From 7292aeb064b53c1a72c57ce8d3ad8e0c7886e446 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 19 May 2022 13:15:47 +0000 Subject: Fold storageapi into storage. --- storageapi/.gitignore | 1 - storageapi/CMakeLists.txt | 26 - storageapi/OWNERS | 1 - storageapi/README | 1 - storageapi/src/.gitignore | 5 - storageapi/src/tests/.gitignore | 10 - storageapi/src/tests/CMakeLists.txt | 18 - storageapi/src/tests/buckets/.gitignore | 9 - storageapi/src/tests/buckets/CMakeLists.txt | 8 - storageapi/src/tests/buckets/bucketinfotest.cpp | 28 - storageapi/src/tests/gtest_runner.cpp | 8 - storageapi/src/tests/mbusprot/.gitignore | 11 - storageapi/src/tests/mbusprot/CMakeLists.txt | 8 - .../mbusprot.4.2.serialization.V_4_2_STABLE | 66 - .../src/tests/mbusprot/storageprotocoltest.cpp | 855 ------------- storageapi/src/tests/message/.gitignore | 9 - storageapi/src/tests/messageapi/.gitignore | 9 - storageapi/src/tests/messageapi/CMakeLists.txt | 8 - .../messageapi/storage_message_address_test.cpp | 40 - storageapi/src/vespa/storageapi/.gitignore | 5 - storageapi/src/vespa/storageapi/CMakeLists.txt | 14 - storageapi/src/vespa/storageapi/app/.gitignore | 14 - storageapi/src/vespa/storageapi/app/CMakeLists.txt | 7 - .../src/vespa/storageapi/app/getbucketid.cpp | 18 - storageapi/src/vespa/storageapi/buckets/.gitignore | 9 - .../src/vespa/storageapi/buckets/CMakeLists.txt | 6 - .../src/vespa/storageapi/buckets/bucketinfo.cpp | 132 -- .../src/vespa/storageapi/buckets/bucketinfo.h | 84 -- storageapi/src/vespa/storageapi/defs.h | 18 - .../src/vespa/storageapi/mbusprot/.gitignore | 10 - .../src/vespa/storageapi/mbusprot/CMakeLists.txt | 34 - .../mbusprot/legacyprotocolserialization.h | 31 - .../storageapi/mbusprot/oldreturncodemapper.h | 68 - .../storageapi/mbusprot/protobuf/common.proto | 68 - .../vespa/storageapi/mbusprot/protobuf/feed.proto | 104 -- .../storageapi/mbusprot/protobuf/inspect.proto | 19 - .../storageapi/mbusprot/protobuf/maintenance.proto | 167 --- .../storageapi/mbusprot/protobuf/visiting.proto | 66 - .../vespa/storageapi/mbusprot/protobuf_includes.h | 16 - .../storageapi/mbusprot/protocolserialization.cpp | 281 ---- .../storageapi/mbusprot/protocolserialization.h | 159 --- .../mbusprot/protocolserialization4_2.cpp | 553 -------- .../storageapi/mbusprot/protocolserialization4_2.h | 72 -- .../mbusprot/protocolserialization5_0.cpp | 624 --------- .../storageapi/mbusprot/protocolserialization5_0.h | 73 -- .../mbusprot/protocolserialization5_1.cpp | 198 --- .../storageapi/mbusprot/protocolserialization5_1.h | 34 - .../mbusprot/protocolserialization5_2.cpp | 64 - .../storageapi/mbusprot/protocolserialization5_2.h | 32 - .../mbusprot/protocolserialization6_0.cpp | 40 - .../storageapi/mbusprot/protocolserialization6_0.h | 24 - .../storageapi/mbusprot/protocolserialization7.cpp | 1351 -------------------- .../storageapi/mbusprot/protocolserialization7.h | 147 --- .../storageapi/mbusprot/serializationhelper.h | 110 -- .../vespa/storageapi/mbusprot/storagecommand.cpp | 11 - .../src/vespa/storageapi/mbusprot/storagecommand.h | 37 - .../vespa/storageapi/mbusprot/storagemessage.cpp | 0 .../src/vespa/storageapi/mbusprot/storagemessage.h | 20 - .../vespa/storageapi/mbusprot/storageprotocol.cpp | 203 --- .../vespa/storageapi/mbusprot/storageprotocol.h | 34 - .../src/vespa/storageapi/mbusprot/storagereply.cpp | 55 - .../src/vespa/storageapi/mbusprot/storagereply.h | 46 - storageapi/src/vespa/storageapi/message/.gitignore | 7 - .../src/vespa/storageapi/message/CMakeLists.txt | 17 - storageapi/src/vespa/storageapi/message/bucket.cpp | 652 ---------- storageapi/src/vespa/storageapi/message/bucket.h | 502 -------- .../vespa/storageapi/message/bucketsplitting.cpp | 134 -- .../src/vespa/storageapi/message/bucketsplitting.h | 120 -- .../src/vespa/storageapi/message/datagram.cpp | 100 -- storageapi/src/vespa/storageapi/message/datagram.h | 77 -- .../vespa/storageapi/message/documentsummary.cpp | 43 - .../src/vespa/storageapi/message/documentsummary.h | 39 - .../src/vespa/storageapi/message/internal.cpp | 45 - storageapi/src/vespa/storageapi/message/internal.h | 69 - .../src/vespa/storageapi/message/persistence.cpp | 345 ----- .../src/vespa/storageapi/message/persistence.h | 333 ----- .../src/vespa/storageapi/message/queryresult.cpp | 45 - .../src/vespa/storageapi/message/queryresult.h | 48 - .../vespa/storageapi/message/removelocation.cpp | 34 - .../src/vespa/storageapi/message/removelocation.h | 35 - .../src/vespa/storageapi/message/searchresult.cpp | 47 - .../src/vespa/storageapi/message/searchresult.h | 37 - storageapi/src/vespa/storageapi/message/stat.cpp | 104 -- storageapi/src/vespa/storageapi/message/stat.h | 90 -- storageapi/src/vespa/storageapi/message/state.cpp | 142 -- storageapi/src/vespa/storageapi/message/state.h | 119 -- .../src/vespa/storageapi/message/visitor.cpp | 233 ---- storageapi/src/vespa/storageapi/message/visitor.h | 244 ---- .../src/vespa/storageapi/messageapi/.gitignore | 6 - .../src/vespa/storageapi/messageapi/CMakeLists.txt | 14 - .../vespa/storageapi/messageapi/bucketcommand.cpp | 46 - .../vespa/storageapi/messageapi/bucketcommand.h | 33 - .../storageapi/messageapi/bucketinfocommand.cpp | 21 - .../storageapi/messageapi/bucketinfocommand.h | 30 - .../storageapi/messageapi/bucketinforeply.cpp | 19 - .../vespa/storageapi/messageapi/bucketinforeply.h | 38 - .../vespa/storageapi/messageapi/bucketreply.cpp | 36 - .../src/vespa/storageapi/messageapi/bucketreply.h | 42 - .../storageapi/messageapi/maintenancecommand.cpp | 9 - .../storageapi/messageapi/maintenancecommand.h | 28 - .../vespa/storageapi/messageapi/messagehandler.h | 196 --- .../src/vespa/storageapi/messageapi/returncode.cpp | 169 --- .../src/vespa/storageapi/messageapi/returncode.h | 117 -- .../vespa/storageapi/messageapi/storagecommand.cpp | 41 - .../vespa/storageapi/messageapi/storagecommand.h | 56 - .../vespa/storageapi/messageapi/storagemessage.cpp | 306 ----- .../vespa/storageapi/messageapi/storagemessage.h | 455 ------- .../vespa/storageapi/messageapi/storagereply.cpp | 38 - .../src/vespa/storageapi/messageapi/storagereply.h | 39 - 109 files changed, 11579 deletions(-) delete mode 100644 storageapi/.gitignore delete mode 100644 storageapi/CMakeLists.txt delete mode 100644 storageapi/OWNERS delete mode 100644 storageapi/README delete mode 100644 storageapi/src/.gitignore delete mode 100644 storageapi/src/tests/.gitignore delete mode 100644 storageapi/src/tests/CMakeLists.txt delete mode 100644 storageapi/src/tests/buckets/.gitignore delete mode 100644 storageapi/src/tests/buckets/CMakeLists.txt delete mode 100644 storageapi/src/tests/buckets/bucketinfotest.cpp delete mode 100644 storageapi/src/tests/gtest_runner.cpp delete mode 100644 storageapi/src/tests/mbusprot/.gitignore delete mode 100644 storageapi/src/tests/mbusprot/CMakeLists.txt delete mode 100644 storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE delete mode 100644 storageapi/src/tests/mbusprot/storageprotocoltest.cpp delete mode 100644 storageapi/src/tests/message/.gitignore delete mode 100644 storageapi/src/tests/messageapi/.gitignore delete mode 100644 storageapi/src/tests/messageapi/CMakeLists.txt delete mode 100644 storageapi/src/tests/messageapi/storage_message_address_test.cpp delete mode 100644 storageapi/src/vespa/storageapi/.gitignore delete mode 100644 storageapi/src/vespa/storageapi/CMakeLists.txt delete mode 100644 storageapi/src/vespa/storageapi/app/.gitignore delete mode 100644 storageapi/src/vespa/storageapi/app/CMakeLists.txt delete mode 100644 storageapi/src/vespa/storageapi/app/getbucketid.cpp delete mode 100644 storageapi/src/vespa/storageapi/buckets/.gitignore delete mode 100644 storageapi/src/vespa/storageapi/buckets/CMakeLists.txt delete mode 100644 storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp delete mode 100644 storageapi/src/vespa/storageapi/buckets/bucketinfo.h delete mode 100644 storageapi/src/vespa/storageapi/defs.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/.gitignore delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protobuf/common.proto delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protobuf/visiting.proto delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.cpp delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/storagecommand.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/storagemessage.cpp delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/storagemessage.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp delete mode 100644 storageapi/src/vespa/storageapi/mbusprot/storagereply.h delete mode 100644 storageapi/src/vespa/storageapi/message/.gitignore delete mode 100644 storageapi/src/vespa/storageapi/message/CMakeLists.txt delete mode 100644 storageapi/src/vespa/storageapi/message/bucket.cpp delete mode 100644 storageapi/src/vespa/storageapi/message/bucket.h delete mode 100644 storageapi/src/vespa/storageapi/message/bucketsplitting.cpp delete mode 100644 storageapi/src/vespa/storageapi/message/bucketsplitting.h delete mode 100644 storageapi/src/vespa/storageapi/message/datagram.cpp delete mode 100644 storageapi/src/vespa/storageapi/message/datagram.h delete mode 100644 storageapi/src/vespa/storageapi/message/documentsummary.cpp delete mode 100644 storageapi/src/vespa/storageapi/message/documentsummary.h delete mode 100644 storageapi/src/vespa/storageapi/message/internal.cpp delete mode 100644 storageapi/src/vespa/storageapi/message/internal.h delete mode 100644 storageapi/src/vespa/storageapi/message/persistence.cpp delete mode 100644 storageapi/src/vespa/storageapi/message/persistence.h delete mode 100644 storageapi/src/vespa/storageapi/message/queryresult.cpp delete mode 100644 storageapi/src/vespa/storageapi/message/queryresult.h delete mode 100644 storageapi/src/vespa/storageapi/message/removelocation.cpp delete mode 100644 storageapi/src/vespa/storageapi/message/removelocation.h delete mode 100644 storageapi/src/vespa/storageapi/message/searchresult.cpp delete mode 100644 storageapi/src/vespa/storageapi/message/searchresult.h delete mode 100644 storageapi/src/vespa/storageapi/message/stat.cpp delete mode 100644 storageapi/src/vespa/storageapi/message/stat.h delete mode 100644 storageapi/src/vespa/storageapi/message/state.cpp delete mode 100644 storageapi/src/vespa/storageapi/message/state.h delete mode 100644 storageapi/src/vespa/storageapi/message/visitor.cpp delete mode 100644 storageapi/src/vespa/storageapi/message/visitor.h delete mode 100644 storageapi/src/vespa/storageapi/messageapi/.gitignore delete mode 100644 storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt delete mode 100644 storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp delete mode 100644 storageapi/src/vespa/storageapi/messageapi/bucketcommand.h delete mode 100644 storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp delete mode 100644 storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h delete mode 100644 storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp delete mode 100644 storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h delete mode 100644 storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp delete mode 100644 storageapi/src/vespa/storageapi/messageapi/bucketreply.h delete mode 100644 storageapi/src/vespa/storageapi/messageapi/maintenancecommand.cpp delete mode 100644 storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h delete mode 100644 storageapi/src/vespa/storageapi/messageapi/messagehandler.h delete mode 100644 storageapi/src/vespa/storageapi/messageapi/returncode.cpp delete mode 100644 storageapi/src/vespa/storageapi/messageapi/returncode.h delete mode 100644 storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp delete mode 100644 storageapi/src/vespa/storageapi/messageapi/storagecommand.h delete mode 100644 storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp delete mode 100644 storageapi/src/vespa/storageapi/messageapi/storagemessage.h delete mode 100644 storageapi/src/vespa/storageapi/messageapi/storagereply.cpp delete mode 100644 storageapi/src/vespa/storageapi/messageapi/storagereply.h (limited to 'storageapi') diff --git a/storageapi/.gitignore b/storageapi/.gitignore deleted file mode 100644 index f3c7a7c5da6..00000000000 --- a/storageapi/.gitignore +++ /dev/null @@ -1 +0,0 @@ -Makefile diff --git a/storageapi/CMakeLists.txt b/storageapi/CMakeLists.txt deleted file mode 100644 index d8e00cc35bf..00000000000 --- a/storageapi/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - fastos - fnet - vespalog - vespalib - document - messagebus - vdslib - documentapi - - LIBS - src/vespa/storageapi - src/vespa/storageapi/app - src/vespa/storageapi/buckets - src/vespa/storageapi/mbusprot - src/vespa/storageapi/message - src/vespa/storageapi/messageapi - - TESTS - src/tests - src/tests/buckets - src/tests/mbusprot - src/tests/messageapi -) diff --git a/storageapi/OWNERS b/storageapi/OWNERS deleted file mode 100644 index dbcff24b338..00000000000 --- a/storageapi/OWNERS +++ /dev/null @@ -1 +0,0 @@ -vekterli diff --git a/storageapi/README b/storageapi/README deleted file mode 100644 index f6332c9f8f0..00000000000 --- a/storageapi/README +++ /dev/null @@ -1 +0,0 @@ -This is the API needed to access the Vespa Storage System. diff --git a/storageapi/src/.gitignore b/storageapi/src/.gitignore deleted file mode 100644 index a735fdb4395..00000000000 --- a/storageapi/src/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.cvsignore -Makefile.ini -config_command.sh -doc -project.dsw diff --git a/storageapi/src/tests/.gitignore b/storageapi/src/tests/.gitignore deleted file mode 100644 index 29911acfe67..00000000000 --- a/storageapi/src/tests/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -*.core -.*.swp -.config.log -.depend -Makefile -docstorage -test.vlog -testrunner -storageapi_gtest_runner_app -storageapi_testrunner_app diff --git a/storageapi/src/tests/CMakeLists.txt b/storageapi/src/tests/CMakeLists.txt deleted file mode 100644 index 18d2b34f499..00000000000 --- a/storageapi/src/tests/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -vespa_add_executable(storageapi_gtest_runner_app TEST - SOURCES - gtest_runner.cpp - DEPENDS - storageapi_testbuckets - storageapi_testmbusprot - storageapi_testmessageapi - storageapi - GTest::GTest -) - -vespa_add_test( - NAME storageapi_gtest_runner_app - COMMAND storageapi_gtest_runner_app -) - diff --git a/storageapi/src/tests/buckets/.gitignore b/storageapi/src/tests/buckets/.gitignore deleted file mode 100644 index 1d859456ef9..00000000000 --- a/storageapi/src/tests/buckets/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.So -*.core -*.so -.*.swp -.config.log -.depend -Makefile -test.vlog -testrunner diff --git a/storageapi/src/tests/buckets/CMakeLists.txt b/storageapi/src/tests/buckets/CMakeLists.txt deleted file mode 100644 index 651b27618a6..00000000000 --- a/storageapi/src/tests/buckets/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageapi_testbuckets - SOURCES - bucketinfotest.cpp - DEPENDS - storageapi - GTest::GTest -) diff --git a/storageapi/src/tests/buckets/bucketinfotest.cpp b/storageapi/src/tests/buckets/bucketinfotest.cpp deleted file mode 100644 index 25648fa7e96..00000000000 --- a/storageapi/src/tests/buckets/bucketinfotest.cpp +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include - -namespace storage::api { - -/** Tests simple operations */ -TEST(BucketInfoTest, testSimple) -{ - BucketInfo info; - - EXPECT_FALSE(info.valid()); - EXPECT_EQ(0u, info.getChecksum()); - EXPECT_EQ(0u, info.getDocumentCount()); - EXPECT_EQ(1u, info.getTotalDocumentSize()); - - info.setChecksum(0xa000bbbb); - info.setDocumentCount(15); - info.setTotalDocumentSize(64000); - - EXPECT_TRUE(info.valid()); - EXPECT_EQ(0xa000bbbb, info.getChecksum()); - EXPECT_EQ(15u, info.getDocumentCount()); - EXPECT_EQ(64000u, info.getTotalDocumentSize()); -}; - -} diff --git a/storageapi/src/tests/gtest_runner.cpp b/storageapi/src/tests/gtest_runner.cpp deleted file mode 100644 index 2ae414830b9..00000000000 --- a/storageapi/src/tests/gtest_runner.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include - -#include -LOG_SETUP("storageapi_gtest_runner"); - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/storageapi/src/tests/mbusprot/.gitignore b/storageapi/src/tests/mbusprot/.gitignore deleted file mode 100644 index 1b779fa320a..00000000000 --- a/storageapi/src/tests/mbusprot/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -*.So -*.core -*.so -.*.swp -.config.log -.depend -Makefile -test.vlog -testrunner -/mbusprot.4.2.serialization.HEAD -/mbusprot.5.0.serialization.5.1 diff --git a/storageapi/src/tests/mbusprot/CMakeLists.txt b/storageapi/src/tests/mbusprot/CMakeLists.txt deleted file mode 100644 index 48469cfd9ea..00000000000 --- a/storageapi/src/tests/mbusprot/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageapi_testmbusprot - SOURCES - storageprotocoltest.cpp - DEPENDS - storageapi - GTest::GTest -) diff --git a/storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE b/storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE deleted file mode 100644 index 5045a98b037..00000000000 --- a/storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE +++ /dev/null @@ -1,66 +0,0 @@ - -MessageType(10, Put) -\00\00\00 -\00\00\00j\00\08\00\00\00ddoc:test:test\00\05testdoctype1\00\00\01\00\00\00=\00\01\05\00= ;This is the contents of the test document. -It ain't much. -\00@\00\00\00\00\00\00Q\00\00\00\00\00\00\00\0e\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(11, Put Reply, reply of Put) -\00\00\00\0b\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 -MessageType(82, Update) -\00\00\00R\00\00\005\00\08doc:test:test\00\01testdoctype1\00\00\01\00\00\00\01\00\00\00\02\00\00\00\01\00\00\10\1b\01\00\00\00\11@\00\00\00\00\00\00Q\00\00\00\00\00\00\00\0e\00\00\00\00\00\00\00 -\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(83, Update Reply, reply of Update) -\00\00\00S\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\00\00\00\00\0e\00\00\00\00\00\00\00\08\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 -MessageType(4, Get) -\00\00\00\04\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\00\00\00\00{\01\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(5, Get Reply, reply of Get) -\00\00\00\05\00\00\00j\00\08\00\00\00ddoc:test:test\00\05testdoctype1\00\00\01\00\00\00=\00\01\05\00= ;This is the contents of the test document. -It ain't much. -\00\00\00\00\00\00\00\00d\00\00\00\0ddoc:test:test\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01 -MessageType(12, Remove) -\00\00\00\0c\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\00\00\00\00\9f\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(13, Remove Reply, reply of Remove) -\00\00\00\0d\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\00\00\00\000\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 -MessageType(14, Revert) -\00\00\00\0e@\00\00\00\00\00\00Q\00\00\00\01\00\00\00\00\00\00\00;\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(15, Revert Reply, reply of Revert) -\00\00\00\0f@\00\00\00\00\00\00Q\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 -MessageType(54, Request bucket info) -\00\00\006\00\00\00\00\00\03\00\00\00\14distributor:3 .1.s:d\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(55, Request bucket info reply, reply of Request bucket info) -\00\00\007\00\00\00\01\00\00\00\00\00\00\00\04\00\00\00+\00\00\00\18\00\00\00{\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01 -MessageType(56, Notify bucket change) -\00\00\008P\00\00\00\00\00\03\e8\00\00\00\02\00\00\00\03\00\00\00\04\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(57, Notify bucket change reply, reply of Notify bucket change) -\00\00\009\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01 -MessageType(26, Create bucket) -\00\00\00\1a\00\00\00\00\00\00\02o\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(27, Create bucket reply, reply of Create bucket) -\00\00\00\1b\00\00\00\00\00\00\02o\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 -MessageType(34, Delete bucket) -\00\00\00"\00\00\00\00\00\00\02o\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(35, Delete bucket reply, reply of Delete bucket) -\00\00\00#\00\00\00\00\00\00\02o\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 -MessageType(32, Merge bucket) -\00\00\00 \00\00\00\00\00\00\02o\00\03\00\04\00\00\0d\01\00\1a\01\00\00\00\00\00\00\04\d2\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(33, Merge bucket reply, reply of Merge bucket) -\00\00\00!\00\00\00\00\00\00\02o\00\03\00\04\00\00\0d\01\00\1a\01\00\00\00\00\00\00\04\d2\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\04\00\00\00 -\00\00\00d -MessageType(50, GetBucketDiff) -\00\00\002\00\00\00\00\00\00\02o\00\02\00\04\00\00\0d\00\00\00\00\00\00\00\04 \00\00\00\01\00\00\00\00\00\01\e2@\00\0c1234567890ab\00\00\00d\00\01\00\00\00\01\00\03\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(51, GetBucketDiff reply, reply of GetBucketDiff) -\00\00\003\00\00\00\00\00\00\02o\00\02\00\04\00\00\0d\00\00\00\00\00\00\00\04 \00\00\00\01\00\00\00\00\00\01\e2@\00\0c1234567890ab\00\00\00d\00\01\00\00\00\01\00\03\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01 -MessageType(52, ApplyBucketDiff) -\00\00\004@\00\00\00\00\00\02o\00\02\00\04\00\00\0d\00\00\00\04\d2\00\00\00\01\00\00\00\00\00\00\00\00\00\0c\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(53, ApplyBucketDiff reply, reply of ApplyBucketDiff) -\00\00\005@\00\00\00\00\00\02o\00\02\00\04\00\00\0d\00\00\00\04\d2\00\00\00\01\00\00\00\00\00\00\00\00\00\0c\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 -MessageType(66, SplitBucket) -\00\00\00B@\00\00\00\00\00\00\00\14(\00\00\03\e8\00\00\00\05\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(67, SplitBucket reply, reply of SplitBucket) -\00\00\00C@\00\00\00\00\00\00\00\00\00\00\02D\00\00\00\00\00\00\00\00\00\00d\00\00\03\e8\00\00'\10D\00\00\00\00\00\00\01\00\00\00e\00\00\03\e9\00\00'\11\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01 -MessageType(18, Visitor Create) -\00\00\00\12\00\00\00\07library\00\00\00\02id\00\00\00\0ddoc selection\00\00\00\01\00\00\00\0bcontroldest\00\00\00\08datadest\00\00\00\02\00\00\00\00\00\00\00{\00\00\00\00\00\00\01\c8\00\00\00\02@\00\00\00\00\00\00\01@\00\00\00\00\00\00\02\00\01\01\00\00\00d\00\00\00\03\00\00\00\0dinto darkness\00\00\00\09bind them\00\00\00\08one ring\00\00\00\10to rule them all\00\00\00\0bone ring to\00\00\00\0dfind them and\00\00\00\00\00\00\00\00\01\ff\ff -MessageType(19, Visitor Create Reply, reply of Visitor Create) -\00\00\00\13\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01 -It ain't much. -\00\00P\00\00\f1\f1\f1\f1\f1\00\00\00\00\00\00\00\00\01\ff\ff diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp deleted file mode 100644 index c130e433285..00000000000 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ /dev/null @@ -1,855 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -using namespace ::testing; - -using std::shared_ptr; -using document::BucketSpace; -using document::ByteBuffer; -using document::Document; -using document::DocumentId; -using document::DocumentType; -using document::DocumentTypeRepo; -using document::FieldUpdate; -using document::FieldPathUpdate; -using document::AssignValueUpdate; -using document::IntFieldValue; -using document::RemoveFieldPathUpdate; -using document::test::makeDocumentBucket; -using document::test::makeBucketSpace; -using storage::lib::ClusterState; -using vespalib::string; - -namespace vespalib { - -// Needed for GTest to properly understand how to print Version values. -// If not present, it will print the byte values of the presumed memory area -// (which will be overrun for some reason, causing Valgrind to scream at us). -void PrintTo(const vespalib::Version& v, std::ostream* os) { - *os << v.toString(); -} - -} - -namespace storage::api { - -struct StorageProtocolTest : TestWithParam { - document::TestDocMan _docMan; - document::Document::SP _testDoc; - document::DocumentId _testDocId; - document::BucketId _bucket_id; - document::Bucket _bucket; - document::BucketId _dummy_remap_bucket{17, 12345}; - BucketInfo _dummy_bucket_info{1,2,3,4,5, true, false, 48}; - mbusprot::StorageProtocol _protocol; - static auto constexpr CONDITION_STRING = "There's just one condition"; - - StorageProtocolTest() - : _docMan(), - _testDoc(_docMan.createDocument()), - _testDocId(_testDoc->getId()), - _bucket_id(16, 0x51), - _bucket(makeDocumentBucket(_bucket_id)), - _protocol(_docMan.getTypeRepoSP()) - { - } - ~StorageProtocolTest(); - - void set_dummy_bucket_info_reply_fields(BucketInfoReply& reply) { - reply.setBucketInfo(_dummy_bucket_info); - reply.remapBucketId(_dummy_remap_bucket); - } - - void assert_bucket_info_reply_fields_propagated(const BucketInfoReply& reply) { - EXPECT_EQ(_dummy_bucket_info, reply.getBucketInfo()); - EXPECT_TRUE(reply.hasBeenRemapped()); - EXPECT_EQ(_dummy_remap_bucket, reply.getBucketId()); - EXPECT_EQ(_bucket_id, reply.getOriginalBucketId()); - } - - template - std::shared_ptr copyCommand(const std::shared_ptr&); - template - std::shared_ptr copyReply(const std::shared_ptr&); -}; - -StorageProtocolTest::~StorageProtocolTest() = default; - -namespace { - -std::string version_as_gtest_string(TestParamInfo info) { - std::ostringstream ss; - auto& p = info.param; - // Dots are not allowed in test names, so convert to underscores. - ss << p.getMajor() << '_' << p.getMinor() << '_' << p.getMicro(); - return ss.str(); -} - -} - -VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(MultiVersionTest, StorageProtocolTest, - Values(vespalib::Version(6, 240, 0), - vespalib::Version(7, 41, 19)), - version_as_gtest_string); - -namespace { - mbus::Message::UP lastCommand; - mbus::Reply::UP lastReply; -} - -TEST_F(StorageProtocolTest, testAddress50) { - vespalib::string cluster("foo"); - StorageMessageAddress address(&cluster, lib::NodeType::STORAGE, 3); - EXPECT_EQ(vespalib::string("storage/cluster.foo/storage/3/default"), - address.to_mbus_route().toString()); -} - -template std::shared_ptr -StorageProtocolTest::copyCommand(const std::shared_ptr& m) -{ - auto mbusMessage = std::make_unique(m); - auto version = GetParam(); - mbus::Blob blob = _protocol.encode(version, *mbusMessage); - mbus::Routable::UP copy(_protocol.decode(version, blob)); - assert(copy.get()); - - auto* copy2 = dynamic_cast(copy.get()); - assert(copy2 != nullptr); - - StorageCommand::SP internalMessage(copy2->getCommand()); - lastCommand = std::move(mbusMessage); - - return std::dynamic_pointer_cast(internalMessage); -} - -template std::shared_ptr -StorageProtocolTest::copyReply(const std::shared_ptr& m) -{ - auto mbusMessage = std::make_unique(m); - auto version = GetParam(); - mbus::Blob blob = _protocol.encode(version, *mbusMessage); - mbus::Routable::UP copy(_protocol.decode(version, blob)); - assert(copy.get()); - - auto* copy2 = dynamic_cast(copy.get()); - assert(copy2 != nullptr); - - copy2->setMessage(std::move(lastCommand)); - auto internalMessage = copy2->getReply(); - lastReply = std::move(mbusMessage); - lastCommand = copy2->getMessage(); - return std::dynamic_pointer_cast(internalMessage); -} - -TEST_P(StorageProtocolTest, put) { - auto cmd = std::make_shared(_bucket, _testDoc, 14); - cmd->setUpdateTimestamp(Timestamp(13)); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - EXPECT_EQ(*_testDoc, *cmd2->getDocument()); - EXPECT_EQ(Timestamp(14), cmd2->getTimestamp()); - EXPECT_EQ(Timestamp(13), cmd2->getUpdateTimestamp()); - - auto reply = std::make_shared(*cmd2); - ASSERT_TRUE(reply->hasDocument()); - EXPECT_EQ(*_testDoc, *reply->getDocument()); - set_dummy_bucket_info_reply_fields(*reply); - auto reply2 = copyReply(reply); - ASSERT_TRUE(reply2->hasDocument()); - EXPECT_EQ(*_testDoc, *reply->getDocument()); - EXPECT_EQ(_testDoc->getId(), reply2->getDocumentId()); - EXPECT_EQ(Timestamp(14), reply2->getTimestamp()); - EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); -} - -TEST_P(StorageProtocolTest, response_without_remapped_bucket_preserves_original_bucket) { - auto cmd = std::make_shared(_bucket, _testDoc, 14); - auto cmd2 = copyCommand(cmd); - auto reply = std::make_shared(*cmd2); - auto reply2 = copyReply(reply); - - EXPECT_FALSE(reply2->hasBeenRemapped()); - EXPECT_EQ(_bucket_id, reply2->getBucketId()); - EXPECT_EQ(document::BucketId(), reply2->getOriginalBucketId()); -} - -TEST_P(StorageProtocolTest, invalid_bucket_info_is_propagated) { - auto cmd = std::make_shared(_bucket, _testDoc, 14); - auto cmd2 = copyCommand(cmd); - auto reply = std::make_shared(*cmd2); - BucketInfo invalid_info; - EXPECT_FALSE(invalid_info.valid()); - reply->setBucketInfo(invalid_info); - auto reply2 = copyReply(reply); - - EXPECT_EQ(invalid_info, reply2->getBucketInfo()); - EXPECT_FALSE(reply2->getBucketInfo().valid()); -} - -TEST_P(StorageProtocolTest, all_zero_bucket_info_is_propagated) { - auto cmd = std::make_shared(_bucket, _testDoc, 14); - auto cmd2 = copyCommand(cmd); - auto reply = std::make_shared(*cmd2); - BucketInfo zero_info(0, 0, 0, 0, 0, false, false, 0); - reply->setBucketInfo(zero_info); - auto reply2 = copyReply(reply); - - EXPECT_EQ(zero_info, reply2->getBucketInfo()); -} - -TEST_P(StorageProtocolTest, request_metadata_is_propagated) { - auto cmd = std::make_shared(_bucket, _testDoc, 14); - cmd->forceMsgId(12345); - cmd->setPriority(50); - cmd->setSourceIndex(321); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(12345, cmd2->getMsgId()); - EXPECT_EQ(50, cmd2->getPriority()); - EXPECT_EQ(321, cmd2->getSourceIndex()); -} - -TEST_P(StorageProtocolTest, response_metadata_is_propagated) { - auto cmd = std::make_shared(_bucket, _testDoc, 14); - auto cmd2 = copyCommand(cmd); - auto reply = std::make_shared(*cmd2); - reply->forceMsgId(1234); - reply->setPriority(101); - ReturnCode result(ReturnCode::TEST_AND_SET_CONDITION_FAILED, "foo is not bar"); - reply->setResult(result); - - auto reply2 = copyReply(reply); - EXPECT_EQ(result, reply2->getResult()); - EXPECT_EQ(1234, reply->getMsgId()); - EXPECT_EQ(101, reply->getPriority()); -} - -TEST_P(StorageProtocolTest, update) { - auto update = std::make_shared(_docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId()); - update->addUpdate(FieldUpdate(_testDoc->getField("headerval")).addUpdate(std::make_unique(std::make_unique(17)))); - - update->addFieldPathUpdate(std::make_unique("headerval", "testdoctype1.headerval > 0")); - - auto cmd = std::make_shared(_bucket, update, 14); - EXPECT_EQ(Timestamp(0), cmd->getOldTimestamp()); - cmd->setOldTimestamp(10); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - EXPECT_EQ(_testDocId, cmd2->getDocumentId()); - EXPECT_EQ(Timestamp(14), cmd2->getTimestamp()); - EXPECT_EQ(Timestamp(10), cmd2->getOldTimestamp()); - EXPECT_EQ(*update, *cmd2->getUpdate()); - - auto reply = std::make_shared(*cmd2, 8); - set_dummy_bucket_info_reply_fields(*reply); - auto reply2 = copyReply(reply); - EXPECT_EQ(_testDocId, reply2->getDocumentId()); - EXPECT_EQ(Timestamp(14), reply2->getTimestamp()); - EXPECT_EQ(Timestamp(8), reply->getOldTimestamp()); - EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); -} - -TEST_P(StorageProtocolTest, get) { - auto cmd = std::make_shared(_bucket, _testDocId, "foo,bar,vekterli", 123); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - EXPECT_EQ(_testDocId, cmd2->getDocumentId()); - EXPECT_EQ(Timestamp(123), cmd2->getBeforeTimestamp()); - EXPECT_EQ(vespalib::string("foo,bar,vekterli"), cmd2->getFieldSet()); - - auto reply = std::make_shared(*cmd2, _testDoc, 100); - set_dummy_bucket_info_reply_fields(*reply); - auto reply2 = copyReply(reply); - ASSERT_TRUE(reply2.get() != nullptr); - ASSERT_TRUE(reply2->getDocument().get() != nullptr); - EXPECT_EQ(*_testDoc, *reply2->getDocument()); - EXPECT_EQ(_testDoc->getId(), reply2->getDocumentId()); - EXPECT_EQ(Timestamp(123), reply2->getBeforeTimestamp()); - EXPECT_EQ(Timestamp(100), reply2->getLastModifiedTimestamp()); - EXPECT_FALSE(reply2->is_tombstone()); - EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); -} - -TEST_P(StorageProtocolTest, get_internal_read_consistency_is_strong_by_default) { - auto cmd = std::make_shared(_bucket, _testDocId, "foo,bar,vekterli", 123); - EXPECT_EQ(cmd->internal_read_consistency(), InternalReadConsistency::Strong); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(cmd2->internal_read_consistency(), InternalReadConsistency::Strong); -} - -TEST_P(StorageProtocolTest, can_set_internal_read_consistency_on_get_commands) { - // Only supported on protocol version 7+. Will default to Strong on older versions, which is what we want. - if (GetParam().getMajor() < 7) { - return; - } - auto cmd = std::make_shared(_bucket, _testDocId, "foo,bar,vekterli", 123); - cmd->set_internal_read_consistency(InternalReadConsistency::Weak); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(cmd2->internal_read_consistency(), InternalReadConsistency::Weak); - - cmd->set_internal_read_consistency(InternalReadConsistency::Strong); - cmd2 = copyCommand(cmd); - EXPECT_EQ(cmd2->internal_read_consistency(), InternalReadConsistency::Strong); -} - -TEST_P(StorageProtocolTest, tombstones_propagated_for_gets) { - // Only supported on protocol version 7+. - if (GetParam().getMajor() < 7) { - return; - } - auto cmd = std::make_shared(_bucket, _testDocId, "foo,bar", 123); - auto reply = std::make_shared(*cmd, std::shared_ptr(), 100, false, true); - set_dummy_bucket_info_reply_fields(*reply); - auto reply2 = copyReply(reply); - - EXPECT_TRUE(reply2->getDocument().get() == nullptr); - EXPECT_EQ(_testDoc->getId(), reply2->getDocumentId()); - EXPECT_EQ(Timestamp(123), reply2->getBeforeTimestamp()); - EXPECT_EQ(Timestamp(100), reply2->getLastModifiedTimestamp()); // In this case, the tombstone timestamp. - EXPECT_TRUE(reply2->is_tombstone()); -} - -// TODO remove this once pre-protobuf serialization is removed -TEST_P(StorageProtocolTest, old_serialization_format_treats_tombstone_get_replies_as_not_found) { - if (GetParam().getMajor() >= 7) { - return; - } - auto cmd = std::make_shared(_bucket, _testDocId, "foo,bar", 123); - auto reply = std::make_shared(*cmd, std::shared_ptr(), 100, false, true); - set_dummy_bucket_info_reply_fields(*reply); - auto reply2 = copyReply(reply); - - EXPECT_TRUE(reply2->getDocument().get() == nullptr); - EXPECT_EQ(_testDoc->getId(), reply2->getDocumentId()); - EXPECT_EQ(Timestamp(123), reply2->getBeforeTimestamp()); - EXPECT_EQ(Timestamp(0), reply2->getLastModifiedTimestamp()); - EXPECT_FALSE(reply2->is_tombstone()); // Protocol version doesn't understand explicit tombstones. -} - -TEST_P(StorageProtocolTest, remove) { - auto cmd = std::make_shared(_bucket, _testDocId, 159); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - EXPECT_EQ(_testDocId, cmd2->getDocumentId()); - EXPECT_EQ(Timestamp(159), cmd2->getTimestamp()); - - auto reply = std::make_shared(*cmd2, 48); - set_dummy_bucket_info_reply_fields(*reply); - - auto reply2 = copyReply(reply); - EXPECT_EQ(_testDocId, reply2->getDocumentId()); - EXPECT_EQ(Timestamp(159), reply2->getTimestamp()); - EXPECT_EQ(Timestamp(48), reply2->getOldTimestamp()); - EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); -} - -TEST_P(StorageProtocolTest, revert) { - std::vector tokens; - tokens.push_back(59); - auto cmd = std::make_shared(_bucket, tokens); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - EXPECT_EQ(tokens, cmd2->getRevertTokens()); - - auto reply = std::make_shared(*cmd2); - set_dummy_bucket_info_reply_fields(*reply); - auto reply2 = copyReply(reply); - EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); -} - -TEST_P(StorageProtocolTest, request_bucket_info) { - { - std::vector ids; - ids.push_back(document::BucketId(3)); - ids.push_back(document::BucketId(7)); - auto cmd = std::make_shared(makeBucketSpace(), ids); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(ids, cmd2->getBuckets()); - EXPECT_FALSE(cmd2->hasSystemState()); - } - { - ClusterState state("distributor:3 .1.s:d"); - auto cmd = std::make_shared(makeBucketSpace(), 3, state, "14"); - auto cmd2 = copyCommand(cmd); - ASSERT_TRUE(cmd2->hasSystemState()); - EXPECT_EQ(uint16_t(3), cmd2->getDistributor()); - EXPECT_EQ(state, cmd2->getSystemState()); - EXPECT_EQ(size_t(0), cmd2->getBuckets().size()); - - auto reply = std::make_shared(*cmd); - RequestBucketInfoReply::Entry e; - e._bucketId = document::BucketId(4); - const uint64_t lastMod = 0x1337cafe98765432ULL; - e._info = BucketInfo(43, 24, 123, 44, 124, false, true, lastMod); - reply->getBucketInfo().push_back(e); - auto reply2 = copyReply(reply); - EXPECT_EQ(size_t(1), reply2->getBucketInfo().size()); - auto& entries(reply2->getBucketInfo()); - EXPECT_EQ(e, entries[0]); - // "Last modified" not counted by operator== for some reason. Testing - // separately until we can figure out if this is by design or not. - EXPECT_EQ(lastMod, entries[0]._info.getLastModified()); - - if (GetParam().getMajor() >= 7) { - EXPECT_TRUE(reply2->supported_node_features().unordered_merge_chaining); - } else { - EXPECT_FALSE(reply2->supported_node_features().unordered_merge_chaining); - } - } -} - -TEST_P(StorageProtocolTest, notify_bucket_change) { - auto cmd = std::make_shared(_bucket, _dummy_bucket_info); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - EXPECT_EQ(_dummy_bucket_info, cmd2->getBucketInfo()); - - auto reply = std::make_shared(*cmd); - auto reply2 = copyReply(reply); -} - -TEST_P(StorageProtocolTest, create_bucket_without_activation) { - auto cmd = std::make_shared(_bucket); - EXPECT_FALSE(cmd->getActive()); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - EXPECT_FALSE(cmd2->getActive()); - - auto reply = std::make_shared(*cmd); - set_dummy_bucket_info_reply_fields(*reply); - auto reply2 = copyReply(reply); - EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); -} - -TEST_P(StorageProtocolTest, create_bucket_propagates_activation_flag) { - auto cmd = std::make_shared(_bucket); - cmd->setActive(true); - auto cmd2 = copyCommand(cmd); - EXPECT_TRUE(cmd2->getActive()); -} - -TEST_P(StorageProtocolTest, delete_bucket) { - auto cmd = std::make_shared(_bucket); - cmd->setBucketInfo(_dummy_bucket_info); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - EXPECT_EQ(_dummy_bucket_info, cmd2->getBucketInfo()); - - auto reply = std::make_shared(*cmd); - // Not set automatically by constructor - reply->setBucketInfo(cmd2->getBucketInfo()); - auto reply2 = copyReply(reply); - EXPECT_EQ(_bucket_id, reply2->getBucketId()); - EXPECT_EQ(_dummy_bucket_info, reply2->getBucketInfo()); -} - -TEST_P(StorageProtocolTest, merge_bucket) { - typedef api::MergeBucketCommand::Node Node; - std::vector nodes; - nodes.push_back(Node(4, false)); - nodes.push_back(Node(13, true)); - nodes.push_back(Node(26, true)); - - std::vector chain; - // Not a valid chain wrt. the nodes, but just want to have unique values - chain.push_back(7); - chain.push_back(14); - - auto cmd = std::make_shared(_bucket, nodes, Timestamp(1234), 567, chain); - cmd->set_use_unordered_forwarding(true); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - EXPECT_EQ(nodes, cmd2->getNodes()); - EXPECT_EQ(Timestamp(1234), cmd2->getMaxTimestamp()); - EXPECT_EQ(uint32_t(567), cmd2->getClusterStateVersion()); - EXPECT_EQ(chain, cmd2->getChain()); - if (GetParam().getMajor() >= 7) { - EXPECT_EQ(cmd2->use_unordered_forwarding(), cmd->use_unordered_forwarding()); - } else { - EXPECT_FALSE(cmd2->use_unordered_forwarding()); - } - - auto reply = std::make_shared(*cmd); - auto reply2 = copyReply(reply); - EXPECT_EQ(_bucket_id, reply2->getBucketId()); - EXPECT_EQ(nodes, reply2->getNodes()); - EXPECT_EQ(Timestamp(1234), reply2->getMaxTimestamp()); - EXPECT_EQ(uint32_t(567), reply2->getClusterStateVersion()); - EXPECT_EQ(chain, reply2->getChain()); -} - -TEST_P(StorageProtocolTest, split_bucket) { - auto cmd = std::make_shared(_bucket); - EXPECT_EQ(0u, cmd->getMinSplitBits()); - EXPECT_EQ(58u, cmd->getMaxSplitBits()); - EXPECT_EQ(std::numeric_limits().max(), cmd->getMinByteSize()); - EXPECT_EQ(std::numeric_limits().max(), cmd->getMinDocCount()); - cmd->setMinByteSize(1000); - cmd->setMinDocCount(5); - cmd->setMaxSplitBits(40); - cmd->setMinSplitBits(20); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - EXPECT_EQ(20u, cmd2->getMinSplitBits()); - EXPECT_EQ(40u, cmd2->getMaxSplitBits()); - EXPECT_EQ(1000u, cmd2->getMinByteSize()); - EXPECT_EQ(5u, cmd2->getMinDocCount()); - - auto reply = std::make_shared(*cmd2); - reply->getSplitInfo().emplace_back(document::BucketId(17, 0), BucketInfo(100, 1000, 10000, true, true)); - reply->getSplitInfo().emplace_back(document::BucketId(17, 1), BucketInfo(101, 1001, 10001, true, true)); - auto reply2 = copyReply(reply); - - EXPECT_EQ(_bucket, reply2->getBucket()); - EXPECT_EQ(size_t(2), reply2->getSplitInfo().size()); - EXPECT_EQ(document::BucketId(17, 0), reply2->getSplitInfo()[0].first); - EXPECT_EQ(document::BucketId(17, 1), reply2->getSplitInfo()[1].first); - EXPECT_EQ(BucketInfo(100, 1000, 10000, true, true), reply2->getSplitInfo()[0].second); - EXPECT_EQ(BucketInfo(101, 1001, 10001, true, true), reply2->getSplitInfo()[1].second); -} - -TEST_P(StorageProtocolTest, join_buckets) { - std::vector sources; - sources.push_back(document::BucketId(17, 0)); - sources.push_back(document::BucketId(17, 1)); - auto cmd = std::make_shared(_bucket); - cmd->getSourceBuckets() = sources; - cmd->setMinJoinBits(3); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - - auto reply = std::make_shared(*cmd2); - reply->setBucketInfo(BucketInfo(3,4,5)); - auto reply2 = copyReply(reply); - - EXPECT_EQ(sources, reply2->getSourceBuckets()); - EXPECT_EQ(3, cmd2->getMinJoinBits()); - EXPECT_EQ(BucketInfo(3,4,5), reply2->getBucketInfo()); - EXPECT_EQ(_bucket, reply2->getBucket()); -} - -TEST_P(StorageProtocolTest, destroy_visitor) { - auto cmd = std::make_shared("instance"); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ("instance", cmd2->getInstanceId()); - - auto reply = std::make_shared(*cmd2); - auto reply2 = copyReply(reply); -} - -TEST_P(StorageProtocolTest, remove_location) { - auto cmd = std::make_shared("id.group == \"mygroup\"", _bucket); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ("id.group == \"mygroup\"", cmd2->getDocumentSelection()); - EXPECT_EQ(_bucket, cmd2->getBucket()); - - uint32_t n_docs_removed = 12345; - auto reply = std::make_shared(*cmd2, n_docs_removed); - auto reply2 = copyReply(reply); - if (GetParam().getMajor() == 7) { - // Statistics are only available for protobuf-enabled version. - EXPECT_EQ(n_docs_removed, reply2->documents_removed()); - } else { - EXPECT_EQ(0, reply2->documents_removed()); - } -} - -TEST_P(StorageProtocolTest, stat_bucket) { - if (GetParam().getMajor() < 7) { - return; // Only available for protobuf-backed protocol version. - } - auto cmd = std::make_shared(_bucket, "id.group == 'mygroup'"); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ("id.group == 'mygroup'", cmd2->getDocumentSelection()); - EXPECT_EQ(_bucket, cmd2->getBucket()); - - auto reply = std::make_shared(*cmd2, "neat bucket info goes here"); - reply->remapBucketId(_dummy_remap_bucket); - auto reply2 = copyReply(reply); - EXPECT_EQ(reply2->getResults(), "neat bucket info goes here"); - EXPECT_TRUE(reply2->hasBeenRemapped()); - EXPECT_EQ(_dummy_remap_bucket, reply2->getBucketId()); - EXPECT_EQ(_bucket_id, reply2->getOriginalBucketId()); -} - -TEST_P(StorageProtocolTest, create_visitor) { - std::vector buckets; - buckets.push_back(document::BucketId(16, 1)); - buckets.push_back(document::BucketId(16, 2)); - - auto cmd = std::make_shared(makeBucketSpace(), "library", "id", "doc selection"); - cmd->setControlDestination("controldest"); - cmd->setDataDestination("datadest"); - cmd->setVisitorCmdId(1); - cmd->getParameters().set("one ring", "to rule them all"); - cmd->getParameters().set("one ring to", "find them and"); - cmd->getParameters().set("into darkness", "bind them"); - cmd->setMaximumPendingReplyCount(2); - cmd->setFromTime(123); - cmd->setToTime(456); - cmd->getBuckets() = buckets; - cmd->setFieldSet("foo,bar,vekterli"); - cmd->setVisitInconsistentBuckets(); - cmd->setQueueTimeout(100ms); - cmd->setPriority(149); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ("library", cmd2->getLibraryName()); - EXPECT_EQ("id", cmd2->getInstanceId()); - EXPECT_EQ("doc selection", cmd2->getDocumentSelection()); - EXPECT_EQ("controldest", cmd2->getControlDestination()); - EXPECT_EQ("datadest", cmd2->getDataDestination()); - EXPECT_EQ(api::Timestamp(123), cmd2->getFromTime()); - EXPECT_EQ(api::Timestamp(456), cmd2->getToTime()); - EXPECT_EQ(2u, cmd2->getMaximumPendingReplyCount()); - EXPECT_EQ(buckets, cmd2->getBuckets()); - EXPECT_EQ("foo,bar,vekterli", cmd2->getFieldSet()); - EXPECT_TRUE(cmd2->visitInconsistentBuckets()); - EXPECT_EQ(149, cmd2->getPriority()); - - auto reply = std::make_shared(*cmd2); - auto reply2 = copyReply(reply); -} - -TEST_P(StorageProtocolTest, get_bucket_diff) { - std::vector nodes; - nodes.push_back(4); - nodes.push_back(13); - std::vector entries; - entries.push_back(GetBucketDiffCommand::Entry()); - entries.back()._gid = document::GlobalId("1234567890abcdef"); - entries.back()._timestamp = 123456; - entries.back()._headerSize = 100; - entries.back()._bodySize = 64_Ki; - entries.back()._flags = 1; - entries.back()._hasMask = 3; - - EXPECT_EQ("Entry(timestamp: 123456, gid(0x313233343536373839306162), hasMask: 0x3,\n" - " header size: 100, body size: 65536, flags 0x1)", - entries.back().toString(true)); - - auto cmd = std::make_shared(_bucket, nodes, 1056); - cmd->getDiff() = entries; - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - - auto reply = std::make_shared(*cmd2); - EXPECT_EQ(entries, reply->getDiff()); - auto reply2 = copyReply(reply); - - EXPECT_EQ(nodes, reply2->getNodes()); - EXPECT_EQ(entries, reply2->getDiff()); - EXPECT_EQ(Timestamp(1056), reply2->getMaxTimestamp()); -} - -namespace { - -ApplyBucketDiffCommand::Entry dummy_apply_entry() { - ApplyBucketDiffCommand::Entry e; - e._docName = "my cool id"; - vespalib::string header_data = "fancy header"; - e._headerBlob.resize(header_data.size()); - memcpy(&e._headerBlob[0], header_data.data(), header_data.size()); - - vespalib::string body_data = "fancier body!"; - e._bodyBlob.resize(body_data.size()); - memcpy(&e._bodyBlob[0], body_data.data(), body_data.size()); - - GetBucketDiffCommand::Entry meta; - meta._timestamp = 567890; - meta._hasMask = 0x3; - meta._flags = 0x1; - meta._headerSize = 12345; - meta._headerSize = header_data.size(); - meta._bodySize = body_data.size(); - - e._entry = meta; - return e; -} - -} - -TEST_P(StorageProtocolTest, apply_bucket_diff) { - std::vector nodes; - nodes.push_back(4); - nodes.push_back(13); - std::vector entries = {dummy_apply_entry()}; - - auto cmd = std::make_shared(_bucket, nodes); - cmd->getDiff() = entries; - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - - auto reply = std::make_shared(*cmd2); - auto reply2 = copyReply(reply); - - EXPECT_EQ(nodes, reply2->getNodes()); - EXPECT_EQ(entries, reply2->getDiff()); -} - -namespace { - struct MyCommand : public api::InternalCommand { - MyCommand() : InternalCommand(101) {} - - api::StorageReply::UP makeReply() override; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override { - out << "MyCommand()"; - if (verbose) { - out << " : "; - InternalCommand::print(out, verbose, indent); - } - } - }; - - struct MyReply : public api::InternalReply { - MyReply(const MyCommand& cmd) : InternalReply(102, cmd) {} - - void print(std::ostream& out, bool verbose, const std::string& indent) const override { - out << "MyReply()"; - if (verbose) { - out << " : "; - InternalReply::print(out, verbose, indent); - } - } - }; - - api::StorageReply::UP MyCommand::makeReply() { - return std::make_unique(*this); - } -} - -TEST_P(StorageProtocolTest, internal_message) { - MyCommand cmd; - MyReply reply(cmd); - // TODO what's this even intended to test? -} - -TEST_P(StorageProtocolTest, set_bucket_state_with_inactive_state) { - auto cmd = std::make_shared(_bucket, SetBucketStateCommand::INACTIVE); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(_bucket, cmd2->getBucket()); - - auto reply = std::make_shared(*cmd2); - auto reply2 = copyReply(reply); - - EXPECT_EQ(SetBucketStateCommand::INACTIVE, cmd2->getState()); - EXPECT_EQ(_bucket, reply2->getBucket()); -} - -TEST_P(StorageProtocolTest, set_bucket_state_with_active_state) { - auto cmd = std::make_shared(_bucket, SetBucketStateCommand::ACTIVE); - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(SetBucketStateCommand::ACTIVE, cmd2->getState()); -} - -TEST_P(StorageProtocolTest, put_command_with_condition) { - auto cmd = std::make_shared(_bucket, _testDoc, 14); - cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); - - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); -} - -TEST_P(StorageProtocolTest, update_command_with_condition) { - auto update = std::make_shared( - _docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId()); - auto cmd = std::make_shared(_bucket, update, 14); - cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); - - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); -} - -TEST_P(StorageProtocolTest, remove_command_with_condition) { - auto cmd = std::make_shared(_bucket, _testDocId, 159); - cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); - - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); -} - -TEST_P(StorageProtocolTest, put_command_with_bucket_space) { - document::Bucket bucket(document::BucketSpace(5), _bucket_id); - auto cmd = std::make_shared(bucket, _testDoc, 14); - - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(bucket, cmd2->getBucket()); -} - -TEST_P(StorageProtocolTest, create_visitor_with_bucket_space) { - document::BucketSpace bucketSpace(5); - auto cmd = std::make_shared(bucketSpace, "library", "id", "doc selection"); - - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(bucketSpace, cmd2->getBucketSpace()); -} - -TEST_P(StorageProtocolTest, request_bucket_info_with_bucket_space) { - document::BucketSpace bucketSpace(5); - std::vector ids = {document::BucketId(3)}; - auto cmd = std::make_shared(bucketSpace, ids); - - auto cmd2 = copyCommand(cmd); - EXPECT_EQ(bucketSpace, cmd2->getBucketSpace()); - EXPECT_EQ(ids, cmd2->getBuckets()); -} - -TEST_P(StorageProtocolTest, serialized_size_is_used_to_set_approx_size_of_storage_message) { - auto cmd = std::make_shared(_bucket, _testDoc, 14); - EXPECT_EQ(50u, cmd->getApproxByteSize()); - - auto cmd2 = copyCommand(cmd); - auto version = GetParam(); - if (version.getMajor() == 7) { // Protobuf-based encoding - EXPECT_EQ(158u, cmd2->getApproxByteSize()); - } else { // Legacy encoding - EXPECT_EQ(181u, cmd2->getApproxByteSize()); - } -} - -TEST_P(StorageProtocolTest, track_memory_footprint_for_some_messages) { - EXPECT_EQ(72u, sizeof(StorageMessage)); - EXPECT_EQ(88u, sizeof(StorageReply)); - EXPECT_EQ(112u, sizeof(BucketReply)); - EXPECT_EQ(8u, sizeof(document::BucketId)); - EXPECT_EQ(16u, sizeof(document::Bucket)); - EXPECT_EQ(32u, sizeof(BucketInfo)); - EXPECT_EQ(144u, sizeof(BucketInfoReply)); - EXPECT_EQ(288u, sizeof(PutReply)); - EXPECT_EQ(272u, sizeof(UpdateReply)); - EXPECT_EQ(264u, sizeof(RemoveReply)); - EXPECT_EQ(352u, sizeof(GetReply)); - EXPECT_EQ(88u, sizeof(StorageCommand)); - EXPECT_EQ(112u, sizeof(BucketCommand)); - EXPECT_EQ(112u, sizeof(BucketInfoCommand)); - EXPECT_EQ(112u + sizeof(vespalib::string), sizeof(TestAndSetCommand)); - EXPECT_EQ(144u + sizeof(vespalib::string), sizeof(PutCommand)); - EXPECT_EQ(144u + sizeof(vespalib::string), sizeof(UpdateCommand)); - EXPECT_EQ(224u + sizeof(vespalib::string), sizeof(RemoveCommand)); - EXPECT_EQ(296u, sizeof(GetCommand)); -} - -} // storage::api diff --git a/storageapi/src/tests/message/.gitignore b/storageapi/src/tests/message/.gitignore deleted file mode 100644 index 1d859456ef9..00000000000 --- a/storageapi/src/tests/message/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.So -*.core -*.so -.*.swp -.config.log -.depend -Makefile -test.vlog -testrunner diff --git a/storageapi/src/tests/messageapi/.gitignore b/storageapi/src/tests/messageapi/.gitignore deleted file mode 100644 index 1d859456ef9..00000000000 --- a/storageapi/src/tests/messageapi/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.So -*.core -*.so -.*.swp -.config.log -.depend -Makefile -test.vlog -testrunner diff --git a/storageapi/src/tests/messageapi/CMakeLists.txt b/storageapi/src/tests/messageapi/CMakeLists.txt deleted file mode 100644 index 5bc89dea4fc..00000000000 --- a/storageapi/src/tests/messageapi/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageapi_testmessageapi - SOURCES - storage_message_address_test.cpp - DEPENDS - storageapi - GTest::GTest -) diff --git a/storageapi/src/tests/messageapi/storage_message_address_test.cpp b/storageapi/src/tests/messageapi/storage_message_address_test.cpp deleted file mode 100644 index ea59fefc924..00000000000 --- a/storageapi/src/tests/messageapi/storage_message_address_test.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include - -using namespace ::testing; - -namespace storage::api { - -namespace { - -size_t hash_of(const vespalib::string & cluster, const lib::NodeType& type, uint16_t index) { - return StorageMessageAddress(&cluster, type, index).internal_storage_hash(); -} - -} - -TEST(StorageMessageAddressTest, storage_hash_covers_all_expected_fields) { - EXPECT_EQ(hash_of("foo", lib::NodeType::STORAGE, 0), - hash_of("foo", lib::NodeType::STORAGE, 0)); - EXPECT_EQ(hash_of("foo", lib::NodeType::DISTRIBUTOR, 0), - hash_of("foo", lib::NodeType::DISTRIBUTOR, 0)); - EXPECT_EQ(hash_of("foo", lib::NodeType::STORAGE, 123), - hash_of("foo", lib::NodeType::STORAGE, 123)); - EXPECT_EQ(hash_of("foo", lib::NodeType::STORAGE, 0), - hash_of("bar", lib::NodeType::STORAGE, 0)); - - // These tests are all true with extremely high probability, though they do - // depend on a hash function that may inherently cause collisions. - EXPECT_NE(hash_of("foo", lib::NodeType::STORAGE, 0), - hash_of("foo", lib::NodeType::DISTRIBUTOR, 0)); - EXPECT_NE(hash_of("foo", lib::NodeType::STORAGE, 0), - hash_of("foo", lib::NodeType::STORAGE, 1)); - - EXPECT_EQ(16u, sizeof(StorageMessageAddress)); - EXPECT_EQ(72u, sizeof(StorageMessage)); - EXPECT_EQ(16u, sizeof(mbus::Trace)); -} - -} // storage::api diff --git a/storageapi/src/vespa/storageapi/.gitignore b/storageapi/src/vespa/storageapi/.gitignore deleted file mode 100644 index 43485abf58c..00000000000 --- a/storageapi/src/vespa/storageapi/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.cvsignore -.depend -Makefile -features.h -/libstorageapi.so.5.1 diff --git a/storageapi/src/vespa/storageapi/CMakeLists.txt b/storageapi/src/vespa/storageapi/CMakeLists.txt deleted file mode 100644 index 7018bbc1b55..00000000000 --- a/storageapi/src/vespa/storageapi/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -vespa_add_library(storageapi - SOURCES - $ - $ - $ - $ - INSTALL lib64 - DEPENDS -) - -vespa_add_target_package_dependency(storageapi Protobuf) - diff --git a/storageapi/src/vespa/storageapi/app/.gitignore b/storageapi/src/vespa/storageapi/app/.gitignore deleted file mode 100644 index fa917bb5ae5..00000000000 --- a/storageapi/src/vespa/storageapi/app/.gitignore +++ /dev/null @@ -1,14 +0,0 @@ -*.lo -.depend -.depend.NEW -.deps -.libs -Makefile -dumpstatusfile -getbucketid -inspectfeedapistatus -testdocstorefile -testdocstorefile2 -testdocstoremem -testresendctrl -storageapi_getbucketid_app diff --git a/storageapi/src/vespa/storageapi/app/CMakeLists.txt b/storageapi/src/vespa/storageapi/app/CMakeLists.txt deleted file mode 100644 index 90e5567d705..00000000000 --- a/storageapi/src/vespa/storageapi/app/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(storageapi_getbucketid_app - SOURCES - getbucketid.cpp - DEPENDS - storageapi -) diff --git a/storageapi/src/vespa/storageapi/app/getbucketid.cpp b/storageapi/src/vespa/storageapi/app/getbucketid.cpp deleted file mode 100644 index 21f7912d1a1..00000000000 --- a/storageapi/src/vespa/storageapi/app/getbucketid.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include -#include -#include - -int main(int argc, char** argv) -{ - if (argc != 2) { - std::cerr << "Usage: getbucketid \n"; - return 1; - } - document::BucketIdFactory factory; - document::BucketId id = factory.getBucketId(document::DocumentId(argv[1])); - - printf("%s has bucketid %s\n", argv[1], id.toString().c_str()); -} - - diff --git a/storageapi/src/vespa/storageapi/buckets/.gitignore b/storageapi/src/vespa/storageapi/buckets/.gitignore deleted file mode 100644 index 7ad21cf1c5e..00000000000 --- a/storageapi/src/vespa/storageapi/buckets/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.lo -.*.swp -.depend -.depend.NEW -.deps -.libs -Makefile -config-stor-bucketidgen.cpp -config-stor-bucketidgen.h diff --git a/storageapi/src/vespa/storageapi/buckets/CMakeLists.txt b/storageapi/src/vespa/storageapi/buckets/CMakeLists.txt deleted file mode 100644 index ee087f6f566..00000000000 --- a/storageapi/src/vespa/storageapi/buckets/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageapi_buckets OBJECT - SOURCES - bucketinfo.cpp - DEPENDS -) diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp deleted file mode 100644 index ff2f40e736b..00000000000 --- a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "bucketinfo.h" -#include -#include -#include - -namespace storage::api { - -static_assert(sizeof(BucketInfo) == 32, "BucketInfo should be 32 bytes"); - -BucketInfo::BucketInfo() noexcept - : _lastModified(0), - _checksum(0), - _docCount(0), - _totDocSize(1), - _metaCount(0), - _usedFileSize(1), - _ready(false), - _active(false) -{} - -BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize) noexcept - : _lastModified(0), - _checksum(checksum), - _docCount(docCount), - _totDocSize(totDocSize), - _metaCount(docCount), - _usedFileSize(totDocSize), - _ready(false), - _active(false) -{} - -BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, - uint32_t totDocSize, uint32_t metaCount, - uint32_t usedFileSize) noexcept - : _lastModified(0), - _checksum(checksum), - _docCount(docCount), - _totDocSize(totDocSize), - _metaCount(metaCount), - _usedFileSize(usedFileSize), - _ready(false), - _active(false) -{} - -BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, - uint32_t totDocSize, uint32_t metaCount, - uint32_t usedFileSize, - bool ready, bool active) noexcept - : _lastModified(0), - _checksum(checksum), - _docCount(docCount), - _totDocSize(totDocSize), - _metaCount(metaCount), - _usedFileSize(usedFileSize), - _ready(ready), - _active(active) -{} - -BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, - uint32_t totDocSize, uint32_t metaCount, - uint32_t usedFileSize, - bool ready, bool active, Timestamp lastModified) noexcept - : _lastModified(lastModified), - _checksum(checksum), - _docCount(docCount), - _totDocSize(totDocSize), - _metaCount(metaCount), - _usedFileSize(usedFileSize), - _ready(ready), - _active(active) -{} - -bool -BucketInfo::operator==(const BucketInfo& info) const noexcept -{ - return (_checksum == info._checksum && - _docCount == info._docCount && - _totDocSize == info._totDocSize && - _metaCount == info._metaCount && - _usedFileSize == info._usedFileSize && - _ready == info._ready && - _active == info._active); -} - -// TODO: add ready/active to printing -vespalib::string -BucketInfo::toString() const -{ - vespalib::asciistream out; - out << "BucketInfo("; - if (valid()) { - out << "crc 0x" << vespalib::hex << _checksum << vespalib::dec - << ", docCount " << _docCount - << ", totDocSize " << _totDocSize; - if (_totDocSize != _usedFileSize) { - out << ", metaCount " << _metaCount - << ", usedFileSize " << _usedFileSize; - } - out << ", ready " << (_ready ? "true" : "false") - << ", active " << (_active ? "true" : "false"); - - if (_lastModified != 0) { - out << ", last modified " << _lastModified; - } - } else { - out << "invalid"; - } - out << ")"; - return out.str(); -} - -void -BucketInfo::printXml(vespalib::XmlOutputStream& xos) const -{ - using namespace vespalib::xml; - xos << XmlAttribute("checksum", _checksum, XmlAttribute::HEX) - << XmlAttribute("docs", _docCount) - << XmlAttribute("size", _totDocSize) - << XmlAttribute("metacount", _metaCount) - << XmlAttribute("usedfilesize", _usedFileSize) - << XmlAttribute("ready", _ready) - << XmlAttribute("active", _active) - << XmlAttribute("lastmodified", _lastModified); -} - -std::ostream & -operator << (std::ostream & os, const BucketInfo & bucketInfo) { - return os << bucketInfo.toString(); -} - -} diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.h b/storageapi/src/vespa/storageapi/buckets/bucketinfo.h deleted file mode 100644 index d7b407185f6..00000000000 --- a/storageapi/src/vespa/storageapi/buckets/bucketinfo.h +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class BucketInfo - * @ingroup bucket - * - * @brief Contains metadata about a bucket. - * - * This class contains metadata about a bucket. It is used to send metadata - * within storage nodes and to distributors. - * - * @version $Id$ - */ - -#pragma once - -#include -#include -#include - -namespace storage::api { - -class BucketInfo -{ - Timestamp _lastModified; - uint32_t _checksum; - uint32_t _docCount; - uint32_t _totDocSize; - uint32_t _metaCount; - uint32_t _usedFileSize; - bool _ready; - bool _active; - -public: - BucketInfo() noexcept; - BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize) noexcept; - BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize, - uint32_t metaCount, uint32_t usedFileSize) noexcept; - BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize, - uint32_t metaCount, uint32_t usedFileSize, - bool ready, bool active) noexcept; - BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize, - uint32_t metaCount, uint32_t usedFileSize, - bool ready, bool active, Timestamp lastModified) noexcept; - - Timestamp getLastModified() const noexcept { return _lastModified; } - uint32_t getChecksum() const noexcept { return _checksum; } - uint32_t getDocumentCount() const noexcept { return _docCount; } - uint32_t getTotalDocumentSize() const noexcept { return _totDocSize; } - uint32_t getMetaCount() const noexcept { return _metaCount; } - uint32_t getUsedFileSize() const noexcept { return _usedFileSize; } - bool isReady() const noexcept { return _ready; } - bool isActive() const noexcept { return _active; } - - void setChecksum(uint32_t crc) noexcept { _checksum = crc; } - void setDocumentCount(uint32_t count) noexcept { _docCount = count; } - void setTotalDocumentSize(uint32_t size) noexcept { _totDocSize = size; } - void setMetaCount(uint32_t count) noexcept { _metaCount = count; } - void setUsedFileSize(uint32_t size) noexcept { _usedFileSize = size; } - void setReady(bool ready = true) noexcept { _ready = ready; } - void setActive(bool active = true) noexcept { _active = active; } - void setLastModified(Timestamp lastModified) noexcept { _lastModified = lastModified; } - - /** - * Only compare checksum, total document count and document - * size, not meta count or used file size. - */ - bool equalDocumentInfo(const BucketInfo& other) const noexcept { - return (_checksum == other._checksum - && _docCount == other._docCount - && _totDocSize == other._totDocSize); - } - - bool operator==(const BucketInfo& info) const noexcept; - bool valid() const noexcept { return (_docCount > 0 || _totDocSize == 0); } - bool empty() const noexcept { - return _metaCount == 0 && _usedFileSize == 0 && _checksum == 0; - } - vespalib::string toString() const; - void printXml(vespalib::XmlOutputStream&) const; -}; - -std::ostream & operator << (std::ostream & os, const BucketInfo & bucketInfo); - -} diff --git a/storageapi/src/vespa/storageapi/defs.h b/storageapi/src/vespa/storageapi/defs.h deleted file mode 100644 index 9ee9fdf2218..00000000000 --- a/storageapi/src/vespa/storageapi/defs.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \file defs.h - * - * \brief Common declarations for the storage API code. - */ -#pragma once - -#include - -namespace storage:: api { - -typedef uint64_t Timestamp; -typedef uint32_t VisitorId; - -const Timestamp MAX_TIMESTAMP = (Timestamp)-1ll; - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/.gitignore b/storageapi/src/vespa/storageapi/mbusprot/.gitignore deleted file mode 100644 index 8e91fe9cab0..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -*.lo -.*.swp -.depend -.depend.NEW -.deps -.libs -Makefile -*.pb.h -*.pb.cc - diff --git a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt deleted file mode 100644 index 43c1da32502..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -find_package(Protobuf REQUIRED) -PROTOBUF_GENERATE_CPP(storageapi_PROTOBUF_SRCS storageapi_PROTOBUF_HDRS - protobuf/common.proto - protobuf/feed.proto - protobuf/inspect.proto - protobuf/visiting.proto - protobuf/maintenance.proto) - -vespa_add_source_target(protobufgen_storageapi_mbusprot DEPENDS ${storageapi_PROTOBUF_SRCS} ${storageapi_PROTOBUF_HDRS}) - -vespa_suppress_warnings_for_protobuf_sources(SOURCES ${storageapi_PROTOBUF_SRCS}) - -# protoc explicitly annotates methods with inline, which triggers -Werror=inline when -# the header file grows over a certain size. -set_source_files_properties(protocolserialization7.cpp PROPERTIES COMPILE_FLAGS "-Wno-inline") - -vespa_add_library(storageapi_mbusprot OBJECT - SOURCES - storagemessage.cpp - storagecommand.cpp - storagereply.cpp - protocolserialization.cpp - storageprotocol.cpp - protocolserialization4_2.cpp - protocolserialization5_0.cpp - protocolserialization5_1.cpp - protocolserialization5_2.cpp - protocolserialization6_0.cpp - protocolserialization7.cpp - ${storageapi_PROTOBUF_SRCS} - DEPENDS -) diff --git a/storageapi/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h deleted file mode 100644 index f3c6a6f6856..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "protocolserialization.h" - -namespace storage::mbusprot { - -/* - * Utility base class for pre-v7 (protobuf) serialization implementations. - * - * TODO remove on Vespa 8 alongside legacy serialization formats. - */ -class LegacyProtocolSerialization : public ProtocolSerialization { - const std::shared_ptr _repo; -public: - explicit LegacyProtocolSerialization(const std::shared_ptr& repo) - : _repo(repo) - {} - - const document::DocumentTypeRepo& getTypeRepo() const { return *_repo; } - const std::shared_ptr getTypeRepoSp() const { return _repo; } - - virtual document::Bucket getBucket(document::ByteBuffer& buf) const = 0; - virtual void putBucket(const document::Bucket& bucket, vespalib::GrowableByteBuffer& buf) const = 0; - virtual document::BucketSpace getBucketSpace(document::ByteBuffer& buf) const = 0; - virtual void putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer& buf) const = 0; - virtual api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const = 0; - virtual void putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const = 0; -}; - -} // storage::mbusprot diff --git a/storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h b/storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h deleted file mode 100644 index 40e09f23697..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \brief Maps return code values used between 4.2 and 5.0 - */ -#pragma once - -namespace storage { -namespace mbusprot { - -int getOldErrorCode(api::ReturnCode::Result newErrorCode) { - switch (newErrorCode) { - case api::ReturnCode::OK: return 1; - case api::ReturnCode::EXISTS: return 1001; - case api::ReturnCode::NOT_READY: return 2000; - case api::ReturnCode::WRONG_DISTRIBUTION: return 2001; - case api::ReturnCode::REJECTED: return 2002; - case api::ReturnCode::ABORTED: return 2003; - case api::ReturnCode::BUCKET_NOT_FOUND: return 2004; - case api::ReturnCode::BUCKET_DELETED: return 2004; - case api::ReturnCode::TIMESTAMP_EXIST: return 2005; - case api::ReturnCode::UNKNOWN_COMMAND: return 3000; - case api::ReturnCode::NOT_IMPLEMENTED: return 3001; - case api::ReturnCode::ILLEGAL_PARAMETERS: return 3002; - case api::ReturnCode::IGNORED: return 3003; - case api::ReturnCode::UNPARSEABLE: return 3004; - case api::ReturnCode::NOT_CONNECTED: return 4000; - case api::ReturnCode::TIMEOUT: return 4003; - case api::ReturnCode::BUSY: return 4004; - case api::ReturnCode::NO_SPACE: return 5000; - case api::ReturnCode::DISK_FAILURE: return 5001; - case api::ReturnCode::IO_FAILURE: return 5003; - case api::ReturnCode::INTERNAL_FAILURE: return 6000; - default: return 6001; - } -} - -api::ReturnCode::Result getNewErrorCode(int oldErrorCode) { - switch (oldErrorCode) { - case 1: return api::ReturnCode::OK; - case 1000: return api::ReturnCode::OK; // NOT_FOUND - case 1001: return api::ReturnCode::EXISTS; - case 2000: return api::ReturnCode::NOT_READY; - case 2001: return api::ReturnCode::WRONG_DISTRIBUTION; - case 2002: return api::ReturnCode::REJECTED; - case 2003: return api::ReturnCode::ABORTED; - case 2004: return api::ReturnCode::BUCKET_NOT_FOUND; - case 2005: return api::ReturnCode::TIMESTAMP_EXIST; - case 3000: return api::ReturnCode::UNKNOWN_COMMAND; - case 3001: return api::ReturnCode::NOT_IMPLEMENTED; - case 3002: return api::ReturnCode::ILLEGAL_PARAMETERS; - case 3003: return api::ReturnCode::IGNORED; - case 3004: return api::ReturnCode::UNPARSEABLE; - case 4000: return api::ReturnCode::NOT_CONNECTED; - case 4001: return api::ReturnCode::BUSY; // OVERLOAD; - case 4002: return api::ReturnCode::NOT_READY; // REMOTE_DISABLED; - case 4003: return api::ReturnCode::TIMEOUT; - case 4004: return api::ReturnCode::BUSY; - case 5000: return api::ReturnCode::NO_SPACE; - case 5001: return api::ReturnCode::DISK_FAILURE; - case 5003: return api::ReturnCode::IO_FAILURE; - case 6000: return api::ReturnCode::INTERNAL_FAILURE; - default: return api::ReturnCode::INTERNAL_FAILURE; - } -} - -} // mbusprot -} // storage - diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/common.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/common.proto deleted file mode 100644 index 49e1a8f8aba..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/common.proto +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -syntax = "proto3"; - -option cc_enable_arenas = true; - -package storage.mbusprot.protobuf; - -// Note: we use a *Request/*Response naming convention rather than *Command/*Reply, -// as the former is the gRPC convention and that's where we intend to move. - -message BucketSpace { - uint64 space_id = 1; -} - -message BucketId { - fixed64 raw_id = 1; -} - -message Bucket { - uint64 space_id = 1; - fixed64 raw_bucket_id = 2; -} - -// Next tag to use: 3 -message BucketInfo { - uint64 last_modified_timestamp = 1; - fixed32 legacy_checksum = 2; - // TODO v2 checksum - uint32 doc_count = 3; - uint32 total_doc_size = 4; - uint32 meta_count = 5; - uint32 used_file_size = 6; - bool ready = 7; - bool active = 8; -} - -message GlobalId { - // 96 bits of GID data in _little_ endian. High entropy, so fixed encoding is better than varint. - // Low 64 bits as if memcpy()ed from bytes [0, 8) of the GID buffer - fixed64 lo_64 = 1; - // High 32 bits as if memcpy()ed from bytes [8, 12) of the GID buffer - fixed32 hi_32 = 2; -} - -// TODO these should ideally be gRPC headers.. -message RequestHeader { - uint64 message_id = 1; - uint32 priority = 2; // Always in range [0, 255] - uint32 source_index = 3; // Always in range [0, 65535] - fixed32 loadtype_id = 4; // It's a hash with high entropy, so fixed encoding is better than varint -} - -// TODO these should ideally be gRPC headers.. -message ResponseHeader { - // TODO this should ideally be gRPC Status... - uint32 return_code_id = 1; - bytes return_code_message = 2; // FIXME it's `bytes` since `string` will check for UTF-8... might not hold... - uint64 message_id = 3; - uint32 priority = 4; // Always in range [0, 255] -} - -message Document { - bytes payload = 1; -} - -message DocumentId { - bytes id = 1; -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto deleted file mode 100644 index cb923db3c3c..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -syntax = "proto3"; - -option cc_enable_arenas = true; - -package storage.mbusprot.protobuf; - -import "common.proto"; - -message TestAndSetCondition { - bytes selection = 1; -} - -message PutRequest { - Bucket bucket = 1; - Document document = 2; - uint64 new_timestamp = 3; - uint64 expected_old_timestamp = 4; // If zero; no expectation. - TestAndSetCondition condition = 5; -} - -message PutResponse { - BucketInfo bucket_info = 1; - BucketId remapped_bucket_id = 2; - bool was_found = 3; -} - -message Update { - bytes payload = 1; -} - -message UpdateRequest { - Bucket bucket = 1; - Update update = 2; - uint64 new_timestamp = 3; - uint64 expected_old_timestamp = 4; // If zero; no expectation. - TestAndSetCondition condition = 5; -} - -message UpdateResponse { - BucketInfo bucket_info = 1; - BucketId remapped_bucket_id = 2; - uint64 updated_timestamp = 3; -} - -message RemoveRequest { - Bucket bucket = 1; - bytes document_id = 2; - uint64 new_timestamp = 3; - TestAndSetCondition condition = 4; -} - -message RemoveResponse { - BucketInfo bucket_info = 1; - BucketId remapped_bucket_id = 2; - uint64 removed_timestamp = 3; -} - -message GetRequest { - Bucket bucket = 1; - bytes document_id = 2; - bytes field_set = 3; - uint64 before_timestamp = 4; - enum InternalReadConsistency { - Strong = 0; // Default for a good reason. - Weak = 1; - } - InternalReadConsistency internal_read_consistency = 5; -} - -message GetResponse { - Document document = 1; - uint64 last_modified_timestamp = 2; - BucketInfo bucket_info = 3; - BucketId remapped_bucket_id = 4; - // Note: last_modified_timestamp and tombstone_timestamp are mutually exclusive. - // Tracked separately (rather than being a flag bool) to avoid issues during rolling upgrades. - uint64 tombstone_timestamp = 5; -} - -message RevertRequest { - Bucket bucket = 1; - repeated uint64 revert_tokens = 2; -} - -message RevertResponse { - BucketInfo bucket_info = 1; - BucketId remapped_bucket_id = 2; -} - -message RemoveLocationRequest { - Bucket bucket = 1; - bytes document_selection = 2; -} - -message RemoveLocationStats { - uint32 documents_removed = 1; -} - -message RemoveLocationResponse { - BucketInfo bucket_info = 1; - BucketId remapped_bucket_id = 2; - RemoveLocationStats stats = 3; -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto deleted file mode 100644 index c3f4b1263a1..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -syntax = "proto3"; - -option cc_enable_arenas = true; - -package storage.mbusprot.protobuf; - -import "common.proto"; - -message StatBucketRequest { - Bucket bucket = 1; - bytes document_selection = 2; -} - -message StatBucketResponse { - BucketId remapped_bucket_id = 1; - bytes results = 2; -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto deleted file mode 100644 index 7f7ab1d7c0b..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -syntax = "proto3"; - -option cc_enable_arenas = true; - -package storage.mbusprot.protobuf; - -import "common.proto"; - -message DeleteBucketRequest { - Bucket bucket = 1; - BucketInfo expected_bucket_info = 2; -} - -message DeleteBucketResponse { - BucketInfo bucket_info = 1; - BucketId remapped_bucket_id = 2; -} - -message CreateBucketRequest { - Bucket bucket = 1; - bool create_as_active = 2; -} - -message CreateBucketResponse { - BucketInfo bucket_info = 1; - BucketId remapped_bucket_id = 2; -} - -message MergeNode { - uint32 index = 1; - bool source_only = 2; -} - -message MergeBucketRequest { - Bucket bucket = 1; - uint32 cluster_state_version = 2; - uint64 max_timestamp = 3; - repeated MergeNode nodes = 4; - repeated uint32 node_chain = 5; - bool unordered_forwarding = 6; -} - -message MergeBucketResponse { - BucketId remapped_bucket_id = 1; -} - -message MetaDiffEntry { - uint64 timestamp = 1; - GlobalId gid = 2; - uint32 header_size = 3; - uint32 body_size = 4; - uint32 flags = 5; - uint32 presence_mask = 6; -} - -message GetBucketDiffRequest { - Bucket bucket = 1; - uint64 max_timestamp = 2; - repeated MergeNode nodes = 3; - repeated MetaDiffEntry diff = 4; -} - -message GetBucketDiffResponse { - BucketId remapped_bucket_id = 1; - repeated MetaDiffEntry diff = 2; -} - -message ApplyDiffEntry { - MetaDiffEntry entry_meta = 1; - bytes document_id = 2; - bytes header_blob = 3; - bytes body_blob = 4; -} - -message ApplyBucketDiffRequest { - Bucket bucket = 1; - repeated MergeNode nodes = 2; - uint32 max_buffer_size = 3; - repeated ApplyDiffEntry entries = 4; -} - -message ApplyBucketDiffResponse { - BucketId remapped_bucket_id = 1; - repeated ApplyDiffEntry entries = 4; -} - -message ExplicitBucketSet { - // `Bucket` is not needed, as the space is inferred from the owning message. - repeated BucketId bucket_ids = 2; -} - -message AllBuckets { - uint32 distributor_index = 1; - bytes cluster_state = 2; - bytes distribution_hash = 3; -} - -message RequestBucketInfoRequest { - BucketSpace bucket_space = 1; - oneof request_for { - ExplicitBucketSet explicit_bucket_set = 2; - AllBuckets all_buckets = 3; - } -} - -message BucketAndBucketInfo { - fixed64 raw_bucket_id = 1; - BucketInfo bucket_info = 2; -} - -message SupportedNodeFeatures { - bool unordered_merge_chaining = 1; -} - -message RequestBucketInfoResponse { - repeated BucketAndBucketInfo bucket_infos = 1; - // Only present for full bucket info fetches (not for explicit buckets) - SupportedNodeFeatures supported_node_features = 2; -} - -message NotifyBucketChangeRequest { - Bucket bucket = 1; - BucketInfo bucket_info = 2; -} - -message NotifyBucketChangeResponse { - // Currently empty -} - -message SplitBucketRequest { - Bucket bucket = 1; - uint32 min_split_bits = 2; - uint32 max_split_bits = 3; - uint32 min_byte_size = 4; - uint32 min_doc_count = 5; -} - -message SplitBucketResponse { - BucketId remapped_bucket_id = 1; - repeated BucketAndBucketInfo split_info = 2; -} - -message JoinBucketsRequest { - Bucket bucket = 1; - repeated BucketId source_buckets = 2; - uint32 min_join_bits = 3; -} - -message JoinBucketsResponse { - BucketInfo bucket_info = 1; - BucketId remapped_bucket_id = 2; -} - -message SetBucketStateRequest { - enum BucketState { - Inactive = 0; - Active = 1; - } - - Bucket bucket = 1; - BucketState state = 2; -} - -message SetBucketStateResponse { - BucketId remapped_bucket_id = 1; -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/visiting.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/visiting.proto deleted file mode 100644 index 35d69bc2d3e..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/visiting.proto +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -syntax = "proto3"; - -option cc_enable_arenas = true; - -package storage.mbusprot.protobuf; - -import "common.proto"; - -message ClientVisitorParameter { - bytes key = 1; - bytes value = 2; -} - -message VisitorConstraints { - bytes document_selection = 1; - uint64 from_time_usec = 2; - uint64 to_time_usec = 3; - bool visit_removes = 4; - bytes field_set = 5; - bool visit_inconsistent_buckets = 6; -} - -message VisitorControlMeta { - bytes instance_id = 1; - bytes library_name = 2; - uint32 visitor_command_id = 3; - bytes control_destination = 4; - bytes data_destination = 5; - - // TODO move? - uint32 max_pending_reply_count = 6; - uint32 queue_timeout = 7; - uint32 max_buckets_per_visitor = 8; -} - -message CreateVisitorRequest { - BucketSpace bucket_space = 1; - repeated BucketId buckets = 2; - - VisitorConstraints constraints = 3; - VisitorControlMeta control_meta = 4; - repeated ClientVisitorParameter client_parameters = 5; -} - -message VisitorStatistics { - uint32 buckets_visited = 1; - uint64 documents_visited = 2; - uint64 bytes_visited = 3; - uint64 documents_returned = 4; - uint64 bytes_returned = 5; - uint64 second_pass_documents_returned = 6; // TODO don't include? orderdoc only - uint64 second_pass_bytes_returned = 7; // TODO don't include? orderdoc only -} - -message CreateVisitorResponse { - VisitorStatistics visitor_statistics = 1; -} - -message DestroyVisitorRequest { - bytes instance_id = 1; -} - -message DestroyVisitorResponse { - // Currently empty -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h b/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h deleted file mode 100644 index 9accfdf75ee..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -// Disable warnings emitted by protoc generated files -#pragma GCC diagnostic push -#ifndef __clang__ -#pragma GCC diagnostic ignored "-Wsuggest-override" -#endif - -#include -#include -#include -#include - -#pragma GCC diagnostic pop diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp deleted file mode 100644 index 4614659c458..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "protocolserialization4_2.h" -#include "serializationhelper.h" -#include "storagecommand.h" -#include "storagereply.h" -#include -#include -#include -#include -#include - - -#include - -#include -LOG_SETUP(".storage.api.mbusprot.serialization.base"); - -namespace storage::mbusprot { - -mbus::Blob -ProtocolSerialization::encode(const api::StorageMessage& msg) const -{ - vespalib::GrowableByteBuffer buf; - - buf.putInt(msg.getType().getId()); - switch (msg.getType().getId()) { - case api::MessageType::PUT_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::PUT_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::UPDATE_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::UPDATE_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::GET_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::GET_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::REMOVE_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::REMOVE_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::REVERT_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::REVERT_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::DELETEBUCKET_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::DELETEBUCKET_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::CREATEBUCKET_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::CREATEBUCKET_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::MERGEBUCKET_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::MERGEBUCKET_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::GETBUCKETDIFF_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::GETBUCKETDIFF_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::APPLYBUCKETDIFF_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::APPLYBUCKETDIFF_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::REQUESTBUCKETINFO_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::REQUESTBUCKETINFO_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::NOTIFYBUCKETCHANGE_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::NOTIFYBUCKETCHANGE_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::SPLITBUCKET_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::SPLITBUCKET_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::JOINBUCKETS_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::JOINBUCKETS_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::VISITOR_CREATE_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::VISITOR_CREATE_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::VISITOR_DESTROY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::VISITOR_DESTROY_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::STATBUCKET_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::STATBUCKET_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::REMOVELOCATION_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::REMOVELOCATION_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::SETBUCKETSTATE_ID: - onEncode(buf, static_cast(msg)); - break; - case api::MessageType::SETBUCKETSTATE_REPLY_ID: - onEncode(buf, static_cast(msg)); - break; - default: - LOG(error, "Trying to encode unhandled type %s", - msg.getType().toString().c_str()); - break; - } - - mbus::Blob retVal(buf.position()); - memcpy(retVal.data(), buf.getBuffer(), buf.position()); - return retVal; -} - -StorageCommand::UP -ProtocolSerialization::decodeCommand(mbus::BlobRef data) const -{ - LOG(spam, "Decode %d bytes of data.", data.size()); - if (data.size() < sizeof(int32_t)) { - std::ostringstream ost; - ost << "Request of size " << data.size() << " is not big enough to be " - "able to store a request."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); - } - - document::ByteBuffer buf(data.data(), data.size()); - int type; - buf.getIntNetwork(type); - SCmd::UP cmd; - switch (type) { - case api::MessageType::PUT_ID: - cmd = onDecodePutCommand(buf); break; - case api::MessageType::UPDATE_ID: - cmd = onDecodeUpdateCommand(buf); break; - case api::MessageType::GET_ID: - cmd = onDecodeGetCommand(buf); break; - case api::MessageType::REMOVE_ID: - cmd = onDecodeRemoveCommand(buf); break; - case api::MessageType::REVERT_ID: - cmd = onDecodeRevertCommand(buf); break; - case api::MessageType::CREATEBUCKET_ID: - cmd = onDecodeCreateBucketCommand(buf); break; - case api::MessageType::DELETEBUCKET_ID: - cmd = onDecodeDeleteBucketCommand(buf); break; - case api::MessageType::MERGEBUCKET_ID: - cmd = onDecodeMergeBucketCommand(buf); break; - case api::MessageType::GETBUCKETDIFF_ID: - cmd = onDecodeGetBucketDiffCommand(buf); break; - case api::MessageType::APPLYBUCKETDIFF_ID: - cmd = onDecodeApplyBucketDiffCommand(buf); break; - case api::MessageType::REQUESTBUCKETINFO_ID: - cmd = onDecodeRequestBucketInfoCommand(buf); break; - case api::MessageType::NOTIFYBUCKETCHANGE_ID: - cmd = onDecodeNotifyBucketChangeCommand(buf); break; - case api::MessageType::SPLITBUCKET_ID: - cmd = onDecodeSplitBucketCommand(buf); break; - case api::MessageType::JOINBUCKETS_ID: - cmd = onDecodeJoinBucketsCommand(buf); break; - case api::MessageType::VISITOR_CREATE_ID: - cmd = onDecodeCreateVisitorCommand(buf); break; - case api::MessageType::VISITOR_DESTROY_ID: - cmd = onDecodeDestroyVisitorCommand(buf); break; - case api::MessageType::STATBUCKET_ID: - cmd = onDecodeStatBucketCommand(buf); break; - case api::MessageType::REMOVELOCATION_ID: - cmd = onDecodeRemoveLocationCommand(buf); break; - case api::MessageType::SETBUCKETSTATE_ID: - cmd = onDecodeSetBucketStateCommand(buf); break; - default: - { - std::ostringstream ost; - ost << "Unknown storage command type " << type; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); - } - } - return std::make_unique(std::move(cmd)); -} - -StorageReply::UP -ProtocolSerialization::decodeReply(mbus::BlobRef data, const api::StorageCommand& cmd) const -{ - LOG(spam, "Decode %d bytes of data.", data.size()); - if (data.size() < sizeof(int32_t)) { - std::ostringstream ost; - ost << "Request of size " << data.size() << " is not big enough to be " - "able to store a request."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); - } - - document::ByteBuffer buf(data.data(), data.size()); - int type; - buf.getIntNetwork(type); - SRep::UP reply; - switch (type) { - case api::MessageType::PUT_REPLY_ID: - reply = onDecodePutReply(cmd, buf); break; - case api::MessageType::UPDATE_REPLY_ID: - reply = onDecodeUpdateReply(cmd, buf); break; - case api::MessageType::GET_REPLY_ID: - reply = onDecodeGetReply(cmd, buf); break; - case api::MessageType::REMOVE_REPLY_ID: - reply = onDecodeRemoveReply(cmd, buf); break; - case api::MessageType::REVERT_REPLY_ID: - reply = onDecodeRevertReply(cmd, buf); break; - case api::MessageType::CREATEBUCKET_REPLY_ID: - reply = onDecodeCreateBucketReply(cmd, buf); break; - case api::MessageType::DELETEBUCKET_REPLY_ID: - reply = onDecodeDeleteBucketReply(cmd, buf); break; - case api::MessageType::MERGEBUCKET_REPLY_ID: - reply = onDecodeMergeBucketReply(cmd, buf); break; - case api::MessageType::GETBUCKETDIFF_REPLY_ID: - reply = onDecodeGetBucketDiffReply(cmd, buf); break; - case api::MessageType::APPLYBUCKETDIFF_REPLY_ID: - reply = onDecodeApplyBucketDiffReply(cmd, buf); break; - case api::MessageType::REQUESTBUCKETINFO_REPLY_ID: - reply = onDecodeRequestBucketInfoReply(cmd, buf); break; - case api::MessageType::NOTIFYBUCKETCHANGE_REPLY_ID: - reply = onDecodeNotifyBucketChangeReply(cmd, buf); break; - case api::MessageType::SPLITBUCKET_REPLY_ID: - reply = onDecodeSplitBucketReply(cmd, buf); break; - case api::MessageType::JOINBUCKETS_REPLY_ID: - reply = onDecodeJoinBucketsReply(cmd, buf); break; - case api::MessageType::VISITOR_CREATE_REPLY_ID: - reply = onDecodeCreateVisitorReply(cmd, buf); break; - case api::MessageType::VISITOR_DESTROY_REPLY_ID: - reply = onDecodeDestroyVisitorReply(cmd, buf); break; - case api::MessageType::STATBUCKET_REPLY_ID: - reply = onDecodeStatBucketReply(cmd, buf); break; - case api::MessageType::REMOVELOCATION_REPLY_ID: - reply = onDecodeRemoveLocationReply(cmd, buf); break; - case api::MessageType::SETBUCKETSTATE_REPLY_ID: - reply = onDecodeSetBucketStateReply(cmd, buf); break; - default: - { - std::ostringstream ost; - ost << "Unknown message type " << type; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); - } - } - return std::make_unique(std::move(reply)); -} - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h deleted file mode 100644 index 653e9ded85a..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include -#include -#include -#include - -namespace document { - class ByteBuffer; -} -namespace mbus { - class Blob; - class BlobRef; -} -namespace vespalib { - class GrowableByteBuffer; -} -namespace storage::api { -class StorageCommand; -class StorageReply; -class PutCommand; -class PutReply; -class GetCommand; -class GetReply; -class RemoveCommand; -class RemoveReply; -class RevertCommand; -class RevertReply; -class DeleteBucketCommand; -class DeleteBucketReply; -class CreateBucketCommand; -class CreateBucketReply; -class MergeBucketCommand; -class MergeBucketReply; -class GetBucketDiffCommand; -class GetBucketDiffReply; -class ApplyBucketDiffCommand; -class ApplyBucketDiffReply; -class RequestBucketInfoCommand; -class RequestBucketInfoReply; -class NotifyBucketChangeCommand; -class NotifyBucketChangeReply; -class SplitBucketCommand; -class SplitBucketReply; -class StatBucketCommand; -class StatBucketReply; -class JoinBucketsCommand; -class JoinBucketsReply; -class SetBucketStateCommand; -class SetBucketStateReply; -class CreateVisitorCommand; -class RemoveLocationCommand; -class RemoveLocationReply; -} - -namespace storage::mbusprot { - -class SerializationHelper; -class StorageCommand; -class StorageReply; - -class ProtocolSerialization { -public: - virtual mbus::Blob encode(const api::StorageMessage&) const; - virtual std::unique_ptr decodeCommand(mbus::BlobRef) const; - virtual std::unique_ptr decodeReply( - mbus::BlobRef, const api::StorageCommand&) const; -protected: - ProtocolSerialization() = default; - virtual ~ProtocolSerialization() = default; - - typedef api::StorageCommand SCmd; - typedef api::StorageReply SRep; - typedef document::ByteBuffer BBuf; - typedef vespalib::GrowableByteBuffer GBBuf; - typedef SerializationHelper SH; - typedef StorageMessage SM; - - virtual void onEncode(GBBuf&, const api::PutCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::PutReply&) const = 0; - virtual void onEncode(GBBuf&, const api::UpdateCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::UpdateReply&) const = 0; - virtual void onEncode(GBBuf&, const api::GetCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::GetReply&) const = 0; - virtual void onEncode(GBBuf&, const api::RemoveCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::RemoveReply&) const = 0; - virtual void onEncode(GBBuf&, const api::RevertCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::RevertReply&) const = 0; - virtual void onEncode(GBBuf&, const api::DeleteBucketCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::DeleteBucketReply&) const = 0; - virtual void onEncode(GBBuf&, const api::CreateBucketCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::CreateBucketReply&) const = 0; - virtual void onEncode(GBBuf&, const api::MergeBucketCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::MergeBucketReply&) const = 0; - virtual void onEncode(GBBuf&, const api::GetBucketDiffCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::GetBucketDiffReply&) const = 0; - virtual void onEncode(GBBuf&, const api::ApplyBucketDiffCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::ApplyBucketDiffReply&) const = 0; - virtual void onEncode(GBBuf&, const api::RequestBucketInfoCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::RequestBucketInfoReply&) const = 0; - virtual void onEncode(GBBuf&, const api::NotifyBucketChangeCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::NotifyBucketChangeReply&) const = 0; - virtual void onEncode(GBBuf&, const api::SplitBucketCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::SplitBucketReply&) const = 0; - virtual void onEncode(GBBuf&, const api::JoinBucketsCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::JoinBucketsReply&) const = 0; - virtual void onEncode(GBBuf&, const api::SetBucketStateCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::SetBucketStateReply&) const = 0; - virtual void onEncode(GBBuf&, const api::CreateVisitorCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::CreateVisitorReply&) const = 0; - virtual void onEncode(GBBuf&, const api::DestroyVisitorCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::DestroyVisitorReply&) const = 0; - virtual void onEncode(GBBuf&, const api::RemoveLocationCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::RemoveLocationReply&) const = 0; - virtual void onEncode(GBBuf&, const api::StatBucketCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::StatBucketReply&) const = 0; - - virtual SCmd::UP onDecodePutCommand(BBuf&) const = 0; - virtual SRep::UP onDecodePutReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeUpdateCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeUpdateReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeGetCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeGetReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeRemoveCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeRemoveReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeRevertCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeRevertReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeDeleteBucketCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeDeleteBucketReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeCreateBucketCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeCreateBucketReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeMergeBucketCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeMergeBucketReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeGetBucketDiffCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeGetBucketDiffReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeApplyBucketDiffCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeApplyBucketDiffReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeRequestBucketInfoCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeRequestBucketInfoReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeNotifyBucketChangeCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeSplitBucketCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeSplitBucketReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeJoinBucketsCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeJoinBucketsReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeSetBucketStateCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeSetBucketStateReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeCreateVisitorReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeDestroyVisitorCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeStatBucketCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeStatBucketReply(const SCmd&, BBuf&) const = 0; -}; - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp deleted file mode 100644 index 1bea0ea74c9..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ /dev/null @@ -1,553 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "protocolserialization4_2.h" -#include "oldreturncodemapper.h" -#include "serializationhelper.h" -#include "storagecommand.h" -#include "storagereply.h" - -#include -#include -#include -#include -#include - -#include -LOG_SETUP(".storage.api.mbusprot.serialization.4_2"); - -using document::BucketSpace; -using document::AllFields; - -namespace storage::mbusprot { - -ProtocolSerialization4_2::ProtocolSerialization4_2( - const std::shared_ptr& repo) - : LegacyProtocolSerialization(repo) -{ -} - -void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::GetCommand& msg) const -{ - buf.putString(msg.getDocumentId().toString()); - putBucket(msg.getBucket(), buf); - buf.putLong(msg.getBeforeTimestamp()); - buf.putBoolean(false); - onEncodeCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization4_2::onDecodeGetCommand(BBuf& buf) const -{ - document::DocumentId did(SH::getString(buf)); - document::Bucket bucket = getBucket(buf); - api::Timestamp beforeTimestamp(SH::getLong(buf)); - bool headerOnly(SH::getBoolean(buf)); // Ignored header only flag - (void) headerOnly; - auto msg = std::make_unique(bucket, did, AllFields::NAME, beforeTimestamp); - onDecodeCommand(buf, *msg); - return msg; -} - -void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveCommand& msg) const -{ - buf.putString(msg.getDocumentId().toString()); - putBucket(msg.getBucket(), buf); - buf.putLong(msg.getTimestamp()); - onEncodeBucketInfoCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization4_2::onDecodeRemoveCommand(BBuf& buf) const -{ - document::DocumentId did(SH::getString(buf)); - document::Bucket bucket = getBucket(buf); - api::Timestamp timestamp(SH::getLong(buf)); - auto msg = std::make_unique(bucket, did, timestamp); - onDecodeBucketInfoCommand(buf, *msg); - return msg; -} - -void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RevertCommand& msg) const -{ - putBucket(msg.getBucket(), buf); - buf.putInt(msg.getRevertTokens().size()); - for (uint32_t i=0, n=msg.getRevertTokens().size(); i tokens(SH::getInt(buf)); - for (uint32_t i=0, n=tokens.size(); i(bucket, tokens); - onDecodeBucketInfoCommand(buf, *msg); - return msg; -} - -void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::CreateBucketCommand& msg) const -{ - putBucket(msg.getBucket(), buf); - onEncodeBucketInfoCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization4_2::onDecodeCreateBucketCommand(BBuf& buf) const -{ - document::Bucket bucket = getBucket(buf); - auto msg = std::make_unique(bucket); - onDecodeBucketInfoCommand(buf, *msg); - return msg; -} - -void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::MergeBucketCommand& msg) const -{ - putBucket(msg.getBucket(), buf); - const std::vector& nodes(msg.getNodes()); - buf.putShort(nodes.size()); - for (uint32_t i=0; i nodes; - nodes.reserve(nodeCount); - for (uint32_t i=0; i(bucket, nodes, timestamp); - onDecodeCommand(buf, *msg); - return msg; -} - -void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::GetBucketDiffCommand& msg) const -{ - putBucket(msg.getBucket(), buf); - const std::vector& nodes(msg.getNodes()); - buf.putShort(nodes.size()); - for (uint32_t i=0; i& entries(msg.getDiff()); - buf.putInt(entries.size()); - for (uint32_t i=0; i nodes; - nodes.reserve(nodeCount); - for (uint32_t i=0; i(bucket, nodes, timestamp); - std::vector& entries(msg->getDiff()); - uint32_t entryCount = SH::getInt(buf); - if (entryCount > buf.getRemaining()) { - // Trigger out of bounds exception rather than out of memory error - buf.incPos(entryCount); - } - entries.resize(entryCount); - for (uint32_t i=0; i& nodes(msg.getNodes()); - buf.putShort(nodes.size()); - for (uint32_t i=0; i& entries(msg.getDiff()); - buf.putInt(entries.size()); - for (uint32_t i=0; i nodes; - nodes.reserve(nodeCount); - for (uint32_t i=0; i(bucket, nodes); - std::vector& entries(msg->getDiff()); - uint32_t entryCount = SH::getInt(buf); - if (entryCount > buf.getRemaining()) { - // Trigger out of bounds exception rather than out of memory error - buf.incPos(entryCount); - } - entries.resize(entryCount); - for (uint32_t i=0; i buf.getRemaining()) { - buf.incPos(headerSize); - } - entries[i]._headerBlob.resize(headerSize); - buf.getBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size()); - uint32_t bodySize = SH::getInt(buf); - if (bodySize > buf.getRemaining()) { - buf.incPos(bodySize); - } - entries[i]._bodyBlob.resize(bodySize); - buf.getBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size()); - } - onDecodeBucketInfoCommand(buf, *msg); - return msg; -} - -void -ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RequestBucketInfoReply& msg) const -{ - buf.putInt(msg.getBucketInfo().size()); - for (const auto & entry : msg.getBucketInfo()) { - buf.putLong(entry._bucketId.getRawId()); - putBucketInfo(entry._info, buf); - } - onEncodeReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd, BBuf& buf) const -{ - auto msg = std::make_unique(static_cast(cmd)); - api::RequestBucketInfoReply::EntryVector & entries(msg->getBucketInfo()); - uint32_t entryCount = SH::getInt(buf); - if (entryCount > buf.getRemaining()) { - // Trigger out of bounds exception rather than out of memory error - buf.incPos(entryCount); - } - entries.resize(entryCount); - for (auto & entry : entries) { - entry._bucketId = document::BucketId(SH::getLong(buf)); - entry._info = getBucketInfo(buf); - } - onDecodeReply(buf, *msg); - return msg; -} - -void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const -{ - putBucket(msg.getBucket(), buf); - putBucketInfo(msg.getBucketInfo(), buf); - onEncodeCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization4_2::onDecodeNotifyBucketChangeCommand(BBuf& buf) const -{ - document::Bucket bucket = getBucket(buf); - api::BucketInfo info(getBucketInfo(buf)); - auto msg = std::make_unique(bucket, info); - onDecodeCommand(buf, *msg); - return msg; -} - -void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::NotifyBucketChangeReply& msg) const -{ - onEncodeReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization4_2::onDecodeNotifyBucketChangeReply(const SCmd& cmd,BBuf& buf) const -{ - auto msg = std::make_unique(static_cast(cmd)); - onDecodeReply(buf, *msg); - return msg; -} - -void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::SplitBucketCommand& msg) const -{ - putBucket(msg.getBucket(), buf); - buf.putByte(msg.getMinSplitBits()); - buf.putByte(msg.getMaxSplitBits()); - buf.putInt(msg.getMinByteSize()); - buf.putInt(msg.getMinDocCount()); - onEncodeCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization4_2::onDecodeSplitBucketCommand(BBuf& buf) const -{ - document::Bucket bucket = getBucket(buf); - auto msg = std::make_unique(bucket); - msg->setMinSplitBits(SH::getByte(buf)); - msg->setMaxSplitBits(SH::getByte(buf)); - msg->setMinByteSize(SH::getInt(buf)); - msg->setMinDocCount(SH::getInt(buf)); - onDecodeCommand(buf, *msg); - return msg; -} - -void ProtocolSerialization4_2::onEncode(GBBuf&, const api::SetBucketStateCommand&) const -{ - throw vespalib::IllegalStateException("Unsupported serialization", VESPA_STRLOC); -} - -api::StorageCommand::UP -ProtocolSerialization4_2::onDecodeSetBucketStateCommand(BBuf&) const -{ - throw vespalib::IllegalStateException("Unsupported deserialization", VESPA_STRLOC); -} - -void ProtocolSerialization4_2::onEncode(GBBuf&, const api::SetBucketStateReply&) const -{ - throw vespalib::IllegalStateException("Unsupported serialization", VESPA_STRLOC); -} - -api::StorageReply::UP -ProtocolSerialization4_2::onDecodeSetBucketStateReply(const SCmd&, BBuf&) const -{ - throw vespalib::IllegalStateException("Unsupported deserialization", VESPA_STRLOC); -} - -void -ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::CreateVisitorCommand& msg) const -{ - putBucketSpace(msg.getBucketSpace(), buf); - buf.putString(msg.getLibraryName()); - buf.putString(msg.getInstanceId()); - buf.putString(msg.getDocumentSelection()); - buf.putInt(msg.getVisitorCmdId()); - buf.putString(msg.getControlDestination()); - buf.putString(msg.getDataDestination()); - buf.putInt(msg.getMaximumPendingReplyCount()); - buf.putLong(msg.getFromTime()); - buf.putLong(msg.getToTime()); - - buf.putInt(msg.getBuckets().size()); - for (uint32_t i = 0; i < msg.getBuckets().size(); i++) { - buf.putLong(msg.getBuckets()[i].getRawId()); - } - - buf.putBoolean(msg.visitRemoves()); - buf.putBoolean(false); - buf.putBoolean(msg.visitInconsistentBuckets()); - buf.putInt(vespalib::count_ms(msg.getQueueTimeout())); - msg.getParameters().serialize(buf); - - onEncodeCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const -{ - BucketSpace bucketSpace = getBucketSpace(buf); - vespalib::stringref libraryName = SH::getString(buf); - vespalib::stringref instanceId = SH::getString(buf); - vespalib::stringref selection = SH::getString(buf); - auto msg = std::make_unique(bucketSpace, libraryName, instanceId, selection); - msg->setVisitorCmdId(SH::getInt(buf)); - msg->setControlDestination(SH::getString(buf)); - msg->setDataDestination(SH::getString(buf)); - msg->setMaximumPendingReplyCount(SH::getInt(buf)); - - msg->setFromTime(SH::getLong(buf)); - msg->setToTime(SH::getLong(buf)); - uint32_t count = SH::getInt(buf); - - if (count > buf.getRemaining()) { - // Trigger out of bounds exception rather than out of memory error - buf.incPos(count); - } - - for (uint32_t i = 0; i < count; i++) { - msg->getBuckets().push_back(document::BucketId(SH::getLong(buf))); - } - - if (SH::getBoolean(buf)) { - msg->setVisitRemoves(); - } - if (SH::getBoolean(buf)) { - msg->setFieldSet(AllFields::NAME); - } - if (SH::getBoolean(buf)) { - msg->setVisitInconsistentBuckets(); - } - msg->setQueueTimeout(std::chrono::milliseconds(SH::getInt(buf))); - msg->getParameters().deserialize(buf); - - onDecodeCommand(buf, *msg); - msg->setVisitorDispatcherVersion(42); - return msg; -} - -void -ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::DestroyVisitorCommand& msg) const -{ - buf.putString(msg.getInstanceId()); - onEncodeCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization4_2::onDecodeDestroyVisitorCommand(BBuf& buf) const -{ - vespalib::stringref instanceId = SH::getString(buf); - auto msg = std::make_unique(instanceId); - onDecodeCommand(buf, *msg); - return msg; -} - -void -ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::DestroyVisitorReply& msg) const -{ - onEncodeReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization4_2::onDecodeDestroyVisitorReply(const SCmd& cmd, BBuf& buf) const -{ - auto msg = std::make_unique(static_cast(cmd)); - onDecodeReply(buf, *msg); - return msg; -} - -void -ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveLocationCommand& msg) const -{ - buf.putString(msg.getDocumentSelection()); - putBucket(msg.getBucket(), buf); - onEncodeCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization4_2::onDecodeRemoveLocationCommand(BBuf& buf) const -{ - vespalib::stringref documentSelection = SH::getString(buf); - document::Bucket bucket = getBucket(buf); - - auto msg = std::make_unique(documentSelection, bucket); - onDecodeCommand(buf, *msg); - return msg; -} - -void -ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveLocationReply& msg) const -{ - onEncodeBucketInfoReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization4_2::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const -{ - auto msg = std::make_unique(static_cast(cmd)); - onDecodeBucketInfoReply(buf, *msg); - return msg; -} - -void ProtocolSerialization4_2::onEncode(GBBuf&, const api::StatBucketCommand&) const { - throw vespalib::IllegalStateException("StatBucketCommand not expected for legacy protocol version", VESPA_STRLOC); -} - -api::StorageCommand::UP -ProtocolSerialization4_2::onDecodeStatBucketCommand(BBuf&) const { - throw vespalib::IllegalStateException("StatBucketCommand not expected for legacy protocol version", VESPA_STRLOC); -} - -void ProtocolSerialization4_2::onEncode(GBBuf&, const api::StatBucketReply&) const { - throw vespalib::IllegalStateException("StatBucketReply not expected for legacy protocol version", VESPA_STRLOC); -} - -api::StorageReply::UP -ProtocolSerialization4_2::onDecodeStatBucketReply(const SCmd&, BBuf&) const { - throw vespalib::IllegalStateException("StatBucketReply not expected for legacy protocol version", VESPA_STRLOC); -} - -// Utility functions for serialization - -void -ProtocolSerialization4_2::onEncodeBucketInfoCommand(GBBuf& buf, const api::BucketInfoCommand& msg) const -{ - onEncodeCommand(buf, msg); -} - -void -ProtocolSerialization4_2::onDecodeBucketInfoCommand(BBuf& buf, api::BucketInfoCommand& msg) const -{ - onDecodeCommand(buf, msg); -} - -void -ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::ReturnCode& rc) const -{ - // Convert error code to codes used in 4.2 - buf.putInt(getOldErrorCode(rc.getResult())); - buf.putString(rc.getMessage()); -} - -void -ProtocolSerialization4_2::onEncodeDiffEntry(GBBuf& buf, const api::GetBucketDiffCommand::Entry& entry) const -{ - buf.putLong(entry._timestamp); - SH::putGlobalId(entry._gid, buf); - buf.putInt(entry._headerSize); - buf.putInt(entry._bodySize); - buf.putShort(entry._flags); - buf.putShort(entry._hasMask); -} - -void -ProtocolSerialization4_2::onDecodeDiffEntry(BBuf& buf, api::GetBucketDiffCommand::Entry& entry) const -{ - entry._timestamp = SH::getLong(buf); - entry._gid = SH::getGlobalId(buf); - entry._headerSize = SH::getInt(buf); - entry._bodySize = SH::getInt(buf); - entry._flags = SH::getShort(buf); - entry._hasMask = SH::getShort(buf); -} - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h deleted file mode 100644 index b8a20e9e401..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "legacyprotocolserialization.h" - -namespace storage::mbusprot { - -class ProtocolSerialization4_2 : public LegacyProtocolSerialization { -public: - explicit ProtocolSerialization4_2(const std::shared_ptr&); - -protected: - void onEncode(GBBuf&, const api::GetCommand&) const override; - void onEncode(GBBuf&, const api::RemoveCommand&) const override; - void onEncode(GBBuf&, const api::RevertCommand&) const override; - void onEncode(GBBuf&, const api::CreateBucketCommand&) const override; - void onEncode(GBBuf&, const api::MergeBucketCommand&) const override; - void onEncode(GBBuf&, const api::GetBucketDiffCommand&) const override; - void onEncode(GBBuf&, const api::ApplyBucketDiffCommand&) const override; - void onEncode(GBBuf&, const api::RequestBucketInfoReply&) const override; - void onEncode(GBBuf&, const api::NotifyBucketChangeCommand&) const override; - void onEncode(GBBuf&, const api::NotifyBucketChangeReply&) const override; - void onEncode(GBBuf&, const api::SplitBucketCommand&) const override; - void onEncode(GBBuf&, const api::CreateVisitorCommand&) const override; - void onEncode(GBBuf&, const api::DestroyVisitorCommand&) const override; - void onEncode(GBBuf&, const api::DestroyVisitorReply&) const override; - void onEncode(GBBuf&, const api::RemoveLocationCommand&) const override; - void onEncode(GBBuf&, const api::RemoveLocationReply&) const override; - void onEncode(GBBuf&, const api::StatBucketCommand&) const override; - void onEncode(GBBuf&, const api::StatBucketReply&) const override; - - // Not supported on 4.2, but implemented here for simplicity. - void onEncode(GBBuf&, const api::SetBucketStateCommand&) const override; - void onEncode(GBBuf&, const api::SetBucketStateReply&) const override; - - virtual void onEncodeBucketInfoCommand(GBBuf&, const api::BucketInfoCommand&) const; - virtual void onEncodeBucketInfoReply(GBBuf&, const api::BucketInfoReply&) const = 0; - virtual void onEncodeCommand(GBBuf&, const api::StorageCommand&) const = 0; - virtual void onEncodeReply(GBBuf&, const api::StorageReply&) const = 0; - - virtual void onEncodeDiffEntry(GBBuf&, const api::GetBucketDiffCommand::Entry&) const; - virtual void onEncode(GBBuf&, const api::ReturnCode&) const; - SCmd::UP onDecodeGetCommand(BBuf&) const override; - SCmd::UP onDecodeRemoveCommand(BBuf&) const override; - SCmd::UP onDecodeRevertCommand(BBuf&) const override; - SCmd::UP onDecodeCreateBucketCommand(BBuf&) const override; - SCmd::UP onDecodeMergeBucketCommand(BBuf&) const override; - SCmd::UP onDecodeGetBucketDiffCommand(BBuf&) const override; - SCmd::UP onDecodeApplyBucketDiffCommand(BBuf&) const override; - SRep::UP onDecodeRequestBucketInfoReply(const SCmd&, BBuf&) const override; - SCmd::UP onDecodeNotifyBucketChangeCommand(BBuf&) const override; - SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&, BBuf&) const override; - SCmd::UP onDecodeSplitBucketCommand(BBuf&) const override; - SCmd::UP onDecodeSetBucketStateCommand(BBuf&) const override; - SRep::UP onDecodeSetBucketStateReply(const SCmd&, BBuf&) const override; - SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const override; - SCmd::UP onDecodeDestroyVisitorCommand(BBuf&) const override; - SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const override; - SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const override; - SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const override; - SCmd::UP onDecodeStatBucketCommand(BBuf&) const override; - SRep::UP onDecodeStatBucketReply(const SCmd&, BBuf&) const override; - - virtual void onDecodeBucketInfoCommand(BBuf&, api::BucketInfoCommand&) const; - virtual void onDecodeBucketInfoReply(BBuf&, api::BucketInfoReply&) const = 0; - virtual void onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const = 0; - virtual void onDecodeReply(BBuf&, api::StorageReply&) const = 0; - - virtual void onDecodeDiffEntry(BBuf&, api::GetBucketDiffCommand::Entry&) const; -}; - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp deleted file mode 100644 index fa400b565b2..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp +++ /dev/null @@ -1,624 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "protocolserialization5_0.h" -#include "serializationhelper.h" -#include "storagecommand.h" -#include "storagereply.h" -#include -#include -#include -#include -#include -#include - -using document::BucketSpace; -using document::FixedBucketSpaces; - -namespace storage::mbusprot { - -document::Bucket -ProtocolSerialization5_0::getBucket(document::ByteBuffer& buf) const -{ - document::BucketId bucketId(SH::getLong(buf)); - return document::Bucket(FixedBucketSpaces::default_space(), bucketId); -} - -void -ProtocolSerialization5_0::putBucket(const document::Bucket& bucket, vespalib::GrowableByteBuffer& buf) const -{ - buf.putLong(bucket.getBucketId().getRawId()); - if (bucket.getBucketSpace() != FixedBucketSpaces::default_space()) { - std::ostringstream ost; - ost << "Bucket with bucket space " << bucket.getBucketSpace() << " cannot be serialized on old storageapi protocol."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); - } -} - -document::BucketSpace -ProtocolSerialization5_0::getBucketSpace(document::ByteBuffer&) const -{ - return FixedBucketSpaces::default_space(); -} - -void -ProtocolSerialization5_0::putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer&) const -{ - if (bucketSpace != FixedBucketSpaces::default_space()) { - std::ostringstream ost; - ost << "Bucket space " << bucketSpace << " cannot be serialized on old storageapi protocol."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); - } -} - -api::BucketInfo -ProtocolSerialization5_0::getBucketInfo(document::ByteBuffer& buf) const -{ - uint32_t crc(SH::getInt(buf)); - uint32_t doccount(SH::getInt(buf)); - uint32_t docsize(SH::getInt(buf)); - uint32_t metacount(SH::getInt(buf)); - uint32_t usedsize(SH::getInt(buf)); - return api::BucketInfo(crc, doccount, docsize, metacount, usedsize); -} - -void -ProtocolSerialization5_0::putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const -{ - buf.putInt(info.getChecksum()); - buf.putInt(info.getDocumentCount()); - buf.putInt(info.getTotalDocumentSize()); - buf.putInt(info.getMetaCount()); - buf.putInt(info.getUsedFileSize()); -} - -void -ProtocolSerialization5_0::onEncodeReply(GBBuf& buf, const api::StorageReply& msg) const -{ - SH::putReturnCode(msg.getResult(), buf); - buf.putLong(msg.getMsgId()); - buf.putByte(msg.getPriority()); -} - -void -ProtocolSerialization5_0::onDecodeReply(BBuf& buf, api::StorageReply& msg) const -{ - msg.setResult(SH::getReturnCode(buf)); - msg.forceMsgId(SH::getLong(buf)); - msg.setPriority(SH::getByte(buf)); -} - -void -ProtocolSerialization5_0::onEncodeCommand(GBBuf& buf, const api::StorageCommand& msg) const -{ - buf.putLong(msg.getMsgId()); - buf.putByte(msg.getPriority()); - buf.putShort(msg.getSourceIndex()); - buf.putInt(0); // LoadType 'default' -} - -void -ProtocolSerialization5_0::onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const -{ - msg.forceMsgId(SH::getLong(buf)); - uint8_t priority = SH::getByte(buf); - msg.setPriority(priority); - msg.setSourceIndex(SH::getShort(buf)); - (void)SH::getInt(buf); // LoadType -} - - -ProtocolSerialization5_0::ProtocolSerialization5_0(const std::shared_ptr& repo) - : ProtocolSerialization4_2(repo) -{ -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::PutReply& msg) const -{ - buf.putBoolean(msg.wasFound()); - onEncodeBucketInfoReply(buf, msg); -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::PutCommand& msg) const -{ - SH::putDocument(msg.getDocument().get(), buf); - putBucket(msg.getBucket(), buf); - buf.putLong(msg.getTimestamp()); - buf.putLong(msg.getUpdateTimestamp()); - onEncodeBucketInfoCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization5_0::onDecodePutCommand(BBuf& buf) const -{ - document::Document::SP doc(SH::getDocument(buf, getTypeRepo())); - document::Bucket bucket = getBucket(buf); - api::Timestamp ts(SH::getLong(buf)); - auto msg = std::make_unique(bucket, doc, ts); - msg->setUpdateTimestamp(SH::getLong(buf)); - onDecodeBucketInfoCommand(buf, *msg); - return msg; -} - -api::StorageReply::UP -ProtocolSerialization5_0::onDecodePutReply(const SCmd& cmd, BBuf& buf) const -{ - bool wasFound = SH::getBoolean(buf); - auto msg = std::make_unique(static_cast(cmd), wasFound); - onDecodeBucketInfoReply(buf, *msg); - return msg; -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::UpdateReply& msg) const -{ - buf.putLong(msg.getOldTimestamp()); - onEncodeBucketInfoReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization5_0::onDecodeUpdateReply(const SCmd& cmd, BBuf& buf) const -{ - api::Timestamp oldTimestamp(SH::getLong(buf)); - auto msg = std::make_unique(static_cast(cmd), oldTimestamp); - onDecodeBucketInfoReply(buf, *msg); - return msg; -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::GetReply& msg) const -{ - SH::putDocument(msg.getDocument().get(), buf); - // Old protocol version doesn't understand tombstones. Make it appear as Not Found. - buf.putLong(msg.is_tombstone() ? api::Timestamp(0) : msg.getLastModifiedTimestamp()); - onEncodeBucketInfoReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization5_0::onDecodeGetReply(const SCmd& cmd, BBuf& buf) const -{ - try { - document::Document::SP doc(SH::getDocument(buf, getTypeRepo())); - api::Timestamp lastModified(SH::getLong(buf)); - auto msg = std::make_unique(static_cast(cmd), doc,lastModified); - onDecodeBucketInfoReply(buf, *msg); - return msg; - } catch (std::exception& e) { - auto msg = std::make_unique(static_cast(cmd), document::Document::SP(),0); - msg->setResult(api::ReturnCode(api::ReturnCode::UNPARSEABLE, e.what())); - return msg; - } -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::RemoveReply& msg) const -{ - buf.putLong(msg.getOldTimestamp()); - onEncodeBucketInfoReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization5_0::onDecodeRemoveReply(const SCmd& cmd, BBuf& buf) const -{ - api::Timestamp oldTimestamp(SH::getLong(buf)); - auto msg = std::make_unique(static_cast(cmd), oldTimestamp); - onDecodeBucketInfoReply(buf, *msg); - return msg; -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::UpdateCommand& msg) const -{ - document::DocumentUpdate* update = msg.getUpdate().get(); - if (update) { - vespalib::nbostream stream; - update->serializeHEAD(stream); - buf.putInt(stream.size()); - buf.putBytes(stream.peek(), stream.size()); - } else { - buf.putInt(0); - } - - putBucket(msg.getBucket(), buf); - buf.putLong(msg.getTimestamp()); - buf.putLong(msg.getOldTimestamp()); - onEncodeBucketInfoCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization5_0::onDecodeUpdateCommand(BBuf& buf) const -{ - document::DocumentUpdate::SP update; - - uint32_t size = SH::getInt(buf); - if (size != 0) { - update = document::DocumentUpdate::createHEAD(getTypeRepo(), vespalib::nbostream(buf.getBufferAtPos(), size)); - buf.incPos(size); - } - - document::Bucket bucket = getBucket(buf); - api::Timestamp timestamp(SH::getLong(buf)); - api::UpdateCommand::UP msg = std::make_unique(bucket, update, timestamp); - msg->setOldTimestamp(SH::getLong(buf)); - onDecodeBucketInfoCommand(buf, *msg); - return msg; -} - - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::RevertReply& msg) const -{ - onEncodeBucketInfoReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization5_0::onDecodeRevertReply(const SCmd& cmd, BBuf& buf) const -{ - auto msg = std::make_unique(static_cast(cmd)); - onDecodeBucketInfoReply(buf, *msg); - return msg; -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::CreateBucketReply& msg) const -{ - onEncodeBucketInfoReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization5_0::onDecodeCreateBucketReply(const SCmd& cmd, BBuf& buf) const -{ - auto msg = std::make_unique(static_cast(cmd)); - onDecodeBucketInfoReply(buf, *msg); - return msg; -} - -void -ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::DeleteBucketCommand& msg) const -{ - putBucket(msg.getBucket(), buf); - onEncodeBucketInfoCommand(buf, msg); - putBucketInfo(msg.getBucketInfo(), buf); -} - -api::StorageCommand::UP -ProtocolSerialization5_0::onDecodeDeleteBucketCommand(BBuf& buf) const -{ - document::Bucket bucket = getBucket(buf); - auto msg = std::make_unique(bucket); - onDecodeBucketInfoCommand(buf, *msg); - if (buf.getRemaining() >= SH::BUCKET_INFO_SERIALIZED_SIZE) { - msg->setBucketInfo(getBucketInfo(buf)); - } - return msg; -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::DeleteBucketReply& msg) const -{ - onEncodeBucketInfoReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization5_0::onDecodeDeleteBucketReply(const SCmd& cmd, BBuf& buf) const -{ - auto msg = std::make_unique(static_cast(cmd)); - onDecodeBucketInfoReply(buf, *msg); - return msg; -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::MergeBucketCommand& msg) const -{ - ProtocolSerialization4_2::onEncode(buf, msg); - - buf.putInt(msg.getClusterStateVersion()); - const std::vector& chain(msg.getChain()); - buf.putShort(chain.size()); - for (std::size_t i = 0; i < chain.size(); ++i) { - buf.putShort(chain[i]); - } -} - -api::StorageCommand::UP -ProtocolSerialization5_0::onDecodeMergeBucketCommand(BBuf& buf) const -{ - api::StorageCommand::UP cmd = ProtocolSerialization4_2::onDecodeMergeBucketCommand(buf); - uint32_t clusterStateVersion = SH::getInt(buf); - uint16_t chainSize = SH::getShort(buf); - std::vector chain; - chain.reserve(chainSize); - for (std::size_t i = 0; i < chainSize; ++i) { - uint16_t index = SH::getShort(buf); - chain.push_back(index); - } - api::MergeBucketCommand& mergeCmd = static_cast(*cmd); - mergeCmd.setChain(chain); - mergeCmd.setClusterStateVersion(clusterStateVersion); - return cmd; -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::MergeBucketReply& msg) const -{ - onEncodeBucketReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization5_0::onDecodeMergeBucketReply(const SCmd& cmd, BBuf& buf) const -{ - auto msg = std::make_unique(static_cast(cmd)); - onDecodeBucketReply(buf, *msg); - return msg; -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::GetBucketDiffReply& msg) const -{ - const std::vector& entries(msg.getDiff()); - buf.putInt(entries.size()); - for (uint32_t i=0; i(static_cast(cmd)); - std::vector& entries(msg->getDiff()); - uint32_t entryCount = SH::getInt(buf); - if (entryCount > buf.getRemaining()) { - // Trigger out of bounds exception rather than out of memory error - buf.incPos(entryCount); - } - entries.resize(entryCount); - for (uint32_t i=0; i& entries(msg.getDiff()); - buf.putInt(entries.size()); - for (uint32_t i=0; i(static_cast(cmd)); - std::vector& entries(msg->getDiff()); - uint32_t entryCount = SH::getInt(buf); - if (entryCount > buf.getRemaining()) { - // Trigger out of bounds exception rather than out of memory error - buf.incPos(entryCount); - } - entries.resize(entryCount); - for (uint32_t i=0; i buf.getRemaining()) { - buf.incPos(headerSize); - } - entries[i]._headerBlob.resize(headerSize); - buf.getBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size()); - uint32_t bodySize = SH::getInt(buf); - if (bodySize > buf.getRemaining()) { - buf.incPos(bodySize); - } - entries[i]._bodyBlob.resize(bodySize); - buf.getBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size()); - } - onDecodeBucketInfoReply(buf, *msg); - return msg; -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::SplitBucketReply& msg) const -{ - const std::vector& entries(msg.getSplitInfo()); - buf.putInt(entries.size()); - for (std::vector::const_iterator it - = entries.begin(); it != entries.end(); ++it) - { - buf.putLong(it->first.getRawId()); - putBucketInfo(it->second, buf); - } - onEncodeBucketReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization5_0::onDecodeSplitBucketReply(const SCmd& cmd, BBuf& buf) const -{ - auto msg = std::make_unique(static_cast(cmd)); - std::vector& entries(msg->getSplitInfo()); - uint32_t targetCount = SH::getInt(buf); - if (targetCount > buf.getRemaining()) { - // Trigger out of bounds exception rather than out of memory error - buf.incPos(targetCount); - } - entries.resize(targetCount); - for (std::vector::iterator it - = entries.begin(); it != entries.end(); ++it) - { - it->first = document::BucketId(SH::getLong(buf)); - it->second = getBucketInfo(buf); - } - onDecodeBucketReply(buf, *msg); - return msg; -} - -void -ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::JoinBucketsCommand& msg) const -{ - putBucket(msg.getBucket(), buf); - buf.putInt(msg.getSourceBuckets().size()); - for (uint32_t i=0, n=msg.getSourceBuckets().size(); i(bucket); - uint32_t size = SH::getInt(buf); - if (size > buf.getRemaining()) { - // Trigger out of bounds exception rather than out of memory error - buf.incPos(size); - } - std::vector& entries(msg->getSourceBuckets()); - for (uint32_t i=0; isetMinJoinBits(SH::getByte(buf)); - onDecodeCommand(buf, *msg); - return msg; -} - -void -ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::JoinBucketsReply& msg) const -{ - putBucketInfo(msg.getBucketInfo(), buf); - onEncodeBucketReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization5_0::onDecodeJoinBucketsReply(const SCmd& cmd, BBuf& buf) const -{ - auto msg = std::make_unique(static_cast(cmd)); - msg->setBucketInfo(getBucketInfo(buf)); - onDecodeBucketReply(buf, *msg); - return msg; -} - -void -ProtocolSerialization5_0::onEncodeBucketInfoReply(GBBuf& buf, const api::BucketInfoReply& msg) const -{ - onEncodeBucketReply(buf, msg); - putBucketInfo(msg.getBucketInfo(), buf); -} - -void -ProtocolSerialization5_0::onDecodeBucketInfoReply(BBuf& buf, api::BucketInfoReply& msg) const -{ - onDecodeBucketReply(buf, msg); - msg.setBucketInfo(getBucketInfo(buf)); -} - -void -ProtocolSerialization5_0::onEncodeBucketReply(GBBuf& buf, const api::BucketReply& msg) const -{ - onEncodeReply(buf, msg); - buf.putLong(msg.hasBeenRemapped() ? msg.getBucketId().getRawId() : 0); -} - -void -ProtocolSerialization5_0::onDecodeBucketReply(BBuf& buf, api::BucketReply& msg) const -{ - onDecodeReply(buf, msg); - document::BucketId bucket(SH::getLong(buf)); - if (bucket.getRawId() != 0) { - msg.remapBucketId(bucket); - } -} - -void -ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::CreateVisitorReply& msg) const -{ - onEncodeReply(buf, msg); - buf.putInt(msg.getVisitorStatistics().getBucketsVisited()); - buf.putLong(msg.getVisitorStatistics().getDocumentsVisited()); - buf.putLong(msg.getVisitorStatistics().getBytesVisited()); - buf.putLong(msg.getVisitorStatistics().getDocumentsReturned()); - buf.putLong(msg.getVisitorStatistics().getBytesReturned()); - // TODO remove second pass concept on Vespa 8 - buf.putLong(msg.getVisitorStatistics().getSecondPassDocumentsReturned()); - buf.putLong(msg.getVisitorStatistics().getSecondPassBytesReturned()); -} - -api::StorageReply::UP -ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) const -{ - auto msg = std::make_unique(static_cast(cmd)); - onDecodeReply(buf, *msg); - - vdslib::VisitorStatistics vs; - vs.setBucketsVisited(SH::getInt(buf)); - vs.setDocumentsVisited(SH::getLong(buf)); - vs.setBytesVisited(SH::getLong(buf)); - vs.setDocumentsReturned(SH::getLong(buf)); - vs.setBytesReturned(SH::getLong(buf)); - // TODO remove second pass concept on Vespa 8 - vs.setSecondPassDocumentsReturned(SH::getLong(buf)); - vs.setSecondPassBytesReturned(SH::getLong(buf)); - msg->setVisitorStatistics(vs); - - return msg; -} - -void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::RequestBucketInfoCommand& msg) const -{ - const std::vector& buckets(msg.getBuckets()); - buf.putInt(buckets.size()); - for (uint32_t i=0; i buckets(SH::getInt(buf)); - for (uint32_t i=0; i(cvc.get())->setMaxBucketsPerVisitor(SH::getInt(buf)); - - static_cast(cvc.get())->setVisitorDispatcherVersion(50); - return cvc; -} - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h deleted file mode 100644 index 8de2edab17f..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "protocolserialization4_2.h" - -namespace storage::mbusprot { - -class ProtocolSerialization5_0 : public ProtocolSerialization4_2 { -public: - ProtocolSerialization5_0(const std::shared_ptr&); - - document::Bucket getBucket(document::ByteBuffer& buf) const override; - void putBucket(const document::Bucket& bucket, vespalib::GrowableByteBuffer& buf) const override; - document::BucketSpace getBucketSpace(document::ByteBuffer& buf) const override; - void putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer& buf) const override; - api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const override; - void putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const override; - - void onEncode(GBBuf&, const api::PutCommand&) const override; - void onEncode(GBBuf&, const api::PutReply&) const override; - void onEncode(GBBuf&, const api::UpdateCommand&) const override; - void onEncode(GBBuf&, const api::UpdateReply&) const override; - void onEncode(GBBuf&, const api::GetReply&) const override; - void onEncode(GBBuf&, const api::RemoveReply&) const override; - void onEncode(GBBuf&, const api::RevertReply&) const override; - void onEncode(GBBuf&, const api::CreateBucketReply&) const override; - void onEncode(GBBuf&, const api::DeleteBucketCommand&) const override; - void onEncode(GBBuf&, const api::DeleteBucketReply&) const override; - void onEncode(GBBuf&, const api::MergeBucketCommand&) const override; - void onEncode(GBBuf&, const api::MergeBucketReply&) const override; - void onEncode(GBBuf&, const api::GetBucketDiffReply&) const override; - void onEncode(GBBuf&, const api::ApplyBucketDiffReply&) const override; - void onEncode(GBBuf&, const api::SplitBucketReply&) const override; - void onEncode(GBBuf&, const api::JoinBucketsCommand&) const override; - void onEncode(GBBuf&, const api::JoinBucketsReply&) const override; - void onEncode(GBBuf&, const api::RequestBucketInfoCommand&) const override; - - void onEncodeBucketInfoReply(GBBuf&, const api::BucketInfoReply&) const override; - virtual void onEncodeBucketReply(GBBuf&, const api::BucketReply&) const; - - void onEncode(GBBuf&, const api::CreateVisitorCommand& msg) const override; - void onEncode(GBBuf&, const api::CreateVisitorReply& msg) const override; - void onEncodeCommand(GBBuf&, const api::StorageCommand&) const override; - void onEncodeReply(GBBuf&, const api::StorageReply&) const override; - - SCmd::UP onDecodePutCommand(BBuf&) const override; - SRep::UP onDecodePutReply(const SCmd&, BBuf&) const override; - SCmd::UP onDecodeUpdateCommand(BBuf&) const override; - SRep::UP onDecodeUpdateReply(const SCmd&, BBuf&) const override; - SRep::UP onDecodeGetReply(const SCmd&, BBuf&) const override; - SRep::UP onDecodeRemoveReply(const SCmd&, BBuf&) const override; - SRep::UP onDecodeRevertReply(const SCmd&, BBuf&) const override; - SRep::UP onDecodeCreateBucketReply(const SCmd&, BBuf&) const override; - SCmd::UP onDecodeDeleteBucketCommand(BBuf&) const override; - SRep::UP onDecodeDeleteBucketReply(const SCmd&, BBuf&) const override; - SCmd::UP onDecodeMergeBucketCommand(BBuf&) const override; - SRep::UP onDecodeMergeBucketReply(const SCmd&, BBuf&) const override; - SRep::UP onDecodeGetBucketDiffReply(const SCmd&, BBuf&) const override; - SRep::UP onDecodeApplyBucketDiffReply(const SCmd&, BBuf&) const override; - SRep::UP onDecodeSplitBucketReply(const SCmd&, BBuf&) const override; - SCmd::UP onDecodeJoinBucketsCommand(BBuf& buf) const override; - SRep::UP onDecodeJoinBucketsReply(const SCmd& cmd, BBuf& buf) const override; - SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const override; - SCmd::UP onDecodeRequestBucketInfoCommand(BBuf& buf) const override; - - void onDecodeBucketInfoReply(BBuf&, api::BucketInfoReply&) const override; - virtual void onDecodeBucketReply(BBuf&, api::BucketReply&) const; - SRep::UP onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) const override; - void onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const override; - void onDecodeReply(BBuf&, api::StorageReply&) const override; -}; - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp deleted file mode 100644 index 97ceceac33d..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "protocolserialization5_1.h" -#include "serializationhelper.h" -#include "storagecommand.h" -#include "storagereply.h" -#include -#include - -using document::BucketSpace; - -namespace storage::mbusprot { - -api::BucketInfo -ProtocolSerialization5_1::getBucketInfo(document::ByteBuffer& buf) const -{ - uint64_t lastModified(SH::getLong(buf)); - uint32_t crc(SH::getInt(buf)); - uint32_t doccount(SH::getInt(buf)); - uint32_t docsize(SH::getInt(buf)); - uint32_t metacount(SH::getInt(buf)); - uint32_t usedsize(SH::getInt(buf)); - uint8_t flags(SH::getByte(buf)); - bool ready = (flags & BUCKET_READY) != 0; - bool active = (flags & BUCKET_ACTIVE) != 0; - return api::BucketInfo(crc, doccount, docsize, - metacount, usedsize, - ready, active, lastModified); -} - -void -ProtocolSerialization5_1::putBucketInfo( - const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const -{ - buf.putLong(info.getLastModified()); - buf.putInt(info.getChecksum()); - buf.putInt(info.getDocumentCount()); - buf.putInt(info.getTotalDocumentSize()); - buf.putInt(info.getMetaCount()); - buf.putInt(info.getUsedFileSize()); - uint8_t flags = (info.isReady() ? BUCKET_READY : 0) | - (info.isActive() ? BUCKET_ACTIVE : 0); - buf.putByte(flags); -} - -ProtocolSerialization5_1::ProtocolSerialization5_1( - const std::shared_ptr& repo) - : ProtocolSerialization5_0(repo) -{ -} - -void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::SetBucketStateCommand& msg) const -{ - putBucket(msg.getBucket(), buf); - buf.putByte(static_cast(msg.getState())); - onEncodeCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization5_1::onDecodeSetBucketStateCommand(BBuf& buf) const -{ - document::Bucket bucket = getBucket(buf); - api::SetBucketStateCommand::BUCKET_STATE state( - static_cast(SH::getByte(buf))); - auto msg = std::make_unique(bucket, state); - onDecodeCommand(buf, *msg); - return msg; -} - -void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::SetBucketStateReply& msg) const -{ - onEncodeBucketReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization5_1::onDecodeSetBucketStateReply(const SCmd& cmd, BBuf& buf) const -{ - auto msg = std::make_unique(static_cast(cmd)); - onDecodeBucketReply(buf, *msg); - return msg; -} - -void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::GetCommand& msg) const -{ - buf.putString(msg.getDocumentId().toString()); - putBucket(msg.getBucket(), buf); - buf.putLong(msg.getBeforeTimestamp()); - buf.putString(msg.getFieldSet()); - onEncodeCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization5_1::onDecodeGetCommand(BBuf& buf) const -{ - document::DocumentId did(SH::getString(buf)); - document::Bucket bucket = getBucket(buf); - api::Timestamp beforeTimestamp(SH::getLong(buf)); - std::string fieldSet(SH::getString(buf)); - auto msg = std::make_unique(bucket, did, fieldSet, beforeTimestamp); - onDecodeCommand(buf, *msg); - return msg; -} - -void -ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::CreateVisitorCommand& msg) const -{ - putBucketSpace(msg.getBucketSpace(), buf); - buf.putString(msg.getLibraryName()); - buf.putString(msg.getInstanceId()); - buf.putString(msg.getDocumentSelection()); - buf.putInt(msg.getVisitorCmdId()); - buf.putString(msg.getControlDestination()); - buf.putString(msg.getDataDestination()); - buf.putInt(msg.getMaximumPendingReplyCount()); - buf.putLong(msg.getFromTime()); - buf.putLong(msg.getToTime()); - - buf.putInt(msg.getBuckets().size()); - for (uint32_t i = 0; i < msg.getBuckets().size(); i++) { - buf.putLong(msg.getBuckets()[i].getRawId()); - } - - buf.putBoolean(msg.visitRemoves()); - buf.putString(msg.getFieldSet()); - buf.putBoolean(msg.visitInconsistentBuckets()); - buf.putInt(vespalib::count_ms(msg.getQueueTimeout())); - msg.getParameters().serialize(buf); - - onEncodeCommand(buf, msg); - - buf.putInt(0); // Unused - buf.putInt(msg.getMaxBucketsPerVisitor()); -} - -api::StorageCommand::UP -ProtocolSerialization5_1::onDecodeCreateVisitorCommand(BBuf& buf) const -{ - BucketSpace bucketSpace = getBucketSpace(buf); - vespalib::stringref libraryName = SH::getString(buf); - vespalib::stringref instanceId = SH::getString(buf); - vespalib::stringref selection = SH::getString(buf); - auto msg = std::make_unique(bucketSpace, libraryName, instanceId, selection); - msg->setVisitorCmdId(SH::getInt(buf)); - msg->setControlDestination(SH::getString(buf)); - msg->setDataDestination(SH::getString(buf)); - msg->setMaximumPendingReplyCount(SH::getInt(buf)); - - msg->setFromTime(SH::getLong(buf)); - msg->setToTime(SH::getLong(buf)); - uint32_t count = SH::getInt(buf); - - if (count > buf.getRemaining()) { - // Trigger out of bounds exception rather than out of memory error - buf.incPos(count); - } - - for (uint32_t i = 0; i < count; i++) { - msg->getBuckets().push_back(document::BucketId(SH::getLong(buf))); - } - - if (SH::getBoolean(buf)) { - msg->setVisitRemoves(); - } - - msg->setFieldSet(SH::getString(buf)); - - if (SH::getBoolean(buf)) { - msg->setVisitInconsistentBuckets(); - } - msg->setQueueTimeout(std::chrono::milliseconds(SH::getInt(buf))); - msg->getParameters().deserialize(buf); - - onDecodeCommand(buf, *msg); - SH::getInt(buf); // Unused - msg->setMaxBucketsPerVisitor(SH::getInt(buf)); - msg->setVisitorDispatcherVersion(50); - return msg; -} - -void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::CreateBucketCommand& msg) const -{ - putBucket(msg.getBucket(), buf); - buf.putBoolean(msg.getActive()); - onEncodeBucketInfoCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization5_1::onDecodeCreateBucketCommand(BBuf& buf) const -{ - document::Bucket bucket = getBucket(buf); - bool setActive = SH::getBoolean(buf); - auto msg = std::make_unique(bucket); - msg->setActive(setActive); - onDecodeBucketInfoCommand(buf, *msg); - return msg; -} - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h deleted file mode 100644 index bfad492e653..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "protocolserialization5_0.h" - -namespace storage::mbusprot { - -class ProtocolSerialization5_1 : public ProtocolSerialization5_0 -{ - enum BucketState { - BUCKET_READY = 0x1, - BUCKET_ACTIVE = 0x2, - }; -public: - ProtocolSerialization5_1(const std::shared_ptr&); - - api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const override; - void putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const override; - -protected: - void onEncode(GBBuf&, const api::SetBucketStateCommand&) const override; - void onEncode(GBBuf&, const api::SetBucketStateReply&) const override; - void onEncode(GBBuf&, const api::GetCommand&) const override; - void onEncode(GBBuf&, const api::CreateVisitorCommand&) const override; - void onEncode(GBBuf&, const api::CreateBucketCommand&) const override; - - SCmd::UP onDecodeSetBucketStateCommand(BBuf&) const override; - SRep::UP onDecodeSetBucketStateReply(const SCmd&, BBuf&) const override; - SCmd::UP onDecodeGetCommand(BBuf&) const override; - SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const override; - SCmd::UP onDecodeCreateBucketCommand(BBuf&) const override; -}; - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp deleted file mode 100644 index 10aa9d69fbc..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// @author Vegard Sjonfjell - -#include "protocolserialization5_2.h" -#include "storagecommand.h" -#include "serializationhelper.h" - -namespace storage::mbusprot { - -using documentapi::TestAndSetCondition; - -void ProtocolSerialization5_2::onEncode(GBBuf & buf, const api::PutCommand & cmd) const -{ - ProtocolSerialization5_0::onEncode(buf, cmd); - encodeTasCondition(buf, cmd); -} - -api::StorageCommand::UP -ProtocolSerialization5_2::onDecodePutCommand(BBuf & buf) const -{ - auto cmd = ProtocolSerialization5_0::onDecodePutCommand(buf); - decodeTasCondition(*cmd, buf); - return cmd; -} - -void ProtocolSerialization5_2::onEncode(GBBuf & buf, const api::RemoveCommand & cmd) const -{ - ProtocolSerialization4_2::onEncode(buf, cmd); - encodeTasCondition(buf, cmd); -} - -api::StorageCommand::UP -ProtocolSerialization5_2::onDecodeRemoveCommand(BBuf & buf) const -{ - auto cmd = ProtocolSerialization4_2::onDecodeRemoveCommand(buf); - decodeTasCondition(*cmd, buf); - return cmd; -} - -void ProtocolSerialization5_2::onEncode(GBBuf & buf, const api::UpdateCommand & cmd) const -{ - ProtocolSerialization5_0::onEncode(buf, cmd); - encodeTasCondition(buf, cmd); -} - -api::StorageCommand::UP -ProtocolSerialization5_2::onDecodeUpdateCommand(BBuf & buf) const -{ - auto cmd = ProtocolSerialization5_0::onDecodeUpdateCommand(buf); - decodeTasCondition(*cmd, buf); - return cmd; -} - -void ProtocolSerialization5_2::decodeTasCondition(api::StorageCommand & storageCmd, BBuf & buf) { - auto & cmd = static_cast(storageCmd); - cmd.setCondition(TestAndSetCondition(SH::getString(buf))); -} - -void ProtocolSerialization5_2::encodeTasCondition(GBBuf & buf, const api::StorageCommand & storageCmd) { - auto & cmd = static_cast(storageCmd); - buf.putString(cmd.getCondition().getSelection()); -} - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h deleted file mode 100644 index e56b9942fa5..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// @author Vegard Sjonfjell - -#pragma once - -#include "protocolserialization5_1.h" -#include -#include - -namespace storage::mbusprot { - -class ProtocolSerialization5_2 : public ProtocolSerialization5_1 -{ -public: - ProtocolSerialization5_2(const std::shared_ptr& repo) - : ProtocolSerialization5_1(repo) - {} - -protected: - void onEncode(GBBuf &, const api::PutCommand &) const override; - void onEncode(GBBuf &, const api::RemoveCommand &) const override; - void onEncode(GBBuf &, const api::UpdateCommand &) const override; - - SCmd::UP onDecodePutCommand(BBuf &) const override; - SCmd::UP onDecodeRemoveCommand(BBuf &) const override; - SCmd::UP onDecodeUpdateCommand(BBuf &) const override; - - static void decodeTasCondition(api::StorageCommand & cmd, BBuf & buf); - static void encodeTasCondition(GBBuf & buf, const api::StorageCommand & cmd); -}; - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.cpp deleted file mode 100644 index 799aff8f8b3..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "protocolserialization6_0.h" -#include "serializationhelper.h" - -namespace storage::mbusprot { - -ProtocolSerialization6_0::ProtocolSerialization6_0(const std::shared_ptr &repo) - : ProtocolSerialization5_2(repo) -{ -} - -document::Bucket -ProtocolSerialization6_0::getBucket(document::ByteBuffer &buf) const -{ - document::BucketSpace bucketSpace(SH::getLong(buf)); - document::BucketId bucketId(SH::getLong(buf)); - return document::Bucket(bucketSpace, bucketId); -} - -void -ProtocolSerialization6_0::putBucket(const document::Bucket &bucket, vespalib::GrowableByteBuffer &buf) const -{ - buf.putLong(bucket.getBucketSpace().getId()); - buf.putLong(bucket.getBucketId().getRawId()); -} - -document::BucketSpace -ProtocolSerialization6_0::getBucketSpace(document::ByteBuffer &buf) const -{ - return document::BucketSpace(SH::getLong(buf)); -} - -void -ProtocolSerialization6_0::putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer &buf) const -{ - buf.putLong(bucketSpace.getId()); -} - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.h deleted file mode 100644 index 5467cf6c5d2..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "protocolserialization5_2.h" - -namespace storage::mbusprot { - -/** - * Protocol serialization version adding decoding and encoding - * of bucket space to almost all commands. - */ -class ProtocolSerialization6_0 : public ProtocolSerialization5_2 -{ -public: - ProtocolSerialization6_0(const std::shared_ptr &repo); - - document::Bucket getBucket(document::ByteBuffer &buf) const override; - void putBucket(const document::Bucket &bucket, vespalib::GrowableByteBuffer &buf) const override; - document::BucketSpace getBucketSpace(document::ByteBuffer &buf) const override; - void putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer &buf) const override; -}; - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp deleted file mode 100644 index 91b5999e34c..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ /dev/null @@ -1,1351 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "protocolserialization7.h" -#include "serializationhelper.h" -#include "protobuf_includes.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace storage::mbusprot { - -ProtocolSerialization7::ProtocolSerialization7(std::shared_ptr repo) - : ProtocolSerialization(), - _repo(std::move(repo)) -{ -} - -namespace { - -void set_bucket(protobuf::Bucket& dest, const document::Bucket& src) { - dest.set_raw_bucket_id(src.getBucketId().getRawId()); - dest.set_space_id(src.getBucketSpace().getId()); -} - -void set_bucket_id(protobuf::BucketId& dest, const document::BucketId& src) { - dest.set_raw_id(src.getRawId()); -} - -document::BucketId get_bucket_id(const protobuf::BucketId& src) { - return document::BucketId(src.raw_id()); -} - -void set_bucket_space(protobuf::BucketSpace& dest, const document::BucketSpace& src) { - dest.set_space_id(src.getId()); -} - -document::BucketSpace get_bucket_space(const protobuf::BucketSpace& src) { - return document::BucketSpace(src.space_id()); -} - -void set_bucket_info(protobuf::BucketInfo& dest, const api::BucketInfo& src) { - dest.set_last_modified_timestamp(src.getLastModified()); - dest.set_legacy_checksum(src.getChecksum()); - dest.set_doc_count(src.getDocumentCount()); - dest.set_total_doc_size(src.getTotalDocumentSize()); - dest.set_meta_count(src.getMetaCount()); - dest.set_used_file_size(src.getUsedFileSize()); - dest.set_active(src.isActive()); - dest.set_ready(src.isReady()); -} - -document::Bucket get_bucket(const protobuf::Bucket& src) { - return document::Bucket(document::BucketSpace(src.space_id()), - document::BucketId(src.raw_bucket_id())); -} - -api::BucketInfo get_bucket_info(const protobuf::BucketInfo& src) { - api::BucketInfo info; - info.setLastModified(src.last_modified_timestamp()); - info.setChecksum(src.legacy_checksum()); - info.setDocumentCount(src.doc_count()); - info.setTotalDocumentSize(src.total_doc_size()); - info.setMetaCount(src.meta_count()); - info.setUsedFileSize(src.used_file_size()); - info.setActive(src.active()); - info.setReady(src.ready()); - return info; -} - -documentapi::TestAndSetCondition get_tas_condition(const protobuf::TestAndSetCondition& src) { - return documentapi::TestAndSetCondition(src.selection()); -} - -void set_tas_condition(protobuf::TestAndSetCondition& dest, const documentapi::TestAndSetCondition& src) { - dest.set_selection(src.getSelection().data(), src.getSelection().size()); -} - -std::shared_ptr get_document(const protobuf::Document& src_doc, - const document::DocumentTypeRepo& type_repo) -{ - if (!src_doc.payload().empty()) { - vespalib::nbostream doc_buf(src_doc.payload().data(), src_doc.payload().size()); - return std::make_shared(type_repo, doc_buf); - } - return std::shared_ptr(); -} - -void set_update(protobuf::Update& dest, const document::DocumentUpdate& src) { - vespalib::nbostream stream; - src.serializeHEAD(stream); - dest.set_payload(stream.peek(), stream.size()); -} - -std::shared_ptr get_update(const protobuf::Update& src, - const document::DocumentTypeRepo& type_repo) -{ - if (!src.payload().empty()) { - return document::DocumentUpdate::createHEAD( - type_repo, vespalib::nbostream(src.payload().data(), src.payload().size())); - } - return std::shared_ptr(); -} - -void write_request_header(vespalib::GrowableByteBuffer& buf, const api::StorageCommand& cmd) { - protobuf::RequestHeader hdr; // Arena alloc not needed since there are no nested messages - hdr.set_message_id(cmd.getMsgId()); - hdr.set_priority(cmd.getPriority()); - hdr.set_source_index(cmd.getSourceIndex()); - - uint8_t dest[128]; // Only primitive fields, should be plenty large enough. - auto encoded_size = static_cast(hdr.ByteSizeLong()); - assert(encoded_size <= sizeof(dest)); - [[maybe_unused]] bool ok = hdr.SerializeWithCachedSizesToArray(dest); - assert(ok); - buf.putInt(encoded_size); - buf.putBytes(reinterpret_cast(dest), encoded_size); -} - -void write_response_header(vespalib::GrowableByteBuffer& buf, const api::StorageReply& reply) { - protobuf::ResponseHeader hdr; // Arena alloc not needed since there are no nested messages - const auto& result = reply.getResult(); - hdr.set_return_code_id(static_cast(result.getResult())); - if (!result.getMessage().empty()) { - hdr.set_return_code_message(result.getMessage().data(), result.getMessage().size()); - } - hdr.set_message_id(reply.getMsgId()); - hdr.set_priority(reply.getPriority()); - - const auto header_size = hdr.ByteSizeLong(); - assert(header_size <= UINT32_MAX); - buf.putInt(static_cast(header_size)); - - auto* dest_buf = reinterpret_cast(buf.allocate(header_size)); - [[maybe_unused]] bool ok = hdr.SerializeWithCachedSizesToArray(dest_buf); - assert(ok); -} - -void decode_request_header(document::ByteBuffer& buf, protobuf::RequestHeader& hdr) { - auto hdr_len = static_cast(SerializationHelper::getInt(buf)); - if (hdr_len > buf.getRemaining()) { - throw document::BufferOutOfBoundsException(buf.getPos(), hdr_len); - } - bool ok = hdr.ParseFromArray(buf.getBufferAtPos(), hdr_len); - if (!ok) { - throw vespalib::IllegalArgumentException("Malformed protobuf request header"); - } - buf.incPos(hdr_len); -} - -void decode_response_header(document::ByteBuffer& buf, protobuf::ResponseHeader& hdr) { - auto hdr_len = static_cast(SerializationHelper::getInt(buf)); - if (hdr_len > buf.getRemaining()) { - throw document::BufferOutOfBoundsException(buf.getPos(), hdr_len); - } - bool ok = hdr.ParseFromArray(buf.getBufferAtPos(), hdr_len); - if (!ok) { - throw vespalib::IllegalArgumentException("Malformed protobuf response header"); - } - buf.incPos(hdr_len); -} - -} // anonymous namespace - -template -class BaseEncoder { - vespalib::GrowableByteBuffer& _out_buf; - ::google::protobuf::Arena _arena; - ProtobufType* _proto_obj; -public: - explicit BaseEncoder(vespalib::GrowableByteBuffer& out_buf) - : _out_buf(out_buf), - _arena(), - _proto_obj(::google::protobuf::Arena::Create(&_arena)) - { - } - - void encode() { - assert(_proto_obj != nullptr); - const auto sz = _proto_obj->ByteSizeLong(); - assert(sz <= UINT32_MAX); - auto* buf = reinterpret_cast(_out_buf.allocate(sz)); - [[maybe_unused]] bool ok = _proto_obj->SerializeWithCachedSizesToArray(buf); - assert(ok); - _proto_obj = nullptr; - } -protected: - vespalib::GrowableByteBuffer& buffer() noexcept { return _out_buf; } - - // Precondition: encode() is not called - ProtobufType& proto_obj() noexcept { return *_proto_obj; } - const ProtobufType& proto_obj() const noexcept { return *_proto_obj; } -}; - -template -class RequestEncoder : public BaseEncoder { -public: - RequestEncoder(vespalib::GrowableByteBuffer& out_buf, const api::StorageCommand& cmd) - : BaseEncoder(out_buf) - { - write_request_header(out_buf, cmd); - } - - // Precondition: encode() is not called - ProtobufType& request() noexcept { return this->proto_obj(); } - const ProtobufType& request() const noexcept { return this->proto_obj(); } -}; - -template -class ResponseEncoder : public BaseEncoder { -public: - ResponseEncoder(vespalib::GrowableByteBuffer& out_buf, const api::StorageReply& reply) - : BaseEncoder(out_buf) - { - write_response_header(out_buf, reply); - } - - // Precondition: encode() is not called - ProtobufType& response() noexcept { return this->proto_obj(); } - const ProtobufType& response() const noexcept { return this->proto_obj(); } -}; - -template -class RequestDecoder { - protobuf::RequestHeader _hdr; - ::google::protobuf::Arena _arena; - ProtobufType* _proto_obj; -public: - RequestDecoder(document::ByteBuffer& in_buf) - : _arena(), - _proto_obj(::google::protobuf::Arena::Create(&_arena)) - { - decode_request_header(in_buf, _hdr); - assert(in_buf.getRemaining() <= INT_MAX); - bool ok = _proto_obj->ParseFromArray(in_buf.getBufferAtPos(), in_buf.getRemaining()); - if (!ok) { - throw vespalib::IllegalArgumentException( - vespalib::make_string("Malformed protobuf request payload for %s", - ProtobufType::descriptor()->full_name().c_str())); - } - } - - void transfer_meta_information_to(api::StorageCommand& dest) { - dest.forceMsgId(_hdr.message_id()); - dest.setPriority(static_cast(_hdr.priority())); - dest.setSourceIndex(static_cast(_hdr.source_index())); - } - - ProtobufType& request() noexcept { return *_proto_obj; } - const ProtobufType& request() const noexcept { return *_proto_obj; } -}; - -template -class ResponseDecoder { - protobuf::ResponseHeader _hdr; - ::google::protobuf::Arena _arena; - ProtobufType* _proto_obj; -public: - explicit ResponseDecoder(document::ByteBuffer& in_buf) - : _arena(), - _proto_obj(::google::protobuf::Arena::Create(&_arena)) - { - decode_response_header(in_buf, _hdr); - assert(in_buf.getRemaining() <= INT_MAX); - bool ok = _proto_obj->ParseFromArray(in_buf.getBufferAtPos(), in_buf.getRemaining()); - if (!ok) { - throw vespalib::IllegalArgumentException( - vespalib::make_string("Malformed protobuf response payload for %s", - ProtobufType::descriptor()->full_name().c_str())); - } - } - - void transfer_meta_information_to(api::StorageReply& dest) { - dest.forceMsgId(_hdr.message_id()); - dest.setPriority(static_cast(_hdr.priority())); - dest.setResult(api::ReturnCode(static_cast(_hdr.return_code_id()), - _hdr.return_code_message())); - } - - ProtobufType& response() noexcept { return *_proto_obj; } - const ProtobufType& response() const noexcept { return *_proto_obj; } -}; - -template -void encode_request(vespalib::GrowableByteBuffer& out_buf, const api::StorageCommand& msg, Func&& f) { - RequestEncoder enc(out_buf, msg); - f(enc.request()); - enc.encode(); -} - -template -void encode_response(vespalib::GrowableByteBuffer& out_buf, const api::StorageReply& reply, Func&& f) { - ResponseEncoder enc(out_buf, reply); - auto& res = enc.response(); - f(res); - enc.encode(); -} - -template -std::unique_ptr -ProtocolSerialization7::decode_request(document::ByteBuffer& in_buf, Func&& f) const { - RequestDecoder dec(in_buf); - const auto& req = dec.request(); - auto cmd = f(req); - dec.transfer_meta_information_to(*cmd); - return cmd; -} - -template -std::unique_ptr -ProtocolSerialization7::decode_response(document::ByteBuffer& in_buf, Func&& f) const { - ResponseDecoder dec(in_buf); - const auto& res = dec.response(); - auto reply = f(res); - dec.transfer_meta_information_to(*reply); - return reply; -} - -template -void encode_bucket_request(vespalib::GrowableByteBuffer& out_buf, const api::BucketCommand& msg, Func&& f) { - encode_request(out_buf, msg, [&](ProtobufType& req) { - set_bucket(*req.mutable_bucket(), msg.getBucket()); - f(req); - }); -} - -template -std::unique_ptr -ProtocolSerialization7::decode_bucket_request(document::ByteBuffer& in_buf, Func&& f) const { - return decode_request(in_buf, [&](const ProtobufType& req) { - if (!req.has_bucket()) { - throw vespalib::IllegalArgumentException( - vespalib::make_string("Malformed protocol buffer request for %s; no bucket", - ProtobufType::descriptor()->full_name().c_str())); - } - const auto bucket = get_bucket(req.bucket()); - return f(req, bucket); - }); -} - -template -void encode_bucket_response(vespalib::GrowableByteBuffer& out_buf, const api::BucketReply& reply, Func&& f) { - encode_response(out_buf, reply, [&](ProtobufType& res) { - if (reply.hasBeenRemapped()) { - set_bucket_id(*res.mutable_remapped_bucket_id(), reply.getBucketId()); - } - f(res); - }); -} - -template -std::unique_ptr -ProtocolSerialization7::decode_bucket_response(document::ByteBuffer& in_buf, Func&& f) const { - return decode_response(in_buf, [&](const ProtobufType& res) { - auto reply = f(res); - if (res.has_remapped_bucket_id()) { - reply->remapBucketId(get_bucket_id(res.remapped_bucket_id())); - } - return reply; - }); -} - -template -void encode_bucket_info_response(vespalib::GrowableByteBuffer& out_buf, const api::BucketInfoReply& reply, Func&& f) { - encode_bucket_response(out_buf, reply, [&](ProtobufType& res) { - set_bucket_info(*res.mutable_bucket_info(), reply.getBucketInfo()); - f(res); - }); -} - -template -std::unique_ptr -ProtocolSerialization7::decode_bucket_info_response(document::ByteBuffer& in_buf, Func&& f) const { - return decode_bucket_response(in_buf, [&](const ProtobufType& res) { - auto reply = f(res); - reply->setBucketInfo(get_bucket_info(res.bucket_info())); // If not present, default of all zeroes is correct - return reply; - }); -} - -// TODO document protobuf ducktyping assumptions - -namespace { -// Inherit from known base class just to avoid having to template this. We don't care about its subtype anyway. -void no_op_encode([[maybe_unused]] ::google::protobuf::Message&) { - // nothing to do here. -} - -void set_document(protobuf::Document& target_doc, const document::Document& src_doc) { - vespalib::nbostream stream; - src_doc.serialize(stream); - target_doc.set_payload(stream.peek(), stream.size()); -} - -} - -// ----------------------------------------------------------------- -// Put -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::PutCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - req.set_new_timestamp(msg.getTimestamp()); - req.set_expected_old_timestamp(msg.getUpdateTimestamp()); - if (msg.getCondition().isPresent()) { - set_tas_condition(*req.mutable_condition(), msg.getCondition()); - } - if (msg.getDocument()) { - set_document(*req.mutable_document(), *msg.getDocument()); - } - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::PutReply& msg) const { - encode_bucket_info_response(buf, msg, [&](auto& res) { - res.set_was_found(msg.wasFound()); - }); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodePutCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - auto document = get_document(req.document(), type_repo()); - auto cmd = std::make_unique(bucket, std::move(document), req.new_timestamp()); - cmd->setUpdateTimestamp(req.expected_old_timestamp()); - if (req.has_condition()) { - cmd->setCondition(get_tas_condition(req.condition())); - } - return cmd; - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodePutReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_info_response(buf, [&](auto& res) { - return std::make_unique(static_cast(cmd), res.was_found()); - }); -} - -// ----------------------------------------------------------------- -// Update -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::UpdateCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - auto* update = msg.getUpdate().get(); - if (update) { - set_update(*req.mutable_update(), *update); - } - req.set_new_timestamp(msg.getTimestamp()); - req.set_expected_old_timestamp(msg.getOldTimestamp()); - if (msg.getCondition().isPresent()) { - set_tas_condition(*req.mutable_condition(), msg.getCondition()); - } - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::UpdateReply& msg) const { - encode_bucket_info_response(buf, msg, [&](auto& res) { - res.set_updated_timestamp(msg.getOldTimestamp()); - }); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeUpdateCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - auto update = get_update(req.update(), type_repo()); - auto cmd = std::make_unique(bucket, std::move(update), req.new_timestamp()); - cmd->setOldTimestamp(req.expected_old_timestamp()); - if (req.has_condition()) { - cmd->setCondition(get_tas_condition(req.condition())); - } - return cmd; - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeUpdateReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_info_response(buf, [&](auto& res) { - return std::make_unique(static_cast(cmd), - res.updated_timestamp()); - }); -} - -// ----------------------------------------------------------------- -// Remove -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - auto doc_id_str = msg.getDocumentId().toString(); - req.set_document_id(doc_id_str.data(), doc_id_str.size()); - req.set_new_timestamp(msg.getTimestamp()); - if (msg.getCondition().isPresent()) { - set_tas_condition(*req.mutable_condition(), msg.getCondition()); - } - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveReply& msg) const { - encode_bucket_info_response(buf, msg, [&](auto& res) { - res.set_removed_timestamp(msg.getOldTimestamp()); - }); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - document::DocumentId doc_id(vespalib::stringref(req.document_id().data(), req.document_id().size())); - auto cmd = std::make_unique(bucket, doc_id, req.new_timestamp()); - if (req.has_condition()) { - cmd->setCondition(get_tas_condition(req.condition())); - } - return cmd; - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeRemoveReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_info_response(buf, [&](auto& res) { - return std::make_unique(static_cast(cmd), - res.removed_timestamp()); - }); -} - -// ----------------------------------------------------------------- -// Get -// ----------------------------------------------------------------- - -namespace { - -protobuf::GetRequest_InternalReadConsistency read_consistency_to_protobuf(api::InternalReadConsistency consistency) { - switch (consistency) { - case api::InternalReadConsistency::Strong: return protobuf::GetRequest_InternalReadConsistency_Strong; - case api::InternalReadConsistency::Weak: return protobuf::GetRequest_InternalReadConsistency_Weak; - default: return protobuf::GetRequest_InternalReadConsistency_Strong; - } -} - -api::InternalReadConsistency read_consistency_from_protobuf(protobuf::GetRequest_InternalReadConsistency consistency) { - switch (consistency) { - case protobuf::GetRequest_InternalReadConsistency_Strong: return api::InternalReadConsistency::Strong; - case protobuf::GetRequest_InternalReadConsistency_Weak: return api::InternalReadConsistency::Weak; - default: return api::InternalReadConsistency::Strong; - } -} - -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - auto doc_id = msg.getDocumentId().toString(); - req.set_document_id(doc_id.data(), doc_id.size()); - req.set_before_timestamp(msg.getBeforeTimestamp()); - if (!msg.getFieldSet().empty()) { - req.set_field_set(msg.getFieldSet().data(), msg.getFieldSet().size()); - } - req.set_internal_read_consistency(read_consistency_to_protobuf(msg.internal_read_consistency())); - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetReply& msg) const { - encode_bucket_info_response(buf, msg, [&](auto& res) { - if (msg.getDocument()) { - set_document(*res.mutable_document(), *msg.getDocument()); - } - if (!msg.is_tombstone()) { - res.set_last_modified_timestamp(msg.getLastModifiedTimestamp()); - } else { - // This field will be ignored by older versions, making the behavior as if - // a timestamp of zero was returned for tombstones, as it the legacy behavior. - res.set_tombstone_timestamp(msg.getLastModifiedTimestamp()); - // Will not be encoded onto the wire, but we include it here to hammer down the - // point that it's intentional to have the last modified time appear as a not - // found document for older versions. - res.set_last_modified_timestamp(0); - } - }); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeGetCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - document::DocumentId doc_id(vespalib::stringref(req.document_id().data(), req.document_id().size())); - auto op = std::make_unique(bucket, std::move(doc_id), - req.field_set(), req.before_timestamp()); - op->set_internal_read_consistency(read_consistency_from_protobuf(req.internal_read_consistency())); - return op; - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeGetReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_info_response(buf, [&](auto& res) { - try { - auto document = get_document(res.document(), type_repo()); - const bool is_tombstone = (res.tombstone_timestamp() != 0); - const auto effective_timestamp = (is_tombstone ? res.tombstone_timestamp() - : res.last_modified_timestamp()); - return std::make_unique(static_cast(cmd), - std::move(document), effective_timestamp, - false, is_tombstone); - } catch (std::exception& e) { - auto reply = std::make_unique(static_cast(cmd), - std::shared_ptr(), 0u); - reply->setResult(api::ReturnCode(api::ReturnCode::UNPARSEABLE, e.what())); - return reply; - } - }); -} - -// ----------------------------------------------------------------- -// Revert -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RevertCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - auto* tokens = req.mutable_revert_tokens(); - assert(msg.getRevertTokens().size() <= INT_MAX); - tokens->Reserve(static_cast(msg.getRevertTokens().size())); - for (auto token : msg.getRevertTokens()) { - tokens->Add(token); - } - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RevertReply& msg) const { - encode_bucket_info_response(buf, msg, no_op_encode); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeRevertCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - std::vector tokens; - tokens.reserve(req.revert_tokens_size()); - for (auto token : req.revert_tokens()) { - tokens.emplace_back(api::Timestamp(token)); - } - return std::make_unique(bucket, std::move(tokens)); - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeRevertReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_info_response(buf, [&]([[maybe_unused]] auto& res) { - return std::make_unique(static_cast(cmd)); - }); -} - -// ----------------------------------------------------------------- -// RemoveLocation -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - req.set_document_selection(msg.getDocumentSelection().data(), msg.getDocumentSelection().size()); - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationReply& msg) const { - encode_bucket_info_response(buf, msg, [&](auto& res) { - res.mutable_stats()->set_documents_removed(msg.documents_removed()); - }); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - return std::make_unique(req.document_selection(), bucket); - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_info_response(buf, [&](auto& res) { - uint32_t documents_removed = (res.has_stats() ? res.stats().documents_removed() : 0u); - return std::make_unique( - static_cast(cmd), - documents_removed); - }); -} - -// ----------------------------------------------------------------- -// DeleteBucket -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::DeleteBucketCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - set_bucket_info(*req.mutable_expected_bucket_info(), msg.getBucketInfo()); - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::DeleteBucketReply& msg) const { - encode_bucket_info_response(buf, msg, no_op_encode); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeDeleteBucketCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - auto cmd = std::make_unique(bucket); - if (req.has_expected_bucket_info()) { - cmd->setBucketInfo(get_bucket_info(req.expected_bucket_info())); - } - return cmd; - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeDeleteBucketReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_info_response(buf, [&]([[maybe_unused]] auto& res) { - return std::make_unique(static_cast(cmd)); - }); -} - -// ----------------------------------------------------------------- -// CreateBucket -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateBucketCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - req.set_create_as_active(msg.getActive()); - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateBucketReply& msg) const { - encode_bucket_info_response(buf, msg, no_op_encode); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeCreateBucketCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - auto cmd = std::make_unique(bucket); - cmd->setActive(req.create_as_active()); - return cmd; - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeCreateBucketReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_info_response(buf, [&]([[maybe_unused]] auto& res) { - return std::make_unique(static_cast(cmd)); - }); -} - -// ----------------------------------------------------------------- -// MergeBucket -// ----------------------------------------------------------------- - -namespace { - -void set_merge_nodes(::google::protobuf::RepeatedPtrField& dest, - const std::vector& src) -{ - dest.Reserve(src.size()); - for (const auto& src_node : src) { - auto* dest_node = dest.Add(); - dest_node->set_index(src_node.index); - dest_node->set_source_only(src_node.sourceOnly); - } -} - -std::vector get_merge_nodes( - const ::google::protobuf::RepeatedPtrField& src) -{ - std::vector nodes; - nodes.reserve(src.size()); - for (const auto& node : src) { - nodes.emplace_back(node.index(), node.source_only()); - } - return nodes; -} - -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::MergeBucketCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - set_merge_nodes(*req.mutable_nodes(), msg.getNodes()); - req.set_max_timestamp(msg.getMaxTimestamp()); - req.set_cluster_state_version(msg.getClusterStateVersion()); - req.set_unordered_forwarding(msg.use_unordered_forwarding()); - for (uint16_t chain_node : msg.getChain()) { - req.add_node_chain(chain_node); - } - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::MergeBucketReply& msg) const { - encode_bucket_response(buf, msg, no_op_encode); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeMergeBucketCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - auto nodes = get_merge_nodes(req.nodes()); - auto cmd = std::make_unique(bucket, std::move(nodes), req.max_timestamp()); - cmd->setClusterStateVersion(req.cluster_state_version()); - std::vector chain; - chain.reserve(req.node_chain_size()); - for (uint16_t node : req.node_chain()) { - chain.emplace_back(node); - } - cmd->setChain(std::move(chain)); - cmd->set_use_unordered_forwarding(req.unordered_forwarding()); - return cmd; - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeMergeBucketReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_response(buf, [&]([[maybe_unused]] auto& res) { - return std::make_unique(static_cast(cmd)); - }); -} - -// ----------------------------------------------------------------- -// GetBucketDiff -// ----------------------------------------------------------------- - -namespace { - -void set_global_id(protobuf::GlobalId& dest, const document::GlobalId& src) { - static_assert(document::GlobalId::LENGTH == 12); - uint64_t lo64; - uint32_t hi32; - memcpy(&lo64, src.get(), sizeof(uint64_t)); - memcpy(&hi32, src.get() + sizeof(uint64_t), sizeof(uint32_t)); - dest.set_lo_64(lo64); - dest.set_hi_32(hi32); -} - -document::GlobalId get_global_id(const protobuf::GlobalId& src) { - static_assert(document::GlobalId::LENGTH == 12); - const uint64_t lo64 = src.lo_64(); - const uint32_t hi32 = src.hi_32(); - - char buf[document::GlobalId::LENGTH]; - memcpy(buf, &lo64, sizeof(uint64_t)); - memcpy(buf + sizeof(uint64_t), &hi32, sizeof(uint32_t)); - return document::GlobalId(buf); -} - -void set_diff_entry(protobuf::MetaDiffEntry& dest, const api::GetBucketDiffCommand::Entry& src) { - dest.set_timestamp(src._timestamp); - set_global_id(*dest.mutable_gid(), src._gid); - dest.set_header_size(src._headerSize); - dest.set_body_size(src._bodySize); - dest.set_flags(src._flags); - dest.set_presence_mask(src._hasMask); -} - -api::GetBucketDiffCommand::Entry get_diff_entry(const protobuf::MetaDiffEntry& src) { - api::GetBucketDiffCommand::Entry e; - e._timestamp = src.timestamp(); - e._gid = get_global_id(src.gid()); - e._headerSize = src.header_size(); - e._bodySize = src.body_size(); - e._flags = src.flags(); - e._hasMask = src.presence_mask(); - return e; -} - -void fill_proto_meta_diff(::google::protobuf::RepeatedPtrField& dest, - const std::vector& src) { - for (const auto& diff_entry : src) { - set_diff_entry(*dest.Add(), diff_entry); - } -} - -void fill_api_meta_diff(std::vector& dest, - const ::google::protobuf::RepeatedPtrField& src) { - // FIXME GetBucketDiffReply ctor copies the diff from the request for some reason - // TODO verify this isn't actually used anywhere and remove this "feature". - dest.clear(); - dest.reserve(src.size()); - for (const auto& diff_entry : src) { - dest.emplace_back(get_diff_entry(diff_entry)); - } -} - -} // anonymous namespace - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetBucketDiffCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - set_merge_nodes(*req.mutable_nodes(), msg.getNodes()); - req.set_max_timestamp(msg.getMaxTimestamp()); - fill_proto_meta_diff(*req.mutable_diff(), msg.getDiff()); - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetBucketDiffReply& msg) const { - encode_bucket_response(buf, msg, [&](auto& res) { - fill_proto_meta_diff(*res.mutable_diff(), msg.getDiff()); - }); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeGetBucketDiffCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - auto nodes = get_merge_nodes(req.nodes()); - auto cmd = std::make_unique(bucket, std::move(nodes), req.max_timestamp()); - fill_api_meta_diff(cmd->getDiff(), req.diff()); - return cmd; - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeGetBucketDiffReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_response(buf, [&](auto& res) { - auto reply = std::make_unique(static_cast(cmd)); - fill_api_meta_diff(reply->getDiff(), res.diff()); - return reply; - }); -} - -// ----------------------------------------------------------------- -// ApplyBucketDiff -// ----------------------------------------------------------------- - -namespace { - -void fill_api_apply_diff_vector(std::vector& diff, - const ::google::protobuf::RepeatedPtrField& src) -{ - // We use the same approach as the legacy protocols here in that we pre-reserve and - // directly write into the vector. This avoids having to ensure all buffer management is movable. - size_t n_entries = src.size(); - diff.resize(n_entries); - for (size_t i = 0; i < n_entries; ++i) { - auto& proto_entry = src.Get(i); - auto& dest = diff[i]; - dest._entry = get_diff_entry(proto_entry.entry_meta()); - dest._docName = proto_entry.document_id(); - // TODO consider making buffers std::strings instead to avoid explicit zeroing-on-resize overhead - dest._headerBlob.resize(proto_entry.header_blob().size()); - memcpy(dest._headerBlob.data(), proto_entry.header_blob().data(), proto_entry.header_blob().size()); - dest._bodyBlob.resize(proto_entry.body_blob().size()); - memcpy(dest._bodyBlob.data(), proto_entry.body_blob().data(), proto_entry.body_blob().size()); - } -} - -void fill_proto_apply_diff_vector(::google::protobuf::RepeatedPtrField& dest, - const std::vector& src) -{ - dest.Reserve(src.size()); - for (const auto& entry : src) { - auto* proto_entry = dest.Add(); - set_diff_entry(*proto_entry->mutable_entry_meta(), entry._entry); - proto_entry->set_document_id(entry._docName.data(), entry._docName.size()); - proto_entry->set_header_blob(entry._headerBlob.data(), entry._headerBlob.size()); - proto_entry->set_body_blob(entry._bodyBlob.data(), entry._bodyBlob.size()); - } -} - -} // anonymous namespace - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - set_merge_nodes(*req.mutable_nodes(), msg.getNodes()); - req.set_max_buffer_size(0x400000); // Unused, GC soon. - fill_proto_apply_diff_vector(*req.mutable_entries(), msg.getDiff()); - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::ApplyBucketDiffReply& msg) const { - encode_bucket_response(buf, msg, [&](auto& res) { - fill_proto_apply_diff_vector(*res.mutable_entries(), msg.getDiff()); - }); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeApplyBucketDiffCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - auto nodes = get_merge_nodes(req.nodes()); - auto cmd = std::make_unique(bucket, std::move(nodes)); - fill_api_apply_diff_vector(cmd->getDiff(), req.entries()); - return cmd; - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeApplyBucketDiffReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_response(buf, [&](auto& res) { - auto reply = std::make_unique(static_cast(cmd)); - fill_api_apply_diff_vector(reply->getDiff(), res.entries()); - return reply; - }); -} - -// ----------------------------------------------------------------- -// RequestBucketInfo -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RequestBucketInfoCommand& msg) const { - encode_request(buf, msg, [&](auto& req) { - set_bucket_space(*req.mutable_bucket_space(), msg.getBucketSpace()); - auto& buckets = msg.getBuckets(); - if (!buckets.empty()) { - auto* proto_buckets = req.mutable_explicit_bucket_set(); - for (const auto& b : buckets) { - set_bucket_id(*proto_buckets->add_bucket_ids(), b); - } - } else { - auto* all_buckets = req.mutable_all_buckets(); - auto cluster_state = msg.getSystemState().toString(); - all_buckets->set_distributor_index(msg.getDistributor()); - all_buckets->set_cluster_state(cluster_state.data(), cluster_state.size()); - all_buckets->set_distribution_hash(msg.getDistributionHash().data(), msg.getDistributionHash().size()); - } - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RequestBucketInfoReply& msg) const { - encode_response(buf, msg, [&](auto& res) { - auto* proto_info = res.mutable_bucket_infos(); - proto_info->Reserve(msg.getBucketInfo().size()); - for (const auto& entry : msg.getBucketInfo()) { - auto* bucket_and_info = proto_info->Add(); - bucket_and_info->set_raw_bucket_id(entry._bucketId.getRawId()); - set_bucket_info(*bucket_and_info->mutable_bucket_info(), entry._info); - } - // We mark features as available at protocol level. Only included for full bucket fetch responses. - if (msg.full_bucket_fetch()) { - res.mutable_supported_node_features()->set_unordered_merge_chaining(true); - } - }); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeRequestBucketInfoCommand(BBuf& buf) const { - return decode_request(buf, [&](auto& req) { - auto bucket_space = get_bucket_space(req.bucket_space()); - if (req.has_explicit_bucket_set()) { - const uint32_t n_buckets = req.explicit_bucket_set().bucket_ids_size(); - std::vector buckets(n_buckets); - const auto& proto_buckets = req.explicit_bucket_set().bucket_ids(); - for (uint32_t i = 0; i < n_buckets; ++i) { - buckets[i] = get_bucket_id(proto_buckets.Get(i)); - } - return std::make_unique(bucket_space, std::move(buckets)); - } else if (req.has_all_buckets()) { - const auto& all_req = req.all_buckets(); - return std::make_unique( - bucket_space, all_req.distributor_index(), - lib::ClusterState(all_req.cluster_state()), all_req.distribution_hash()); - } else { - throw vespalib::IllegalArgumentException("RequestBucketInfo does not have any applicable fields set"); - } - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeRequestBucketInfoReply(const SCmd& cmd, BBuf& buf) const { - return decode_response(buf, [&](auto& res) { - auto reply = std::make_unique(static_cast(cmd)); - auto& dest_entries = reply->getBucketInfo(); - uint32_t n_entries = res.bucket_infos_size(); - dest_entries.resize(n_entries); - for (uint32_t i = 0; i < n_entries; ++i) { - const auto& proto_entry = res.bucket_infos(i); - dest_entries[i]._bucketId = document::BucketId(proto_entry.raw_bucket_id()); - dest_entries[i]._info = get_bucket_info(proto_entry.bucket_info()); - } - if (res.has_supported_node_features()) { - const auto& src_features = res.supported_node_features(); - auto& dest_features = reply->supported_node_features(); - dest_features.unordered_merge_chaining = src_features.unordered_merge_chaining(); - } - return reply; - }); -} - -// ----------------------------------------------------------------- -// NotifyBucketChange -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - set_bucket_info(*req.mutable_bucket_info(), msg.getBucketInfo()); - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::NotifyBucketChangeReply& msg) const { - encode_response(buf, msg, no_op_encode); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeNotifyBucketChangeCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - auto bucket_info = get_bucket_info(req.bucket_info()); - return std::make_unique(bucket, bucket_info); - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeNotifyBucketChangeReply(const SCmd& cmd, BBuf& buf) const { - return decode_response(buf, [&]([[maybe_unused]] auto& res) { - return std::make_unique(static_cast(cmd)); - }); -} - -// ----------------------------------------------------------------- -// SplitBucket -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::SplitBucketCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - req.set_min_split_bits(msg.getMinSplitBits()); - req.set_max_split_bits(msg.getMaxSplitBits()); - req.set_min_byte_size(msg.getMinByteSize()); - req.set_min_doc_count(msg.getMinDocCount()); - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::SplitBucketReply& msg) const { - encode_bucket_response(buf, msg, [&](auto& res) { - for (const auto& split_info : msg.getSplitInfo()) { - auto* proto_info = res.add_split_info(); - proto_info->set_raw_bucket_id(split_info.first.getRawId()); - set_bucket_info(*proto_info->mutable_bucket_info(), split_info.second); - } - }); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeSplitBucketCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - auto cmd = std::make_unique(bucket); - cmd->setMinSplitBits(static_cast(req.min_split_bits())); - cmd->setMaxSplitBits(static_cast(req.max_split_bits())); - cmd->setMinByteSize(req.min_byte_size()); - cmd->setMinDocCount(req.min_doc_count()); - return cmd; - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeSplitBucketReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_response(buf, [&](auto& res) { - auto reply = std::make_unique(static_cast(cmd)); - auto& dest_info = reply->getSplitInfo(); - dest_info.reserve(res.split_info_size()); - for (const auto& proto_info : res.split_info()) { - dest_info.emplace_back(document::BucketId(proto_info.raw_bucket_id()), - get_bucket_info(proto_info.bucket_info())); - } - return reply; - }); -} - -// ----------------------------------------------------------------- -// JoinBuckets -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::JoinBucketsCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - for (const auto& source : msg.getSourceBuckets()) { - set_bucket_id(*req.add_source_buckets(), source); - } - req.set_min_join_bits(msg.getMinJoinBits()); - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::JoinBucketsReply& msg) const { - encode_bucket_info_response(buf, msg, no_op_encode); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeJoinBucketsCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - auto cmd = std::make_unique(bucket); - auto& entries = cmd->getSourceBuckets(); - for (const auto& proto_bucket : req.source_buckets()) { - entries.emplace_back(get_bucket_id(proto_bucket)); - } - cmd->setMinJoinBits(static_cast(req.min_join_bits())); - return cmd; - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeJoinBucketsReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_info_response(buf, [&]([[maybe_unused]] auto& res) { - return std::make_unique(static_cast(cmd)); - }); -} - -// ----------------------------------------------------------------- -// SetBucketState -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::SetBucketStateCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - auto state = (msg.getState() == api::SetBucketStateCommand::BUCKET_STATE::ACTIVE - ? protobuf::SetBucketStateRequest_BucketState_Active - : protobuf::SetBucketStateRequest_BucketState_Inactive); - req.set_state(state); - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::SetBucketStateReply& msg) const { - // SetBucketStateReply is _technically_ a BucketInfoReply, but the legacy protocol impls - // do _not_ encode bucket info as part of the wire format (and it's not used on the distributor), - // so we follow that here and only encode remapping information. - encode_bucket_response(buf, msg, no_op_encode); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeSetBucketStateCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - auto state = (req.state() == protobuf::SetBucketStateRequest_BucketState_Active - ? api::SetBucketStateCommand::BUCKET_STATE::ACTIVE - : api::SetBucketStateCommand::BUCKET_STATE::INACTIVE); - return std::make_unique(bucket, state); - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeSetBucketStateReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_response(buf, [&]([[maybe_unused]] auto& res) { - return std::make_unique(static_cast(cmd)); - }); -} - -// ----------------------------------------------------------------- -// CreateVisitor -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateVisitorCommand& msg) const { - encode_request(buf, msg, [&](auto& req) { - set_bucket_space(*req.mutable_bucket_space(), msg.getBucketSpace()); - for (const auto& bucket : msg.getBuckets()) { - set_bucket_id(*req.add_buckets(), bucket); - } - - auto* ctrl_meta = req.mutable_control_meta(); - ctrl_meta->set_library_name(msg.getLibraryName().data(), msg.getLibraryName().size()); - ctrl_meta->set_instance_id(msg.getInstanceId().data(), msg.getInstanceId().size()); - ctrl_meta->set_visitor_command_id(msg.getVisitorCmdId()); - ctrl_meta->set_control_destination(msg.getControlDestination().data(), msg.getControlDestination().size()); - ctrl_meta->set_data_destination(msg.getDataDestination().data(), msg.getDataDestination().size()); - ctrl_meta->set_queue_timeout(vespalib::count_ms(msg.getQueueTimeout())); - ctrl_meta->set_max_pending_reply_count(msg.getMaximumPendingReplyCount()); - ctrl_meta->set_max_buckets_per_visitor(msg.getMaxBucketsPerVisitor()); - - auto* constraints = req.mutable_constraints(); - constraints->set_document_selection(msg.getDocumentSelection().data(), msg.getDocumentSelection().size()); - constraints->set_from_time_usec(msg.getFromTime()); - constraints->set_to_time_usec(msg.getToTime()); - constraints->set_visit_inconsistent_buckets(msg.visitInconsistentBuckets()); - constraints->set_visit_removes(msg.visitRemoves()); - constraints->set_field_set(msg.getFieldSet().data(), msg.getFieldSet().size()); - - for (const auto& param : msg.getParameters()) { - auto* proto_param = req.add_client_parameters(); - proto_param->set_key(param.first.data(), param.first.size()); - proto_param->set_value(param.second.data(), param.second.size()); - } - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateVisitorReply& msg) const { - encode_response(buf, msg, [&](auto& res) { - auto& stats = msg.getVisitorStatistics(); - auto* proto_stats = res.mutable_visitor_statistics(); - proto_stats->set_buckets_visited(stats.getBucketsVisited()); - proto_stats->set_documents_visited(stats.getDocumentsVisited()); - proto_stats->set_bytes_visited(stats.getBytesVisited()); - proto_stats->set_documents_returned(stats.getDocumentsReturned()); - proto_stats->set_bytes_returned(stats.getBytesReturned()); - proto_stats->set_second_pass_documents_returned(stats.getSecondPassDocumentsReturned()); // TODO remove on Vespa 8 - proto_stats->set_second_pass_bytes_returned(stats.getSecondPassBytesReturned()); // TODO remove on Vespa 8 - }); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeCreateVisitorCommand(BBuf& buf) const { - return decode_request(buf, [&](auto& req) { - auto bucket_space = get_bucket_space(req.bucket_space()); - auto& ctrl_meta = req.control_meta(); - auto& constraints = req.constraints(); - auto cmd = std::make_unique(bucket_space, ctrl_meta.library_name(), - ctrl_meta.instance_id(), constraints.document_selection()); - for (const auto& proto_bucket : req.buckets()) { - cmd->getBuckets().emplace_back(get_bucket_id(proto_bucket)); - } - - cmd->setVisitorCmdId(ctrl_meta.visitor_command_id()); - cmd->setControlDestination(ctrl_meta.control_destination()); - cmd->setDataDestination(ctrl_meta.data_destination()); - cmd->setMaximumPendingReplyCount(ctrl_meta.max_pending_reply_count()); - cmd->setQueueTimeout(std::chrono::milliseconds(ctrl_meta.queue_timeout())); - cmd->setMaxBucketsPerVisitor(ctrl_meta.max_buckets_per_visitor()); - cmd->setVisitorDispatcherVersion(50); // FIXME this magic number is lifted verbatim from the 5.1 protocol impl - - for (const auto& proto_param : req.client_parameters()) { - cmd->getParameters().set(proto_param.key(), proto_param.value()); - } - - cmd->setFromTime(constraints.from_time_usec()); - cmd->setToTime(constraints.to_time_usec()); - cmd->setVisitRemoves(constraints.visit_removes()); - cmd->setFieldSet(constraints.field_set()); - cmd->setVisitInconsistentBuckets(constraints.visit_inconsistent_buckets()); - return cmd; - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) const { - return decode_response(buf, [&](auto& res) { - auto reply = std::make_unique(static_cast(cmd)); - vdslib::VisitorStatistics vs; - const auto& proto_stats = res.visitor_statistics(); - vs.setBucketsVisited(proto_stats.buckets_visited()); - vs.setDocumentsVisited(proto_stats.documents_visited()); - vs.setBytesVisited(proto_stats.bytes_visited()); - vs.setDocumentsReturned(proto_stats.documents_returned()); - vs.setBytesReturned(proto_stats.bytes_returned()); - vs.setSecondPassDocumentsReturned(proto_stats.second_pass_documents_returned()); // TODO remove on Vespa 8 - vs.setSecondPassBytesReturned(proto_stats.second_pass_bytes_returned()); // TODO remove on Vespa 8 - reply->setVisitorStatistics(vs); - return reply; - }); -} - -// ----------------------------------------------------------------- -// DestroyVisitor -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::DestroyVisitorCommand& msg) const { - encode_request(buf, msg, [&](auto& req) { - req.set_instance_id(msg.getInstanceId().data(), msg.getInstanceId().size()); - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::DestroyVisitorReply& msg) const { - encode_response(buf, msg, no_op_encode); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeDestroyVisitorCommand(BBuf& buf) const { - return decode_request(buf, [&](auto& req) { - return std::make_unique(req.instance_id()); - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeDestroyVisitorReply(const SCmd& cmd, BBuf& buf) const { - return decode_response(buf, [&]([[maybe_unused]] auto& res) { - return std::make_unique(static_cast(cmd)); - }); -} - -// ----------------------------------------------------------------- -// StatBucket -// ----------------------------------------------------------------- - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::StatBucketCommand& msg) const { - encode_bucket_request(buf, msg, [&](auto& req) { - req.set_document_selection(msg.getDocumentSelection().data(), msg.getDocumentSelection().size()); - }); -} - -void ProtocolSerialization7::onEncode(GBBuf& buf, const api::StatBucketReply& msg) const { - encode_bucket_response(buf, msg, [&](auto& res) { - res.set_results(msg.getResults().data(), msg.getResults().size()); - }); -} - -api::StorageCommand::UP ProtocolSerialization7::onDecodeStatBucketCommand(BBuf& buf) const { - return decode_bucket_request(buf, [&](auto& req, auto& bucket) { - return std::make_unique(bucket, req.document_selection()); - }); -} - -api::StorageReply::UP ProtocolSerialization7::onDecodeStatBucketReply(const SCmd& cmd, BBuf& buf) const { - return decode_bucket_response(buf, [&](auto& res) { - return std::make_unique(static_cast(cmd), res.results()); - }); -} - -} // storage::mbusprot diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h deleted file mode 100644 index a61397c85ac..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "protocolserialization.h" - -namespace storage::mbusprot { - -/** - * Protocol serialization version that uses Protocol Buffers for all its binary - * encoding and decoding. - */ -class ProtocolSerialization7 final : public ProtocolSerialization { - const std::shared_ptr _repo; -public: - explicit ProtocolSerialization7(std::shared_ptr repo); - - const document::DocumentTypeRepo& type_repo() const noexcept { return *_repo; } - - // Put - void onEncode(GBBuf&, const api::PutCommand&) const override; - void onEncode(GBBuf&, const api::PutReply&) const override; - SCmd::UP onDecodePutCommand(BBuf&) const override; - SRep::UP onDecodePutReply(const SCmd&, BBuf&) const override; - - // Update - void onEncode(GBBuf&, const api::UpdateCommand&) const override; - void onEncode(GBBuf&, const api::UpdateReply&) const override; - SCmd::UP onDecodeUpdateCommand(BBuf&) const override; - SRep::UP onDecodeUpdateReply(const SCmd&, BBuf&) const override; - - // Remove - void onEncode(GBBuf&, const api::RemoveCommand&) const override; - void onEncode(GBBuf&, const api::RemoveReply&) const override; - SCmd::UP onDecodeRemoveCommand(BBuf&) const override; - SRep::UP onDecodeRemoveReply(const SCmd&, BBuf&) const override; - - // Get - void onEncode(GBBuf&, const api::GetCommand&) const override; - void onEncode(GBBuf&, const api::GetReply&) const override; - SCmd::UP onDecodeGetCommand(BBuf&) const override; - SRep::UP onDecodeGetReply(const SCmd&, BBuf&) const override; - - // Revert - TODO this is deprecated, no? - void onEncode(GBBuf&, const api::RevertCommand&) const override; - void onEncode(GBBuf&, const api::RevertReply&) const override; - SCmd::UP onDecodeRevertCommand(BBuf&) const override; - SRep::UP onDecodeRevertReply(const SCmd&, BBuf&) const override; - - // DeleteBucket - void onEncode(GBBuf&, const api::DeleteBucketCommand&) const override; - void onEncode(GBBuf&, const api::DeleteBucketReply&) const override; - SCmd::UP onDecodeDeleteBucketCommand(BBuf&) const override; - SRep::UP onDecodeDeleteBucketReply(const SCmd&, BBuf&) const override; - - // CreateBucket - void onEncode(GBBuf&, const api::CreateBucketCommand&) const override; - void onEncode(GBBuf&, const api::CreateBucketReply&) const override; - SCmd::UP onDecodeCreateBucketCommand(BBuf&) const override; - SRep::UP onDecodeCreateBucketReply(const SCmd&, BBuf&) const override; - - // MergeBucket - void onEncode(GBBuf&, const api::MergeBucketCommand&) const override; - void onEncode(GBBuf&, const api::MergeBucketReply&) const override; - SCmd::UP onDecodeMergeBucketCommand(BBuf&) const override; - SRep::UP onDecodeMergeBucketReply(const SCmd&, BBuf&) const override; - - // GetBucketDiff - void onEncode(GBBuf&, const api::GetBucketDiffCommand&) const override; - void onEncode(GBBuf&, const api::GetBucketDiffReply&) const override; - SCmd::UP onDecodeGetBucketDiffCommand(BBuf&) const override; - SRep::UP onDecodeGetBucketDiffReply(const SCmd&, BBuf&) const override; - - // ApplyBucketDiff - void onEncode(GBBuf&, const api::ApplyBucketDiffCommand&) const override; - void onEncode(GBBuf&, const api::ApplyBucketDiffReply&) const override; - SCmd::UP onDecodeApplyBucketDiffCommand(BBuf&) const override; - SRep::UP onDecodeApplyBucketDiffReply(const SCmd&, BBuf&) const override; - - // RequestBucketInfo - void onEncode(GBBuf&, const api::RequestBucketInfoCommand&) const override; - void onEncode(GBBuf&, const api::RequestBucketInfoReply&) const override; - SCmd::UP onDecodeRequestBucketInfoCommand(BBuf&) const override; - SRep::UP onDecodeRequestBucketInfoReply(const SCmd&, BBuf&) const override; - - // NotifyBucketChange - void onEncode(GBBuf&, const api::NotifyBucketChangeCommand&) const override; - void onEncode(GBBuf&, const api::NotifyBucketChangeReply&) const override; - SCmd::UP onDecodeNotifyBucketChangeCommand(BBuf&) const override; - SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&, BBuf&) const override; - - // SplitBucket - void onEncode(GBBuf&, const api::SplitBucketCommand&) const override; - void onEncode(GBBuf&, const api::SplitBucketReply&) const override; - SCmd::UP onDecodeSplitBucketCommand(BBuf&) const override; - SRep::UP onDecodeSplitBucketReply(const SCmd&, BBuf&) const override; - - // JoinBuckets - void onEncode(GBBuf&, const api::JoinBucketsCommand&) const override; - void onEncode(GBBuf&, const api::JoinBucketsReply&) const override; - SCmd::UP onDecodeJoinBucketsCommand(BBuf&) const override; - SRep::UP onDecodeJoinBucketsReply(const SCmd&, BBuf&) const override; - - // SetBucketState - void onEncode(GBBuf&, const api::SetBucketStateCommand&) const override; - void onEncode(GBBuf&, const api::SetBucketStateReply&) const override; - SCmd::UP onDecodeSetBucketStateCommand(BBuf&) const override; - SRep::UP onDecodeSetBucketStateReply(const SCmd&, BBuf&) const override; - - // CreateVisitor - void onEncode(GBBuf&, const api::CreateVisitorCommand&) const override; - void onEncode(GBBuf&, const api::CreateVisitorReply&) const override; - SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const override; - SRep::UP onDecodeCreateVisitorReply(const SCmd&, BBuf&) const override; - - // DestroyVisitor - void onEncode(GBBuf&, const api::DestroyVisitorCommand&) const override; - void onEncode(GBBuf&, const api::DestroyVisitorReply&) const override; - SCmd::UP onDecodeDestroyVisitorCommand(BBuf&) const override; - SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const override; - - // RemoveLocation - void onEncode(GBBuf&, const api::RemoveLocationCommand&) const override; - void onEncode(GBBuf&, const api::RemoveLocationReply&) const override; - SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const override; - SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const override; - - // StatBucket - void onEncode(GBBuf&, const api::StatBucketCommand&) const override; - void onEncode(GBBuf&, const api::StatBucketReply&) const override; - SCmd::UP onDecodeStatBucketCommand(BBuf&) const override; - SRep::UP onDecodeStatBucketReply(const SCmd&, BBuf&) const override; - -private: - template - std::unique_ptr decode_request(document::ByteBuffer& in_buf, Func&& f) const; - template - std::unique_ptr decode_response(document::ByteBuffer& in_buf, Func&& f) const; - template - std::unique_ptr decode_bucket_request(document::ByteBuffer& in_buf, Func&& f) const; - template - std::unique_ptr decode_bucket_response(document::ByteBuffer& in_buf, Func&& f) const; - template - std::unique_ptr decode_bucket_info_response(document::ByteBuffer& in_buf, Func&& f) const; -}; - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h deleted file mode 100644 index 457a6178704..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace storage::mbusprot { - -class SerializationHelper -{ -public: - static int64_t getLong(document::ByteBuffer& buf) { - int64_t tmp; - buf.getLongNetwork(tmp); - return tmp; - } - - static int32_t getInt(document::ByteBuffer& buf) { - int32_t tmp; - buf.getIntNetwork(tmp); - return tmp; - } - - static int16_t getShort(document::ByteBuffer& buf) { - int16_t tmp; - buf.getShortNetwork(tmp); - return tmp; - } - - static uint8_t getByte(document::ByteBuffer& buf) { - uint8_t tmp; - buf.getByte(tmp); - return tmp; - } - - static vespalib::stringref getString(document::ByteBuffer& buf) { - uint32_t tmp; - buf.getIntNetwork((int32_t&) tmp); - const char * p = buf.getBufferAtPos(); - buf.incPos(tmp); - vespalib::stringref s(p, tmp); - return s; - } - - static bool getBoolean(document::ByteBuffer& buf) { - uint8_t tmp; - buf.getByte(tmp); - return (tmp == 1); - } - - static api::ReturnCode getReturnCode(document::ByteBuffer& buf) { - api::ReturnCode::Result result = (api::ReturnCode::Result) getInt(buf); - vespalib::stringref message = getString(buf); - return api::ReturnCode(result, message); - } - - static void putReturnCode(const api::ReturnCode& code, vespalib::GrowableByteBuffer& buf) - { - buf.putInt(code.getResult()); - buf.putString(code.getMessage()); - } - - static const uint32_t BUCKET_INFO_SERIALIZED_SIZE = sizeof(uint32_t) * 3; - - static document::GlobalId getGlobalId(document::ByteBuffer& buf) { - std::vector buffer(getShort(buf)); - for (uint32_t i=0; i(repo, stream); - } - } - - static void putDocument(document::Document* doc, vespalib::GrowableByteBuffer& buf) - { - if (doc) { - vespalib::nbostream stream; - doc->serialize(stream); - buf.putInt(stream.size()); - buf.putBytes(stream.peek(), stream.size()); - } else { - buf.putInt(0); - } - } - -}; - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp deleted file mode 100644 index 34fd0992adb..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "storagecommand.h" - -namespace storage::mbusprot { - -StorageCommand::StorageCommand(api::StorageCommand::SP cmd) - : mbus::Message(), - _cmd(std::move(cmd)) -{ } - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h b/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h deleted file mode 100644 index e65c0295e31..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "storagemessage.h" -#include "storageprotocol.h" -#include -#include - -namespace storage::mbusprot { - -class StorageCommand : public mbus::Message, public StorageMessage { -public: - typedef std::unique_ptr UP; - - explicit StorageCommand(api::StorageCommand::SP); - - const mbus::string & getProtocol() const override { return StorageProtocol::NAME; } - uint32_t getType() const override { return _cmd->getType().getId(); } - const api::StorageCommand::SP& getCommand() { return _cmd; } - api::StorageCommand::CSP getCommand() const { return _cmd; } - api::StorageMessage::SP getInternalMessage() override { return _cmd; } - api::StorageMessage::CSP getInternalMessage() const override { return _cmd; } - - bool has_command() const noexcept { return (_cmd.get() != nullptr); } - api::StorageCommand::SP steal_command() { return std::move(_cmd); } - - bool hasBucketSequence() const override { return false; } - - uint8_t priority() const override { - return ((getInternalMessage()->getPriority()) / 255) * 16; - } - -private: - api::StorageCommand::SP _cmd; -}; - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagemessage.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagemessage.cpp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagemessage.h b/storageapi/src/vespa/storageapi/mbusprot/storagemessage.h deleted file mode 100644 index 61323222b89..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/storagemessage.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include - -namespace storage::mbusprot { - -class StorageMessage { -public: - typedef std::unique_ptr UP; - - virtual ~StorageMessage() {} - - virtual api::StorageMessage::SP getInternalMessage() = 0; - virtual api::StorageMessage::CSP getInternalMessage() const = 0; - -}; - -} - diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp deleted file mode 100644 index 3cf54860f7c..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "storageprotocol.h" -#include "serializationhelper.h" -#include "storagecommand.h" -#include "storagereply.h" -#include -#include -#include -#include - -#include -LOG_SETUP(".storage.api.mbusprot.protocol"); - -namespace storage::mbusprot { - -mbus::string StorageProtocol::NAME = "StorageProtocol"; - -StorageProtocol::StorageProtocol(const std::shared_ptr repo) - : _serializer5_0(repo), - _serializer5_1(repo), - _serializer5_2(repo), - _serializer6_0(repo), - _serializer7_0(repo) -{ -} - -StorageProtocol::~StorageProtocol() = default; - -mbus::IRoutingPolicy::UP -StorageProtocol::createPolicy(const mbus::string&, const mbus::string&) const -{ - return mbus::IRoutingPolicy::UP(); -} - -namespace { - vespalib::Version version7_0(7, 41, 19); - vespalib::Version version6_0(6, 240, 0); - vespalib::Version version5_2(5, 93, 30); - vespalib::Version version5_1(5, 1, 0); - vespalib::Version version5_0(5, 0, 12); - vespalib::Version version5_0beta(4, 3, 0); -} - - -static bool -suppressEncodeWarning(const api::StorageMessage *msg) -{ - const auto *req = dynamic_cast(msg); - return ((req != nullptr) && (req->getBucketSpace() != document::FixedBucketSpaces::default_space())); -} - -static mbus::Blob -encodeMessage(const ProtocolSerialization & serializer, - const mbus::Routable & routable, - const StorageMessage & message, - const vespalib::Version & serializerVersion, - const vespalib::Version & actualVersion) -{ - mbus::Blob blob(serializer.encode(*message.getInternalMessage())); - - if (LOG_WOULD_LOG(spam)) { - std::ostringstream messageStream; - document::StringUtil::printAsHex(messageStream, blob.data(), blob.size()); - - LOG(spam, "Encoded message of protocol %s type %s using " - "%s serialization as version is %s:\n%s", - routable.getProtocol().c_str(), - message.getInternalMessage()->getType().toString().c_str(), - serializerVersion.toString().c_str(), - actualVersion.toString().c_str(), - messageStream.str().c_str()); - } - - return blob; -} - - -mbus::Blob -StorageProtocol::encode(const vespalib::Version& version, - const mbus::Routable& routable) const -{ - const StorageMessage & message(dynamic_cast(routable)); - - try { - if (message.getInternalMessage().get() == 0) { - throw vespalib::IllegalArgumentException( - "Given storage message wrapper does not contain a " - "storage message.", - VESPA_STRLOC); - } - - if (version < version5_1) { - if (version < version5_0beta) { - LOGBP(warning, - "No support for using messagebus for version %s." - "Minimum version is %s. Thus we cannot serialize %s.", - version.toString().c_str(), - version5_0beta.toString().c_str(), - message.getInternalMessage()->toString().c_str()); - - return mbus::Blob(0); - } else { - return encodeMessage(_serializer5_0, routable, message, version5_0, version); - } - } else if (version < version5_2) { - return encodeMessage(_serializer5_1, routable, message, version5_1, version); - } else { - if (version < version6_0) { - return encodeMessage(_serializer5_2, routable, message, version5_2, version); - } else if (version < version7_0) { - return encodeMessage(_serializer6_0, routable, message, version6_0, version); - } else { - return encodeMessage(_serializer7_0, routable, message, version7_0, version); - } - } - - } catch (std::exception & e) { - if (!(version < version6_0 && - suppressEncodeWarning(message.getInternalMessage().get()))) { - LOGBP(warning, "Failed to encode %s storage protocol message %s: %s", - version.toString().c_str(), - message.getInternalMessage()->toString().c_str(), - e.what()); - } - } - - return mbus::Blob(0); -} - -static mbus::Routable::UP -decodeMessage(const ProtocolSerialization & serializer, - mbus::BlobRef data, - const api::MessageType & type, - const vespalib::Version & serializerVersion, - const vespalib::Version & actualVersion) -{ - if (LOG_WOULD_LOG(spam)) { - std::ostringstream messageStream; - document::StringUtil::printAsHex(messageStream, data.data(), data.size()); - - LOG(spam, - "Decoding %s of version %s " - "using %s decoder from:\n%s", - type.toString().c_str(), - actualVersion.toString().c_str(), - serializerVersion.toString().c_str(), - messageStream.str().c_str()); - } - - if (type.isReply()) { - return std::make_unique(data, serializer); - } else { - auto command = serializer.decodeCommand(data); - if (command && command->getInternalMessage()) { - command->getInternalMessage()->setApproxByteSize(data.size()); - } - return mbus::Routable::UP(command.release()); - } -} - -mbus::Routable::UP -StorageProtocol::decode(const vespalib::Version & version, - mbus::BlobRef data) const -{ - try { - document::ByteBuffer buf(data.data(), data.size()); - auto & type = api::MessageType::get( - static_cast(SerializationHelper::getInt(buf))); - - StorageMessage::UP message; - if (version < version5_1) { - if (version < version5_0beta) { - LOGBP(error, - "No support for using messagebus for version %s." - "Minimum version is %s.", - version.toString().c_str(), - version5_0beta.toString().c_str()); - } else { - return decodeMessage(_serializer5_0, data, type, version5_0, version); - } - } else if (version < version5_2) { - return decodeMessage(_serializer5_1, data, type, version5_1, version); - } else { - if (version < version6_0) { - return decodeMessage(_serializer5_2, data, type, version5_2, version); - } else if (version < version7_0) { - return decodeMessage(_serializer6_0, data, type, version6_0, version); - } else { - return decodeMessage(_serializer7_0, data, type, version7_0, version); - } - } - } catch (std::exception & e) { - std::ostringstream ost; - ost << "Failed to decode " << version.toString() << " messagebus " - << "storage protocol message: " << e.what() << "\n"; - document::StringUtil::printAsHex(ost, data.data(), data.size()); - LOGBP(warning, "%s", ost.str().c_str()); - } - - return mbus::Routable::UP(); -} - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h deleted file mode 100644 index 3f36ac42117..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "protocolserialization5_2.h" -#include "protocolserialization6_0.h" -#include "protocolserialization7.h" -#include - -namespace storage::mbusprot { - -class StorageProtocol final : public mbus::IProtocol -{ -public: - typedef std::shared_ptr SP; - - static mbus::string NAME; - - explicit StorageProtocol(const std::shared_ptr); - ~StorageProtocol() override; - - const mbus::string& getName() const override { return NAME; } - mbus::IRoutingPolicy::UP createPolicy(const mbus::string& name, const mbus::string& param) const override; - mbus::Blob encode(const vespalib::Version&, const mbus::Routable&) const override; - mbus::Routable::UP decode(const vespalib::Version&, mbus::BlobRef) const override; - bool requireSequencing() const override { return true; } -private: - ProtocolSerialization5_0 _serializer5_0; - ProtocolSerialization5_1 _serializer5_1; - ProtocolSerialization5_2 _serializer5_2; - ProtocolSerialization6_0 _serializer6_0; - ProtocolSerialization7 _serializer7_0; -}; - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp deleted file mode 100644 index 1db6912dd33..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "storagereply.h" -#include "storagecommand.h" -#include -#include - - -using vespalib::alloc::Alloc; -using vespalib::IllegalStateException; - -namespace storage::mbusprot { - -StorageReply::StorageReply(mbus::BlobRef data, const ProtocolSerialization& serializer) - : _serializer(&serializer), - _sz(data.size()), - _buffer(Alloc::alloc(_sz)), - _mbusType(0), - _reply() -{ - memcpy(_buffer.get(), data.data(), _sz); - vespalib::nbostream nbo(data.data(), _sz); - nbo >> _mbusType; -} - -StorageReply::StorageReply(api::StorageReply::SP reply) - : _serializer(0), - _sz(0), - _buffer(), - _mbusType(reply->getType().getId()), - _reply(std::move(reply)) -{} - -StorageReply::~StorageReply() = default; - -void -StorageReply::deserialize() const -{ - if (_reply.get()) return; - StorageReply& reply(const_cast(*this)); - mbus::Message::UP msg(reply.getMessage()); - if (msg.get() == 0) { - throw IllegalStateException("Cannot deserialize storage reply before message have been set", VESPA_STRLOC); - } - const StorageCommand* cmd(dynamic_cast(msg.get())); - reply.setMessage(std::move(msg)); - if (cmd == 0) { - throw IllegalStateException("Storage reply get message did not return a storage command", VESPA_STRLOC); - } - mbus::BlobRef blobRef(static_cast(_buffer.get()), _sz); - _reply = _serializer->decodeReply(blobRef, *cmd->getCommand())->getReply(); - Alloc().swap(_buffer); -} - -} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagereply.h b/storageapi/src/vespa/storageapi/mbusprot/storagereply.h deleted file mode 100644 index 538b7caa678..00000000000 --- a/storageapi/src/vespa/storageapi/mbusprot/storagereply.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "storagemessage.h" -#include "storageprotocol.h" -#include -#include - -namespace storage::mbusprot { - -class StorageReply : public mbus::Reply, public StorageMessage { - const ProtocolSerialization* _serializer; - size_t _sz; - mutable vespalib::alloc::Alloc _buffer; - uint32_t _mbusType; - mutable api::StorageReply::SP _reply; - -public: - typedef std::unique_ptr UP; - - StorageReply(mbus::BlobRef data, const ProtocolSerialization&); - StorageReply(api::StorageReply::SP reply); - ~StorageReply(); - - const mbus::string& getProtocol() const override { return StorageProtocol::NAME; } - - uint32_t getType() const override { return _mbusType; } - - const api::StorageReply::SP& getReply() { deserialize(); return _reply; } - api::StorageReply::CSP getReply() const { deserialize(); return _reply; } - - api::StorageMessage::SP getInternalMessage() override { deserialize(); return _reply; } - api::StorageMessage::CSP getInternalMessage() const override { deserialize(); return _reply; } - - uint8_t priority() const override { - if (_reply) { - return _reply->getPriority(); - } - return 0; - } - -private: - void deserialize() const; -}; - -} diff --git a/storageapi/src/vespa/storageapi/message/.gitignore b/storageapi/src/vespa/storageapi/message/.gitignore deleted file mode 100644 index 526f91c6668..00000000000 --- a/storageapi/src/vespa/storageapi/message/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -*.lo -.*.swp -.depend -.depend.NEW -.deps -.libs -Makefile diff --git a/storageapi/src/vespa/storageapi/message/CMakeLists.txt b/storageapi/src/vespa/storageapi/message/CMakeLists.txt deleted file mode 100644 index 2a761921dff..00000000000 --- a/storageapi/src/vespa/storageapi/message/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageapi_message OBJECT - SOURCES - datagram.cpp - persistence.cpp - bucket.cpp - visitor.cpp - state.cpp - searchresult.cpp - bucketsplitting.cpp - documentsummary.cpp - stat.cpp - removelocation.cpp - queryresult.cpp - internal.cpp - DEPENDS -) diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp deleted file mode 100644 index 520f1aa2741..00000000000 --- a/storageapi/src/vespa/storageapi/message/bucket.cpp +++ /dev/null @@ -1,652 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bucket.h" -#include -#include -#include -#include -#include -#include -#include - -namespace storage::api { - -IMPLEMENT_COMMAND(CreateBucketCommand, CreateBucketReply) -IMPLEMENT_REPLY(CreateBucketReply) -IMPLEMENT_COMMAND(DeleteBucketCommand, DeleteBucketReply) -IMPLEMENT_REPLY(DeleteBucketReply) -IMPLEMENT_COMMAND(MergeBucketCommand, MergeBucketReply) -IMPLEMENT_REPLY(MergeBucketReply) -IMPLEMENT_COMMAND(GetBucketDiffCommand, GetBucketDiffReply) -IMPLEMENT_REPLY(GetBucketDiffReply) -IMPLEMENT_COMMAND(ApplyBucketDiffCommand, ApplyBucketDiffReply) -IMPLEMENT_REPLY(ApplyBucketDiffReply) -IMPLEMENT_COMMAND(RequestBucketInfoCommand, RequestBucketInfoReply) -IMPLEMENT_REPLY(RequestBucketInfoReply) -IMPLEMENT_COMMAND(NotifyBucketChangeCommand, NotifyBucketChangeReply) -IMPLEMENT_REPLY(NotifyBucketChangeReply) -IMPLEMENT_COMMAND(SetBucketStateCommand, SetBucketStateReply) -IMPLEMENT_REPLY(SetBucketStateReply) - -CreateBucketCommand::CreateBucketCommand(const document::Bucket &bucket) - : MaintenanceCommand(MessageType::CREATEBUCKET, bucket), - _active(false) -{ } - -void -CreateBucketCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "CreateBucketCommand(" << getBucketId(); - if (_active) { - out << ", active"; - } else { - out << ", inactive"; - } - out << ")"; - out << " Reasons to start: " << _reason; - if (verbose) { - out << " : "; - MaintenanceCommand::print(out, verbose, indent); - } -} - -CreateBucketReply::CreateBucketReply(const CreateBucketCommand& cmd) - : BucketInfoReply(cmd) -{ -} - -void -CreateBucketReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "CreateBucketReply(" << getBucketId() << ")"; - if (verbose) { - out << " : "; - BucketInfoReply::print(out, verbose, indent); - } -} - -DeleteBucketCommand::DeleteBucketCommand(const document::Bucket &bucket) - : MaintenanceCommand(MessageType::DELETEBUCKET, bucket) -{ } - -void -DeleteBucketCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "DeleteBucketCommand(" << getBucketId() << ")"; - out << " Reasons to start: " << _reason; - if (verbose) { - out << " : "; - MaintenanceCommand::print(out, verbose, indent); - } -} - -DeleteBucketReply::DeleteBucketReply(const DeleteBucketCommand& cmd) - : BucketInfoReply(cmd) -{ -} - -void -DeleteBucketReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "DeleteBucketReply(" << getBucketId() << ")"; - if (verbose) { - out << " : "; - BucketInfoReply::print(out, verbose, indent); - } -} - -MergeBucketCommand::MergeBucketCommand( - const document::Bucket &bucket, const std::vector& nodes, - Timestamp maxTimestamp, uint32_t clusterStateVersion, - const std::vector& chain) - : MaintenanceCommand(MessageType::MERGEBUCKET, bucket), - _nodes(nodes), - _maxTimestamp(maxTimestamp), - _clusterStateVersion(clusterStateVersion), - _chain(chain), - _use_unordered_forwarding(false) -{} - -MergeBucketCommand::~MergeBucketCommand() = default; - -void -MergeBucketCommand::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "MergeBucketCommand(" << getBucketId() << ", to time " - << _maxTimestamp << ", cluster state version: " - << _clusterStateVersion << ", nodes: ["; - for (uint32_t i=0; i<_nodes.size(); ++i) { - if (i != 0) out << ", "; - out << _nodes[i]; - } - out << "], chain: ["; - for (uint32_t i = 0; i < _chain.size(); ++i) { - if (i != 0) out << ", "; - out << _chain[i]; - } - out << "]"; - if (_use_unordered_forwarding) { - out << " (unordered forwarding)"; - } - out << ", reasons to start: " << _reason; - out << ")"; - if (verbose) { - out << " : "; - BucketCommand::print(out, verbose, indent); - } -} - -std::ostream& -operator<<(std::ostream& out, const MergeBucketCommand::Node& n) { - return out << n.index << (n.sourceOnly ? " (source only)" : ""); -} - -MergeBucketReply::MergeBucketReply(const MergeBucketCommand& cmd) - : BucketReply(cmd), - _nodes(cmd.getNodes()), - _maxTimestamp(cmd.getMaxTimestamp()), - _clusterStateVersion(cmd.getClusterStateVersion()), - _chain(cmd.getChain()) -{ -} - -void -MergeBucketReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "MergeBucketReply(" << getBucketId() << ", to time " - << _maxTimestamp << ", cluster state version: " - << _clusterStateVersion << ", nodes: ["; - for (uint32_t i=0; i<_nodes.size(); ++i) { - if (i != 0) out << ", "; - out << _nodes[i]; - } - out << "], chain: ["; - for (uint32_t i = 0; i < _chain.size(); ++i) { - if (i != 0) out << ", "; - out << _chain[i]; - } - out << "])"; - if (verbose) { - out << " : "; - BucketReply::print(out, verbose, indent); - } -} - -GetBucketDiffCommand::Entry::Entry() - : _timestamp(0), - _gid(), - _headerSize(0), - _bodySize(0), - _flags(0), - _hasMask(0) -{ -} - -void GetBucketDiffCommand::Entry::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "Entry(timestamp: " << _timestamp - << ", " << _gid.toString() << ", hasMask: 0x" << _hasMask; - if (verbose) { - out << ",\n" << indent << " " << "header size: " - << std::dec << _headerSize << ", body size: " << _bodySize - << ", flags 0x" << std::hex << _flags << std::dec; - } - out << ")"; -} - -bool GetBucketDiffCommand::Entry::operator==(const Entry& e) const -{ - return (_timestamp == e._timestamp && - _headerSize == e._headerSize && - _bodySize == e._bodySize && - _gid == e._gid && - _flags == e._flags); -} - -GetBucketDiffCommand::GetBucketDiffCommand( - const document::Bucket &bucket, const std::vector& nodes, - Timestamp maxTimestamp) - : BucketCommand(MessageType::GETBUCKETDIFF, bucket), - _nodes(nodes), - _maxTimestamp(maxTimestamp) -{} - -GetBucketDiffCommand::~GetBucketDiffCommand() = default; - -void -GetBucketDiffCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "GetBucketDiffCommand(" << getBucketId() << ", to time " - << _maxTimestamp << ", nodes: ["; - for (uint32_t i=0; i<_nodes.size(); ++i) { - if (i != 0) out << ", "; - out << _nodes[i]; - } - - if (_diff.empty()) { - out << "], no entries"; - } else if (verbose) { - out << "],"; - for (uint32_t i=0; i<_diff.size(); ++i) { - out << "\n" << indent << " "; - _diff[i].print(out, verbose, indent + " "); - } - } else { - out << ", " << _diff.size() << " entries"; - out << ", id " << _msgId; - } - out << ")"; - if (verbose) { - out << " : "; - BucketCommand::print(out, verbose, indent); - } -} - -GetBucketDiffReply::GetBucketDiffReply(const GetBucketDiffCommand& cmd) - : BucketReply(cmd), - _nodes(cmd.getNodes()), - _maxTimestamp(cmd.getMaxTimestamp()), - _diff(cmd.getDiff()) -{} - -GetBucketDiffReply::~GetBucketDiffReply() = default; - -void -GetBucketDiffReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "GetBucketDiffReply(" << getBucketId() << ", to time " - << _maxTimestamp << ", nodes: ["; - for (uint32_t i=0; i<_nodes.size(); ++i) { - if (i != 0) out << ", "; - out << _nodes[i]; - } - if (_diff.empty()) { - out << "], no entries"; - } else if (verbose) { - out << "],"; - for (uint32_t i=0; i<_diff.size(); ++i) { - out << "\n" << indent << " "; - _diff[i].print(out, verbose, indent + " "); - } - } else { - out << ", " << _diff.size() << " entries"; - out << ", id " << _msgId; - } - out << ")"; - if (verbose) { - out << " : "; - BucketReply::print(out, verbose, indent); - } -} - -ApplyBucketDiffCommand::Entry::Entry() - : _entry(), - _docName(), - _headerBlob(), - _bodyBlob(), - _repo() -{} - -ApplyBucketDiffCommand::Entry::Entry(const GetBucketDiffCommand::Entry& e) - : _entry(e), - _docName(), - _headerBlob(), - _bodyBlob(), - _repo() -{} - -ApplyBucketDiffCommand::Entry::~Entry() = default; -ApplyBucketDiffCommand::Entry::Entry(const Entry &) = default; -ApplyBucketDiffCommand::Entry & ApplyBucketDiffCommand::Entry::operator = (const Entry &) = default; - -bool -ApplyBucketDiffCommand::Entry::filled() const -{ - return ((_headerBlob.size() > 0 || - (_entry._headerSize == 0 && !_docName.empty())) && - (_bodyBlob.size() > 0 || - _entry._bodySize == 0)); -} - -void -ApplyBucketDiffCommand::Entry::print( - std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "ApplyEntry("; - _entry.print(out, verbose, indent + " "); - out << ",\n" << indent << " name(" << _docName - << "), headerBlob(" << _headerBlob.size() - << "), bodyBlob(" << _bodyBlob.size() << ")"; - if (_headerBlob.size() > 0) { - vespalib::nbostream buf(&_headerBlob[0], - _headerBlob.size()); - if (_repo) { - document::Document doc(*_repo, buf); - out << ",\n" << indent << " " << doc.getId().getGlobalId().toString(); - } else { - out << ",\n" << indent << " unknown global id. (repo missing)"; - } - } - out << ")"; -} - -bool -ApplyBucketDiffCommand::Entry::operator==(const Entry& e) const -{ - return (_entry == e._entry && - _headerBlob == e._headerBlob && - _bodyBlob == e._bodyBlob); -} - -ApplyBucketDiffCommand::ApplyBucketDiffCommand( - const document::Bucket &bucket, const std::vector& nodes) - : BucketInfoCommand(MessageType::APPLYBUCKETDIFF, bucket), - _nodes(nodes), - _diff() -{} - -ApplyBucketDiffCommand::~ApplyBucketDiffCommand() = default; - -void -ApplyBucketDiffCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - uint32_t totalSize = 0; - uint32_t filled = 0; - for (std::vector::const_iterator it = _diff.begin(); - it != _diff.end(); ++it) - { - totalSize += it->_headerBlob.size(); - totalSize += it->_bodyBlob.size(); - if (it->filled()) ++filled; - } - out << "ApplyBucketDiffCommand(" << getBucketId() << ", nodes: ["; - for (uint32_t i=0; i<_nodes.size(); ++i) { - if (i != 0) out << ", "; - out << _nodes[i]; - } - out << "], " << _diff.size() << " entries of " << totalSize << " bytes, " - << (100.0 * filled / _diff.size()) << " \% filled)"; - if (_diff.empty()) { - out << ", no entries"; - } else if (verbose) { - out << ","; - for (uint32_t i=0; i<_diff.size(); ++i) { - out << "\n" << indent << " "; - _diff[i].print(out, verbose, indent + " "); - } - } else { - out << ", " << _diff.size() << " entries"; - out << ", id " << _msgId; - } - out << ")"; - if (verbose) { - out << " : "; - BucketCommand::print(out, verbose, indent); - } -} - -ApplyBucketDiffReply::ApplyBucketDiffReply(const ApplyBucketDiffCommand& cmd) - : BucketInfoReply(cmd), - _nodes(cmd.getNodes()), - _diff(cmd.getDiff()) -{} - -ApplyBucketDiffReply::~ApplyBucketDiffReply() = default; - -void -ApplyBucketDiffReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - uint32_t totalSize = 0; - uint32_t filled = 0; - for (const Entry & entry : _diff) { - totalSize += entry._headerBlob.size(); - totalSize += entry._bodyBlob.size(); - if (entry.filled()) ++filled; - } - out << "ApplyBucketDiffReply(" << getBucketId() << ", nodes: ["; - for (uint32_t i=0; i<_nodes.size(); ++i) { - if (i != 0) out << ", "; - out << _nodes[i]; - } - out << "], " << _diff.size() << " entries of " << totalSize << " bytes, " - << (100.0 * filled / _diff.size()) << " \% filled)"; - if (_diff.empty()) { - out << ", no entries"; - } else if (verbose) { - out << ","; - for (uint32_t i=0; i<_diff.size(); ++i) { - out << "\n" << indent << " "; - _diff[i].print(out, verbose, indent + " "); - } - } else { - out << ", " << _diff.size() << " entries"; - out << ", id " << _msgId; - } - out << ")"; - if (verbose) { - out << " : "; - BucketInfoReply::print(out, verbose, indent); - } -} - -RequestBucketInfoCommand::RequestBucketInfoCommand( - document::BucketSpace bucketSpace, - const std::vector& buckets) - : StorageCommand(MessageType::REQUESTBUCKETINFO), - _bucketSpace(bucketSpace), - _buckets(buckets), - _state(), - _distributor(0xFFFF) -{ -} - -RequestBucketInfoCommand::RequestBucketInfoCommand( - document::BucketSpace bucketSpace, - uint16_t distributor, const lib::ClusterState& state, - vespalib::stringref distributionHash) - : StorageCommand(MessageType::REQUESTBUCKETINFO), - _bucketSpace(bucketSpace), - _buckets(), - _state(new lib::ClusterState(state)), - _distributor(distributor), - _distributionHash(distributionHash) -{ -} - -RequestBucketInfoCommand::RequestBucketInfoCommand( - document::BucketSpace bucketSpace, - uint16_t distributor, const lib::ClusterState& state) - : StorageCommand(MessageType::REQUESTBUCKETINFO), - _bucketSpace(bucketSpace), - _buckets(), - _state(new lib::ClusterState(state)), - _distributor(distributor), - _distributionHash("") -{ -} - -RequestBucketInfoCommand::~RequestBucketInfoCommand() = default; - -document::Bucket -RequestBucketInfoCommand::getBucket() const -{ - return document::Bucket(_bucketSpace, document::BucketId()); -} - -document::BucketId -RequestBucketInfoCommand::super_bucket_id() const -{ - return _buckets.empty() ? document::BucketId() : _buckets[0]; -} - -void -RequestBucketInfoCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "RequestBucketInfoCommand("; - if ( ! _buckets.empty()) { - out << _buckets.size() << " buckets"; - } - if (hasSystemState()) { - out << "distributor " << _distributor << " in "; - _state->print(out, verbose, indent + " "); - } else if (super_bucket_id().isSet()) { - out << ", super bucket " << super_bucket_id() << ". "; - } - if (verbose && !_buckets.empty()) { - out << "\n" << indent << " Specified buckets:\n" << indent << " "; - std::copy(_buckets.begin(), _buckets.end(), - std::ostream_iterator( - out, ("\n" + indent + " ").c_str())); - } - out << ")"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -std::ostream& operator<<(std::ostream& out, const RequestBucketInfoReply::Entry& e) -{ - return out << e._bucketId << " - " << e._info; -} - - -RequestBucketInfoReply::RequestBucketInfoReply(const RequestBucketInfoCommand& cmd) - : StorageReply(cmd), - _buckets(), - _full_bucket_fetch(cmd.hasSystemState()), - _super_bucket_id(cmd.super_bucket_id()) -{ } - -RequestBucketInfoReply::~RequestBucketInfoReply() = default; - -void -RequestBucketInfoReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "RequestBucketInfoReply(" << _buckets.size(); - if (_full_bucket_fetch) { - out << ", full fetch"; - } else if (super_bucket_id().isSet()) { - out << ", super bucket " << super_bucket_id(); - } - if (verbose) { - out << "\n" << indent << " "; - std::copy(_buckets.begin(), _buckets.end(), - std::ostream_iterator(out, - ("\n" + indent + " ").c_str())); - } - out << ")"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -NotifyBucketChangeCommand::NotifyBucketChangeCommand( - const document::Bucket &bucket, const BucketInfo& info) - : BucketCommand(MessageType::NOTIFYBUCKETCHANGE, bucket), - _info(info) -{ -} - -void -NotifyBucketChangeCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "NotifyBucketChangeCommand(" << getBucketId() << ", "; - out << _info; - out << ")"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -NotifyBucketChangeReply::NotifyBucketChangeReply( - const NotifyBucketChangeCommand& cmd) - : BucketReply(cmd) -{ -} - -void -NotifyBucketChangeReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "NotifyBucketChangeReply(" << getBucketId() << ")"; - if (verbose) { - out << " : "; - BucketReply::print(out, verbose, indent); - } -} - -SetBucketStateCommand::SetBucketStateCommand( - const document::Bucket &bucket, - BUCKET_STATE state) - : MaintenanceCommand(MessageType::SETBUCKETSTATE, bucket), - _state(state) -{ -} - -void -SetBucketStateCommand::print(std::ostream& out, - bool verbose, - const std::string& indent) const -{ - out << "SetBucketStateCommand(" << getBucketId() << ", "; - switch (_state) { - case INACTIVE: - out << "INACTIVE"; - break; - case ACTIVE: - out << "ACTIVE"; - break; - } - out << ")"; - if (verbose) { - out << " : "; - MaintenanceCommand::print(out, verbose, indent); - } -} - -vespalib::string -SetBucketStateCommand::getSummary() const -{ - vespalib::asciistream stream; - stream << "SetBucketStateCommand(" << getBucketId().toString() << ", " - << ((_state == ACTIVE) ? "ACTIVE" : "INACTIVE") << ")"; - return stream.str(); -} - -SetBucketStateReply::SetBucketStateReply( - const SetBucketStateCommand& cmd) - : BucketInfoReply(cmd) -{ -} - -void -SetBucketStateReply::print(std::ostream& out, - bool verbose, - const std::string& indent) const -{ - out << "SetBucketStateReply(" << getBucketId() << ")"; - if (verbose) { - out << " : "; - BucketInfoReply::print(out, verbose, indent); - } -} - -} - -template class vespalib::Array; diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storageapi/src/vespa/storageapi/message/bucket.h deleted file mode 100644 index 47785a92039..00000000000 --- a/storageapi/src/vespa/storageapi/message/bucket.h +++ /dev/null @@ -1,502 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file bucketinfo.h - * - * Bucket related commands. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace document { class DocumentTypeRepo; } - -namespace storage::lib { class ClusterState; } - -namespace storage::api { - -/** - * @class CreateBucketCommand - * @ingroup message - * - * @brief Command for creating a new bucket on a storage node. - */ -class CreateBucketCommand : public MaintenanceCommand { - bool _active; - -public: - explicit CreateBucketCommand(const document::Bucket &bucket); - void setActive(bool active) { _active = active; } - bool getActive() const { return _active; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(CreateBucketCommand, onCreateBucket) -}; - -/** - * @class CreateBucketReply - * @ingroup message - * - * @brief Reply of a create bucket command. - */ -class CreateBucketReply : public BucketInfoReply { -public: - explicit CreateBucketReply(const CreateBucketCommand& cmd); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(CreateBucketReply, onCreateBucketReply); -}; - -/** - * @class DeleteBucketCommand - * @ingroup message - * - * @brief Command for deleting a bucket from one or more storage nodes. - */ -class DeleteBucketCommand : public MaintenanceCommand { - BucketInfo _info; -public: - explicit DeleteBucketCommand(const document::Bucket &bucket); - - const BucketInfo& getBucketInfo() const { return _info; } - void setBucketInfo(const BucketInfo& info) { _info = info; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(DeleteBucketCommand, onDeleteBucket) -}; - -/** - * @class DeleteBucketReply - * @ingroup message - * - * @brief Reply of a delete bucket command. - */ -class DeleteBucketReply : public BucketInfoReply { -public: - explicit DeleteBucketReply(const DeleteBucketCommand& cmd); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(DeleteBucketReply, onDeleteBucketReply) -}; - - -/** - * @class MergeBucketCommand - * @ingroup message - * - * @brief Merge a bucket - * - * Merges given bucket copies, held on the given node list. A maximum timestamp - * should be given, such that the buckets may be used during merge. If not - * given, storage will set current time for it, but distributors should really - * set it, as they have the reference clock for a bucket. - * - * An optional "only for source" node list can be provided. In this case, the - * nodes in that list are only used for sources in the merge, and never as - * targets, even if they are missing documents from the other nodes. - * - */ -class MergeBucketCommand : public MaintenanceCommand { -public: - struct Node { - uint16_t index; - bool sourceOnly; - - Node(uint16_t index_, bool sourceOnly_ = false) noexcept - : index(index_), sourceOnly(sourceOnly_) {} - - bool operator==(const Node& n) const noexcept - { return (index == n.index && sourceOnly == n.sourceOnly); } - }; - -private: - std::vector _nodes; - Timestamp _maxTimestamp; - uint32_t _clusterStateVersion; - std::vector _chain; - bool _use_unordered_forwarding; - -public: - MergeBucketCommand(const document::Bucket &bucket, - const std::vector&, - Timestamp maxTimestamp, - uint32_t clusterStateVersion = 0, - const std::vector& chain = std::vector()); - ~MergeBucketCommand() override; - - const std::vector& getNodes() const { return _nodes; } - Timestamp getMaxTimestamp() const { return _maxTimestamp; } - const std::vector& getChain() const { return _chain; } - uint32_t getClusterStateVersion() const { return _clusterStateVersion; } - void setClusterStateVersion(uint32_t version) { _clusterStateVersion = version; } - void setChain(const std::vector& chain) { _chain = chain; } - void set_use_unordered_forwarding(bool unordered_forwarding) noexcept { - _use_unordered_forwarding = unordered_forwarding; - } - [[nodiscard]] bool use_unordered_forwarding() const noexcept { return _use_unordered_forwarding; } - [[nodiscard]] bool from_distributor() const noexcept { return _chain.empty(); } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(MergeBucketCommand, onMergeBucket) -}; - -std::ostream& -operator<<(std::ostream& out, const MergeBucketCommand::Node& n); - -/** - * @class MergeBucketReply - * @ingroup message - * - * @brief Reply of a merge bucket command. - */ -class MergeBucketReply : public BucketReply { -public: - typedef MergeBucketCommand::Node Node; - -private: - std::vector _nodes; - Timestamp _maxTimestamp; - uint32_t _clusterStateVersion; - std::vector _chain; - -public: - explicit MergeBucketReply(const MergeBucketCommand& cmd); - - const std::vector& getNodes() const { return _nodes; } - Timestamp getMaxTimestamp() const { return _maxTimestamp; } - const std::vector& getChain() const { return _chain; } - uint32_t getClusterStateVersion() const { return _clusterStateVersion; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGEREPLY(MergeBucketReply, onMergeBucketReply) -}; - -/** - * @class GetBucketDiff - * @ingroup message - * - * @brief Message sent between storage nodes as the first step of merge. - */ -class GetBucketDiffCommand : public BucketCommand { -public: - typedef MergeBucketCommand::Node Node; - - struct Entry : public document::Printable { - Timestamp _timestamp; - document::GlobalId _gid; - uint32_t _headerSize; - uint32_t _bodySize; - uint16_t _flags; - uint16_t _hasMask; - - Entry(); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - bool operator==(const Entry&) const; - bool operator<(const Entry& e) const - { return (_timestamp < e._timestamp); } - }; -private: - std::vector _nodes; - Timestamp _maxTimestamp; - std::vector _diff; - -public: - GetBucketDiffCommand(const document::Bucket &bucket, - const std::vector&, - Timestamp maxTimestamp); - ~GetBucketDiffCommand() override; - - const std::vector& getNodes() const { return _nodes; } - Timestamp getMaxTimestamp() const { return _maxTimestamp; } - const std::vector& getDiff() const { return _diff; } - std::vector& getDiff() { return _diff; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGECOMMAND(GetBucketDiffCommand, onGetBucketDiff) -}; - -/** - * @class GetBucketDiffReply - * @ingroup message - * - * @brief Reply of GetBucketDiffCommand - */ -class GetBucketDiffReply : public BucketReply { -public: - typedef MergeBucketCommand::Node Node; - typedef GetBucketDiffCommand::Entry Entry; - -private: - std::vector _nodes; - Timestamp _maxTimestamp; - std::vector _diff; - -public: - explicit GetBucketDiffReply(const GetBucketDiffCommand& cmd); - ~GetBucketDiffReply(); - - const std::vector& getNodes() const { return _nodes; } - Timestamp getMaxTimestamp() const { return _maxTimestamp; } - const std::vector& getDiff() const { return _diff; } - std::vector& getDiff() { return _diff; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGEREPLY(GetBucketDiffReply, onGetBucketDiffReply) -}; - -/** - * @class ApplyBucketDiff - * @ingroup message - * - * @brief Sends a chunk of document entries, which the bucket copies can use - * to update themselves. - */ -class ApplyBucketDiffCommand : public BucketInfoCommand { -public: - typedef MergeBucketCommand::Node Node; - struct Entry : public document::Printable { - GetBucketDiffCommand::Entry _entry; - vespalib::string _docName; - std::vector _headerBlob; - // TODO: In theory the body blob could be removed now as all is in one blob - // That will enable simplification of code in document. - std::vector _bodyBlob; - const document::DocumentTypeRepo *_repo; - - Entry(); - Entry(const GetBucketDiffCommand::Entry&); - Entry(const Entry &); - Entry & operator = (const Entry &); - Entry(Entry &&) = default; - Entry & operator = (Entry &&) = default; - ~Entry(); - - bool filled() const; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - bool operator==(const Entry&) const; - }; -private: - std::vector _nodes; - std::vector _diff; - -public: - ApplyBucketDiffCommand(const document::Bucket &bucket, - const std::vector& nodes); - ~ApplyBucketDiffCommand() override; - - const std::vector& getNodes() const { return _nodes; } - const std::vector& getDiff() const { return _diff; } - std::vector& getDiff() { return _diff; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGECOMMAND(ApplyBucketDiffCommand, onApplyBucketDiff) -}; - -/** - * @class ApplyBucketDiffReply - * @ingroup message - * - * @brief Reply of ApplyBucketDiffCommand - */ -class ApplyBucketDiffReply : public BucketInfoReply { -public: - typedef MergeBucketCommand::Node Node; - typedef ApplyBucketDiffCommand::Entry Entry; - -private: - std::vector _nodes; - std::vector _diff; - -public: - explicit ApplyBucketDiffReply(const ApplyBucketDiffCommand& cmd); - ~ApplyBucketDiffReply() override; - - const std::vector& getNodes() const { return _nodes; } - const std::vector& getDiff() const { return _diff; } - std::vector& getDiff() { return _diff; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGEREPLY(ApplyBucketDiffReply, onApplyBucketDiffReply) -}; - -/** - * @class RequestBucketInfoCommand - * @ingroup message - * - * @brief Command for getting bucket info. - * - * Used to get checksums of buckets from a storage node. - * If list of buckets for which to retrieve info is given. If it is empty, - * it means all buckets. - * A system state and a distributor index may be given. If given, only info for - * the buckets that belong to the given distributor should be returned. - */ -class RequestBucketInfoCommand : public StorageCommand { - document::BucketSpace _bucketSpace; - std::vector _buckets; - std::unique_ptr _state; - uint16_t _distributor; - vespalib::string _distributionHash; - -public: - RequestBucketInfoCommand(document::BucketSpace bucketSpace, - const std::vector& buckets); - RequestBucketInfoCommand(document::BucketSpace bucketSpace, - uint16_t distributor, - const lib::ClusterState& state, - vespalib::stringref _distributionHash); - - RequestBucketInfoCommand(document::BucketSpace bucketSpace, - uint16_t distributor, - const lib::ClusterState& state); - ~RequestBucketInfoCommand() override; - - const std::vector& getBuckets() const { return _buckets; } - - bool hasSystemState() const { return (_state.get() != 0); } - uint16_t getDistributor() const { return _distributor; } - const lib::ClusterState& getSystemState() const { return *_state; } - - const vespalib::string& getDistributionHash() const { return _distributionHash; } - document::BucketSpace getBucketSpace() const { return _bucketSpace; } - document::Bucket getBucket() const override; - document::BucketId super_bucket_id() const; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGECOMMAND(RequestBucketInfoCommand, onRequestBucketInfo) -}; - - -/** - * @class RequestBucketInfoReply - * @ingroup message - * - * @brief Answer of a bucket info command. - */ -class RequestBucketInfoReply : public StorageReply { -public: - struct Entry { - document::BucketId _bucketId; - BucketInfo _info; - - bool operator==(const Entry& e) const { return (_bucketId == e._bucketId && _info == e._info); } - bool operator!=(const Entry& e) const { return !(*this == e); } - Entry() : _bucketId(), _info() {} - Entry(const document::BucketId& id, const BucketInfo& info) - : _bucketId(id), _info(info) {} - friend std::ostream& operator<<(std::ostream& os, const Entry&); - }; - struct SupportedNodeFeatures { - bool unordered_merge_chaining = false; - }; - using EntryVector = vespalib::Array; -private: - EntryVector _buckets; - bool _full_bucket_fetch; - document::BucketId _super_bucket_id; - SupportedNodeFeatures _supported_node_features; - -public: - - explicit RequestBucketInfoReply(const RequestBucketInfoCommand& cmd); - ~RequestBucketInfoReply() override; - const EntryVector & getBucketInfo() const { return _buckets; } - EntryVector & getBucketInfo() { return _buckets; } - [[nodiscard]] bool full_bucket_fetch() const noexcept { return _full_bucket_fetch; } - // Only contains useful information if full_bucket_fetch() == true - [[nodiscard]] const SupportedNodeFeatures& supported_node_features() const noexcept { - return _supported_node_features; - } - [[nodiscard]] SupportedNodeFeatures& supported_node_features() noexcept { - return _supported_node_features; - } - const document::BucketId& super_bucket_id() const { return _super_bucket_id; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(RequestBucketInfoReply, onRequestBucketInfoReply) -}; - -/** - * @class NotifyBucketChangeCommand - * @ingroup message - * - * @brief Command for letting others know a bucket have been altered. - * - * When the persistence layer notices a bucket has been corrupted, such that - * it needs to be repaired, this message will be sent to notify others - * of change. Others being bucket database on storage node, and possibly - * distributor. - */ -class NotifyBucketChangeCommand : public BucketCommand { - BucketInfo _info; -public: - NotifyBucketChangeCommand(const document::Bucket &bucket, - const BucketInfo& bucketInfo); - const BucketInfo& getBucketInfo() const { return _info; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(NotifyBucketChangeCommand, onNotifyBucketChange) -}; - - -/** - * @class NotifyBucketChangeReply - * @ingroup message - * - * @brief Answer of notify bucket command. - * - * Noone will resend these messages, and they're not needed, but all commands - * need to have a reply. - */ -class NotifyBucketChangeReply : public BucketReply { -public: - explicit NotifyBucketChangeReply(const NotifyBucketChangeCommand& cmd); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(NotifyBucketChangeReply, onNotifyBucketChangeReply) -}; - -/** - * @class SetBucketStateCommand - * @ingroup message - * - * @brief Sent by distributor to set the ready/active state of a bucket. - */ -class SetBucketStateCommand : public MaintenanceCommand -{ -public: - enum BUCKET_STATE - { - INACTIVE, - ACTIVE - }; -private: - BUCKET_STATE _state; -public: - SetBucketStateCommand(const document::Bucket &bucket, BUCKET_STATE state); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - BUCKET_STATE getState() const { return _state; } - DECLARE_STORAGECOMMAND(SetBucketStateCommand, onSetBucketState) -private: - - vespalib::string getSummary() const override; -}; - -/** - * @class SetBucketStateReply - * @ingroup message - * - * @brief Answer to SetBucketStateCommand. - */ -class SetBucketStateReply : public BucketInfoReply -{ -public: - explicit SetBucketStateReply(const SetBucketStateCommand&); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(SetBucketStateReply, onSetBucketStateReply) -}; - -} diff --git a/storageapi/src/vespa/storageapi/message/bucketsplitting.cpp b/storageapi/src/vespa/storageapi/message/bucketsplitting.cpp deleted file mode 100644 index 784ba6edbd1..00000000000 --- a/storageapi/src/vespa/storageapi/message/bucketsplitting.cpp +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bucketsplitting.h" -#include -#include - -namespace storage::api { - -IMPLEMENT_COMMAND(SplitBucketCommand, SplitBucketReply) -IMPLEMENT_REPLY(SplitBucketReply) -IMPLEMENT_COMMAND(JoinBucketsCommand, JoinBucketsReply) -IMPLEMENT_REPLY(JoinBucketsReply) - -SplitBucketCommand::SplitBucketCommand(const document::Bucket &bucket) - : MaintenanceCommand(MessageType::SPLITBUCKET, bucket), - _minSplitBits(0), - _maxSplitBits(58), - _minByteSize(std::numeric_limits::max()), - _minDocCount(std::numeric_limits::max()) -{ - // By default, set very large sizes, to ensure we trigger 'already big - // enough' behaviour, only splitting one step by default. The distributor - // should always overwrite one of these values to get correct behaviour. -} - -void -SplitBucketCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "SplitBucketCommand(" << getBucketId(); - if (_minDocCount != std::numeric_limits::max() - || _minByteSize != std::numeric_limits::max()) - { - out << "Max doc count: " << _minDocCount - << ", Max total doc size: " << _minByteSize; - } else if (_maxSplitBits != 58) { - out << "Max split bits to use: " << _maxSplitBits; - } - out << ")"; - out << " Reasons to start: " << _reason; - if (verbose) { - out << " : "; - BucketCommand::print(out, verbose, indent); - } -} - -SplitBucketReply::SplitBucketReply(const SplitBucketCommand& cmd) - : BucketReply(cmd) -{ -} - -void -SplitBucketReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "SplitBucketReply(" << getBucketId(); - if (_result.empty()) { - out << " - No target files created."; - } else { - out << " ->"; - for (uint32_t i=0; i<_result.size(); ++i) { - out << "\n" << indent << " " << _result[i].first << ": " - << _result[i].second; - } - } - out << ")"; - if (verbose) { - out << " : "; - BucketReply::print(out, verbose, indent); - } -} - -JoinBucketsCommand::JoinBucketsCommand(const document::Bucket &target) - : MaintenanceCommand(MessageType::JOINBUCKETS, target), - _minJoinBits(0) -{ -} - -void -JoinBucketsCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "JoinBucketsCommand(" << getBucketId(); - if (_sources.empty()) { - out << " - No files to join."; - } else { - out << " <-"; - for (uint32_t i=0; i<_sources.size(); ++i) { - out << " " << _sources[i]; - } - } - out << ")"; - out << " Reasons to start: " << _reason; - if (verbose) { - out << " : "; - BucketCommand::print(out, verbose, indent); - } -} - - -JoinBucketsReply::JoinBucketsReply(const JoinBucketsCommand& cmd) - : BucketInfoReply(cmd), - _sources(cmd.getSourceBuckets()) -{ -} - -JoinBucketsReply::JoinBucketsReply(const JoinBucketsCommand& cmd, const BucketInfo& bucketInfo) - : BucketInfoReply(cmd), - _sources(cmd.getSourceBuckets()) -{ - setBucketInfo(bucketInfo); -} - -void -JoinBucketsReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "JoinBucketsReply(" << getBucketId(); - if (_sources.empty()) { - out << " - No files to join."; - } else { - out << " <-"; - for (uint32_t i=0; i<_sources.size(); ++i) { - out << " " << _sources[i]; - } - } - out << ")"; - if (verbose) { - out << " : "; - BucketReply::print(out, verbose, indent); - } -} - -} diff --git a/storageapi/src/vespa/storageapi/message/bucketsplitting.h b/storageapi/src/vespa/storageapi/message/bucketsplitting.h deleted file mode 100644 index 584cdfd5638..00000000000 --- a/storageapi/src/vespa/storageapi/message/bucketsplitting.h +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include -#include -#include -#include - -namespace storage::api { - -/** - * @class SplitBucketCommand - * @ingroup message - * - * @brief Split a bucket - * - * Splits a bucket into two parts using the next split bit that is unused. - * - * Distributors can issue splits for multiple reasons: - * - Inconsistent buckets, so we need to split buckets containing others until - * they are either split equally, or no longer contains others. - * - Buckets that are too large are split to reduce file size. - * - Buckets with too many entries are split to reduce amount of metadata. - * - * In the first case, min and max split bits can be set. This will make storage - * able to split several bits at a time, but know where to stop. - * - * In the second case, min byte size can be set, to ensure that we don't split - * bucket more one step if the copy at the time of processing is - * actually smaller. Since removes can happen in the meantime, the min byte size - * should be smaller than the limit we use for splitting. Suggesting half. - * - * Similarily we can do as the second case in the third case too, just using - * min doc count as limiter instead. - * - * If neither are specified, min/max split bits limits nothing, but the sizes - * are set to max, which ensures that only one split step is taken. - */ -class SplitBucketCommand : public MaintenanceCommand { -private: - uint8_t _minSplitBits; - uint8_t _maxSplitBits; - uint32_t _minByteSize; - uint32_t _minDocCount; - -public: - SplitBucketCommand(const document::Bucket& bucket); - - uint8_t getMinSplitBits() const { return _minSplitBits; } - uint8_t getMaxSplitBits() const { return _maxSplitBits; } - uint32_t getMinByteSize() const { return _minByteSize; } - uint32_t getMinDocCount() const { return _minDocCount; } - - void setMinSplitBits(uint8_t v) { _minSplitBits = v; } - void setMaxSplitBits(uint8_t v) { _maxSplitBits = v; } - void setMinByteSize(uint32_t v) { _minByteSize = v; } - void setMinDocCount(uint32_t v) { _minDocCount = v; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGECOMMAND(SplitBucketCommand, onSplitBucket) -}; - -/** - * @class SplitBucketReply - * @ingroup message - * - * @brief Reply of a split bucket command. - */ -class SplitBucketReply : public BucketReply { -public: - typedef std::pair Entry; - explicit SplitBucketReply(const SplitBucketCommand& cmd); - std::vector& getSplitInfo() { return _result; } - const std::vector& getSplitInfo() const { return _result; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(SplitBucketReply, onSplitBucketReply) -private: - std::vector _result; -}; - -/** - * @class JoinBucketCommand - * @ingroup message - * - * @brief Join two buckets - * - * Joins two buckets on the same node into a bucket with one fewer split bit. - */ -class JoinBucketsCommand : public MaintenanceCommand { - std::vector _sources; - uint8_t _minJoinBits; -public: - explicit JoinBucketsCommand(const document::Bucket &target); - std::vector& getSourceBuckets() { return _sources; } - const std::vector& getSourceBuckets() const { return _sources; } - void setMinJoinBits(uint8_t minJoinBits) { _minJoinBits = minJoinBits; } - uint8_t getMinJoinBits() const { return _minJoinBits; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(JoinBucketsCommand, onJoinBuckets) -}; - -/** - * @class JoinBucketsReply - * @ingroup message - * - * @brief Reply of a join bucket command. - */ -class JoinBucketsReply : public BucketInfoReply { - std::vector _sources; -public: - explicit JoinBucketsReply(const JoinBucketsCommand& cmd); - JoinBucketsReply(const JoinBucketsCommand& cmd, const BucketInfo& bucketInfo); - const std::vector& getSourceBuckets() const { return _sources; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(JoinBucketsReply, onJoinBucketsReply) -}; - -} diff --git a/storageapi/src/vespa/storageapi/message/datagram.cpp b/storageapi/src/vespa/storageapi/message/datagram.cpp deleted file mode 100644 index 546e0edecc1..00000000000 --- a/storageapi/src/vespa/storageapi/message/datagram.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "datagram.h" -#include - -using document::BucketSpace; - -namespace storage { -namespace api { - -IMPLEMENT_COMMAND(MapVisitorCommand, MapVisitorReply) -IMPLEMENT_REPLY(MapVisitorReply) -IMPLEMENT_COMMAND(EmptyBucketsCommand, EmptyBucketsReply) -IMPLEMENT_REPLY(EmptyBucketsReply) - -MapVisitorCommand::MapVisitorCommand() - : StorageCommand(MessageType::MAPVISITOR) -{ -} - -void -MapVisitorCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "MapVisitor(" << _statistics.size() << " entries"; - if (verbose) { - for (vdslib::Parameters::ParametersMap::const_iterator it - = _statistics.begin(); it != _statistics.end(); ++it) - { - out << ",\n" << indent << " " << it->first << ": " - << vespalib::stringref(it->second.c_str(), it->second.length()); - } - out << ") : "; - StorageCommand::print(out, verbose, indent); - } else { - out << ")"; - } -} - -MapVisitorReply::MapVisitorReply(const MapVisitorCommand& cmd) - : StorageReply(cmd) -{ -} - -void -MapVisitorReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "MapVisitorReply()"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -EmptyBucketsCommand::EmptyBucketsCommand( - const std::vector& buckets) - : StorageCommand(MessageType::EMPTYBUCKETS), - _buckets(buckets) -{ -} - -void -EmptyBucketsCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "EmptyBuckets("; - if (verbose) { - for (uint32_t i=0; i<_buckets.size(); ++i) { - out << "\n" << indent << " "; - out << _buckets[i]; - } - } else { - out << _buckets.size() << " buckets"; - } - out << ")"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -EmptyBucketsReply::EmptyBucketsReply(const EmptyBucketsCommand& cmd) - : StorageReply(cmd) -{ -} - -void -EmptyBucketsReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "EmptyBucketsReply()"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -} // api -} // storage diff --git a/storageapi/src/vespa/storageapi/message/datagram.h b/storageapi/src/vespa/storageapi/message/datagram.h deleted file mode 100644 index e0f5a9f7b30..00000000000 --- a/storageapi/src/vespa/storageapi/message/datagram.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "visitor.h" -#include -#include -#include - -namespace storage::api { - -/** - * @class MapStorageCommand - * @ingroup message - * - * @brief Sends a map of data to a visitor. - * - * This is a generic way to transfer data to the visitor data handler. - * It is for instance used when doing a specialized visitor to gather statistics - * on usage of document types and namespaces. - */ -class MapVisitorCommand : public StorageCommand { - vdslib::Parameters _statistics; -public: - MapVisitorCommand(); - vdslib::Parameters& getData() { return _statistics; }; - const vdslib::Parameters& getData() const { return _statistics; }; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(MapVisitorCommand, onMapVisitor) -}; - -/** - * @class MapStorageReply - * @ingroup message - * - * @brief Confirm that a given map visitor command has been received. - */ -class MapVisitorReply : public StorageReply { -public: - explicit MapVisitorReply(const MapVisitorCommand&); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(MapVisitorReply, onMapVisitorReply) -}; - -/** - * @class EmptyBucketsCommand - * @ingroup message - * - * @brief Sends a vector of bucket ids to a visitor. - * - * This message is used in synchronization to tell the synchronization client - * that a bucket contains no data at all. This is needed to let the follower be - * able to delete documents from these buckets, as they would otherwise be - * ignored by the synch agent. - */ -class EmptyBucketsCommand : public StorageCommand { - std::vector _buckets; -public: - EmptyBucketsCommand(const std::vector&); - const std::vector& getBuckets() const { return _buckets; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(EmptyBucketsCommand, onEmptyBuckets) -}; - -/** - * @class EmptyBucketsReply - * @ingroup message - * - * @brief Confirm that a given emptybucketscommad has been received. - */ -class EmptyBucketsReply : public StorageReply { -public: - explicit EmptyBucketsReply(const EmptyBucketsCommand&); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(EmptyBucketsReply, onEmptyBucketsReply) -}; - -} diff --git a/storageapi/src/vespa/storageapi/message/documentsummary.cpp b/storageapi/src/vespa/storageapi/message/documentsummary.cpp deleted file mode 100644 index 6909b4d223c..00000000000 --- a/storageapi/src/vespa/storageapi/message/documentsummary.cpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "documentsummary.h" -#include - -namespace storage { -namespace api { - -IMPLEMENT_COMMAND(DocumentSummaryCommand, DocumentSummaryReply) -IMPLEMENT_REPLY(DocumentSummaryReply) - -DocumentSummaryCommand::DocumentSummaryCommand() - : StorageCommand(MessageType::DOCUMENTSUMMARY), - DocumentSummary() -{ } - -void -DocumentSummaryCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "DocumentSummary(" << getSummaryCount() << " summaries)"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -DocumentSummaryReply::DocumentSummaryReply(const DocumentSummaryCommand& cmd) - : StorageReply(cmd) -{ } - -void -DocumentSummaryReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "DocumentSummaryReply()"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -} // api -} // storage diff --git a/storageapi/src/vespa/storageapi/message/documentsummary.h b/storageapi/src/vespa/storageapi/message/documentsummary.h deleted file mode 100644 index 5e2c1af3cfd..00000000000 --- a/storageapi/src/vespa/storageapi/message/documentsummary.h +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "visitor.h" -#include - -namespace storage { -namespace api { - -/** - * @class DocumentSummaryCommand - * @ingroup message - * - * @brief The result of a searchvisitor. - */ -class DocumentSummaryCommand : public StorageCommand, - public vdslib::DocumentSummary -{ -public: - explicit DocumentSummaryCommand(); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(DocumentSummaryCommand, onDocumentSummary) -}; - -/** - * @class DocumentSummaryReply - * @ingroup message - * - * @brief Response to a document summary command. - */ -class DocumentSummaryReply : public StorageReply { -public: - explicit DocumentSummaryReply(const DocumentSummaryCommand& command); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(DocumentSummaryReply, onDocumentSummaryReply) -}; - -} // api -} // storage diff --git a/storageapi/src/vespa/storageapi/message/internal.cpp b/storageapi/src/vespa/storageapi/message/internal.cpp deleted file mode 100644 index 5e1daaaf3be..00000000000 --- a/storageapi/src/vespa/storageapi/message/internal.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "internal.h" -#include - - -namespace storage::api { - -InternalCommand::InternalCommand(uint32_t type) - : StorageCommand(MessageType::INTERNAL), - _type(type) -{ } - -InternalCommand::~InternalCommand() = default; - -InternalReply::InternalReply(uint32_t type, const InternalCommand& cmd) - : StorageReply(cmd), - _type(type) -{ } - -InternalReply::~InternalReply() = default; - -void -InternalCommand::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "InternalCommand(" << _type << ")"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -void -InternalReply::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "InternalReply(" << _type << ")"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - - -} - diff --git a/storageapi/src/vespa/storageapi/message/internal.h b/storageapi/src/vespa/storageapi/message/internal.h deleted file mode 100644 index e0fee3e5494..00000000000 --- a/storageapi/src/vespa/storageapi/message/internal.h +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file internal.h - * - * Internal commands, used in storage. These are commands that don't need to - * be serialized as they never leave a node, implemented within storage itself - * to be able to include storage types not defined in the API. - * - * Historically these messages also existed so we could alter internal messages - * without recompiling clients, but currently no clients use storage API for - * communication anymore so this is no longer an issue. - */ -#pragma once - -#include -#include - -namespace storage::api { - -/** - * @class InternalCommand - * @ingroup message - * - * @brief Base class for commands local to a VDS node. - * - * This is the base class for internal server commands. They can not be - * serialized, so any attempt of sending such a command away from a storage - * node will fail. - */ -class InternalCommand : public StorageCommand { - uint32_t _type; - -public: - InternalCommand(uint32_t type); - ~InternalCommand() override; - - uint32_t getType() const { return _type; } - - bool callHandler(MessageHandler& h, const std::shared_ptr & m) const override { - return h.onInternal(std::static_pointer_cast(m)); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -/** - * @class InternalReply - * @ingroup message - * - * @brief Response of an internal command. - */ -class InternalReply : public StorageReply { - uint32_t _type; - -public: - InternalReply(uint32_t type, const InternalCommand& cmd); - ~InternalReply() override; - - uint32_t getType() const { return _type; } - - bool callHandler(MessageHandler& h, const std::shared_ptr & m) const override { - return h.onInternalReply(std::static_pointer_cast(m)); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -} - diff --git a/storageapi/src/vespa/storageapi/message/persistence.cpp b/storageapi/src/vespa/storageapi/message/persistence.cpp deleted file mode 100644 index 41a53449b67..00000000000 --- a/storageapi/src/vespa/storageapi/message/persistence.cpp +++ /dev/null @@ -1,345 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "persistence.h" -#include -#include -#include -#include -#include - -namespace storage::api { - -IMPLEMENT_COMMAND(PutCommand, PutReply) -IMPLEMENT_REPLY(PutReply) -IMPLEMENT_COMMAND(UpdateCommand, UpdateReply) -IMPLEMENT_REPLY(UpdateReply) -IMPLEMENT_COMMAND(GetCommand, GetReply) -IMPLEMENT_REPLY(GetReply) -IMPLEMENT_COMMAND(RemoveCommand, RemoveReply) -IMPLEMENT_REPLY(RemoveReply) -IMPLEMENT_COMMAND(RevertCommand, RevertReply) -IMPLEMENT_REPLY(RevertReply) - -TestAndSetCommand::TestAndSetCommand(const MessageType & messageType, const document::Bucket &bucket) - : BucketInfoCommand(messageType, bucket) -{} -TestAndSetCommand::~TestAndSetCommand() = default; - -PutCommand::PutCommand(const document::Bucket &bucket, const DocumentSP& doc, Timestamp time) - : TestAndSetCommand(MessageType::PUT, bucket), - _doc(doc), - _timestamp(time), - _updateTimestamp(0) -{ - if ( !_doc ) { - throw vespalib::IllegalArgumentException("Cannot put a null document", VESPA_STRLOC); - } -} - -PutCommand::~PutCommand() = default; - -const document::DocumentId& -PutCommand::getDocumentId() const { - return _doc->getId(); -} - -const document::DocumentType * -PutCommand::getDocumentType() const { - return &_doc->getType(); -} - -vespalib::string -PutCommand::getSummary() const -{ - vespalib::asciistream stream; - stream << "Put(BucketId(0x" << vespalib::hex << getBucketId().getId() << "), " - << _doc->getId().toString() - << ", timestamp " << vespalib::dec << _timestamp - << ')'; - - return stream.str(); -} - -void -PutCommand::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "Put(" << getBucketId() << ", " << _doc->getId() - << ", timestamp " << _timestamp << ", size " - << _doc->serialize().size() << ")"; - if (verbose) { - out << " {\n" << indent << " "; - _doc->print(out, verbose, indent + " "); - out << "\n" << indent << "}" << " : "; - BucketInfoCommand::print(out, verbose, indent); - } -} - -PutReply::PutReply(const PutCommand& cmd, bool wasFoundFlag) - : BucketInfoReply(cmd), - _docId(cmd.getDocumentId()), - _document(cmd.getDocument()), - _timestamp(cmd.getTimestamp()), - _updateTimestamp(cmd.getUpdateTimestamp()), - _wasFound(wasFoundFlag) -{ -} - -PutReply::~PutReply() = default; - -void -PutReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "PutReply(" << _docId << ", " << getBucketId() << ", timestamp " << _timestamp; - - if (hasBeenRemapped()) { - out << " (was remapped)"; - } - - out << ")"; - if (verbose) { - out << " : "; - BucketInfoReply::print(out, verbose, indent); - } -} - -UpdateCommand::UpdateCommand(const document::Bucket &bucket, const document::DocumentUpdate::SP& update, Timestamp time) - : TestAndSetCommand(MessageType::UPDATE, bucket), - _update(update), - _timestamp(time), - _oldTimestamp(0) -{ - if ( ! _update) { - throw vespalib::IllegalArgumentException("Cannot update a null update", VESPA_STRLOC); - } -} - -const document::DocumentType * -UpdateCommand::getDocumentType() const { - return &_update->getType(); -} - -UpdateCommand::~UpdateCommand() = default; - -const document::DocumentId& -UpdateCommand::getDocumentId() const { - return _update->getId(); -} - -vespalib::string -UpdateCommand::getSummary() const { - vespalib::asciistream stream; - stream << "Update(BucketId(0x" << vespalib::hex << getBucketId().getId() << "), " - << _update->getId().toString() << ", timestamp " << vespalib::dec << _timestamp; - if (_oldTimestamp != 0) { - stream << ", old timestamp " << _oldTimestamp; - } - stream << ')'; - - return stream.str(); -} - -void -UpdateCommand::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "Update(" << getBucketId() << ", " << _update->getId() << ", timestamp " << _timestamp; - if (_oldTimestamp != 0) { - out << ", old timestamp " << _oldTimestamp; - } - out << ")"; - if (verbose) { - out << " {\n" << indent << " "; - _update->print(out, verbose, indent + " "); - out << "\n" << indent << "} : "; - BucketInfoCommand::print(out, verbose, indent); - } -} - -UpdateReply::UpdateReply(const UpdateCommand& cmd, Timestamp oldTimestamp) - : BucketInfoReply(cmd), - _docId(cmd.getDocumentId()), - _timestamp(cmd.getTimestamp()), - _oldTimestamp(oldTimestamp), - _consistentNode((uint16_t)-1) -{ -} - -UpdateReply::~UpdateReply() = default; - -void -UpdateReply::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "UpdateReply(" - << _docId << ", " << getBucketId() << ", timestamp " - << _timestamp << ", timestamp of updated doc: " << _oldTimestamp; - - if (_consistentNode != (uint16_t)-1) { - out << " Was inconsistent (best node " << _consistentNode << ")"; - } - - out << ")"; - - if (verbose) { - out << " : "; - BucketInfoReply::print(out, verbose, indent); - } -} - -GetCommand::GetCommand(const document::Bucket &bucket, const document::DocumentId& docId, - vespalib::stringref fieldSet, Timestamp before) - : BucketInfoCommand(MessageType::GET, bucket), - _docId(docId), - _beforeTimestamp(before), - _fieldSet(fieldSet), - _internal_read_consistency(InternalReadConsistency::Strong) -{ -} - -GetCommand::~GetCommand() = default; - -vespalib::string -GetCommand::getSummary() const -{ - vespalib::asciistream stream; - stream << "Get(BucketId(" << vespalib::hex << getBucketId().getId() << "), " << _docId.toString() - << ", beforetimestamp " << vespalib::dec << _beforeTimestamp << ')'; - - return stream.str(); -} - - -void -GetCommand::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "Get(" << getBucketId() << ", " << _docId << ")"; - if (verbose) { - out << " : "; - BucketCommand::print(out, verbose, indent); - } -} - -GetReply::GetReply(const GetCommand& cmd, - const DocumentSP& doc, - Timestamp lastModified, - bool had_consistent_replicas, - bool is_tombstone) - : BucketInfoReply(cmd), - _docId(cmd.getDocumentId()), - _fieldSet(cmd.getFieldSet()), - _doc(doc), - _beforeTimestamp(cmd.getBeforeTimestamp()), - _lastModifiedTime(lastModified), - _had_consistent_replicas(had_consistent_replicas), - _is_tombstone(is_tombstone) -{ -} - -GetReply::~GetReply() = default; - -void -GetReply::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "GetReply(" << getBucketId() << ", " << _docId << ", timestamp " << _lastModifiedTime << ")"; - if (verbose) { - out << " : "; - BucketReply::print(out, verbose, indent); - } -} - -RemoveCommand::RemoveCommand(const document::Bucket &bucket, const document::DocumentId& docId, Timestamp timestamp) - : TestAndSetCommand(MessageType::REMOVE, bucket), - _docId(docId), - _timestamp(timestamp) -{ -} - -RemoveCommand::~RemoveCommand() = default; - -vespalib::string -RemoveCommand::getSummary() const { - vespalib::asciistream stream; - stream << "Remove(BucketId(0x" << vespalib::hex << getBucketId().getId() << "), " - << _docId.toString() << ", timestamp " << vespalib::dec << _timestamp << ')'; - - return stream.str(); -} -void -RemoveCommand::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "Remove(" << getBucketId() << ", " << _docId << ", timestamp " << _timestamp << ")"; - if (verbose) { - out << " : "; - BucketInfoCommand::print(out, verbose, indent); - } -} - -RemoveReply::RemoveReply(const RemoveCommand& cmd, Timestamp oldTimestamp) - : BucketInfoReply(cmd), - _docId(cmd.getDocumentId()), - _timestamp(cmd.getTimestamp()), - _oldTimestamp(oldTimestamp) -{ -} - -RemoveReply::~RemoveReply() = default; - -void -RemoveReply::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "RemoveReply(" << getBucketId() << ", " << _docId << ", timestamp " << _timestamp; - if (_oldTimestamp != 0) { - out << ", removed doc from " << _oldTimestamp; - } else { - out << ", not found"; - } - out << ")"; - if (verbose) { - out << " : "; - BucketInfoReply::print(out, verbose, indent); - } -} - -RevertCommand::RevertCommand(const document::Bucket &bucket, const std::vector& revertTokens) - : BucketInfoCommand(MessageType::REVERT, bucket), - _tokens(revertTokens) -{ -} - -RevertCommand::~RevertCommand() = default; - -void -RevertCommand::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "Revert(" << getBucketId(); - if (verbose) { - out << ","; - for (Timestamp token : _tokens) { - out << "\n" << indent << " " << token; - } - } - out << ")"; - if (verbose) { - out << " : "; - BucketInfoCommand::print(out, verbose, indent); - } -} - -RevertReply::RevertReply(const RevertCommand& cmd) - : BucketInfoReply(cmd), - _tokens(cmd.getRevertTokens()) -{ -} - -RevertReply::~RevertReply() = default; - -void -RevertReply::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "RevertReply(" << getBucketId() << ")"; - if (verbose) { - out << " : "; - BucketInfoReply::print(out, verbose, indent); - } -} - -} diff --git a/storageapi/src/vespa/storageapi/message/persistence.h b/storageapi/src/vespa/storageapi/message/persistence.h deleted file mode 100644 index 1d8cfd9d277..00000000000 --- a/storageapi/src/vespa/storageapi/message/persistence.h +++ /dev/null @@ -1,333 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file persistence.h - * - * Persistence related commands, like put, get & remove - */ -#pragma once - -#include -#include -#include -#include - -namespace document { - class DocumentUpdate; - class Document; -} -namespace storage::api { - -using documentapi::TestAndSetCondition; -using DocumentSP = std::shared_ptr; - -class TestAndSetCommand : public BucketInfoCommand { - TestAndSetCondition _condition; -public: - TestAndSetCommand(const MessageType & messageType, const document::Bucket &bucket); - ~TestAndSetCommand() override; - - void setCondition(const TestAndSetCondition & condition) { _condition = condition; } - const TestAndSetCondition & getCondition() const { return _condition; } - - /** - * Uniform interface to get document id - * Used by test and set to retrieve already existing document - */ - virtual const document::DocumentId & getDocumentId() const = 0; - virtual const document::DocumentType * getDocumentType() const { return nullptr; } -}; - -/** - * @class PutCommand - * @ingroup message - * - * @brief Command for adding a document to the storage system. - */ -class PutCommand : public TestAndSetCommand { - DocumentSP _doc; - Timestamp _timestamp; - Timestamp _updateTimestamp; - -public: - PutCommand(const document::Bucket &bucket, const DocumentSP&, Timestamp); - ~PutCommand() override; - - void setTimestamp(Timestamp ts) { _timestamp = ts; } - - /** - * If set, this PUT will only update the header of an existing document, - * rather than writing an entire new PUT. It will only perform the write if - * there exists a document already with the given timestamp. - */ - void setUpdateTimestamp(Timestamp ts) { _updateTimestamp = ts; } - Timestamp getUpdateTimestamp() const { return _updateTimestamp; } - - const DocumentSP& getDocument() const { return _doc; } - const document::DocumentId& getDocumentId() const override; - Timestamp getTimestamp() const { return _timestamp; } - const document::DocumentType * getDocumentType() const override; - - vespalib::string getSummary() const override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGECOMMAND(PutCommand, onPut); -}; - -/** - * @class PutReply - * @ingroup message - * - * @brief Reply of a put command. - */ -class PutReply : public BucketInfoReply { - document::DocumentId _docId; - DocumentSP _document; // Not serialized - Timestamp _timestamp; - Timestamp _updateTimestamp; - bool _wasFound; - -public: - explicit PutReply(const PutCommand& cmd, bool wasFound = true); - ~PutReply() override; - - const document::DocumentId& getDocumentId() const { return _docId; } - bool hasDocument() const { return _document.get(); } - const DocumentSP& getDocument() const { return _document; } - Timestamp getTimestamp() const { return _timestamp; }; - Timestamp getUpdateTimestamp() const { return _updateTimestamp; } - - bool wasFound() const { return _wasFound; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGEREPLY(PutReply, onPutReply) -}; - -/** - * @class UpdateCommand - * @ingroup message - * - * @brief Command for updating a document to the storage system. - */ -class UpdateCommand : public TestAndSetCommand { - std::shared_ptr _update; - Timestamp _timestamp; - Timestamp _oldTimestamp; - -public: - UpdateCommand(const document::Bucket &bucket, - const std::shared_ptr&, Timestamp); - ~UpdateCommand() override; - - void setTimestamp(Timestamp ts) { _timestamp = ts; } - void setOldTimestamp(Timestamp ts) { _oldTimestamp = ts; } - - const std::shared_ptr& getUpdate() const { return _update; } - const document::DocumentId& getDocumentId() const override; - Timestamp getTimestamp() const { return _timestamp; } - Timestamp getOldTimestamp() const { return _oldTimestamp; } - - const document::DocumentType * getDocumentType() const override; - vespalib::string getSummary() const override; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGECOMMAND(UpdateCommand, onUpdate); -}; - -/** - * @class UpdateReply - * @ingroup message - * - * @brief Reply of a update command. - */ -class UpdateReply : public BucketInfoReply { - document::DocumentId _docId; - Timestamp _timestamp; - Timestamp _oldTimestamp; - uint16_t _consistentNode; - -public: - UpdateReply(const UpdateCommand& cmd, Timestamp oldTimestamp = 0); - ~UpdateReply() override; - - void setOldTimestamp(Timestamp ts) { _oldTimestamp = ts; } - - const document::DocumentId& getDocumentId() const { return _docId; } - Timestamp getTimestamp() const { return _timestamp; } - Timestamp getOldTimestamp() const { return _oldTimestamp; } - - bool wasFound() const { return (_oldTimestamp != 0); } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - /** - * If this update was inconsistent (multiple different timestamps returned), - * set the "best" node. - */ - void setNodeWithNewestTimestamp(uint16_t node) { _consistentNode = node; } - - uint16_t getNodeWithNewestTimestamp() { return _consistentNode; } - - DECLARE_STORAGEREPLY(UpdateReply, onUpdateReply) -}; - - -/** - * @class GetCommand - * @ingroup message - * - * @brief Command for returning a single document. - * - * Normally, the newest version of a document is retrieved. The timestamp can - * be used to retrieve the newest copy, which is not newer than the given - * timestamp. - */ -class GetCommand : public BucketInfoCommand { - document::DocumentId _docId; - Timestamp _beforeTimestamp; - vespalib::string _fieldSet; - InternalReadConsistency _internal_read_consistency; -public: - GetCommand(const document::Bucket &bucket, const document::DocumentId&, - vespalib::stringref fieldSet, Timestamp before = MAX_TIMESTAMP); - ~GetCommand() override; - void setBeforeTimestamp(Timestamp ts) { _beforeTimestamp = ts; } - const document::DocumentId& getDocumentId() const { return _docId; } - Timestamp getBeforeTimestamp() const { return _beforeTimestamp; } - const vespalib::string& getFieldSet() const { return _fieldSet; } - void setFieldSet(vespalib::stringref fieldSet) { _fieldSet = fieldSet; } - InternalReadConsistency internal_read_consistency() const noexcept { - return _internal_read_consistency; - } - void set_internal_read_consistency(InternalReadConsistency consistency) noexcept { - _internal_read_consistency = consistency; - } - - vespalib::string getSummary() const override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - api::LockingRequirements lockingRequirements() const noexcept override { - return api::LockingRequirements::Shared; - } - - DECLARE_STORAGECOMMAND(GetCommand, onGet) -}; - -/** - * @class GetReply - * @ingroup message - * - * @brief Reply for a get command. - */ -class GetReply : public BucketInfoReply { - document::DocumentId _docId; // In case of not found, we want id still - vespalib::string _fieldSet; - DocumentSP _doc; // Null pointer if not found - Timestamp _beforeTimestamp; - Timestamp _lastModifiedTime; - bool _had_consistent_replicas; - bool _is_tombstone; -public: - explicit GetReply(const GetCommand& cmd, - const DocumentSP& doc = DocumentSP(), - Timestamp lastModified = 0, - bool had_consistent_replicas = false, - bool is_tombstone = false); - - ~GetReply() override; - - const DocumentSP& getDocument() const { return _doc; } - const document::DocumentId& getDocumentId() const { return _docId; } - const vespalib::string& getFieldSet() const { return _fieldSet; } - - Timestamp getLastModifiedTimestamp() const noexcept { return _lastModifiedTime; } - Timestamp getBeforeTimestamp() const noexcept { return _beforeTimestamp; } - - [[nodiscard]] bool had_consistent_replicas() const noexcept { return _had_consistent_replicas; } - [[nodiscard]] bool is_tombstone() const noexcept { return _is_tombstone; } - - bool wasFound() const { return (_doc.get() != nullptr); } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(GetReply, onGetReply) -}; - -/** - * @class RemoveCommand - * @ingroup message - * - * @brief Command for removing a document. - */ -class RemoveCommand : public TestAndSetCommand { - document::DocumentId _docId; - Timestamp _timestamp; - -public: - RemoveCommand(const document::Bucket &bucket, const document::DocumentId& docId, Timestamp timestamp); - ~RemoveCommand() override; - - void setTimestamp(Timestamp ts) { _timestamp = ts; } - const document::DocumentId& getDocumentId() const override { return _docId; } - Timestamp getTimestamp() const { return _timestamp; } - vespalib::string getSummary() const override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(RemoveCommand, onRemove) -}; - -/** - * @class RemoveReply - * @ingroup message - * - * @brief Reply for a remove command. - */ -class RemoveReply : public BucketInfoReply { - document::DocumentId _docId; - Timestamp _timestamp; - Timestamp _oldTimestamp; -public: - explicit RemoveReply(const RemoveCommand& cmd, Timestamp oldTimestamp = 0); - ~RemoveReply() override; - - const document::DocumentId& getDocumentId() const { return _docId; } - Timestamp getTimestamp() { return _timestamp; }; - Timestamp getOldTimestamp() const { return _oldTimestamp; } - void setOldTimestamp(Timestamp oldTimestamp) { _oldTimestamp = oldTimestamp; } - bool wasFound() const { return (_oldTimestamp != 0); } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(RemoveReply, onRemoveReply) -}; - -/** - * @class RevertCommand - * @ingroup message - * - * @brief Command for reverting a write or remove operation. - */ -class RevertCommand : public BucketInfoCommand { - std::vector _tokens; -public: - RevertCommand(const document::Bucket &bucket, - const std::vector& revertTokens); - ~RevertCommand() override; - const std::vector& getRevertTokens() const { return _tokens; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(RevertCommand, onRevert) -}; - -/** - * @class RevertReply - * @ingroup message - * - * @brief Reply for a revert command. - */ -class RevertReply : public BucketInfoReply { - std::vector _tokens; -public: - explicit RevertReply(const RevertCommand& cmd); - ~RevertReply() override; - const std::vector& getRevertTokens() const { return _tokens; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(RevertReply, onRevertReply) -}; - -} diff --git a/storageapi/src/vespa/storageapi/message/queryresult.cpp b/storageapi/src/vespa/storageapi/message/queryresult.cpp deleted file mode 100644 index b5d29cf4e02..00000000000 --- a/storageapi/src/vespa/storageapi/message/queryresult.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "queryresult.h" -#include - -namespace storage { -namespace api { - -IMPLEMENT_COMMAND(QueryResultCommand, QueryResultReply) -IMPLEMENT_REPLY(QueryResultReply) - -QueryResultCommand::QueryResultCommand() - : StorageCommand(MessageType::QUERYRESULT), - _searchResult(), - _summary() -{ } - -void -QueryResultCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "QueryResultCommand(" << _searchResult.getHitCount() << " hits)"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -QueryResultReply::QueryResultReply(const QueryResultCommand& cmd) - : StorageReply(cmd) -{ } - -void -QueryResultReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "QueryResultReply()"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -} // api -} // storage diff --git a/storageapi/src/vespa/storageapi/message/queryresult.h b/storageapi/src/vespa/storageapi/message/queryresult.h deleted file mode 100644 index c3bbbdc47ce..00000000000 --- a/storageapi/src/vespa/storageapi/message/queryresult.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "visitor.h" -#include -#include - -namespace storage { -namespace api { - -/** - * @class QueryResultCommand - * @ingroup message - * - * @brief The result of a searchvisitor. - */ -class QueryResultCommand : public StorageCommand { -public: - QueryResultCommand(); - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - const vdslib::SearchResult & getSearchResult() const { return _searchResult; } - vdslib::SearchResult & getSearchResult() { return _searchResult; } - const vdslib::DocumentSummary & getDocumentSummary() const { return _summary; } - vdslib::DocumentSummary & getDocumentSummary() { return _summary; } - - DECLARE_STORAGECOMMAND(QueryResultCommand, onQueryResult) -private: - vdslib::SearchResult _searchResult; - vdslib::DocumentSummary _summary; -}; - -/** - * @class QueryResultReply - * @ingroup message - * - * @brief Response to a search result command. - */ -class QueryResultReply : public StorageReply { -public: - explicit QueryResultReply(const QueryResultCommand& command); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(QueryResultReply, onQueryResultReply) -}; - -} // api -} // storage diff --git a/storageapi/src/vespa/storageapi/message/removelocation.cpp b/storageapi/src/vespa/storageapi/message/removelocation.cpp deleted file mode 100644 index 7b7ed894b2c..00000000000 --- a/storageapi/src/vespa/storageapi/message/removelocation.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "removelocation.h" -#include - -namespace storage::api { - -IMPLEMENT_COMMAND(RemoveLocationCommand, RemoveLocationReply) -IMPLEMENT_REPLY(RemoveLocationReply) - -RemoveLocationCommand::RemoveLocationCommand(vespalib::stringref documentSelection, - const document::Bucket &bucket) - : BucketInfoCommand(MessageType::REMOVELOCATION, bucket), - _documentSelection(documentSelection) -{} - -RemoveLocationCommand::~RemoveLocationCommand() {} - -void -RemoveLocationCommand::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - if (_documentSelection.length()) { - out << "Remove selection(" << _documentSelection << "): "; - } - BucketInfoCommand::print(out, verbose, indent); -} - -RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed) - : BucketInfoReply(cmd), - _documents_removed(docs_removed) -{ -} - -} diff --git a/storageapi/src/vespa/storageapi/message/removelocation.h b/storageapi/src/vespa/storageapi/message/removelocation.h deleted file mode 100644 index 276b090cc57..00000000000 --- a/storageapi/src/vespa/storageapi/message/removelocation.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include -#include -#include - -namespace storage::api { - -class RemoveLocationCommand : public BucketInfoCommand -{ -public: - RemoveLocationCommand(vespalib::stringref documentSelection, const document::Bucket &bucket); - ~RemoveLocationCommand() override; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - const vespalib::string& getDocumentSelection() const { return _documentSelection; } - DECLARE_STORAGECOMMAND(RemoveLocationCommand, onRemoveLocation); -private: - vespalib::string _documentSelection; -}; - -class RemoveLocationReply : public BucketInfoReply -{ - uint32_t _documents_removed; -public: - explicit RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed = 0); - void set_documents_removed(uint32_t docs_removed) noexcept { - _documents_removed = docs_removed; - } - uint32_t documents_removed() const noexcept { return _documents_removed; } - DECLARE_STORAGEREPLY(RemoveLocationReply, onRemoveLocationReply) -}; - -} diff --git a/storageapi/src/vespa/storageapi/message/searchresult.cpp b/storageapi/src/vespa/storageapi/message/searchresult.cpp deleted file mode 100644 index b2cf04b0410..00000000000 --- a/storageapi/src/vespa/storageapi/message/searchresult.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "searchresult.h" -#include - -using vdslib::SearchResult; - -namespace storage { -namespace api { - -IMPLEMENT_COMMAND(SearchResultCommand, SearchResultReply) -IMPLEMENT_REPLY(SearchResultReply) - -SearchResultCommand::SearchResultCommand() - : StorageCommand(MessageType::SEARCHRESULT), - SearchResult() -{ -} - -void -SearchResultCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "SearchResultCommand(" << getHitCount() << " hits)"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -SearchResultReply::SearchResultReply(const SearchResultCommand& cmd) - : StorageReply(cmd) -{ } - -void -SearchResultReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "SearchResultReply()"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -} // api -} // storage diff --git a/storageapi/src/vespa/storageapi/message/searchresult.h b/storageapi/src/vespa/storageapi/message/searchresult.h deleted file mode 100644 index 4795876102d..00000000000 --- a/storageapi/src/vespa/storageapi/message/searchresult.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "visitor.h" -#include - -namespace storage { -namespace api { - -/** - * @class SearchResultCommand - * @ingroup message - * - * @brief The result of a searchvisitor. - */ -class SearchResultCommand : public StorageCommand, public vdslib::SearchResult { -public: - SearchResultCommand(); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(SearchResultCommand, onSearchResult) -}; - -/** - * @class SearchResultReply - * @ingroup message - * - * @brief Response to a search result command. - */ -class SearchResultReply : public StorageReply { -public: - explicit SearchResultReply(const SearchResultCommand& command); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(SearchResultReply, onSearchResultReply) -}; - -} // api -} // storage diff --git a/storageapi/src/vespa/storageapi/message/stat.cpp b/storageapi/src/vespa/storageapi/message/stat.cpp deleted file mode 100644 index 3b97f4f5541..00000000000 --- a/storageapi/src/vespa/storageapi/message/stat.cpp +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "stat.h" -#include - -namespace storage::api { - -IMPLEMENT_COMMAND(StatBucketCommand, StatBucketReply) -IMPLEMENT_REPLY(StatBucketReply) -IMPLEMENT_COMMAND(GetBucketListCommand, GetBucketListReply) -IMPLEMENT_REPLY(GetBucketListReply) - -StatBucketCommand::StatBucketCommand(const document::Bucket& bucket, - vespalib::stringref documentSelection) - : BucketCommand(MessageType::STATBUCKET, bucket), - _docSelection(documentSelection) -{ -} - -StatBucketCommand::~StatBucketCommand() = default; - -void -StatBucketCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "StatBucketCommand(" << getBucketId() - << ", selection: " << _docSelection << ")"; - if (verbose) { - out << " : "; - BucketCommand::print(out, verbose, indent); - } -} - -StatBucketReply::StatBucketReply(const StatBucketCommand& cmd, - vespalib::stringref results) - : BucketReply(cmd), - _results(results) -{ -} - -void -StatBucketReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "StatBucketReply(" << getBucketId(); - if (verbose) { - out << ", result: " << _results << ") : "; - BucketReply::print(out, verbose, indent); - } else { - vespalib::string::size_type pos = _results.find('\n'); - vespalib::string overview; - if (pos != vespalib::string::npos) { - overview = _results.substr(0, pos) + " ..."; - } else { - overview = _results; - } - out << ", result: " << overview << ")"; - } -} - -GetBucketListCommand::GetBucketListCommand(const document::Bucket &bucket) - : BucketCommand(MessageType::GETBUCKETLIST, bucket) -{ -} - -void -GetBucketListCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "GetBucketList(" << getBucketId() << ")"; - if (verbose) { - out << " : "; - BucketCommand::print(out, verbose, indent); - } -} - -GetBucketListReply::GetBucketListReply(const GetBucketListCommand& cmd) - : BucketReply(cmd), - _buckets() -{} - -GetBucketListReply::~GetBucketListReply() {} - -void -GetBucketListReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "GetBucketListReply(" << getBucketId() << ", Info on " - << _buckets.size() << " buckets)"; - if (verbose) { - out << " : "; - BucketReply::print(out, verbose, indent); - } -} - -std::ostream& -operator<<(std::ostream& out, const GetBucketListReply::BucketInfo& instance) -{ - out << "BucketInfo(" << instance._bucket << ": " - << instance._bucketInformation << ")"; - return out; -} - -} diff --git a/storageapi/src/vespa/storageapi/message/stat.h b/storageapi/src/vespa/storageapi/message/stat.h deleted file mode 100644 index 0797ae43799..00000000000 --- a/storageapi/src/vespa/storageapi/message/stat.h +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include -#include - -namespace storage::api { - -/** - * \class StatBucketCommand - * \ingroup stat - * - * \brief Command used to get information about a given bucket.. - * - * Command used by stat to get detailed information about a bucket. - */ -class StatBucketCommand : public BucketCommand { -private: - vespalib::string _docSelection; -public: - StatBucketCommand(const document::Bucket &bucket, - vespalib::stringref documentSelection); - ~StatBucketCommand() override; - - const vespalib::string& getDocumentSelection() const { return _docSelection; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(StatBucketCommand, onStatBucket); -}; - -class StatBucketReply : public BucketReply { - vespalib::string _results; -public: - explicit StatBucketReply(const StatBucketCommand&, vespalib::stringref results = ""); - const vespalib::string& getResults() const noexcept { return _results; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(StatBucketReply, onStatBucketReply) -}; - -/** - * \class GetBucketListCommand - * \ingroup stat - * - * \brief Command used to find actual buckets related to a given one. - * - * Command used by stat to query distributor to find actual buckets contained - * by the given bucket, or buckets that contain the given bucket. (getAll() call - * on bucket database) - */ -class GetBucketListCommand : public BucketCommand { -public: - explicit GetBucketListCommand(const document::Bucket &bucket); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(GetBucketListCommand, onGetBucketList); -}; - -class GetBucketListReply : public BucketReply { -public: - struct BucketInfo { - document::BucketId _bucket; - vespalib::string _bucketInformation; - - BucketInfo(const document::BucketId& id, - vespalib::stringref bucketInformation) - : _bucket(id), - _bucketInformation(bucketInformation) - {} - - bool operator==(const BucketInfo& other) const { - return (_bucket == other._bucket - && _bucketInformation == other._bucketInformation); - } - }; - -private: - std::vector _buckets; - -public: - explicit GetBucketListReply(const GetBucketListCommand&); - ~GetBucketListReply() override; - std::vector& getBuckets() { return _buckets; } - const std::vector& getBuckets() const { return _buckets; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGEREPLY(GetBucketListReply, onGetBucketListReply) - -}; - -std::ostream& operator<<(std::ostream& out, const GetBucketListReply::BucketInfo& instance); - -} diff --git a/storageapi/src/vespa/storageapi/message/state.cpp b/storageapi/src/vespa/storageapi/message/state.cpp deleted file mode 100644 index 23bd766ac2a..00000000000 --- a/storageapi/src/vespa/storageapi/message/state.cpp +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "state.h" -#include -#include -#include - -namespace storage { -namespace api { - -IMPLEMENT_COMMAND(GetNodeStateCommand, GetNodeStateReply) -IMPLEMENT_REPLY(GetNodeStateReply) -IMPLEMENT_COMMAND(SetSystemStateCommand, SetSystemStateReply) -IMPLEMENT_REPLY(SetSystemStateReply) -IMPLEMENT_COMMAND(ActivateClusterStateVersionCommand, ActivateClusterStateVersionReply) -IMPLEMENT_REPLY(ActivateClusterStateVersionReply) - -GetNodeStateCommand::GetNodeStateCommand(lib::NodeState::UP expectedState) - : StorageCommand(MessageType::GETNODESTATE), - _expectedState(std::move(expectedState)) -{ -} - -void -GetNodeStateCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "GetNodeStateCommand("; - if (_expectedState.get() != 0) { - out << "Expected state: " << *_expectedState; - } - out << ")"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -GetNodeStateReply::GetNodeStateReply(const GetNodeStateCommand& cmd) - : StorageReply(cmd), - _state() -{ -} - -GetNodeStateReply::GetNodeStateReply(const GetNodeStateCommand& cmd, - const lib::NodeState& state) - : StorageReply(cmd), - _state(new lib::NodeState(state)) -{ -} - -void -GetNodeStateReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "GetNodeStateReply("; - if (_state.get()) { - out << "State: " << *_state; - } - out << ")"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -SetSystemStateCommand::SetSystemStateCommand(const lib::ClusterStateBundle& state) - : StorageCommand(MessageType::SETSYSTEMSTATE), - _state(state) -{ -} - -SetSystemStateCommand::SetSystemStateCommand(const lib::ClusterState& state) - : StorageCommand(MessageType::SETSYSTEMSTATE), - _state(state) -{ -} - -void -SetSystemStateCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "SetSystemStateCommand(" << *_state.getBaselineClusterState() << ")"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -SetSystemStateReply::SetSystemStateReply(const SetSystemStateCommand& cmd) - : StorageReply(cmd), - _state(cmd.getClusterStateBundle()) -{ -} - -void -SetSystemStateReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "SetSystemStateReply()"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -ActivateClusterStateVersionCommand::ActivateClusterStateVersionCommand(uint32_t version) - : StorageCommand(MessageType::ACTIVATE_CLUSTER_STATE_VERSION), - _version(version) -{ -} - -void ActivateClusterStateVersionCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "ActivateClusterStateVersionCommand(" << _version << ")"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -ActivateClusterStateVersionReply::ActivateClusterStateVersionReply(const ActivateClusterStateVersionCommand& cmd) - : StorageReply(cmd), - _activateVersion(cmd.version()), - _actualVersion(0) // Must be set explicitly -{ -} - -void ActivateClusterStateVersionReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "ActivateClusterStateVersionReply(activate " << _activateVersion - << ", actual " << _actualVersion << ")"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -} // api -} // storage diff --git a/storageapi/src/vespa/storageapi/message/state.h b/storageapi/src/vespa/storageapi/message/state.h deleted file mode 100644 index aa562c77ef9..00000000000 --- a/storageapi/src/vespa/storageapi/message/state.h +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include -#include -#include -#include - -namespace storage::api { - -/** - * @class GetNodeStateCommand - * @ingroup message - * - * @brief Command for setting node state. No payload - */ -class GetNodeStateCommand : public StorageCommand { - lib::NodeState::UP _expectedState; - -public: - explicit GetNodeStateCommand(lib::NodeState::UP expectedState); - - const lib::NodeState* getExpectedState() const { return _expectedState.get(); } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGECOMMAND(GetNodeStateCommand, onGetNodeState) -}; - -/** - * @class GetNodeStateReply - * @ingroup message - * - * @brief Reply to GetNodeStateCommand - */ -class GetNodeStateReply : public StorageReply { - lib::NodeState::UP _state; - std::string _nodeInfo; - -public: - GetNodeStateReply(const GetNodeStateCommand&); // Only used on makeReply() - GetNodeStateReply(const GetNodeStateCommand&, const lib::NodeState&); - - bool hasNodeState() const { return (_state.get() != 0); } - const lib::NodeState& getNodeState() const { return *_state; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - void setNodeInfo(const std::string& info) { _nodeInfo = info; } - const std::string& getNodeInfo() const { return _nodeInfo; } - - DECLARE_STORAGEREPLY(GetNodeStateReply, onGetNodeStateReply) -}; - -/** - * @class SetSystemStateCommand - * @ingroup message - * - * @brief Command for telling a node about the system state - state of each node - * in the system and state of the system (all ok, no merging, block - * put/get/remove etx) - */ -class SetSystemStateCommand : public StorageCommand { - lib::ClusterStateBundle _state; - -public: - explicit SetSystemStateCommand(const lib::ClusterStateBundle &state); - explicit SetSystemStateCommand(const lib::ClusterState &state); - const lib::ClusterState& getSystemState() const { return *_state.getBaselineClusterState(); } - const lib::ClusterStateBundle& getClusterStateBundle() const { return _state; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGECOMMAND(SetSystemStateCommand, onSetSystemState) -}; - -/** - * @class SetSystemStateReply - * @ingroup message - * - * @brief Reply received after a SetSystemStateCommand. - */ -class SetSystemStateReply : public StorageReply { - lib::ClusterStateBundle _state; - -public: - explicit SetSystemStateReply(const SetSystemStateCommand& cmd); - - // Not serialized. Available locally - const lib::ClusterState& getSystemState() const { return *_state.getBaselineClusterState(); } - const lib::ClusterStateBundle& getClusterStateBundle() const { return _state; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGEREPLY(SetSystemStateReply, onSetSystemStateReply) -}; - -class ActivateClusterStateVersionCommand : public StorageCommand { - uint32_t _version; -public: - explicit ActivateClusterStateVersionCommand(uint32_t version); - uint32_t version() const noexcept { return _version; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGECOMMAND(ActivateClusterStateVersionCommand, onActivateClusterStateVersion); -}; - -class ActivateClusterStateVersionReply : public StorageReply { - uint32_t _activateVersion; - uint32_t _actualVersion; -public: - explicit ActivateClusterStateVersionReply(const ActivateClusterStateVersionCommand&); - uint32_t activateVersion() const noexcept { return _activateVersion; } - void setActualVersion(uint32_t version) noexcept { _actualVersion = version; } - uint32_t actualVersion() const noexcept { return _actualVersion; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGEREPLY(ActivateClusterStateVersionReply, onActivateClusterStateVersionReply); -}; - -} diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp deleted file mode 100644 index fb3274273ac..00000000000 --- a/storageapi/src/vespa/storageapi/message/visitor.cpp +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "visitor.h" -#include -#include -#include -#include - -namespace storage::api { - -IMPLEMENT_COMMAND(CreateVisitorCommand, CreateVisitorReply) -IMPLEMENT_REPLY(CreateVisitorReply) -IMPLEMENT_COMMAND(DestroyVisitorCommand, DestroyVisitorReply) -IMPLEMENT_REPLY(DestroyVisitorReply) -IMPLEMENT_COMMAND(VisitorInfoCommand, VisitorInfoReply) -IMPLEMENT_REPLY(VisitorInfoReply) - -CreateVisitorCommand::CreateVisitorCommand(document::BucketSpace bucketSpace, - vespalib::stringref libraryName, - vespalib::stringref instanceId, - vespalib::stringref docSelection) - : StorageCommand(MessageType::VISITOR_CREATE), - _bucketSpace(bucketSpace), - _libName(libraryName), - _params(), - _controlDestination(), - _dataDestination(), - _docSelection(docSelection), - _buckets(), - _fromTime(0), - _toTime(api::MAX_TIMESTAMP), - _visitorCmdId(getMsgId()), - _instanceId(instanceId), - _visitorId(0), - _visitRemoves(false), - _fieldSet(document::AllFields::NAME), - _visitInconsistentBuckets(false), - _queueTimeout(2000ms), - _maxPendingReplyCount(2), - _version(50), - _maxBucketsPerVisitor(1) -{ -} - -CreateVisitorCommand::CreateVisitorCommand(const CreateVisitorCommand& o) - : StorageCommand(o), - _bucketSpace(o._bucketSpace), - _libName(o._libName), - _params(o._params), - _controlDestination(o._controlDestination), - _dataDestination(o._dataDestination), - _docSelection(o._docSelection), - _buckets(o._buckets), - _fromTime(o._fromTime), - _toTime(o._toTime), - _visitorCmdId(getMsgId()), - _instanceId(o._instanceId), - _visitorId(o._visitorId), - _visitRemoves(o._visitRemoves), - _fieldSet(o._fieldSet), - _visitInconsistentBuckets(o._visitInconsistentBuckets), - _queueTimeout(o._queueTimeout), - _maxPendingReplyCount(o._maxPendingReplyCount), - _version(o._version), - _maxBucketsPerVisitor(o._maxBucketsPerVisitor) -{ -} - -CreateVisitorCommand::~CreateVisitorCommand() = default; - -document::Bucket -CreateVisitorCommand::getBucket() const -{ - return document::Bucket(_bucketSpace, document::BucketId()); -} - -document::BucketId -CreateVisitorCommand::super_bucket_id() const -{ - if (_buckets.empty()) { - // TODO STRIPE: Is this actually an error situation? Should be fixed elsewhere. - return document::BucketId(); - } - return _buckets[0]; -} - -void -CreateVisitorCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "CreateVisitorCommand(" << _libName << ", " << _docSelection; - if (verbose) { - out << ") {"; - out << "\n" << indent << " Library name: '" << _libName << "'"; - out << "\n" << indent << " Instance Id: '" << _instanceId << "'"; - out << "\n" << indent << " Control Destination: '" << _controlDestination << "'"; - out << "\n" << indent << " Data Destination: '" << _dataDestination << "'"; - out << "\n" << indent << " Doc Selection: '" << _docSelection << "'"; - out << "\n" << indent << " Max pending: '" << _maxPendingReplyCount << "'"; - out << "\n" << indent << " Timeout: " << vespalib::count_ms(getTimeout()) << " ms"; - out << "\n" << indent << " Queue timeout: " << vespalib::count_ms(_queueTimeout) << " ms"; - out << "\n" << indent << " VisitorDispatcher version: '" << _version << "'"; - if (visitRemoves()) { - out << "\n" << indent << " Visiting remove entries too"; - } - - out << "\n" << indent << " Returning fields: " << _fieldSet; - - if (visitInconsistentBuckets()) { - out << "\n" << indent << " Visiting inconsistent buckets"; - } - out << "\n" << indent << " From " << _fromTime << " to " << _toTime; - for (std::vector::const_iterator it - = _buckets.begin(); it != _buckets.end(); ++it) - { - out << "\n" << indent << " " << (*it); - } - out << "\n" << indent << " "; - _params.print(out, verbose, indent + " "); - out << "\n" << indent << " Max buckets: '" << _maxBucketsPerVisitor << "'"; - out << "\n" << indent << "} : "; - StorageCommand::print(out, verbose, indent); - } else if (_buckets.size() == 2) { - out << ", top " << _buckets[0] << ", progress " << _buckets[1] << ")"; - } else { - out << ", " << _buckets.size() << " buckets)"; - } - -} - -CreateVisitorReply::CreateVisitorReply(const CreateVisitorCommand& cmd) - : StorageReply(cmd), - _super_bucket_id(cmd.super_bucket_id()), - _lastBucket(document::BucketId(INT_MAX)) -{ -} - -void -CreateVisitorReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "CreateVisitorReply(last=" << _lastBucket << ")"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -DestroyVisitorCommand::DestroyVisitorCommand(vespalib::stringref instanceId) - : StorageCommand(MessageType::VISITOR_DESTROY), - _instanceId(instanceId) -{ -} - -void -DestroyVisitorCommand::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "DestroyVisitorCommand(" << _instanceId << ")"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -DestroyVisitorReply::DestroyVisitorReply(const DestroyVisitorCommand& cmd) - : StorageReply(cmd) -{ -} - -void -DestroyVisitorReply::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "DestroyVisitorReply()"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -VisitorInfoCommand::VisitorInfoCommand() - : StorageCommand(MessageType::VISITOR_INFO), - _completed(false), - _bucketsCompleted(), - _error(ReturnCode::OK) -{ -} - -VisitorInfoCommand::~VisitorInfoCommand() = default; - -void -VisitorInfoCommand::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "VisitorInfoCommand("; - if (_completed) { out << "completed"; } - if (_error.failed()) { - out << _error; - } - if (verbose) { - out << ") : "; - StorageCommand::print(out, verbose, indent); - } else { - if (!_bucketsCompleted.empty()) { - out << _bucketsCompleted.size() << " buckets completed"; - } - out << ")"; - } -} - -VisitorInfoReply::VisitorInfoReply(const VisitorInfoCommand& cmd) - : StorageReply(cmd), - _completed(cmd.visitorCompleted()) -{ -} - -void -VisitorInfoReply::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "VisitorInfoReply("; - if (_completed) { out << "completed"; } - if (verbose) { - out << ") : "; - StorageReply::print(out, verbose, indent); - } else { - out << ")"; - } -} - -std::ostream& -operator<<(std::ostream& out, const VisitorInfoCommand::BucketTimestampPair& pair) { - return out << pair.bucketId << " - " << pair.timestamp; -} - -} diff --git a/storageapi/src/vespa/storageapi/message/visitor.h b/storageapi/src/vespa/storageapi/message/visitor.h deleted file mode 100644 index e6835405768..00000000000 --- a/storageapi/src/vespa/storageapi/message/visitor.h +++ /dev/null @@ -1,244 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file storageapi/message/visitor.h - * - * Messages related to visitors, used by the visitor manager. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace storage::api { - -/** - * @class CreateVisitorCommand - * @ingroup message - * - * @brief Command for creating a visitor. - */ -class CreateVisitorCommand : public StorageCommand { -private: - document::BucketSpace _bucketSpace; - vespalib::string _libName; // Name of visitor library to use, ie. DumpVisitor.so - vdslib::Parameters _params; - - vespalib::string _controlDestination; - vespalib::string _dataDestination; - - vespalib::string _docSelection; - std::vector _buckets; - Timestamp _fromTime; - Timestamp _toTime; - - uint32_t _visitorCmdId; - vespalib::string _instanceId; - VisitorId _visitorId; // Set on storage node - - bool _visitRemoves; - vespalib::string _fieldSet; - bool _visitInconsistentBuckets; - - duration _queueTimeout; - uint32_t _maxPendingReplyCount; - uint32_t _version; - - uint32_t _maxBucketsPerVisitor; - -public: - CreateVisitorCommand(document::BucketSpace bucketSpace, - vespalib::stringref libraryName, - vespalib::stringref instanceId, - vespalib::stringref docSelection); - - /** Create another command with similar visitor settings. */ - CreateVisitorCommand(const CreateVisitorCommand& template_); - ~CreateVisitorCommand(); - - void setVisitorCmdId(uint32_t id) { _visitorCmdId = id; } - void setControlDestination(vespalib::stringref d) { _controlDestination = d; } - void setDataDestination(vespalib::stringref d) { _dataDestination = d; } - void setParameters(const vdslib::Parameters& params) { _params = params; } - void setMaximumPendingReplyCount(uint32_t count) { _maxPendingReplyCount = count; } - void setFieldSet(vespalib::stringref fieldSet) { _fieldSet = fieldSet; } - void setVisitRemoves(bool value = true) { _visitRemoves = value; } - void setVisitInconsistentBuckets(bool visitInconsistent = true) { _visitInconsistentBuckets = visitInconsistent; } - void addBucketToBeVisited(const document::BucketId& id) { _buckets.push_back(id); } - void setVisitorId(const VisitorId id) { _visitorId = id; } - void setInstanceId(vespalib::stringref id) { _instanceId = id; } - void setQueueTimeout(duration milliSecs) { _queueTimeout = milliSecs; } - void setFromTime(Timestamp ts) { _fromTime = ts; } - void setToTime(Timestamp ts) { _toTime = ts; } - - VisitorId getVisitorId() const { return _visitorId; } - uint32_t getVisitorCmdId() const { return _visitorCmdId; } - document::BucketSpace getBucketSpace() const { return _bucketSpace; } - document::Bucket getBucket() const override; - document::BucketId super_bucket_id() const; - const vespalib::string & getLibraryName() const { return _libName; } - const vespalib::string & getInstanceId() const { return _instanceId; } - const vespalib::string & getControlDestination() const { return _controlDestination; } - const vespalib::string & getDataDestination() const { return _dataDestination; } - const vespalib::string & getDocumentSelection() const { return _docSelection; } - const vdslib::Parameters& getParameters() const { return _params; } - vdslib::Parameters& getParameters() { return _params; } - uint32_t getMaximumPendingReplyCount() const { return _maxPendingReplyCount; } - const std::vector& getBuckets() const { return _buckets; } - Timestamp getFromTime() const { return _fromTime; } - Timestamp getToTime() const { return _toTime; } - std::vector& getBuckets() { return _buckets; } - bool visitRemoves() const { return _visitRemoves; } - const vespalib::string& getFieldSet() const { return _fieldSet; } - bool visitInconsistentBuckets() const { return _visitInconsistentBuckets; } - duration getQueueTimeout() const { return _queueTimeout; } - - void setVisitorDispatcherVersion(uint32_t version) { _version = version; } - uint32_t getVisitorDispatcherVersion() const { return _version; } - - void setMaxBucketsPerVisitor(uint32_t max) { _maxBucketsPerVisitor = max; } - uint32_t getMaxBucketsPerVisitor() const { return _maxBucketsPerVisitor; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - DECLARE_STORAGECOMMAND(CreateVisitorCommand, onCreateVisitor) -}; - -/** - * @class CreateVisitorReply - * @ingroup message - * - * @brief Response to a create visitor command. - */ -class CreateVisitorReply : public StorageReply { -private: - document::BucketId _super_bucket_id; - document::BucketId _lastBucket; - vdslib::VisitorStatistics _visitorStatistics; - -public: - explicit CreateVisitorReply(const CreateVisitorCommand& cmd); - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - void setLastBucket(const document::BucketId& lastBucket) { _lastBucket = lastBucket; } - - const document::BucketId& super_bucket_id() const { return _super_bucket_id; } - const document::BucketId& getLastBucket() const { return _lastBucket; } - - void setVisitorStatistics(const vdslib::VisitorStatistics& stats) { _visitorStatistics = stats; } - - const vdslib::VisitorStatistics& getVisitorStatistics() const { return _visitorStatistics; } - - DECLARE_STORAGEREPLY(CreateVisitorReply, onCreateVisitorReply) -}; - -/** - * @class DestroyVisitorCommand - * @ingroup message - * - * @brief Command for removing a visitor. - */ -class DestroyVisitorCommand : public StorageCommand { -private: - vespalib::string _instanceId; - -public: - explicit DestroyVisitorCommand(vespalib::stringref instanceId); - - const vespalib::string & getInstanceId() const { return _instanceId; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGECOMMAND(DestroyVisitorCommand, onDestroyVisitor) -}; - -/** - * @class DestroyVisitorReply - * @ingroup message - * - * @brief Response to a destroy visitor command. - */ -class DestroyVisitorReply : public StorageReply { -public: - explicit DestroyVisitorReply(const DestroyVisitorCommand& cmd); - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGEREPLY(DestroyVisitorReply, onDestroyVisitorReply) -}; - -/** - * @class VisitorInfoCommand - * @ingroup message - * - * @brief Sends status information of an ongoing visitor. - * - * Includes three different kinds of data. - * - Notification when visiting is complete. - * - Notification when individual buckets have been completely visited. - * (Including the timestamp of the newest document visited) - * - Notification that some error condition arose during visiting. - */ -class VisitorInfoCommand : public StorageCommand { -public: - struct BucketTimestampPair { - document::BucketId bucketId; - Timestamp timestamp; - - BucketTimestampPair() noexcept : bucketId(), timestamp(0) {} - BucketTimestampPair(const document::BucketId& bucket, const Timestamp& ts) noexcept - : bucketId(bucket), timestamp(ts) - {} - - bool operator==(const BucketTimestampPair& other) const noexcept { - return (bucketId == other.bucketId && timestamp && other.timestamp); - } - }; - -private: - bool _completed; - std::vector _bucketsCompleted; - ReturnCode _error; - -public: - VisitorInfoCommand(); - ~VisitorInfoCommand() override; - - void setErrorCode(ReturnCode && code) { _error = std::move(code); } - void setCompleted() { _completed = true; } - void setBucketCompleted(const document::BucketId& id, Timestamp lastVisited) { - _bucketsCompleted.push_back(BucketTimestampPair(id, lastVisited)); - } - void setBucketsCompleted(const std::vector& bc) { - _bucketsCompleted = bc; - } - - const ReturnCode& getErrorCode() const { return _error; } - const std::vector& getCompletedBucketsList() const { - return _bucketsCompleted; - } - bool visitorCompleted() const { return _completed; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGECOMMAND(VisitorInfoCommand, onVisitorInfo) -}; - -std::ostream& operator<<(std::ostream& out, const VisitorInfoCommand::BucketTimestampPair& pair); - -class VisitorInfoReply : public StorageReply { - bool _completed; - -public: - VisitorInfoReply(const VisitorInfoCommand& cmd); - bool visitorCompleted() const { return _completed; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGEREPLY(VisitorInfoReply, onVisitorInfoReply) -}; - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/.gitignore b/storageapi/src/vespa/storageapi/messageapi/.gitignore deleted file mode 100644 index 6e555686642..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*.lo -.depend -.depend.NEW -.deps -.libs -Makefile diff --git a/storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt b/storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt deleted file mode 100644 index 6454bb1de7d..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageapi_messageapi OBJECT - SOURCES - bucketcommand.cpp - bucketreply.cpp - bucketinfocommand.cpp - bucketinforeply.cpp - maintenancecommand.cpp - returncode.cpp - storagemessage.cpp - storagecommand.cpp - storagereply.cpp - DEPENDS -) diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp deleted file mode 100644 index c75267f560d..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bucketcommand.h" -#include - -using document::Bucket; -using document::BucketId; -using document::BucketSpace; - -namespace storage { -namespace api { - -BucketCommand::BucketCommand(const MessageType& type, const Bucket &bucket) - : StorageCommand(type), - _bucket(bucket), - _originalBucket() -{ -} - -void -BucketCommand::remapBucketId(const BucketId& bucket) -{ - if (_originalBucket.getRawId() == 0) { - _originalBucket = _bucket.getBucketId(); - } - Bucket newBucket(_bucket.getBucketSpace(), bucket); - _bucket = newBucket; -} - -void -BucketCommand::print(std::ostream& out, - bool verbose, const std::string& indent) const -{ - out << "BucketCommand(" << _bucket.getBucketId(); - if (hasBeenRemapped()) { - out << " <- " << _originalBucket; - } - out << ")"; - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -} // api -} // storage diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketcommand.h b/storageapi/src/vespa/storageapi/messageapi/bucketcommand.h deleted file mode 100644 index 605653681b5..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/bucketcommand.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::api::BucketCommand - * @ingroup messageapi - * - * @brief Superclass for storage commands that operate towards a single bucket. - */ - -#pragma once - -#include "storagecommand.h" -#include - -namespace storage::api { - -class BucketCommand : public StorageCommand { - document::Bucket _bucket; - document::BucketId _originalBucket; - -protected: - BucketCommand(const MessageType& type, const document::Bucket &bucket); - -public: - DECLARE_POINTER_TYPEDEFS(BucketCommand); - - void remapBucketId(const document::BucketId& bucket); - document::Bucket getBucket() const override { return _bucket; } - bool hasBeenRemapped() const { return (_originalBucket.getRawId() != 0); } - const document::BucketId& getOriginalBucketId() const { return _originalBucket; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp deleted file mode 100644 index 8d5ad85b8aa..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bucketinfocommand.h" -#include - -namespace storage { -namespace api { - -void -BucketInfoCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "BucketInfoCommand()"; - if (verbose) { - out << " : "; - BucketCommand::print(out, verbose, indent); - } -} - -} // api -} // storage diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h b/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h deleted file mode 100644 index 0f6627328a9..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::api::BucketInfoCommand - * @ingroup messageapi - * - * @brief Superclass for storage commands that returns bucket info. - * - * This class doesn't add any functionality now, other than being able to check - * if a message is an instance of this class. But we want commands and replies - * to be in the same inheritance structure, and the reply adds functionality. - */ - -#pragma once - -#include "bucketcommand.h" - -namespace storage::api { - -class BucketInfoCommand : public BucketCommand { -protected: - BucketInfoCommand(const MessageType& type, const document::Bucket &bucket) - : BucketCommand(type, bucket) {} - -public: - DECLARE_POINTER_TYPEDEFS(BucketInfoCommand); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -} - diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp deleted file mode 100644 index 6eb5f96e888..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bucketinforeply.h" -#include - -namespace storage::api { - -void -BucketInfoReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "BucketInfoReply(" << _result << ")"; - if (verbose) { - out << " : "; - BucketReply::print(out, verbose, indent); - } -} - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h b/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h deleted file mode 100644 index 961e63a7ac8..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::api::BucketInfoReply - * @ingroup messageapi - * - * @brief Superclass for storage replies which returns bucket info in reply. - * - * A bucket info reply contains information about the state of a bucket. This - * can be altered from before the operation if this was a write operation or if - * the bucket was repaired in the process. - */ - -#pragma once - -#include "bucketreply.h" -#include "bucketinfocommand.h" -#include - -namespace storage::api { - -class BucketInfoReply : public BucketReply { - BucketInfo _result; - -protected: - BucketInfoReply(const BucketInfoCommand& cmd) - : BucketReply(cmd), - _result() - {} - -public: - DECLARE_POINTER_TYPEDEFS(BucketInfoReply); - - const BucketInfo& getBucketInfo() const { return _result; }; - void setBucketInfo(const BucketInfo& info) { _result = info; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp deleted file mode 100644 index 08b5effbd11..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bucketreply.h" -#include "bucketcommand.h" -#include - -using document::Bucket; -using document::BucketId; - -namespace storage::api { - -void -BucketReply::remapBucketId(const BucketId& bucket) { - if (_originalBucket.getRawId() == 0) { - _originalBucket = _bucket.getBucketId(); - } - Bucket newBucket(_bucket.getBucketSpace(), bucket); - _bucket = newBucket; -} - -void -BucketReply::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "BucketReply(" << _bucket.getBucketId(); - if (hasBeenRemapped()) { - out << " <- " << _originalBucket; - } - out << ")"; - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketreply.h b/storageapi/src/vespa/storageapi/messageapi/bucketreply.h deleted file mode 100644 index e7ded37c14d..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/bucketreply.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::api::BucketReply - * @ingroup messageapi - * - * @brief Superclass for storage replies which operates on single bucket. - */ - -#pragma once - -#include "storagereply.h" -#include "bucketcommand.h" - -namespace storage::api { - -class BucketCommand; - -class BucketReply : public StorageReply { - document::Bucket _bucket; - document::BucketId _originalBucket; - -protected: - BucketReply(const BucketCommand& cmd) - : StorageReply(cmd), - _bucket(cmd.getBucket()), - _originalBucket(cmd.getOriginalBucketId()) - { } - -public: - DECLARE_POINTER_TYPEDEFS(BucketReply); - - document::Bucket getBucket() const override { return _bucket; } - - bool hasBeenRemapped() const { return (_originalBucket.getRawId() != 0); } - const document::BucketId& getOriginalBucketId() const { return _originalBucket; } - - /** The deserialization code need access to set the remapping. */ - void remapBucketId(const document::BucketId& bucket); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.cpp b/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.cpp deleted file mode 100644 index 91551be0987..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.cpp +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "maintenancecommand.h" - -namespace storage::api { - -MaintenanceCommand::~MaintenanceCommand() {} - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h b/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h deleted file mode 100644 index 6bb36d0d32f..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "bucketinfocommand.h" - -namespace storage { -namespace api { - -class MaintenanceCommand : public BucketInfoCommand -{ -public: - MaintenanceCommand(const MessageType& type, const document::Bucket &bucket) - : BucketInfoCommand(type, bucket) - {} - MaintenanceCommand(const MaintenanceCommand &) = default; - MaintenanceCommand(MaintenanceCommand &&) = default; - MaintenanceCommand & operator = (const MaintenanceCommand &) = delete; - MaintenanceCommand & operator = (MaintenanceCommand &&) = delete; - ~MaintenanceCommand(); - - const vespalib::string& getReason() const { return _reason; }; - void setReason(vespalib::stringref reason) { _reason = reason; }; -protected: - vespalib::string _reason; -}; - -} -} diff --git a/storageapi/src/vespa/storageapi/messageapi/messagehandler.h b/storageapi/src/vespa/storageapi/messageapi/messagehandler.h deleted file mode 100644 index 9ba8542e9db..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/messagehandler.h +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::api::MessageHandler - * @ingroup messageapi - * - * @brief Class to prevent manual casting and switches of message types. - * - * MessageHandler defines an interface for processing StorageMessage objects - * of various subclasses. - * - * @version $Id$ - */ - -#pragma once - -#include - -namespace storage::api { - -// Commands - -class GetCommand; // Retrieve document -class PutCommand; // Add document -class UpdateCommand; // Update document -class RemoveCommand; // Remove document -class RevertCommand; // Revert put/remove operation - -class CreateVisitorCommand; // Create a new visitor -class DestroyVisitorCommand; // Destroy a running visitor -class VisitorInfoCommand; // Sends visitor info to visitor controller -class MapVisitorCommand; -class SearchResultCommand; -class DocumentSummaryCommand; -class QueryResultCommand; - -class InternalCommand; - -class CreateBucketCommand; -class DeleteBucketCommand; -class MergeBucketCommand; -class GetBucketDiffCommand; -class ApplyBucketDiffCommand; -class SplitBucketCommand; -class JoinBucketsCommand; -class SetBucketStateCommand; - -class RequestBucketInfoCommand; -class NotifyBucketChangeCommand; -class SetNodeStateCommand; -class GetNodeStateCommand; -class SetSystemStateCommand; -class ActivateClusterStateVersionCommand; -class ActivateClusterStateVersionReply; -class GetSystemStateCommand; -class BucketsAddedCommand; -class BucketsRemovedCommand; - -// Replies - -class GetReply; -class PutReply; -class UpdateReply; -class RemoveReply; -class RevertReply; - -class CreateVisitorReply; -class DestroyVisitorReply; -class VisitorInfoReply; -class MapVisitorReply; -class SearchResultReply; -class DocumentSummaryReply; -class QueryResultReply; - -class InternalReply; - -class CreateBucketReply; -class DeleteBucketReply; -class MergeBucketReply; -class GetBucketDiffReply; -class ApplyBucketDiffReply; -class SplitBucketReply; -class JoinBucketsReply; -class SetBucketStateReply; - -class RequestBucketInfoReply; -class NotifyBucketChangeReply; -class SetNodeStateReply; -class GetNodeStateReply; -class SetSystemStateReply; -class GetSystemStateReply; -class BucketsAddedReply; -class BucketsRemovedReply; - -class StatBucketCommand; -class StatBucketReply; -class GetBucketListCommand; -class GetBucketListReply; - -class EmptyBucketsCommand; -class EmptyBucketsReply; - -class RemoveLocationCommand; -class RemoveLocationReply; - -#define _INTERNAL_DEF_ON_MC(m, c) bool m(const std::shared_ptr & ) override -#define _INTERNAL_DEF_IMPL_ON_MC(m, c) bool m(const std::shared_ptr & ) override { return false; } -#define _INTERNAL_IMPL_ON_MC(cl, m, c, p) bool cl::m(const std::shared_ptr & p) -#define DEF_IMPL_MSG_COMMAND_H(m) _INTERNAL_DEF_IMPL_ON_MC(on##m, m##Command) -#define DEF_IMPL_MSG_REPLY_H(m) _INTERNAL_DEF_IMPL_ON_MC(on##m##Reply, m##Reply) -#define DEF_MSG_COMMAND_H(m) _INTERNAL_DEF_ON_MC(on##m, m##Command) -#define DEF_MSG_REPLY_H(m) _INTERNAL_DEF_ON_MC(on##m##Reply, m##Reply) -#define IMPL_MSG_COMMAND_ARG_H(cl, m, p) _INTERNAL_IMPL_ON_MC(cl, on##m, m##Command, p) -#define IMPL_MSG_REPLY_ARG_H(cl, m, p) _INTERNAL_IMPL_ON_MC(cl, on##m##Reply, m##Reply, p) -#define IMPL_MSG_COMMAND_H(cl, m) IMPL_MSG_COMMAND_ARG_H(cl, m, cmd) -#define IMPL_MSG_REPLY_H(cl, m) IMPL_MSG_REPLY_ARG_H(cl, m, reply) -#define ON_M(m) DEF_IMPL_MSG_COMMAND_H(m); DEF_IMPL_MSG_REPLY_H(m) - -class MessageHandler { -public: - // Basic operations - virtual bool onGet(const std::shared_ptr&) { return false; } - virtual bool onGetReply(const std::shared_ptr&) { return false; } - virtual bool onPut(const std::shared_ptr&) { return false; } - virtual bool onPutReply(const std::shared_ptr&) { return false; } - virtual bool onUpdate(const std::shared_ptr&) { return false; } - virtual bool onUpdateReply(const std::shared_ptr&) { return false; } - virtual bool onRemove(const std::shared_ptr&) { return false; } - virtual bool onRemoveReply(const std::shared_ptr&) { return false; } - virtual bool onRevert(const std::shared_ptr&) { return false; } - virtual bool onRevertReply(const std::shared_ptr&) { return false; } - - virtual bool onCreateVisitor(const std::shared_ptr&) { return false; } - virtual bool onCreateVisitorReply(const std::shared_ptr&) { return false; } - virtual bool onDestroyVisitor(const std::shared_ptr&) { return false; } - virtual bool onDestroyVisitorReply(const std::shared_ptr&) { return false; } - virtual bool onVisitorInfo(const std::shared_ptr&) { return false; } - virtual bool onVisitorInfoReply(const std::shared_ptr&) { return false; } - virtual bool onMapVisitor(const std::shared_ptr&) { return false; } - virtual bool onMapVisitorReply(const std::shared_ptr&) { return false; } - virtual bool onSearchResult(const std::shared_ptr&) { return false; } - virtual bool onSearchResultReply(const std::shared_ptr&) { return false; } - virtual bool onQueryResult(const std::shared_ptr&) { return false; } - virtual bool onQueryResultReply(const std::shared_ptr&) { return false; } - virtual bool onDocumentSummary(const std::shared_ptr&) { return false; } - virtual bool onDocumentSummaryReply(const std::shared_ptr&) { return false; } - virtual bool onEmptyBuckets(const std::shared_ptr&) { return false; } - virtual bool onEmptyBucketsReply(const std::shared_ptr&) { return false; } - virtual bool onInternal(const std::shared_ptr&) { return false; } - virtual bool onInternalReply(const std::shared_ptr&) { return false; } - virtual bool onCreateBucket(const std::shared_ptr&) { return false; } - virtual bool onCreateBucketReply(const std::shared_ptr&) { return false; } - virtual bool onDeleteBucket(const std::shared_ptr&) { return false; } - virtual bool onDeleteBucketReply(const std::shared_ptr&) { return false; } - virtual bool onMergeBucket(const std::shared_ptr&) { return false; } - virtual bool onMergeBucketReply(const std::shared_ptr&) { return false; } - virtual bool onGetBucketDiff(const std::shared_ptr&) { return false; } - virtual bool onGetBucketDiffReply(const std::shared_ptr&) { return false; } - virtual bool onApplyBucketDiff(const std::shared_ptr&) { return false; } - virtual bool onApplyBucketDiffReply(const std::shared_ptr&) { return false; } - virtual bool onSplitBucket(const std::shared_ptr&) { return false; } - virtual bool onSplitBucketReply(const std::shared_ptr&) { return false; } - virtual bool onJoinBuckets(const std::shared_ptr&) { return false; } - virtual bool onJoinBucketsReply(const std::shared_ptr&) { return false; } - virtual bool onSetBucketState(const std::shared_ptr&) { return false; } - virtual bool onSetBucketStateReply(const std::shared_ptr&) { return false; } - virtual bool onRequestBucketInfo(const std::shared_ptr&) { return false; } - virtual bool onRequestBucketInfoReply(const std::shared_ptr&) { return false; } - virtual bool onNotifyBucketChange(const std::shared_ptr&) { return false; } - virtual bool onNotifyBucketChangeReply(const std::shared_ptr&) { return false; } - virtual bool onSetNodeState(const std::shared_ptr&) { return false; } - virtual bool onSetNodeStateReply(const std::shared_ptr&) { return false; } - virtual bool onGetNodeState(const std::shared_ptr&) { return false; } - virtual bool onGetNodeStateReply(const std::shared_ptr&) { return false; } - virtual bool onSetSystemState(const std::shared_ptr&) { return false; } - virtual bool onSetSystemStateReply(const std::shared_ptr&) { return false; } - virtual bool onActivateClusterStateVersion(const std::shared_ptr&) { return false; } - virtual bool onActivateClusterStateVersionReply(const std::shared_ptr&) { return false; } - virtual bool onGetSystemState(const std::shared_ptr&) { return false; } - virtual bool onGetSystemStateReply(const std::shared_ptr&) { return false; } - virtual bool onBucketsAdded(const std::shared_ptr&) { return false; } - virtual bool onBucketsAddedReply(const std::shared_ptr&) { return false; } - virtual bool onBucketsRemoved(const std::shared_ptr&) { return false; } - virtual bool onBucketsRemovedReply(const std::shared_ptr&) { return false; } - virtual bool onStatBucket(const std::shared_ptr&) { return false; } - virtual bool onStatBucketReply(const std::shared_ptr&) { return false; } - virtual bool onGetBucketList(const std::shared_ptr&) { return false; } - virtual bool onGetBucketListReply(const std::shared_ptr&) { return false; } - virtual bool onRemoveLocation(const std::shared_ptr&) { return false; } - virtual bool onRemoveLocationReply(const std::shared_ptr&) { return false; } - - virtual ~MessageHandler() = default; -}; - -#undef ON_M - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp deleted file mode 100644 index ef587968515..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "returncode.h" -#include - -namespace storage::api { - -ReturnCode & ReturnCode::operator = (ReturnCode &&) noexcept = default; - -ReturnCode::ReturnCode(Result result, vespalib::stringref msg) - : _result(result), - _message() -{ - if ( ! msg.empty()) { - _message = std::make_unique(msg); - } -} - -ReturnCode::ReturnCode(const ReturnCode & rhs) - : _result(rhs._result), - _message() -{ - if (rhs._message) { - _message = std::make_unique(*rhs._message); - } -} - -ReturnCode & -ReturnCode::operator = (const ReturnCode & rhs) { - return operator=(ReturnCode(rhs)); -} - -vespalib::string -ReturnCode::getResultString(Result result) { - return documentapi::DocumentProtocol::getErrorName(result); -} - -vespalib::string -ReturnCode::toString() const { - vespalib::string ret = "ReturnCode("; - ret += getResultString(_result); - if ( _message && ! _message->empty()) { - ret += ", "; - ret += *_message; - } - ret += ")"; - return ret; -} - -std::ostream & -operator << (std::ostream & os, const ReturnCode & returnCode) { - return os << returnCode.toString(); -} - -bool -ReturnCode::isBusy() const -{ - // Casting to suppress -Wswitch since we're comparing against enum values - // not present in the ReturnCode::Result enum. - switch (static_cast(_result)) { - case mbus::ErrorCode::SEND_QUEUE_FULL: - case mbus::ErrorCode::SESSION_BUSY: - case mbus::ErrorCode::TIMEOUT: - case Protocol::ERROR_BUSY: - return true; - default: - return false; - } -} - -bool -ReturnCode::isNodeDownOrNetwork() const -{ - switch (static_cast(_result)) { - case mbus::ErrorCode::NO_ADDRESS_FOR_SERVICE: - case mbus::ErrorCode::CONNECTION_ERROR: - case mbus::ErrorCode::UNKNOWN_SESSION: - case mbus::ErrorCode::HANDSHAKE_FAILED: - case mbus::ErrorCode::NO_SERVICES_FOR_ROUTE: - case mbus::ErrorCode::NETWORK_ERROR: - case mbus::ErrorCode::UNKNOWN_PROTOCOL: - case Protocol::ERROR_NODE_NOT_READY: - case Protocol::ERROR_NOT_CONNECTED: - return true; - default: - return false; - } -} - -bool -ReturnCode::isCriticalForMaintenance() const -{ - if (_result >= static_cast(mbus::ErrorCode::FATAL_ERROR)) { - return true; - } - - switch (static_cast(_result)) { - case Protocol::ERROR_INTERNAL_FAILURE: - case Protocol::ERROR_NO_SPACE: - case Protocol::ERROR_UNPARSEABLE: - case Protocol::ERROR_ILLEGAL_PARAMETERS: - case Protocol::ERROR_NOT_IMPLEMENTED: - case Protocol::ERROR_UNKNOWN_COMMAND: - case Protocol::ERROR_PROCESSING_FAILURE: - case Protocol::ERROR_IGNORED: - return true; - default: - return false; - } -} - -bool -ReturnCode::isCriticalForVisitor() const -{ - return isCriticalForMaintenance(); -} - -bool -ReturnCode::isCriticalForVisitorDispatcher() const -{ - return isCriticalForMaintenance(); -} - -bool -ReturnCode::isNonCriticalForIntegrityChecker() const -{ - switch (static_cast(_result)) { - case Protocol::ERROR_ABORTED: - case Protocol::ERROR_BUCKET_DELETED: - case Protocol::ERROR_BUCKET_NOT_FOUND: - return true; - default: - return false; - } -} - -bool -ReturnCode::isShutdownRelated() const -{ - switch (static_cast(_result)) { - case Protocol::ERROR_ABORTED: - return true; - default: - return false; - } -} - -bool -ReturnCode::isBucketDisappearance() const -{ - switch (static_cast(_result)) { - case Protocol::ERROR_BUCKET_NOT_FOUND: - case Protocol::ERROR_BUCKET_DELETED: - return true; - default: - return false; - } -} - -bool -ReturnCode::operator==(const ReturnCode& code) const { - return (_result == code._result) && (getMessage() == code.getMessage()); -} - -bool -ReturnCode::operator!=(const ReturnCode& code) const { - return (_result != code._result) || (getMessage() != code.getMessage()); -} -} diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.h b/storageapi/src/vespa/storageapi/messageapi/returncode.h deleted file mode 100644 index f60cbe2b840..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/returncode.h +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::api::ReturnCode - * @ingroup messageapi - * - * @brief Class for representing return values from the processing chain - * - * @version $Id$ - */ - -#pragma once - -#include - - -namespace storage::api { - -class ReturnCode { -public: - typedef documentapi::DocumentProtocol Protocol; - - /** Return status codes */ - enum Result { - OK = mbus::ErrorCode::NONE, - ENCODE_ERROR = mbus::ErrorCode::ENCODE_ERROR, - - EXISTS = Protocol::ERROR_EXISTS, - - NOT_READY = Protocol::ERROR_NODE_NOT_READY, - WRONG_DISTRIBUTION = Protocol::ERROR_WRONG_DISTRIBUTION, - REJECTED = Protocol::ERROR_REJECTED, - ABORTED = Protocol::ERROR_ABORTED, - BUCKET_NOT_FOUND = Protocol::ERROR_BUCKET_NOT_FOUND, - BUCKET_DELETED = Protocol::ERROR_BUCKET_DELETED, - TIMESTAMP_EXIST = Protocol::ERROR_TIMESTAMP_EXIST, - STALE_TIMESTAMP = Protocol::ERROR_STALE_TIMESTAMP, - TEST_AND_SET_CONDITION_FAILED = Protocol::ERROR_TEST_AND_SET_CONDITION_FAILED, - - // Wrong use - UNKNOWN_COMMAND = Protocol::ERROR_UNKNOWN_COMMAND, - NOT_IMPLEMENTED = Protocol::ERROR_NOT_IMPLEMENTED, - ILLEGAL_PARAMETERS = Protocol::ERROR_ILLEGAL_PARAMETERS, - IGNORED = Protocol::ERROR_IGNORED, - UNPARSEABLE = Protocol::ERROR_UNPARSEABLE, - - // Network failure - NOT_CONNECTED = Protocol::ERROR_NOT_CONNECTED, - TIMEOUT = mbus::ErrorCode::TIMEOUT, - BUSY = Protocol::ERROR_BUSY, - - // Disk operations - NO_SPACE = Protocol::ERROR_NO_SPACE, - DISK_FAILURE = Protocol::ERROR_DISK_FAILURE, - IO_FAILURE = Protocol::ERROR_IO_FAILURE, - - // Don't know what happened (catch-all) - INTERNAL_FAILURE = Protocol::ERROR_INTERNAL_FAILURE - }; - -private: - Result _result; - std::unique_ptr _message; -public: - ReturnCode() - : _result(OK), - _message() - { } - explicit ReturnCode(Result result) - : _result(result), - _message() - {} - ReturnCode(Result result, vespalib::stringref msg); - ReturnCode(const ReturnCode &); - ReturnCode & operator = (const ReturnCode &); - ReturnCode(ReturnCode &&) noexcept = default; - ReturnCode & operator = (ReturnCode &&) noexcept; - - vespalib::stringref getMessage() const { - return _message - ? _message->operator vespalib::stringref() - : vespalib::stringref(); - } - - Result getResult() const { return _result; } - - /** - * Translate from status code to human-readable string - * @param result Status code returned from getResult() - */ - static vespalib::string getResultString(Result result); - - bool failed() const { return (_result != OK); } - bool success() const { return (_result == OK); } - - bool operator==(Result res) const { return _result == res; } - bool operator!=(Result res) const { return _result != res; } - bool operator==(const ReturnCode& code) const; - bool operator!=(const ReturnCode& code) const; - - // To avoid lots of code matching various return codes in storage, we define - // some functions they can use to match those codes that corresponds to what - // they want to match. - - bool isBusy() const; - bool isNodeDownOrNetwork() const; - bool isCriticalForMaintenance() const; - bool isCriticalForVisitor() const; - bool isCriticalForVisitorDispatcher() const; - bool isShutdownRelated() const; - bool isBucketDisappearance() const; - bool isNonCriticalForIntegrityChecker() const; - vespalib::string toString() const; -}; - -std::ostream & operator << (std::ostream & os, const ReturnCode & returnCode); - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp b/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp deleted file mode 100644 index 1e797ba4792..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "storagecommand.h" -#include -#include - -namespace storage::api { - -StorageCommand::StorageCommand(const StorageCommand& other) - : StorageMessage(other, generateMsgId()), - _timeout(other._timeout), - _sourceIndex(other._sourceIndex) -{ -} - -StorageCommand::StorageCommand(const MessageType& type, Priority p) - : StorageMessage(type, generateMsgId()), - // Default timeout is unlimited. Set from mbus message. Some internal - // use want unlimited timeout, (such as readbucketinfo, repair bucket - // etc) - _timeout(duration::max()), - _sourceIndex(0xFFFF) -{ - setPriority(p); -} - -StorageCommand::~StorageCommand() = default; - -void -StorageCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "StorageCommand(" << _type.getName(); - if (_priority != NORMAL) out << ", priority = " << static_cast(_priority); - if (_sourceIndex != 0xFFFF) out << ", source = " << _sourceIndex; - out << ", timeout = " << vespalib::count_ms(_timeout) << " ms"; - out << ")"; -} - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagecommand.h b/storageapi/src/vespa/storageapi/messageapi/storagecommand.h deleted file mode 100644 index 30d59e5fe4b..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/storagecommand.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::api::StorageCommand - * @ingroup messageapi - * - * @brief Superclass for all storage commands. - * - * A storage command is a storage message you will get a storage reply for. - * - * @version $Id$ - */ - -#pragma once - -#include "storagemessage.h" - -namespace storage::api { - -class StorageReply; - -class StorageCommand : public StorageMessage { - duration _timeout; /** Timeout of command in milliseconds */ - /** Sets what node this message origins from. 0xFFFF is unset. */ - uint16_t _sourceIndex; - -protected: - explicit StorageCommand(const StorageCommand& other); - explicit StorageCommand(const MessageType& type, Priority p = NORMAL); - -public: - DECLARE_POINTER_TYPEDEFS(StorageCommand); - - ~StorageCommand() override; - - bool sourceIndexSet() const { return (_sourceIndex != 0xffff); } - void setSourceIndex(uint16_t sourceIndex) { _sourceIndex = sourceIndex; } - uint16_t getSourceIndex() const { return _sourceIndex; } - - void setTimeout(duration timeout) { _timeout = timeout; } - duration getTimeout() const { return _timeout; } - - /** Used to set a new id so the message can be resent. */ - void setNewId() { StorageMessage::setNewMsgId(); } - - /** Overload this to get more descriptive message output. */ - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - /** - * A way for someone to make a reply to a storage message without - * knowing the type of the message. Should just call reply constructor - * taking command as input. - */ - virtual std::unique_ptr makeReply() = 0; -}; - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp deleted file mode 100644 index db5c86af989..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "storagemessage.h" -#include -#include -#include -#include -#include -#include -#include - -namespace storage::api { - -namespace { - -std::atomic _G_lastMsgId(1000); - -} - -static const vespalib::string STORAGEADDRESS_PREFIX = "storage/cluster."; - -const char* -StorageMessage::getPriorityString(Priority p) { - switch (p) { - case LOW: return "LOW"; - case NORMAL: return "NORMAL"; - case HIGH: return "HIGH"; - case VERYHIGH: return "VERYHIGH"; - default: return "UNKNOWN"; - } -} - -std::map MessageType::_codes; - -const MessageType MessageType::DOCBLOCK("DocBlock", DOCBLOCK_ID); -const MessageType MessageType::DOCBLOCK_REPLY("DocBlock Reply", DOCBLOCK_REPLY_ID, &MessageType::DOCBLOCK); -const MessageType MessageType::GET("Get", GET_ID); -const MessageType MessageType::GET_REPLY("Get Reply", GET_REPLY_ID, &MessageType::GET); -const MessageType MessageType::INTERNAL("Internal", INTERNAL_ID); -const MessageType MessageType::INTERNAL_REPLY("Internal Reply", INTERNAL_REPLY_ID, &MessageType::INTERNAL); -const MessageType MessageType::PUT("Put", PUT_ID); -const MessageType MessageType::PUT_REPLY("Put Reply", PUT_REPLY_ID, &MessageType::PUT); -const MessageType MessageType::UPDATE("Update", UPDATE_ID); -const MessageType MessageType::UPDATE_REPLY("Update Reply", UPDATE_REPLY_ID, &MessageType::UPDATE); -const MessageType MessageType::REMOVE("Remove", REMOVE_ID); -const MessageType MessageType::REMOVE_REPLY("Remove Reply", REMOVE_REPLY_ID, &MessageType::REMOVE); -const MessageType MessageType::REVERT("Revert", REVERT_ID); -const MessageType MessageType::REVERT_REPLY("Revert Reply", REVERT_REPLY_ID, &MessageType::REVERT); -const MessageType MessageType::VISITOR_CREATE("Visitor Create", VISITOR_CREATE_ID); -const MessageType MessageType::VISITOR_CREATE_REPLY("Visitor Create Reply", VISITOR_CREATE_REPLY_ID, &MessageType::VISITOR_CREATE); -const MessageType MessageType::VISITOR_DESTROY("Visitor Destroy", VISITOR_DESTROY_ID); -const MessageType MessageType::VISITOR_DESTROY_REPLY("Visitor Destroy Reply", VISITOR_DESTROY_REPLY_ID, &MessageType::VISITOR_DESTROY); -const MessageType MessageType::REQUESTBUCKETINFO("Request bucket info", REQUESTBUCKETINFO_ID); -const MessageType MessageType::REQUESTBUCKETINFO_REPLY("Request bucket info reply", REQUESTBUCKETINFO_REPLY_ID, &MessageType::REQUESTBUCKETINFO); -const MessageType MessageType::NOTIFYBUCKETCHANGE("Notify bucket change", NOTIFYBUCKETCHANGE_ID); -const MessageType MessageType::NOTIFYBUCKETCHANGE_REPLY("Notify bucket change reply", NOTIFYBUCKETCHANGE_REPLY_ID, &MessageType::NOTIFYBUCKETCHANGE); -const MessageType MessageType::CREATEBUCKET("Create bucket", CREATEBUCKET_ID); -const MessageType MessageType::CREATEBUCKET_REPLY("Create bucket reply", CREATEBUCKET_REPLY_ID, &MessageType::CREATEBUCKET); -const MessageType MessageType::MERGEBUCKET("Merge bucket", MERGEBUCKET_ID); -const MessageType MessageType::MERGEBUCKET_REPLY("Merge bucket reply", MERGEBUCKET_REPLY_ID, &MessageType::MERGEBUCKET); -const MessageType MessageType::DELETEBUCKET("Delete bucket", DELETEBUCKET_ID); -const MessageType MessageType::DELETEBUCKET_REPLY("Delete bucket reply", DELETEBUCKET_REPLY_ID, &MessageType::DELETEBUCKET); -const MessageType MessageType::SETNODESTATE("Set node state", SETNODESTATE_ID); -const MessageType MessageType::SETNODESTATE_REPLY("Set node state reply", SETNODESTATE_REPLY_ID, &MessageType::SETNODESTATE); -const MessageType MessageType::GETNODESTATE("Get node state", GETNODESTATE_ID); -const MessageType MessageType::GETNODESTATE_REPLY("Get node state reply", GETNODESTATE_REPLY_ID, &MessageType::GETNODESTATE); -const MessageType MessageType::SETSYSTEMSTATE("Set system state", SETSYSTEMSTATE_ID); -const MessageType MessageType::SETSYSTEMSTATE_REPLY("Set system state reply", SETSYSTEMSTATE_REPLY_ID, &MessageType::SETSYSTEMSTATE); -const MessageType MessageType::GETSYSTEMSTATE("Get system state", GETSYSTEMSTATE_ID); -const MessageType MessageType::GETSYSTEMSTATE_REPLY("get system state reply", GETSYSTEMSTATE_REPLY_ID, &MessageType::GETSYSTEMSTATE); -const MessageType MessageType::ACTIVATE_CLUSTER_STATE_VERSION("Activate cluster state version", ACTIVATE_CLUSTER_STATE_VERSION_ID); -const MessageType MessageType::ACTIVATE_CLUSTER_STATE_VERSION_REPLY("Activate cluster state version reply", ACTIVATE_CLUSTER_STATE_VERSION_REPLY_ID, &MessageType::ACTIVATE_CLUSTER_STATE_VERSION); -const MessageType MessageType::GETBUCKETDIFF("GetBucketDiff", GETBUCKETDIFF_ID); -const MessageType MessageType::GETBUCKETDIFF_REPLY("GetBucketDiff reply", GETBUCKETDIFF_REPLY_ID, &MessageType::GETBUCKETDIFF); -const MessageType MessageType::APPLYBUCKETDIFF("ApplyBucketDiff", APPLYBUCKETDIFF_ID); -const MessageType MessageType::APPLYBUCKETDIFF_REPLY("ApplyBucketDiff reply", APPLYBUCKETDIFF_REPLY_ID, &MessageType::APPLYBUCKETDIFF); -const MessageType MessageType::VISITOR_INFO("VisitorInfo", VISITOR_INFO_ID); -const MessageType MessageType::VISITOR_INFO_REPLY("VisitorInfo reply", VISITOR_INFO_REPLY_ID, &MessageType::VISITOR_INFO); -const MessageType MessageType::SEARCHRESULT("SearchResult", SEARCHRESULT_ID); -const MessageType MessageType::SEARCHRESULT_REPLY("SearchResult reply", SEARCHRESULT_REPLY_ID, &MessageType::SEARCHRESULT); -const MessageType MessageType::DOCUMENTSUMMARY("DocumentSummary", DOCUMENTSUMMARY_ID); -const MessageType MessageType::DOCUMENTSUMMARY_REPLY("DocumentSummary reply", DOCUMENTSUMMARY_REPLY_ID, &MessageType::DOCUMENTSUMMARY); -const MessageType MessageType::MAPVISITOR("Mapvisitor", MAPVISITOR_ID); -const MessageType MessageType::MAPVISITOR_REPLY("Mapvisitor reply", MAPVISITOR_REPLY_ID, &MessageType::MAPVISITOR); -const MessageType MessageType::SPLITBUCKET("SplitBucket", SPLITBUCKET_ID); -const MessageType MessageType::SPLITBUCKET_REPLY("SplitBucket reply", SPLITBUCKET_REPLY_ID, &MessageType::SPLITBUCKET); -const MessageType MessageType::JOINBUCKETS("Joinbuckets", JOINBUCKETS_ID); -const MessageType MessageType::JOINBUCKETS_REPLY("Joinbuckets reply", JOINBUCKETS_REPLY_ID, &MessageType::JOINBUCKETS); -const MessageType MessageType::STATBUCKET("Statbucket", STATBUCKET_ID); -const MessageType MessageType::STATBUCKET_REPLY("Statbucket Reply", STATBUCKET_REPLY_ID, &MessageType::STATBUCKET); -const MessageType MessageType::GETBUCKETLIST("Getbucketlist", GETBUCKETLIST_ID); -const MessageType MessageType::GETBUCKETLIST_REPLY("Getbucketlist Reply", GETBUCKETLIST_REPLY_ID, &MessageType::GETBUCKETLIST); -const MessageType MessageType::DOCUMENTLIST("documentlist", DOCUMENTLIST_ID); -const MessageType MessageType::DOCUMENTLIST_REPLY("documentlist Reply", DOCUMENTLIST_REPLY_ID, &MessageType::DOCUMENTLIST); -const MessageType MessageType::EMPTYBUCKETS("Emptybuckets", EMPTYBUCKETS_ID); -const MessageType MessageType::EMPTYBUCKETS_REPLY("Emptybuckets Reply", EMPTYBUCKETS_REPLY_ID, &MessageType::EMPTYBUCKETS); -const MessageType MessageType::REMOVELOCATION("Removelocation", REMOVELOCATION_ID); -const MessageType MessageType::REMOVELOCATION_REPLY("Removelocation Reply", REMOVELOCATION_REPLY_ID, &MessageType::REMOVELOCATION); -const MessageType MessageType::QUERYRESULT("QueryResult", QUERYRESULT_ID); -const MessageType MessageType::QUERYRESULT_REPLY("QueryResult reply", QUERYRESULT_REPLY_ID, &MessageType::QUERYRESULT); -const MessageType MessageType::SETBUCKETSTATE("SetBucketState", SETBUCKETSTATE_ID); -const MessageType MessageType::SETBUCKETSTATE_REPLY("SetBucketStateReply", SETBUCKETSTATE_REPLY_ID, &MessageType::SETBUCKETSTATE); - -const MessageType& -MessageType::MessageType::get(Id id) -{ - auto it = _codes.find(id); - if (it == _codes.end()) { - std::ostringstream ost; - ost << "No message type with id " << id << "."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); - } - return *it->second; -} -MessageType::MessageType(vespalib::stringref name, Id id, - const MessageType* replyOf) - : _name(name), _id(id), _reply(nullptr), _replyOf(replyOf) -{ - _codes[id] = this; - if (_replyOf) { - assert(_replyOf->_reply == nullptr); - // Ugly cast to let initialization work - auto& type = const_cast(*_replyOf); - type._reply = this; - } -} - -MessageType::~MessageType() = default; - -void -MessageType::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "MessageType(" << _id << ", " << _name; - if (_replyOf) { - out << ", reply of " << _replyOf->getName(); - } - out << ")"; -} - -std::ostream & operator << (std::ostream & os, const StorageMessageAddress & addr) { - return os << addr.toString(); -} - -namespace { - -vespalib::string -createAddress(vespalib::stringref cluster, const lib::NodeType &type, uint16_t index) { - vespalib::asciistream os; - os << STORAGEADDRESS_PREFIX << cluster << '/' << type.toString() << '/' << index << "/default"; - return os.str(); -} - -uint32_t -calculate_node_hash(const lib::NodeType &type, uint16_t index) { - uint16_t buf[] = {type, index}; - size_t hash = vespalib::hashValue(&buf, sizeof(buf)); - return uint32_t(hash & 0xffffffffl) ^ uint32_t(hash >> 32); -} - -vespalib::string Empty; - -} - -// TODO we ideally want this removed. Currently just in place to support usage as map key when emplacement not available -StorageMessageAddress::StorageMessageAddress() noexcept - : _cluster(&Empty), - _precomputed_storage_hash(0), - _type(lib::NodeType::Type::UNKNOWN), - _protocol(Protocol::STORAGE), - _index(0) -{} - -StorageMessageAddress::StorageMessageAddress(const vespalib::string * cluster, const lib::NodeType& type, uint16_t index) noexcept - : StorageMessageAddress(cluster, type, index, Protocol::STORAGE) -{ } - -StorageMessageAddress::StorageMessageAddress(const vespalib::string * cluster, const lib::NodeType& type, - uint16_t index, Protocol protocol) noexcept - : _cluster(cluster), - _precomputed_storage_hash(calculate_node_hash(type, index)), - _type(type.getType()), - _protocol(protocol), - _index(index) -{ } - -StorageMessageAddress::~StorageMessageAddress() = default; - -mbus::Route -StorageMessageAddress::to_mbus_route() const -{ - mbus::Route result; - auto address_as_str = createAddress(getCluster(), lib::NodeType::get(_type), _index); - std::vector directives; - directives.emplace_back(std::make_shared(std::move(address_as_str))); - result.addHop(mbus::Hop(std::move(directives), false)); - return result; -} - -bool -StorageMessageAddress::operator==(const StorageMessageAddress& other) const noexcept -{ - if (_protocol != other._protocol) return false; - if (_type != other._type) return false; - if (_index != other._index) return false; - if (getCluster() != other.getCluster()) return false; - return true; -} - -vespalib::string -StorageMessageAddress::toString() const -{ - vespalib::asciistream os; - print(os); - return os.str(); -} - -void -StorageMessageAddress::print(vespalib::asciistream & out) const -{ - out << "StorageMessageAddress("; - if (_protocol == Protocol::STORAGE) { - out << "Storage protocol"; - } else { - out << "Document protocol"; - } - if (_type == lib::NodeType::Type::UNKNOWN) { - out << ", " << to_mbus_route().toString() << ")"; - } else { - out << ", cluster " << getCluster() << ", nodetype " << lib::NodeType::get(_type) - << ", index " << _index << ")"; - } -} - -TransportContext::~TransportContext() = default; - -StorageMessage::Id -StorageMessage::generateMsgId() noexcept -{ - return _G_lastMsgId.fetch_add(1, std::memory_order_relaxed); -} - -StorageMessage::StorageMessage(const MessageType& type, Id id) noexcept - : _type(type), - _msgId(id), - _address(), - _trace(), - _approxByteSize(50), - _priority(NORMAL) -{ -} - -StorageMessage::StorageMessage(const StorageMessage& other, Id id) noexcept - : _type(other._type), - _msgId(id), - _address(), - _trace(other.getTrace().getLevel()), - _approxByteSize(other._approxByteSize), - _priority(other._priority) -{ -} - -StorageMessage::~StorageMessage() = default; - -void -StorageMessage::setNewMsgId() noexcept -{ - _msgId = generateMsgId(); -} - -vespalib::string -StorageMessage::getSummary() const { - return toString(); -} - -const char* -to_string(LockingRequirements req) noexcept { - switch (req) { - case LockingRequirements::Exclusive: return "Exclusive"; - case LockingRequirements::Shared: return "Shared"; - default: abort(); - } -} - -std::ostream& -operator<<(std::ostream& os, LockingRequirements req) { - os << to_string(req); - return os; -} - -const char* -to_string(InternalReadConsistency consistency) noexcept { - switch (consistency) { - case InternalReadConsistency::Strong: return "Strong"; - case InternalReadConsistency::Weak: return "Weak"; - default: abort(); - } -} - -std::ostream& -operator<<(std::ostream& os, InternalReadConsistency consistency) { - os << to_string(consistency); - return os; -} - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h deleted file mode 100644 index 71567192bd9..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h +++ /dev/null @@ -1,455 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -/** - * @class storage::api::StorageMessage - * @ingroup messageapi - * - * @brief Superclass for all storage messages. - * - * @version $Id$ - */ - -#pragma once - -#include "messagehandler.h" -#include -#include -#include -#include -#include -#include -#include - -namespace vespalib { class asciistream; } -// The following macros are provided as a way to write storage messages simply. -// They implement the parts of the code that can easily be automaticly -// generated. - -/** - * Adds a messagehandler callback and some utilities - */ -#define DECLARE_POINTER_TYPEDEFS(message) \ - typedef std::unique_ptr UP; \ - typedef std::shared_ptr SP; \ - typedef std::shared_ptr CSP; - -#define DECLARE_STORAGEREPLY(reply, callback) \ -public: \ - DECLARE_POINTER_TYPEDEFS(reply) \ -private: \ - bool callHandler(storage::api::MessageHandler& h, \ - const std::shared_ptr& m) const override \ - { \ - return h.callback(std::static_pointer_cast(m)); \ - } - -/** Commands also has a command to implement to create the reply. */ -#define DECLARE_STORAGECOMMAND(command, callback) \ -public: \ - std::unique_ptr makeReply() override; \ - DECLARE_STORAGEREPLY(command, callback) - -/** This macro implements common stuff for all storage messages. */ -#define IMPLEMENT_COMMON(message) \ - -/** This macro is used to implement common storage reply functionality. */ -#define IMPLEMENT_REPLY(reply) \ - IMPLEMENT_COMMON(reply) \ - -/** This macro is used to implement common storage command functionality. */ -#define IMPLEMENT_COMMAND(command, reply) \ - IMPLEMENT_COMMON(command) \ - std::unique_ptr \ - storage::api::command::makeReply() \ - { \ - return std::make_unique(*this); \ - } - -namespace storage::api { - -using duration = vespalib::duration; - -/** - * @class MessageType - * @ingroup messageapi - * - * @brief This class defines the different message types we have. - * - * This is used to be able to deserialize messages of various classes. - */ -class MessageType : public vespalib::Printable { -public: - enum Id { - GET_ID = 4, - GET_REPLY_ID = 5, - INTERNAL_ID = 6, - INTERNAL_REPLY_ID = 7, - PUT_ID = 10, - PUT_REPLY_ID = 11, - REMOVE_ID = 12, - REMOVE_REPLY_ID = 13, - REVERT_ID = 14, - REVERT_REPLY_ID = 15, - STAT_ID = 16, - STAT_REPLY_ID = 17, - VISITOR_CREATE_ID = 18, - VISITOR_CREATE_REPLY_ID = 19, - VISITOR_DESTROY_ID = 20, - VISITOR_DESTROY_REPLY_ID = 21, - CREATEBUCKET_ID = 26, - CREATEBUCKET_REPLY_ID = 27, - MERGEBUCKET_ID = 32, - MERGEBUCKET_REPLY_ID = 33, - DELETEBUCKET_ID = 34, - DELETEBUCKET_REPLY_ID = 35, - SETNODESTATE_ID = 36, - SETNODESTATE_REPLY_ID = 37, - GETNODESTATE_ID = 38, - GETNODESTATE_REPLY_ID = 39, - SETSYSTEMSTATE_ID = 40, - SETSYSTEMSTATE_REPLY_ID = 41, - GETSYSTEMSTATE_ID = 42, - GETSYSTEMSTATE_REPLY_ID = 43, - GETBUCKETDIFF_ID = 50, - GETBUCKETDIFF_REPLY_ID = 51, - APPLYBUCKETDIFF_ID = 52, - APPLYBUCKETDIFF_REPLY_ID = 53, - REQUESTBUCKETINFO_ID = 54, - REQUESTBUCKETINFO_REPLY_ID = 55, - NOTIFYBUCKETCHANGE_ID = 56, - NOTIFYBUCKETCHANGE_REPLY_ID = 57, - DOCBLOCK_ID = 58, - DOCBLOCK_REPLY_ID = 59, - VISITOR_INFO_ID = 60, - VISITOR_INFO_REPLY_ID = 61, - SEARCHRESULT_ID = 64, - SEARCHRESULT_REPLY_ID = 65, - SPLITBUCKET_ID = 66, - SPLITBUCKET_REPLY_ID = 67, - JOINBUCKETS_ID = 68, - JOINBUCKETS_REPLY_ID = 69, - DOCUMENTSUMMARY_ID = 72, - DOCUMENTSUMMARY_REPLY_ID = 73, - MAPVISITOR_ID = 74, - MAPVISITOR_REPLY_ID = 75, - STATBUCKET_ID = 76, - STATBUCKET_REPLY_ID = 77, - GETBUCKETLIST_ID = 78, - GETBUCKETLIST_REPLY_ID = 79, - DOCUMENTLIST_ID = 80, - DOCUMENTLIST_REPLY_ID = 81, - UPDATE_ID = 82, - UPDATE_REPLY_ID = 83, - EMPTYBUCKETS_ID = 84, - EMPTYBUCKETS_REPLY_ID = 85, - REMOVELOCATION_ID = 86, - REMOVELOCATION_REPLY_ID = 87, - QUERYRESULT_ID = 88, - QUERYRESULT_REPLY_ID = 89, - SETBUCKETSTATE_ID = 94, - SETBUCKETSTATE_REPLY_ID = 95, - ACTIVATE_CLUSTER_STATE_VERSION_ID = 96, - ACTIVATE_CLUSTER_STATE_VERSION_REPLY_ID = 97, - MESSAGETYPE_MAX_ID - }; - -private: - static std::map _codes; - const vespalib::string _name; - Id _id; - MessageType *_reply; - const MessageType *_replyOf; - - MessageType(vespalib::stringref name, Id id, const MessageType* replyOf = 0); -public: - static const MessageType DOCBLOCK; - static const MessageType DOCBLOCK_REPLY; - static const MessageType GET; - static const MessageType GET_REPLY; - static const MessageType INTERNAL; - static const MessageType INTERNAL_REPLY; - static const MessageType PUT; - static const MessageType PUT_REPLY; - static const MessageType REMOVE; - static const MessageType REMOVE_REPLY; - static const MessageType REVERT; - static const MessageType REVERT_REPLY; - static const MessageType VISITOR_CREATE; - static const MessageType VISITOR_CREATE_REPLY; - static const MessageType VISITOR_DESTROY; - static const MessageType VISITOR_DESTROY_REPLY; - static const MessageType REQUESTBUCKETINFO; - static const MessageType REQUESTBUCKETINFO_REPLY; - static const MessageType NOTIFYBUCKETCHANGE; - static const MessageType NOTIFYBUCKETCHANGE_REPLY; - static const MessageType CREATEBUCKET; - static const MessageType CREATEBUCKET_REPLY; - static const MessageType MERGEBUCKET; - static const MessageType MERGEBUCKET_REPLY; - static const MessageType DELETEBUCKET; - static const MessageType DELETEBUCKET_REPLY; - static const MessageType SETNODESTATE; - static const MessageType SETNODESTATE_REPLY; - static const MessageType GETNODESTATE; - static const MessageType GETNODESTATE_REPLY; - static const MessageType SETSYSTEMSTATE; - static const MessageType SETSYSTEMSTATE_REPLY; - static const MessageType GETSYSTEMSTATE; - static const MessageType GETSYSTEMSTATE_REPLY; - static const MessageType ACTIVATE_CLUSTER_STATE_VERSION; - static const MessageType ACTIVATE_CLUSTER_STATE_VERSION_REPLY; - static const MessageType BUCKETSADDED; - static const MessageType BUCKETSADDED_REPLY; - static const MessageType BUCKETSREMOVED; - static const MessageType BUCKETSREMOVED_REPLY; - static const MessageType GETBUCKETDIFF; - static const MessageType GETBUCKETDIFF_REPLY; - static const MessageType APPLYBUCKETDIFF; - static const MessageType APPLYBUCKETDIFF_REPLY; - static const MessageType VISITOR_INFO; - static const MessageType VISITOR_INFO_REPLY; - static const MessageType SEARCHRESULT; - static const MessageType SEARCHRESULT_REPLY; - static const MessageType SPLITBUCKET; - static const MessageType SPLITBUCKET_REPLY; - static const MessageType JOINBUCKETS; - static const MessageType JOINBUCKETS_REPLY; - static const MessageType DOCUMENTSUMMARY; - static const MessageType DOCUMENTSUMMARY_REPLY; - static const MessageType MAPVISITOR; - static const MessageType MAPVISITOR_REPLY; - static const MessageType STATBUCKET; - static const MessageType STATBUCKET_REPLY; - static const MessageType GETBUCKETLIST; - static const MessageType GETBUCKETLIST_REPLY; - static const MessageType DOCUMENTLIST; - static const MessageType DOCUMENTLIST_REPLY; - static const MessageType UPDATE; - static const MessageType UPDATE_REPLY; - static const MessageType EMPTYBUCKETS; - static const MessageType EMPTYBUCKETS_REPLY; - static const MessageType REMOVELOCATION; - static const MessageType REMOVELOCATION_REPLY; - static const MessageType QUERYRESULT; - static const MessageType QUERYRESULT_REPLY; - static const MessageType SETBUCKETSTATE; - static const MessageType SETBUCKETSTATE_REPLY; - - static const MessageType& get(Id id); - - MessageType(const MessageType &) = delete; - MessageType& operator=(const MessageType &) = delete; - ~MessageType(); - Id getId() const noexcept { return _id; } - static Id getMaxId() noexcept { return MESSAGETYPE_MAX_ID; } - const vespalib::string& getName() const noexcept { return _name; } - bool isReply() const noexcept { return (_replyOf != 0); } - /** Only valid to call on replies. */ - const MessageType& getCommandType() const noexcept { return *_replyOf; } - /** Only valid to call on commands. */ - const MessageType& getReplyType() const noexcept { return *_reply; } - bool operator==(const MessageType& type) const noexcept { return (_id == type._id); } - bool operator!=(const MessageType& type) const noexcept { return (_id != type._id); } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -/** - * Represent an address we can send a storage message to. - * We have two kinds of addresses: - * - A VDS address used to send to a single VDS node. - * - An external mbus route, used to send to an external source. - */ -class StorageMessageAddress { -public: - enum class Protocol : uint8_t { STORAGE, DOCUMENT }; - -private: - const vespalib::string *_cluster; - // Used for internal VDS addresses only - uint32_t _precomputed_storage_hash; - lib::NodeType::Type _type; - Protocol _protocol; - uint16_t _index; - -public: - StorageMessageAddress() noexcept; // Only to be used when transient default ctor semantics are needed by containers - StorageMessageAddress(const vespalib::string * cluster, const lib::NodeType& type, uint16_t index) noexcept; - StorageMessageAddress(const vespalib::string * cluster, const lib::NodeType& type, uint16_t index, Protocol protocol) noexcept; - ~StorageMessageAddress(); - - void setProtocol(Protocol p) noexcept { _protocol = p; } - - mbus::Route to_mbus_route() const; - Protocol getProtocol() const noexcept { return _protocol; } - uint16_t getIndex() const noexcept { return _index; } - lib::NodeType::Type getNodeType() const noexcept { return _type; } - const vespalib::string& getCluster() const noexcept { return *_cluster; } - - // Returns precomputed hash over pair. Other fields not included. - [[nodiscard]] uint32_t internal_storage_hash() const noexcept { - return _precomputed_storage_hash; - } - - bool operator==(const StorageMessageAddress& other) const noexcept; - vespalib::string toString() const; - friend std::ostream & operator << (std::ostream & os, const StorageMessageAddress & addr); - static StorageMessageAddress create(const vespalib::string * cluster, const lib::NodeType& type, uint16_t index) noexcept { - return api::StorageMessageAddress(cluster, type, index); - } - static StorageMessageAddress createDocApi(const vespalib::string * cluster, const lib::NodeType& type, uint16_t index) noexcept { - return api::StorageMessageAddress(cluster, type, index, Protocol::DOCUMENT); - } -private: - void print(vespalib::asciistream & out) const; -}; - -struct TransportContext { - virtual ~TransportContext() = 0; -}; - -enum class LockingRequirements : uint8_t { - // Operations with exclusive locking can only be executed iff no other - // exclusive or shared locks are taken for its bucket. - Exclusive = 0, - // Operations with shared locking can only be executed iff no exclusive - // lock is taken for its bucket. Should only be used for read-only operations - // that cannot mutate a bucket's state. - Shared -}; - -const char* to_string(LockingRequirements req) noexcept; -std::ostream& operator<<(std::ostream&, LockingRequirements); - -// This mirrors spi::ReadConsistency and has the same semantics, but is -// decoupled to avoid extra cross-module dependencies. -// Note that the name _internal_ read consistency is intentional to lessen -// any ambiguities on whether this is consistency in a distributed systems -// setting (i.e. linearizability) on internally in the persistence provider. -enum class InternalReadConsistency : uint8_t { - Strong = 0, - Weak -}; - -const char* to_string(InternalReadConsistency consistency) noexcept; -std::ostream& operator<<(std::ostream&, InternalReadConsistency); - -class StorageMessage : public vespalib::Printable -{ - friend class StorageMessageTest; // Used for testing only -public: - DECLARE_POINTER_TYPEDEFS(StorageMessage); - typedef uint64_t Id; - typedef uint8_t Priority; - - enum LegacyPriorityValues { - LOW = 225, - NORMAL = 127, - HIGH = 50, - VERYHIGH = 0 - }; // FIXME - //static const unsigned int NUM_PRIORITIES = UINT8_MAX; - static const char* getPriorityString(Priority); - -private: - static document::Bucket getDummyBucket() noexcept { return document::Bucket(document::BucketSpace::invalid(), document::BucketId()); } - mutable std::unique_ptr _transportContext; - -protected: - static Id generateMsgId() noexcept; - - const MessageType& _type; - Id _msgId; - StorageMessageAddress _address; - vespalib::Trace _trace; - uint32_t _approxByteSize; - Priority _priority; - - StorageMessage(const MessageType& code, Id id) noexcept; - StorageMessage(const StorageMessage&, Id id) noexcept; - -public: - StorageMessage& operator=(const StorageMessage&) = delete; - StorageMessage(const StorageMessage&) = delete; - ~StorageMessage() override; - - Id getMsgId() const noexcept { return _msgId; } - - /** Method used by storage commands to set a new id. */ - void setNewMsgId() noexcept; - - /** - * Set the id of this message. Typically used to set the id to a - * unique value previously generated with the generateMsgId method. - **/ - void forceMsgId(Id msgId) noexcept { _msgId = msgId; } - - const MessageType& getType() const noexcept { return _type; } - - void setPriority(Priority p) noexcept { _priority = p; } - Priority getPriority() const noexcept { return _priority; } - - const StorageMessageAddress* getAddress() const noexcept { return (_address.getNodeType() != lib::NodeType::Type::UNKNOWN) ? &_address : nullptr; } - - void setAddress(const StorageMessageAddress& address) noexcept { - _address = address; - } - - /** - * Returns the approximate memory footprint (in bytes) of a storage message. - * By default, returns 50 bytes. - */ - uint32_t getApproxByteSize() const noexcept { - return _approxByteSize; - } - - void setApproxByteSize(uint32_t value) noexcept { - _approxByteSize = value; - } - - /** - * Used by storage to remember the context in which this message was - * created, whether it was a storageprotocol message, a documentprotocol - * message, or an RPC call. - */ - void setTransportContext(std::unique_ptr context) noexcept { - _transportContext = std::move(context); - } - - std::unique_ptr getTransportContext() const noexcept { - return std::move(_transportContext); - } - - bool has_transport_context() const noexcept { - return (_transportContext.get() != nullptr); - } - - /** - * This method is overloaded in subclasses and will call the correct - * method in the MessageHandler interface. - */ - virtual bool callHandler(MessageHandler&, const StorageMessage::SP&) const = 0; - - mbus::Trace && steal_trace() noexcept { return std::move(_trace); } - mbus::Trace& getTrace() noexcept { return _trace; } - const mbus::Trace& getTrace() const noexcept { return _trace; } - - /** - Sets the trace object for this message. - */ - void setTrace(vespalib::Trace && trace) noexcept { _trace = std::move(trace); } - - /** - * Cheap version of tostring(). - */ - virtual vespalib::string getSummary() const; - - virtual document::Bucket getBucket() const { return getDummyBucket(); } - document::BucketId getBucketId() const noexcept { return getBucket().getBucketId(); } - virtual LockingRequirements lockingRequirements() const noexcept { - // Safe default: assume exclusive locking is required. - return LockingRequirements::Exclusive; - } -}; - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp deleted file mode 100644 index a6fefe57e08..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "storagereply.h" -#include "storagecommand.h" -#include - -namespace storage::api { - -StorageReply::StorageReply(const StorageCommand& cmd) - : StorageReply(cmd, ReturnCode()) -{} - -StorageReply::StorageReply(const StorageCommand& cmd, ReturnCode code) - : StorageMessage(cmd.getType().getReplyType(), cmd.getMsgId()), - _result(std::move(code)) -{ - setPriority(cmd.getPriority()); - if (cmd.getAddress()) { - setAddress(*cmd.getAddress()); - } - // TODD do we really need copy construction - if ( ! cmd.getTrace().isEmpty()) { - setTrace(vespalib::Trace(cmd.getTrace())); - } else { - getTrace().setLevel(cmd.getTrace().getLevel()); - } - setTransportContext(cmd.getTransportContext()); -} - -StorageReply::~StorageReply() = default; - -void -StorageReply::print(std::ostream& out, bool , const std::string& ) const -{ - out << "StorageReply(" << _type.getName() << ", " << _result << ")"; -} - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.h b/storageapi/src/vespa/storageapi/messageapi/storagereply.h deleted file mode 100644 index 9128617096d..00000000000 --- a/storageapi/src/vespa/storageapi/messageapi/storagereply.h +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::api::StorageReply - * @ingroup messageapi - * - * @brief Superclass for all storage replies. - * - * A storage reply is a storage message sent in reply to a storage command. - * - * @version $Id$ - */ - -#pragma once - -#include "returncode.h" -#include "storagemessage.h" - -namespace storage::api { - -class StorageCommand; - -class StorageReply : public StorageMessage { - ReturnCode _result; - -protected: - explicit StorageReply(const StorageCommand& cmd); - StorageReply(const StorageCommand& cmd, ReturnCode code); - -public: - ~StorageReply() override; - DECLARE_POINTER_TYPEDEFS(StorageReply); - - void setResult(ReturnCode r) { _result = std::move(r); } - void setResult(ReturnCode::Result r) { _result = ReturnCode(r); } - const ReturnCode& getResult() const { return _result; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -} -- cgit v1.2.3