diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /storageapi |
Publish
Diffstat (limited to 'storageapi')
99 files changed, 11137 insertions, 0 deletions
diff --git a/storageapi/.gitignore b/storageapi/.gitignore new file mode 100644 index 00000000000..a9b20e8992d --- /dev/null +++ b/storageapi/.gitignore @@ -0,0 +1,2 @@ +Makefile +Testing diff --git a/storageapi/CMakeLists.txt b/storageapi/CMakeLists.txt new file mode 100644 index 00000000000..bf57c21035d --- /dev/null +++ b/storageapi/CMakeLists.txt @@ -0,0 +1,27 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + DEPENDS + fastos + fnet + vespalog + vespalib + document + messagebus + vdslib + storageframework + documentapi + + LIBS + src/vespa/storageapi + src/vespa/storageapi/app + src/vespa/storageapi/buckets + src/vespa/storageapi/mbusprot + src/vespa/storageapi/message + src/vespa/storageapi/messageapi + + TESTS + src/tests + src/tests/buckets + src/tests/mbusprot + src/tests/messageapi +) diff --git a/storageapi/OWNERS b/storageapi/OWNERS new file mode 100644 index 00000000000..97c35339850 --- /dev/null +++ b/storageapi/OWNERS @@ -0,0 +1,2 @@ +vekterli +dybdahl diff --git a/storageapi/README b/storageapi/README new file mode 100644 index 00000000000..f6332c9f8f0 --- /dev/null +++ b/storageapi/README @@ -0,0 +1 @@ +This is the API needed to access the Vespa Storage System. diff --git a/storageapi/src/.gitignore b/storageapi/src/.gitignore new file mode 100644 index 00000000000..3578c0b3853 --- /dev/null +++ b/storageapi/src/.gitignore @@ -0,0 +1,6 @@ +.cvsignore +Makefile.ini +config_command.sh +doc +project.dsw +/storageapi.mak diff --git a/storageapi/src/tests/.gitignore b/storageapi/src/tests/.gitignore new file mode 100644 index 00000000000..37b74259b4a --- /dev/null +++ b/storageapi/src/tests/.gitignore @@ -0,0 +1,9 @@ +*.core +.*.swp +.config.log +.depend +Makefile +docstorage +test.vlog +testrunner +storageapi_testrunner_app diff --git a/storageapi/src/tests/CMakeLists.txt b/storageapi/src/tests/CMakeLists.txt new file mode 100644 index 00000000000..eeeb701f8b5 --- /dev/null +++ b/storageapi/src/tests/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(storageapi_testrunner_app + SOURCES + testrunner.cpp + DEPENDS + storageapi_testbuckets + storageapi_testmessageapi + storageapi_testmbusprot + storageapi +) +vespa_add_test(NAME storageapi_testrunner_app COMMAND storageapi_testrunner_app) diff --git a/storageapi/src/tests/buckets/.gitignore b/storageapi/src/tests/buckets/.gitignore new file mode 100644 index 00000000000..1d859456ef9 --- /dev/null +++ b/storageapi/src/tests/buckets/.gitignore @@ -0,0 +1,9 @@ +*.So +*.core +*.so +.*.swp +.config.log +.depend +Makefile +test.vlog +testrunner diff --git a/storageapi/src/tests/buckets/CMakeLists.txt b/storageapi/src/tests/buckets/CMakeLists.txt new file mode 100644 index 00000000000..aa25f630f53 --- /dev/null +++ b/storageapi/src/tests/buckets/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storageapi_testbuckets + SOURCES + bucketinfotest.cpp + DEPENDS + cppunit +) diff --git a/storageapi/src/tests/buckets/bucketinfotest.cpp b/storageapi/src/tests/buckets/bucketinfotest.cpp new file mode 100644 index 00000000000..bc079ce8cad --- /dev/null +++ b/storageapi/src/tests/buckets/bucketinfotest.cpp @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <vespa/storageapi/buckets/bucketinfo.h> + +namespace storage { +namespace api { + +struct BucketInfo_Test : public CppUnit::TestFixture { + + void testSimple(); + + CPPUNIT_TEST_SUITE(BucketInfo_Test); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(BucketInfo_Test); + +/** Tests simple operations */ +void BucketInfo_Test::testSimple() +{ + BucketInfo info; + + CPPUNIT_ASSERT_EQUAL(false, info.valid()); + CPPUNIT_ASSERT_EQUAL(0u, info.getChecksum()); + CPPUNIT_ASSERT_EQUAL(0u, info.getDocumentCount()); + CPPUNIT_ASSERT_EQUAL(1u, info.getTotalDocumentSize()); + + info.setChecksum(0xa000bbbb); + info.setDocumentCount(15); + info.setTotalDocumentSize(64000); + + CPPUNIT_ASSERT_EQUAL(true, info.valid()); + CPPUNIT_ASSERT_EQUAL(0xa000bbbb, info.getChecksum()); + CPPUNIT_ASSERT_EQUAL(15u, info.getDocumentCount()); + CPPUNIT_ASSERT_EQUAL(64000u, info.getTotalDocumentSize()); +}; + +} // api +} // storage diff --git a/storageapi/src/tests/mbusprot/.gitignore b/storageapi/src/tests/mbusprot/.gitignore new file mode 100644 index 00000000000..1b779fa320a --- /dev/null +++ b/storageapi/src/tests/mbusprot/.gitignore @@ -0,0 +1,11 @@ +*.So +*.core +*.so +.*.swp +.config.log +.depend +Makefile +test.vlog +testrunner +/mbusprot.4.2.serialization.HEAD +/mbusprot.5.0.serialization.5.1 diff --git a/storageapi/src/tests/mbusprot/CMakeLists.txt b/storageapi/src/tests/mbusprot/CMakeLists.txt new file mode 100644 index 00000000000..d7d376ae105 --- /dev/null +++ b/storageapi/src/tests/mbusprot/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storageapi_testmbusprot + SOURCES + storageprotocoltest.cpp + DEPENDS + vdstestlib +) diff --git a/storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE b/storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE new file mode 100644 index 00000000000..4ee14c628b4 --- /dev/null +++ b/storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE @@ -0,0 +1,70 @@ + +MessageType(10, Put) +\00\00\00 +\00\00\00j\00\08\00\00\00ddoc:test:test\00\05testdoctype1\00\00\01\00\00\00=\00\01\05\00= ;This is the contents of the test document. +It ain't much. +\00@\00\00\00\00\00\00Q\00\00\00\00\00\00\00\0e\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(11, Put Reply, reply of Put) +\00\00\00\0b\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(82, Update) +\00\00\00R\00\00\005\00\08doc:test:test\00\01testdoctype1\00\00\01\00\00\00\01\00\00\00\02\00\00\00\01\00\00\10\1b\01\00\00\00\11@\00\00\00\00\00\00Q\00\00\00\00\00\00\00\0e\00\00\00\00\00\00\00 +\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(83, Update Reply, reply of Update) +\00\00\00S\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\00\00\00\00\0e\00\00\00\00\00\00\00\08\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(4, Get) +\00\00\00\04\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\00\00\00\00{\01\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(5, Get Reply, reply of Get) +\00\00\00\05\00\00\00j\00\08\00\00\00ddoc:test:test\00\05testdoctype1\00\00\01\00\00\00=\00\01\05\00= ;This is the contents of the test document. +It ain't much. +\00\00\00\00\00\00\00\00d\00\00\00\0ddoc:test:test\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(12, Remove) +\00\00\00\0c\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\00\00\00\00\9f\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(13, Remove Reply, reply of Remove) +\00\00\00\0d\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\00\00\00\000\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(14, Revert) +\00\00\00\0e@\00\00\00\00\00\00Q\00\00\00\01\00\00\00\00\00\00\00;\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(15, Revert Reply, reply of Revert) +\00\00\00\0f@\00\00\00\00\00\00Q\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(54, Request bucket info) +\00\00\006\00\00\00\00\00\03\00\00\00\14distributor:3 .1.s:d\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(55, Request bucket info reply, reply of Request bucket info) +\00\00\007\00\00\00\01\00\00\00\00\00\00\00\04\00\00\00+\00\00\00\18\00\00\00{\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(56, Notify bucket change) +\00\00\008P\00\00\00\00\00\03\e8\00\00\00\02\00\00\00\03\00\00\00\04\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(57, Notify bucket change reply, reply of Notify bucket change) +\00\00\009\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(26, Create bucket) +\00\00\00\1a\00\00\00\00\00\00\02o\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(27, Create bucket reply, reply of Create bucket) +\00\00\00\1b\00\00\00\00\00\00\02o\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(34, Delete bucket) +\00\00\00"\00\00\00\00\00\00\02o\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(35, Delete bucket reply, reply of Delete bucket) +\00\00\00#\00\00\00\00\00\00\02o\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(32, Merge bucket) +\00\00\00 \00\00\00\00\00\00\02o\00\03\00\04\00\00\0d\01\00\1a\01\00\00\00\00\00\00\04\d2\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(33, Merge bucket reply, reply of Merge bucket) +\00\00\00!\00\00\00\00\00\00\02o\00\03\00\04\00\00\0d\01\00\1a\01\00\00\00\00\00\00\04\d2\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\04\00\00\00 +\00\00\00d +MessageType(50, GetBucketDiff) +\00\00\002\00\00\00\00\00\00\02o\00\02\00\04\00\00\0d\00\00\00\00\00\00\00\04 \00\00\00\01\00\00\00\00\00\01\e2@\00\0c1234567890ab\00\00\00d\00\01\00\00\00\01\00\03\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(51, GetBucketDiff reply, reply of GetBucketDiff) +\00\00\003\00\00\00\00\00\00\02o\00\02\00\04\00\00\0d\00\00\00\00\00\00\00\04 \00\00\00\01\00\00\00\00\00\01\e2@\00\0c1234567890ab\00\00\00d\00\01\00\00\00\01\00\03\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(52, ApplyBucketDiff) +\00\00\004@\00\00\00\00\00\02o\00\02\00\04\00\00\0d\00\00\00\04\d2\00\00\00\01\00\00\00\00\00\00\00\00\00\0c\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(53, ApplyBucketDiff reply, reply of ApplyBucketDiff) +\00\00\005@\00\00\00\00\00\02o\00\02\00\04\00\00\0d\00\00\00\04\d2\00\00\00\01\00\00\00\00\00\00\00\00\00\0c\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(66, SplitBucket) +\00\00\00B@\00\00\00\00\00\00\00\14(\00\00\03\e8\00\00\00\05\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(67, SplitBucket reply, reply of SplitBucket) +\00\00\00C@\00\00\00\00\00\00\00\00\00\00\02D\00\00\00\00\00\00\00\00\00\00d\00\00\03\e8\00\00'\10D\00\00\00\00\00\00\01\00\00\00e\00\00\03\e9\00\00'\11\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(18, Visitor Create) +\00\00\00\12\00\00\00\07library\00\00\00\02id\00\00\00\0ddoc selection\00\00\00\01\00\00\00\0bcontroldest\00\00\00\08datadest\00\00\00\02\00\00\00\00\00\00\00{\00\00\00\00\00\00\01\c8\00\00\00\02@\00\00\00\00\00\00\01@\00\00\00\00\00\00\02\00\01\01\00\00\00d\00\00\00\03\00\00\00\0dinto darkness\00\00\00\09bind them\00\00\00\08one ring\00\00\00\10to rule them all\00\00\00\0bone ring to\00\00\00\0dfind them and\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(19, Visitor Create Reply, reply of Visitor Create) +\00\00\00\13\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01 +MessageType(70, MultiOperation) +\00\00\00F\00\00'\10\01\00\00\00\00\00\00\00\00\00\00\00\a6&\00\00$\00\00\00\ca&\00\00F\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\08\00\00\00\1edoc:test:test\00\01testdoctype1\00\00\01\00\00\00=\00\01\05\00= ;This is the contents of the test document. +It ain't much. +\00\00P\00\00\f1\f1\f1\f1\f1\00\00\00\00\00\00\00\00\01\ff\ff +MessageType(71, MultiOperation Reply, reply of MultiOperation) +\00\00\00GP\00\00\f1\f1\f1\f1\f1\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01
\ No newline at end of file diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp new file mode 100644 index 00000000000..738fe9ceeee --- /dev/null +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -0,0 +1,972 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/document/base/testdocman.h> +#include <vespa/document/document.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/update/fieldpathupdates.h> +#include <iomanip> +#include <sstream> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storageapi/message/internal.h> +#include <vespa/storageapi/message/removelocation.h> +#include <vespa/storageapi/message/multioperation.h> +#include <vespa/storageapi/message/batch.h> +#include <vespa/storageapi/mbusprot/storageprotocol.h> +#include <vespa/storageapi/mbusprot/storagecommand.h> +#include <vespa/storageapi/mbusprot/storagereply.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/storageapi/message/visitor.h> + +using std::shared_ptr; +using document::ByteBuffer; +using document::Document; +using document::DocumentId; +using document::DocumentType; +using document::DocumentTypeRepo; +using storage::lib::ClusterState; +using vespalib::string; + +namespace storage { +namespace api { + +struct StorageProtocolTest : public CppUnit::TestFixture { + document::TestDocMan _docMan; + document::Document::SP _testDoc; + document::DocumentId _testDocId; + document::BucketId _bucket{16, 0x51}; + vespalib::Version _version5_0{5, 0, 12}; + vespalib::Version _version5_1{5, 1, 0}; + vespalib::Version _version5_2{5, 93, 30}; + documentapi::LoadTypeSet _loadTypes; + mbusprot::StorageProtocol _protocol; + static std::vector<std::string> _nonVerboseMessageStrings; + static std::vector<std::string> _verboseMessageStrings; + static std::vector<char> _serialization50; + static auto constexpr CONDITION_STRING = "There's just one condition"; + + StorageProtocolTest() + : _docMan(), + _testDoc(_docMan.createDocument()), + _testDocId(_testDoc->getId()), + _protocol(_docMan.getTypeRepoSP(), _loadTypes) + { + _loadTypes.addLoadType(34, "foo", documentapi::Priority::PRI_NORMAL_2); + } + + template<typename Command> + std::shared_ptr<Command> copyCommand(const std::shared_ptr<Command>&, vespalib::Version); + template<typename Reply> + std::shared_ptr<Reply> copyReply(const std::shared_ptr<Reply>&); + void recordOutput(const api::StorageMessage& msg); + + void recordSerialization50(); + + void testWriteSerialization50(); + void testAddress50(); + void testStringOutputs(); + + void testPut51(); + void testUpdate51(); + void testGet51(); + void testRemove51(); + void testRevert51(); + void testRequestBucketInfo51(); + void testNotifyBucketChange51(); + void testCreateBucket51(); + void testDeleteBucket51(); + void testMergeBucket51(); + void testGetBucketDiff51(); + void testApplyBucketDiff51(); + void testSplitBucket51(); + void testSplitBucketChain51(); + void testJoinBuckets51(); + void testMultiOperation51(); + void testBatchPutRemove51(); + void testCreateVisitor51(); + void testDestroyVisitor51(); + void testRemoveLocation51(); + void testInternalMessage(); + void testSetBucketState51(); + + void testPutCommand52(); + void testUpdateCommand52(); + void testRemoveCommand52(); + + CPPUNIT_TEST_SUITE(StorageProtocolTest); + + // Enable to see string outputs of messages + // CPPUNIT_TEST_DISABLED(testStringOutputs); + + // Enable this to write 5.0 serialization to disk + // CPPUNIT_TEST_DISABLED(testWriteSerialization50); + // CPPUNIT_TEST_DISABLED(testAddress50); + + // 5.1 tests + CPPUNIT_TEST(testPut51); + CPPUNIT_TEST(testUpdate51); + CPPUNIT_TEST(testGet51); + CPPUNIT_TEST(testRemove51); + CPPUNIT_TEST(testRevert51); + CPPUNIT_TEST(testRequestBucketInfo51); + CPPUNIT_TEST(testNotifyBucketChange51); + CPPUNIT_TEST(testCreateBucket51); + CPPUNIT_TEST(testDeleteBucket51); + CPPUNIT_TEST(testMergeBucket51); + CPPUNIT_TEST(testGetBucketDiff51); + CPPUNIT_TEST(testApplyBucketDiff51); + CPPUNIT_TEST(testSplitBucket51); + CPPUNIT_TEST(testJoinBuckets51); + CPPUNIT_TEST(testCreateVisitor51); + CPPUNIT_TEST(testDestroyVisitor51); + CPPUNIT_TEST(testRemoveLocation51); + CPPUNIT_TEST(testMultiOperation51); + CPPUNIT_TEST(testBatchPutRemove51); + CPPUNIT_TEST(testInternalMessage); + CPPUNIT_TEST(testSetBucketState51); + + // 5.2 tests + CPPUNIT_TEST(testPutCommand52); + CPPUNIT_TEST(testUpdateCommand52); + CPPUNIT_TEST(testRemoveCommand52); + + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(StorageProtocolTest); + +std::vector<std::string> StorageProtocolTest::_nonVerboseMessageStrings; +std::vector<std::string> StorageProtocolTest::_verboseMessageStrings; +std::vector<char> StorageProtocolTest::_serialization50; + +void +StorageProtocolTest::recordOutput(const api::StorageMessage& msg) +{ + std::ostringstream ost; + ost << " "; + msg.print(ost, false, " "); + _nonVerboseMessageStrings.push_back(ost.str()); + ost.str(""); + ost << " "; + msg.print(ost, true, " "); + _verboseMessageStrings.push_back(ost.str()); +} + +namespace { + + bool debug = false; + + struct ScopedName { + std::string _name; + + ScopedName(const std::string& s) : _name(s) { + if (debug) std::cerr << "Starting test " << _name << "\n"; + } + ~ScopedName() { + if (debug) std::cerr << "Finished test " << _name << "\n"; + } + }; + +} // Anonymous namespace + +namespace { + mbus::Message::UP lastCommand; + mbus::Reply::UP lastReply; +} + +void +StorageProtocolTest::testAddress50() +{ + StorageMessageAddress address("foo", lib::NodeType::STORAGE, 3); + CPPUNIT_ASSERT_EQUAL(vespalib::string("storage/cluster.foo/storage/3/default"), + address.getRoute().toString()); +} + +template<typename Command> std::shared_ptr<Command> +StorageProtocolTest::copyCommand(const std::shared_ptr<Command>& m, vespalib::Version version) +{ + mbus::Message::UP mbusMessage(new mbusprot::StorageCommand(m)); + mbus::Blob blob = _protocol.encode(version, *mbusMessage); + mbus::Routable::UP copy(_protocol.decode(version, blob)); + + CPPUNIT_ASSERT(copy.get()); + + mbusprot::StorageCommand* copy2(dynamic_cast<mbusprot::StorageCommand*>(copy.get())); + CPPUNIT_ASSERT(copy2 != 0); + + StorageCommand::SP internalMessage(copy2->getCommand()); + lastCommand = std::move(mbusMessage); + + return std::dynamic_pointer_cast<Command>(internalMessage); +} + +template<typename Reply> std::shared_ptr<Reply> +StorageProtocolTest::copyReply(const std::shared_ptr<Reply>& m) +{ + mbus::Reply::UP mbusMessage(new mbusprot::StorageReply(m)); + mbus::Blob blob = _protocol.encode(_version5_1, *mbusMessage); + mbus::Routable::UP copy(_protocol.decode(_version5_1, blob)); + CPPUNIT_ASSERT(copy.get()); + mbusprot::StorageReply* copy2( + dynamic_cast<mbusprot::StorageReply*>(copy.get())); + CPPUNIT_ASSERT(copy2 != 0); + copy2->setMessage(std::move(lastCommand)); + StorageReply::SP internalMessage(copy2->getReply()); + lastReply = std::move(mbusMessage); + lastCommand = copy2->getMessage(); + return std::dynamic_pointer_cast<Reply>(internalMessage); +} + +void +StorageProtocolTest::recordSerialization50() +{ + assert(lastCommand.get()); + assert(lastReply.get()); + for (uint32_t j=0; j<2; ++j) { + mbusprot::StorageMessage& msg(j == 0 + ? dynamic_cast<mbusprot::StorageMessage&>(*lastCommand) + : dynamic_cast<mbusprot::StorageMessage&>(*lastReply)); + msg.getInternalMessage()->forceMsgId(0); + mbus::Routable& routable(j == 0 + ? dynamic_cast<mbus::Routable&>(*lastCommand) + : dynamic_cast<mbus::Routable&>(*lastReply)); + mbus::Blob blob = _protocol.encode(_version5_0, routable); + _serialization50.push_back('\n'); + std::string type(msg.getInternalMessage()->getType().toString()); + for (uint32_t i=0, n=type.size(); i<n; ++i) { + _serialization50.push_back(type[i]); + } + _serialization50.push_back('\n'); + + for (uint32_t i=0, n=blob.size(); i<n; ++i) { + _serialization50.push_back(blob.data()[i]); + } + } +} + +void +StorageProtocolTest::testPut51() +{ + ScopedName test("testPut51"); + PutCommand::SP cmd(new PutCommand(_bucket, _testDoc, 14)); + cmd->setUpdateTimestamp(Timestamp(13)); + cmd->setLoadType(_loadTypes["foo"]); + PutCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(*_testDoc, *cmd2->getDocument()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("foo"), cmd2->getLoadType().getName()); + CPPUNIT_ASSERT_EQUAL(Timestamp(14), cmd2->getTimestamp()); + CPPUNIT_ASSERT_EQUAL(Timestamp(13), cmd2->getUpdateTimestamp()); + + PutReply::SP reply(new PutReply(*cmd2)); + CPPUNIT_ASSERT(reply->hasDocument()); + CPPUNIT_ASSERT_EQUAL(*_testDoc, *reply->getDocument()); + PutReply::SP reply2(copyReply(reply)); + CPPUNIT_ASSERT(reply2->hasDocument()); + CPPUNIT_ASSERT_EQUAL(*_testDoc, *reply->getDocument()); + CPPUNIT_ASSERT_EQUAL(_testDoc->getId(), reply2->getDocumentId()); + CPPUNIT_ASSERT_EQUAL(Timestamp(14), reply2->getTimestamp()); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testUpdate51() +{ + ScopedName test("testUpdate51"); + document::DocumentUpdate::SP update( + new document::DocumentUpdate(*_testDoc->getDataType(), + _testDoc->getId())); + std::shared_ptr<document::AssignValueUpdate> assignUpdate( + new document::AssignValueUpdate(document::IntFieldValue(17))); + document::FieldUpdate fieldUpdate(_testDoc->getField("headerval")); + fieldUpdate.addUpdate(*assignUpdate); + update->addUpdate(fieldUpdate); + + update->addFieldPathUpdate( + document::FieldPathUpdate::CP( + new document::RemoveFieldPathUpdate( + _docMan.getTypeRepo(), *_testDoc->getDataType(), + "headerval", "testdoctype1.headerval > 0"))); + + UpdateCommand::SP cmd(new UpdateCommand(_bucket, update, 14)); + CPPUNIT_ASSERT_EQUAL(Timestamp(0), cmd->getOldTimestamp()); + cmd->setOldTimestamp(10); + UpdateCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(_testDocId, cmd2->getDocumentId()); + CPPUNIT_ASSERT_EQUAL(Timestamp(14), cmd2->getTimestamp()); + CPPUNIT_ASSERT_EQUAL(Timestamp(10), cmd2->getOldTimestamp()); + CPPUNIT_ASSERT_EQUAL(*update, *cmd2->getUpdate()); + + UpdateReply::SP reply(new UpdateReply(*cmd2, 8)); + UpdateReply::SP reply2(copyReply(reply)); + CPPUNIT_ASSERT_EQUAL(_testDocId, reply2->getDocumentId()); + CPPUNIT_ASSERT_EQUAL(Timestamp(14), reply2->getTimestamp()); + CPPUNIT_ASSERT_EQUAL(Timestamp(8), reply->getOldTimestamp()); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testGet51() +{ + ScopedName test("testGet51"); + GetCommand::SP cmd(new GetCommand(_bucket, _testDocId, "foo,bar,vekterli", 123)); + GetCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(_testDocId, cmd2->getDocumentId()); + CPPUNIT_ASSERT_EQUAL(Timestamp(123), cmd2->getBeforeTimestamp()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("foo,bar,vekterli"), cmd2->getFieldSet()); + + GetReply::SP reply(new GetReply(*cmd2, _testDoc, 100)); + GetReply::SP reply2(copyReply(reply)); + CPPUNIT_ASSERT(reply2.get()); + CPPUNIT_ASSERT(reply2->getDocument().get()); + CPPUNIT_ASSERT_EQUAL(*_testDoc, *reply2->getDocument()); + CPPUNIT_ASSERT_EQUAL(_testDoc->getId(), reply2->getDocumentId()); + CPPUNIT_ASSERT_EQUAL(Timestamp(123), reply2->getBeforeTimestamp()); + CPPUNIT_ASSERT_EQUAL(Timestamp(100), reply2->getLastModifiedTimestamp()); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testRemove51() +{ + ScopedName test("testRemove51"); + RemoveCommand::SP cmd(new RemoveCommand(_bucket, _testDocId, 159)); + RemoveCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(_testDocId, cmd2->getDocumentId()); + CPPUNIT_ASSERT_EQUAL(Timestamp(159), cmd2->getTimestamp()); + + RemoveReply::SP reply(new RemoveReply(*cmd2, 48)); + reply->setBucketInfo(BucketInfo(1,2,3,4,5, true, false, 48)); + + RemoveReply::SP reply2(copyReply(reply)); + CPPUNIT_ASSERT_EQUAL(_testDocId, reply2->getDocumentId()); + CPPUNIT_ASSERT_EQUAL(Timestamp(159), reply2->getTimestamp()); + CPPUNIT_ASSERT_EQUAL(Timestamp(48), reply2->getOldTimestamp()); + CPPUNIT_ASSERT_EQUAL(BucketInfo(1,2,3,4,5, true, false, 48), + reply2->getBucketInfo()); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testRevert51() +{ + ScopedName test("testRevertCommand51"); + std::vector<Timestamp> tokens; + tokens.push_back(59); + RevertCommand::SP cmd(new RevertCommand(_bucket, tokens)); + RevertCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(tokens, cmd2->getRevertTokens()); + + RevertReply::SP reply(new RevertReply(*cmd2)); + BucketInfo info(0x12345432, 101, 520); + reply->setBucketInfo(info); + RevertReply::SP reply2(copyReply(reply)); + + CPPUNIT_ASSERT_EQUAL(info, reply2->getBucketInfo()); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testRequestBucketInfo51() +{ + ScopedName test("testRequestBucketInfo51"); + { + std::vector<document::BucketId> ids; + ids.push_back(document::BucketId(3)); + ids.push_back(document::BucketId(7)); + RequestBucketInfoCommand::SP cmd(new RequestBucketInfoCommand(ids)); + RequestBucketInfoCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(ids, cmd2->getBuckets()); + CPPUNIT_ASSERT(!cmd2->hasSystemState()); + + recordOutput(*cmd2); + } + { + ClusterState state("distributor:3 .1.s:d"); + RequestBucketInfoCommand::SP cmd(new RequestBucketInfoCommand( + 3, state, "14")); + RequestBucketInfoCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT(cmd2->hasSystemState()); + CPPUNIT_ASSERT_EQUAL(uint16_t(3), cmd2->getDistributor()); + CPPUNIT_ASSERT_EQUAL(state, cmd2->getSystemState()); + CPPUNIT_ASSERT_EQUAL(size_t(0), cmd2->getBuckets().size()); + + RequestBucketInfoReply::SP reply(new RequestBucketInfoReply(*cmd)); + RequestBucketInfoReply::Entry e; + e._bucketId = document::BucketId(4); + const uint64_t lastMod = 0x1337cafe98765432ULL; + e._info = BucketInfo(43, 24, 123, 44, 124, false, true, lastMod); + reply->getBucketInfo().push_back(e); + RequestBucketInfoReply::SP reply2(copyReply(reply)); + CPPUNIT_ASSERT_EQUAL(size_t(1), reply2->getBucketInfo().size()); + auto& entries(reply2->getBucketInfo()); + CPPUNIT_ASSERT_EQUAL(e, entries[0]); + // "Last modified" not counted by operator== for some reason. Testing + // separately until we can figure out if this is by design or not. + CPPUNIT_ASSERT_EQUAL(lastMod, entries[0]._info.getLastModified()); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); + } +} + +void +StorageProtocolTest::testNotifyBucketChange51() +{ + ScopedName test("testNotifyBucketChange51"); + BucketInfo info(2, 3, 4); + NotifyBucketChangeCommand::SP cmd(new NotifyBucketChangeCommand( + document::BucketId(20, 1000), info)); + NotifyBucketChangeCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(document::BucketId(20, 1000), + cmd2->getBucketId()); + CPPUNIT_ASSERT_EQUAL(info, cmd2->getBucketInfo()); + + NotifyBucketChangeReply::SP reply(new NotifyBucketChangeReply(*cmd)); + NotifyBucketChangeReply::SP reply2(copyReply(reply)); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testCreateBucket51() +{ + ScopedName test("testCreateBucket51"); + document::BucketId id(623); + + CreateBucketCommand::SP cmd(new CreateBucketCommand(id)); + CreateBucketCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(id, cmd2->getBucketId()); + + CreateBucketReply::SP reply(new CreateBucketReply(*cmd)); + CreateBucketReply::SP reply2(copyReply(reply)); + CPPUNIT_ASSERT_EQUAL(id, reply2->getBucketId()); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testDeleteBucket51() +{ + ScopedName test("testDeleteBucket51"); + document::BucketId id(623); + + DeleteBucketCommand::SP cmd(new DeleteBucketCommand(id)); + BucketInfo info(0x100, 200, 300); + cmd->setBucketInfo(info); + DeleteBucketCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(id, cmd2->getBucketId()); + CPPUNIT_ASSERT_EQUAL(info, cmd2->getBucketInfo()); + + DeleteBucketReply::SP reply(new DeleteBucketReply(*cmd)); + // Not set automatically by constructor + reply->setBucketInfo(cmd2->getBucketInfo()); + DeleteBucketReply::SP reply2(copyReply(reply)); + CPPUNIT_ASSERT_EQUAL(id, reply2->getBucketId()); + CPPUNIT_ASSERT_EQUAL(info, reply2->getBucketInfo()); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testMergeBucket51() +{ + ScopedName test("testMergeBucket51"); + document::BucketId id(623); + + typedef api::MergeBucketCommand::Node Node; + std::vector<Node> nodes; + nodes.push_back(Node(4, false)); + nodes.push_back(Node(13, true)); + nodes.push_back(Node(26, true)); + + std::vector<uint16_t> chain; + // Not a valid chain wrt. the nodes, but just want to have unique values + chain.push_back(7); + chain.push_back(14); + + MergeBucketCommand::SP cmd( + new MergeBucketCommand(id, nodes, Timestamp(1234), 567, chain)); + MergeBucketCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(id, cmd2->getBucketId()); + CPPUNIT_ASSERT_EQUAL(nodes, cmd2->getNodes()); + CPPUNIT_ASSERT_EQUAL(Timestamp(1234), cmd2->getMaxTimestamp()); + CPPUNIT_ASSERT_EQUAL(uint32_t(567), cmd2->getClusterStateVersion()); + CPPUNIT_ASSERT_EQUAL(chain, cmd2->getChain()); + + MergeBucketReply::SP reply(new MergeBucketReply(*cmd)); + MergeBucketReply::SP reply2(copyReply(reply)); + CPPUNIT_ASSERT_EQUAL(id, reply2->getBucketId()); + CPPUNIT_ASSERT_EQUAL(nodes, reply2->getNodes()); + CPPUNIT_ASSERT_EQUAL(Timestamp(1234), reply2->getMaxTimestamp()); + CPPUNIT_ASSERT_EQUAL(uint32_t(567), reply2->getClusterStateVersion()); + CPPUNIT_ASSERT_EQUAL(chain, reply2->getChain()); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testSplitBucket51() +{ + ScopedName test("testSplitBucket51"); + + document::BucketId bucket(16, 0); + SplitBucketCommand::SP cmd(new SplitBucketCommand(bucket)); + CPPUNIT_ASSERT_EQUAL(0u, (uint32_t) cmd->getMinSplitBits()); + CPPUNIT_ASSERT_EQUAL(58u, (uint32_t) cmd->getMaxSplitBits()); + CPPUNIT_ASSERT_EQUAL(std::numeric_limits<uint32_t>().max(), + cmd->getMinByteSize()); + CPPUNIT_ASSERT_EQUAL(std::numeric_limits<uint32_t>().max(), + cmd->getMinDocCount()); + cmd->setMinByteSize(1000); + cmd->setMinDocCount(5); + cmd->setMaxSplitBits(40); + cmd->setMinSplitBits(20); + SplitBucketCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(20u, (uint32_t) cmd2->getMinSplitBits()); + CPPUNIT_ASSERT_EQUAL(40u, (uint32_t) cmd2->getMaxSplitBits()); + CPPUNIT_ASSERT_EQUAL(1000u, cmd2->getMinByteSize()); + CPPUNIT_ASSERT_EQUAL(5u, cmd2->getMinDocCount()); + + SplitBucketReply::SP reply(new SplitBucketReply(*cmd2)); + reply->getSplitInfo().push_back(SplitBucketReply::Entry( + document::BucketId(17, 0), BucketInfo(100, 1000, 10000, true, true))); + reply->getSplitInfo().push_back(SplitBucketReply::Entry( + document::BucketId(17, 1), BucketInfo(101, 1001, 10001, true, true))); + SplitBucketReply::SP reply2(copyReply(reply)); + + CPPUNIT_ASSERT_EQUAL(bucket, reply2->getBucketId()); + CPPUNIT_ASSERT_EQUAL(size_t(2), reply2->getSplitInfo().size()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(17, 0), + reply2->getSplitInfo()[0].first); + CPPUNIT_ASSERT_EQUAL(document::BucketId(17, 1), + reply2->getSplitInfo()[1].first); + CPPUNIT_ASSERT_EQUAL(BucketInfo(100, 1000, 10000, true, true), + reply2->getSplitInfo()[0].second); + CPPUNIT_ASSERT_EQUAL(BucketInfo(101, 1001, 10001, true, true), + reply2->getSplitInfo()[1].second); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testJoinBuckets51() +{ + ScopedName test("testJoinBuckets51"); + document::BucketId bucket(16, 0); + std::vector<document::BucketId> sources; + sources.push_back(document::BucketId(17, 0)); + sources.push_back(document::BucketId(17, 1)); + JoinBucketsCommand::SP cmd(new JoinBucketsCommand(bucket)); + cmd->getSourceBuckets() = sources; + cmd->setMinJoinBits(3); + JoinBucketsCommand::SP cmd2(copyCommand(cmd, _version5_1)); + + JoinBucketsReply::SP reply(new JoinBucketsReply(*cmd2)); + reply->setBucketInfo(BucketInfo(3,4,5)); + JoinBucketsReply::SP reply2(copyReply(reply)); + + CPPUNIT_ASSERT_EQUAL(sources, reply2->getSourceBuckets()); + CPPUNIT_ASSERT_EQUAL(3, (int)cmd2->getMinJoinBits()); + CPPUNIT_ASSERT_EQUAL(BucketInfo(3,4,5), reply2->getBucketInfo()); + CPPUNIT_ASSERT_EQUAL(bucket, reply2->getBucketId()); + + recordOutput(*cmd2); + recordOutput(*reply2); +} + +void +StorageProtocolTest::testDestroyVisitor51() +{ + ScopedName test("testDestroyVisitor51"); + + DestroyVisitorCommand::SP cmd( + new DestroyVisitorCommand("instance")); + DestroyVisitorCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(string("instance"), cmd2->getInstanceId()); + + DestroyVisitorReply::SP reply(new DestroyVisitorReply(*cmd2)); + DestroyVisitorReply::SP reply2(copyReply(reply)); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testRemoveLocation51() +{ + ScopedName test("testRemoveLocation51"); + + RemoveLocationCommand::SP cmd( + new RemoveLocationCommand("id.group == \"mygroup\"", document::BucketId(16, 1234))); + RemoveLocationCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(vespalib::string("id.group == \"mygroup\""), cmd2->getDocumentSelection()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1234), cmd2->getBucketId()); + + RemoveLocationReply::SP reply(new RemoveLocationReply(*cmd2)); + RemoveLocationReply::SP reply2(copyReply(reply)); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testCreateVisitor51() +{ + ScopedName test("testCreateVisitor51"); + + std::vector<document::BucketId> buckets; + buckets.push_back(document::BucketId(16, 1)); + buckets.push_back(document::BucketId(16, 2)); + + CreateVisitorCommand::SP cmd( + new CreateVisitorCommand("library", "id", "doc selection")); + cmd->setControlDestination("controldest"); + cmd->setDataDestination("datadest"); + cmd->setVisitorCmdId(1); + cmd->getParameters().set("one ring", "to rule them all"); + cmd->getParameters().set("one ring to", "find them and"); + cmd->getParameters().set("into darkness", "bind them"); + cmd->setMaximumPendingReplyCount(2); + cmd->setFromTime(123); + cmd->setToTime(456); + cmd->getBuckets() = buckets; + cmd->setFieldSet("foo,bar,vekterli"); + cmd->setVisitInconsistentBuckets(); + cmd->setQueueTimeout(100); + cmd->setVisitorOrdering(document::OrderingSpecification::DESCENDING); + cmd->setPriority(149); + CreateVisitorCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(string("library"), cmd2->getLibraryName()); + CPPUNIT_ASSERT_EQUAL(string("id"), cmd2->getInstanceId()); + CPPUNIT_ASSERT_EQUAL(string("doc selection"), + cmd2->getDocumentSelection()); + CPPUNIT_ASSERT_EQUAL(string("controldest"), + cmd2->getControlDestination()); + CPPUNIT_ASSERT_EQUAL(string("datadest"), cmd2->getDataDestination()); + CPPUNIT_ASSERT_EQUAL(api::Timestamp(123), cmd2->getFromTime()); + CPPUNIT_ASSERT_EQUAL(api::Timestamp(456), cmd2->getToTime()); + CPPUNIT_ASSERT_EQUAL(2u, cmd2->getMaximumPendingReplyCount()); + CPPUNIT_ASSERT_EQUAL(buckets, cmd2->getBuckets()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("foo,bar,vekterli"), cmd2->getFieldSet()); + CPPUNIT_ASSERT(cmd2->visitInconsistentBuckets()); + CPPUNIT_ASSERT_EQUAL(document::OrderingSpecification::DESCENDING, cmd2->getVisitorOrdering()); + CPPUNIT_ASSERT_EQUAL(149, (int)cmd2->getPriority()); + + CreateVisitorReply::SP reply(new CreateVisitorReply(*cmd2)); + CreateVisitorReply::SP reply2(copyReply(reply)); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testGetBucketDiff51() +{ + ScopedName test("testGetBucketDiff51"); + document::BucketId id(623); + + std::vector<api::MergeBucketCommand::Node> nodes; + nodes.push_back(4); + nodes.push_back(13); + std::vector<GetBucketDiffCommand::Entry> entries; + entries.push_back(GetBucketDiffCommand::Entry()); + entries.back()._gid = document::GlobalId("1234567890abcdef"); + entries.back()._timestamp = 123456; + entries.back()._headerSize = 100; + entries.back()._bodySize = 65536; + entries.back()._flags = 1; + entries.back()._hasMask = 3; + + CPPUNIT_ASSERT_EQUAL(std::string( + "Entry(timestamp: 123456, gid(0x313233343536373839306162), " + "hasMask: 0x3,\n" + " header size: 100, body size: 65536, flags 0x1)"), + entries.back().toString(true)); + + GetBucketDiffCommand::SP cmd(new GetBucketDiffCommand(id, nodes, 1056)); + cmd->getDiff() = entries; + GetBucketDiffCommand::SP cmd2(copyCommand(cmd, _version5_1)); + + GetBucketDiffReply::SP reply(new GetBucketDiffReply(*cmd2)); + CPPUNIT_ASSERT_EQUAL(entries, reply->getDiff()); + GetBucketDiffReply::SP reply2(copyReply(reply)); + + CPPUNIT_ASSERT_EQUAL(nodes, reply2->getNodes()); + CPPUNIT_ASSERT_EQUAL(entries, reply2->getDiff()); + CPPUNIT_ASSERT_EQUAL(Timestamp(1056), reply2->getMaxTimestamp()); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testApplyBucketDiff51() +{ + ScopedName test("testApplyBucketDiff51"); + document::BucketId id(16, 623); + + std::vector<api::MergeBucketCommand::Node> nodes; + nodes.push_back(4); + nodes.push_back(13); + std::vector<ApplyBucketDiffCommand::Entry> entries; + entries.push_back(ApplyBucketDiffCommand::Entry()); + + ApplyBucketDiffCommand::SP cmd(new ApplyBucketDiffCommand(id, nodes, 1234)); + cmd->getDiff() = entries; + ApplyBucketDiffCommand::SP cmd2(copyCommand(cmd, _version5_1)); + + ApplyBucketDiffReply::SP reply(new ApplyBucketDiffReply(*cmd2)); + ApplyBucketDiffReply::SP reply2(copyReply(reply)); + + CPPUNIT_ASSERT_EQUAL(nodes, reply2->getNodes()); + CPPUNIT_ASSERT_EQUAL(entries, reply2->getDiff()); + CPPUNIT_ASSERT_EQUAL(1234u, reply2->getMaxBufferSize()); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testMultiOperation51() +{ + ScopedName test("testMultiOperation51"); + + document::BucketId bucket(20, 0xf1f1f1f1f1ull); + DocumentTypeRepo::SP repo(new DocumentTypeRepo); + MultiOperationCommand::SP + cmd(new MultiOperationCommand(repo, bucket, 10000)); + cmd->getOperations().addPut(*_testDoc); + MultiOperationCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(bucket, cmd2->getBucketId()); + CPPUNIT_ASSERT_EQUAL(*_testDoc, + *cmd2->getOperations().begin()->getDocument()); + + MultiOperationReply::SP reply(new MultiOperationReply(*cmd2)); + MultiOperationReply::SP reply2(copyReply(reply)); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +void +StorageProtocolTest::testBatchPutRemove51() +{ + ScopedName test("testBatchPutRemove51"); + + document::BucketId bucket(20, 0xf1f1f1f1f1ull); + BatchPutRemoveCommand::SP cmd(new BatchPutRemoveCommand(bucket)); + cmd->addPut(_testDoc, 100); + cmd->addHeaderUpdate(_testDoc, 101, 1234); + cmd->addRemove(_testDoc->getId(), 102); + cmd->forceMsgId(556677); + BatchPutRemoveCommand::SP cmd2(copyCommand(cmd, _version5_1)); + CPPUNIT_ASSERT_EQUAL(bucket, cmd2->getBucketId()); + CPPUNIT_ASSERT_EQUAL(3, (int)cmd2->getOperationCount()); + CPPUNIT_ASSERT_EQUAL(*_testDoc, *(dynamic_cast<const BatchPutRemoveCommand::PutOperation&>(cmd2->getOperation(0)).document)); + CPPUNIT_ASSERT_EQUAL((uint64_t)100, cmd2->getOperation(0).timestamp); + { + vespalib::nbostream header; + _testDoc->serializeHeader(header); + document::Document headerDoc(_docMan.getTypeRepo(), header); + CPPUNIT_ASSERT_EQUAL( + headerDoc, + *(dynamic_cast<const BatchPutRemoveCommand::HeaderUpdateOperation&>( + cmd2->getOperation(1)).document)); + } + CPPUNIT_ASSERT_EQUAL((uint64_t)101, cmd2->getOperation(1).timestamp); + CPPUNIT_ASSERT_EQUAL(1234, (int)dynamic_cast<const BatchPutRemoveCommand::HeaderUpdateOperation&>(cmd2->getOperation(1)).timestampToUpdate); + CPPUNIT_ASSERT_EQUAL(_testDoc->getId(), dynamic_cast<const BatchPutRemoveCommand::RemoveOperation&>(cmd2->getOperation(2)).documentId); + CPPUNIT_ASSERT_EQUAL((uint64_t)102, cmd2->getOperation(2).timestamp); + CPPUNIT_ASSERT_EQUAL(uint64_t(556677), cmd2->getMsgId()); + + BatchPutRemoveReply::SP reply(new BatchPutRemoveReply(*cmd2)); + reply->getDocumentsNotFound().push_back(document::DocumentId("userdoc:footype:1234:foo1")); + reply->getDocumentsNotFound().push_back(document::DocumentId("userdoc:footype:1234:foo2")); + reply->getDocumentsNotFound().push_back(document::DocumentId("userdoc:footype:1234:foo3")); + + BatchPutRemoveReply::SP reply2(copyReply(reply)); + + CPPUNIT_ASSERT_EQUAL(3, (int)reply2->getDocumentsNotFound().size()); + CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:footype:1234:foo1"), reply2->getDocumentsNotFound()[0]); + CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:footype:1234:foo2"), reply2->getDocumentsNotFound()[1]); + CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:footype:1234:foo3"), reply2->getDocumentsNotFound()[2]); + + recordOutput(*cmd2); + recordOutput(*reply2); + recordSerialization50(); +} + +namespace { + struct MyCommand : public api::InternalCommand { + MyCommand() : InternalCommand(101) {} + + api::StorageReply::UP makeReply(); + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const + { + out << "MyCommand()"; + if (verbose) { + out << " : "; + InternalCommand::print(out, verbose, indent); + } + } + }; + + struct MyReply : public api::InternalReply { + MyReply(const MyCommand& cmd) : InternalReply(102, cmd) {} + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const + { + out << "MyReply()"; + if (verbose) { + out << " : "; + InternalReply::print(out, verbose, indent); + } + } + }; + + api::StorageReply::UP MyCommand::makeReply() { + return api::StorageReply::UP(new MyReply(*this)); + } +} + +void +StorageProtocolTest::testInternalMessage() +{ + ScopedName test("testInternal51"); + MyCommand cmd; + MyReply reply(cmd); + + recordOutput(cmd); + recordOutput(reply); +} + +void +StorageProtocolTest::testSetBucketState51() +{ + ScopedName test("testSetBucketState51"); + document::BucketId bucket(16, 0); + SetBucketStateCommand::SP cmd( + new SetBucketStateCommand(bucket, SetBucketStateCommand::ACTIVE)); + SetBucketStateCommand::SP cmd2(copyCommand(cmd, _version5_1)); + + SetBucketStateReply::SP reply(new SetBucketStateReply(*cmd2)); + SetBucketStateReply::SP reply2(copyReply(reply)); + + CPPUNIT_ASSERT_EQUAL(SetBucketStateCommand::ACTIVE, cmd2->getState()); + CPPUNIT_ASSERT_EQUAL(bucket, cmd2->getBucketId()); + CPPUNIT_ASSERT_EQUAL(bucket, reply2->getBucketId()); + + recordOutput(*cmd2); + recordOutput(*reply2); +} + +void +StorageProtocolTest::testPutCommand52() +{ + ScopedName test("testPutCommand52"); + + PutCommand::SP cmd(new PutCommand(_bucket, _testDoc, 14)); + cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); + + PutCommand::SP cmd2(copyCommand(cmd, _version5_2)); + CPPUNIT_ASSERT_EQUAL(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); +} + +void +StorageProtocolTest::testUpdateCommand52() +{ + ScopedName test("testUpdateCommand52"); + + document::DocumentUpdate::SP update(new document::DocumentUpdate(*_testDoc->getDataType(), _testDoc->getId())); + UpdateCommand::SP cmd(new UpdateCommand(_bucket, update, 14)); + cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); + + UpdateCommand::SP cmd2(copyCommand(cmd, _version5_2)); + CPPUNIT_ASSERT_EQUAL(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); +} + +void +StorageProtocolTest::testRemoveCommand52() +{ + ScopedName test("testRemoveCommand52"); + + RemoveCommand::SP cmd(new RemoveCommand(_bucket, _testDocId, 159)); + cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); + + RemoveCommand::SP cmd2(copyCommand(cmd, _version5_2)); + CPPUNIT_ASSERT_EQUAL(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); +} + +void +StorageProtocolTest::testStringOutputs() +{ + std::cerr << "\nNon verbose output:\n"; + for (uint32_t i=0, n=_nonVerboseMessageStrings.size(); i<n; ++i) { + std::cerr << _nonVerboseMessageStrings[i] << "\n"; + } + std::cerr << "\nVerbose output:\n"; + for (uint32_t i=0, n=_verboseMessageStrings.size(); i<n; ++i) { + std::cerr << _verboseMessageStrings[i] << "\n"; + } +} + +void +StorageProtocolTest::testWriteSerialization50() +{ + std::ofstream of("mbusprot/mbusprot.5.0.serialization.5.1"); + of << std::hex << std::setfill('0'); + for (uint32_t i=0, n=_serialization50.size(); i<n; ++i) { + char c = _serialization50[i]; + if (c > 126 || (c < 32 && c != 10)) { + int32_t num = static_cast<int32_t>(c); + if (num < 0) num += 256; + of << '\\' << std::setw(2) << num; + } else if (c == '\\') { + of << "\\\\"; + } else { + of << c; + } + } + of.close(); +} + +} // mbusprot +} // storage diff --git a/storageapi/src/tests/message/.gitignore b/storageapi/src/tests/message/.gitignore new file mode 100644 index 00000000000..1d859456ef9 --- /dev/null +++ b/storageapi/src/tests/message/.gitignore @@ -0,0 +1,9 @@ +*.So +*.core +*.so +.*.swp +.config.log +.depend +Makefile +test.vlog +testrunner diff --git a/storageapi/src/tests/messageapi/.gitignore b/storageapi/src/tests/messageapi/.gitignore new file mode 100644 index 00000000000..1d859456ef9 --- /dev/null +++ b/storageapi/src/tests/messageapi/.gitignore @@ -0,0 +1,9 @@ +*.So +*.core +*.so +.*.swp +.config.log +.depend +Makefile +test.vlog +testrunner diff --git a/storageapi/src/tests/messageapi/CMakeLists.txt b/storageapi/src/tests/messageapi/CMakeLists.txt new file mode 100644 index 00000000000..b4b6669d3e5 --- /dev/null +++ b/storageapi/src/tests/messageapi/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storageapi_testmessageapi INTERFACE + SOURCES + DEPENDS +) diff --git a/storageapi/src/tests/testrunner.cpp b/storageapi/src/tests/testrunner.cpp new file mode 100644 index 00000000000..d6a42e57fb4 --- /dev/null +++ b/storageapi/src/tests/testrunner.cpp @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/vdstestlib/cppunit/cppunittestrunner.h> + +LOG_SETUP("storageapicppunittestrunner"); + +int +main(int argc, char **argv) +{ + vdstestlib::CppUnitTestRunner testRunner; + return testRunner.run(argc, argv); +} diff --git a/storageapi/src/vespa/storageapi/.gitignore b/storageapi/src/vespa/storageapi/.gitignore new file mode 100644 index 00000000000..43485abf58c --- /dev/null +++ b/storageapi/src/vespa/storageapi/.gitignore @@ -0,0 +1,5 @@ +.cvsignore +.depend +Makefile +features.h +/libstorageapi.so.5.1 diff --git a/storageapi/src/vespa/storageapi/CMakeLists.txt b/storageapi/src/vespa/storageapi/CMakeLists.txt new file mode 100644 index 00000000000..32b02e73c81 --- /dev/null +++ b/storageapi/src/vespa/storageapi/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storageapi + SOURCES + $<TARGET_OBJECTS:storageapi_message> + $<TARGET_OBJECTS:storageapi_buckets> + $<TARGET_OBJECTS:storageapi_messageapi> + $<TARGET_OBJECTS:storageapi_mbusprot> + INSTALL lib64 + DEPENDS +) diff --git a/storageapi/src/vespa/storageapi/app/.gitignore b/storageapi/src/vespa/storageapi/app/.gitignore new file mode 100644 index 00000000000..fa917bb5ae5 --- /dev/null +++ b/storageapi/src/vespa/storageapi/app/.gitignore @@ -0,0 +1,14 @@ +*.lo +.depend +.depend.NEW +.deps +.libs +Makefile +dumpstatusfile +getbucketid +inspectfeedapistatus +testdocstorefile +testdocstorefile2 +testdocstoremem +testresendctrl +storageapi_getbucketid_app diff --git a/storageapi/src/vespa/storageapi/app/CMakeLists.txt b/storageapi/src/vespa/storageapi/app/CMakeLists.txt new file mode 100644 index 00000000000..45f48e4d1a0 --- /dev/null +++ b/storageapi/src/vespa/storageapi/app/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(storageapi_getbucketid_app + SOURCES + getbucketid.cpp + INSTALL bin + DEPENDS + storageapi +) diff --git a/storageapi/src/vespa/storageapi/app/getbucketid.cpp b/storageapi/src/vespa/storageapi/app/getbucketid.cpp new file mode 100644 index 00000000000..49e6ecf6ec7 --- /dev/null +++ b/storageapi/src/vespa/storageapi/app/getbucketid.cpp @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/document/base/documentid.h> + +int main(int argc, char** argv) +{ + if (argc != 2) { + std::cerr << "Usage: getbucketid <documentid>\n"; + exit(1); + } + document::BucketIdFactory factory; + document::BucketId id = factory.getBucketId(document::DocumentId(argv[1])); + + printf("%s has bucketid %s\n", argv[1], id.toString().c_str()); +} + + diff --git a/storageapi/src/vespa/storageapi/buckets/.gitignore b/storageapi/src/vespa/storageapi/buckets/.gitignore new file mode 100644 index 00000000000..7ad21cf1c5e --- /dev/null +++ b/storageapi/src/vespa/storageapi/buckets/.gitignore @@ -0,0 +1,9 @@ +*.lo +.*.swp +.depend +.depend.NEW +.deps +.libs +Makefile +config-stor-bucketidgen.cpp +config-stor-bucketidgen.h diff --git a/storageapi/src/vespa/storageapi/buckets/CMakeLists.txt b/storageapi/src/vespa/storageapi/buckets/CMakeLists.txt new file mode 100644 index 00000000000..9e4554b26a8 --- /dev/null +++ b/storageapi/src/vespa/storageapi/buckets/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storageapi_buckets OBJECT + SOURCES + bucketinfo.cpp + DEPENDS +) diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp new file mode 100644 index 00000000000..e4c89b81f18 --- /dev/null +++ b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/buckets/bucketinfo.h> + +namespace storage { +namespace api { + +BucketInfo::BucketInfo() + : _lastModified(0), + _checksum(0), + _docCount(0), + _totDocSize(1), + _metaCount(0), + _usedFileSize(1), + _ready(false), + _active(false) +{ +} + +BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, + uint32_t totDocSize) + : _lastModified(0), + _checksum(checksum), + _docCount(docCount), + _totDocSize(totDocSize), + _metaCount(docCount), + _usedFileSize(totDocSize), + _ready(false), + _active(false) +{ +} + +BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, + uint32_t totDocSize, uint32_t metaCount, + uint32_t usedFileSize) + : _lastModified(0), + _checksum(checksum), + _docCount(docCount), + _totDocSize(totDocSize), + _metaCount(metaCount), + _usedFileSize(usedFileSize), + _ready(false), + _active(false) +{ +} + +BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, + uint32_t totDocSize, uint32_t metaCount, + uint32_t usedFileSize, + bool ready, bool active) + : _lastModified(0), + _checksum(checksum), + _docCount(docCount), + _totDocSize(totDocSize), + _metaCount(metaCount), + _usedFileSize(usedFileSize), + _ready(ready), + _active(active) +{ +} + +BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, + uint32_t totDocSize, uint32_t metaCount, + uint32_t usedFileSize, + bool ready, bool active, Timestamp lastModified) + : _lastModified(lastModified), + _checksum(checksum), + _docCount(docCount), + _totDocSize(totDocSize), + _metaCount(metaCount), + _usedFileSize(usedFileSize), + _ready(ready), + _active(active) +{ +} + +bool +BucketInfo::operator==(const BucketInfo& info) const +{ + return (_checksum == info._checksum && + _docCount == info._docCount && + _totDocSize == info._totDocSize && + _metaCount == info._metaCount && + _usedFileSize == info._usedFileSize && + _ready == info._ready && + _active == info._active); +} + +// TODO: add ready/active to printing +void +BucketInfo::print(vespalib::asciistream& out, const PrintProperties&) const +{ + out << "BucketInfo("; + if (valid()) { + std::ostringstream ost; + ost << std::hex << _checksum; + out << "crc 0x" << ost.str() + << ", docCount " << _docCount + << ", totDocSize " << _totDocSize; + if (_totDocSize != _usedFileSize) { + out << ", metaCount " << _metaCount + << ", usedFileSize " << _usedFileSize; + } + out << ", ready " << (_ready ? "true" : "false") + << ", active " << (_active ? "true" : "false"); + + if (_lastModified != 0) { + out << ", last modified " << _lastModified; + } + } else { + out << "invalid"; + } + out << ")"; +} + +void +BucketInfo::printXml(vespalib::XmlOutputStream& xos) const +{ + using namespace vespalib::xml; + xos << XmlAttribute("checksum", _checksum, XmlAttribute::HEX) + << XmlAttribute("docs", _docCount) + << XmlAttribute("size", _totDocSize) + << XmlAttribute("metacount", _metaCount) + << XmlAttribute("usedfilesize", _usedFileSize) + << XmlAttribute("ready", _ready) + << XmlAttribute("active", _active) + << XmlAttribute("lastmodified", _lastModified); +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.h b/storageapi/src/vespa/storageapi/buckets/bucketinfo.h new file mode 100644 index 00000000000..4b5a5a724d0 --- /dev/null +++ b/storageapi/src/vespa/storageapi/buckets/bucketinfo.h @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class BucketInfo + * @ingroup bucket + * + * @brief Contains metadata about a bucket. + * + * This class contains metadata about a bucket. It is used to send metadata + * within storage nodes and to distributors. + * + * @version $Id$ + */ + +#pragma once + +#include <stdint.h> +#include <vespa/storageapi/defs.h> +#include <vespa/vespalib/util/printable.h> +#include <vespa/vespalib/util/xmlserializable.h> + +namespace storage { +namespace api { + +class BucketInfo : public vespalib::AsciiPrintable +{ + Timestamp _lastModified; + uint32_t _checksum; + uint32_t _docCount; + uint32_t _totDocSize; + uint32_t _metaCount; + uint32_t _usedFileSize; + bool _ready; + bool _active; + +public: + BucketInfo(); + BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize); + BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize, + uint32_t metaCount, uint32_t usedFileSize); + BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize, + uint32_t metaCount, uint32_t usedFileSize, + bool ready, bool active); + BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize, + uint32_t metaCount, uint32_t usedFileSize, + bool ready, bool active, Timestamp lastModified); + + Timestamp getLastModified() const { return _lastModified; } + uint32_t getChecksum() const { return _checksum; } + uint32_t getDocumentCount() const { return _docCount; } + uint32_t getTotalDocumentSize() const { return _totDocSize; } + uint32_t getMetaCount() const { return _metaCount; } + uint32_t getUsedFileSize() const { return _usedFileSize; } + bool isReady() const { return _ready; } + bool isActive() const { return _active; } + + void setChecksum(uint32_t crc) { _checksum = crc; } + void setDocumentCount(uint32_t count) { _docCount = count; } + void setTotalDocumentSize(uint32_t size) { _totDocSize = size; } + void setMetaCount(uint32_t count) { _metaCount = count; } + void setUsedFileSize(uint32_t size) { _usedFileSize = size; } + void setReady(bool ready = true) { _ready = ready; } + void setActive(bool active = true) { _active = active; } + void setLastModified(Timestamp lastModified) { _lastModified = lastModified; } + + /** + * Only compare checksum, total document count and document + * size, not meta count or used file size. + */ + bool equalDocumentInfo(const BucketInfo& other) const { + return (_checksum == other._checksum + && _docCount == other._docCount + && _totDocSize == other._totDocSize); + } + + bool operator==(const BucketInfo& info) const; + bool valid() const { return (_docCount > 0 || _totDocSize == 0); } + bool empty() const { + return _metaCount == 0 && _usedFileSize == 0 && _checksum == 0; + } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const + { + vespalib::AsciiPrintable::print(out, verbose, indent); + } + virtual void print(vespalib::asciistream&, const PrintProperties&) const; + + void printXml(vespalib::XmlOutputStream&) const; +}; + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/defs.h b/storageapi/src/vespa/storageapi/defs.h new file mode 100644 index 00000000000..787e7bdfbee --- /dev/null +++ b/storageapi/src/vespa/storageapi/defs.h @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \file defs.h + * + * \brief Common declarations for the storage API code. + */ +#pragma once + +#include <vespa/fastos/fastos.h> + +namespace storage { +namespace api { + +typedef uint64_t Timestamp; +typedef uint32_t VisitorId; + +const Timestamp MAX_TIMESTAMP = (Timestamp)-1ll; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/mbusprot/.gitignore b/storageapi/src/vespa/storageapi/mbusprot/.gitignore new file mode 100644 index 00000000000..526f91c6668 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/.gitignore @@ -0,0 +1,7 @@ +*.lo +.*.swp +.depend +.depend.NEW +.deps +.libs +Makefile diff --git a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt new file mode 100644 index 00000000000..3fea219677d --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storageapi_mbusprot OBJECT + SOURCES + storagemessage.cpp + storagecommand.cpp + storagereply.cpp + protocolserialization.cpp + storageprotocol.cpp + protocolserialization4_2.cpp + protocolserialization5_0.cpp + protocolserialization5_1.cpp + protocolserialization5_2.cpp + DEPENDS +) diff --git a/storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h b/storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h new file mode 100644 index 00000000000..9d2b13bff10 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \brief Maps return code values used between 4.2 and 5.0 + */ +#pragma once + +namespace storage { +namespace mbusprot { + +int getOldErrorCode(api::ReturnCode::Result newErrorCode) { + switch (newErrorCode) { + case api::ReturnCode::OK: return 1; + case api::ReturnCode::EXISTS: return 1001; + case api::ReturnCode::NOT_READY: return 2000; + case api::ReturnCode::WRONG_DISTRIBUTION: return 2001; + case api::ReturnCode::REJECTED: return 2002; + case api::ReturnCode::ABORTED: return 2003; + case api::ReturnCode::BUCKET_NOT_FOUND: return 2004; + case api::ReturnCode::BUCKET_DELETED: return 2004; + case api::ReturnCode::TIMESTAMP_EXIST: return 2005; + case api::ReturnCode::UNKNOWN_COMMAND: return 3000; + case api::ReturnCode::NOT_IMPLEMENTED: return 3001; + case api::ReturnCode::ILLEGAL_PARAMETERS: return 3002; + case api::ReturnCode::IGNORED: return 3003; + case api::ReturnCode::UNPARSEABLE: return 3004; + case api::ReturnCode::NOT_CONNECTED: return 4000; + case api::ReturnCode::TIMEOUT: return 4003; + case api::ReturnCode::BUSY: return 4004; + case api::ReturnCode::NO_SPACE: return 5000; + case api::ReturnCode::DISK_FAILURE: return 5001; + case api::ReturnCode::IO_FAILURE: return 5003; + case api::ReturnCode::INTERNAL_FAILURE: return 6000; + default: return 6001; + } +} + +api::ReturnCode::Result getNewErrorCode(int oldErrorCode) { + switch (oldErrorCode) { + case 1: return api::ReturnCode::OK; + case 1000: return api::ReturnCode::OK; // NOT_FOUND + case 1001: return api::ReturnCode::EXISTS; + case 2000: return api::ReturnCode::NOT_READY; + case 2001: return api::ReturnCode::WRONG_DISTRIBUTION; + case 2002: return api::ReturnCode::REJECTED; + case 2003: return api::ReturnCode::ABORTED; + case 2004: return api::ReturnCode::BUCKET_NOT_FOUND; + case 2005: return api::ReturnCode::TIMESTAMP_EXIST; + case 3000: return api::ReturnCode::UNKNOWN_COMMAND; + case 3001: return api::ReturnCode::NOT_IMPLEMENTED; + case 3002: return api::ReturnCode::ILLEGAL_PARAMETERS; + case 3003: return api::ReturnCode::IGNORED; + case 3004: return api::ReturnCode::UNPARSEABLE; + case 4000: return api::ReturnCode::NOT_CONNECTED; + case 4001: return api::ReturnCode::BUSY; // OVERLOAD; + case 4002: return api::ReturnCode::NOT_READY; // REMOTE_DISABLED; + case 4003: return api::ReturnCode::TIMEOUT; + case 4004: return api::ReturnCode::BUSY; + case 5000: return api::ReturnCode::NO_SPACE; + case 5001: return api::ReturnCode::DISK_FAILURE; + case 5003: return api::ReturnCode::IO_FAILURE; + case 6000: return api::ReturnCode::INTERNAL_FAILURE; + default: return api::ReturnCode::INTERNAL_FAILURE; + } +} + +} // mbusprot +} // storage + diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp new file mode 100644 index 00000000000..1cbe0d09071 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp @@ -0,0 +1,302 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/mbusprot/protocolserialization4_2.h> + +#include <vespa/log/log.h> +#include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/multioperation.h> +#include <vespa/storageapi/message/batch.h> +#include <vespa/storageapi/message/removelocation.h> +#include <vespa/storageapi/mbusprot/serializationhelper.h> +#include <vespa/storageapi/mbusprot/storagecommand.h> +#include <vespa/storageapi/mbusprot/storagereply.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + +LOG_SETUP(".storage.api.mbusprot.serialization.base"); + +namespace storage { +namespace mbusprot { + +ProtocolSerialization::ProtocolSerialization( + const document::DocumentTypeRepo::SP& repo) + : _repo(repo) +{ +} + +mbus::Blob +ProtocolSerialization::encode(const api::StorageMessage& msg) const +{ + vespalib::GrowableByteBuffer buf; + + buf.putInt(msg.getType().getId()); + switch (msg.getType().getId()) { + case api::MessageType::PUT_ID: + onEncode(buf, static_cast<const api::PutCommand&>(msg)); + break; + case api::MessageType::PUT_REPLY_ID: + onEncode(buf, static_cast<const api::PutReply&>(msg)); + break; + case api::MessageType::UPDATE_ID: + onEncode(buf, static_cast<const api::UpdateCommand&>(msg)); + break; + case api::MessageType::UPDATE_REPLY_ID: + onEncode(buf, static_cast<const api::UpdateReply&>(msg)); + break; + case api::MessageType::GET_ID: + onEncode(buf, static_cast<const api::GetCommand&>(msg)); + break; + case api::MessageType::GET_REPLY_ID: + onEncode(buf, static_cast<const api::GetReply&>(msg)); + break; + case api::MessageType::REMOVE_ID: + onEncode(buf, static_cast<const api::RemoveCommand&>(msg)); + break; + case api::MessageType::REMOVE_REPLY_ID: + onEncode(buf, static_cast<const api::RemoveReply&>(msg)); + break; + case api::MessageType::REVERT_ID: + onEncode(buf, static_cast<const api::RevertCommand&>(msg)); + break; + case api::MessageType::REVERT_REPLY_ID: + onEncode(buf, static_cast<const api::RevertReply&>(msg)); + break; + case api::MessageType::DELETEBUCKET_ID: + onEncode(buf, static_cast<const api::DeleteBucketCommand&>(msg)); + break; + case api::MessageType::DELETEBUCKET_REPLY_ID: + onEncode(buf, static_cast<const api::DeleteBucketReply&>(msg)); + break; + case api::MessageType::CREATEBUCKET_ID: + onEncode(buf, static_cast<const api::CreateBucketCommand&>(msg)); + break; + case api::MessageType::CREATEBUCKET_REPLY_ID: + onEncode(buf, static_cast<const api::CreateBucketReply&>(msg)); + break; + case api::MessageType::MERGEBUCKET_ID: + onEncode(buf, static_cast<const api::MergeBucketCommand&>(msg)); + break; + case api::MessageType::MERGEBUCKET_REPLY_ID: + onEncode(buf, static_cast<const api::MergeBucketReply&>(msg)); + break; + case api::MessageType::GETBUCKETDIFF_ID: + onEncode(buf, static_cast<const api::GetBucketDiffCommand&>(msg)); + break; + case api::MessageType::GETBUCKETDIFF_REPLY_ID: + onEncode(buf, static_cast<const api::GetBucketDiffReply&>(msg)); + break; + case api::MessageType::APPLYBUCKETDIFF_ID: + onEncode(buf, static_cast<const api::ApplyBucketDiffCommand&>(msg)); + break; + case api::MessageType::APPLYBUCKETDIFF_REPLY_ID: + onEncode(buf, static_cast<const api::ApplyBucketDiffReply&>(msg)); + break; + case api::MessageType::REQUESTBUCKETINFO_ID: + onEncode(buf, static_cast<const api::RequestBucketInfoCommand&>(msg)); + break; + case api::MessageType::REQUESTBUCKETINFO_REPLY_ID: + onEncode(buf, static_cast<const api::RequestBucketInfoReply&>(msg)); + break; + case api::MessageType::NOTIFYBUCKETCHANGE_ID: + onEncode(buf, static_cast<const api::NotifyBucketChangeCommand&>(msg)); + break; + case api::MessageType::NOTIFYBUCKETCHANGE_REPLY_ID: + onEncode(buf, static_cast<const api::NotifyBucketChangeReply&>(msg)); + break; + case api::MessageType::SPLITBUCKET_ID: + onEncode(buf, static_cast<const api::SplitBucketCommand&>(msg)); + break; + case api::MessageType::SPLITBUCKET_REPLY_ID: + onEncode(buf, static_cast<const api::SplitBucketReply&>(msg)); + break; + case api::MessageType::JOINBUCKETS_ID: + onEncode(buf, static_cast<const api::JoinBucketsCommand&>(msg)); + break; + case api::MessageType::JOINBUCKETS_REPLY_ID: + onEncode(buf, static_cast<const api::JoinBucketsReply&>(msg)); + break; + case api::MessageType::MULTIOPERATION_ID: + onEncode(buf, static_cast<const api::MultiOperationCommand&>(msg)); + break; + case api::MessageType::MULTIOPERATION_REPLY_ID: + onEncode(buf, static_cast<const api::MultiOperationReply&>(msg)); + break; + case api::MessageType::VISITOR_CREATE_ID: + onEncode(buf, static_cast<const api::CreateVisitorCommand&>(msg)); + break; + case api::MessageType::VISITOR_CREATE_REPLY_ID: + onEncode(buf, static_cast<const api::CreateVisitorReply&>(msg)); + break; + case api::MessageType::VISITOR_DESTROY_ID: + onEncode(buf, static_cast<const api::DestroyVisitorCommand&>(msg)); + break; + case api::MessageType::VISITOR_DESTROY_REPLY_ID: + onEncode(buf, static_cast<const api::DestroyVisitorReply&>(msg)); + break; + case api::MessageType::REMOVELOCATION_ID: + onEncode(buf, static_cast<const api::RemoveLocationCommand&>(msg)); + break; + case api::MessageType::REMOVELOCATION_REPLY_ID: + onEncode(buf, static_cast<const api::RemoveLocationReply&>(msg)); + break; + case api::MessageType::BATCHPUTREMOVE_ID: + onEncode(buf, static_cast<const api::BatchPutRemoveCommand&>(msg)); + break; + case api::MessageType::BATCHPUTREMOVE_REPLY_ID: + onEncode(buf, static_cast<const api::BatchPutRemoveReply&>(msg)); + break; + case api::MessageType::SETBUCKETSTATE_ID: + onEncode(buf, static_cast<const api::SetBucketStateCommand&>(msg)); + break; + case api::MessageType::SETBUCKETSTATE_REPLY_ID: + onEncode(buf, static_cast<const api::SetBucketStateReply&>(msg)); + break; + default: + LOG(error, "Trying to encode unhandled type %s", + msg.getType().toString().c_str()); + break; + } + + mbus::Blob retVal(buf.position()); + memcpy(retVal.data(), buf.getBuffer(), buf.position()); + return retVal; +} + +StorageCommand::UP +ProtocolSerialization::decodeCommand(mbus::BlobRef data) const +{ + LOG(spam, "Decode %d bytes of data.", data.size()); + if (data.size() < sizeof(int32_t)) { + std::ostringstream ost; + ost << "Request of size " << data.size() << " is not big enough to be " + "able to store a request."; + throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + } + + document::ByteBuffer buf(data.data(), data.size()); + int type; + buf.getIntNetwork(type); + SCmd::UP cmd; + switch (type) { + case api::MessageType::PUT_ID: + cmd = onDecodePutCommand(buf); break; + case api::MessageType::UPDATE_ID: + cmd = onDecodeUpdateCommand(buf); break; + case api::MessageType::GET_ID: + cmd = onDecodeGetCommand(buf); break; + case api::MessageType::REMOVE_ID: + cmd = onDecodeRemoveCommand(buf); break; + case api::MessageType::REVERT_ID: + cmd = onDecodeRevertCommand(buf); break; + case api::MessageType::CREATEBUCKET_ID: + cmd = onDecodeCreateBucketCommand(buf); break; + case api::MessageType::DELETEBUCKET_ID: + cmd = onDecodeDeleteBucketCommand(buf); break; + case api::MessageType::MERGEBUCKET_ID: + cmd = onDecodeMergeBucketCommand(buf); break; + case api::MessageType::GETBUCKETDIFF_ID: + cmd = onDecodeGetBucketDiffCommand(buf); break; + case api::MessageType::APPLYBUCKETDIFF_ID: + cmd = onDecodeApplyBucketDiffCommand(buf); break; + case api::MessageType::REQUESTBUCKETINFO_ID: + cmd = onDecodeRequestBucketInfoCommand(buf); break; + case api::MessageType::NOTIFYBUCKETCHANGE_ID: + cmd = onDecodeNotifyBucketChangeCommand(buf); break; + case api::MessageType::SPLITBUCKET_ID: + cmd = onDecodeSplitBucketCommand(buf); break; + case api::MessageType::JOINBUCKETS_ID: + cmd = onDecodeJoinBucketsCommand(buf); break; + case api::MessageType::MULTIOPERATION_ID: + cmd = onDecodeMultiOperationCommand(buf); break; + case api::MessageType::VISITOR_CREATE_ID: + cmd = onDecodeCreateVisitorCommand(buf); break; + case api::MessageType::VISITOR_DESTROY_ID: + cmd = onDecodeDestroyVisitorCommand(buf); break; + case api::MessageType::REMOVELOCATION_ID: + cmd = onDecodeRemoveLocationCommand(buf); break; + case api::MessageType::BATCHPUTREMOVE_ID: + cmd = onDecodeBatchPutRemoveCommand(buf); break; + case api::MessageType::SETBUCKETSTATE_ID: + cmd = onDecodeSetBucketStateCommand(buf); break; + default: + { + std::ostringstream ost; + ost << "Unknown storage command type " << type; + throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + } + } + return StorageCommand::UP(new StorageCommand(SCmd::SP(cmd.release()))); +} + +StorageReply::UP +ProtocolSerialization::decodeReply(mbus::BlobRef data, + const api::StorageCommand& cmd) const +{ + LOG(spam, "Decode %d bytes of data.", data.size()); + if (data.size() < sizeof(int32_t)) { + std::ostringstream ost; + ost << "Request of size " << data.size() << " is not big enough to be " + "able to store a request."; + throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + } + + document::ByteBuffer buf(data.data(), data.size()); + int type; + buf.getIntNetwork(type); + SRep::UP reply; + switch (type) { + case api::MessageType::PUT_REPLY_ID: + reply = onDecodePutReply(cmd, buf); break; + case api::MessageType::UPDATE_REPLY_ID: + reply = onDecodeUpdateReply(cmd, buf); break; + case api::MessageType::GET_REPLY_ID: + reply = onDecodeGetReply(cmd, buf); break; + case api::MessageType::REMOVE_REPLY_ID: + reply = onDecodeRemoveReply(cmd, buf); break; + case api::MessageType::REVERT_REPLY_ID: + reply = onDecodeRevertReply(cmd, buf); break; + case api::MessageType::CREATEBUCKET_REPLY_ID: + reply = onDecodeCreateBucketReply(cmd, buf); break; + case api::MessageType::DELETEBUCKET_REPLY_ID: + reply = onDecodeDeleteBucketReply(cmd, buf); break; + case api::MessageType::MERGEBUCKET_REPLY_ID: + reply = onDecodeMergeBucketReply(cmd, buf); break; + case api::MessageType::GETBUCKETDIFF_REPLY_ID: + reply = onDecodeGetBucketDiffReply(cmd, buf); break; + case api::MessageType::APPLYBUCKETDIFF_REPLY_ID: + reply = onDecodeApplyBucketDiffReply(cmd, buf); break; + case api::MessageType::REQUESTBUCKETINFO_REPLY_ID: + reply = onDecodeRequestBucketInfoReply(cmd, buf); break; + case api::MessageType::NOTIFYBUCKETCHANGE_REPLY_ID: + reply = onDecodeNotifyBucketChangeReply(cmd, buf); break; + case api::MessageType::SPLITBUCKET_REPLY_ID: + reply = onDecodeSplitBucketReply(cmd, buf); break; + case api::MessageType::JOINBUCKETS_REPLY_ID: + reply = onDecodeJoinBucketsReply(cmd, buf); break; + case api::MessageType::MULTIOPERATION_REPLY_ID: + reply = onDecodeMultiOperationReply(cmd, buf); break; + case api::MessageType::VISITOR_CREATE_REPLY_ID: + reply = onDecodeCreateVisitorReply(cmd, buf); break; + case api::MessageType::VISITOR_DESTROY_REPLY_ID: + reply = onDecodeDestroyVisitorReply(cmd, buf); break; + case api::MessageType::REMOVELOCATION_REPLY_ID: + reply = onDecodeRemoveLocationReply(cmd, buf); break; + case api::MessageType::BATCHPUTREMOVE_REPLY_ID: + reply = onDecodeBatchPutRemoveReply(cmd, buf); break; + case api::MessageType::SETBUCKETSTATE_REPLY_ID: + reply = onDecodeSetBucketStateReply(cmd, buf); break; + default: + { + std::ostringstream ost; + ost << "Unknown message type " << type; + throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + } + } + return StorageReply::UP(new StorageReply(SRep::SP(reply.release()))); +} + +} // mbusprot +} // storage diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h new file mode 100644 index 00000000000..c7740a84938 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h @@ -0,0 +1,186 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/messagebus/routable.h> +#include <vespa/storageapi/mbusprot/storagemessage.h> +#include <vespa/storageapi/message/bucket.h> + +namespace document { + class ByteBuffer; +} +namespace mbus { + class Blob; + class BlobRef; +} +namespace vespalib { + class GrowableByteBuffer; +} +namespace storage { +namespace api { +class StorageCommand; +class StorageReply; +class PutCommand; +class PutReply; +class GetCommand; +class GetReply; +class RemoveCommand; +class RemoveReply; +class RevertCommand; +class RevertReply; +class DeleteBucketCommand; +class DeleteBucketReply; +class CreateBucketCommand; +class CreateBucketReply; +class MergeBucketCommand; +class MergeBucketReply; +class GetBucketDiffCommand; +class GetBucketDiffReply; +class ApplyBucketDiffCommand; +class ApplyBucketDiffReply; +class RequestBucketInfoCommand; +class RequestBucketInfoReply; +class NotifyBucketChangeCommand; +class NotifyBucketChangeReply; +class SplitBucketCommand; +class SplitBucketReply; +class JoinBucketsCommand; +class JoinBucketsReply; +class SetBucketStateCommand; +class SetBucketStateReply; +class MultiOperationCommand; +class MultiOperationReply; +class CreateVisitorCommand; +class RemoveLocationCommand; +class RemoveLocationReply; +class BatchPutRemoveCommand; +class BatchPutRemoveReply; +class BatchDocumentUpdateCommand; +class BatchDocumentUpdateReply; +} + +namespace mbusprot { + +class SerializationHelper; +class StorageCommand; +class StorageReply; + +class ProtocolSerialization { + const document::DocumentTypeRepo::SP _repo; + +public: + virtual mbus::Blob encode(const api::StorageMessage&) const; + virtual std::unique_ptr<StorageCommand> decodeCommand(mbus::BlobRef) const; + virtual std::unique_ptr<StorageReply> decodeReply( + mbus::BlobRef, const api::StorageCommand&) const; + +protected: + const document::DocumentTypeRepo& getTypeRepo() const { return *_repo; } + const document::DocumentTypeRepo::SP getTypeRepoSp() const + { return _repo; } + + ProtocolSerialization(const document::DocumentTypeRepo::SP &repo); + virtual ~ProtocolSerialization() {} + + typedef api::StorageCommand SCmd; + typedef api::StorageReply SRep; + typedef document::ByteBuffer BBuf; + typedef vespalib::GrowableByteBuffer GBBuf; + typedef SerializationHelper SH; + typedef StorageMessage SM; + + virtual void onEncode(GBBuf&, const api::PutCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::PutReply&) const = 0; + virtual void onEncode(GBBuf&, const api::UpdateCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::UpdateReply&) const = 0; + virtual void onEncode(GBBuf&, const api::GetCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::GetReply&) const = 0; + virtual void onEncode(GBBuf&, const api::RemoveCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::RemoveReply&) const = 0; + virtual void onEncode(GBBuf&, const api::RevertCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::RevertReply&) const = 0; + virtual void onEncode(GBBuf&, const api::DeleteBucketCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::DeleteBucketReply&) const = 0; + virtual void onEncode(GBBuf&, const api::CreateBucketCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::CreateBucketReply&) const = 0; + virtual void onEncode(GBBuf&, const api::MergeBucketCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::MergeBucketReply&) const = 0; + virtual void onEncode(GBBuf&, const api::GetBucketDiffCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::GetBucketDiffReply&) const = 0; + virtual void onEncode(GBBuf&, const api::ApplyBucketDiffCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::ApplyBucketDiffReply&) const = 0; + virtual void onEncode(GBBuf&, + const api::RequestBucketInfoCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::RequestBucketInfoReply&) const = 0; + virtual void onEncode(GBBuf&, + const api::NotifyBucketChangeCommand&) const = 0; + virtual void onEncode(GBBuf&, + const api::NotifyBucketChangeReply&) const = 0; + virtual void onEncode(GBBuf&, const api::SplitBucketCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::SplitBucketReply&) const = 0; + virtual void onEncode(GBBuf&, const api::JoinBucketsCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::JoinBucketsReply&) const = 0; + virtual void onEncode(GBBuf&, const api::SetBucketStateCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::SetBucketStateReply&) const = 0; + virtual void onEncode(GBBuf&, const api::MultiOperationCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::MultiOperationReply&) const = 0; + virtual void onEncode(GBBuf&, const api::CreateVisitorCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::CreateVisitorReply&) const = 0; + virtual void onEncode(GBBuf&, const api::DestroyVisitorCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::DestroyVisitorReply&) const = 0; + virtual void onEncode(GBBuf&, const api::RemoveLocationCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::RemoveLocationReply&) const = 0; + virtual void onEncode(GBBuf&, const api::BatchPutRemoveCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::BatchPutRemoveReply&) const = 0; + + virtual SCmd::UP onDecodePutCommand(BBuf&) const = 0; + virtual SRep::UP onDecodePutReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeUpdateCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeUpdateReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeGetCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeGetReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeRemoveCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeRemoveReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeRevertCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeRevertReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeDeleteBucketCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeDeleteBucketReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeCreateBucketCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeCreateBucketReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeMergeBucketCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeMergeBucketReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeGetBucketDiffCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeGetBucketDiffReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeApplyBucketDiffCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeApplyBucketDiffReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeRequestBucketInfoCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeRequestBucketInfoReply(const SCmd&, + BBuf&) const = 0; + virtual SCmd::UP onDecodeNotifyBucketChangeCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&, + BBuf&) const = 0; + virtual SCmd::UP onDecodeSplitBucketCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeSplitBucketReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeJoinBucketsCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeJoinBucketsReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeSetBucketStateCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeSetBucketStateReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeMultiOperationCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeMultiOperationReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeCreateVisitorReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeDestroyVisitorCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const = 0; + virtual SCmd::UP onDecodeBatchPutRemoveCommand(BBuf&) const = 0; + virtual SRep::UP onDecodeBatchPutRemoveReply(const SCmd&, BBuf&) const = 0; + + virtual api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const = 0; + virtual void putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const = 0; + +}; + +} // mbusprot +} // storage + diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp new file mode 100644 index 00000000000..4965a6d7df9 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp @@ -0,0 +1,705 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/mbusprot/protocolserialization4_2.h> + +#include <vespa/log/log.h> +#include <vespa/messagebus/blob.h> +#include <vespa/messagebus/blobref.h> +#include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/batch.h> +#include <vespa/storageapi/message/multioperation.h> +#include <vespa/storageapi/mbusprot/oldreturncodemapper.h> +#include <vespa/storageapi/mbusprot/serializationhelper.h> +#include <vespa/storageapi/mbusprot/storagecommand.h> +#include <vespa/storageapi/mbusprot/storagereply.h> +#include <vespa/storageapi/mbusprot/storageprotocol.h> +#include <vespa/storageapi/message/removelocation.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + +LOG_SETUP(".storage.api.mbusprot.serialization.4_2"); + +namespace storage { +namespace mbusprot { + +ProtocolSerialization4_2::ProtocolSerialization4_2( + const document::DocumentTypeRepo::SP& repo) + : ProtocolSerialization(repo) +{ +} + +void ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::MultiOperationCommand& msg) const +{ + uint64_t docBlockSize = msg.getOperations().spaceNeeded(); + buf.putInt(docBlockSize); + char* pos = buf.allocate(docBlockSize); + vdslib::DocumentList copy(msg.getOperations(), pos, docBlockSize); + buf.putBoolean(msg.keepTimeStamps()); + buf.putLong(msg.getBucketId().getRawId()); + onEncodeBucketInfoCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeMultiOperationCommand(BBuf& buf) const +{ + uint32_t length = SH::getInt(buf); + std::vector<char> buffer(length); + buf.getBytes(&buffer[0], length); + bool keepTimestamps = SH::getBoolean(buf); + document::BucketId bucket(SH::getLong(buf)); + api::MultiOperationCommand::UP msg( + new api::MultiOperationCommand(getTypeRepoSp(), + bucket, buffer, keepTimestamps)); + onDecodeBucketInfoCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void +ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::BatchPutRemoveCommand& msg) const +{ + // Serialization format - allow different types of serialization depending on source. + buf.putByte(0); + buf.putLong(msg.getBucketId().getRawId()); + buf.putInt(msg.getOperationCount()); + + for (uint32_t i = 0; i < msg.getOperationCount(); i++) { + const api::BatchPutRemoveCommand::Operation& op = msg.getOperation(i); + buf.putByte((uint8_t)op.type); + buf.putLong(op.timestamp); + + switch (op.type) { + case api::BatchPutRemoveCommand::Operation::REMOVE: + buf.putString(static_cast<const api::BatchPutRemoveCommand::RemoveOperation&>(op).documentId.toString()); + break; + case api::BatchPutRemoveCommand::Operation::HEADERUPDATE: + { + buf.putLong(static_cast<const api::BatchPutRemoveCommand::HeaderUpdateOperation&>(op).timestampToUpdate); + + vespalib::nbostream stream; + static_cast<const api::BatchPutRemoveCommand::HeaderUpdateOperation&>(op).document->serializeHeader(stream); + buf.putInt(stream.size()); + buf.putBytes(stream.peek(), stream.size()); + break; + } + case api::BatchPutRemoveCommand::Operation::PUT: + SH::putDocument(static_cast<const api::BatchPutRemoveCommand::PutOperation&>(op).document.get(), buf); + break; + } + } + onEncodeBucketInfoCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeBatchPutRemoveCommand(BBuf& buf) const +{ + SH::getByte(buf); + std::unique_ptr<api::BatchPutRemoveCommand> cmd(new api::BatchPutRemoveCommand(document::BucketId(SH::getLong(buf)))); + int length = SH::getInt(buf); + + for (int i = 0; i < length; i++) { + int type = SH::getByte(buf); + long timestamp = SH::getLong(buf); + + switch (type) { + case api::BatchPutRemoveCommand::Operation::REMOVE: + cmd->addRemove(document::DocumentId(SH::getString(buf)), timestamp); + break; + case api::BatchPutRemoveCommand::Operation::HEADERUPDATE: + { + long newTimestamp = SH::getLong(buf); + cmd->addHeaderUpdate(document::Document::SP( + SH::getDocument(buf, getTypeRepo())), + timestamp, newTimestamp); + break; + } + case api::BatchPutRemoveCommand::Operation::PUT: + cmd->addPut(document::Document::SP(SH::getDocument( + buf, getTypeRepo())), timestamp); + break; + } + } + + onDecodeBucketInfoCommand(buf, *cmd); + + return api::StorageCommand::UP(cmd.release()); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::BatchPutRemoveReply& msg) const +{ + buf.putInt(msg.getDocumentsNotFound().size()); + for (uint32_t i = 0; i < msg.getDocumentsNotFound().size(); i++) { + buf.putString(msg.getDocumentsNotFound()[i].toString()); + } + + onEncodeBucketInfoReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization4_2::onDecodeBatchPutRemoveReply(const SCmd& cmd, + BBuf& buf) const +{ + api::BatchPutRemoveReply::UP msg(new api::BatchPutRemoveReply( + static_cast<const api::BatchPutRemoveCommand&>(cmd))); + uint32_t count = SH::getInt(buf); + for (uint32_t i = 0; i < count; i++) { + msg->getDocumentsNotFound().push_back(document::DocumentId(SH::getString(buf))); + } + + onDecodeBucketInfoReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::GetCommand& msg) const +{ + buf.putString(msg.getDocumentId().toString()); + buf.putLong(msg.getBucketId().getRawId()); + buf.putLong(msg.getBeforeTimestamp()); + buf.putBoolean(msg.getFieldSet() == "[header]"); + onEncodeCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeGetCommand(BBuf& buf) const +{ + document::DocumentId did(SH::getString(buf)); + document::BucketId bucket(SH::getLong(buf)); + api::Timestamp beforeTimestamp(SH::getLong(buf)); + bool headerOnly(SH::getBoolean(buf)); + api::GetCommand::UP msg( + new api::GetCommand(bucket, did, headerOnly ? "[header]" : "[all]", beforeTimestamp)); + onDecodeCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::RemoveCommand& msg) const +{ + buf.putString(msg.getDocumentId().toString()); + buf.putLong(msg.getBucketId().getRawId()); + buf.putLong(msg.getTimestamp()); + onEncodeBucketInfoCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeRemoveCommand(BBuf& buf) const +{ + document::DocumentId did(SH::getString(buf)); + document::BucketId bucket(SH::getLong(buf)); + api::Timestamp timestamp(SH::getLong(buf)); + api::RemoveCommand::UP msg(new api::RemoveCommand(bucket, did, timestamp)); + onDecodeBucketInfoCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::RevertCommand& msg) const +{ + buf.putLong(msg.getBucketId().getRawId()); + buf.putInt(msg.getRevertTokens().size()); + for (uint32_t i=0, n=msg.getRevertTokens().size(); i<n; ++i) { + buf.putLong(msg.getRevertTokens()[i]); + } + onEncodeBucketInfoCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeRevertCommand(BBuf& buf) const +{ + document::BucketId bid(SH::getLong(buf)); + std::vector<api::Timestamp> tokens(SH::getInt(buf)); + for (uint32_t i=0, n=tokens.size(); i<n; ++i) { + tokens[i] = SH::getLong(buf); + } + api::RevertCommand::UP msg(new api::RevertCommand(bid, tokens)); + onDecodeBucketInfoCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::CreateBucketCommand& msg) const +{ + buf.putLong(msg.getBucketId().getRawId()); + onEncodeBucketInfoCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeCreateBucketCommand(BBuf& buf) const +{ + document::BucketId bid(SH::getLong(buf)); + api::CreateBucketCommand::UP msg(new api::CreateBucketCommand(bid)); + onDecodeBucketInfoCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::MergeBucketCommand& msg) const +{ + buf.putLong(msg.getBucketId().getRawId()); + const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); + buf.putShort(nodes.size()); + for (uint32_t i=0; i<nodes.size(); ++i) { + buf.putShort(nodes[i].index); + buf.putBoolean(nodes[i].sourceOnly); + } + buf.putLong(msg.getMaxTimestamp()); + onEncodeCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeMergeBucketCommand(BBuf& buf) const +{ + typedef api::MergeBucketCommand::Node Node; + document::BucketId bid(SH::getLong(buf)); + uint16_t nodeCount = SH::getShort(buf); + std::vector<Node> nodes; + nodes.reserve(nodeCount); + for (uint32_t i=0; i<nodeCount; ++i) { + uint16_t index(SH::getShort(buf)); + bool sourceOnly = SH::getBoolean(buf); + nodes.push_back(Node(index, sourceOnly)); + } + api::Timestamp timestamp(SH::getLong(buf)); + api::MergeBucketCommand::UP msg( + new api::MergeBucketCommand(bid, nodes, timestamp)); + onDecodeCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::GetBucketDiffCommand& msg) const +{ + buf.putLong(msg.getBucketId().getRawId()); + const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); + buf.putShort(nodes.size()); + for (uint32_t i=0; i<nodes.size(); ++i) { + buf.putShort(nodes[i].index); + buf.putBoolean(nodes[i].sourceOnly); + } + buf.putLong(msg.getMaxTimestamp()); + const std::vector<api::GetBucketDiffCommand::Entry>& entries(msg.getDiff()); + buf.putInt(entries.size()); + for (uint32_t i=0; i<entries.size(); ++i) { + onEncodeDiffEntry(buf, entries[i]); + } + onEncodeCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeGetBucketDiffCommand(BBuf& buf) const +{ + typedef api::MergeBucketCommand::Node Node; + document::BucketId bid(SH::getLong(buf)); + uint16_t nodeCount = SH::getShort(buf); + std::vector<Node> nodes; + nodes.reserve(nodeCount); + for (uint32_t i=0; i<nodeCount; ++i) { + uint16_t index(SH::getShort(buf)); + bool sourceOnly = SH::getBoolean(buf); + nodes.push_back(Node(index, sourceOnly)); + } + api::Timestamp timestamp = SH::getLong(buf); + api::GetBucketDiffCommand::UP msg( + new api::GetBucketDiffCommand(bid, nodes, timestamp)); + std::vector<api::GetBucketDiffCommand::Entry>& entries(msg->getDiff()); + uint32_t entryCount = SH::getInt(buf); + if (entryCount > buf.getRemaining()) { + // Trigger out of bounds exception rather than out of memory error + buf.incPos(entryCount); + } + entries.resize(entryCount); + for (uint32_t i=0; i<entries.size(); ++i) { + onDecodeDiffEntry(buf, entries[i]); + } + onDecodeCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const +{ + buf.putLong(msg.getBucketId().getRawId()); + const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); + buf.putShort(nodes.size()); + for (uint32_t i=0; i<nodes.size(); ++i) { + buf.putShort(nodes[i].index); + buf.putBoolean(nodes[i].sourceOnly); + } + buf.putInt(msg.getMaxBufferSize()); + const std::vector<api::ApplyBucketDiffCommand::Entry>& entries( + msg.getDiff()); + buf.putInt(entries.size()); + for (uint32_t i=0; i<entries.size(); ++i) { + onEncodeDiffEntry(buf, entries[i]._entry); + buf.putString(entries[i]._docName); + buf.putInt(entries[i]._headerBlob.size()); + buf.putBytes(&entries[i]._headerBlob[0], + entries[i]._headerBlob.size()); + buf.putInt(entries[i]._bodyBlob.size()); + buf.putBytes(&entries[i]._bodyBlob[0], + entries[i]._bodyBlob.size()); + } + onEncodeBucketInfoCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeApplyBucketDiffCommand(BBuf& buf) const +{ + typedef api::MergeBucketCommand::Node Node; + document::BucketId bid(SH::getLong(buf)); + uint16_t nodeCount = SH::getShort(buf); + std::vector<Node> nodes; + nodes.reserve(nodeCount); + for (uint32_t i=0; i<nodeCount; ++i) { + uint16_t index(SH::getShort(buf)); + bool sourceOnly = SH::getBoolean(buf); + nodes.push_back(Node(index, sourceOnly)); + } + uint32_t maxBufferSize(SH::getInt(buf)); + api::ApplyBucketDiffCommand::UP msg( + new api::ApplyBucketDiffCommand(bid, nodes, maxBufferSize)); + std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg->getDiff()); + uint32_t entryCount = SH::getInt(buf); + if (entryCount > buf.getRemaining()) { + // Trigger out of bounds exception rather than out of memory error + buf.incPos(entryCount); + } + entries.resize(entryCount); + for (uint32_t i=0; i<entries.size(); ++i) { + onDecodeDiffEntry(buf, entries[i]._entry); + entries[i]._docName = SH::getString(buf); + uint32_t headerSize = SH::getInt(buf); + if (headerSize > buf.getRemaining()) { + buf.incPos(headerSize); + } + entries[i]._headerBlob.resize(headerSize); + buf.getBytes(&entries[i]._headerBlob[0], + entries[i]._headerBlob.size()); + uint32_t bodySize = SH::getInt(buf); + if (bodySize > buf.getRemaining()) { + buf.incPos(bodySize); + } + entries[i]._bodyBlob.resize(bodySize); + buf.getBytes(&entries[i]._bodyBlob[0], + entries[i]._bodyBlob.size()); + } + onDecodeBucketInfoCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void +ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RequestBucketInfoReply& msg) const +{ + buf.putInt(msg.getBucketInfo().size()); + for (const auto & entry : msg.getBucketInfo()) { + buf.putLong(entry._bucketId.getRawId()); + putBucketInfo(entry._info, buf); + } + onEncodeReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd, + BBuf& buf) const +{ + api::RequestBucketInfoReply::UP msg(new api::RequestBucketInfoReply( + static_cast<const api::RequestBucketInfoCommand&>(cmd))); + api::RequestBucketInfoReply::EntryVector & entries(msg->getBucketInfo()); + uint32_t entryCount = SH::getInt(buf); + if (entryCount > buf.getRemaining()) { + // Trigger out of bounds exception rather than out of memory error + buf.incPos(entryCount); + } + entries.resize(entryCount); + for (auto & entry : entries) { + entry._bucketId = document::BucketId(SH::getLong(buf)); + entry._info = getBucketInfo(buf); + } + onDecodeReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const +{ + buf.putLong(msg.getBucketId().getRawId()); + putBucketInfo(msg.getBucketInfo(), buf); + onEncodeCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeNotifyBucketChangeCommand(BBuf& buf) const +{ + document::BucketId bucket(SH::getLong(buf)); + api::BucketInfo info(getBucketInfo(buf)); + api::NotifyBucketChangeCommand::UP msg( + new api::NotifyBucketChangeCommand(bucket, info)); + onDecodeCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::NotifyBucketChangeReply& msg) const +{ + onEncodeReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization4_2::onDecodeNotifyBucketChangeReply(const SCmd& cmd, + BBuf& buf) const +{ + api::NotifyBucketChangeReply::UP msg(new api::NotifyBucketChangeReply( + static_cast<const api::NotifyBucketChangeCommand&>(cmd))); + onDecodeReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::SplitBucketCommand& msg) const +{ + buf.putLong(msg.getBucketId().getRawId()); + buf.putByte(msg.getMinSplitBits()); + buf.putByte(msg.getMaxSplitBits()); + buf.putInt(msg.getMinByteSize()); + buf.putInt(msg.getMinDocCount()); + onEncodeCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeSplitBucketCommand(BBuf& buf) const +{ + document::BucketId id(SH::getLong(buf)); + api::SplitBucketCommand::UP msg(new api::SplitBucketCommand(id)); + msg->setMinSplitBits(SH::getByte(buf)); + msg->setMaxSplitBits(SH::getByte(buf)); + msg->setMinByteSize(SH::getInt(buf)); + msg->setMinDocCount(SH::getInt(buf)); + onDecodeCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf&, const api::SetBucketStateCommand&) const +{ + throw vespalib::IllegalStateException("Unsupported serialization", + VESPA_STRLOC); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeSetBucketStateCommand(BBuf&) const +{ + throw vespalib::IllegalStateException("Unsupported deserialization", + VESPA_STRLOC); +} + +void ProtocolSerialization4_2::onEncode( + GBBuf&, const api::SetBucketStateReply&) const +{ + throw vespalib::IllegalStateException("Unsupported serialization", + VESPA_STRLOC); +} + +api::StorageReply::UP +ProtocolSerialization4_2::onDecodeSetBucketStateReply(const SCmd&, + BBuf&) const +{ + throw vespalib::IllegalStateException("Unsupported deserialization", + VESPA_STRLOC); +} + +void +ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::CreateVisitorCommand& msg) const +{ + buf.putString(msg.getLibraryName()); + buf.putString(msg.getInstanceId()); + buf.putString(msg.getDocumentSelection()); + buf.putInt(msg.getVisitorCmdId()); + buf.putString(msg.getControlDestination()); + buf.putString(msg.getDataDestination()); + buf.putInt(msg.getMaximumPendingReplyCount()); + buf.putLong(msg.getFromTime()); + buf.putLong(msg.getToTime()); + + buf.putInt(msg.getBuckets().size()); + for (uint32_t i = 0; i < msg.getBuckets().size(); i++) { + buf.putLong(msg.getBuckets()[i].getRawId()); + } + + buf.putBoolean(msg.visitRemoves()); + buf.putBoolean(msg.getFieldSet() == "[header]"); + buf.putBoolean(msg.visitInconsistentBuckets()); + buf.putInt(msg.getQueueTimeout()); + + uint32_t size = msg.getParameters().getSerializedSize(); + char* docBuffer = buf.allocate(size); + document::ByteBuffer bbuf(docBuffer, size); + msg.getParameters().serialize(bbuf); + + onEncodeCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const +{ + vespalib::stringref libraryName = SH::getString(buf); + vespalib::stringref instanceId = SH::getString(buf); + vespalib::stringref selection = SH::getString(buf); + api::CreateVisitorCommand::UP msg( + new api::CreateVisitorCommand(libraryName, instanceId, selection)); + msg->setVisitorCmdId(SH::getInt(buf)); + msg->setControlDestination(SH::getString(buf)); + msg->setDataDestination(SH::getString(buf)); + msg->setMaximumPendingReplyCount(SH::getInt(buf)); + + msg->setFromTime(SH::getLong(buf)); + msg->setToTime(SH::getLong(buf)); + uint32_t count = SH::getInt(buf); + + if (count > buf.getRemaining()) { + // Trigger out of bounds exception rather than out of memory error + buf.incPos(count); + } + + for (uint32_t i = 0; i < count; i++) { + msg->getBuckets().push_back(document::BucketId(SH::getLong(buf))); + } + + if (SH::getBoolean(buf)) { + msg->setVisitRemoves(); + } + if (SH::getBoolean(buf)) { + msg->setFieldSet("[header]"); + } + if (SH::getBoolean(buf)) { + msg->setVisitInconsistentBuckets(); + } + msg->setQueueTimeout(SH::getInt(buf)); + msg->getParameters().deserialize(getTypeRepo(), buf); + + onDecodeCommand(buf, *msg); + msg->setVisitorDispatcherVersion(42); + return api::StorageCommand::UP(msg.release()); +} + +void +ProtocolSerialization4_2::onEncode( + GBBuf& buf, const api::DestroyVisitorCommand& msg) const +{ + buf.putString(msg.getInstanceId()); + onEncodeCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeDestroyVisitorCommand(BBuf& buf) const +{ + vespalib::stringref instanceId = SH::getString(buf); + api::DestroyVisitorCommand::UP msg(new api::DestroyVisitorCommand(instanceId)); + onDecodeCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void +ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::DestroyVisitorReply& msg) const +{ + onEncodeReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization4_2::onDecodeDestroyVisitorReply(const SCmd& cmd, BBuf& buf) const +{ + api::DestroyVisitorReply::UP msg(new api::DestroyVisitorReply(static_cast<const api::DestroyVisitorCommand&>(cmd))); + onDecodeReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void +ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveLocationCommand& msg) const +{ + buf.putString(msg.getDocumentSelection()); + buf.putLong(msg.getBucketId().getRawId()); + onEncodeCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization4_2::onDecodeRemoveLocationCommand(BBuf& buf) const +{ + vespalib::stringref documentSelection = SH::getString(buf); + uint64_t bucketId = SH::getLong(buf); + + api::RemoveLocationCommand::UP msg; + msg.reset(new api::RemoveLocationCommand(documentSelection, document::BucketId(bucketId))); + onDecodeCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void +ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveLocationReply& msg) const +{ + onEncodeBucketInfoReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization4_2::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const +{ + api::RemoveLocationReply::UP msg(new api::RemoveLocationReply(static_cast<const api::RemoveLocationCommand&>(cmd))); + onDecodeBucketInfoReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +// Utility functions for serialization + +void +ProtocolSerialization4_2::onEncodeBucketInfoCommand( + GBBuf& buf, const api::BucketInfoCommand& msg) const +{ + onEncodeCommand(buf, msg); +} + +void +ProtocolSerialization4_2::onDecodeBucketInfoCommand( + BBuf& buf, api::BucketInfoCommand& msg) const +{ + onDecodeCommand(buf, msg); +} + +void +ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::ReturnCode& rc) const +{ + // Convert error code to codes used in 4.2 + buf.putInt(getOldErrorCode(rc.getResult())); + buf.putString(rc.getMessage()); +} + +void +ProtocolSerialization4_2::onEncodeDiffEntry( + GBBuf& buf, const api::GetBucketDiffCommand::Entry& entry) const +{ + buf.putLong(entry._timestamp); + SH::putGlobalId(entry._gid, buf); + buf.putInt(entry._headerSize); + buf.putInt(entry._bodySize); + buf.putShort(entry._flags); + buf.putShort(entry._hasMask); +} + +void +ProtocolSerialization4_2::onDecodeDiffEntry( + BBuf& buf, api::GetBucketDiffCommand::Entry& entry) const +{ + entry._timestamp = SH::getLong(buf); + entry._gid = SH::getGlobalId(buf); + entry._headerSize = SH::getInt(buf); + entry._bodySize = SH::getInt(buf); + entry._flags = SH::getShort(buf); + entry._hasMask = SH::getShort(buf); +} + +} // mbusprot +} // storage diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h new file mode 100644 index 00000000000..97a56b5da22 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storageapi/mbusprot/protocolserialization.h> + +namespace storage { +namespace mbusprot { + +class ProtocolSerialization4_2 : public ProtocolSerialization { +public: + ProtocolSerialization4_2(const document::DocumentTypeRepo::SP&); + +protected: + virtual void onEncode(GBBuf&, const api::GetCommand&) const; + virtual void onEncode(GBBuf&, const api::RemoveCommand&) const; + virtual void onEncode(GBBuf&, const api::RevertCommand&) const; + virtual void onEncode(GBBuf&, const api::CreateBucketCommand&) const; + virtual void onEncode(GBBuf&, const api::MergeBucketCommand&) const; + virtual void onEncode(GBBuf&, const api::GetBucketDiffCommand&) const; + virtual void onEncode(GBBuf&, const api::ApplyBucketDiffCommand&) const; + virtual void onEncode(GBBuf&, const api::RequestBucketInfoReply&) const; + virtual void onEncode(GBBuf&, const api::NotifyBucketChangeCommand&) const; + virtual void onEncode(GBBuf&, const api::NotifyBucketChangeReply&) const; + virtual void onEncode(GBBuf&, const api::SplitBucketCommand&) const; + virtual void onEncode(GBBuf&, const api::MultiOperationCommand&) const; + virtual void onEncode(GBBuf&, const api::CreateVisitorCommand&) const; + virtual void onEncode(GBBuf&, const api::DestroyVisitorCommand&) const; + virtual void onEncode(GBBuf&, const api::DestroyVisitorReply&) const; + virtual void onEncode(GBBuf&, const api::RemoveLocationCommand&) const; + virtual void onEncode(GBBuf&, const api::RemoveLocationReply&) const; + + // Not supported on 4.2, but implemented here for simplicity. + virtual void onEncode(GBBuf&, const api::BatchPutRemoveCommand&) const; + virtual void onEncode(GBBuf&, const api::BatchPutRemoveReply&) const; + virtual void onEncode(GBBuf&, const api::SetBucketStateCommand&) const; + virtual void onEncode(GBBuf&, const api::SetBucketStateReply&) const; + + virtual void onEncodeBucketInfoCommand(GBBuf&, const api::BucketInfoCommand&) const; + virtual void onEncodeBucketInfoReply(GBBuf&, const api::BucketInfoReply&) const = 0; + virtual void onEncodeCommand(GBBuf&, const api::StorageCommand&) const = 0; + virtual void onEncodeReply(GBBuf&, const api::StorageReply&) const = 0; + + virtual void onEncodeDiffEntry(GBBuf&, const api::GetBucketDiffCommand::Entry&) const; + virtual void onEncode(GBBuf&, const api::ReturnCode&) const; + virtual SCmd::UP onDecodeGetCommand(BBuf&) const; + virtual SCmd::UP onDecodeRemoveCommand(BBuf&) const; + virtual SCmd::UP onDecodeRevertCommand(BBuf&) const; + virtual SCmd::UP onDecodeCreateBucketCommand(BBuf&) const; + virtual SCmd::UP onDecodeMergeBucketCommand(BBuf&) const; + virtual SCmd::UP onDecodeGetBucketDiffCommand(BBuf&) const; + virtual SCmd::UP onDecodeApplyBucketDiffCommand(BBuf&) const; + virtual SRep::UP onDecodeRequestBucketInfoReply(const SCmd&, BBuf&) const; + virtual SCmd::UP onDecodeNotifyBucketChangeCommand(BBuf&) const; + virtual SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&, BBuf&) const; + virtual SCmd::UP onDecodeSplitBucketCommand(BBuf&) const; + virtual SCmd::UP onDecodeSetBucketStateCommand(BBuf&) const; + virtual SRep::UP onDecodeSetBucketStateReply(const SCmd&, BBuf&) const; + virtual SCmd::UP onDecodeMultiOperationCommand(BBuf&) const; + virtual SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const; + virtual SCmd::UP onDecodeDestroyVisitorCommand(BBuf&) const; + virtual SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const; + virtual SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const; + virtual SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const; + + // Not supported on 4.2, but implemented here for simplicity. + virtual SCmd::UP onDecodeBatchPutRemoveCommand(BBuf&) const; + virtual SRep::UP onDecodeBatchPutRemoveReply(const SCmd&, BBuf&) const; + + virtual void onDecodeBucketInfoCommand(BBuf&, api::BucketInfoCommand&) const; + virtual void onDecodeBucketInfoReply(BBuf&, api::BucketInfoReply&) const = 0; + virtual void onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const = 0; + virtual void onDecodeReply(BBuf&, api::StorageReply&) const = 0; + + virtual void onDecodeDiffEntry(BBuf&, api::GetBucketDiffCommand::Entry&) const; +}; + +} // mbusprot +} // storage + diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp new file mode 100644 index 00000000000..1b3427b5784 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp @@ -0,0 +1,684 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/mbusprot/protocolserialization5_0.h> + +#include <vespa/log/log.h> +#include <vespa/messagebus/blob.h> +#include <vespa/messagebus/blobref.h> +#include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/multioperation.h> +#include <vespa/storageapi/mbusprot/serializationhelper.h> +#include <vespa/storageapi/mbusprot/storagecommand.h> +#include <vespa/storageapi/mbusprot/storagereply.h> +#include <vespa/storageapi/mbusprot/storageprotocol.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/document/select/orderingspecification.h> +#include <vespa/storageapi/messageapi/returncode.h> + +LOG_SETUP(".storage.api.mbusprot.serialization.5_0"); + +namespace storage { +namespace mbusprot { + +api::BucketInfo +ProtocolSerialization5_0::getBucketInfo(document::ByteBuffer& buf) const +{ + uint32_t crc(SH::getInt(buf)); + uint32_t doccount(SH::getInt(buf)); + uint32_t docsize(SH::getInt(buf)); + uint32_t metacount(SH::getInt(buf)); + uint32_t usedsize(SH::getInt(buf)); + return api::BucketInfo(crc, doccount, docsize, metacount, usedsize); +} + +void +ProtocolSerialization5_0::putBucketInfo( + const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const +{ + buf.putInt(info.getChecksum()); + buf.putInt(info.getDocumentCount()); + buf.putInt(info.getTotalDocumentSize()); + buf.putInt(info.getMetaCount()); + buf.putInt(info.getUsedFileSize()); +} + +void +ProtocolSerialization5_0::onEncodeReply( + GBBuf& buf, const api::StorageReply& msg) const +{ + SH::putReturnCode(msg.getResult(), buf); + buf.putLong(msg.getMsgId()); + buf.putByte(msg.getPriority()); +} + +void +ProtocolSerialization5_0::onDecodeReply(BBuf& buf, + api::StorageReply& msg) const +{ + msg.setResult(SH::getReturnCode(buf)); + msg.forceMsgId(SH::getLong(buf)); + msg.setPriority(SH::getByte(buf)); +} + +void +ProtocolSerialization5_0::onEncodeCommand( + GBBuf& buf, const api::StorageCommand& msg) const +{ + buf.putLong(msg.getMsgId()); + buf.putByte(msg.getPriority()); + buf.putShort(msg.getSourceIndex()); + buf.putInt(msg.getLoadType().getId()); +} + +void +ProtocolSerialization5_0::onDecodeCommand(BBuf& buf, + api::StorageCommand& msg) const +{ + msg.forceMsgId(SH::getLong(buf)); + uint8_t priority = SH::getByte(buf); + msg.setPriority(priority); + msg.setSourceIndex(SH::getShort(buf)); + msg.setLoadType(_loadTypes[SH::getInt(buf)]); +} + + +ProtocolSerialization5_0::ProtocolSerialization5_0( + const document::DocumentTypeRepo::SP& repo, + const documentapi::LoadTypeSet& loadTypes) + : ProtocolSerialization4_2(repo), + _loadTypes(loadTypes) +{ +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::PutReply& msg) const +{ + buf.putBoolean(msg.wasFound()); + onEncodeBucketInfoReply(buf, msg); +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::PutCommand& msg) const +{ + SH::putDocument(msg.getDocument().get(), buf); + buf.putLong(msg.getBucketId().getRawId()); + buf.putLong(msg.getTimestamp()); + buf.putLong(msg.getUpdateTimestamp()); + onEncodeBucketInfoCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization5_0::onDecodePutCommand(BBuf& buf) const +{ + document::Document::SP doc(SH::getDocument(buf, getTypeRepo())); + document::BucketId id(SH::getLong(buf)); + api::Timestamp ts(SH::getLong(buf)); + api::PutCommand::UP msg(new api::PutCommand(id, doc, ts)); + msg->setUpdateTimestamp(SH::getLong(buf)); + onDecodeBucketInfoCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodePutReply(const SCmd& cmd, BBuf& buf) const +{ + bool wasFound = SH::getBoolean(buf); + api::PutReply::UP msg(new api::PutReply( + static_cast<const api::PutCommand&>(cmd), wasFound)); + onDecodeBucketInfoReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::UpdateReply& msg) const +{ + buf.putLong(msg.getOldTimestamp()); + onEncodeBucketInfoReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeUpdateReply(const SCmd& cmd, BBuf& buf) const +{ + api::Timestamp oldTimestamp(SH::getLong(buf)); + api::UpdateReply::UP msg(new api::UpdateReply( + static_cast<const api::UpdateCommand&>(cmd), oldTimestamp)); + onDecodeBucketInfoReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::GetReply& msg) const +{ + SH::putDocument(msg.getDocument().get(), buf); + buf.putLong(msg.getLastModifiedTimestamp()); + onEncodeBucketInfoReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeGetReply(const SCmd& cmd, BBuf& buf) const +{ + try { + document::Document::SP doc(SH::getDocument(buf, getTypeRepo())); + api::Timestamp lastModified(SH::getLong(buf)); + api::GetReply::UP msg( + new api::GetReply( + static_cast<const api::GetCommand&>(cmd), + doc, + lastModified)); + onDecodeBucketInfoReply(buf, *msg); + return api::StorageReply::UP(msg.release()); + } catch (std::exception& e) { + api::GetReply::UP msg( + new api::GetReply( + static_cast<const api::GetCommand&>(cmd), + document::Document::SP(), + 0)); + msg->setResult(api::ReturnCode( + api::ReturnCode::UNPARSEABLE, + e.what())); + return api::StorageReply::UP(msg.release()); + } +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::RemoveReply& msg) const +{ + buf.putLong(msg.getOldTimestamp()); + onEncodeBucketInfoReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeRemoveReply(const SCmd& cmd, BBuf& buf) const +{ + api::Timestamp oldTimestamp(SH::getLong(buf)); + api::RemoveReply::UP msg(new api::RemoveReply( + static_cast<const api::RemoveCommand&>(cmd), oldTimestamp)); + onDecodeBucketInfoReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::UpdateCommand& msg) const +{ + document::DocumentUpdate* update = msg.getUpdate().get(); + if (update) { + vespalib::nbostream stream; + update->serializeHEAD(stream); + buf.putInt(stream.size()); + buf.putBytes(stream.peek(), stream.size()); + } else { + buf.putInt(0); + } + + buf.putLong(msg.getBucketId().getRawId()); + buf.putLong(msg.getTimestamp()); + buf.putLong(msg.getOldTimestamp()); + onEncodeBucketInfoCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization5_0::onDecodeUpdateCommand(BBuf& buf) const +{ + document::DocumentUpdate::SP update; + + uint32_t size = SH::getInt(buf); + if (size != 0) { + document::ByteBuffer bbuf(buf.getBufferAtPos(), size); + buf.incPos(size); + update.reset(new document::DocumentUpdate(getTypeRepo(), bbuf, + document::DocumentUpdate:: + SerializeVersion:: + SERIALIZE_HEAD)); + } + + document::BucketId bucket(SH::getLong(buf)); + api::Timestamp timestamp(SH::getLong(buf)); + api::UpdateCommand::UP msg( + new api::UpdateCommand(bucket, update, timestamp)); + msg->setOldTimestamp(SH::getLong(buf)); + onDecodeBucketInfoCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::RevertReply& msg) const +{ + onEncodeBucketInfoReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeRevertReply(const SCmd& cmd, BBuf& buf) const +{ + api::RevertReply::UP msg(new api::RevertReply( + static_cast<const api::RevertCommand&>(cmd))); + onDecodeBucketInfoReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::CreateBucketReply& msg) const +{ + onEncodeBucketInfoReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeCreateBucketReply(const SCmd& cmd, + BBuf& buf) const +{ + api::CreateBucketReply::UP msg(new api::CreateBucketReply( + static_cast<const api::CreateBucketCommand&>(cmd))); + onDecodeBucketInfoReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void +ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::DeleteBucketCommand& msg) const +{ + buf.putLong(msg.getBucketId().getRawId()); + onEncodeBucketInfoCommand(buf, msg); + putBucketInfo(msg.getBucketInfo(), buf); +} + +api::StorageCommand::UP +ProtocolSerialization5_0::onDecodeDeleteBucketCommand(BBuf& buf) const +{ + document::BucketId bid(SH::getLong(buf)); + api::DeleteBucketCommand::UP msg(new api::DeleteBucketCommand(bid)); + onDecodeBucketInfoCommand(buf, *msg); + if (buf.getRemaining() >= SH::BUCKET_INFO_SERIALIZED_SIZE) { + msg->setBucketInfo(getBucketInfo(buf)); + } + return api::StorageCommand::UP(msg.release()); +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::DeleteBucketReply& msg) const +{ + onEncodeBucketInfoReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeDeleteBucketReply(const SCmd& cmd, + BBuf& buf) const +{ + api::DeleteBucketReply::UP msg(new api::DeleteBucketReply( + static_cast<const api::DeleteBucketCommand&>(cmd))); + onDecodeBucketInfoReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::MergeBucketCommand& msg) const +{ + ProtocolSerialization4_2::onEncode(buf, msg); + + buf.putInt(msg.getClusterStateVersion()); + const std::vector<uint16_t>& chain(msg.getChain()); + buf.putShort(chain.size()); + for (std::size_t i = 0; i < chain.size(); ++i) { + buf.putShort(chain[i]); + } +} + +api::StorageCommand::UP +ProtocolSerialization5_0::onDecodeMergeBucketCommand(BBuf& buf) const +{ + api::StorageCommand::UP cmd + = ProtocolSerialization4_2::onDecodeMergeBucketCommand(buf); + uint32_t clusterStateVersion = SH::getInt(buf); + uint16_t chainSize = SH::getShort(buf); + std::vector<uint16_t> chain; + chain.reserve(chainSize); + for (std::size_t i = 0; i < chainSize; ++i) { + uint16_t index = SH::getShort(buf); + chain.push_back(index); + } + api::MergeBucketCommand& mergeCmd = static_cast<api::MergeBucketCommand&>(*cmd); + mergeCmd.setChain(chain); + mergeCmd.setClusterStateVersion(clusterStateVersion); + return cmd; +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::MergeBucketReply& msg) const +{ + onEncodeBucketReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeMergeBucketReply(const SCmd& cmd, + BBuf& buf) const +{ + api::MergeBucketReply::UP msg(new api::MergeBucketReply( + static_cast<const api::MergeBucketCommand&>(cmd))); + onDecodeBucketReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::GetBucketDiffReply& msg) const +{ + const std::vector<api::GetBucketDiffCommand::Entry>& entries(msg.getDiff()); + buf.putInt(entries.size()); + for (uint32_t i=0; i<entries.size(); ++i) { + onEncodeDiffEntry(buf, entries[i]); + } + onEncodeBucketReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeGetBucketDiffReply(const SCmd& cmd, + BBuf& buf) const +{ + api::GetBucketDiffReply::UP msg(new api::GetBucketDiffReply( + static_cast<const api::GetBucketDiffCommand&>(cmd))); + std::vector<api::GetBucketDiffCommand::Entry>& entries(msg->getDiff()); + uint32_t entryCount = SH::getInt(buf); + if (entryCount > buf.getRemaining()) { + // Trigger out of bounds exception rather than out of memory error + buf.incPos(entryCount); + } + entries.resize(entryCount); + for (uint32_t i=0; i<entries.size(); ++i) { + onDecodeDiffEntry(buf, entries[i]); + } + onDecodeBucketReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::ApplyBucketDiffReply& msg) const +{ + const std::vector<api::ApplyBucketDiffCommand::Entry>& entries( + msg.getDiff()); + buf.putInt(entries.size()); + for (uint32_t i=0; i<entries.size(); ++i) { + onEncodeDiffEntry(buf, entries[i]._entry); + buf.putString(entries[i]._docName); + buf.putInt(entries[i]._headerBlob.size()); + buf.putBytes(&entries[i]._headerBlob[0], + entries[i]._headerBlob.size()); + buf.putInt(entries[i]._bodyBlob.size()); + buf.putBytes(&entries[i]._bodyBlob[0], + entries[i]._bodyBlob.size()); + } + onEncodeBucketInfoReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeApplyBucketDiffReply(const SCmd& cmd, + BBuf& buf) const +{ + api::ApplyBucketDiffReply::UP msg(new api::ApplyBucketDiffReply( + static_cast<const api::ApplyBucketDiffCommand&>(cmd))); + std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg->getDiff()); + uint32_t entryCount = SH::getInt(buf); + if (entryCount > buf.getRemaining()) { + // Trigger out of bounds exception rather than out of memory error + buf.incPos(entryCount); + } + entries.resize(entryCount); + for (uint32_t i=0; i<entries.size(); ++i) { + onDecodeDiffEntry(buf, entries[i]._entry); + entries[i]._docName = SH::getString(buf); + uint32_t headerSize = SH::getInt(buf); + if (headerSize > buf.getRemaining()) { + buf.incPos(headerSize); + } + entries[i]._headerBlob.resize(headerSize); + buf.getBytes(&entries[i]._headerBlob[0], + entries[i]._headerBlob.size()); + uint32_t bodySize = SH::getInt(buf); + if (bodySize > buf.getRemaining()) { + buf.incPos(bodySize); + } + entries[i]._bodyBlob.resize(bodySize); + buf.getBytes(&entries[i]._bodyBlob[0], + entries[i]._bodyBlob.size()); + } + onDecodeBucketInfoReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::SplitBucketReply& msg) const +{ + const std::vector<api::SplitBucketReply::Entry>& entries( + msg.getSplitInfo()); + buf.putInt(entries.size()); + for (std::vector<api::SplitBucketReply::Entry>::const_iterator it + = entries.begin(); it != entries.end(); ++it) + { + buf.putLong(it->first.getRawId()); + putBucketInfo(it->second, buf); + } + onEncodeBucketReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeSplitBucketReply(const SCmd& cmd, + BBuf& buf) const +{ + api::SplitBucketReply::UP msg(new api::SplitBucketReply( + static_cast<const api::SplitBucketCommand&>(cmd))); + std::vector<api::SplitBucketReply::Entry>& entries(msg->getSplitInfo()); + uint32_t targetCount = SH::getInt(buf); + if (targetCount > buf.getRemaining()) { + // Trigger out of bounds exception rather than out of memory error + buf.incPos(targetCount); + } + entries.resize(targetCount); + for (std::vector<api::SplitBucketReply::Entry>::iterator it + = entries.begin(); it != entries.end(); ++it) + { + it->first = document::BucketId(SH::getLong(buf)); + it->second = getBucketInfo(buf); + } + onDecodeBucketReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::MultiOperationReply& msg) const +{ + onEncodeBucketInfoReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeMultiOperationReply(const SCmd& cmd, + BBuf& buf) const +{ + api::MultiOperationReply::UP msg(new api::MultiOperationReply( + static_cast<const api::MultiOperationCommand&>(cmd))); + onDecodeBucketInfoReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void +ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::JoinBucketsCommand& msg) const +{ + buf.putLong(msg.getBucketId().getRawId()); + buf.putInt(msg.getSourceBuckets().size()); + for (uint32_t i=0, n=msg.getSourceBuckets().size(); i<n; ++i) { + buf.putLong(msg.getSourceBuckets()[i].getRawId()); + } + buf.putByte(msg.getMinJoinBits()); + onEncodeCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization5_0::onDecodeJoinBucketsCommand(BBuf& buf) const +{ + document::BucketId bucket(SH::getLong(buf)); + api::JoinBucketsCommand::UP msg(new api::JoinBucketsCommand(bucket)); + uint32_t size = SH::getInt(buf); + if (size > buf.getRemaining()) { + // Trigger out of bounds exception rather than out of memory error + buf.incPos(size); + } + std::vector<document::BucketId>& entries(msg->getSourceBuckets()); + for (uint32_t i=0; i<size; ++i) { + entries.push_back(document::BucketId(SH::getLong(buf))); + } + msg->setMinJoinBits(SH::getByte(buf)); + onDecodeCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void +ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::JoinBucketsReply& msg) const +{ + putBucketInfo(msg.getBucketInfo(), buf); + onEncodeBucketReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeJoinBucketsReply(const SCmd& cmd, + BBuf& buf) const +{ + api::JoinBucketsReply::UP msg(new api::JoinBucketsReply( + static_cast<const api::JoinBucketsCommand&>(cmd))); + msg->setBucketInfo(getBucketInfo(buf)); + onDecodeBucketReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void +ProtocolSerialization5_0::onEncodeBucketInfoReply( + GBBuf& buf, const api::BucketInfoReply& msg) const +{ + onEncodeBucketReply(buf, msg); + putBucketInfo(msg.getBucketInfo(), buf); +} + +void +ProtocolSerialization5_0::onDecodeBucketInfoReply( + BBuf& buf, api::BucketInfoReply& msg) const +{ + onDecodeBucketReply(buf, msg); + msg.setBucketInfo(getBucketInfo(buf)); +} + +void +ProtocolSerialization5_0::onEncodeBucketReply( + GBBuf& buf, const api::BucketReply& msg) const +{ + onEncodeReply(buf, msg); + buf.putLong(msg.hasBeenRemapped() ? msg.getBucketId().getRawId() : 0); +} + +void +ProtocolSerialization5_0::onDecodeBucketReply( + BBuf& buf, api::BucketReply& msg) const +{ + onDecodeReply(buf, msg); + document::BucketId bucket(SH::getLong(buf)); + if (bucket.getRawId() != 0) { + msg.remapBucketId(bucket); + } +} + +void +ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::CreateVisitorReply& msg) const +{ + onEncodeReply(buf, msg); + buf.putInt(msg.getVisitorStatistics().getBucketsVisited()); + buf.putLong(msg.getVisitorStatistics().getDocumentsVisited()); + buf.putLong(msg.getVisitorStatistics().getBytesVisited()); + buf.putLong(msg.getVisitorStatistics().getDocumentsReturned()); + buf.putLong(msg.getVisitorStatistics().getBytesReturned()); + buf.putLong(msg.getVisitorStatistics().getSecondPassDocumentsReturned()); + buf.putLong(msg.getVisitorStatistics().getSecondPassBytesReturned()); +} + +api::StorageReply::UP +ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd, + BBuf& buf) const +{ + api::CreateVisitorReply::UP msg(new api::CreateVisitorReply( + static_cast<const api::CreateVisitorCommand&>(cmd))); + onDecodeReply(buf, *msg); + + vdslib::VisitorStatistics vs; + vs.setBucketsVisited(SH::getInt(buf)); + vs.setDocumentsVisited(SH::getLong(buf)); + vs.setBytesVisited(SH::getLong(buf)); + vs.setDocumentsReturned(SH::getLong(buf)); + vs.setBytesReturned(SH::getLong(buf)); + vs.setSecondPassDocumentsReturned(SH::getLong(buf)); + vs.setSecondPassBytesReturned(SH::getLong(buf)); + msg->setVisitorStatistics(vs); + + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization5_0::onEncode( + GBBuf& buf, const api::RequestBucketInfoCommand& msg) const +{ + const std::vector<document::BucketId>& buckets(msg.getBuckets()); + buf.putInt(buckets.size()); + for (uint32_t i=0; i<buckets.size(); ++i) { + buf.putLong(buckets[i].getRawId()); + } + if (buckets.size() == 0) { + buf.putShort(msg.getDistributor()); + buf.putString(msg.getSystemState().toString()); + buf.putString(msg.getDistributionHash()); + } + + onEncodeCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization5_0::onDecodeRequestBucketInfoCommand(BBuf& buf) const +{ + std::vector<document::BucketId> buckets(SH::getInt(buf)); + for (uint32_t i=0; i<buckets.size(); ++i) { + buckets[i] = document::BucketId(SH::getLong(buf)); + } + api::RequestBucketInfoCommand::UP msg; + if (buckets.size() != 0) { + msg.reset(new api::RequestBucketInfoCommand(buckets)); + } else { + int distributor = SH::getShort(buf); + lib::ClusterState state(SH::getString(buf)); + msg.reset(new api::RequestBucketInfoCommand(distributor, state, SH::getString(buf))); + } + onDecodeCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void +ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::CreateVisitorCommand& cmd) const +{ + ProtocolSerialization4_2::onEncode(buf, cmd); + + buf.putInt(cmd.getVisitorOrdering()); + buf.putInt(cmd.getMaxBucketsPerVisitor()); +} + +api::StorageCommand::UP +ProtocolSerialization5_0::onDecodeCreateVisitorCommand(BBuf& buf) const +{ + api::StorageCommand::UP cvc = ProtocolSerialization4_2::onDecodeCreateVisitorCommand(buf); + + static_cast<api::CreateVisitorCommand*>(cvc.get())->setVisitorOrdering( + (document::OrderingSpecification::Order)SH::getInt(buf)); + + static_cast<api::CreateVisitorCommand*>(cvc.get())->setMaxBucketsPerVisitor(SH::getInt(buf)); + + static_cast<api::CreateVisitorCommand*>(cvc.get())->setVisitorDispatcherVersion(50); + return cvc; +} + +} // mbusprot +} // storage diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h new file mode 100644 index 00000000000..61de1d97d37 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storageapi/mbusprot/protocolserialization4_2.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> + +namespace storage { +namespace mbusprot { + +class ProtocolSerialization5_0 : public ProtocolSerialization4_2 { +private: + const documentapi::LoadTypeSet& _loadTypes; + +public: + ProtocolSerialization5_0(const document::DocumentTypeRepo::SP&, + const documentapi::LoadTypeSet& loadTypes); + + virtual api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const; + virtual void putBucketInfo(const api::BucketInfo& info, + vespalib::GrowableByteBuffer& buf) const; + + virtual void onEncode(GBBuf&, const api::PutCommand&) const; + virtual void onEncode(GBBuf&, const api::PutReply&) const; + virtual void onEncode(GBBuf&, const api::UpdateCommand&) const; + virtual void onEncode(GBBuf&, const api::UpdateReply&) const; + virtual void onEncode(GBBuf&, const api::GetReply&) const; + virtual void onEncode(GBBuf&, const api::RemoveReply&) const; + virtual void onEncode(GBBuf&, const api::RevertReply&) const; + virtual void onEncode(GBBuf&, const api::CreateBucketReply&) const; + virtual void onEncode(GBBuf&, const api::DeleteBucketCommand&) const; + virtual void onEncode(GBBuf&, const api::DeleteBucketReply&) const; + virtual void onEncode(GBBuf&, const api::MergeBucketCommand&) const; + virtual void onEncode(GBBuf&, const api::MergeBucketReply&) const; + virtual void onEncode(GBBuf&, const api::GetBucketDiffReply&) const; + virtual void onEncode(GBBuf&, const api::ApplyBucketDiffReply&) const; + virtual void onEncode(GBBuf&, const api::SplitBucketReply&) const; + virtual void onEncode(GBBuf&, const api::MultiOperationReply&) const; + virtual void onEncode(GBBuf&, const api::JoinBucketsCommand&) const; + virtual void onEncode(GBBuf&, const api::JoinBucketsReply&) const; + virtual void onEncode(GBBuf&, const api::RequestBucketInfoCommand&) const; + + virtual void onEncodeBucketInfoReply(GBBuf&, const api::BucketInfoReply&) const; + virtual void onEncodeBucketReply(GBBuf&, const api::BucketReply&) const; + + virtual void onEncode(GBBuf&, const api::CreateVisitorCommand& msg) const; + virtual void onEncode(GBBuf&, const api::CreateVisitorReply& msg) const; + virtual void onEncodeCommand(GBBuf&, const api::StorageCommand&) const; + virtual void onEncodeReply(GBBuf&, const api::StorageReply&) const; + + virtual SCmd::UP onDecodePutCommand(BBuf&) const; + virtual SRep::UP onDecodePutReply(const SCmd&, BBuf&) const; + virtual SCmd::UP onDecodeUpdateCommand(BBuf&) const; + virtual SRep::UP onDecodeUpdateReply(const SCmd&, BBuf&) const; + virtual SRep::UP onDecodeGetReply(const SCmd&, BBuf&) const; + virtual SRep::UP onDecodeRemoveReply(const SCmd&, BBuf&) const; + virtual SRep::UP onDecodeRevertReply(const SCmd&, BBuf&) const; + virtual SRep::UP onDecodeCreateBucketReply(const SCmd&, BBuf&) const; + virtual SCmd::UP onDecodeDeleteBucketCommand(BBuf&) const; + virtual SRep::UP onDecodeDeleteBucketReply(const SCmd&, BBuf&) const; + virtual SCmd::UP onDecodeMergeBucketCommand(BBuf&) const; + virtual SRep::UP onDecodeMergeBucketReply(const SCmd&, BBuf&) const; + virtual SRep::UP onDecodeGetBucketDiffReply(const SCmd&, BBuf&) const; + virtual SRep::UP onDecodeApplyBucketDiffReply(const SCmd&, BBuf&) const; + virtual SRep::UP onDecodeSplitBucketReply(const SCmd&, BBuf&) const; + virtual SRep::UP onDecodeMultiOperationReply(const SCmd&, BBuf&) const; + virtual SCmd::UP onDecodeJoinBucketsCommand(BBuf& buf) const; + virtual SRep::UP onDecodeJoinBucketsReply(const SCmd& cmd, BBuf& buf) const; + virtual SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const; + virtual SCmd::UP onDecodeRequestBucketInfoCommand(BBuf& buf) const; + + virtual void onDecodeBucketInfoReply(BBuf&, api::BucketInfoReply&) const; + virtual void onDecodeBucketReply(BBuf&, api::BucketReply&) const; + virtual SRep::UP onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) const; + virtual void onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const; + virtual void onDecodeReply(BBuf&, api::StorageReply&) const; +}; + +} // mbusprot +} // storage + diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp new file mode 100644 index 00000000000..9244189da1f --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp @@ -0,0 +1,228 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/mbusprot/protocolserialization5_1.h> + +#include <vespa/log/log.h> +#include <vespa/messagebus/blob.h> +#include <vespa/messagebus/blobref.h> +#include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/multioperation.h> +#include <vespa/storageapi/mbusprot/serializationhelper.h> +#include <vespa/storageapi/mbusprot/storagecommand.h> +#include <vespa/storageapi/mbusprot/storagereply.h> +#include <vespa/storageapi/mbusprot/storageprotocol.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/document/select/orderingspecification.h> +#include <vespa/storageapi/messageapi/returncode.h> + +LOG_SETUP(".storage.api.mbusprot.serialization.5_1"); + +namespace storage { +namespace mbusprot { + +api::BucketInfo +ProtocolSerialization5_1::getBucketInfo(document::ByteBuffer& buf) const +{ + uint64_t lastModified(SH::getLong(buf)); + uint32_t crc(SH::getInt(buf)); + uint32_t doccount(SH::getInt(buf)); + uint32_t docsize(SH::getInt(buf)); + uint32_t metacount(SH::getInt(buf)); + uint32_t usedsize(SH::getInt(buf)); + uint8_t flags(SH::getByte(buf)); + bool ready = (flags & BUCKET_READY) != 0; + bool active = (flags & BUCKET_ACTIVE) != 0; + return api::BucketInfo(crc, doccount, docsize, + metacount, usedsize, + ready, active, lastModified); +} + +void +ProtocolSerialization5_1::putBucketInfo( + const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const +{ + buf.putLong(info.getLastModified()); + buf.putInt(info.getChecksum()); + buf.putInt(info.getDocumentCount()); + buf.putInt(info.getTotalDocumentSize()); + buf.putInt(info.getMetaCount()); + buf.putInt(info.getUsedFileSize()); + uint8_t flags = (info.isReady() ? BUCKET_READY : 0) | + (info.isActive() ? BUCKET_ACTIVE : 0); + buf.putByte(flags); +} + +ProtocolSerialization5_1::ProtocolSerialization5_1( + const document::DocumentTypeRepo::SP& repo, + const documentapi::LoadTypeSet& loadTypes) + : ProtocolSerialization5_0(repo, loadTypes) +{ +} + +void ProtocolSerialization5_1::onEncode( + GBBuf& buf, const api::SetBucketStateCommand& msg) const +{ + buf.putLong(msg.getBucketId().getRawId()); + buf.putByte(static_cast<uint8_t>(msg.getState())); + onEncodeCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization5_1::onDecodeSetBucketStateCommand(BBuf& buf) const +{ + document::BucketId bucket(SH::getLong(buf)); + api::SetBucketStateCommand::BUCKET_STATE state( + static_cast<api::SetBucketStateCommand::BUCKET_STATE>( + SH::getByte(buf))); + api::SetBucketStateCommand::UP msg( + new api::SetBucketStateCommand(bucket, state)); + onDecodeCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void ProtocolSerialization5_1::onEncode( + GBBuf& buf, const api::SetBucketStateReply& msg) const +{ + onEncodeBucketReply(buf, msg); +} + +api::StorageReply::UP +ProtocolSerialization5_1::onDecodeSetBucketStateReply(const SCmd& cmd, + BBuf& buf) const +{ + api::SetBucketStateReply::UP msg(new api::SetBucketStateReply( + static_cast<const api::SetBucketStateCommand&>(cmd))); + onDecodeBucketReply(buf, *msg); + return api::StorageReply::UP(msg.release()); +} + +void ProtocolSerialization5_1::onEncode( + GBBuf& buf, const api::GetCommand& msg) const +{ + buf.putString(msg.getDocumentId().toString()); + buf.putLong(msg.getBucketId().getRawId()); + buf.putLong(msg.getBeforeTimestamp()); + buf.putString(msg.getFieldSet()); + onEncodeCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization5_1::onDecodeGetCommand(BBuf& buf) const +{ + document::DocumentId did(SH::getString(buf)); + document::BucketId bucket(SH::getLong(buf)); + api::Timestamp beforeTimestamp(SH::getLong(buf)); + std::string fieldSet(SH::getString(buf)); + api::GetCommand::UP msg( + new api::GetCommand(bucket, did, fieldSet, beforeTimestamp)); + onDecodeCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + +void +ProtocolSerialization5_1::onEncode( + GBBuf& buf, const api::CreateVisitorCommand& msg) const +{ + buf.putString(msg.getLibraryName()); + buf.putString(msg.getInstanceId()); + buf.putString(msg.getDocumentSelection()); + buf.putInt(msg.getVisitorCmdId()); + buf.putString(msg.getControlDestination()); + buf.putString(msg.getDataDestination()); + buf.putInt(msg.getMaximumPendingReplyCount()); + buf.putLong(msg.getFromTime()); + buf.putLong(msg.getToTime()); + + buf.putInt(msg.getBuckets().size()); + for (uint32_t i = 0; i < msg.getBuckets().size(); i++) { + buf.putLong(msg.getBuckets()[i].getRawId()); + } + + buf.putBoolean(msg.visitRemoves()); + buf.putString(msg.getFieldSet()); + buf.putBoolean(msg.visitInconsistentBuckets()); + buf.putInt(msg.getQueueTimeout()); + + uint32_t size = msg.getParameters().getSerializedSize(); + char* docBuffer = buf.allocate(size); + document::ByteBuffer bbuf(docBuffer, size); + msg.getParameters().serialize(bbuf); + + onEncodeCommand(buf, msg); + + buf.putInt(msg.getVisitorOrdering()); + buf.putInt(msg.getMaxBucketsPerVisitor()); +} + +api::StorageCommand::UP +ProtocolSerialization5_1::onDecodeCreateVisitorCommand(BBuf& buf) const +{ + vespalib::stringref libraryName = SH::getString(buf); + vespalib::stringref instanceId = SH::getString(buf); + vespalib::stringref selection = SH::getString(buf); + api::CreateVisitorCommand::UP msg( + new api::CreateVisitorCommand(libraryName, instanceId, selection)); + msg->setVisitorCmdId(SH::getInt(buf)); + msg->setControlDestination(SH::getString(buf)); + msg->setDataDestination(SH::getString(buf)); + msg->setMaximumPendingReplyCount(SH::getInt(buf)); + + msg->setFromTime(SH::getLong(buf)); + msg->setToTime(SH::getLong(buf)); + uint32_t count = SH::getInt(buf); + + if (count > buf.getRemaining()) { + // Trigger out of bounds exception rather than out of memory error + buf.incPos(count); + } + + for (uint32_t i = 0; i < count; i++) { + msg->getBuckets().push_back(document::BucketId(SH::getLong(buf))); + } + + if (SH::getBoolean(buf)) { + msg->setVisitRemoves(); + } + + msg->setFieldSet(SH::getString(buf)); + + if (SH::getBoolean(buf)) { + msg->setVisitInconsistentBuckets(); + } + msg->setQueueTimeout(SH::getInt(buf)); + msg->getParameters().deserialize(getTypeRepo(), buf); + + onDecodeCommand(buf, *msg); + msg->setVisitorOrdering( + (document::OrderingSpecification::Order)SH::getInt(buf)); + msg->setMaxBucketsPerVisitor(SH::getInt(buf)); + msg->setVisitorDispatcherVersion(50); + return api::StorageCommand::UP(msg.release()); +} + +void ProtocolSerialization5_1::onEncode( + GBBuf& buf, const api::CreateBucketCommand& msg) const +{ + buf.putLong(msg.getBucketId().getRawId()); + buf.putBoolean(msg.getActive()); + onEncodeBucketInfoCommand(buf, msg); +} + +api::StorageCommand::UP +ProtocolSerialization5_1::onDecodeCreateBucketCommand(BBuf& buf) const +{ + document::BucketId bid(SH::getLong(buf)); + bool setActive = SH::getBoolean(buf); + api::CreateBucketCommand::UP msg(new api::CreateBucketCommand(bid)); + msg->setActive(setActive); + onDecodeBucketInfoCommand(buf, *msg); + return api::StorageCommand::UP(msg.release()); +} + + +} // mbusprot +} // storage diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h new file mode 100644 index 00000000000..491dd5e02eb --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storageapi/mbusprot/protocolserialization5_0.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> + +namespace storage { +namespace mbusprot { + +class ProtocolSerialization5_1 : public ProtocolSerialization5_0 +{ + enum BucketState { + BUCKET_READY = 0x1, + BUCKET_ACTIVE = 0x2, + }; +public: + ProtocolSerialization5_1(const document::DocumentTypeRepo::SP&, + const documentapi::LoadTypeSet& loadTypes); + + virtual api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const; + virtual void putBucketInfo(const api::BucketInfo& info, + vespalib::GrowableByteBuffer& buf) const; + +protected: + virtual void onEncode(GBBuf&, const api::SetBucketStateCommand&) const; + virtual void onEncode(GBBuf&, const api::SetBucketStateReply&) const; + virtual void onEncode(GBBuf&, const api::GetCommand&) const; + virtual void onEncode(GBBuf&, const api::CreateVisitorCommand&) const; + virtual void onEncode(GBBuf&, const api::CreateBucketCommand&) const; + + virtual SCmd::UP onDecodeSetBucketStateCommand(BBuf&) const; + virtual SRep::UP onDecodeSetBucketStateReply(const SCmd&, BBuf&) const; + virtual SCmd::UP onDecodeGetCommand(BBuf&) const; + virtual SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const; + virtual SCmd::UP onDecodeCreateBucketCommand(BBuf&) const; +}; + +} // mbusprot +} // storage + diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp new file mode 100644 index 00000000000..68b8449cf4c --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/mbusprot/protocolserialization5_2.h> +#include <vespa/storageapi/mbusprot/storagecommand.h> +#include <vespa/storageapi/mbusprot/serializationhelper.h> + +#include <vespa/log/log.h> +LOG_SETUP(".storage.api.mbusprot.serialization.5_2"); + +namespace storage { +namespace mbusprot { + +using documentapi::TestAndSetCondition; + +void ProtocolSerialization5_2::onEncode(GBBuf & buf, const api::PutCommand & cmd) const +{ + ProtocolSerialization5_0::onEncode(buf, cmd); + encodeTasCondition(buf, cmd); +} + +api::StorageCommand::UP +ProtocolSerialization5_2::onDecodePutCommand(BBuf & buf) const +{ + auto cmd = ProtocolSerialization5_0::onDecodePutCommand(buf); + decodeTasCondition(*cmd, buf); + return cmd; +} + +void ProtocolSerialization5_2::onEncode(GBBuf & buf, const api::RemoveCommand & cmd) const +{ + ProtocolSerialization4_2::onEncode(buf, cmd); + encodeTasCondition(buf, cmd); +} + +api::StorageCommand::UP +ProtocolSerialization5_2::onDecodeRemoveCommand(BBuf & buf) const +{ + auto cmd = ProtocolSerialization4_2::onDecodeRemoveCommand(buf); + decodeTasCondition(*cmd, buf); + return cmd; +} + +void ProtocolSerialization5_2::onEncode(GBBuf & buf, const api::UpdateCommand & cmd) const +{ + ProtocolSerialization5_0::onEncode(buf, cmd); + encodeTasCondition(buf, cmd); +} + +api::StorageCommand::UP +ProtocolSerialization5_2::onDecodeUpdateCommand(BBuf & buf) const +{ + auto cmd = ProtocolSerialization5_0::onDecodeUpdateCommand(buf); + decodeTasCondition(*cmd, buf); + return cmd; +} + +void ProtocolSerialization5_2::decodeTasCondition(api::StorageCommand & storageCmd, BBuf & buf) { + auto & cmd = static_cast<api::TestAndSetCommand &>(storageCmd); + cmd.setCondition(TestAndSetCondition(SH::getString(buf))); +} + +void ProtocolSerialization5_2::encodeTasCondition(GBBuf & buf, const api::StorageCommand & storageCmd) { + auto & cmd = static_cast<const api::TestAndSetCommand &>(storageCmd); + buf.putString(cmd.getCondition().getSelection()); +} + +} // mbusprot +} // storage diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h new file mode 100644 index 00000000000..35af05d192b --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell + +#pragma once + +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> +#include <vespa/storageapi/mbusprot/protocolserialization5_1.h> +#include <vespa/storageapi/message/persistence.h> + +namespace storage { +namespace mbusprot { + +class ProtocolSerialization5_2 : public ProtocolSerialization5_1 +{ +public: + ProtocolSerialization5_2( + const document::DocumentTypeRepo::SP& repo, + const documentapi::LoadTypeSet & loadTypes) + : ProtocolSerialization5_1(repo, loadTypes) + {} + +protected: + virtual void onEncode(GBBuf &, const api::PutCommand &) const override; + virtual void onEncode(GBBuf &, const api::RemoveCommand &) const override; + virtual void onEncode(GBBuf &, const api::UpdateCommand &) const override; + + virtual SCmd::UP onDecodePutCommand(BBuf &) const override; + virtual SCmd::UP onDecodeRemoveCommand(BBuf &) const override; + virtual SCmd::UP onDecodeUpdateCommand(BBuf &) const override; + + static void decodeTasCondition(api::StorageCommand & cmd, BBuf & buf); + static void encodeTasCondition(GBBuf & buf, const api::StorageCommand & cmd); +}; + +} // mbusprot +} // storage diff --git a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h new file mode 100644 index 00000000000..dea5ca70e9a --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h @@ -0,0 +1,151 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/types.h> +#include <vespa/document/base/globalid.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + +namespace storage { +namespace mbusprot { + +class SerializationHelper +{ +public: + static int64_t getLong(document::ByteBuffer& buf) { + int64_t tmp; + buf.getLongNetwork(tmp); + return tmp; + } + + static int32_t getInt(document::ByteBuffer& buf) { + int32_t tmp; + buf.getIntNetwork(tmp); + return tmp; + } + + static int16_t getShort(document::ByteBuffer& buf) { + int16_t tmp; + buf.getShortNetwork(tmp); + return tmp; + } + + static uint8_t getByte(document::ByteBuffer& buf) { + uint8_t tmp; + buf.getByte(tmp); + return tmp; + } + + static vespalib::stringref getString(document::ByteBuffer& buf) { + uint32_t tmp; + buf.getIntNetwork((int32_t&) tmp); + const char * p = buf.getBufferAtPos(); + buf.incPos(tmp); + vespalib::stringref s(p, tmp); + return s; + } + + static bool getBoolean(document::ByteBuffer& buf) { + uint8_t tmp; + buf.getByte(tmp); + return (tmp == 1); + } + + static api::ReturnCode getReturnCode(document::ByteBuffer& buf) { + api::ReturnCode::Result result = (api::ReturnCode::Result) getInt(buf); + vespalib::stringref message = getString(buf); + return api::ReturnCode(result, message); + } + + static void putReturnCode(const api::ReturnCode& code, + vespalib::GrowableByteBuffer& buf) + { + buf.putInt(code.getResult()); + buf.putString(code.getMessage()); + } + + static const uint32_t BUCKET_INFO_SERIALIZED_SIZE = sizeof(uint32_t) * 3; + + static document::GlobalId getGlobalId(document::ByteBuffer& buf) { + std::vector<char> buffer(getShort(buf)); + for (uint32_t i=0; i<buffer.size(); ++i) { + buffer[i] = getByte(buf); + } + return document::GlobalId(&buffer[0]); + } + + static void putGlobalId(const document::GlobalId& gid, + vespalib::GrowableByteBuffer& buf) + { + buf.putShort(document::GlobalId::LENGTH); + for (uint32_t i=0; i<document::GlobalId::LENGTH; ++i) { + buf.putByte(gid.get()[i]); + } + } + + static document::Document::UP getDocument( + document::ByteBuffer& buf, + const document::DocumentTypeRepo& repo) + { + uint32_t size = getInt(buf); + if (size == 0) { + return document::Document::UP(); + } else { + document::ByteBuffer bbuf(buf.getBufferAtPos(), size); + buf.incPos(size); + return document::Document::UP(new document::Document(repo, bbuf)); + } + } + + static document::DocumentUpdate::UP getUpdate( + document::ByteBuffer& buf, + const document::DocumentTypeRepo& repo) + { + uint32_t size = getInt(buf); + if (size == 0) { + return document::DocumentUpdate::UP(); + } else { + document::ByteBuffer bbuf(buf.getBufferAtPos(), size); + buf.incPos(size); + return document::DocumentUpdate::UP( + new document::DocumentUpdate(repo, bbuf, + document::DocumentUpdate:: + SerializeVersion:: + SERIALIZE_42)); + } + } + + static void putDocument(document::Document* doc, + vespalib::GrowableByteBuffer& buf) + { + if (doc) { + vespalib::nbostream stream; + doc->serialize(stream); + buf.putInt(stream.size()); + buf.putBytes(stream.peek(), stream.size()); + } else { + buf.putInt(0); + } + } + + static void putUpdate(document::DocumentUpdate* update, + vespalib::GrowableByteBuffer& buf) + { + if (update) { + vespalib::nbostream stream; + update->serialize42(stream); + buf.putInt(stream.size()); + buf.putBytes(stream.peek(), stream.size()); + } else { + buf.putInt(0); + } + } + +}; + +} // mbusprot +} // storage + diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp new file mode 100644 index 00000000000..04d92b8fa9e --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/mbusprot/storagecommand.h> + +namespace storage { +namespace mbusprot { + +StorageCommand::StorageCommand(const storage::api::StorageCommand::SP& cmd) + : mbus::Message(), + _cmd(cmd) +{ +} + +} // mbusprot +} // storage diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h b/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h new file mode 100644 index 00000000000..11ef23a1254 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/messagebus/message.h> +#include <vespa/storageapi/mbusprot/storagemessage.h> +#include <vespa/storageapi/mbusprot/storageprotocol.h> +#include <vespa/storageapi/messageapi/storagecommand.h> + +namespace storage { +namespace mbusprot { + +class StorageCommand : public mbus::Message, public StorageMessage { +public: + typedef std::unique_ptr<StorageCommand> UP; + + StorageCommand(const storage::api::StorageCommand::SP&); + + const mbus::string & getProtocol() const { return StorageProtocol::NAME; } + + uint32_t getType() const { return _cmd->getType().getId(); } + + const api::StorageCommand::SP& getCommand() { return _cmd; } + api::StorageCommand::CSP getCommand() const { return _cmd; } + virtual api::StorageMessage::SP getInternalMessage() { return _cmd; } + virtual api::StorageMessage::CSP getInternalMessage() const { return _cmd; } + + virtual uint8_t priority() const { + return ((getInternalMessage()->getPriority()) / 255) * 16; + } + +private: + api::StorageCommand::SP _cmd; +}; + +} // mbusprot +} // storage + diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagemessage.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagemessage.cpp new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/storagemessage.cpp diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagemessage.h b/storageapi/src/vespa/storageapi/mbusprot/storagemessage.h new file mode 100644 index 00000000000..8424712b7c9 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/storagemessage.h @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storageapi/messageapi/storagemessage.h> + +namespace storage { +namespace mbusprot { + +class StorageMessage { +public: + typedef std::unique_ptr<StorageMessage> UP; + + virtual ~StorageMessage() {} + + virtual api::StorageMessage::SP getInternalMessage() = 0; + virtual storage::api::StorageMessage::CSP getInternalMessage() const = 0; + +}; + +} // protocol +} // storage + diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp new file mode 100644 index 00000000000..7e1e3815dc5 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp @@ -0,0 +1,173 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/mbusprot/storageprotocol.h> + +#include <vespa/log/log.h> +#include <vespa/storageapi/mbusprot/protocolserialization.h> +#include <vespa/storageapi/mbusprot/serializationhelper.h> +#include <vespa/storageapi/mbusprot/storagecommand.h> +#include <vespa/storageapi/mbusprot/storagereply.h> +#include <vespa/vespalib/util/exceptions.h> + +LOG_SETUP(".storage.api.mbusprot.protocol"); + +namespace storage { +namespace mbusprot { + +mbus::string StorageProtocol::NAME = "StorageProtocol"; + +StorageProtocol::StorageProtocol(const document::DocumentTypeRepo::SP repo, + const documentapi::LoadTypeSet& loadTypes) + : _serializer5_0(repo, loadTypes), + _serializer5_1(repo, loadTypes), + _serializer5_2(repo, loadTypes) +{ +} + +mbus::IRoutingPolicy::UP +StorageProtocol::createPolicy(const mbus::string&, const mbus::string&) const +{ + return mbus::IRoutingPolicy::UP(); +} + +namespace { + vespalib::Version version5_2(5, 93, 30); + vespalib::Version version5_1(5, 1, 0); + vespalib::Version version5_0(5, 0, 12); + vespalib::Version version5_0beta(4, 3, 0); +} + +static mbus::Blob +encodeMessage(const ProtocolSerialization & serializer, + const mbus::Routable & routable, + const StorageMessage & message, + const vespalib::Version & serializerVersion, + const vespalib::Version & actualVersion) +{ + mbus::Blob blob(serializer.encode(*message.getInternalMessage())); + + if (LOG_WOULD_LOG(spam)) { + std::ostringstream messageStream; + document::StringUtil::printAsHex(messageStream, blob.data(), blob.size()); + + LOG(spam, "Encoded message of protocol %s type %s using " + "%s serialization as version is %s:\n%s", + routable.getProtocol().c_str(), + message.getInternalMessage()->getType().toString().c_str(), + serializerVersion.toString().c_str(), + actualVersion.toString().c_str(), + messageStream.str().c_str()); + } + + return blob; +} + + +mbus::Blob +StorageProtocol::encode(const vespalib::Version& version, + const mbus::Routable& routable) const +{ + const StorageMessage & message(dynamic_cast<const StorageMessage &>(routable)); + + try { + if (message.getInternalMessage().get() == 0) { + throw vespalib::IllegalArgumentException( + "Given storage message wrapper does not contain a " + "storage message.", + VESPA_STRLOC); + } + + if (version < version5_1) { + if (version < version5_0beta) { + LOGBP(warning, + "No support for using messagebus for version %s." + "Minimum version is %s. Thus we cannot serialize %s.", + version.toString().c_str(), + version5_0beta.toString().c_str(), + message.getInternalMessage()->toString().c_str()); + + return mbus::Blob(0); + } else { + return encodeMessage(_serializer5_0, routable, message, version5_0, version); + } + } else if (version < version5_2) { + return encodeMessage(_serializer5_1, routable, message, version5_1, version); + } else { + return encodeMessage(_serializer5_2, routable, message, version5_2, version); + } + + } catch (std::exception & e) { + LOGBP(warning, "Failed to encode %s storage protocol message %s: %s", + version.toString().c_str(), + message.getInternalMessage()->toString().c_str(), + e.what()); + } + + return mbus::Blob(0); +} + +static mbus::Routable::UP +decodeMessage(const ProtocolSerialization & serializer, + mbus::BlobRef data, + const api::MessageType & type, + const vespalib::Version & serializerVersion, + const vespalib::Version & actualVersion) +{ + if (LOG_WOULD_LOG(spam)) { + std::ostringstream messageStream; + document::StringUtil::printAsHex(messageStream, data.data(), data.size()); + + LOG(spam, + "Decoding %s of version %s " + "using %s decoder from:\n%s", + type.toString().c_str(), + actualVersion.toString().c_str(), + serializerVersion.toString().c_str(), + messageStream.str().c_str()); + } + + if (type.isReply()) { + return std::make_unique<StorageReply>(data, serializer); + } else { + return mbus::Routable::UP(serializer.decodeCommand(data).release()); + } +} + +mbus::Routable::UP +StorageProtocol::decode(const vespalib::Version & version, + mbus::BlobRef data) const +{ + try { + document::ByteBuffer buf(data.data(), data.size()); + auto & type = api::MessageType::get( + static_cast<api::MessageType::Id>(SerializationHelper::getInt(buf))); + + StorageMessage::UP message; + if (version < version5_1) { + if (version < version5_0beta) { + LOGBP(error, + "No support for using messagebus for version %s." + "Minimum version is %s.", + version.toString().c_str(), + version5_0beta.toString().c_str()); + } else { + return decodeMessage(_serializer5_0, data, type, version5_0, version); + } + } else if (version < version5_2) { + return decodeMessage(_serializer5_1, data, type, version5_1, version); + } else { + return decodeMessage(_serializer5_2, data, type, version5_2, version); + } + } catch (std::exception & e) { + std::ostringstream ost; + ost << "Failed to decode " << version.toString() << " messagebus " + << "storage protocol message: " << e.what() << "\n"; + document::StringUtil::printAsHex(ost, data.data(), data.size()); + LOGBP(warning, "%s", ost.str().c_str()); + } + + return mbus::Routable::UP(); +} + +} // mbusprot +} // storage diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h new file mode 100644 index 00000000000..c490a17555a --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/messagebus/iprotocol.h> +#include <string> +#include <vespa/storageapi/mbusprot/protocolserialization5_0.h> +#include <vespa/storageapi/mbusprot/protocolserialization5_1.h> +#include <vespa/storageapi/mbusprot/protocolserialization5_2.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> + +namespace storage { +namespace mbusprot { + +class StorageProtocol : public mbus::IProtocol +{ +public: + typedef std::shared_ptr<StorageProtocol> SP; + + static mbus::string NAME; + + StorageProtocol(const document::DocumentTypeRepo::SP, + const documentapi::LoadTypeSet& loadTypes); + + // Implements IProtocol. + const mbus::string& getName() const { return NAME; } + + // Implements IProtocol. + mbus::IRoutingPolicy::UP createPolicy(const mbus::string& name, + const mbus::string& param) const; + + // Implements IProtocol. + mbus::Blob encode(const vespalib::Version&, const mbus::Routable&) const; + + // Implements IProtocol. + mbus::Routable::UP decode(const vespalib::Version&, mbus::BlobRef) const; + +private: + ProtocolSerialization5_0 _serializer5_0; + ProtocolSerialization5_1 _serializer5_1; + ProtocolSerialization5_2 _serializer5_2; +}; + +} // mbusprot +} // storage + + + diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp new file mode 100644 index 00000000000..b9fde51eaef --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/mbusprot/storagereply.h> + +#include <vespa/storageapi/mbusprot/storagecommand.h> + +namespace storage { +namespace mbusprot { + +StorageReply::StorageReply(const mbus::BlobRef& data, + const ProtocolSerialization& serializer) + : _serializer(&serializer), + _buffer(data.size()), + _mbusType(0), + _reply() +{ + memcpy(_buffer.get(), data.data(), _buffer.size()); + document::ByteBuffer buf(data.data(), data.size()); + buf.getIntNetwork(reinterpret_cast<int32_t&>(_mbusType)); +} + +StorageReply::StorageReply(const api::StorageReply::SP& reply) + : _serializer(0), + _buffer(), + _mbusType(reply->getType().getId()), + _reply(reply) +{ +} + +StorageReply::~StorageReply() +{ +} + +void +StorageReply::deserialize() const +{ + if (_reply.get()) return; + StorageReply& reply(const_cast<StorageReply&>(*this)); + mbus::Message::UP msg(reply.getMessage()); + if (msg.get() == 0) { + throw vespalib::IllegalStateException( + "Cannot deserialize storage reply before message have been set", + VESPA_STRLOC); + } + const StorageCommand* cmd(dynamic_cast<const StorageCommand*>(msg.get())); + reply.setMessage(std::move(msg)); + if (cmd == 0) { + throw vespalib::IllegalStateException( + "Storage reply get message did not return a storage command", + VESPA_STRLOC); + } + mbus::BlobRef blobRef(static_cast<char *>(_buffer.get()), _buffer.size()); + _reply = _serializer->decodeReply(blobRef, *cmd->getCommand())->getReply(); + vespalib::DefaultAlloc().swap(_buffer); +} + +} // mbusprot +} // storage diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagereply.h b/storageapi/src/vespa/storageapi/mbusprot/storagereply.h new file mode 100644 index 00000000000..52f7efc50cc --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/storagereply.h @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/messagebus/reply.h> +#include <vespa/storageapi/mbusprot/storagemessage.h> +#include <vespa/storageapi/mbusprot/storageprotocol.h> +#include <vespa/storageapi/messageapi/storagereply.h> + +namespace storage { +namespace mbusprot { + +class StorageReply : public mbus::Reply, public StorageMessage { + const ProtocolSerialization* _serializer; + mutable vespalib::DefaultAlloc _buffer; + uint32_t _mbusType; + mutable api::StorageReply::SP _reply; + +public: + typedef std::unique_ptr<StorageReply> UP; + + StorageReply(const mbus::BlobRef& data, const ProtocolSerialization&); + StorageReply(const api::StorageReply::SP& reply); + virtual ~StorageReply(); + + virtual const mbus::string& getProtocol() const + { return StorageProtocol::NAME; } + + uint32_t getType() const { return _mbusType; } + + const api::StorageReply::SP& getReply() { deserialize(); return _reply; } + api::StorageReply::CSP getReply() const { deserialize(); return _reply; } + virtual api::StorageMessage::SP getInternalMessage() + { deserialize(); return _reply; } + virtual api::StorageMessage::CSP getInternalMessage() const + { deserialize(); return _reply; } + + virtual uint8_t priority() const { + if (_reply.get()) { + return _reply->getPriority(); + } + return 0; + } + +private: + + void deserialize() const; +}; + +} // mbusprot +} // storage + diff --git a/storageapi/src/vespa/storageapi/message/.gitignore b/storageapi/src/vespa/storageapi/message/.gitignore new file mode 100644 index 00000000000..526f91c6668 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/.gitignore @@ -0,0 +1,7 @@ +*.lo +.*.swp +.depend +.depend.NEW +.deps +.libs +Makefile diff --git a/storageapi/src/vespa/storageapi/message/CMakeLists.txt b/storageapi/src/vespa/storageapi/message/CMakeLists.txt new file mode 100644 index 00000000000..a263f495a1d --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storageapi_message OBJECT + SOURCES + datagram.cpp + persistence.cpp + bucket.cpp + visitor.cpp + state.cpp + searchresult.cpp + bucketsplitting.cpp + multioperation.cpp + documentsummary.cpp + stat.cpp + removelocation.cpp + queryresult.cpp + batch.cpp + DEPENDS +) diff --git a/storageapi/src/vespa/storageapi/message/batch.cpp b/storageapi/src/vespa/storageapi/message/batch.cpp new file mode 100644 index 00000000000..6da213efaf5 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/batch.cpp @@ -0,0 +1,171 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <algorithm> +#include <vespa/storageapi/message/batch.h> +#include <vespa/document/bucket/bucketidfactory.h> + +using namespace storage::api; + +IMPLEMENT_COMMAND(BatchPutRemoveCommand, BatchPutRemoveReply) +IMPLEMENT_REPLY(BatchPutRemoveReply) +IMPLEMENT_COMMAND(BatchDocumentUpdateCommand, BatchDocumentUpdateReply) +IMPLEMENT_REPLY(BatchDocumentUpdateReply) + + +BatchPutRemoveCommand::Operation::Operation(uint64_t ts, Type tp) + : timestamp(ts), + type(tp) +{ +} + +BatchPutRemoveCommand::PutOperation::PutOperation(document::Document::SP doc, uint64_t ts) + : Operation(ts, PUT), + document(doc) +{ +} + +BatchPutRemoveCommand::HeaderUpdateOperation::HeaderUpdateOperation(document::Document::SP doc, uint64_t newTimestamp, uint64_t timestampToUpdate_) + : Operation(newTimestamp, HEADERUPDATE), + document(doc), + timestampToUpdate(timestampToUpdate_) +{ +} + +BatchPutRemoveCommand::RemoveOperation::RemoveOperation(const document::DocumentId& docId, uint64_t ts) + : Operation(ts, REMOVE), + documentId(docId) +{ +} + +BatchPutRemoveCommand::BatchPutRemoveCommand(const document::BucketId& bucketId) + : BucketInfoCommand(MessageType::BATCHPUTREMOVE, bucketId), + _approxSize(0) +{ +} + +void +BatchPutRemoveCommand::addPut(document::Document::SP document, uint64_t ts) +{ + _operations.push_back(vespalib::LinkedPtr<Operation>(new PutOperation(document, ts))); + _approxSize += document->serialize()->getLength(); +} + +void +BatchPutRemoveCommand::addHeaderUpdate(document::Document::SP document, uint64_t ts, uint64_t timestampToUpdate) +{ + _operations.push_back(vespalib::LinkedPtr<Operation>(new HeaderUpdateOperation(document, ts, timestampToUpdate))); + _approxSize += document->serialize()->getLength(); +} + +void +BatchPutRemoveCommand::addRemove(const document::DocumentId& docId, uint64_t ts) +{ + _operations.push_back(vespalib::LinkedPtr<Operation>(new RemoveOperation(docId, ts))); + _approxSize += docId.toString().length(); +} + +void +BatchPutRemoveCommand::addOperation(const Operation& op, bool cloneDocument) +{ + switch (op.type) { + case Operation::PUT: + { + document::Document::SP doc; + if (!cloneDocument) { + doc = static_cast<const PutOperation&>(op).document; + } else { + doc.reset(static_cast<const PutOperation&>(op).document->clone()); + } + addPut(doc, op.timestamp); + break; + } + case Operation::REMOVE: + addRemove(static_cast<const RemoveOperation&>(op).documentId, op.timestamp); + break; + case Operation::HEADERUPDATE: + { + const HeaderUpdateOperation& hup = static_cast<const HeaderUpdateOperation&>(op); + document::Document::SP doc; + if (!cloneDocument) { + doc = hup.document; + } else { + doc.reset(hup.document->clone()); + } + addHeaderUpdate(doc, op.timestamp, hup.timestampToUpdate); + break; + } + } +} + +void +BatchPutRemoveCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const { + out << "BatchPutRemove(" << getBucketId() << ", " << _operations.size() << " operations)"; + + if (verbose) { + out << " : "; + BucketInfoCommand::print(out, verbose, indent); + } +} + +BatchPutRemoveReply::BatchPutRemoveReply(const BatchPutRemoveCommand& cmd) + : BucketInfoReply(cmd) +{ +} + +void +BatchPutRemoveReply::print(std::ostream& out, bool verbose, + const std::string& indent) const { + out << "BatchPutRemoveReply("; + out << _documentsNotFound.size() << " documents not found)"; + + if (verbose) { + out << " {"; + for (std::vector<document::DocumentId>::const_iterator it = + _documentsNotFound.begin(); + it != _documentsNotFound.end(); ++it) + { + out << "\n" << indent << " " << (*it); + } + out << "\n" << indent << "} : "; + BucketInfoReply::print(out, verbose, indent); + } +} + +BatchDocumentUpdateCommand::BatchDocumentUpdateCommand(const UpdateList& updates) + : StorageCommand(MessageType::BATCHDOCUMENTUPDATE), + _updates(updates) +{ + document::BucketIdFactory factory; + _bucketId = factory.getBucketId(updates[0]->getId()); +} + +void +BatchDocumentUpdateCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const { + out << "BatchDocumentUpdate(" << _updates.size() << " operations)"; + + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +BatchDocumentUpdateReply::BatchDocumentUpdateReply(const BatchDocumentUpdateCommand& cmd) + : StorageReply(cmd), + _documentsNotFound() +{ +} + +void +BatchDocumentUpdateReply::print(std::ostream& out, bool verbose, + const std::string& indent) const { + out << "BatchDocumentUpdateReply(" + << std::count(_documentsNotFound.begin(), _documentsNotFound.end(), true) + << " not found)"; + + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} diff --git a/storageapi/src/vespa/storageapi/message/batch.h b/storageapi/src/vespa/storageapi/message/batch.h new file mode 100644 index 00000000000..a1b92fa1f0e --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/batch.h @@ -0,0 +1,204 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/storageapi/messageapi/bucketinfocommand.h> +#include <vespa/storageapi/messageapi/bucketinforeply.h> + +namespace storage { +namespace api { + +/** + * @class BatchPutRemoveCommand + * @ingroup message + * + * @brief Sends a batch of puts and removes + */ +class BatchPutRemoveCommand : public BucketInfoCommand { +public: + class Operation { + public: + enum Type { + REMOVE, // Removes a document + HEADERUPDATE, // Updates the header of a document, if it already exists. + PUT // Inserts a new document. + }; + + Operation(uint64_t ts, Type type); + virtual ~Operation() {}; + + uint64_t timestamp; + Type type; + + virtual const document::DocumentId& getDocumentId() const = 0; + }; + + explicit BatchPutRemoveCommand(const document::BucketId& id); + + class PutOperation : public Operation { + public: + PutOperation(document::Document::SP document, uint64_t timestamp); + + document::Document::SP document; + + const document::DocumentId& getDocumentId() const { + return document->getId(); + } + }; + + class HeaderUpdateOperation : public Operation { + public: + HeaderUpdateOperation(document::Document::SP document, uint64_t newTimestamp, uint64_t timestampToUpdate); + + document::Document::SP document; + uint64_t timestampToUpdate; + + const document::DocumentId& getDocumentId() const { + return document->getId(); + } + }; + + class RemoveOperation : public Operation { + public: + RemoveOperation(const document::DocumentId& docId, uint64_t timestamp); + + document::DocumentId documentId; + + const document::DocumentId& getDocumentId() const { + return documentId; + } + }; + + /** + Adds a PUT operation to be performed. + */ + void addPut(document::Document::SP document, uint64_t timestamp); + + /** + Adds a PUT operation to be performed. + */ + void addHeaderUpdate(document::Document::SP document, uint64_t newTimestamp, uint64_t timestampToUpdate); + + /** + Adds a REMOVE operation to be performed. + */ + void addRemove(const document::DocumentId& docId, uint64_t timestamp); + + /** + * Adds an operation to be performed. Optionally deep-clones the + * operation's document. + */ + void addOperation(const Operation& op, bool cloneDocument = false); + + /** + Returns the number of operations in this batch. + */ + uint32_t getOperationCount() const { return _operations.size(); } + + /** + Returns the nth operation in this batch. + */ + const Operation& getOperation(uint32_t index) const { return *_operations[index]; } + + /** + Returns the nth operation in this batch. + */ + Operation& getOperation(uint32_t index) { return *_operations[index]; } + + /** + Returns an approximate size of this message. + */ + uint32_t getMemoryFootprint() const { return _approxSize + 20; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(BatchPutRemoveCommand, onBatchPutRemove) + +private: + std::vector<vespalib::LinkedPtr<Operation> > _operations; + uint32_t _approxSize; +}; + +/** + * @class BatchPutRemoveReply + * @ingroup message + * + * @brief Confirm that a given docoperations have been received. + */ +class BatchPutRemoveReply : public BucketInfoReply { +private: + std::vector<document::DocumentId> _documentsNotFound; + +public: + explicit BatchPutRemoveReply(const BatchPutRemoveCommand&); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + const std::vector<document::DocumentId>& getDocumentsNotFound() const { return _documentsNotFound; } + std::vector<document::DocumentId>& getDocumentsNotFound() { return _documentsNotFound; } + + DECLARE_STORAGEREPLY(BatchPutRemoveReply, onBatchPutRemoveReply) +}; + +class BatchDocumentUpdateCommand : public StorageCommand +{ +public: + typedef std::vector<document::DocumentUpdate::SP > UpdateList; + + /** + Creates a batch update message containing the given updates. + */ + BatchDocumentUpdateCommand(const UpdateList& updates); + + /** + @return Returns a list of the updates to be performed. + */ + const UpdateList& getUpdates() const { return _updates; }; + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + /** + Returns a bucket id suitable for routing this message. + */ + document::BucketId getBucketId() const { return _bucketId; } + virtual bool hasSingleBucketId() const { return true; } + + DECLARE_STORAGECOMMAND(BatchDocumentUpdateCommand, onBatchDocumentUpdate) + +private: + UpdateList _updates; + document::BucketId _bucketId; +}; + +/** + * @class BatchDocumentUpdateReply + * @ingroup message + * + * @brief Confirm that a given docoperations have been received. + */ +class BatchDocumentUpdateReply : public StorageReply { + // 1-1 mapping of found/not found state for documents + std::vector<bool> _documentsNotFound; +public: + explicit BatchDocumentUpdateReply(const BatchDocumentUpdateCommand&); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + const std::vector<bool>& getDocumentsNotFound() const { return _documentsNotFound; } + std::vector<bool>& getDocumentsNotFound() { return _documentsNotFound; } + + DECLARE_STORAGEREPLY(BatchDocumentUpdateReply, onBatchDocumentUpdateReply) +}; + + + +} +} + + diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp new file mode 100644 index 00000000000..e2450ca17ca --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/bucket.cpp @@ -0,0 +1,615 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/message/bucket.h> + +#include <vespa/document/fieldvalue/document.h> + +namespace storage { +namespace api { + +IMPLEMENT_COMMAND(CreateBucketCommand, CreateBucketReply) +IMPLEMENT_REPLY(CreateBucketReply) +IMPLEMENT_COMMAND(DeleteBucketCommand, DeleteBucketReply) +IMPLEMENT_REPLY(DeleteBucketReply) +IMPLEMENT_COMMAND(MergeBucketCommand, MergeBucketReply) +IMPLEMENT_REPLY(MergeBucketReply) +IMPLEMENT_COMMAND(GetBucketDiffCommand, GetBucketDiffReply) +IMPLEMENT_REPLY(GetBucketDiffReply) +IMPLEMENT_COMMAND(ApplyBucketDiffCommand, ApplyBucketDiffReply) +IMPLEMENT_REPLY(ApplyBucketDiffReply) +IMPLEMENT_COMMAND(RequestBucketInfoCommand, RequestBucketInfoReply) +IMPLEMENT_REPLY(RequestBucketInfoReply) +IMPLEMENT_COMMAND(NotifyBucketChangeCommand, NotifyBucketChangeReply) +IMPLEMENT_REPLY(NotifyBucketChangeReply) +IMPLEMENT_COMMAND(SetBucketStateCommand, SetBucketStateReply) +IMPLEMENT_REPLY(SetBucketStateReply) + +CreateBucketCommand::CreateBucketCommand(const document::BucketId& id) + : MaintenanceCommand(MessageType::CREATEBUCKET, id), + _active(false) +{ +} + +void +CreateBucketCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "CreateBucketCommand(" << getBucketId(); + if (_active) { + out << ", active"; + } else { + out << ", inactive"; + } + out << ")"; + out << " Reasons to start: " << _reason; + if (verbose) { + out << " : "; + MaintenanceCommand::print(out, verbose, indent); + } +} + +CreateBucketReply::CreateBucketReply(const CreateBucketCommand& cmd) + : BucketInfoReply(cmd) +{ +} + +void +CreateBucketReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "CreateBucketReply(" << getBucketId() << ")"; + if (verbose) { + out << " : "; + BucketInfoReply::print(out, verbose, indent); + } +} + +DeleteBucketCommand::DeleteBucketCommand(const document::BucketId& id) + : MaintenanceCommand(MessageType::DELETEBUCKET, id) +{ +} + +void +DeleteBucketCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "DeleteBucketCommand(" << getBucketId() << ")"; + out << " Reasons to start: " << _reason; + if (verbose) { + out << " : "; + MaintenanceCommand::print(out, verbose, indent); + } +} + +DeleteBucketReply::DeleteBucketReply(const DeleteBucketCommand& cmd) + : BucketInfoReply(cmd) +{ +} + +void +DeleteBucketReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "DeleteBucketReply(" << getBucketId() << ")"; + if (verbose) { + out << " : "; + BucketInfoReply::print(out, verbose, indent); + } +} + +MergeBucketCommand::MergeBucketCommand( + const document::BucketId& id, const std::vector<Node>& nodes, + Timestamp maxTimestamp, uint32_t clusterStateVersion, + const std::vector<uint16_t>& chain) + : MaintenanceCommand(MessageType::MERGEBUCKET, id), + _nodes(nodes), + _maxTimestamp(maxTimestamp), + _clusterStateVersion(clusterStateVersion), + _chain(chain) +{ +} + +void +MergeBucketCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "MergeBucketCommand(" << getBucketId() << ", to time " + << _maxTimestamp << ", cluster state version: " + << _clusterStateVersion << ", nodes: ["; + for (uint32_t i=0; i<_nodes.size(); ++i) { + if (i != 0) out << ", "; + out << _nodes[i]; + } + out << "], chain: ["; + for (uint32_t i = 0; i < _chain.size(); ++i) { + if (i != 0) out << ", "; + out << _chain[i]; + } + out << "]"; + out << ", reasons to start: " << _reason; + out << ")"; + if (verbose) { + out << " : "; + BucketCommand::print(out, verbose, indent); + } +} + +MergeBucketReply::MergeBucketReply(const MergeBucketCommand& cmd) + : BucketReply(cmd), + _nodes(cmd.getNodes()), + _maxTimestamp(cmd.getMaxTimestamp()), + _clusterStateVersion(cmd.getClusterStateVersion()), + _chain(cmd.getChain()) +{ +} + +void +MergeBucketReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "MergeBucketReply(" << getBucketId() << ", to time " + << _maxTimestamp << ", cluster state version: " + << _clusterStateVersion << ", nodes: "; + for (uint32_t i=0; i<_nodes.size(); ++i) { + if (i != 0) out << ", "; + out << _nodes[i]; + } + out << "], chain: ["; + for (uint32_t i = 0; i < _chain.size(); ++i) { + if (i != 0) out << ", "; + out << _chain[i]; + } + out << "])"; + if (verbose) { + out << " : "; + BucketReply::print(out, verbose, indent); + } +} + +GetBucketDiffCommand::Entry::Entry() + : _timestamp(0), + _gid(), + _headerSize(0), + _bodySize(0), + _flags(0), + _hasMask(0) +{ +} + +void GetBucketDiffCommand::Entry::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "Entry(timestamp: " << _timestamp + << ", " << _gid << ", hasMask: 0x" << _hasMask; + if (verbose) { + out << ",\n" << indent << " " << "header size: " + << std::dec << _headerSize << ", body size: " << _bodySize + << ", flags 0x" << std::hex << _flags << std::dec; + } + out << ")"; +} + +bool GetBucketDiffCommand::Entry::operator==(const Entry& e) const +{ + return (_timestamp == e._timestamp && + _headerSize == e._headerSize && + _bodySize == e._bodySize && + _gid == e._gid && + _flags == e._flags); +} + +GetBucketDiffCommand::GetBucketDiffCommand( + const document::BucketId& id, const std::vector<Node>& nodes, + Timestamp maxTimestamp) + : BucketCommand(MessageType::GETBUCKETDIFF, id), + _nodes(nodes), + _maxTimestamp(maxTimestamp) +{ +} + +void +GetBucketDiffCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "GetBucketDiffCommand(" << getBucketId() << ", to time " + << _maxTimestamp << ", nodes: "; + for (uint32_t i=0; i<_nodes.size(); ++i) { + if (i != 0) out << ", "; + out << _nodes[i]; + } + if (_diff.empty()) { + out << ", no entries"; + } else if (verbose) { + out << ","; + for (uint32_t i=0; i<_diff.size(); ++i) { + out << "\n" << indent << " "; + _diff[i].print(out, verbose, indent + " "); + } + } else { + out << ", " << _diff.size() << " entries"; + out << ", id " << _msgId; + } + out << ")"; + if (verbose) { + out << " : "; + BucketCommand::print(out, verbose, indent); + } +} + +GetBucketDiffReply::GetBucketDiffReply(const GetBucketDiffCommand& cmd) + : BucketReply(cmd), + _nodes(cmd.getNodes()), + _maxTimestamp(cmd.getMaxTimestamp()), + _diff(cmd.getDiff()) +{ +} + +void +GetBucketDiffReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "GetBucketDiffReply(" << getBucketId() << ", to time " + << _maxTimestamp << ", nodes: "; + for (uint32_t i=0; i<_nodes.size(); ++i) { + if (i != 0) out << ", "; + out << _nodes[i]; + } + if (_diff.empty()) { + out << ", no entries"; + } else if (verbose) { + out << ","; + for (uint32_t i=0; i<_diff.size(); ++i) { + out << "\n" << indent << " "; + _diff[i].print(out, verbose, indent + " "); + } + } else { + out << ", " << _diff.size() << " entries"; + out << ", id " << _msgId; + } + out << ")"; + if (verbose) { + out << " : "; + BucketReply::print(out, verbose, indent); + } +} + +ApplyBucketDiffCommand::Entry::Entry() + : _entry(), + _docName(), + _headerBlob(), + _bodyBlob(), + _repo() +{ +} + +ApplyBucketDiffCommand::Entry::Entry(const GetBucketDiffCommand::Entry& e) + : _entry(e), + _docName(), + _headerBlob(), + _bodyBlob(), + _repo() +{ +} + +bool ApplyBucketDiffCommand::Entry::filled() const +{ + return ((_headerBlob.size() > 0 || + (_entry._headerSize == 0 && !_docName.empty())) && + (_bodyBlob.size() > 0 || + _entry._bodySize == 0)); +} + +void ApplyBucketDiffCommand::Entry::print( + std::ostream& out, bool verbose, const std::string& indent) const +{ + out << "ApplyEntry("; + _entry.print(out, verbose, indent + " "); + out << ",\n" << indent << " name(" << _docName + << "), headerBlob(" << _headerBlob.size() + << "), bodyBlob(" << _bodyBlob.size() << ")"; + if (_headerBlob.size() > 0) { + document::ByteBuffer buf(&_headerBlob[0], + _headerBlob.size()); + if (_repo) { + document::Document doc(*_repo, buf); + out << ",\n" << indent << " " << doc.getId().getGlobalId(); + } else { + out << ",\n" << indent << " unknown global id. (repo missing)"; + } + } + out << ")"; +} + +bool ApplyBucketDiffCommand::Entry::operator==(const Entry& e) const +{ + return (_entry == e._entry && + _headerBlob == e._headerBlob && + _bodyBlob == e._bodyBlob); +} + +ApplyBucketDiffCommand::ApplyBucketDiffCommand( + const document::BucketId& id, const std::vector<Node>& nodes, + uint32_t maxBufferSize) + : BucketInfoCommand(MessageType::APPLYBUCKETDIFF, id), + _nodes(nodes), + _diff(), + _maxBufferSize(maxBufferSize) +{ +} + +void +ApplyBucketDiffCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + uint32_t totalSize = 0; + uint32_t filled = 0; + for (std::vector<Entry>::const_iterator it = _diff.begin(); + it != _diff.end(); ++it) + { + totalSize += it->_headerBlob.size(); + totalSize += it->_bodyBlob.size(); + if (it->filled()) ++filled; + } + out << "ApplyBucketDiffCommand(" << getBucketId() << ", nodes: "; + for (uint32_t i=0; i<_nodes.size(); ++i) { + if (i != 0) out << ", "; + out << _nodes[i]; + } + out << ", max buffer size " << _maxBufferSize << " bytes" + << ", " << _diff.size() << " entries of " << totalSize << " bytes, " + << (100.0 * filled / _diff.size()) << " \% filled)"; + if (_diff.empty()) { + out << ", no entries"; + } else if (verbose) { + out << ","; + for (uint32_t i=0; i<_diff.size(); ++i) { + out << "\n" << indent << " "; + _diff[i].print(out, verbose, indent + " "); + } + } else { + out << ", " << _diff.size() << " entries"; + out << ", id " << _msgId; + } + out << ")"; + if (verbose) { + out << " : "; + BucketCommand::print(out, verbose, indent); + } +} + +ApplyBucketDiffReply::ApplyBucketDiffReply(const ApplyBucketDiffCommand& cmd) + : BucketInfoReply(cmd), + _nodes(cmd.getNodes()), + _diff(cmd.getDiff()), + _maxBufferSize(cmd.getMaxBufferSize()) +{ +} + +void +ApplyBucketDiffReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + uint32_t totalSize = 0; + uint32_t filled = 0; + for (std::vector<Entry>::const_iterator it = _diff.begin(); + it != _diff.end(); ++it) + { + totalSize += it->_headerBlob.size(); + totalSize += it->_bodyBlob.size(); + if (it->filled()) ++filled; + } + out << "ApplyBucketDiffReply(" << getBucketId() << ", nodes: "; + for (uint32_t i=0; i<_nodes.size(); ++i) { + if (i != 0) out << ", "; + out << _nodes[i]; + } + out << ", max buffer size " << _maxBufferSize << " bytes" + << ", " << _diff.size() << " entries of " << totalSize << " bytes, " + << (100.0 * filled / _diff.size()) << " \% filled)"; + if (_diff.empty()) { + out << ", no entries"; + } else if (verbose) { + out << ","; + for (uint32_t i=0; i<_diff.size(); ++i) { + out << "\n" << indent << " "; + _diff[i].print(out, verbose, indent + " "); + } + } else { + out << ", " << _diff.size() << " entries"; + out << ", id " << _msgId; + } + out << ")"; + if (verbose) { + out << " : "; + BucketInfoReply::print(out, verbose, indent); + } +} + +RequestBucketInfoCommand::RequestBucketInfoCommand( + const std::vector<document::BucketId>& buckets) + : StorageCommand(MessageType::REQUESTBUCKETINFO), + _buckets(buckets), + _state(), + _distributor(0xFFFF) +{ +} + +RequestBucketInfoCommand::RequestBucketInfoCommand( + uint16_t distributor, const lib::ClusterState& state, + const vespalib::stringref & distributionHash) + : StorageCommand(MessageType::REQUESTBUCKETINFO), + _buckets(), + _state(new lib::ClusterState(state)), + _distributor(distributor), + _distributionHash(distributionHash) +{ +} + +RequestBucketInfoCommand::RequestBucketInfoCommand( + uint16_t distributor, const lib::ClusterState& state) + : StorageCommand(MessageType::REQUESTBUCKETINFO), + _buckets(), + _state(new lib::ClusterState(state)), + _distributor(distributor), + _distributionHash("") +{ +} + +void +RequestBucketInfoCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "RequestBucketInfoCommand("; + if (_buckets.size() != 0) { + out << _buckets.size() << " buckets"; + } + if (hasSystemState()) { + out << "distributor " << _distributor << " in "; + _state->print(out, verbose, indent + " "); + } + if (verbose && _buckets.size() > 0) { + out << "\n" << indent << " Specified buckets:\n" << indent << " "; + std::copy(_buckets.begin(), _buckets.end(), + std::ostream_iterator<document::BucketId>( + out, ("\n" + indent + " ").c_str())); + } + out << ")"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +std::ostream& operator<<(std::ostream& out, + const RequestBucketInfoReply::Entry& e) +{ + return out << e._bucketId << " - " << e._info; +} + + +RequestBucketInfoReply::RequestBucketInfoReply( + const RequestBucketInfoCommand& cmd) + : StorageReply(cmd), + _buckets() +{ +} + +uint32_t +RequestBucketInfoReply::getMemoryFootprint() const +{ + return sizeof(Entry) * _buckets.capacity(); +} + +void +RequestBucketInfoReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "RequestBucketInfoReply(" << _buckets.size(); + if (verbose) { + out << "\n" << indent << " "; + std::copy(_buckets.begin(), _buckets.end(), + std::ostream_iterator<Entry>(out, + ("\n" + indent + " ").c_str())); + } + out << ")"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +NotifyBucketChangeCommand::NotifyBucketChangeCommand( + const document::BucketId& id, const BucketInfo& info) + : BucketCommand(MessageType::NOTIFYBUCKETCHANGE, id), + _info(info) +{ +} + +void +NotifyBucketChangeCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "NotifyBucketChangeCommand(" << getBucketId() << ", "; + _info.print(out, verbose, indent); + out << ")"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +NotifyBucketChangeReply::NotifyBucketChangeReply( + const NotifyBucketChangeCommand& cmd) + : BucketReply(cmd) +{ +} + +void +NotifyBucketChangeReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "NotifyBucketChangeReply(" << getBucketId() << ")"; + if (verbose) { + out << " : "; + BucketReply::print(out, verbose, indent); + } +} + +SetBucketStateCommand::SetBucketStateCommand( + const document::BucketId& bucket, + BUCKET_STATE state) + : MaintenanceCommand(MessageType::SETBUCKETSTATE, bucket), + _state(state) +{ +} + +void +SetBucketStateCommand::print(std::ostream& out, + bool verbose, + const std::string& indent) const +{ + out << "SetBucketStateCommand(" << getBucketId() << ", "; + switch (_state) { + case INACTIVE: + out << "INACTIVE"; + break; + case ACTIVE: + out << "ACTIVE"; + break; + } + out << ")"; + if (verbose) { + out << " : "; + MaintenanceCommand::print(out, verbose, indent); + } +} + +vespalib::string +SetBucketStateCommand::getSummary() const +{ + vespalib::asciistream stream; + stream << "SetBucketStateCommand(" << getBucketId().toString() << ", " + << ((_state == ACTIVE) ? "ACTIVE" : "INACTIVE") << ")"; + return stream.str(); +} + +SetBucketStateReply::SetBucketStateReply( + const SetBucketStateCommand& cmd) + : BucketInfoReply(cmd) +{ +} + +void +SetBucketStateReply::print(std::ostream& out, + bool verbose, + const std::string& indent) const +{ + out << "SetBucketStateReply(" << getBucketId() << ")"; + if (verbose) { + out << " : "; + BucketInfoReply::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storageapi/src/vespa/storageapi/message/bucket.h new file mode 100644 index 00000000000..28b3ba36f30 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/bucket.h @@ -0,0 +1,528 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @file bucketinfo.h + * + * Bucket related commands. + */ + +#pragma once + +#include <vespa/document/base/globalid.h> +#include <vespa/vdslib/state/clusterstate.h> +#include <vespa/storageapi/defs.h> +#include <vespa/storageapi/messageapi/bucketcommand.h> +#include <vespa/storageapi/messageapi/bucketreply.h> +#include <vespa/storageapi/messageapi/bucketinfocommand.h> +#include <vespa/storageapi/messageapi/bucketinforeply.h> +#include <vespa/storageapi/messageapi/maintenancecommand.h> + +namespace document { class DocumentTypeRepo; } + +namespace storage { +namespace api { + +/** + * @class CreateBucketCommand + * @ingroup message + * + * @brief Command for creating a new bucket on a storage node. + */ +class CreateBucketCommand : public MaintenanceCommand { + bool _active; + +public: + explicit CreateBucketCommand(const document::BucketId& id); + + void setActive(bool active) { _active = active; } + bool getActive() const { return _active; } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(CreateBucketCommand, onCreateBucket) +}; + +/** + * @class CreateBucketReply + * @ingroup message + * + * @brief Reply of a create bucket command. + */ +class CreateBucketReply : public BucketInfoReply { +public: + explicit CreateBucketReply(const CreateBucketCommand& cmd); + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(CreateBucketReply, onCreateBucketReply); +}; + +/** + * @class DeleteBucketCommand + * @ingroup message + * + * @brief Command for deleting a bucket from one or more storage nodes. + */ +class DeleteBucketCommand : public MaintenanceCommand { + BucketInfo _info; +public: + explicit DeleteBucketCommand(const document::BucketId& id); + + const BucketInfo& getBucketInfo() const { return _info; } + void setBucketInfo(const BucketInfo& info) { _info = info; } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(DeleteBucketCommand, onDeleteBucket) +}; + +/** + * @class DeleteBucketReply + * @ingroup message + * + * @brief Reply of a delete bucket command. + */ +class DeleteBucketReply : public BucketInfoReply { +public: + explicit DeleteBucketReply(const DeleteBucketCommand& cmd); + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(DeleteBucketReply, onDeleteBucketReply) +}; + + +/** + * @class MergeBucketCommand + * @ingroup message + * + * @brief Merge a bucket + * + * Merges given bucket copies, held on the given node list. A maximum timestamp + * should be given, such that the buckets may be used during merge. If not + * given, storage will set current time for it, but distributors should really + * set it, as they have the reference clock for a bucket. + * + * An optional "only for source" node list can be provided. In this case, the + * nodes in that list are only used for sources in the merge, and never as + * targets, even if they are missing documents from the other nodes. + * + */ +class MergeBucketCommand : public MaintenanceCommand { +public: + struct Node { + uint16_t index; + bool sourceOnly; + + Node(uint16_t index_, bool sourceOnly_ = false) + : index(index_), sourceOnly(sourceOnly_) {} + + bool operator==(const Node& n) const + { return (index == n.index && sourceOnly == n.sourceOnly); } + }; + +private: + std::vector<Node> _nodes; + Timestamp _maxTimestamp; + uint32_t _clusterStateVersion; + std::vector<uint16_t> _chain; + +public: + MergeBucketCommand(const document::BucketId&, + const std::vector<Node>&, + Timestamp maxTimestamp, + uint32_t clusterStateVersion = 0, + const std::vector<uint16_t>& chain = std::vector<uint16_t>()); + + const std::vector<Node>& getNodes() const { return _nodes; } + Timestamp getMaxTimestamp() const { return _maxTimestamp; } + const std::vector<uint16_t>& getChain() const { return _chain; } + + uint32_t getClusterStateVersion() const { return _clusterStateVersion; } + + void setClusterStateVersion(uint32_t version) { _clusterStateVersion = version; } + void setChain(const std::vector<uint16_t>& chain) { _chain = chain; } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(MergeBucketCommand, onMergeBucket) +}; + +inline std::ostream& +operator<<(std::ostream& out, const MergeBucketCommand::Node& n) { + return out << n.index << (n.sourceOnly ? " (source only)" : ""); +} + +/** + * @class MergeBucketReply + * @ingroup message + * + * @brief Reply of a merge bucket command. + */ +class MergeBucketReply : public BucketReply { +public: + typedef MergeBucketCommand::Node Node; + +private: + std::vector<Node> _nodes; + Timestamp _maxTimestamp; + uint32_t _clusterStateVersion; + std::vector<uint16_t> _chain; + +public: + explicit MergeBucketReply(const MergeBucketCommand& cmd); + + const std::vector<Node>& getNodes() const { return _nodes; } + Timestamp getMaxTimestamp() const { return _maxTimestamp; } + const std::vector<uint16_t>& getChain() const { return _chain; } + uint32_t getClusterStateVersion() const { return _clusterStateVersion; } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(MergeBucketReply, onMergeBucketReply) +}; + +/** + * @class GetBucketDiff + * @ingroup message + * + * @brief Message sent between storage nodes as the first step of merge. + */ +class GetBucketDiffCommand : public BucketCommand { +public: + typedef MergeBucketCommand::Node Node; + + struct Entry : public document::Printable { + Timestamp _timestamp; + document::GlobalId _gid; + uint32_t _headerSize; + uint32_t _bodySize; + uint16_t _flags; + uint16_t _hasMask; + + Entry(); + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + bool operator==(const Entry&) const; + bool operator<(const Entry& e) const + { return (_timestamp < e._timestamp); } + }; +private: + std::vector<Node> _nodes; + Timestamp _maxTimestamp; + std::vector<Entry> _diff; + +public: + GetBucketDiffCommand(const document::BucketId&, + const std::vector<Node>&, + Timestamp maxTimestamp); + + const std::vector<Node>& getNodes() const { return _nodes; } + Timestamp getMaxTimestamp() const { return _maxTimestamp; } + const std::vector<Entry>& getDiff() const { return _diff; } + std::vector<Entry>& getDiff() { return _diff; } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(GetBucketDiffCommand, onGetBucketDiff) +}; + +/** + * @class GetBucketDiffReply + * @ingroup message + * + * @brief Reply of GetBucketDiffCommand + */ +class GetBucketDiffReply : public BucketReply { +public: + typedef MergeBucketCommand::Node Node; + typedef GetBucketDiffCommand::Entry Entry; + +private: + std::vector<Node> _nodes; + Timestamp _maxTimestamp; + std::vector<Entry> _diff; + +public: + explicit GetBucketDiffReply(const GetBucketDiffCommand& cmd); + + const std::vector<Node>& getNodes() const { return _nodes; } + Timestamp getMaxTimestamp() const { return _maxTimestamp; } + const std::vector<Entry>& getDiff() const { return _diff; } + std::vector<Entry>& getDiff() { return _diff; } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(GetBucketDiffReply, onGetBucketDiffReply) +}; + +/** + * @class ApplyBucketDiff + * @ingroup message + * + * @brief Sends a chunk of document entries, which the bucket copies can use + * to update themselves. + */ +class ApplyBucketDiffCommand : public BucketInfoCommand { +public: + typedef MergeBucketCommand::Node Node; + struct Entry : public document::Printable { + GetBucketDiffCommand::Entry _entry; + vespalib::string _docName; + std::vector<char> _headerBlob; + std::vector<char> _bodyBlob; + const document::DocumentTypeRepo *_repo; + + Entry(); + Entry(const GetBucketDiffCommand::Entry&); + + bool filled() const; + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + bool operator==(const Entry&) const; + }; +private: + std::vector<Node> _nodes; + std::vector<Entry> _diff; + // We may send more metadata entries that should fit in one apply bucket + // diff command, to let node pick which ones it wants to fill in first. + // Nodes should verify that they don't fill a command up with more than + // this number of bytes. + uint32_t _maxBufferSize; + +public: + ApplyBucketDiffCommand(const document::BucketId& id, + const std::vector<Node>& nodes, + uint32_t maxBufferSize); + + const std::vector<Node>& getNodes() const { return _nodes; } + const std::vector<Entry>& getDiff() const { return _diff; } + std::vector<Entry>& getDiff() { return _diff; } + uint32_t getMaxBufferSize() const { return _maxBufferSize; } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(ApplyBucketDiffCommand, onApplyBucketDiff) +}; + +/** + * @class ApplyBucketDiffReply + * @ingroup message + * + * @brief Reply of ApplyBucketDiffCommand + */ +class ApplyBucketDiffReply : public BucketInfoReply { +public: + typedef MergeBucketCommand::Node Node; + typedef ApplyBucketDiffCommand::Entry Entry; + +private: + std::vector<Node> _nodes; + std::vector<Entry> _diff; + uint32_t _maxBufferSize; + +public: + explicit ApplyBucketDiffReply(const ApplyBucketDiffCommand& cmd); + + const std::vector<Node>& getNodes() const { return _nodes; } + const std::vector<Entry>& getDiff() const { return _diff; } + std::vector<Entry>& getDiff() { return _diff; } + uint32_t getMaxBufferSize() const { return _maxBufferSize; } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(ApplyBucketDiffReply, onApplyBucketDiffReply) +}; + +/** + * @class RequestBucketInfoCommand + * @ingroup message + * + * @brief Command for getting bucket info. + * + * Used to get checksums of buckets from a storage node. + * If list of buckets for which to retrieve info is given. If it is empty, + * it means all buckets. + * A system state and a distributor index may be given. If given, only info for + * the buckets that belong to the given distributor should be returned. + */ +class RequestBucketInfoCommand : public StorageCommand { + std::vector<document::BucketId> _buckets; + std::unique_ptr<lib::ClusterState> _state; + uint16_t _distributor; + vespalib::string _distributionHash; + +public: + explicit RequestBucketInfoCommand( + const std::vector<document::BucketId>& buckets); + RequestBucketInfoCommand(uint16_t distributor, + const lib::ClusterState& state, + const vespalib::stringref & _distributionHash); + + RequestBucketInfoCommand(uint16_t distributor, + const lib::ClusterState& state); + + const std::vector<document::BucketId>& getBuckets() const + { return _buckets; } + + bool hasSystemState() const { return (_state.get() != 0); } + uint16_t getDistributor() const { return _distributor; } + const lib::ClusterState& getSystemState() const + { return *_state; } + + const vespalib::string& getDistributionHash() const { return _distributionHash; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(RequestBucketInfoCommand, onRequestBucketInfo) +}; + + +/** + * @class RequestBucketInfoReply + * @ingroup message + * + * @brief Answer of a bucket info command. + */ +class RequestBucketInfoReply : public StorageReply { +public: + struct Entry { + document::BucketId _bucketId; + BucketInfo _info; + + bool operator==(const Entry& e) const + { return (_bucketId == e._bucketId && _info == e._info); } + Entry() : _bucketId(), _info() {} + Entry(const document::BucketId& id, const BucketInfo& info) + : _bucketId(id), _info(info) {} + friend std::ostream& operator<<(std::ostream& os, const Entry&); + }; + typedef vespalib::Array<Entry, vespalib::DefaultAlloc> EntryVector; +private: + EntryVector _buckets; + +public: + + explicit RequestBucketInfoReply(const RequestBucketInfoCommand& cmd); + + const EntryVector & getBucketInfo() const { return _buckets; } + EntryVector & getBucketInfo() { return _buckets; } + + uint32_t getMemoryFootprint() const; + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(RequestBucketInfoReply, onRequestBucketInfoReply) +}; + +/** + * @class NotifyBucketChangeCommand + * @ingroup message + * + * @brief Command for letting others know a bucket have been altered. + * + * When the persistence layer notices a bucket has been corrupted, such that + * it needs to be repaired, this message will be sent to notify others + * of change. Others being bucket database on storage node, and possibly + * distributor. + */ +class NotifyBucketChangeCommand : public BucketCommand { + BucketInfo _info; + +public: + NotifyBucketChangeCommand(const document::BucketId& bucket, + const BucketInfo& bucketInfo); + + const BucketInfo& getBucketInfo() const { return _info; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(NotifyBucketChangeCommand, onNotifyBucketChange) +}; + + +/** + * @class NotifyBucketChangeReply + * @ingroup message + * + * @brief Answer of notify bucket command. + * + * Noone will resend these messages, and they're not needed, but all commands + * need to have a reply. + */ +class NotifyBucketChangeReply : public BucketReply { +public: + explicit NotifyBucketChangeReply(const NotifyBucketChangeCommand& cmd); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(NotifyBucketChangeReply, onNotifyBucketChangeReply) +}; + +/** + * @class SetBucketStateCommand + * @ingroup message + * + * @brief Sent by distributor to set the ready/active state of a bucket. + */ +class SetBucketStateCommand : public MaintenanceCommand +{ +public: + enum BUCKET_STATE + { + INACTIVE, + ACTIVE + }; +private: + BUCKET_STATE _state; +public: + SetBucketStateCommand(const document::BucketId& bucket, + BUCKET_STATE state); + + void print(std::ostream& out, + bool verbose, + const std::string& indent) const; + + BUCKET_STATE getState() const { return _state; } + + DECLARE_STORAGECOMMAND(SetBucketStateCommand, onSetBucketState) +private: + virtual vespalib::string getSummary() const; +}; + +/** + * @class SetBucketStateReply + * @ingroup message + * + * @brief Answer to SetBucketStateCommand. + */ +class SetBucketStateReply : public BucketInfoReply +{ +public: + explicit SetBucketStateReply(const SetBucketStateCommand&); + + void print(std::ostream& out, + bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(SetBucketStateReply, onSetBucketStateReply) +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/message/bucketsplitting.cpp b/storageapi/src/vespa/storageapi/message/bucketsplitting.cpp new file mode 100644 index 00000000000..aa692ae2d57 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/bucketsplitting.cpp @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/message/bucketsplitting.h> + +namespace storage { +namespace api { + +IMPLEMENT_COMMAND(SplitBucketCommand, SplitBucketReply) +IMPLEMENT_REPLY(SplitBucketReply) +IMPLEMENT_COMMAND(JoinBucketsCommand, JoinBucketsReply) +IMPLEMENT_REPLY(JoinBucketsReply) + +SplitBucketCommand::SplitBucketCommand(const document::BucketId& id) + : MaintenanceCommand(MessageType::SPLITBUCKET, id), + _minSplitBits(0), + _maxSplitBits(58), + _minByteSize(std::numeric_limits<uint32_t>::max()), + _minDocCount(std::numeric_limits<uint32_t>::max()) +{ + // By default, set very large sizes, to ensure we trigger 'already big + // enough' behaviour, only splitting one step by default. The distributor + // should always overwrite one of these values to get correct behaviour. +} + +void +SplitBucketCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "SplitBucketCommand(" << getBucketId(); + if (_minDocCount != std::numeric_limits<uint32_t>::max() + || _minByteSize != std::numeric_limits<uint32_t>::max()) + { + out << "Max doc count: " << _minDocCount + << ", Max total doc size: " << _minByteSize; + } else if (_maxSplitBits != 58) { + out << "Max split bits to use: " << _maxSplitBits; + } + out << ")"; + out << " Reasons to start: " << _reason; + if (verbose) { + out << " : "; + BucketCommand::print(out, verbose, indent); + } +} + +SplitBucketReply::SplitBucketReply(const SplitBucketCommand& cmd) + : BucketReply(cmd) +{ +} + +void +SplitBucketReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "SplitBucketReply(" << getBucketId(); + if (_result.empty()) { + out << " - No target files created."; + } else { + out << " ->"; + for (uint32_t i=0; i<_result.size(); ++i) { + out << "\n" << indent << " " << _result[i].first << ": " + << _result[i].second; + } + } + out << ")"; + if (verbose) { + out << " : "; + BucketReply::print(out, verbose, indent); + } +} + +JoinBucketsCommand::JoinBucketsCommand(const document::BucketId& target) + : MaintenanceCommand(MessageType::JOINBUCKETS, target), + _minJoinBits(0) +{ +} + +void +JoinBucketsCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "JoinBucketsCommand(" << getBucketId(); + if (_sources.empty()) { + out << " - No files to join."; + } else { + out << " <-"; + for (uint32_t i=0; i<_sources.size(); ++i) { + out << " " << _sources[i]; + } + } + out << ")"; + out << " Reasons to start: " << _reason; + if (verbose) { + out << " : "; + BucketCommand::print(out, verbose, indent); + } +} + + +JoinBucketsReply::JoinBucketsReply(const JoinBucketsCommand& cmd) + : BucketInfoReply(cmd), + _sources(cmd.getSourceBuckets()) +{ +} + +JoinBucketsReply::JoinBucketsReply(const JoinBucketsCommand& cmd, const BucketInfo& bucketInfo) + : BucketInfoReply(cmd), + _sources(cmd.getSourceBuckets()) +{ + setBucketInfo(bucketInfo); +} + +void +JoinBucketsReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "JoinBucketsReply(" << getBucketId(); + if (_sources.empty()) { + out << " - No files to join."; + } else { + out << " <-"; + for (uint32_t i=0; i<_sources.size(); ++i) { + out << " " << _sources[i]; + } + } + out << ")"; + if (verbose) { + out << " : "; + BucketReply::print(out, verbose, indent); + } +} + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/message/bucketsplitting.h b/storageapi/src/vespa/storageapi/message/bucketsplitting.h new file mode 100644 index 00000000000..cbddfc8eaf6 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/bucketsplitting.h @@ -0,0 +1,144 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storageapi/buckets/bucketinfo.h> +#include <vespa/storageapi/messageapi/bucketcommand.h> +#include <vespa/storageapi/messageapi/bucketinforeply.h> +#include <vespa/storageapi/messageapi/maintenancecommand.h> + +namespace storage { +namespace api { + +/** + * @class SplitBucketCommand + * @ingroup message + * + * @brief Split a bucket + * + * Splits a bucket into two parts using the next split bit that is unused. + * + * Distributors can issue splits for multiple reasons: + * - Inconsistent buckets, so we need to split buckets containing others until + * they are either split equally, or no longer contains others. + * - Buckets that are too large are split to reduce file size. + * - Buckets with too many entries are split to reduce amount of metadata. + * + * In the first case, min and max split bits can be set. This will make storage + * able to split several bits at a time, but know where to stop. + * + * In the second case, min byte size can be set, to ensure that we don't split + * bucket more one step if the copy at the time of processing is + * actually smaller. Since removes can happen in the meantime, the min byte size + * should be smaller than the limit we use for splitting. Suggesting half. + * + * Similarily we can do as the second case in the third case too, just using + * min doc count as limiter instead. + * + * If neither are specified, min/max split bits limits nothing, but the sizes + * are set to max, which ensures that only one split step is taken. + */ +class SplitBucketCommand : public MaintenanceCommand { +private: + uint8_t _minSplitBits; + uint8_t _maxSplitBits; + uint32_t _minByteSize; + uint32_t _minDocCount; + +public: + SplitBucketCommand(const document::BucketId& id); + + uint8_t getMinSplitBits() const { return _minSplitBits; } + uint8_t getMaxSplitBits() const { return _maxSplitBits; } + uint32_t getMinByteSize() const { return _minByteSize; } + uint32_t getMinDocCount() const { return _minDocCount; } + + void setMinSplitBits(uint8_t v) { _minSplitBits = v; } + void setMaxSplitBits(uint8_t v) { _maxSplitBits = v; } + void setMinByteSize(uint32_t v) { _minByteSize = v; } + void setMinDocCount(uint32_t v) { _minDocCount = v; } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(SplitBucketCommand, onSplitBucket) +}; + +/** + * @class SplitBucketReply + * @ingroup message + * + * @brief Reply of a split bucket command. + */ +class SplitBucketReply : public BucketReply { +public: + typedef std::pair<document::BucketId, BucketInfo> Entry; + +private: + std::vector<Entry> _result; + +public: + explicit SplitBucketReply(const SplitBucketCommand& cmd); + + std::vector<Entry>& getSplitInfo() { return _result; } + const std::vector<Entry>& getSplitInfo() const { return _result; } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(SplitBucketReply, onSplitBucketReply) +}; + +/** + * @class JoinBucketCommand + * @ingroup message + * + * @brief Join two buckets + * + * Joins two buckets on the same node into a bucket with one fewer split bit. + */ +class JoinBucketsCommand : public MaintenanceCommand { + std::vector<document::BucketId> _sources; + uint8_t _minJoinBits; + +public: + explicit JoinBucketsCommand(const document::BucketId& target); + + std::vector<document::BucketId>& getSourceBuckets() { return _sources; } + const std::vector<document::BucketId>& getSourceBuckets() const + { return _sources; } + + void setMinJoinBits(uint8_t minJoinBits) { _minJoinBits = minJoinBits; } + uint8_t getMinJoinBits() const { return _minJoinBits; } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(JoinBucketsCommand, onJoinBuckets) +}; + +/** + * @class JoinBucketsReply + * @ingroup message + * + * @brief Reply of a join bucket command. + */ +class JoinBucketsReply : public BucketInfoReply { + std::vector<document::BucketId> _sources; + +public: + explicit JoinBucketsReply(const JoinBucketsCommand& cmd); + + JoinBucketsReply(const JoinBucketsCommand& cmd, const BucketInfo& bucketInfo); + + const std::vector<document::BucketId>& getSourceBuckets() const + { return _sources; } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(JoinBucketsReply, onJoinBucketsReply) +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/message/datagram.cpp b/storageapi/src/vespa/storageapi/message/datagram.cpp new file mode 100644 index 00000000000..bc0dadc3fb3 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/datagram.cpp @@ -0,0 +1,189 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <cassert> +#include <vespa/storageapi/message/datagram.h> + +namespace storage { +namespace api { + +IMPLEMENT_COMMAND(DocBlockCommand, DocBlockReply) +IMPLEMENT_REPLY(DocBlockReply) +IMPLEMENT_COMMAND(MapVisitorCommand, MapVisitorReply) +IMPLEMENT_REPLY(MapVisitorReply) +IMPLEMENT_COMMAND(DocumentListCommand, DocumentListReply) +IMPLEMENT_REPLY(DocumentListReply) +IMPLEMENT_COMMAND(EmptyBucketsCommand, EmptyBucketsReply) +IMPLEMENT_REPLY(EmptyBucketsReply) + +DocBlockCommand::DocBlockCommand(const document::BucketId& bucketId, + const vdslib::DocumentList& block, + const std::shared_ptr<void>& buffer) + : StorageCommand(MessageType::DOCBLOCK), + _bucketId(bucketId), + _docBlock(block), + _buffer(buffer), + _keepTimeStamps(false) +{ +} + +void +DocBlockCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "DocBlockCommand(" + << "size " << _docBlock.getBufferSize() << ", used space " + << (_docBlock.getBufferSize() - _docBlock.countFree()) << ", doccount " + << _docBlock.size() << ")"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +DocBlockReply::DocBlockReply(const DocBlockCommand& cmd) + : StorageReply(cmd) +{ +} + +void +DocBlockReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "DocBlockReply()"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +MapVisitorCommand::MapVisitorCommand() + : StorageCommand(MessageType::MAPVISITOR) +{ +} + +void +MapVisitorCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "MapVisitor(" << _statistics.size() << " entries"; + if (verbose) { + for (vdslib::Parameters::ParametersMap::const_iterator it + = _statistics.begin(); it != _statistics.end(); ++it) + { + out << ",\n" << indent << " " << it->first << ": " + << vespalib::stringref(it->second.c_str(), it->second.length()); + } + out << ") : "; + StorageCommand::print(out, verbose, indent); + } else { + out << ")"; + } +} + +MapVisitorReply::MapVisitorReply(const MapVisitorCommand& cmd) + : StorageReply(cmd) +{ +} + +void +MapVisitorReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "MapVisitorReply()"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +DocumentListCommand::DocumentListCommand(const document::BucketId& bid) + : StorageCommand(MessageType::DOCUMENTLIST), + _bucketId(bid), + _documents() +{ +} + +void +DocumentListCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "DocumentList(" << _bucketId; + if (_documents.empty()) { + out << ", empty"; + } else if (verbose) { + out << ","; + for (uint32_t i=0; i<_documents.size(); ++i) { + out << "\n" << indent << " "; + out << ":" << _documents[i]; + } + } else { + out << ", " << _documents.size() << " documents"; + } + out << ")"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +DocumentListReply::DocumentListReply(const DocumentListCommand& cmd) + : StorageReply(cmd) +{ +} + +void +DocumentListReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "DocumentListReply()"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +EmptyBucketsCommand::EmptyBucketsCommand( + const std::vector<document::BucketId>& buckets) + : StorageCommand(MessageType::EMPTYBUCKETS), + _buckets(buckets) +{ +} + +void +EmptyBucketsCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "EmptyBuckets("; + if (verbose) { + for (uint32_t i=0; i<_buckets.size(); ++i) { + out << "\n" << indent << " "; + out << _buckets[i]; + } + } else { + out << _buckets.size() << " buckets"; + } + out << ")"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +EmptyBucketsReply::EmptyBucketsReply(const EmptyBucketsCommand& cmd) + : StorageReply(cmd) +{ +} + +void +EmptyBucketsReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "EmptyBucketsReply()"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/message/datagram.h b/storageapi/src/vespa/storageapi/message/datagram.h new file mode 100644 index 00000000000..69563bb32f8 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/datagram.h @@ -0,0 +1,226 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/document.h> +#include <vespa/vdslib/container/documentlist.h> +#include <vespa/storageapi/messageapi/storagecommand.h> +#include <vespa/storageapi/messageapi/storagereply.h> +#include <vespa/storageapi/defs.h> +#include <vespa/storageapi/message/visitor.h> + +namespace storage { +namespace api { + +/** + * @class DocBlockCommand + * @ingroup message + * + * @brief Sends a docblock to a visitor or subscriber. + */ +class DocBlockCommand : public StorageCommand { + document::BucketId _bucketId; + vdslib::DocumentList _docBlock; + std::shared_ptr<void> _buffer; // Owns data in docblock + bool _keepTimeStamps; // Used for recovery/synchronization where we want to + // keep the timestamps of the origin. + +public: + DocBlockCommand(const document::BucketId& bucketId, + const vdslib::DocumentList& block, + const std::shared_ptr<void>& buffer); + + vdslib::DocumentList& getDocumentBlock() + { assert(_docBlock.getBufferSize() > 0); return _docBlock; } + const vdslib::DocumentList& getDocumentBlock() const + { assert(_docBlock.getBufferSize() > 0); return _docBlock; } + + void setDocumentBlock(vdslib::DocumentList& block) { + _docBlock = block; + } + + document::BucketId getBucketId() const { return _bucketId; } + virtual bool hasSingleBucketId() const { return true; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + bool keepTimeStamps() const { return _keepTimeStamps; } + void keepTimeStamps(bool keepTime) { _keepTimeStamps = keepTime; } + + DECLARE_STORAGECOMMAND(DocBlockCommand, onDocBlock) +}; + +/** + * @class DocBlockReply + * @ingroup message + * + * @brief Confirm that a given docblock have been received. + */ +class DocBlockReply : public StorageReply { +public: + explicit DocBlockReply(const DocBlockCommand&); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(DocBlockReply, onDocBlockReply) +}; + +/** + * @class MapStorageCommand + * @ingroup message + * + * @brief Sends a map of data to a visitor. + * + * This is a generic way to transfer data to the visitor data handler. + * It is for instance used when doing a specialized visitor to gather statistics + * on usage of document types and namespaces. + */ +class MapVisitorCommand : public StorageCommand { + vdslib::Parameters _statistics; + +public: + MapVisitorCommand(); + + vdslib::Parameters& getData() { return _statistics; }; + const vdslib::Parameters& getData() const { return _statistics; }; + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(MapVisitorCommand, onMapVisitor) +}; + +/** + * @class MapStorageReply + * @ingroup message + * + * @brief Confirm that a given map visitor command has been received. + */ +class MapVisitorReply : public StorageReply { +public: + explicit MapVisitorReply(const MapVisitorCommand&); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(MapVisitorReply, onMapVisitorReply) +}; + +/** + * @class DocumentListCommand + * @ingroup message + * + * @brief Sends a list of documents to the visitor data handler. + * + * This is used in synchronization in order to transfer minimal amount of data + * to the synchronization agent. + */ +class DocumentListCommand : public StorageCommand { +public: + struct Entry { + document::Document::SP _doc; + int64_t _lastModified; + bool _removeEntry; + + Entry() : _doc(), _lastModified(0), _removeEntry(false) {} + + Entry(const document::Document::SP& doc, int64_t lastModified, + bool removeEntry) + : _doc(doc), + _lastModified(lastModified), + _removeEntry(removeEntry) + { + } + }; + +private: + document::BucketId _bucketId; + std::vector<Entry> _documents; + +public: + DocumentListCommand(const document::BucketId& bid); + + const document::BucketId& getBucketId() { return _bucketId; } + std::vector<Entry>& getDocuments() { return _documents; } + const std::vector<Entry>& getDocuments() const { return _documents; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(DocumentListCommand, onDocumentList) + +}; + +inline std::ostream& operator<<(std::ostream& out, + const DocumentListCommand::Entry& e) +{ + out << e._doc->getId(); + if (e._removeEntry) out << " - removed"; + out << ", last modified at " << e._lastModified; + return out; +} + +/** + * @class DocumentListReply + * @ingroup message + * + * @brief Confirm that a given visitorstatisticscommand has been received. + */ +class DocumentListReply : public StorageReply { +public: + explicit DocumentListReply(const DocumentListCommand&); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(DocumentListReply, onDocumentListReply) +}; + +/** + * @class EmptyBucketsCommand + * @ingroup message + * + * @brief Sends a vector of bucket ids to a visitor. + * + * This message is used in synchronization to tell the synchronization client + * that a bucket contains no data at all. This is needed to let the follower be + * able to delete documents from these buckets, as they would otherwise be + * ignored by the synch agent. + */ +class EmptyBucketsCommand : public StorageCommand { + std::vector<document::BucketId> _buckets; + +public: + EmptyBucketsCommand(const std::vector<document::BucketId>&); + + const std::vector<document::BucketId>& getBuckets() const + { return _buckets; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(EmptyBucketsCommand, onEmptyBuckets) + +private: +}; + +/** + * @class EmptyBucketsReply + * @ingroup message + * + * @brief Confirm that a given emptybucketscommad has been received. + */ +class EmptyBucketsReply : public StorageReply { +public: + explicit EmptyBucketsReply(const EmptyBucketsCommand&); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(EmptyBucketsReply, onEmptyBucketsReply) +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/message/documentsummary.cpp b/storageapi/src/vespa/storageapi/message/documentsummary.cpp new file mode 100644 index 00000000000..cfb56489459 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/documentsummary.cpp @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/message/documentsummary.h> + +namespace storage { +namespace api { + +IMPLEMENT_COMMAND(DocumentSummaryCommand, DocumentSummaryReply) +IMPLEMENT_REPLY(DocumentSummaryReply) + +DocumentSummaryCommand::DocumentSummaryCommand() + : StorageCommand(MessageType::DOCUMENTSUMMARY), + DocumentSummary() +{ +} + +void +DocumentSummaryCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "DocumentSummary(" << getSummaryCount() << " summaries)"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +DocumentSummaryReply::DocumentSummaryReply(const DocumentSummaryCommand& cmd) + : StorageReply(cmd) +{ +} + +void +DocumentSummaryReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "DocumentSummaryReply()"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/message/documentsummary.h b/storageapi/src/vespa/storageapi/message/documentsummary.h new file mode 100644 index 00000000000..2aa3c4765c5 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/documentsummary.h @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vdslib/container/documentsummary.h> +#include <vespa/storageapi/message/visitor.h> + +namespace storage { +namespace api { + +/** + * @class DocumentSummaryCommand + * @ingroup message + * + * @brief The result of a searchvisitor. + */ +class DocumentSummaryCommand : public StorageCommand, + public vdslib::DocumentSummary +{ +public: + explicit DocumentSummaryCommand(); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + uint32_t getMemoryFootprint() const { + return getSerializedSize(); + } + + DECLARE_STORAGECOMMAND(DocumentSummaryCommand, onDocumentSummary) +}; + +/** + * @class DocumentSummaryReply + * @ingroup message + * + * @brief Response to a document summary command. + */ +class DocumentSummaryReply : public StorageReply { +public: + explicit DocumentSummaryReply(const DocumentSummaryCommand& command); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(DocumentSummaryReply, onDocumentSummaryReply) +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/message/internal.h b/storageapi/src/vespa/storageapi/message/internal.h new file mode 100644 index 00000000000..9b92d956919 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/internal.h @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @file internal.h + * + * Internal commands, used in storage. These are commands that don't need to + * be serialized as they never leave a node, implemented within storage itself + * to be able to include storage types not defined in the API. + * + * Historically these messages also existed so we could alter internal messages + * without recompiling clients, but currently no clients use storage API for + * communication anymore so this is no longer an issue. + */ +#pragma once + +#include <vespa/storageapi/messageapi/storagecommand.h> +#include <vespa/storageapi/messageapi/storagereply.h> + +namespace storage { +namespace api { + +/** + * @class InternalCommand + * @ingroup message + * + * @brief Base class for commands local to a VDS node. + * + * This is the base class for internal server commands. They can not be + * serialized, so any attempt of sending such a command away from a storage + * node will fail. + */ +class InternalCommand : public StorageCommand { + uint32_t _type; + +public: + InternalCommand(uint32_t type) + : StorageCommand(MessageType::INTERNAL), _type(type) {} + + uint32_t getType() const { return _type; } + + virtual bool callHandler(MessageHandler& h, + const std::shared_ptr<StorageMessage> & m) const + { + return h.onInternal(std::static_pointer_cast<InternalCommand>(m)); + } + + /** + * Enforcing that subclasses implement print such that we can see what kind + * of message it is when debugging. + */ + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const = 0; +}; + +/** + * @class InternalReply + * @ingroup message + * + * @brief Response of an internal command. + */ +class InternalReply : public StorageReply { + uint32_t _type; + +public: + InternalReply(uint32_t type, const InternalCommand& cmd) + : StorageReply(cmd), + _type(type) + { + } + + uint32_t getType() const { return _type; } + + virtual bool callHandler(MessageHandler& h, + const std::shared_ptr<StorageMessage> & m) const + { + return h.onInternalReply(std::static_pointer_cast<InternalReply>(m)); + } + + /** + * Enforcing that subclasses implement print such that we can see what kind + * of message it is when debugging. + */ + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const = 0; +}; + +inline void +InternalCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "InternalCommand(" << _type << ")"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +inline void +InternalReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "InternalReply(" << _type << ")"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/message/multioperation.cpp b/storageapi/src/vespa/storageapi/message/multioperation.cpp new file mode 100644 index 00000000000..35e4075e3ab --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/multioperation.cpp @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/message/multioperation.h> + +using document::DocumentTypeRepo; + +namespace storage { +namespace api { + +IMPLEMENT_COMMAND(MultiOperationCommand, MultiOperationReply) +IMPLEMENT_REPLY(MultiOperationReply) + +MultiOperationCommand::MultiOperationCommand(const DocumentTypeRepo::SP &repo, + const document::BucketId& id, + int bufferSize, + bool keepTimeStamps_) + : BucketInfoCommand(MessageType::MULTIOPERATION, id), + _buffer(), + _operations(repo, 0, 0), + _keepTimeStamps(keepTimeStamps_) +{ + _buffer.resize(bufferSize); + if (_buffer.size() > 0) { + _operations = vdslib::WritableDocumentList(_operations.getTypeRepo(), + &_buffer[0], _buffer.size(), false); + } +} + +MultiOperationCommand::MultiOperationCommand(const DocumentTypeRepo::SP &repo, + const document::BucketId& id, + const std::vector<char>& buffer, + bool keepTimeStamps_) + : BucketInfoCommand(MessageType::MULTIOPERATION, id), + _buffer(buffer), + _operations(repo, 0, 0), + _keepTimeStamps(keepTimeStamps_) +{ + if (_buffer.size() > 0) { + _operations = vdslib::WritableDocumentList(_operations.getTypeRepo(), + &_buffer[0], _buffer.size(), true); + } +} + +MultiOperationCommand::MultiOperationCommand(const MultiOperationCommand& o) + : BucketInfoCommand(MessageType::MULTIOPERATION, o.getBucketId()), + _buffer(o._buffer), + _operations(o._operations.getTypeRepo(),0, 0), + _keepTimeStamps(o._keepTimeStamps) +{ + setTimeout(o.getTimeout()); + setSourceIndex(o.getSourceIndex()); + setPriority(o.getPriority()); + if (_buffer.size() > 0) { + _operations = vdslib::WritableDocumentList(_operations.getTypeRepo(), + &_buffer[0], _buffer.size(), true); + } +} + +void +MultiOperationCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "MultiOperationCommand(" << getBucketId() + << ", size " << _operations.getBufferSize() << ", used space " + << (_operations.getBufferSize() - _operations.countFree()) + << ", doccount " << _operations.size() << ", keepTimeStamps " + << _keepTimeStamps << ")"; + if (verbose) { + out << " {"; + bool first = true; + for(vdslib::DocumentList::const_iterator it = _operations.begin(); + it != _operations.end(); ++it) + { + if (!first) { out << ","; } else { first = false; } + out << "\n" << indent << " "; + if (it->isRemoveEntry()) { + out << "Remove(" << it->getDocumentId() << ")"; + } else if (it->isUpdateEntry()) { + out << "Update(" << it->getDocumentId() << ")"; + } else { + out << "Put(" << it->getDocumentId() << ")"; + } + } + out << "\n" << indent << "} : "; + BucketInfoCommand::print(out, verbose, indent); + } +} + +MultiOperationReply::MultiOperationReply(const MultiOperationCommand& cmd) + : BucketInfoReply(cmd), + _highestModificationTimestamp(0) +{ +} + +void +MultiOperationReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "MultiOperationReply(" << getBucketId() << ")"; + if (verbose) { + out << " : "; + BucketInfoReply::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/message/multioperation.h b/storageapi/src/vespa/storageapi/message/multioperation.h new file mode 100644 index 00000000000..31db78458f9 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/multioperation.h @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/document/bucket/bucketid.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/storageapi/messageapi/bucketinfocommand.h> +#include <vespa/storageapi/messageapi/bucketinforeply.h> +#include <vespa/storageapi/defs.h> +#include <vespa/storageapi/message/visitor.h> +#include <vespa/vdslib/container/writabledocumentlist.h> + +namespace storage { +namespace api { + +/** + * @class MultiOperationCommand + * @ingroup message + * + * @brief Sends a documentlist + */ +class MultiOperationCommand : public BucketInfoCommand { +private: + std::vector<char> _buffer; // Used to hold data refered to by document list + // if message is to own its data. + vdslib::WritableDocumentList _operations; + bool _keepTimeStamps; + +public: + explicit MultiOperationCommand(const document::DocumentTypeRepo::SP &repo, + const document::BucketId& id, + int bufferSize, + bool keepTimeStamps = false); + explicit MultiOperationCommand(const document::DocumentTypeRepo::SP &repo, + const document::BucketId& id, + const std::vector<char>& buffer, + bool keepTimeStamps = false); + explicit MultiOperationCommand(const MultiOperationCommand& template_); + + std::vector<char>& getBuffer() { return _buffer; }; + const std::vector<char>& getBuffer() const { return _buffer; }; + + vdslib::WritableDocumentList& getOperations() + { assert(_operations.getBufferSize() > 0); return _operations; } + const vdslib::WritableDocumentList& getOperations() const + { assert(_operations.getBufferSize() > 0); return _operations; } + + void setOperations(vdslib::WritableDocumentList& operations) { + _buffer.clear(); + _operations = operations; + } + + uint32_t getMemoryFootprint() const { + return _buffer.size() + 20; + } + + bool keepTimeStamps() const { return _keepTimeStamps; } + void keepTimeStamps(bool keepTime) { _keepTimeStamps = keepTime; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(MultiOperationCommand, onMultiOperation) +}; + +/** + * @class MultiOperationReply + * @ingroup message + * + * @brief Confirm that a given docoperations have been received. + */ +class MultiOperationReply : public BucketInfoReply { +private: + // No need to serialize this, as it's only used internally in the distributor. + uint64_t _highestModificationTimestamp; + +public: + explicit MultiOperationReply(const MultiOperationCommand&); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + void setHighestModificationTimestamp(uint64_t highestModificationTimestamp) { + _highestModificationTimestamp = highestModificationTimestamp; + } + uint64_t getHighestModificationTimestamp() const { return _highestModificationTimestamp; } + + DECLARE_STORAGEREPLY(MultiOperationReply, onMultiOperationReply) +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/message/persistence.cpp b/storageapi/src/vespa/storageapi/message/persistence.cpp new file mode 100644 index 00000000000..e39c03cf440 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/persistence.cpp @@ -0,0 +1,358 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/message/persistence.h> + +#include <vespa/vespalib/util/exceptions.h> + +namespace storage { +namespace api { + +IMPLEMENT_COMMAND(PutCommand, PutReply) +IMPLEMENT_REPLY(PutReply) +IMPLEMENT_COMMAND(UpdateCommand, UpdateReply) +IMPLEMENT_REPLY(UpdateReply) +IMPLEMENT_COMMAND(GetCommand, GetReply) +IMPLEMENT_REPLY(GetReply) +IMPLEMENT_COMMAND(RemoveCommand, RemoveReply) +IMPLEMENT_REPLY(RemoveReply) +IMPLEMENT_COMMAND(RevertCommand, RevertReply) +IMPLEMENT_REPLY(RevertReply) + +PutCommand::PutCommand(const document::BucketId& id, + const document::Document::SP& doc, Timestamp time) + : TestAndSetCommand(MessageType::PUT, id), + _doc(doc), + _timestamp(time), + _updateTimestamp(0) +{ + if (_doc.get() == 0) { + throw vespalib::IllegalArgumentException( + "Cannot put a null document", VESPA_STRLOC); + } +} + +StorageCommand::UP +PutCommand::createCopyToForward( + const document::BucketId& bucket, uint64_t timestamp) const +{ + StorageCommand::UP cmd(new PutCommand(bucket, _doc, timestamp)); + static_cast<PutCommand&>(*cmd).setUpdateTimestamp(_updateTimestamp); + return cmd; +} + +vespalib::string +PutCommand::getSummary() const +{ + vespalib::asciistream stream; + stream << "Put(BucketId(0x" << vespalib::hex << getBucketId().getId() << "), " + << _doc->getId().toString() + << ", timestamp " << vespalib::dec << _timestamp + << ')'; + + return stream.str(); +} + +void +PutCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "Put(" << getBucketId() << ", " << _doc->getId() + << ", timestamp " << _timestamp << ", size " + << _doc->serialize()->getLength() << ")"; + if (verbose) { + out << " {\n" << indent << " "; + _doc->print(out, verbose, indent + " "); + out << "\n" << indent << "}" << " : "; + BucketInfoCommand::print(out, verbose, indent); + } +} + +PutReply::PutReply(const PutCommand& cmd, bool wasFoundFlag) + : BucketInfoReply(cmd), + _docId(cmd.getDocumentId()), + _document(cmd.getDocument()), + _timestamp(cmd.getTimestamp()), + _updateTimestamp(cmd.getUpdateTimestamp()), + _wasFound(wasFoundFlag) +{ +} + +void +PutReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "PutReply(" + << _docId.toString() << ", " << getBucketId() << ", timestamp " + << _timestamp; + + if (hasBeenRemapped()) { + out << " (was remapped)"; + } + + out << ")"; + if (verbose) { + out << " : "; + BucketInfoReply::print(out, verbose, indent); + } +} + +UpdateCommand::UpdateCommand(const document::BucketId& id, + const document::DocumentUpdate::SP& update, + Timestamp time) + : TestAndSetCommand(MessageType::UPDATE, id), + _update(update), + _timestamp(time), + _oldTimestamp(0) +{ + if (_update.get() == 0) { + throw vespalib::IllegalArgumentException( + "Cannot update a null update", VESPA_STRLOC); + } +} + +vespalib::string +UpdateCommand::getSummary() const { + vespalib::asciistream stream; + stream << "Update(BucketId(0x" << vespalib::hex << getBucketId().getId() << "), " + << _update->getId().toString() + << ", timestamp " << vespalib::dec << _timestamp; + if (_oldTimestamp != 0) { + stream << ", old timestamp " << _oldTimestamp; + } + stream << ')'; + + return stream.str(); +} + +void +UpdateCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "Update(" << getBucketId() << ", " << _update->getId() + << ", timestamp " << _timestamp; + if (_oldTimestamp != 0) { + out << ", old timestamp " << _oldTimestamp; + } + out << ")"; + if (verbose) { + out << " {\n" << indent << " "; + _update->print(out, verbose, indent + " "); + out << "\n" << indent << "} : "; + BucketInfoCommand::print(out, verbose, indent); + } +} + +StorageCommand::UP +UpdateCommand::createCopyToForward( + const document::BucketId& bucket, uint64_t timestamp) const +{ + StorageCommand::UP cmd(new UpdateCommand(bucket, _update, timestamp)); + static_cast<UpdateCommand&>(*cmd).setOldTimestamp(_oldTimestamp); + return cmd; +} + +UpdateReply::UpdateReply(const UpdateCommand& cmd, Timestamp oldTimestamp) + : BucketInfoReply(cmd), + _docId(cmd.getDocumentId()), + _timestamp(cmd.getTimestamp()), + _oldTimestamp(oldTimestamp), + _consistentNode((uint16_t)-1) +{ +} + +void +UpdateReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "UpdateReply(" + << _docId.toString() << ", " << getBucketId() << ", timestamp " + << _timestamp << ", timestamp of updated doc: " << _oldTimestamp; + + if (_consistentNode != (uint16_t)-1) { + out << " Was inconsistent (best node " << _consistentNode << ")"; + } + + out << ")"; + + if (verbose) { + out << " : "; + BucketInfoReply::print(out, verbose, indent); + } +} + +GetCommand::GetCommand(const document::BucketId& bid, + const document::DocumentId& docId, + const vespalib::stringref & fieldSet, Timestamp before) + : BucketInfoCommand(MessageType::GET, bid), + _docId(docId), + _beforeTimestamp(before), + _fieldSet(fieldSet) +{ +} + +StorageCommand::UP +GetCommand::createCopyToForward( + const document::BucketId& bucket, uint64_t timestamp) const +{ + (void) timestamp; + StorageCommand::UP cmd(new GetCommand( + bucket, _docId, _fieldSet, _beforeTimestamp)); + return cmd; +} + +vespalib::string +GetCommand::getSummary() const +{ + vespalib::asciistream stream; + stream << "Get(BucketId(" << vespalib::hex << getBucketId().getId() << "), " + << _docId.toString() + << ", beforetimestamp " << vespalib::dec << _beforeTimestamp + << ')'; + + return stream.str(); +} + + +void +GetCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "Get(" << getBucketId() << ", " << _docId << ")"; + if (verbose) { + out << " : "; + BucketCommand::print(out, verbose, indent); + } +} + +GetReply::GetReply(const GetCommand& cmd, + const document::Document::SP& doc, + Timestamp lastModified) + : BucketInfoReply(cmd), + _docId(cmd.getDocumentId()), + _fieldSet(cmd.getFieldSet()), + _doc(doc), + _beforeTimestamp(cmd.getBeforeTimestamp()), + _lastModifiedTime(lastModified) +{ +} + +void +GetReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "GetReply(" << getBucketId() << ", " << _docId + << ", timestamp " << _lastModifiedTime << ")"; + if (verbose) { + out << " : "; + BucketReply::print(out, verbose, indent); + } +} + +RemoveCommand::RemoveCommand(const document::BucketId& bid, + const document::DocumentId& docId, + Timestamp timestamp) + : TestAndSetCommand(MessageType::REMOVE, bid), + _docId(docId), + _timestamp(timestamp) +{ +} + +StorageCommand::UP +RemoveCommand::createCopyToForward( + const document::BucketId& bucket, uint64_t timestamp) const +{ + StorageCommand::UP cmd(new RemoveCommand(bucket, _docId, timestamp)); + return cmd; +} + +vespalib::string +RemoveCommand::getSummary() const { + vespalib::asciistream stream; + stream << "Remove(BucketId(0x" << vespalib::hex << getBucketId().getId() << "), " + << _docId.toString() << ", timestamp " << vespalib::dec << _timestamp << ')'; + + return stream.str(); +} +void +RemoveCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "Remove(" << getBucketId() << ", " << _docId + << ", timestamp " << _timestamp << ")"; + if (verbose) { + out << " : "; + BucketInfoCommand::print(out, verbose, indent); + } +} + +RemoveReply::RemoveReply(const RemoveCommand& cmd, Timestamp oldTimestamp) + : BucketInfoReply(cmd), + _docId(cmd.getDocumentId()), + _timestamp(cmd.getTimestamp()), + _oldTimestamp(oldTimestamp) +{ +} + +void +RemoveReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "RemoveReply(" << getBucketId() << ", " << _docId + << ", timestamp " << _timestamp; + if (_oldTimestamp != 0) { + out << ", removed doc from " << _oldTimestamp; + } else { + out << ", not found"; + } + out << ")"; + if (verbose) { + out << " : "; + BucketInfoReply::print(out, verbose, indent); + } +} + +RevertCommand::RevertCommand(const document::BucketId& id, + const std::vector<Timestamp>& revertTokens) + : BucketInfoCommand(MessageType::REVERT, id), + _tokens(revertTokens) +{ +} + +void +RevertCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "Revert(" << getBucketId(); + if (verbose) { + out << ","; + for (uint32_t i=0; i<_tokens.size(); ++i) { + out << "\n" << indent << " " << _tokens[i]; + } + } + out << ")"; + if (verbose) { + out << " : "; + BucketInfoCommand::print(out, verbose, indent); + } +} + +RevertReply::RevertReply(const RevertCommand& cmd) + : BucketInfoReply(cmd), + _tokens(cmd.getRevertTokens()) +{ +} + +void +RevertReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "RevertReply(" << getBucketId() << ")"; + if (verbose) { + out << " : "; + BucketInfoReply::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/message/persistence.h b/storageapi/src/vespa/storageapi/message/persistence.h new file mode 100644 index 00000000000..12d371bf561 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/persistence.h @@ -0,0 +1,363 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @file persistence.h + * + * Persistence related commands, like put, get & remove + */ +#pragma once + +#include <vespa/document/document.h> +#include <vespa/storageapi/messageapi/bucketinforeply.h> +#include <vespa/storageapi/defs.h> +#include <vespa/documentapi/messagebus/messages/testandsetcondition.h> + +namespace storage { +namespace api { + +using documentapi::TestAndSetCondition; + +class TestAndSetCommand : public BucketInfoCommand { + TestAndSetCondition _condition; + +public: + TestAndSetCommand(const MessageType & messageType, + const document::BucketId & id) + : BucketInfoCommand(messageType, id) {} + + void setCondition(const TestAndSetCondition & condition) { _condition = condition; } + const TestAndSetCondition & getCondition() const { return _condition; } + + /** + * Uniform interface to get document id + * Used by test and set to retrieve already existing document + */ + virtual const document::DocumentId & getDocumentId() const = 0; +}; + +/** + * @class PutCommand + * @ingroup message + * + * @brief Command for adding a document to the storage system. + */ +class PutCommand : public TestAndSetCommand { + document::Document::SP _doc; + Timestamp _timestamp; + Timestamp _updateTimestamp; + +public: + PutCommand(const document::BucketId&, const document::Document::SP&, + Timestamp); + + void setTimestamp(Timestamp ts) { _timestamp = ts; } + + /** + * If set, this PUT will only update the header of an existing document, + * rather than writing an entire new PUT. It will only perform the write if + * there exists a document already with the given timestamp. + */ + void setUpdateTimestamp(Timestamp ts) { _updateTimestamp = ts; } + Timestamp getUpdateTimestamp() const { return _updateTimestamp; } + + const document::Document::SP& getDocument() const { return _doc; } + const document::DocumentId& getDocumentId() const override { return _doc->getId(); } + Timestamp getTimestamp() const { return _timestamp; } + + uint32_t getMemoryFootprint() const { + return (_doc.get() ? 4096 : 0) + 20; + } + + vespalib::string getSummary() const override; + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + virtual StorageCommand::UP createCopyToForward( + const document::BucketId& bucket, uint64_t timestamp) const; + + DECLARE_STORAGECOMMAND(PutCommand, onPut); +}; + +/** + * @class PutReply + * @ingroup message + * + * @brief Reply of a put command. + */ +class PutReply : public BucketInfoReply { + document::DocumentId _docId; + document::Document::SP _document; // Not serialized + Timestamp _timestamp; + Timestamp _updateTimestamp; + bool _wasFound; + +public: + explicit PutReply(const PutCommand& cmd, bool wasFound = true); + + const document::DocumentId& getDocumentId() const { return _docId; } + bool hasDocument() const { return _document.get(); } + const document::Document::SP& getDocument() const { return _document; } + Timestamp getTimestamp() const { return _timestamp; }; + Timestamp getUpdateTimestamp() const { return _updateTimestamp; } + + bool isHeadersOnlyPut() const { return (_updateTimestamp != 0); } + bool wasFound() const { return _wasFound; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(PutReply, onPutReply) +}; + +/** + * @class UpdateCommand + * @ingroup message + * + * @brief Command for updating a document to the storage system. + */ +class UpdateCommand : public TestAndSetCommand { + document::DocumentUpdate::SP _update; + Timestamp _timestamp; + Timestamp _oldTimestamp; + +public: + UpdateCommand(const document::BucketId&, + const document::DocumentUpdate::SP&, Timestamp); + + void setTimestamp(Timestamp ts) { _timestamp = ts; } + void setOldTimestamp(Timestamp ts) { _oldTimestamp = ts; } + + const document::DocumentUpdate::SP& getUpdate() const { return _update; } + const document::DocumentId& getDocumentId() const override + { return _update->getId(); } + Timestamp getTimestamp() const { return _timestamp; } + Timestamp getOldTimestamp() const { return _oldTimestamp; } + + uint32_t getMemoryFootprint() const { + return (_update.get() ? 1024 : 0) + 30; + } + + vespalib::string getSummary() const override; + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + virtual StorageCommand::UP createCopyToForward( + const document::BucketId& bucket, uint64_t timestamp) const; + + DECLARE_STORAGECOMMAND(UpdateCommand, onUpdate); +}; + +/** + * @class UpdateReply + * @ingroup message + * + * @brief Reply of a update command. + */ +class UpdateReply : public BucketInfoReply { + document::DocumentId _docId; + Timestamp _timestamp; + Timestamp _oldTimestamp; + uint16_t _consistentNode; + +public: + UpdateReply(const UpdateCommand& cmd, Timestamp oldTimestamp = 0); + + void setOldTimestamp(Timestamp ts) { _oldTimestamp = ts; } + + const document::DocumentId& getDocumentId() const { return _docId; } + Timestamp getTimestamp() const { return _timestamp; } + Timestamp getOldTimestamp() const { return _oldTimestamp; } + + bool wasFound() const { return (_oldTimestamp != 0); } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + /** + * If this update was inconsistent (multiple different timestamps returned), + * set the "best" node. + */ + void setNodeWithNewestTimestamp(uint16_t node) { _consistentNode = node; } + + uint16_t getNodeWithNewestTimestamp() { return _consistentNode; } + + DECLARE_STORAGEREPLY(UpdateReply, onUpdateReply) +}; + + +/** + * @class GetCommand + * @ingroup message + * + * @brief Command for returning a single document. + * + * Normally, the newest version of a document is retrieved. The timestamp can + * be used to retrieve the newest copy, which is not newer than the given + * timestamp. + */ +class GetCommand : public BucketInfoCommand { + document::DocumentId _docId; + Timestamp _beforeTimestamp; + vespalib::string _fieldSet; + +public: + GetCommand(const document::BucketId&, const document::DocumentId&, + const vespalib::stringref & fieldSet, Timestamp before = MAX_TIMESTAMP); + + void setBeforeTimestamp(Timestamp ts) { _beforeTimestamp = ts; } + + const document::DocumentId& getDocumentId() const { return _docId; } + Timestamp getBeforeTimestamp() const { return _beforeTimestamp; } + const vespalib::string& getFieldSet() const { return _fieldSet; } + void setFieldSet(const vespalib::stringref & fieldSet) { _fieldSet = fieldSet; } + + vespalib::string getSummary() const override; + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + virtual StorageCommand::UP createCopyToForward( + const document::BucketId& bucket, uint64_t timestamp) const; + + DECLARE_STORAGECOMMAND(GetCommand, onGet) +}; + +/** + * @class GetReply + * @ingroup message + * + * @brief Reply for a get command. + */ +class GetReply : public BucketInfoReply { + document::DocumentId _docId; // In case of not found, we want id still + vespalib::string _fieldSet; + document::Document::SP _doc; // Null pointer if not found + Timestamp _beforeTimestamp; + Timestamp _lastModifiedTime; + +public: + GetReply(const GetCommand& cmd, + const document::Document::SP& doc = document::Document::SP(), + Timestamp lastModified = 0); + + const document::Document::SP& getDocument() const { return _doc; } + const document::DocumentId& getDocumentId() const { return _docId; } + const vespalib::string& getFieldSet() const { return _fieldSet; } + + Timestamp getLastModifiedTimestamp() const { return _lastModifiedTime; } + Timestamp getBeforeTimestamp() const { return _beforeTimestamp; } + + bool wasFound() const { return (_doc.get() != 0); } + + uint32_t getMemoryFootprint() const { + return (_doc.get() ? 4096 : 0) + 30; + } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(GetReply, onGetReply) +}; + +/** + * @class RemoveCommand + * @ingroup message + * + * @brief Command for removing a document. + */ +class RemoveCommand : public TestAndSetCommand { + document::DocumentId _docId; + Timestamp _timestamp; + +public: + RemoveCommand(const document::BucketId&, const document::DocumentId& docId, + Timestamp timestamp); + + void setTimestamp(Timestamp ts) { _timestamp = ts; } + + const document::DocumentId& getDocumentId() const override { return _docId; } + Timestamp getTimestamp() const { return _timestamp; } + + vespalib::string getSummary() const override; + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + virtual StorageCommand::UP createCopyToForward( + const document::BucketId& bucket, uint64_t timestamp) const; + + DECLARE_STORAGECOMMAND(RemoveCommand, onRemove) +}; + +/** + * @class RemoveReply + * @ingroup message + * + * @brief Reply for a remove command. + */ +class RemoveReply : public BucketInfoReply { + document::DocumentId _docId; + Timestamp _timestamp; + Timestamp _oldTimestamp; + +public: + explicit RemoveReply(const RemoveCommand& cmd, Timestamp oldTimestamp = 0); + + const document::DocumentId& getDocumentId() const { return _docId; } + Timestamp getTimestamp() { return _timestamp; }; + Timestamp getOldTimestamp() const { return _oldTimestamp; } + + void setOldTimestamp(Timestamp oldTimestamp) { _oldTimestamp = oldTimestamp; } + + bool wasFound() const { return (_oldTimestamp != 0); } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(RemoveReply, onRemoveReply) +}; + +/** + * @class RevertCommand + * @ingroup message + * + * @brief Command for reverting a write or remove operation. + */ +class RevertCommand : public BucketInfoCommand { + std::vector<Timestamp> _tokens; + +public: + RevertCommand(const document::BucketId& bucket, + const std::vector<Timestamp>& revertTokens); + + const std::vector<Timestamp>& getRevertTokens() const { return _tokens; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(RevertCommand, onRevert) +}; + +/** + * @class RevertReply + * @ingroup message + * + * @brief Reply for a revert command. + */ +class RevertReply : public BucketInfoReply { + std::vector<Timestamp> _tokens; + +public: + explicit RevertReply(const RevertCommand& cmd); + + const std::vector<Timestamp>& getRevertTokens() const { return _tokens; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(RevertReply, onRevertReply) +}; + +} // api +} // storage + + diff --git a/storageapi/src/vespa/storageapi/message/queryresult.cpp b/storageapi/src/vespa/storageapi/message/queryresult.cpp new file mode 100644 index 00000000000..0bef270b6ce --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/queryresult.cpp @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/message/queryresult.h> + +namespace storage { +namespace api { + +IMPLEMENT_COMMAND(QueryResultCommand, QueryResultReply) +IMPLEMENT_REPLY(QueryResultReply) + +QueryResultCommand::QueryResultCommand() + : StorageCommand(MessageType::QUERYRESULT), + _searchResult(), + _summary() +{ +} + +void +QueryResultCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "QueryResultCommand(" << _searchResult.getHitCount() << " hits)"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +QueryResultReply::QueryResultReply(const QueryResultCommand& cmd) + : StorageReply(cmd) +{ +} + +void +QueryResultReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "QueryResultReply()"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/message/queryresult.h b/storageapi/src/vespa/storageapi/message/queryresult.h new file mode 100644 index 00000000000..c0bed1eee54 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/queryresult.h @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vdslib/container/searchresult.h> +#include <vespa/vdslib/container/documentsummary.h> +#include <vespa/storageapi/message/visitor.h> + +namespace storage { +namespace api { + +/** + * @class QueryResultCommand + * @ingroup message + * + * @brief The result of a searchvisitor. + */ +class QueryResultCommand : public StorageCommand { +public: + QueryResultCommand(); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + uint32_t getMemoryFootprint() const { + return getSearchResult().getSerializedSize() + getDocumentSummary().getSerializedSize(); + } + const vdslib::SearchResult & getSearchResult() const { return _searchResult; } + vdslib::SearchResult & getSearchResult() { return _searchResult; } + const vdslib::DocumentSummary & getDocumentSummary() const { return _summary; } + vdslib::DocumentSummary & getDocumentSummary() { return _summary; } + + DECLARE_STORAGECOMMAND(QueryResultCommand, onQueryResult) +private: + vdslib::SearchResult _searchResult; + vdslib::DocumentSummary _summary; +}; + +/** + * @class QueryResultReply + * @ingroup message + * + * @brief Response to a search result command. + */ +class QueryResultReply : public StorageReply { +public: + explicit QueryResultReply(const QueryResultCommand& command); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(QueryResultReply, onQueryResultReply) +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/message/removelocation.cpp b/storageapi/src/vespa/storageapi/message/removelocation.cpp new file mode 100644 index 00000000000..e0c5d438b6e --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/removelocation.cpp @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/message/removelocation.h> + +namespace storage { +namespace api { + +IMPLEMENT_COMMAND(RemoveLocationCommand, RemoveLocationReply) +IMPLEMENT_REPLY(RemoveLocationReply) + +RemoveLocationCommand::RemoveLocationCommand( + const vespalib::stringref & documentSelection, const document::BucketId& id) + : BucketInfoCommand(MessageType::REMOVELOCATION, id), + _documentSelection(documentSelection) +{ +} + +void +RemoveLocationCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + if (_documentSelection.length()) { + out << "Remove selection(" << _documentSelection << "): "; + } + BucketInfoCommand::print(out, verbose, indent); +} + +RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd) + : BucketInfoReply(cmd) +{ +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/message/removelocation.h b/storageapi/src/vespa/storageapi/message/removelocation.h new file mode 100644 index 00000000000..77e8d42b052 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/removelocation.h @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storageapi/defs.h> +#include <vespa/storageapi/messageapi/storagecommand.h> +#include <vespa/storageapi/messageapi/bucketinforeply.h> + + +namespace storage { +namespace api { + +class RemoveLocationCommand : public BucketInfoCommand +{ +public: + RemoveLocationCommand(const vespalib::stringref & documentSelection, + const document::BucketId&); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + uint32_t getMemoryFootprint() const { + return _documentSelection.length(); + } + + const vespalib::string& getDocumentSelection() const { return _documentSelection; } + + DECLARE_STORAGECOMMAND(RemoveLocationCommand, onRemoveLocation); + +private: + vespalib::string _documentSelection; +}; + +class RemoveLocationReply : public BucketInfoReply +{ +public: + RemoveLocationReply(const RemoveLocationCommand& cmd); + + DECLARE_STORAGEREPLY(RemoveLocationReply, onRemoveLocationReply) +}; + +} +} + diff --git a/storageapi/src/vespa/storageapi/message/searchresult.cpp b/storageapi/src/vespa/storageapi/message/searchresult.cpp new file mode 100644 index 00000000000..b344ce8566a --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/searchresult.cpp @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/message/searchresult.h> + +using vdslib::SearchResult; + +namespace storage { +namespace api { + +IMPLEMENT_COMMAND(SearchResultCommand, SearchResultReply) +IMPLEMENT_REPLY(SearchResultReply) + +SearchResultCommand::SearchResultCommand() + : StorageCommand(MessageType::SEARCHRESULT), + SearchResult() +{ +} + +void +SearchResultCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "SearchResultCommand(" << getHitCount() << " hits)"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +SearchResultReply::SearchResultReply(const SearchResultCommand& cmd) + : StorageReply(cmd) +{ +} + +void +SearchResultReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "SearchResultReply()"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/message/searchresult.h b/storageapi/src/vespa/storageapi/message/searchresult.h new file mode 100644 index 00000000000..ea5a47cf37c --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/searchresult.h @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vdslib/container/searchresult.h> +#include <vespa/storageapi/message/visitor.h> + +namespace storage { +namespace api { + +/** + * @class SearchResultCommand + * @ingroup message + * + * @brief The result of a searchvisitor. + */ +class SearchResultCommand : public StorageCommand, public vdslib::SearchResult { +public: + SearchResultCommand(); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + uint32_t getMemoryFootprint() const { + return getSerializedSize(); + } + + DECLARE_STORAGECOMMAND(SearchResultCommand, onSearchResult) +}; + +/** + * @class SearchResultReply + * @ingroup message + * + * @brief Response to a search result command. + */ +class SearchResultReply : public StorageReply { +public: + explicit SearchResultReply(const SearchResultCommand& command); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(SearchResultReply, onSearchResultReply) +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/message/stat.cpp b/storageapi/src/vespa/storageapi/message/stat.cpp new file mode 100644 index 00000000000..e006b95ed16 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/stat.cpp @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/message/stat.h> + +namespace storage { +namespace api { + +IMPLEMENT_COMMAND(StatBucketCommand, StatBucketReply) +IMPLEMENT_REPLY(StatBucketReply) +IMPLEMENT_COMMAND(GetBucketListCommand, GetBucketListReply) +IMPLEMENT_REPLY(GetBucketListReply) + +StatBucketCommand::StatBucketCommand(const document::BucketId& id, + const vespalib::stringref & documentSelection) + : BucketCommand(MessageType::STATBUCKET, id), + _docSelection(documentSelection) +{ +} + +void +StatBucketCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "StatBucketCommand(" << getBucketId() + << ", selection: " << _docSelection << ")"; + if (verbose) { + out << " : "; + BucketCommand::print(out, verbose, indent); + } +} + +StorageCommand::UP +StatBucketCommand::createCopyToForward( + const document::BucketId& bucket, uint64_t) const +{ + StatBucketCommand::UP cmd(new StatBucketCommand(bucket, _docSelection)); + return std::move(cmd); +} + +StatBucketReply::StatBucketReply(const StatBucketCommand& cmd, + const vespalib::stringref & results) + : BucketReply(cmd), + _results(results) +{ +} + +void +StatBucketReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "StatBucketReply(" << getBucketId(); + if (verbose) { + out << ", result: " << _results << ") : "; + BucketReply::print(out, verbose, indent); + } else { + vespalib::string::size_type pos = _results.find('\n'); + vespalib::string overview; + if (pos != vespalib::string::npos) { + overview = _results.substr(0, pos) + " ..."; + } else { + overview = _results; + } + out << ", result: " << overview << ")"; + } +} + +GetBucketListCommand::GetBucketListCommand(const document::BucketId& id) + : BucketCommand(MessageType::GETBUCKETLIST, id) +{ +} + +void +GetBucketListCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "GetBucketList(" << getBucketId() << ")"; + if (verbose) { + out << " : "; + BucketCommand::print(out, verbose, indent); + } +} + +GetBucketListReply::GetBucketListReply(const GetBucketListCommand& cmd) + : BucketReply(cmd), + _buckets() +{ +} + +void +GetBucketListReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "GetBucketListReply(" << getBucketId() << ", Info on " + << _buckets.size() << " buckets)"; + if (verbose) { + out << " : "; + BucketReply::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/message/stat.h b/storageapi/src/vespa/storageapi/message/stat.h new file mode 100644 index 00000000000..dbad3e9cd7b --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/stat.h @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/storageapi/messageapi/bucketcommand.h> +#include <vespa/storageapi/messageapi/bucketreply.h> + +namespace storage { + +namespace api { + +/** + * \class StatBucketCommand + * \ingroup stat + * + * \brief Command used to get information about a given bucket.. + * + * Command used by stat to get detailed information about a bucket. + */ +class StatBucketCommand : public BucketCommand { +private: + vespalib::string _docSelection; + +public: + StatBucketCommand(const document::BucketId& bucket, + const vespalib::stringref & documentSelection); + + const vespalib::string& getDocumentSelection() const { return _docSelection; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + virtual StorageCommand::UP createCopyToForward( + const document::BucketId&, uint64_t timestamp) const; + + DECLARE_STORAGECOMMAND(StatBucketCommand, onStatBucket); +}; + +class StatBucketReply : public BucketReply { + vespalib::string _results; + +public: + StatBucketReply(const StatBucketCommand&, const vespalib::stringref & results = ""); + + const vespalib::string& getResults() { return _results; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(StatBucketReply, onStatBucketReply) +}; + +/** + * \class GetBucketListCommand + * \ingroup stat + * + * \brief Command used to find actual buckets related to a given one. + * + * Command used by stat to query distributor to find actual buckets contained + * by the given bucket, or buckets that contain the given bucket. (getAll() call + * on bucket database) + */ +class GetBucketListCommand : public BucketCommand { +public: + GetBucketListCommand(const document::BucketId& bucket); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(GetBucketListCommand, onGetBucketList); +}; + +class GetBucketListReply : public BucketReply { +public: + struct BucketInfo { + document::BucketId _bucket; + vespalib::string _bucketInformation; + + BucketInfo(const document::BucketId& id, + const vespalib::stringref & bucketInformation) + : _bucket(id), + _bucketInformation(bucketInformation) + { + } + + bool operator==(const BucketInfo& other) const { + return (_bucket == other._bucket + && _bucketInformation == other._bucketInformation); + } + }; + +private: + std::vector<BucketInfo> _buckets; + +public: + GetBucketListReply(const GetBucketListCommand&); + + std::vector<BucketInfo>& getBuckets() { return _buckets; } + const std::vector<BucketInfo>& getBuckets() const { return _buckets; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(GetBucketListReply, onGetBucketListReply) + +}; + +inline std::ostream& operator<<(std::ostream& out, + const GetBucketListReply::BucketInfo& instance) +{ + out << "BucketInfo(" << instance._bucket << ": " + << instance._bucketInformation << ")"; + return out; +} + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/message/state.cpp b/storageapi/src/vespa/storageapi/message/state.cpp new file mode 100644 index 00000000000..5b81ee72aa4 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/state.cpp @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/storageapi/messageapi/storagemessage.h> + +namespace storage { +namespace api { + +IMPLEMENT_COMMAND(GetNodeStateCommand, GetNodeStateReply) +IMPLEMENT_REPLY(GetNodeStateReply) +IMPLEMENT_COMMAND(SetSystemStateCommand, SetSystemStateReply) +IMPLEMENT_REPLY(SetSystemStateReply) + +GetNodeStateCommand::GetNodeStateCommand(lib::NodeState::UP expectedState) + : StorageCommand(MessageType::GETNODESTATE), + _expectedState(std::move(expectedState)) +{ +} + +void +GetNodeStateCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "GetNodeStateCommand("; + if (_expectedState.get() != 0) { + out << "Expected state: " << *_expectedState; + } + out << ")"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +GetNodeStateReply::GetNodeStateReply(const GetNodeStateCommand& cmd) + : StorageReply(cmd), + _state() +{ +} + +GetNodeStateReply::GetNodeStateReply(const GetNodeStateCommand& cmd, + const lib::NodeState& state) + : StorageReply(cmd), + _state(new lib::NodeState(state)) +{ +} + +void +GetNodeStateReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "GetNodeStateReply("; + if (_state.get()) { + out << "State: " << *_state; + } + out << ")"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +SetSystemStateCommand::SetSystemStateCommand(const lib::ClusterState& state) + : StorageCommand(MessageType::SETSYSTEMSTATE), + _state(state) +{ +} + +void +SetSystemStateCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "SetSystemStateCommand(" << _state << ")"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +SetSystemStateReply::SetSystemStateReply(const SetSystemStateCommand& cmd) + : StorageReply(cmd), + _state(cmd.getSystemState()) +{ +} + +void +SetSystemStateReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "SetSystemStateReply()"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/message/state.h b/storageapi/src/vespa/storageapi/message/state.h new file mode 100644 index 00000000000..d3120943619 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/state.h @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @file state.h + * + * State commands. + */ +#pragma once + +#include <vespa/storageapi/messageapi/storagecommand.h> +#include <vespa/storageapi/messageapi/storagereply.h> +#include <vespa/vdslib/state/clusterstate.h> + +namespace storage { +namespace api { + +/** + * @class GetNodeStateCommand + * @ingroup message + * + * @brief Command for setting node state. No payload + */ +class GetNodeStateCommand : public StorageCommand { + lib::NodeState::UP _expectedState; + +public: + explicit GetNodeStateCommand(lib::NodeState::UP expectedState); + + const lib::NodeState* getExpectedState() const + { return _expectedState.get(); } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(GetNodeStateCommand, onGetNodeState) +}; + +/** + * @class GetNodeStateReply + * @ingroup message + * + * @brief Reply to GetNodeStateCommand + */ +class GetNodeStateReply : public StorageReply { + lib::NodeState::UP _state; + std::string _nodeInfo; + +public: + GetNodeStateReply(const GetNodeStateCommand&); // Only used on makeReply() + GetNodeStateReply(const GetNodeStateCommand&, const lib::NodeState&); + + bool hasNodeState() const { return (_state.get() != 0); } + const lib::NodeState& getNodeState() const { return *_state; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + void setNodeInfo(const std::string& info) { _nodeInfo = info; } + const std::string& getNodeInfo() const { return _nodeInfo; } + + DECLARE_STORAGEREPLY(GetNodeStateReply, onGetNodeStateReply) +}; + +/** + * @class SetSystemStateCommand + * @ingroup message + * + * @brief Command for telling a node about the system state - state of each node + * in the system and state of the system (all ok, no merging, block + * put/get/remove etx) + */ +class SetSystemStateCommand : public StorageCommand { + lib::ClusterState _state; + +public: + explicit SetSystemStateCommand(const lib::ClusterState&); + + const lib::ClusterState& getSystemState() const { return _state; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(SetSystemStateCommand, onSetSystemState) +}; + +/** + * @class SetSystemStateReply + * @ingroup message + * + * @brief Reply received after a SetSystemStateCommand. + */ +class SetSystemStateReply : public StorageReply { + lib::ClusterState _state; + +public: + explicit SetSystemStateReply(const SetSystemStateCommand& cmd); + + // Not serialized. Available locally + const lib::ClusterState& getSystemState() const { return _state; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(SetSystemStateReply, onSetSystemStateReply) +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp new file mode 100644 index 00000000000..f2ffab76326 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/visitor.cpp @@ -0,0 +1,227 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/* $Id$ */ + + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/message/visitor.h> + +namespace storage { +namespace api { + +IMPLEMENT_COMMAND(CreateVisitorCommand, CreateVisitorReply) +IMPLEMENT_REPLY(CreateVisitorReply) +IMPLEMENT_COMMAND(DestroyVisitorCommand, DestroyVisitorReply) +IMPLEMENT_REPLY(DestroyVisitorReply) +IMPLEMENT_COMMAND(VisitorInfoCommand, VisitorInfoReply) +IMPLEMENT_REPLY(VisitorInfoReply) + +CreateVisitorCommand::CreateVisitorCommand(const vespalib::stringref & libraryName, + const vespalib::stringref & instanceId, + const vespalib::stringref & docSelection) + : StorageCommand(MessageType::VISITOR_CREATE), + _libName(libraryName), + _params(), + _controlDestination(), + _dataDestination(), + _docSelection(docSelection), + _buckets(), + _fromTime(0), + _toTime(api::MAX_TIMESTAMP), + _visitorCmdId(getMsgId()), + _instanceId(instanceId), + _visitorId(0), + _visitRemoves(false), + _fieldSet("[all]"), + _visitInconsistentBuckets(false), + _queueTimeout(2000), + _maxPendingReplyCount(2), + _version(50), + _ordering(document::OrderingSpecification::ASCENDING), + _maxBucketsPerVisitor(1) +{ +} + +CreateVisitorCommand::CreateVisitorCommand(const CreateVisitorCommand& o) + : StorageCommand(o), + _libName(o._libName), + _params(o._params), + _controlDestination(o._controlDestination), + _dataDestination(o._dataDestination), + _docSelection(o._docSelection), + _buckets(o._buckets), + _fromTime(o._fromTime), + _toTime(o._toTime), + _visitorCmdId(getMsgId()), + _instanceId(o._instanceId), + _visitorId(o._visitorId), + _visitRemoves(o._visitRemoves), + _fieldSet(o._fieldSet), + _visitInconsistentBuckets(o._visitInconsistentBuckets), + _queueTimeout(o._queueTimeout), + _maxPendingReplyCount(o._maxPendingReplyCount), + _version(o._version), + _ordering(o._ordering), + _maxBucketsPerVisitor(o._maxBucketsPerVisitor) +{ +} + +StorageCommand::UP +CreateVisitorCommand::createCopyToForward( + const document::BucketId& bucket, uint64_t) const +{ + CreateVisitorCommand::UP cmd(new CreateVisitorCommand(*this)); + cmd->_buckets.clear(); + cmd->_buckets.push_back(bucket); + return std::move(cmd); +} + +void +CreateVisitorCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "CreateVisitorCommand(" << _libName << ", " << _docSelection; + if (verbose) { + out << ") {"; + out << "\n" << indent << " Library name: '" << _libName << "'"; + out << "\n" << indent << " Instance Id: '" << _instanceId << "'"; + out << "\n" << indent << " Control Destination: '" + << _controlDestination << "'"; + out << "\n" << indent << " Data Destination: '" + << _dataDestination << "'"; + out << "\n" << indent << " Doc Selection: '" << _docSelection << "'"; + out << "\n" << indent << " Max pending: '" + << _maxPendingReplyCount << "'"; + out << "\n" << indent << " Timeout: " << getTimeout(); + out << "\n" << indent << " Queue timeout: " << _queueTimeout << " ms"; + out << "\n" << indent << " VisitorDispatcher version: '" << _version << "'"; + if (visitRemoves()) { + out << "\n" << indent << " Visiting remove entries too"; + } + + out << "\n" << indent << " Returning fields: " << _fieldSet; + + if (visitInconsistentBuckets()) { + out << "\n" << indent << " Visiting inconsistent buckets"; + } + out << "\n" << indent << " From " << _fromTime << " to " << _toTime; + for (std::vector<document::BucketId>::const_iterator it + = _buckets.begin(); it != _buckets.end(); ++it) + { + out << "\n" << indent << " " << (*it); + } + out << "\n" << indent << " "; + _params.print(out, verbose, indent + " "); + out << "\n" << indent << " " << "ordering(" << _ordering << ")"; + out << "\n" << indent << " Max buckets: '" + << _maxBucketsPerVisitor << "'"; + out << "\n" << indent << "} : "; + StorageCommand::print(out, verbose, indent); + } else if (_buckets.size() == 2) { + out << ", top " << _buckets[0] << ", progress " << _buckets[1] << ")"; + } else { + out << ", " << _buckets.size() << " buckets)"; + } + +} + +CreateVisitorReply::CreateVisitorReply(const CreateVisitorCommand& cmd) + : StorageReply(cmd), + _lastBucket(document::BucketId(INT_MAX)) +{ +} + +void +CreateVisitorReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "CreateVisitorReply(last=" << _lastBucket << ")"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +DestroyVisitorCommand::DestroyVisitorCommand(const vespalib::stringref & instanceId) + : StorageCommand(MessageType::VISITOR_DESTROY), + _instanceId(instanceId) +{ +} + +void +DestroyVisitorCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "DestroyVisitorCommand(" << _instanceId << ")"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +DestroyVisitorReply::DestroyVisitorReply(const DestroyVisitorCommand& cmd) + : StorageReply(cmd) +{ +} + +void +DestroyVisitorReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "DestroyVisitorReply()"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +VisitorInfoCommand::VisitorInfoCommand() + : StorageCommand(MessageType::VISITOR_INFO), + _completed(false), + _bucketsCompleted(), + _error(ReturnCode::OK) +{ +} + +void +VisitorInfoCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "VisitorInfoCommand("; + if (_completed) { out << "completed"; } + if (_error.failed()) { + out << _error; + } + if (verbose) { + out << ") : "; + StorageCommand::print(out, verbose, indent); + } else { + if (!_bucketsCompleted.empty()) { + out << _bucketsCompleted.size() << " buckets completed"; + } + out << ")"; + } +} + +VisitorInfoReply::VisitorInfoReply(const VisitorInfoCommand& cmd) + : StorageReply(cmd), + _completed(cmd.visitorCompleted()) +{ +} + +void +VisitorInfoReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "VisitorInfoReply("; + if (_completed) { out << "completed"; } + if (verbose) { + out << ") : "; + StorageReply::print(out, verbose, indent); + } else { + out << ")"; + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/message/visitor.h b/storageapi/src/vespa/storageapi/message/visitor.h new file mode 100644 index 00000000000..10e7aa65e40 --- /dev/null +++ b/storageapi/src/vespa/storageapi/message/visitor.h @@ -0,0 +1,269 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @file storageapi/message/visitor.h + * + * Messages related to visitors, used by the visitor manager. + */ + +#pragma once + +#include <vespa/storageapi/defs.h> +#include <vespa/document/bucket/bucketid.h> +#include <vespa/vdslib/container/parameters.h> +#include <vespa/vdslib/container/visitorstatistics.h> +#include <vespa/storageapi/messageapi/storagecommand.h> +#include <vespa/storageapi/messageapi/storagereply.h> +#include <vespa/document/select/orderingspecification.h> + +namespace storage { +namespace api { + +/** + * @class CreateVisitorCommand + * @ingroup message + * + * @brief Command for creating a visitor. + */ +class CreateVisitorCommand : public StorageCommand { +private: + vespalib::string _libName; // Name of visitor library to use, ie. DumpVisitor.so + vdslib::Parameters _params; + + vespalib::string _controlDestination; + vespalib::string _dataDestination; + + vespalib::string _docSelection; + std::vector<document::BucketId> _buckets; + Timestamp _fromTime; + Timestamp _toTime; + + uint32_t _visitorCmdId; + vespalib::string _instanceId; + VisitorId _visitorId; // Set on storage node + + bool _visitRemoves; + vespalib::string _fieldSet; + bool _visitInconsistentBuckets; + + uint32_t _queueTimeout; + uint32_t _maxPendingReplyCount; + uint32_t _version; + + document::OrderingSpecification::Order _ordering; + uint32_t _maxBucketsPerVisitor; + +public: + CreateVisitorCommand(const vespalib::stringref & libraryName, + const vespalib::stringref & instanceId, + const vespalib::stringref & docSelection); + + /** Create another command with similar visitor settings. */ + CreateVisitorCommand(const CreateVisitorCommand& template_); + + void setVisitorCmdId(uint32_t id) { _visitorCmdId = id; } + void setControlDestination(const vespalib::stringref & d) + { _controlDestination = d; } + void setDataDestination(const vespalib::stringref & d) { _dataDestination = d; } + void setParameters(const vdslib::Parameters& params) { _params = params; } + void setMaximumPendingReplyCount(uint32_t count) + { _maxPendingReplyCount = count; } + void setFieldSet(const vespalib::stringref& fieldSet) + { _fieldSet = fieldSet; } + void setVisitRemoves(bool value = true) { _visitRemoves = value; } + void setVisitInconsistentBuckets(bool visitInconsistent = true) + { _visitInconsistentBuckets = visitInconsistent; } + void addBucketToBeVisited(const document::BucketId& id) + { _buckets.push_back(id); } + void setVisitorId(const VisitorId id) { _visitorId = id; } + void setInstanceId(const vespalib::stringref & id) { _instanceId = id; } + void setQueueTimeout(uint32_t milliSecs) { _queueTimeout = milliSecs; } + void setFromTime(Timestamp ts) { _fromTime = ts; } + void setToTime(Timestamp ts) { _toTime = ts; } + + VisitorId getVisitorId() const { return _visitorId; } + uint32_t getVisitorCmdId() const { return _visitorCmdId; } + const vespalib::string & getLibraryName() const { return _libName; } + const vespalib::string & getInstanceId() const { return _instanceId; } + const vespalib::string & getControlDestination() const + { return _controlDestination; } + const vespalib::string & getDataDestination() const { return _dataDestination; } + const vespalib::string & getDocumentSelection() const { return _docSelection; } + const vdslib::Parameters& getParameters() const { return _params; } + vdslib::Parameters& getParameters() { return _params; } + uint32_t getMaximumPendingReplyCount() const + { return _maxPendingReplyCount; } + const std::vector<document::BucketId>& getBuckets() const + { return _buckets; } + Timestamp getFromTime() const { return _fromTime; } + Timestamp getToTime() const { return _toTime; } + std::vector<document::BucketId>& getBuckets() { return _buckets; } + bool visitRemoves() const { return _visitRemoves; } + const vespalib::string& getFieldSet() const { return _fieldSet; } + bool visitInconsistentBuckets() const { return _visitInconsistentBuckets; } + // In millisec + uint32_t getQueueTimeout() const { return _queueTimeout; } + + void setVisitorDispatcherVersion(uint32_t version) { _version = version; } + uint32_t getVisitorDispatcherVersion() const { return _version; } + + void setVisitorOrdering(document::OrderingSpecification::Order ordering) { _ordering = ordering; } + document::OrderingSpecification::Order getVisitorOrdering() const { return _ordering; } + void setMaxBucketsPerVisitor(uint32_t max) + { _maxBucketsPerVisitor = max; } + uint32_t getMaxBucketsPerVisitor() const + { return _maxBucketsPerVisitor; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + virtual StorageCommand::UP createCopyToForward( + const document::BucketId&, uint64_t timestamp) const; + + DECLARE_STORAGECOMMAND(CreateVisitorCommand, onCreateVisitor) +}; + +/** + * @class CreateVisitorReply + * @ingroup message + * + * @brief Response to a create visitor command. + */ +class CreateVisitorReply : public StorageReply { +private: + document::BucketId _lastBucket; + vdslib::VisitorStatistics _visitorStatistics; + +public: + explicit CreateVisitorReply(const CreateVisitorCommand& cmd); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + void setLastBucket(const document::BucketId& lastBucket) { _lastBucket = lastBucket; } + + const document::BucketId& getLastBucket() const { return _lastBucket; } + + void setVisitorStatistics(const vdslib::VisitorStatistics& stats) { _visitorStatistics = stats; } + + const vdslib::VisitorStatistics& getVisitorStatistics() const { return _visitorStatistics; } + + DECLARE_STORAGEREPLY(CreateVisitorReply, onCreateVisitorReply) +}; + +/** + * @class DestroyVisitorCommand + * @ingroup message + * + * @brief Command for removing a visitor. + */ +class DestroyVisitorCommand : public StorageCommand { +private: + vespalib::string _instanceId; + +public: + explicit DestroyVisitorCommand(const vespalib::stringref & instanceId); + + const vespalib::string & getInstanceId() const { return _instanceId; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(DestroyVisitorCommand, onDestroyVisitor) +}; + +/** + * @class DestroyVisitorReply + * @ingroup message + * + * @brief Response to a destroy visitor command. + */ +class DestroyVisitorReply : public StorageReply { +public: + explicit DestroyVisitorReply(const DestroyVisitorCommand& cmd); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(DestroyVisitorReply, onDestroyVisitorReply) +}; + +/** + * @class VisitorInfoCommand + * @ingroup message + * + * @brief Sends status information of an ongoing visitor. + * + * Includes three different kinds of data. + * - Notification when visiting is complete. + * - Notification when individual buckets have been completely visited. + * (Including the timestamp of the newest document visited) + * - Notification that some error condition arose during visiting. + */ +class VisitorInfoCommand : public StorageCommand { +public: + struct BucketTimestampPair { + document::BucketId bucketId; + Timestamp timestamp; + + BucketTimestampPair() : bucketId(), timestamp(0) {} + BucketTimestampPair(const document::BucketId& bucket, + const Timestamp& ts) + : bucketId(bucket), timestamp(ts) {} + + bool operator==(const BucketTimestampPair& other) const { + return (bucketId == other.bucketId && timestamp && other.timestamp); + } + }; + +private: + bool _completed; + std::vector<BucketTimestampPair> _bucketsCompleted; + ReturnCode _error; + +public: + VisitorInfoCommand(); + + void setErrorCode(const ReturnCode& code) { _error = code; } + void setCompleted() { _completed = true; } + void setBucketCompleted(const document::BucketId& id, Timestamp lastVisited) + { + _bucketsCompleted.push_back(BucketTimestampPair(id, lastVisited)); + } + void setBucketsCompleted(const std::vector<BucketTimestampPair>& bc) + { _bucketsCompleted = bc; } + + const ReturnCode& getErrorCode() const { return _error; } + const std::vector<BucketTimestampPair>& getCompletedBucketsList() const + { return _bucketsCompleted; } + bool visitorCompleted() const { return _completed; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGECOMMAND(VisitorInfoCommand, onVisitorInfo) +}; + +inline std::ostream& +operator<<(std::ostream& out, + const VisitorInfoCommand::BucketTimestampPair& pair) +{ + return out << pair.bucketId << " - " << pair.timestamp; +} + +class VisitorInfoReply : public StorageReply { + bool _completed; + +public: + VisitorInfoReply(const VisitorInfoCommand& cmd); + + bool visitorCompleted() const { return _completed; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + DECLARE_STORAGEREPLY(VisitorInfoReply, onVisitorInfoReply) +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/messageapi/.gitignore b/storageapi/src/vespa/storageapi/messageapi/.gitignore new file mode 100644 index 00000000000..6e555686642 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/.gitignore @@ -0,0 +1,6 @@ +*.lo +.depend +.depend.NEW +.deps +.libs +Makefile diff --git a/storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt b/storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt new file mode 100644 index 00000000000..7b272d672ed --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt @@ -0,0 +1,13 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storageapi_messageapi OBJECT + SOURCES + returncode.cpp + storagemessage.cpp + storagecommand.cpp + storagereply.cpp + bucketcommand.cpp + bucketreply.cpp + bucketinfocommand.cpp + bucketinforeply.cpp + DEPENDS +) diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp new file mode 100644 index 00000000000..465833a7abf --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/messageapi/bucketcommand.h> + +namespace storage { +namespace api { + +void +BucketCommand::print(std::ostream& out, + bool verbose, const std::string& indent) const +{ + out << "BucketCommand(" << _bucket; + if (hasBeenRemapped()) { + out << " <- " << _originalBucket; + } + out << ")"; + if (verbose) { + out << " : "; + StorageCommand::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketcommand.h b/storageapi/src/vespa/storageapi/messageapi/bucketcommand.h new file mode 100644 index 00000000000..f4f5909fccd --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/bucketcommand.h @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class storage::api::BucketCommand + * @ingroup messageapi + * + * @brief Superclass for storage commands that operate towards a single bucket. + */ + +#pragma once + +#include <vespa/document/bucket/bucketid.h> +#include <vespa/storageapi/messageapi/storagecommand.h> + +namespace storage { +namespace api { + +class BucketCommand : public StorageCommand { + document::BucketId _bucket; + document::BucketId _originalBucket; + +protected: + BucketCommand(const MessageType& type, const document::BucketId& id) + : StorageCommand(type), _bucket(id), _originalBucket() {} + +public: + DECLARE_POINTER_TYPEDEFS(BucketCommand); + + void remapBucketId(const document::BucketId& bucket) { + if (_originalBucket.getRawId() == 0) _originalBucket = _bucket; + _bucket = bucket; + } + + document::BucketId getBucketId() const { return _bucket; } + bool hasBeenRemapped() const { return (_originalBucket.getRawId() != 0); } + const document::BucketId& getOriginalBucketId() const + { return _originalBucket; } + + virtual void print(std::ostream& out, + bool verbose, const std::string& indent) const; + + virtual bool hasSingleBucketId() const { return true; } + +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp new file mode 100644 index 00000000000..feb03e58c71 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/messageapi/bucketinfocommand.h> + +#include <vespa/storageapi/message/persistence.h> + +namespace storage { +namespace api { + +void +BucketInfoCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "BucketInfoCommand()"; + if (verbose) { + out << " : "; + BucketCommand::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h b/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h new file mode 100644 index 00000000000..4579d83cfed --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class storage::api::BucketInfoCommand + * @ingroup messageapi + * + * @brief Superclass for storage commands that returns bucket info. + * + * This class doesn't add any functionality now, other than being able to check + * if a message is an instance of this class. But we want commands and replies + * to be in the same inheritance structure, and the reply adds functionality. + */ + +#pragma once + +#include <vespa/storageapi/messageapi/bucketcommand.h> + +namespace storage { +namespace api { + +class BucketInfoCommand : public BucketCommand { +protected: + BucketInfoCommand(const MessageType& type, const document::BucketId& id) + : BucketCommand(type, id) {} + +public: + DECLARE_POINTER_TYPEDEFS(BucketInfoCommand); + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp new file mode 100644 index 00000000000..99736e7eaef --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/messageapi/bucketinforeply.h> + +namespace storage { +namespace api { + +BucketInfoReply::BucketInfoReply(const BucketInfoCommand& cmd, + const ReturnCode& code) + : BucketReply(cmd, code), + _result() +{ +} + +void +BucketInfoReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "BucketInfoReply(" << _result << ")"; + if (verbose) { + out << " : "; + BucketReply::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h b/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h new file mode 100644 index 00000000000..0a5c435ade3 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class storage::api::BucketInfoReply + * @ingroup messageapi + * + * @brief Superclass for storage replies which returns bucket info in reply. + * + * A bucket info reply contains information about the state of a bucket. This + * can be altered from before the operation if this was a write operation or if + * the bucket was repaired in the process. + */ + +#pragma once + +#include <vespa/storageapi/buckets/bucketinfo.h> +#include <vespa/storageapi/messageapi/bucketreply.h> +#include <vespa/storageapi/messageapi/bucketinfocommand.h> + +namespace storage { +namespace api { + +class BucketInfoReply : public BucketReply { + BucketInfo _result; + +protected: + BucketInfoReply(const BucketInfoCommand& cmd, + const ReturnCode& code = ReturnCode(ReturnCode::OK)); + +public: + DECLARE_POINTER_TYPEDEFS(BucketInfoReply); + + const BucketInfo& getBucketInfo() const { return _result; }; + void setBucketInfo(const BucketInfo& info) { _result = info; } + + /** Overload this to get more descriptive message output. */ + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp new file mode 100644 index 00000000000..573d0e9fbbb --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/messageapi/bucketreply.h> + +#include <vespa/storageapi/messageapi/bucketcommand.h> + +namespace storage { +namespace api { + +BucketReply::BucketReply(const BucketCommand& cmd, + const ReturnCode& code) + : StorageReply(cmd, code), + _bucket(cmd.getBucketId()), + _originalBucket(cmd.getOriginalBucketId()) +{ +} + +void +BucketReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + out << "BucketReply(" << _bucket; + if (hasBeenRemapped()) { + out << " <- " << _originalBucket; + } + out << ")"; + if (verbose) { + out << " : "; + StorageReply::print(out, verbose, indent); + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketreply.h b/storageapi/src/vespa/storageapi/messageapi/bucketreply.h new file mode 100644 index 00000000000..d2605604568 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/bucketreply.h @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class storage::api::BucketReply + * @ingroup messageapi + * + * @brief Superclass for storage replies which operates on single bucket. + */ + +#pragma once + +#include <vespa/document/bucket/bucketid.h> +#include <vespa/storageapi/messageapi/storagereply.h> + +namespace storage { +namespace api { + +class BucketCommand; + +class BucketReply : public StorageReply { + document::BucketId _bucket; + document::BucketId _originalBucket; + +protected: + BucketReply(const BucketCommand& cmd, + const ReturnCode& code = ReturnCode(ReturnCode::OK)); + +public: + DECLARE_POINTER_TYPEDEFS(BucketReply); + + document::BucketId getBucketId() const { return _bucket; } + virtual bool hasSingleBucketId() const { return true; } + + bool hasBeenRemapped() const { return (_originalBucket.getRawId() != 0); } + const document::BucketId& getOriginalBucketId() const + { return _originalBucket; } + + /** The deserialization code need access to set the remapping. */ + void remapBucketId(const document::BucketId& bucket) { + if (_originalBucket.getRawId() == 0) _originalBucket = _bucket; + _bucket = bucket; + } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h b/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h new file mode 100644 index 00000000000..0687200f9fb --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storageapi/messageapi/bucketinfocommand.h> + +namespace storage { +namespace api { + +class MaintenanceCommand : public BucketInfoCommand +{ +public: + MaintenanceCommand(const MessageType& type, const document::BucketId& id) + : BucketInfoCommand(type, id) {} + + const vespalib::string& getReason() const { return _reason; }; + + void setReason(const vespalib::stringref & reason) { _reason = reason; }; + +protected: + vespalib::string _reason; +}; + +} + +} + diff --git a/storageapi/src/vespa/storageapi/messageapi/messagehandler.h b/storageapi/src/vespa/storageapi/messageapi/messagehandler.h new file mode 100644 index 00000000000..66873dfba27 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/messagehandler.h @@ -0,0 +1,367 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class storage::api::MessageHandler + * @ingroup messageapi + * + * @brief Class to prevent manual casting and switches of message types. + * + * MessageHandler defines an interface for processing StorageMessage objects + * of various subclasses. + * + * @version $Id$ + */ + +#pragma once + + +namespace storage { +namespace api { + +// Commands + +class GetCommand; // Retrieve document +class PutCommand; // Add document +class UpdateCommand; // Update document +class RemoveCommand; // Remove document +class RevertCommand; // Revert put/remove operation +class MultiOperationCommand; // Multi put/remove/update operation +class BatchPutRemoveCommand; +class BatchDocumentUpdateCommand; + +class CreateVisitorCommand; // Create a new visitor +class DestroyVisitorCommand; // Destroy a running visitor +class VisitorInfoCommand; // Sends visitor info to visitor controller +class DocBlockCommand; // A block of documents visited +class MapVisitorCommand; +class SearchResultCommand; +class DocumentSummaryCommand; +class QueryResultCommand; + +class InternalCommand; + +class CreateBucketCommand; +class DeleteBucketCommand; +class MergeBucketCommand; +class GetBucketDiffCommand; +class ApplyBucketDiffCommand; +class SplitBucketCommand; +class JoinBucketsCommand; +class SetBucketStateCommand; + +class RequestBucketInfoCommand; +class NotifyBucketChangeCommand; +class SetNodeStateCommand; +class GetNodeStateCommand; +class SetSystemStateCommand; +class GetSystemStateCommand; +class GetBucketNodesCommand; +class BucketsAddedCommand; +class BucketsRemovedCommand; + +// Replies + +class GetReply; +class PutReply; +class UpdateReply; +class RemoveReply; +class RevertReply; +class MultiOperationReply; +class BatchPutRemoveReply; +class BatchDocumentUpdateReply; + +class CreateVisitorReply; +class DestroyVisitorReply; +class VisitorInfoReply; +class DocBlockReply; +class MapVisitorReply; +class SearchResultReply; +class DocumentSummaryReply; +class QueryResultReply; + +class InternalReply; + +class CreateBucketReply; +class DeleteBucketReply; +class MergeBucketReply; +class GetBucketDiffReply; +class ApplyBucketDiffReply; +class SplitBucketReply; +class JoinBucketsReply; +class SetBucketStateReply; + +class RequestBucketInfoReply; +class NotifyBucketChangeReply; +class SetNodeStateReply; +class GetNodeStateReply; +class SetSystemStateReply; +class GetSystemStateReply; +class GetBucketNodesReply; +class BucketsAddedReply; +class BucketsRemovedReply; + +class StatBucketCommand; +class StatBucketReply; +class GetBucketListCommand; +class GetBucketListReply; + +class DocumentListCommand; +class DocumentListReply; + +class EmptyBucketsCommand; +class EmptyBucketsReply; + +class RemoveLocationCommand; +class RemoveLocationReply; + +#define _INTERNAL_DEF_ON_MC(m, c) virtual bool m(const std::shared_ptr<storage::api::c> & ) +#define _INTERNAL_DEF_IMPL_ON_MC(m, c) virtual bool m(const std::shared_ptr<storage::api::c> & ) { return false; } +#define _INTERNAL_IMPL_ON_MC(cl, m, c, p) bool cl::m(const std::shared_ptr<storage::api::c> & p) +#define DEF_IMPL_MSG_COMMAND_H(m) _INTERNAL_DEF_IMPL_ON_MC(on##m, m##Command) +#define DEF_IMPL_MSG_REPLY_H(m) _INTERNAL_DEF_IMPL_ON_MC(on##m##Reply, m##Reply) +#define DEF_MSG_COMMAND_H(m) _INTERNAL_DEF_ON_MC(on##m, m##Command) +#define DEF_MSG_REPLY_H(m) _INTERNAL_DEF_ON_MC(on##m##Reply, m##Reply) +#define IMPL_MSG_COMMAND_ARG_H(cl, m, p) _INTERNAL_IMPL_ON_MC(cl, on##m, m##Command, p) +#define IMPL_MSG_REPLY_ARG_H(cl, m, p) _INTERNAL_IMPL_ON_MC(cl, on##m##Reply, m##Reply, p) +#define IMPL_MSG_COMMAND_H(cl, m) IMPL_MSG_COMMAND_ARG_H(cl, m, cmd) +#define IMPL_MSG_REPLY_H(cl, m) IMPL_MSG_REPLY_ARG_H(cl, m, reply) +#define ON_M(m) DEF_IMPL_MSG_COMMAND_H(m); DEF_IMPL_MSG_REPLY_H(m) + +class MessageHandler { +public: + // Basic operations + virtual bool onGet(const std::shared_ptr<api::GetCommand>&) + { return false; } + virtual bool onGetReply(const std::shared_ptr<api::GetReply>&) + { return false; } + virtual bool onPut(const std::shared_ptr<api::PutCommand>&) + { return false; } + virtual bool onPutReply(const std::shared_ptr<api::PutReply>&) + { return false; } + virtual bool onUpdate(const std::shared_ptr<api::UpdateCommand>&) + { return false; } + virtual bool onUpdateReply(const std::shared_ptr<api::UpdateReply>&) + { return false; } + virtual bool onRemove(const std::shared_ptr<api::RemoveCommand>&) + { return false; } + virtual bool onRemoveReply(const std::shared_ptr<api::RemoveReply>&) + { return false; } + virtual bool onRevert(const std::shared_ptr<api::RevertCommand>&) + { return false; } + virtual bool onRevertReply(const std::shared_ptr<api::RevertReply>&) + { return false; } + virtual bool onMultiOperation( + const std::shared_ptr<api::MultiOperationCommand>&) + { return false; } + virtual bool onMultiOperationReply( + const std::shared_ptr<api::MultiOperationReply>&) + { return false; } + virtual bool onBatchPutRemove( + const std::shared_ptr<api::BatchPutRemoveCommand>&) + { return false; } + virtual bool onBatchPutRemoveReply( + const std::shared_ptr<api::BatchPutRemoveReply>&) + { return false; } + virtual bool onBatchDocumentUpdate( + const std::shared_ptr<api::BatchDocumentUpdateCommand>&) + { return false; } + virtual bool onBatchDocumentUpdateReply( + const std::shared_ptr<api::BatchDocumentUpdateReply>&) + { return false; } + + // Visiting + virtual bool onCreateVisitor( + const std::shared_ptr<api::CreateVisitorCommand>&) + { return false; } + virtual bool onCreateVisitorReply( + const std::shared_ptr<api::CreateVisitorReply>&) + { return false; } + virtual bool onDestroyVisitor( + const std::shared_ptr<api::DestroyVisitorCommand>&) + { return false; } + virtual bool onDestroyVisitorReply( + const std::shared_ptr<api::DestroyVisitorReply>&) + { return false; } + virtual bool onVisitorInfo( + const std::shared_ptr<api::VisitorInfoCommand>&) + { return false; } + virtual bool onVisitorInfoReply( + const std::shared_ptr<api::VisitorInfoReply>&) + { return false; } + virtual bool onDocBlock( + const std::shared_ptr<api::DocBlockCommand>&) + { return false; } + virtual bool onDocBlockReply( + const std::shared_ptr<api::DocBlockReply>&) + { return false; } + virtual bool onMapVisitor( + const std::shared_ptr<api::MapVisitorCommand>&) + { return false; } + virtual bool onMapVisitorReply( + const std::shared_ptr<api::MapVisitorReply>&) + { return false; } + virtual bool onSearchResult( + const std::shared_ptr<api::SearchResultCommand>&) + { return false; } + virtual bool onSearchResultReply( + const std::shared_ptr<api::SearchResultReply>&) + { return false; } + virtual bool onQueryResult( + const std::shared_ptr<api::QueryResultCommand>&) + { return false; } + virtual bool onQueryResultReply( + const std::shared_ptr<api::QueryResultReply>&) + { return false; } + virtual bool onDocumentSummary( + const std::shared_ptr<api::DocumentSummaryCommand>&) + { return false; } + virtual bool onDocumentSummaryReply( + const std::shared_ptr<api::DocumentSummaryReply>&) + { return false; } + virtual bool onDocumentList( + const std::shared_ptr<api::DocumentListCommand>&) + { return false; } + virtual bool onDocumentListReply( + const std::shared_ptr<api::DocumentListReply>&) + { return false; } + virtual bool onEmptyBuckets( + const std::shared_ptr<api::EmptyBucketsCommand>&) + { return false; } + virtual bool onEmptyBucketsReply( + const std::shared_ptr<api::EmptyBucketsReply>&) + { return false; } + + + virtual bool onInternal(const std::shared_ptr<api::InternalCommand>&) + { return false; } + virtual bool onInternalReply( + const std::shared_ptr<api::InternalReply>&) + { return false; } + + virtual bool onCreateBucket( + const std::shared_ptr<api::CreateBucketCommand>&) + { return false; } + virtual bool onCreateBucketReply( + const std::shared_ptr<api::CreateBucketReply>&) + { return false; } + virtual bool onDeleteBucket( + const std::shared_ptr<api::DeleteBucketCommand>&) + { return false; } + virtual bool onDeleteBucketReply( + const std::shared_ptr<api::DeleteBucketReply>&) + { return false; } + virtual bool onMergeBucket( + const std::shared_ptr<api::MergeBucketCommand>&) + { return false; } + virtual bool onMergeBucketReply( + const std::shared_ptr<api::MergeBucketReply>&) + { return false; } + virtual bool onGetBucketDiff( + const std::shared_ptr<api::GetBucketDiffCommand>&) + { return false; } + virtual bool onGetBucketDiffReply( + const std::shared_ptr<api::GetBucketDiffReply>&) + { return false; } + virtual bool onApplyBucketDiff( + const std::shared_ptr<api::ApplyBucketDiffCommand>&) + { return false; } + virtual bool onApplyBucketDiffReply( + const std::shared_ptr<api::ApplyBucketDiffReply>&) + { return false; } + virtual bool onSplitBucket( + const std::shared_ptr<api::SplitBucketCommand>&) + { return false; } + virtual bool onSplitBucketReply( + const std::shared_ptr<api::SplitBucketReply>&) + { return false; } + virtual bool onJoinBuckets( + const std::shared_ptr<api::JoinBucketsCommand>&) + { return false; } + virtual bool onJoinBucketsReply( + const std::shared_ptr<api::JoinBucketsReply>&) + { return false; } + virtual bool onSetBucketState( + const std::shared_ptr<api::SetBucketStateCommand>&) + { return false; } + virtual bool onSetBucketStateReply( + const std::shared_ptr<api::SetBucketStateReply>&) + { return false; } + + virtual bool onRequestBucketInfo( + const std::shared_ptr<api::RequestBucketInfoCommand>&) + { return false; } + virtual bool onRequestBucketInfoReply( + const std::shared_ptr<api::RequestBucketInfoReply>&) + { return false; } + virtual bool onNotifyBucketChange( + const std::shared_ptr<api::NotifyBucketChangeCommand>&) + { return false; } + virtual bool onNotifyBucketChangeReply( + const std::shared_ptr<api::NotifyBucketChangeReply>&) + { return false; } + virtual bool onSetNodeState( + const std::shared_ptr<api::SetNodeStateCommand>&) + { return false; } + virtual bool onSetNodeStateReply( + const std::shared_ptr<api::SetNodeStateReply>&) + { return false; } + virtual bool onGetNodeState( + const std::shared_ptr<api::GetNodeStateCommand>&) + { return false; } + virtual bool onGetNodeStateReply( + const std::shared_ptr<api::GetNodeStateReply>&) + { return false; } + virtual bool onSetSystemState( + const std::shared_ptr<api::SetSystemStateCommand>&) + { return false; } + virtual bool onSetSystemStateReply( + const std::shared_ptr<api::SetSystemStateReply>&) + { return false; } + virtual bool onGetSystemState( + const std::shared_ptr<api::GetSystemStateCommand>&) + { return false; } + virtual bool onGetSystemStateReply( + const std::shared_ptr<api::GetSystemStateReply>&) + { return false; } + virtual bool onBucketsAdded( + const std::shared_ptr<api::BucketsAddedCommand>&) + { return false; } + virtual bool onBucketsAddedReply( + const std::shared_ptr<api::BucketsAddedReply>&) + { return false; } + virtual bool onBucketsRemoved( + const std::shared_ptr<api::BucketsRemovedCommand>&) + { return false; } + virtual bool onBucketsRemovedReply( + const std::shared_ptr<api::BucketsRemovedReply>&) + { return false; } + + virtual bool onStatBucket( + const std::shared_ptr<api::StatBucketCommand>&) + { return false; } + virtual bool onStatBucketReply( + const std::shared_ptr<api::StatBucketReply>&) + { return false; } + + virtual bool onGetBucketList( + const std::shared_ptr<api::GetBucketListCommand>&) + { return false; } + virtual bool onGetBucketListReply( + const std::shared_ptr<api::GetBucketListReply>&) + { return false; } + + virtual bool onRemoveLocation( + const std::shared_ptr<api::RemoveLocationCommand>&) + { return false; } + + virtual bool onRemoveLocationReply( + const std::shared_ptr<api::RemoveLocationReply>&) + { return false; } + + virtual ~MessageHandler() {} +}; + +#undef ON_M + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp new file mode 100644 index 00000000000..6b2a80563c8 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp @@ -0,0 +1,181 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/messageapi/returncode.h> +#include <vespa/messagebus/errorcode.h> + +#include <vespa/document/util/bytebuffer.h> +#include <vespa/fnet/frt/error.h> +#include <iostream> +#include <vector> + +namespace storage { +namespace api { + +ReturnCode::ReturnCode() + : _result(OK), + _message() +{ +} + +ReturnCode::ReturnCode(Result result, const vespalib::stringref & msg) + : _result(result), + _message(msg) +{ +} + +ReturnCode::ReturnCode(const document::DocumentTypeRepo &repo, + document::ByteBuffer& buffer) + : _result(OK), + _message() +{ + deserialize(repo, buffer); +} + +void ReturnCode:: +onDeserialize(const document::DocumentTypeRepo &, document::ByteBuffer& buffer) +{ + int32_t result; + buffer.getInt(result); + _result = static_cast<Result>(result); + int32_t size; + buffer.getInt(size); + const char * p = buffer.getBufferAtPos(); + buffer.incPos(size); + _message.assign(p, size); +} + +void ReturnCode::onSerialize(document::ByteBuffer& buffer) const +{ + buffer.putInt(_result); + buffer.putInt(_message.size()); + buffer.putBytes(_message.c_str(), _message.size()); +} + +size_t ReturnCode::getSerializedSize() const +{ + return 2 * sizeof(int32_t) + _message.size(); +} + +void +ReturnCode::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + out << "ReturnCode(" << ReturnCode::getResultString(getResult()); + if (getMessage().size() > 0) out << ", " << getMessage(); + out << ")"; +} + +vespalib::string ReturnCode::getResultString(Result result) { + return documentapi::DocumentProtocol::getErrorName(result); +} + +bool +ReturnCode::isBusy() const +{ + // Casting to suppress -Wswitch since we're comparing against enum values + // not present in the ReturnCode::Result enum. + switch (static_cast<uint32_t>(_result)) { + case mbus::ErrorCode::SEND_QUEUE_FULL: + case mbus::ErrorCode::SESSION_BUSY: + case mbus::ErrorCode::TIMEOUT: + case Protocol::ERROR_BUSY: + return true; + default: + return false; + } +} + +bool +ReturnCode::isNodeDownOrNetwork() const +{ + switch (static_cast<uint32_t>(_result)) { + case mbus::ErrorCode::NO_ADDRESS_FOR_SERVICE: + case mbus::ErrorCode::CONNECTION_ERROR: + case mbus::ErrorCode::UNKNOWN_SESSION: + case mbus::ErrorCode::HANDSHAKE_FAILED: + case mbus::ErrorCode::NO_SERVICES_FOR_ROUTE: + case mbus::ErrorCode::SERVICE_OOS: + case mbus::ErrorCode::NETWORK_ERROR: + case mbus::ErrorCode::UNKNOWN_PROTOCOL: + case Protocol::ERROR_NODE_NOT_READY: + case Protocol::ERROR_NOT_CONNECTED: + return true; + default: + return false; + } +} + +bool +ReturnCode::isCriticalForMaintenance() const +{ + if (_result >= static_cast<uint32_t>(mbus::ErrorCode::FATAL_ERROR)) { + return true; + } + + switch (static_cast<uint32_t>(_result)) { + case Protocol::ERROR_INTERNAL_FAILURE: + case Protocol::ERROR_NO_SPACE: + case Protocol::ERROR_UNPARSEABLE: + case Protocol::ERROR_ILLEGAL_PARAMETERS: + case Protocol::ERROR_NOT_IMPLEMENTED: + case Protocol::ERROR_UNKNOWN_COMMAND: + case Protocol::ERROR_PROCESSING_FAILURE: + case Protocol::ERROR_IGNORED: + return true; + default: + return false; + } +} + +bool +ReturnCode::isCriticalForVisitor() const +{ + return isCriticalForMaintenance(); +} + +bool +ReturnCode::isCriticalForVisitorDispatcher() const +{ + return isCriticalForMaintenance(); +} + +bool +ReturnCode::isNonCriticalForIntegrityChecker() const +{ + switch (static_cast<uint32_t>(_result)) { + case Protocol::ERROR_ABORTED: + case Protocol::ERROR_BUCKET_DELETED: + case Protocol::ERROR_BUCKET_NOT_FOUND: + return true; + default: + return false; + } +} + +bool +ReturnCode::isShutdownRelated() const +{ + switch (static_cast<uint32_t>(_result)) { + case Protocol::ERROR_ABORTED: + return true; + default: + return false; + } +} + +bool +ReturnCode::isBucketDisappearance() const +{ + switch (static_cast<uint32_t>(_result)) { + case Protocol::ERROR_BUCKET_NOT_FOUND: + case Protocol::ERROR_BUCKET_DELETED: + return true; + default: + return false; + } +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.h b/storageapi/src/vespa/storageapi/messageapi/returncode.h new file mode 100644 index 00000000000..997487853e8 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/returncode.h @@ -0,0 +1,127 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class storage::api::ReturnCode + * @ingroup messageapi + * + * @brief Class for representing return values from the processing chain + * + * @version $Id$ + */ + +#pragma once + +#include <vespa/vespalib/util/printable.h> +#include <vespa/document/util/serializable.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <iosfwd> +#include <string> + +namespace document { + class ByteBuffer; +} + +namespace storage { +namespace api { + +class ReturnCode : public document::Deserializable, + public vespalib::Printable { +public: + typedef documentapi::DocumentProtocol Protocol; + + /** Return status codes */ + enum Result { + OK = mbus::ErrorCode::NONE, + + EXISTS = Protocol::ERROR_EXISTS, + + NOT_READY = Protocol::ERROR_NODE_NOT_READY, + WRONG_DISTRIBUTION = Protocol::ERROR_WRONG_DISTRIBUTION, + REJECTED = Protocol::ERROR_REJECTED, + ABORTED = Protocol::ERROR_ABORTED, + BUCKET_NOT_FOUND = Protocol::ERROR_BUCKET_NOT_FOUND, + BUCKET_DELETED = Protocol::ERROR_BUCKET_DELETED, + TIMESTAMP_EXIST = Protocol::ERROR_TIMESTAMP_EXIST, + STALE_TIMESTAMP = Protocol::ERROR_STALE_TIMESTAMP, + TEST_AND_SET_CONDITION_FAILED = Protocol::ERROR_TEST_AND_SET_CONDITION_FAILED, + + // Wrong use + UNKNOWN_COMMAND = Protocol::ERROR_UNKNOWN_COMMAND, + NOT_IMPLEMENTED = Protocol::ERROR_NOT_IMPLEMENTED, + ILLEGAL_PARAMETERS = Protocol::ERROR_ILLEGAL_PARAMETERS, + IGNORED = Protocol::ERROR_IGNORED, + UNPARSEABLE = Protocol::ERROR_UNPARSEABLE, + + // Network failure + NOT_CONNECTED = Protocol::ERROR_NOT_CONNECTED, + TIMEOUT = mbus::ErrorCode::TIMEOUT, + BUSY = Protocol::ERROR_BUSY, + + // Disk operations + NO_SPACE = Protocol::ERROR_NO_SPACE, + DISK_FAILURE = Protocol::ERROR_DISK_FAILURE, + IO_FAILURE = Protocol::ERROR_IO_FAILURE, + + // Don't know what happened (catch-all) + INTERNAL_FAILURE = Protocol::ERROR_INTERNAL_FAILURE + }; + +private: + Result _result; + vespalib::string _message; + + void onDeserialize(const document::DocumentTypeRepo &repo, + document::ByteBuffer& buffer); + void onSerialize(document::ByteBuffer& buffer) const; + +public: + ReturnCode(); + explicit ReturnCode(Result result, const vespalib::stringref & msg = ""); + ReturnCode(const document::DocumentTypeRepo &repo, + document::ByteBuffer& buffer); + + ReturnCode* clone() const { return new ReturnCode(*this); } + + size_t getSerializedSize() const; + + const vespalib::string& getMessage() const { return _message; } + void setMessage(const vespalib::stringref & message) { _message = message; } + + Result getResult() const { return _result; } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + /** + * Translate from status code to human-readable string + * @param result Status code returned from getResult() + */ + static vespalib::string getResultString(Result result); + + bool failed() const { return (_result != OK); } + bool success() const { return (_result == OK); } + + bool operator==(Result res) const { return _result == res; } + bool operator!=(Result res) const { return _result != res; } + bool operator==(const ReturnCode& code) const + { return _result == code._result && _message == code._message; } + bool operator!=(const ReturnCode& code) const + { return _result != code._result || _message != code._message; } + + // To avoid lots of code matching various return codes in storage, we define + // some functions they can use to match those codes that corresponds to what + // they want to match. + + bool isBusy() const; + bool isNodeDownOrNetwork() const; + bool isCriticalForMaintenance() const; + bool isCriticalForVisitor() const; + bool isCriticalForVisitorDispatcher() const; + bool isShutdownRelated() const; + bool isBucketDisappearance() const; + bool isNonCriticalForIntegrityChecker() const; +}; + + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp b/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp new file mode 100644 index 00000000000..9578c347b92 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <limits> +#include <vespa/storageapi/messageapi/storagecommand.h> + +namespace storage { +namespace api { + +StorageCommand::StorageCommand(const StorageCommand& other) + : StorageMessage(other, generateMsgId()), + _timeout(other._timeout), + _sourceIndex(other._sourceIndex) +{ + setTrace(other.getTrace()); +} + +StorageCommand::StorageCommand(const MessageType& type, Priority p) + : StorageMessage(type, generateMsgId()), + // Default timeout is unlimited. Set from mbus message. Some internal + // use want unlimited timeout, (such as readbucketinfo, repair bucket + // etc) + _timeout(std::numeric_limits<uint32_t>().max()), + _sourceIndex(0xFFFF) +{ + setPriority(p); +} + +void +StorageCommand::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + out << "StorageCommand(" << _type.getName(); + if (_priority != NORMAL) out << ", priority = " << static_cast<int>(_priority); + if (_sourceIndex != 0xFFFF) out << ", source = " << _sourceIndex; + out << ", timeout = " << _timeout << " ms"; + out << ")"; +} + +StorageCommand::UP +StorageCommand::createCopyToForward(const document::BucketId&, uint64_t) const +{ + throw vespalib::IllegalStateException( + "Command " + _type.getName() + " does not support forwarding.", + VESPA_STRLOC); +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/messageapi/storagecommand.h b/storageapi/src/vespa/storageapi/messageapi/storagecommand.h new file mode 100644 index 00000000000..c71a5097bc4 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/storagecommand.h @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class storage::api::StorageCommand + * @ingroup messageapi + * + * @brief Superclass for all storage commands. + * + * A storage command is a storage message you will get a storage reply for. + * + * @version $Id$ + */ + +#pragma once + +#include <vespa/storageapi/messageapi/storagemessage.h> + +namespace storage { +namespace api { + +class StorageReply; + +class StorageCommand : public StorageMessage { + uint32_t _timeout; /** Timeout of command in milliseconds */ + /** Sets what node this message origins from. 0xFFFF is unset. */ + uint16_t _sourceIndex; + +protected: + explicit StorageCommand(const StorageCommand& other); + explicit StorageCommand(const MessageType& type, Priority p = NORMAL); + +public: + DECLARE_POINTER_TYPEDEFS(StorageCommand); + + virtual ~StorageCommand() {} + + bool sourceIndexSet() const { return (_sourceIndex != 0xffff); } + void setSourceIndex(uint16_t sourceIndex) { _sourceIndex = sourceIndex; } + uint16_t getSourceIndex() const { return _sourceIndex; } + + /** Set timeout in milliseconds. */ + void setTimeout(uint32_t milliseconds) { _timeout = milliseconds; } + /** Get timeout in milliseconds. */ + uint32_t getTimeout() const { return _timeout; } + + /** Used to set a new id so the message can be resent. */ + void setNewId() { StorageMessage::setNewMsgId(); } + + /** Overload this to get more descriptive message output. */ + virtual void print(std::ostream& out, + bool verbose, + const std::string& indent) const; + + /** + * A way for someone to make a reply to a storage message without + * knowing the type of the message. Should just call reply constructor + * taking command as input. + */ + virtual std::unique_ptr<StorageReply> makeReply() = 0; + + /** + * Distributors need a way to create copies of messages to send to forward + * to different nodes. Only messages sent through the distributor needs to + * have an actual implementation of this. + */ + virtual StorageCommand::UP createCopyToForward( + const document::BucketId& bucket, uint64_t timestamp) const; + +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp new file mode 100644 index 00000000000..80b25111488 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -0,0 +1,334 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/messageapi/storagemessage.h> + +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/messagebus/routing/verbatimdirective.h> + +namespace storage { +namespace api { + +static const vespalib::string STORAGEADDRESS_PREFIX = "storage/cluster."; + +const char* +StorageMessage::getPriorityString(Priority p) { + switch (p) { + case LOW: return "LOW"; + case NORMAL: return "NORMAL"; + case HIGH: return "HIGH"; + case VERYHIGH: return "VERYHIGH"; + default: return "UNKNOWN"; + } +} + +std::map<MessageType::Id, MessageType*> MessageType::_codes; + +const MessageType MessageType::DOCBLOCK("DocBlock", DOCBLOCK_ID); +const MessageType MessageType::DOCBLOCK_REPLY( + "DocBlock Reply", DOCBLOCK_REPLY_ID, &MessageType::DOCBLOCK); +const MessageType MessageType::GET("Get", GET_ID); +const MessageType MessageType::GET_REPLY( + "Get Reply", GET_REPLY_ID, &MessageType::GET); +const MessageType MessageType::INTERNAL("Internal", INTERNAL_ID); +const MessageType MessageType::INTERNAL_REPLY( + "Internal Reply", INTERNAL_REPLY_ID, &MessageType::INTERNAL); +const MessageType MessageType::PUT("Put", PUT_ID); +const MessageType MessageType::PUT_REPLY( + "Put Reply", PUT_REPLY_ID, &MessageType::PUT); +const MessageType MessageType::UPDATE("Update", UPDATE_ID); +const MessageType MessageType::UPDATE_REPLY( + "Update Reply", UPDATE_REPLY_ID, &MessageType::UPDATE); +const MessageType MessageType::REMOVE("Remove", REMOVE_ID); +const MessageType MessageType::REMOVE_REPLY( + "Remove Reply", REMOVE_REPLY_ID, &MessageType::REMOVE); +const MessageType MessageType::REVERT("Revert", REVERT_ID); +const MessageType MessageType::REVERT_REPLY( + "Revert Reply", REVERT_REPLY_ID, &MessageType::REVERT); +const MessageType MessageType::VISITOR_CREATE( + "Visitor Create", VISITOR_CREATE_ID); +const MessageType MessageType::VISITOR_CREATE_REPLY( + "Visitor Create Reply", VISITOR_CREATE_REPLY_ID, + &MessageType::VISITOR_CREATE); +const MessageType MessageType::VISITOR_DESTROY( + "Visitor Destroy", VISITOR_DESTROY_ID); +const MessageType MessageType::VISITOR_DESTROY_REPLY( + "Visitor Destroy Reply", VISITOR_DESTROY_REPLY_ID, + &MessageType::VISITOR_DESTROY); +const MessageType MessageType::REQUESTBUCKETINFO("Request bucket info", + REQUESTBUCKETINFO_ID); +const MessageType MessageType::REQUESTBUCKETINFO_REPLY( + "Request bucket info reply", REQUESTBUCKETINFO_REPLY_ID, + &MessageType::REQUESTBUCKETINFO); +const MessageType MessageType::NOTIFYBUCKETCHANGE("Notify bucket change", + NOTIFYBUCKETCHANGE_ID); +const MessageType MessageType::NOTIFYBUCKETCHANGE_REPLY( + "Notify bucket change reply", NOTIFYBUCKETCHANGE_REPLY_ID, + &MessageType::NOTIFYBUCKETCHANGE); +const MessageType MessageType::CREATEBUCKET("Create bucket", CREATEBUCKET_ID); +const MessageType MessageType::CREATEBUCKET_REPLY( + "Create bucket reply", CREATEBUCKET_REPLY_ID, + &MessageType::CREATEBUCKET); +const MessageType MessageType::MERGEBUCKET("Merge bucket", MERGEBUCKET_ID); +const MessageType MessageType::MERGEBUCKET_REPLY( + "Merge bucket reply", MERGEBUCKET_REPLY_ID, + &MessageType::MERGEBUCKET); +const MessageType MessageType::DELETEBUCKET("Delete bucket", DELETEBUCKET_ID); +const MessageType MessageType::DELETEBUCKET_REPLY( + "Delete bucket reply", DELETEBUCKET_REPLY_ID, + &MessageType::DELETEBUCKET); +const MessageType MessageType::SETNODESTATE("Set node state", SETNODESTATE_ID); +const MessageType MessageType::SETNODESTATE_REPLY( + "Set node state reply", SETNODESTATE_REPLY_ID, + &MessageType::SETNODESTATE); +const MessageType MessageType::GETNODESTATE("Get node state", GETNODESTATE_ID); +const MessageType MessageType::GETNODESTATE_REPLY( + "Get node state reply", GETNODESTATE_REPLY_ID, + &MessageType::GETNODESTATE); +const MessageType MessageType::SETSYSTEMSTATE("Set system state", SETSYSTEMSTATE_ID); +const MessageType MessageType::SETSYSTEMSTATE_REPLY( + "Set system state reply", SETSYSTEMSTATE_REPLY_ID, + &MessageType::SETSYSTEMSTATE); +const MessageType MessageType::GETSYSTEMSTATE("Get system state", GETSYSTEMSTATE_ID); +const MessageType MessageType::GETSYSTEMSTATE_REPLY( + "get system state reply", GETSYSTEMSTATE_REPLY_ID, + &MessageType::GETSYSTEMSTATE); +const MessageType MessageType::GETBUCKETDIFF("GetBucketDiff", GETBUCKETDIFF_ID); +const MessageType MessageType::GETBUCKETDIFF_REPLY( + "GetBucketDiff reply", GETBUCKETDIFF_REPLY_ID, + &MessageType::GETBUCKETDIFF); +const MessageType MessageType::APPLYBUCKETDIFF("ApplyBucketDiff", + APPLYBUCKETDIFF_ID); +const MessageType MessageType::APPLYBUCKETDIFF_REPLY( + "ApplyBucketDiff reply", APPLYBUCKETDIFF_REPLY_ID, + &MessageType::APPLYBUCKETDIFF); +const MessageType MessageType::VISITOR_INFO("VisitorInfo", + VISITOR_INFO_ID); +const MessageType MessageType::VISITOR_INFO_REPLY( + "VisitorInfo reply", VISITOR_INFO_REPLY_ID, + &MessageType::VISITOR_INFO); +const MessageType MessageType::SEARCHRESULT("SearchResult", SEARCHRESULT_ID); +const MessageType MessageType::SEARCHRESULT_REPLY( + "SearchResult reply", SEARCHRESULT_REPLY_ID, + &MessageType::SEARCHRESULT); +const MessageType MessageType::DOCUMENTSUMMARY("DocumentSummary", DOCUMENTSUMMARY_ID); +const MessageType MessageType::DOCUMENTSUMMARY_REPLY( + "DocumentSummary reply", DOCUMENTSUMMARY_REPLY_ID, + &MessageType::DOCUMENTSUMMARY); +const MessageType MessageType::MAPVISITOR("Mapvisitor", MAPVISITOR_ID); +const MessageType MessageType::MAPVISITOR_REPLY( + "Mapvisitor reply", MAPVISITOR_REPLY_ID, + &MessageType::MAPVISITOR); +const MessageType MessageType::SPLITBUCKET("SplitBucket", SPLITBUCKET_ID); +const MessageType MessageType::SPLITBUCKET_REPLY( + "SplitBucket reply", SPLITBUCKET_REPLY_ID, + &MessageType::SPLITBUCKET); +const MessageType MessageType::JOINBUCKETS("Joinbuckets", JOINBUCKETS_ID); +const MessageType MessageType::JOINBUCKETS_REPLY( + "Joinbuckets reply", JOINBUCKETS_REPLY_ID, + &MessageType::JOINBUCKETS); +const MessageType MessageType::MULTIOPERATION("MultiOperation", MULTIOPERATION_ID); +const MessageType MessageType::MULTIOPERATION_REPLY( + "MultiOperation Reply", MULTIOPERATION_REPLY_ID, &MessageType::MULTIOPERATION); +const MessageType MessageType::STATBUCKET("Statbucket", STATBUCKET_ID); +const MessageType MessageType::STATBUCKET_REPLY( + "Statbucket Reply", STATBUCKET_REPLY_ID, &MessageType::STATBUCKET); +const MessageType MessageType::GETBUCKETLIST("Getbucketlist", GETBUCKETLIST_ID); +const MessageType MessageType::GETBUCKETLIST_REPLY( + "Getbucketlist Reply", GETBUCKETLIST_REPLY_ID, &MessageType::GETBUCKETLIST); +const MessageType MessageType::DOCUMENTLIST("documentlist", DOCUMENTLIST_ID); +const MessageType MessageType::DOCUMENTLIST_REPLY( + "documentlist Reply", DOCUMENTLIST_REPLY_ID, &MessageType::DOCUMENTLIST); +const MessageType MessageType::EMPTYBUCKETS("Emptybuckets", EMPTYBUCKETS_ID); +const MessageType MessageType::EMPTYBUCKETS_REPLY( + "Emptybuckets Reply", EMPTYBUCKETS_REPLY_ID, &MessageType::EMPTYBUCKETS); +const MessageType MessageType::REMOVELOCATION("Removelocation", REMOVELOCATION_ID); +const MessageType MessageType::REMOVELOCATION_REPLY( + "Removelocation Reply", REMOVELOCATION_REPLY_ID, &MessageType::REMOVELOCATION); +const MessageType MessageType::QUERYRESULT("QueryResult", QUERYRESULT_ID); +const MessageType MessageType::QUERYRESULT_REPLY( + "QueryResult reply", QUERYRESULT_REPLY_ID, + &MessageType::QUERYRESULT); +const MessageType MessageType::BATCHPUTREMOVE("BatchPutRemove", BATCHPUTREMOVE_ID); +const MessageType MessageType::BATCHPUTREMOVE_REPLY( + "BatchPutRemove reply", BATCHPUTREMOVE_REPLY_ID, + &MessageType::BATCHPUTREMOVE); +const MessageType MessageType::BATCHDOCUMENTUPDATE("BatchDocumentUpdate", BATCHDOCUMENTUPDATE_ID); +const MessageType MessageType::BATCHDOCUMENTUPDATE_REPLY( + "BatchDocumentUpdate reply", BATCHDOCUMENTUPDATE_REPLY_ID, + &MessageType::BATCHDOCUMENTUPDATE); +const MessageType MessageType::SETBUCKETSTATE( + "SetBucketState", + SETBUCKETSTATE_ID); +const MessageType MessageType::SETBUCKETSTATE_REPLY( + "SetBucketStateReply", + SETBUCKETSTATE_REPLY_ID, + &MessageType::SETBUCKETSTATE); + +const MessageType& MessageType::get(Id id) +{ + std::map<Id, MessageType*>::const_iterator it = _codes.find(id); + if (it == _codes.end()) { + std::ostringstream ost; + ost << "No message type with id " << id << "."; + throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + } + return *it->second; +} + +StorageMessageAddress::StorageMessageAddress(const mbus::Route& route) + : _route(route), + _retryEnabled(false), + _protocol(DOCUMENT), + _cluster(""), + _type(0), + _index(0xFFFF) +{ +} + +static vespalib::string +createAddress(const vespalib::stringref & cluster, const lib::NodeType& type, + uint16_t index) +{ + vespalib::asciistream os; + os << STORAGEADDRESS_PREFIX << cluster << '/' << type.toString() << '/' << index << "/default"; + return os.str(); +} + +StorageMessageAddress::StorageMessageAddress( + const vespalib::stringref & cluster, const lib::NodeType& type, + uint16_t index, Protocol protocol) + : _retryEnabled(false), + _protocol(protocol), + _cluster(cluster), + _type(&type), + _index(index) +{ + mbus::IHopDirective::SP directive(new mbus::VerbatimDirective( + createAddress(cluster, type, index))); + + std::vector<mbus::IHopDirective::SP> directives; + directives.push_back(directive); + mbus::Hop hop(directives, false); + _route.addHop(hop); +} + +uint16_t +StorageMessageAddress::getIndex() const +{ + if (_type == 0) { + throw vespalib::IllegalStateException( + "Cannot retrieve node index out of external address", + VESPA_STRLOC); + } + return _index; +} + +const lib::NodeType& +StorageMessageAddress::getNodeType() const +{ + if (_type == 0) { + throw vespalib::IllegalStateException( + "Cannot retrieve node type out of external address", + VESPA_STRLOC); + } + return *_type; +} + +const vespalib::string& +StorageMessageAddress::getCluster() const +{ + if (_type == 0) { + throw vespalib::IllegalStateException( + "Cannot retrieve cluster out of external address", + VESPA_STRLOC); + } + return _cluster; +} + +bool +StorageMessageAddress::operator==(const StorageMessageAddress& other) const +{ + if (_protocol != other._protocol) return false; + if (_retryEnabled != other._retryEnabled) return false; + if (_type != other._type) return false; + if (_type != 0) { + if (_cluster != other._cluster) return false; + if (_index != other._index) return false; + if (_type != other._type) return false; + } + return true; +} + +vespalib::string +StorageMessageAddress::toString() const +{ + vespalib::asciistream os; + print(os); + return os.str(); +} + +void +StorageMessageAddress::print(vespalib::asciistream & out) const +{ + out << "StorageMessageAddress("; + if (_protocol == STORAGE) { + out << "Storage protocol"; + } else { + out << "Document protocol"; + } + if (_retryEnabled) { + out << ", retry enabled"; + } + if (_type == 0) { + out << ", " << _route.toString() << ")"; + } else { + out << ", cluster " << _cluster << ", nodetype " << *_type + << ", index " << _index << ")"; + } +} + +TransportContext::~TransportContext() +{ +} + +vespalib::Lock StorageMessage::_msgIdLock; +StorageMessage::Id StorageMessage::_lastMsgId = 1000; + +StorageMessage::Id +StorageMessage::generateMsgId() +{ + vespalib::LockGuard sync(_msgIdLock); + Id msgId = _lastMsgId++; + _lastMsgId &= ((Id(-1) << 8) >> 8); + return msgId; +} + +StorageMessage::StorageMessage(const MessageType& type, Id id) + : _type(type), + _msgId(id), + _priority(NORMAL), + _address(), + _loadType(documentapi::LoadType::DEFAULT) +{ +} + +StorageMessage::StorageMessage(const StorageMessage& other, Id id) + : _type(other._type), + _msgId(id), + _priority(other._priority), + _address(), + _loadType(other._loadType) +{ + +} + +void StorageMessage::setNewMsgId() +{ + vespalib::LockGuard sync(_msgIdLock); + _msgId = _lastMsgId++; + _lastMsgId &= ((Id(-1) << 8) >> 8); +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h new file mode 100644 index 00000000000..85cb53c3681 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h @@ -0,0 +1,458 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * @class storage::api::StorageMessage + * @ingroup messageapi + * + * @brief Superclass for all storage messages. + * + * @version $Id$ + */ + +#pragma once + +#include <map> +#include <vespa/vespalib/util/printable.h> +#include <vespa/messagebus/routing/route.h> +#include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/storageapi/messageapi/messagehandler.h> +#include <vespa/vdslib/state/nodetype.h> +#include <vespa/messagebus/trace.h> +#include <vespa/documentapi/loadtypes/loadtype.h> +#include <vespa/storageframework/generic/memory/memorytoken.h> +#include <vespa/document/bucket/bucketid.h> + +// The following macros are provided as a way to write storage messages simply. +// They implement the parts of the code that can easily be automaticly +// generated. + +/** + * Adds a messagehandler callback and some utilities + */ +#define DECLARE_POINTER_TYPEDEFS(message) \ + typedef std::unique_ptr<message> UP; \ + typedef std::shared_ptr<message> SP; \ + typedef std::shared_ptr<const message> CSP; + +#define DECLARE_STORAGEREPLY(reply, callback) \ +public: \ + DECLARE_POINTER_TYPEDEFS(reply) \ +private: \ + virtual bool callHandler( \ + MessageHandler& h, \ + const std::shared_ptr<StorageMessage>& m) const \ + { \ + return h.callback(std::static_pointer_cast<reply>(m)); \ + } + +/** Commands also has a command to implement to create the reply. */ +#define DECLARE_STORAGECOMMAND(command, callback) \ +public: \ + std::unique_ptr<StorageReply> makeReply(); \ + DECLARE_STORAGEREPLY(command, callback) + +/** This macro implements common stuff for all storage messages. */ +#define IMPLEMENT_COMMON(message) \ + +/** This macro is used to implement common storage reply functionality. */ +#define IMPLEMENT_REPLY(reply) \ + IMPLEMENT_COMMON(reply) \ + +/** This macro is used to implement common storage command functionality. */ +#define IMPLEMENT_COMMAND(command, reply) \ + IMPLEMENT_COMMON(command) \ + std::unique_ptr<storage::api::StorageReply> \ + storage::api::command::makeReply() \ + { \ + return std::unique_ptr<storage::api::StorageReply>(new reply(*this)); \ + } + + +namespace storage { + +namespace api { + +/** + * @class MessageType + * @ingroup messageapi + * + * @brief This class defines the different message types we have. + * + * This is used to be able to deserialize messages of various classes. + */ +class MessageType : public vespalib::Printable { +private: + MessageType(const MessageType &); + MessageType& operator=(const MessageType &); + +public: + enum Id { + GET_ID = 4, + GET_REPLY_ID = 5, + INTERNAL_ID = 6, + INTERNAL_REPLY_ID = 7, + PUT_ID = 10, + PUT_REPLY_ID = 11, + REMOVE_ID = 12, + REMOVE_REPLY_ID = 13, + REVERT_ID = 14, + REVERT_REPLY_ID = 15, + STAT_ID = 16, + STAT_REPLY_ID = 17, + VISITOR_CREATE_ID = 18, + VISITOR_CREATE_REPLY_ID = 19, + VISITOR_DESTROY_ID = 20, + VISITOR_DESTROY_REPLY_ID = 21, + CREATEBUCKET_ID = 26, + CREATEBUCKET_REPLY_ID = 27, + MERGEBUCKET_ID = 32, + MERGEBUCKET_REPLY_ID = 33, + DELETEBUCKET_ID = 34, + DELETEBUCKET_REPLY_ID = 35, + SETNODESTATE_ID = 36, + SETNODESTATE_REPLY_ID = 37, + GETNODESTATE_ID = 38, + GETNODESTATE_REPLY_ID = 39, + SETSYSTEMSTATE_ID = 40, + SETSYSTEMSTATE_REPLY_ID = 41, + GETSYSTEMSTATE_ID = 42, + GETSYSTEMSTATE_REPLY_ID = 43, + GETBUCKETDIFF_ID = 50, + GETBUCKETDIFF_REPLY_ID = 51, + APPLYBUCKETDIFF_ID = 52, + APPLYBUCKETDIFF_REPLY_ID = 53, + REQUESTBUCKETINFO_ID = 54, + REQUESTBUCKETINFO_REPLY_ID = 55, + NOTIFYBUCKETCHANGE_ID = 56, + NOTIFYBUCKETCHANGE_REPLY_ID = 57, + DOCBLOCK_ID = 58, + DOCBLOCK_REPLY_ID = 59, + VISITOR_INFO_ID = 60, + VISITOR_INFO_REPLY_ID = 61, + SEARCHRESULT_ID = 64, + SEARCHRESULT_REPLY_ID = 65, + SPLITBUCKET_ID = 66, + SPLITBUCKET_REPLY_ID = 67, + JOINBUCKETS_ID = 68, + JOINBUCKETS_REPLY_ID = 69, + MULTIOPERATION_ID = 70, + MULTIOPERATION_REPLY_ID = 71, + DOCUMENTSUMMARY_ID = 72, + DOCUMENTSUMMARY_REPLY_ID = 73, + MAPVISITOR_ID = 74, + MAPVISITOR_REPLY_ID = 75, + STATBUCKET_ID = 76, + STATBUCKET_REPLY_ID = 77, + GETBUCKETLIST_ID = 78, + GETBUCKETLIST_REPLY_ID = 79, + DOCUMENTLIST_ID = 80, + DOCUMENTLIST_REPLY_ID = 81, + UPDATE_ID = 82, + UPDATE_REPLY_ID = 83, + EMPTYBUCKETS_ID = 84, + EMPTYBUCKETS_REPLY_ID = 85, + REMOVELOCATION_ID = 86, + REMOVELOCATION_REPLY_ID = 87, + QUERYRESULT_ID = 88, + QUERYRESULT_REPLY_ID = 89, + BATCHPUTREMOVE_ID = 90, + BATCHPUTREMOVE_REPLY_ID = 91, + BATCHDOCUMENTUPDATE_ID = 92, + BATCHDOCUMENTUPDATE_REPLY_ID = 93, + SETBUCKETSTATE_ID = 94, + SETBUCKETSTATE_REPLY_ID = 95, + MESSAGETYPE_MAX_ID + }; + +private: + static std::map<Id, MessageType*> _codes; + const vespalib::string _name; + Id _id; + MessageType *_reply; + const MessageType *_replyOf; + + MessageType(const vespalib::stringref & name, Id id, + const MessageType* replyOf = 0) + : _name(name), _id(id), _reply(NULL), _replyOf(replyOf) + { + _codes[id] = this; + if (_replyOf != 0) { + assert(_replyOf->_reply == 0); + // Ugly cast to let initialization work + MessageType& type = const_cast<MessageType&>(*_replyOf); + type._reply = this; + } + } +public: + static const MessageType DOCBLOCK; + static const MessageType DOCBLOCK_REPLY; + static const MessageType GET; + static const MessageType GET_REPLY; + static const MessageType INTERNAL; + static const MessageType INTERNAL_REPLY; + static const MessageType PUT; + static const MessageType PUT_REPLY; + static const MessageType REMOVE; + static const MessageType REMOVE_REPLY; + static const MessageType REVERT; + static const MessageType REVERT_REPLY; + static const MessageType VISITOR_CREATE; + static const MessageType VISITOR_CREATE_REPLY; + static const MessageType VISITOR_DESTROY; + static const MessageType VISITOR_DESTROY_REPLY; + static const MessageType REQUESTBUCKETINFO; + static const MessageType REQUESTBUCKETINFO_REPLY; + static const MessageType NOTIFYBUCKETCHANGE; + static const MessageType NOTIFYBUCKETCHANGE_REPLY; + static const MessageType CREATEBUCKET; + static const MessageType CREATEBUCKET_REPLY; + static const MessageType MERGEBUCKET; + static const MessageType MERGEBUCKET_REPLY; + static const MessageType DELETEBUCKET; + static const MessageType DELETEBUCKET_REPLY; + static const MessageType SETNODESTATE; + static const MessageType SETNODESTATE_REPLY; + static const MessageType GETNODESTATE; + static const MessageType GETNODESTATE_REPLY; + static const MessageType SETSYSTEMSTATE; + static const MessageType SETSYSTEMSTATE_REPLY; + static const MessageType GETSYSTEMSTATE; + static const MessageType GETSYSTEMSTATE_REPLY; + static const MessageType BUCKETSADDED; + static const MessageType BUCKETSADDED_REPLY; + static const MessageType BUCKETSREMOVED; + static const MessageType BUCKETSREMOVED_REPLY; + static const MessageType GETBUCKETDIFF; + static const MessageType GETBUCKETDIFF_REPLY; + static const MessageType APPLYBUCKETDIFF; + static const MessageType APPLYBUCKETDIFF_REPLY; + static const MessageType VISITOR_INFO; + static const MessageType VISITOR_INFO_REPLY; + static const MessageType SEARCHRESULT; + static const MessageType SEARCHRESULT_REPLY; + static const MessageType SPLITBUCKET; + static const MessageType SPLITBUCKET_REPLY; + static const MessageType JOINBUCKETS; + static const MessageType JOINBUCKETS_REPLY; + static const MessageType MULTIOPERATION; + static const MessageType MULTIOPERATION_REPLY; + static const MessageType DOCUMENTSUMMARY; + static const MessageType DOCUMENTSUMMARY_REPLY; + static const MessageType MAPVISITOR; + static const MessageType MAPVISITOR_REPLY; + static const MessageType STATBUCKET; + static const MessageType STATBUCKET_REPLY; + static const MessageType GETBUCKETLIST; + static const MessageType GETBUCKETLIST_REPLY; + static const MessageType DOCUMENTLIST; + static const MessageType DOCUMENTLIST_REPLY; + static const MessageType UPDATE; + static const MessageType UPDATE_REPLY; + static const MessageType EMPTYBUCKETS; + static const MessageType EMPTYBUCKETS_REPLY; + static const MessageType REMOVELOCATION; + static const MessageType REMOVELOCATION_REPLY; + static const MessageType QUERYRESULT; + static const MessageType QUERYRESULT_REPLY; + static const MessageType BATCHPUTREMOVE; + static const MessageType BATCHPUTREMOVE_REPLY; + static const MessageType BATCHDOCUMENTUPDATE; + static const MessageType BATCHDOCUMENTUPDATE_REPLY; + static const MessageType SETBUCKETSTATE; + static const MessageType SETBUCKETSTATE_REPLY; + + static const MessageType& get(Id id); + + Id getId() const { return _id; } + static Id getMaxId() { return MESSAGETYPE_MAX_ID; } + const vespalib::string& getName() const { return _name; } + bool isReply() const { return (_replyOf != 0); } + /** Only valid to call on replies. */ + const MessageType& getCommandType() const + { assert(_replyOf); return *_replyOf; } + /** Only valid to call on commands. */ + const MessageType& getReplyType() const + { assert(_reply); return *_reply; } + bool operator==(const MessageType& type) const { return (_id == type._id); } + bool operator!=(const MessageType& type) const { return (_id != type._id); } + + void print(std::ostream& out, bool verbose, const std::string& indent) const + { + (void) verbose; (void) indent; + out << "MessageType(" << _id << ", " << _name; + if (_replyOf) { + out << ", reply of " << _replyOf->getName(); + } + out << ")"; + } +}; + +/** + * Represent an address we can send a storage message to. + * We have two kinds of addresses: + * - A VDS address used to send to a single VDS node. + * - An external mbus route, used to send to an external source. + */ +class StorageMessageAddress { +public: + enum Protocol { STORAGE, DOCUMENT }; + +private: + mbus::Route _route; + bool _retryEnabled; + Protocol _protocol; + // Used for internal VDS addresses only + vespalib::string _cluster; + const lib::NodeType* _type; + uint16_t _index; + +public: + StorageMessageAddress(const mbus::Route& route); + StorageMessageAddress(const vespalib::stringref & clusterName, + const lib::NodeType& type, uint16_t index, + Protocol protocol = STORAGE); + + void setProtocol(Protocol p) { _protocol = p; } + void enableRetry(bool enable = true) { _retryEnabled = enable; } + + const mbus::Route& getRoute() const { return _route; } + bool retryEnabled() const { return _retryEnabled; } + Protocol getProtocol() const { return _protocol; } + uint16_t getIndex() const; + const lib::NodeType& getNodeType() const; + const vespalib::string& getCluster() const; + + bool operator==(const StorageMessageAddress& other) const; + vespalib::string toString() const; + friend std::ostream & operator << + (std::ostream & os, const StorageMessageAddress & addr) { + return os << addr.toString(); + } + +private: + void print(vespalib::asciistream & out) const; +}; + +struct TransportContext { + virtual ~TransportContext() = 0; +}; + +class StorageMessage : public vespalib::Printable +{ + friend class StorageMessageTest; // Used for testing only +public: + DECLARE_POINTER_TYPEDEFS(StorageMessage); + typedef uint64_t Id; + typedef uint8_t Priority; + + enum LegacyPriorityValues { + LOW = 225, + NORMAL = 127, + HIGH = 50, + VERYHIGH = 0 + }; // FIXME + //static const unsigned int NUM_PRIORITIES = UINT8_MAX; + static const char* getPriorityString(Priority); + +private: + static vespalib::Lock _msgIdLock; + static Id _lastMsgId; + + StorageMessage& operator=(const StorageMessage&); + StorageMessage(const StorageMessage&); + + mutable std::unique_ptr<TransportContext> _transportContext; + std::unique_ptr<framework::MemoryToken> _memoryToken; + +protected: + static Id generateMsgId(); + + const MessageType& _type; + Id _msgId; + Priority _priority; + std::unique_ptr<StorageMessageAddress> _address; + documentapi::LoadType _loadType; + mbus::Trace _trace; + + StorageMessage(const MessageType& code, Id id); + StorageMessage(const StorageMessage&, Id id); + +public: + virtual ~StorageMessage() {} + + Id getMsgId() const { return _msgId; } + + /** Method used by storage commands to set a new id. */ + void setNewMsgId(); + + /** + * Set the id of this message. Typically used to set the id to a + * unique value previously generated with the generateMsgId method. + **/ + void forceMsgId(Id msgId) { _msgId = msgId; } + + const MessageType& getType() const { return _type; } + + void setPriority(Priority p) { _priority = p; } + Priority getPriority() const { return _priority; } + + const StorageMessageAddress* getAddress() const { return _address.get(); } + + void setAddress(const StorageMessageAddress& address) + { _address.reset(new StorageMessageAddress(address)); } + + void setMemoryToken(std::unique_ptr<framework::MemoryToken> token) { + _memoryToken = std::move(token); + } + + /** + Returns the approximate memory footprint of a storage message. + By default, returns 50 bytes. This only needs to be overriden if the + message potentially can be much larger than this. + */ + virtual uint32_t getMemoryFootprint() const { + return 50; + } + + /** + * Used by storage to remember the context in which this message was + * created, whether it was a storageprotocol message, a documentprotocol + * message, or an RPC call. + */ + void setTransportContext(std::unique_ptr<TransportContext> context) + { _transportContext = std::move(context); } + + std::unique_ptr<TransportContext> getTransportContext() const + { return std::move(_transportContext); } + + /** + * This method is overloaded in subclasses and will call the correct + * method in the MessageHandler interface. + */ + virtual bool callHandler(MessageHandler&, + const StorageMessage::SP&) const = 0; + + const documentapi::LoadType& getLoadType() const { return _loadType; } + void setLoadType(const documentapi::LoadType& type) { _loadType = type; } + + mbus::Trace& getTrace() { return _trace; } + const mbus::Trace& getTrace() const { return _trace; } + + /** + Sets the trace object for this message. + */ + void setTrace(const mbus::Trace &trace) { _trace = trace; } + + /** + * Cheap version of tostring(). + */ + virtual vespalib::string getSummary() const { return toString(); }; + + virtual document::BucketId getBucketId() const { return document::BucketId(); } + virtual bool hasSingleBucketId() const { return false; } +}; + +} // api +} // storage + diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp new file mode 100644 index 00000000000..aa473cd90a1 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storageapi/messageapi/storagereply.h> + +#include <vespa/storageapi/messageapi/storagecommand.h> + +namespace storage { +namespace api { + +StorageReply::StorageReply(const StorageCommand& cmd, ReturnCode code) + : StorageMessage(cmd.getType().getReplyType(), cmd.getMsgId()), + _result(code) +{ + setPriority(cmd.getPriority()); + if (cmd.getAddress()) { + setAddress(*cmd.getAddress()); + } + setTrace(cmd.getTrace()); + setTransportContext(cmd.getTransportContext()); +} + +void +StorageReply::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; (void) indent; + out << "StorageReply(" << _type.getName() << ", " + << _result.toString() << ")"; +} + +} // api +} // storage diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.h b/storageapi/src/vespa/storageapi/messageapi/storagereply.h new file mode 100644 index 00000000000..65475335519 --- /dev/null +++ b/storageapi/src/vespa/storageapi/messageapi/storagereply.h @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class storage::api::StorageReply + * @ingroup messageapi + * + * @brief Superclass for all storage replies. + * + * A storage reply is a storage message sent in reply to a storage command. + * + * @version $Id$ + */ + +#pragma once + +#include <vespa/storageapi/messageapi/returncode.h> +#include <vespa/storageapi/messageapi/storagemessage.h> + +namespace storage { +namespace api { + +class StorageCommand; + +class StorageReply : public StorageMessage { + ReturnCode _result; + +protected: + explicit StorageReply(const StorageCommand& cmd, + ReturnCode code = ReturnCode(ReturnCode::OK)); + +public: + DECLARE_POINTER_TYPEDEFS(StorageReply); + + void setResult(const ReturnCode& r) { _result = r; } + void setResult(ReturnCode::Result r) { _result = ReturnCode(r); } + const ReturnCode& getResult() const { return _result; } + + /** Overload this to get more descriptive output. */ + virtual void print(std::ostream& out, + bool verbose, + const std::string& indent) const; +}; + +} // api +} // storage + |