summaryrefslogtreecommitdiffstats
path: root/storageapi
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
committerJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
commit72231250ed81e10d66bfe70701e64fa5fe50f712 (patch)
tree2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /storageapi
Publish
Diffstat (limited to 'storageapi')
-rw-r--r--storageapi/.gitignore2
-rw-r--r--storageapi/CMakeLists.txt27
-rw-r--r--storageapi/OWNERS2
-rw-r--r--storageapi/README1
-rw-r--r--storageapi/src/.gitignore6
-rw-r--r--storageapi/src/tests/.gitignore9
-rw-r--r--storageapi/src/tests/CMakeLists.txt11
-rw-r--r--storageapi/src/tests/buckets/.gitignore9
-rw-r--r--storageapi/src/tests/buckets/CMakeLists.txt7
-rw-r--r--storageapi/src/tests/buckets/bucketinfotest.cpp42
-rw-r--r--storageapi/src/tests/mbusprot/.gitignore11
-rw-r--r--storageapi/src/tests/mbusprot/CMakeLists.txt7
-rw-r--r--storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE70
-rw-r--r--storageapi/src/tests/mbusprot/storageprotocoltest.cpp972
-rw-r--r--storageapi/src/tests/message/.gitignore9
-rw-r--r--storageapi/src/tests/messageapi/.gitignore9
-rw-r--r--storageapi/src/tests/messageapi/CMakeLists.txt5
-rw-r--r--storageapi/src/tests/testrunner.cpp14
-rw-r--r--storageapi/src/vespa/storageapi/.gitignore5
-rw-r--r--storageapi/src/vespa/storageapi/CMakeLists.txt10
-rw-r--r--storageapi/src/vespa/storageapi/app/.gitignore14
-rw-r--r--storageapi/src/vespa/storageapi/app/CMakeLists.txt8
-rw-r--r--storageapi/src/vespa/storageapi/app/getbucketid.cpp18
-rw-r--r--storageapi/src/vespa/storageapi/buckets/.gitignore9
-rw-r--r--storageapi/src/vespa/storageapi/buckets/CMakeLists.txt6
-rw-r--r--storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp131
-rw-r--r--storageapi/src/vespa/storageapi/buckets/bucketinfo.h92
-rw-r--r--storageapi/src/vespa/storageapi/defs.h21
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/.gitignore7
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt14
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h68
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp302
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h186
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp705
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h79
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp684
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h80
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp228
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h40
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp70
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h37
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h151
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp15
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/storagecommand.h37
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/storagemessage.cpp0
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/storagemessage.h22
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp173
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h47
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp58
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/storagereply.h51
-rw-r--r--storageapi/src/vespa/storageapi/message/.gitignore7
-rw-r--r--storageapi/src/vespa/storageapi/message/CMakeLists.txt18
-rw-r--r--storageapi/src/vespa/storageapi/message/batch.cpp171
-rw-r--r--storageapi/src/vespa/storageapi/message/batch.h204
-rw-r--r--storageapi/src/vespa/storageapi/message/bucket.cpp615
-rw-r--r--storageapi/src/vespa/storageapi/message/bucket.h528
-rw-r--r--storageapi/src/vespa/storageapi/message/bucketsplitting.cpp135
-rw-r--r--storageapi/src/vespa/storageapi/message/bucketsplitting.h144
-rw-r--r--storageapi/src/vespa/storageapi/message/datagram.cpp189
-rw-r--r--storageapi/src/vespa/storageapi/message/datagram.h226
-rw-r--r--storageapi/src/vespa/storageapi/message/documentsummary.cpp45
-rw-r--r--storageapi/src/vespa/storageapi/message/documentsummary.h50
-rw-r--r--storageapi/src/vespa/storageapi/message/internal.h110
-rw-r--r--storageapi/src/vespa/storageapi/message/multioperation.cpp108
-rw-r--r--storageapi/src/vespa/storageapi/message/multioperation.h93
-rw-r--r--storageapi/src/vespa/storageapi/message/persistence.cpp358
-rw-r--r--storageapi/src/vespa/storageapi/message/persistence.h363
-rw-r--r--storageapi/src/vespa/storageapi/message/queryresult.cpp46
-rw-r--r--storageapi/src/vespa/storageapi/message/queryresult.h56
-rw-r--r--storageapi/src/vespa/storageapi/message/removelocation.cpp34
-rw-r--r--storageapi/src/vespa/storageapi/message/removelocation.h43
-rw-r--r--storageapi/src/vespa/storageapi/message/searchresult.cpp47
-rw-r--r--storageapi/src/vespa/storageapi/message/searchresult.h48
-rw-r--r--storageapi/src/vespa/storageapi/message/stat.cpp102
-rw-r--r--storageapi/src/vespa/storageapi/message/stat.h118
-rw-r--r--storageapi/src/vespa/storageapi/message/state.cpp99
-rw-r--r--storageapi/src/vespa/storageapi/message/state.h108
-rw-r--r--storageapi/src/vespa/storageapi/message/visitor.cpp227
-rw-r--r--storageapi/src/vespa/storageapi/message/visitor.h269
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/.gitignore6
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt13
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp25
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/bucketcommand.h47
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp23
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h34
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp28
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h42
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp35
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/bucketreply.h49
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h26
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/messagehandler.h367
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/returncode.cpp181
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/returncode.h127
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp50
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/storagecommand.h72
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp334
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/storagemessage.h458
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/storagereply.cpp33
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/storagereply.h45
99 files changed, 11137 insertions, 0 deletions
diff --git a/storageapi/.gitignore b/storageapi/.gitignore
new file mode 100644
index 00000000000..a9b20e8992d
--- /dev/null
+++ b/storageapi/.gitignore
@@ -0,0 +1,2 @@
+Makefile
+Testing
diff --git a/storageapi/CMakeLists.txt b/storageapi/CMakeLists.txt
new file mode 100644
index 00000000000..bf57c21035d
--- /dev/null
+++ b/storageapi/CMakeLists.txt
@@ -0,0 +1,27 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_define_module(
+ DEPENDS
+ fastos
+ fnet
+ vespalog
+ vespalib
+ document
+ messagebus
+ vdslib
+ storageframework
+ documentapi
+
+ LIBS
+ src/vespa/storageapi
+ src/vespa/storageapi/app
+ src/vespa/storageapi/buckets
+ src/vespa/storageapi/mbusprot
+ src/vespa/storageapi/message
+ src/vespa/storageapi/messageapi
+
+ TESTS
+ src/tests
+ src/tests/buckets
+ src/tests/mbusprot
+ src/tests/messageapi
+)
diff --git a/storageapi/OWNERS b/storageapi/OWNERS
new file mode 100644
index 00000000000..97c35339850
--- /dev/null
+++ b/storageapi/OWNERS
@@ -0,0 +1,2 @@
+vekterli
+dybdahl
diff --git a/storageapi/README b/storageapi/README
new file mode 100644
index 00000000000..f6332c9f8f0
--- /dev/null
+++ b/storageapi/README
@@ -0,0 +1 @@
+This is the API needed to access the Vespa Storage System.
diff --git a/storageapi/src/.gitignore b/storageapi/src/.gitignore
new file mode 100644
index 00000000000..3578c0b3853
--- /dev/null
+++ b/storageapi/src/.gitignore
@@ -0,0 +1,6 @@
+.cvsignore
+Makefile.ini
+config_command.sh
+doc
+project.dsw
+/storageapi.mak
diff --git a/storageapi/src/tests/.gitignore b/storageapi/src/tests/.gitignore
new file mode 100644
index 00000000000..37b74259b4a
--- /dev/null
+++ b/storageapi/src/tests/.gitignore
@@ -0,0 +1,9 @@
+*.core
+.*.swp
+.config.log
+.depend
+Makefile
+docstorage
+test.vlog
+testrunner
+storageapi_testrunner_app
diff --git a/storageapi/src/tests/CMakeLists.txt b/storageapi/src/tests/CMakeLists.txt
new file mode 100644
index 00000000000..eeeb701f8b5
--- /dev/null
+++ b/storageapi/src/tests/CMakeLists.txt
@@ -0,0 +1,11 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(storageapi_testrunner_app
+ SOURCES
+ testrunner.cpp
+ DEPENDS
+ storageapi_testbuckets
+ storageapi_testmessageapi
+ storageapi_testmbusprot
+ storageapi
+)
+vespa_add_test(NAME storageapi_testrunner_app COMMAND storageapi_testrunner_app)
diff --git a/storageapi/src/tests/buckets/.gitignore b/storageapi/src/tests/buckets/.gitignore
new file mode 100644
index 00000000000..1d859456ef9
--- /dev/null
+++ b/storageapi/src/tests/buckets/.gitignore
@@ -0,0 +1,9 @@
+*.So
+*.core
+*.so
+.*.swp
+.config.log
+.depend
+Makefile
+test.vlog
+testrunner
diff --git a/storageapi/src/tests/buckets/CMakeLists.txt b/storageapi/src/tests/buckets/CMakeLists.txt
new file mode 100644
index 00000000000..aa25f630f53
--- /dev/null
+++ b/storageapi/src/tests/buckets/CMakeLists.txt
@@ -0,0 +1,7 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(storageapi_testbuckets
+ SOURCES
+ bucketinfotest.cpp
+ DEPENDS
+ cppunit
+)
diff --git a/storageapi/src/tests/buckets/bucketinfotest.cpp b/storageapi/src/tests/buckets/bucketinfotest.cpp
new file mode 100644
index 00000000000..bc079ce8cad
--- /dev/null
+++ b/storageapi/src/tests/buckets/bucketinfotest.cpp
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <cppunit/extensions/HelperMacros.h>
+#include <vespa/storageapi/buckets/bucketinfo.h>
+
+namespace storage {
+namespace api {
+
+struct BucketInfo_Test : public CppUnit::TestFixture {
+
+ void testSimple();
+
+ CPPUNIT_TEST_SUITE(BucketInfo_Test);
+ CPPUNIT_TEST(testSimple);
+ CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(BucketInfo_Test);
+
+/** Tests simple operations */
+void BucketInfo_Test::testSimple()
+{
+ BucketInfo info;
+
+ CPPUNIT_ASSERT_EQUAL(false, info.valid());
+ CPPUNIT_ASSERT_EQUAL(0u, info.getChecksum());
+ CPPUNIT_ASSERT_EQUAL(0u, info.getDocumentCount());
+ CPPUNIT_ASSERT_EQUAL(1u, info.getTotalDocumentSize());
+
+ info.setChecksum(0xa000bbbb);
+ info.setDocumentCount(15);
+ info.setTotalDocumentSize(64000);
+
+ CPPUNIT_ASSERT_EQUAL(true, info.valid());
+ CPPUNIT_ASSERT_EQUAL(0xa000bbbb, info.getChecksum());
+ CPPUNIT_ASSERT_EQUAL(15u, info.getDocumentCount());
+ CPPUNIT_ASSERT_EQUAL(64000u, info.getTotalDocumentSize());
+};
+
+} // api
+} // storage
diff --git a/storageapi/src/tests/mbusprot/.gitignore b/storageapi/src/tests/mbusprot/.gitignore
new file mode 100644
index 00000000000..1b779fa320a
--- /dev/null
+++ b/storageapi/src/tests/mbusprot/.gitignore
@@ -0,0 +1,11 @@
+*.So
+*.core
+*.so
+.*.swp
+.config.log
+.depend
+Makefile
+test.vlog
+testrunner
+/mbusprot.4.2.serialization.HEAD
+/mbusprot.5.0.serialization.5.1
diff --git a/storageapi/src/tests/mbusprot/CMakeLists.txt b/storageapi/src/tests/mbusprot/CMakeLists.txt
new file mode 100644
index 00000000000..d7d376ae105
--- /dev/null
+++ b/storageapi/src/tests/mbusprot/CMakeLists.txt
@@ -0,0 +1,7 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(storageapi_testmbusprot
+ SOURCES
+ storageprotocoltest.cpp
+ DEPENDS
+ vdstestlib
+)
diff --git a/storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE b/storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE
new file mode 100644
index 00000000000..4ee14c628b4
--- /dev/null
+++ b/storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE
@@ -0,0 +1,70 @@
+
+MessageType(10, Put)
+\00\00\00
+\00\00\00j\00\08\00\00\00ddoc:test:test\00\05testdoctype1\00\00\01\00\00\00=\00\01\05\00= ;This is the contents of the test document.
+It ain't much.
+\00@\00\00\00\00\00\00Q\00\00\00\00\00\00\00\0e\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(11, Put Reply, reply of Put)
+\00\00\00\0b\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(82, Update)
+\00\00\00R\00\00\005\00\08doc:test:test\00\01testdoctype1\00\00\01\00\00\00\01\00\00\00\02\00\00\00\01\00\00\10\1b\01\00\00\00\11@\00\00\00\00\00\00Q\00\00\00\00\00\00\00\0e\00\00\00\00\00\00\00
+\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(83, Update Reply, reply of Update)
+\00\00\00S\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\00\00\00\00\0e\00\00\00\00\00\00\00\08\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(4, Get)
+\00\00\00\04\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\00\00\00\00{\01\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(5, Get Reply, reply of Get)
+\00\00\00\05\00\00\00j\00\08\00\00\00ddoc:test:test\00\05testdoctype1\00\00\01\00\00\00=\00\01\05\00= ;This is the contents of the test document.
+It ain't much.
+\00\00\00\00\00\00\00\00d\00\00\00\0ddoc:test:test\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(12, Remove)
+\00\00\00\0c\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\00\00\00\00\9f\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(13, Remove Reply, reply of Remove)
+\00\00\00\0d\00\00\00\0ddoc:test:test@\00\00\00\00\00\00Q\00\00\00\00\00\00\000\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(14, Revert)
+\00\00\00\0e@\00\00\00\00\00\00Q\00\00\00\01\00\00\00\00\00\00\00;\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(15, Revert Reply, reply of Revert)
+\00\00\00\0f@\00\00\00\00\00\00Q\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(54, Request bucket info)
+\00\00\006\00\00\00\00\00\03\00\00\00\14distributor:3 .1.s:d\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(55, Request bucket info reply, reply of Request bucket info)
+\00\00\007\00\00\00\01\00\00\00\00\00\00\00\04\00\00\00+\00\00\00\18\00\00\00{\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(56, Notify bucket change)
+\00\00\008P\00\00\00\00\00\03\e8\00\00\00\02\00\00\00\03\00\00\00\04\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(57, Notify bucket change reply, reply of Notify bucket change)
+\00\00\009\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(26, Create bucket)
+\00\00\00\1a\00\00\00\00\00\00\02o\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(27, Create bucket reply, reply of Create bucket)
+\00\00\00\1b\00\00\00\00\00\00\02o\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(34, Delete bucket)
+\00\00\00"\00\00\00\00\00\00\02o\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(35, Delete bucket reply, reply of Delete bucket)
+\00\00\00#\00\00\00\00\00\00\02o\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(32, Merge bucket)
+\00\00\00 \00\00\00\00\00\00\02o\00\03\00\04\00\00\0d\01\00\1a\01\00\00\00\00\00\00\04\d2\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(33, Merge bucket reply, reply of Merge bucket)
+\00\00\00!\00\00\00\00\00\00\02o\00\03\00\04\00\00\0d\01\00\1a\01\00\00\00\00\00\00\04\d2\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\04\00\00\00
+\00\00\00d
+MessageType(50, GetBucketDiff)
+\00\00\002\00\00\00\00\00\00\02o\00\02\00\04\00\00\0d\00\00\00\00\00\00\00\04 \00\00\00\01\00\00\00\00\00\01\e2@\00\0c1234567890ab\00\00\00d\00\01\00\00\00\01\00\03\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(51, GetBucketDiff reply, reply of GetBucketDiff)
+\00\00\003\00\00\00\00\00\00\02o\00\02\00\04\00\00\0d\00\00\00\00\00\00\00\04 \00\00\00\01\00\00\00\00\00\01\e2@\00\0c1234567890ab\00\00\00d\00\01\00\00\00\01\00\03\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(52, ApplyBucketDiff)
+\00\00\004@\00\00\00\00\00\02o\00\02\00\04\00\00\0d\00\00\00\04\d2\00\00\00\01\00\00\00\00\00\00\00\00\00\0c\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(53, ApplyBucketDiff reply, reply of ApplyBucketDiff)
+\00\00\005@\00\00\00\00\00\02o\00\02\00\04\00\00\0d\00\00\00\04\d2\00\00\00\01\00\00\00\00\00\00\00\00\00\0c\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(66, SplitBucket)
+\00\00\00B@\00\00\00\00\00\00\00\14(\00\00\03\e8\00\00\00\05\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(67, SplitBucket reply, reply of SplitBucket)
+\00\00\00C@\00\00\00\00\00\00\00\00\00\00\02D\00\00\00\00\00\00\00\00\00\00d\00\00\03\e8\00\00'\10D\00\00\00\00\00\00\01\00\00\00e\00\00\03\e9\00\00'\11\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(18, Visitor Create)
+\00\00\00\12\00\00\00\07library\00\00\00\02id\00\00\00\0ddoc selection\00\00\00\01\00\00\00\0bcontroldest\00\00\00\08datadest\00\00\00\02\00\00\00\00\00\00\00{\00\00\00\00\00\00\01\c8\00\00\00\02@\00\00\00\00\00\00\01@\00\00\00\00\00\00\02\00\01\01\00\00\00d\00\00\00\03\00\00\00\0dinto darkness\00\00\00\09bind them\00\00\00\08one ring\00\00\00\10to rule them all\00\00\00\0bone ring to\00\00\00\0dfind them and\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(19, Visitor Create Reply, reply of Visitor Create)
+\00\00\00\13\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01
+MessageType(70, MultiOperation)
+\00\00\00F\00\00'\10\01\00\00\00\00\00\00\00\00\00\00\00\a6&\00\00$\00\00\00\ca&\00\00F\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\08\00\00\00\1edoc:test:test\00\01testdoctype1\00\00\01\00\00\00=\00\01\05\00= ;This is the contents of the test document.
+It ain't much.
+\00\00P\00\00\f1\f1\f1\f1\f1\00\00\00\00\00\00\00\00\01\ff\ff
+MessageType(71, MultiOperation Reply, reply of MultiOperation)
+\00\00\00GP\00\00\f1\f1\f1\f1\f1\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01 \ No newline at end of file
diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
new file mode 100644
index 00000000000..738fe9ceeee
--- /dev/null
+++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
@@ -0,0 +1,972 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/base/testdocman.h>
+#include <vespa/document/document.h>
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/update/fieldpathupdates.h>
+#include <iomanip>
+#include <sstream>
+#include <vespa/storageapi/message/persistence.h>
+#include <vespa/storageapi/message/bucket.h>
+#include <vespa/storageapi/message/bucketsplitting.h>
+#include <vespa/storageapi/message/internal.h>
+#include <vespa/storageapi/message/removelocation.h>
+#include <vespa/storageapi/message/multioperation.h>
+#include <vespa/storageapi/message/batch.h>
+#include <vespa/storageapi/mbusprot/storageprotocol.h>
+#include <vespa/storageapi/mbusprot/storagecommand.h>
+#include <vespa/storageapi/mbusprot/storagereply.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/util/growablebytebuffer.h>
+#include <vespa/storageapi/message/visitor.h>
+
+using std::shared_ptr;
+using document::ByteBuffer;
+using document::Document;
+using document::DocumentId;
+using document::DocumentType;
+using document::DocumentTypeRepo;
+using storage::lib::ClusterState;
+using vespalib::string;
+
+namespace storage {
+namespace api {
+
+struct StorageProtocolTest : public CppUnit::TestFixture {
+ document::TestDocMan _docMan;
+ document::Document::SP _testDoc;
+ document::DocumentId _testDocId;
+ document::BucketId _bucket{16, 0x51};
+ vespalib::Version _version5_0{5, 0, 12};
+ vespalib::Version _version5_1{5, 1, 0};
+ vespalib::Version _version5_2{5, 93, 30};
+ documentapi::LoadTypeSet _loadTypes;
+ mbusprot::StorageProtocol _protocol;
+ static std::vector<std::string> _nonVerboseMessageStrings;
+ static std::vector<std::string> _verboseMessageStrings;
+ static std::vector<char> _serialization50;
+ static auto constexpr CONDITION_STRING = "There's just one condition";
+
+ StorageProtocolTest()
+ : _docMan(),
+ _testDoc(_docMan.createDocument()),
+ _testDocId(_testDoc->getId()),
+ _protocol(_docMan.getTypeRepoSP(), _loadTypes)
+ {
+ _loadTypes.addLoadType(34, "foo", documentapi::Priority::PRI_NORMAL_2);
+ }
+
+ template<typename Command>
+ std::shared_ptr<Command> copyCommand(const std::shared_ptr<Command>&, vespalib::Version);
+ template<typename Reply>
+ std::shared_ptr<Reply> copyReply(const std::shared_ptr<Reply>&);
+ void recordOutput(const api::StorageMessage& msg);
+
+ void recordSerialization50();
+
+ void testWriteSerialization50();
+ void testAddress50();
+ void testStringOutputs();
+
+ void testPut51();
+ void testUpdate51();
+ void testGet51();
+ void testRemove51();
+ void testRevert51();
+ void testRequestBucketInfo51();
+ void testNotifyBucketChange51();
+ void testCreateBucket51();
+ void testDeleteBucket51();
+ void testMergeBucket51();
+ void testGetBucketDiff51();
+ void testApplyBucketDiff51();
+ void testSplitBucket51();
+ void testSplitBucketChain51();
+ void testJoinBuckets51();
+ void testMultiOperation51();
+ void testBatchPutRemove51();
+ void testCreateVisitor51();
+ void testDestroyVisitor51();
+ void testRemoveLocation51();
+ void testInternalMessage();
+ void testSetBucketState51();
+
+ void testPutCommand52();
+ void testUpdateCommand52();
+ void testRemoveCommand52();
+
+ CPPUNIT_TEST_SUITE(StorageProtocolTest);
+
+ // Enable to see string outputs of messages
+ // CPPUNIT_TEST_DISABLED(testStringOutputs);
+
+ // Enable this to write 5.0 serialization to disk
+ // CPPUNIT_TEST_DISABLED(testWriteSerialization50);
+ // CPPUNIT_TEST_DISABLED(testAddress50);
+
+ // 5.1 tests
+ CPPUNIT_TEST(testPut51);
+ CPPUNIT_TEST(testUpdate51);
+ CPPUNIT_TEST(testGet51);
+ CPPUNIT_TEST(testRemove51);
+ CPPUNIT_TEST(testRevert51);
+ CPPUNIT_TEST(testRequestBucketInfo51);
+ CPPUNIT_TEST(testNotifyBucketChange51);
+ CPPUNIT_TEST(testCreateBucket51);
+ CPPUNIT_TEST(testDeleteBucket51);
+ CPPUNIT_TEST(testMergeBucket51);
+ CPPUNIT_TEST(testGetBucketDiff51);
+ CPPUNIT_TEST(testApplyBucketDiff51);
+ CPPUNIT_TEST(testSplitBucket51);
+ CPPUNIT_TEST(testJoinBuckets51);
+ CPPUNIT_TEST(testCreateVisitor51);
+ CPPUNIT_TEST(testDestroyVisitor51);
+ CPPUNIT_TEST(testRemoveLocation51);
+ CPPUNIT_TEST(testMultiOperation51);
+ CPPUNIT_TEST(testBatchPutRemove51);
+ CPPUNIT_TEST(testInternalMessage);
+ CPPUNIT_TEST(testSetBucketState51);
+
+ // 5.2 tests
+ CPPUNIT_TEST(testPutCommand52);
+ CPPUNIT_TEST(testUpdateCommand52);
+ CPPUNIT_TEST(testRemoveCommand52);
+
+ CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(StorageProtocolTest);
+
+std::vector<std::string> StorageProtocolTest::_nonVerboseMessageStrings;
+std::vector<std::string> StorageProtocolTest::_verboseMessageStrings;
+std::vector<char> StorageProtocolTest::_serialization50;
+
+void
+StorageProtocolTest::recordOutput(const api::StorageMessage& msg)
+{
+ std::ostringstream ost;
+ ost << " ";
+ msg.print(ost, false, " ");
+ _nonVerboseMessageStrings.push_back(ost.str());
+ ost.str("");
+ ost << " ";
+ msg.print(ost, true, " ");
+ _verboseMessageStrings.push_back(ost.str());
+}
+
+namespace {
+
+ bool debug = false;
+
+ struct ScopedName {
+ std::string _name;
+
+ ScopedName(const std::string& s) : _name(s) {
+ if (debug) std::cerr << "Starting test " << _name << "\n";
+ }
+ ~ScopedName() {
+ if (debug) std::cerr << "Finished test " << _name << "\n";
+ }
+ };
+
+} // Anonymous namespace
+
+namespace {
+ mbus::Message::UP lastCommand;
+ mbus::Reply::UP lastReply;
+}
+
+void
+StorageProtocolTest::testAddress50()
+{
+ StorageMessageAddress address("foo", lib::NodeType::STORAGE, 3);
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("storage/cluster.foo/storage/3/default"),
+ address.getRoute().toString());
+}
+
+template<typename Command> std::shared_ptr<Command>
+StorageProtocolTest::copyCommand(const std::shared_ptr<Command>& m, vespalib::Version version)
+{
+ mbus::Message::UP mbusMessage(new mbusprot::StorageCommand(m));
+ mbus::Blob blob = _protocol.encode(version, *mbusMessage);
+ mbus::Routable::UP copy(_protocol.decode(version, blob));
+
+ CPPUNIT_ASSERT(copy.get());
+
+ mbusprot::StorageCommand* copy2(dynamic_cast<mbusprot::StorageCommand*>(copy.get()));
+ CPPUNIT_ASSERT(copy2 != 0);
+
+ StorageCommand::SP internalMessage(copy2->getCommand());
+ lastCommand = std::move(mbusMessage);
+
+ return std::dynamic_pointer_cast<Command>(internalMessage);
+}
+
+template<typename Reply> std::shared_ptr<Reply>
+StorageProtocolTest::copyReply(const std::shared_ptr<Reply>& m)
+{
+ mbus::Reply::UP mbusMessage(new mbusprot::StorageReply(m));
+ mbus::Blob blob = _protocol.encode(_version5_1, *mbusMessage);
+ mbus::Routable::UP copy(_protocol.decode(_version5_1, blob));
+ CPPUNIT_ASSERT(copy.get());
+ mbusprot::StorageReply* copy2(
+ dynamic_cast<mbusprot::StorageReply*>(copy.get()));
+ CPPUNIT_ASSERT(copy2 != 0);
+ copy2->setMessage(std::move(lastCommand));
+ StorageReply::SP internalMessage(copy2->getReply());
+ lastReply = std::move(mbusMessage);
+ lastCommand = copy2->getMessage();
+ return std::dynamic_pointer_cast<Reply>(internalMessage);
+}
+
+void
+StorageProtocolTest::recordSerialization50()
+{
+ assert(lastCommand.get());
+ assert(lastReply.get());
+ for (uint32_t j=0; j<2; ++j) {
+ mbusprot::StorageMessage& msg(j == 0
+ ? dynamic_cast<mbusprot::StorageMessage&>(*lastCommand)
+ : dynamic_cast<mbusprot::StorageMessage&>(*lastReply));
+ msg.getInternalMessage()->forceMsgId(0);
+ mbus::Routable& routable(j == 0
+ ? dynamic_cast<mbus::Routable&>(*lastCommand)
+ : dynamic_cast<mbus::Routable&>(*lastReply));
+ mbus::Blob blob = _protocol.encode(_version5_0, routable);
+ _serialization50.push_back('\n');
+ std::string type(msg.getInternalMessage()->getType().toString());
+ for (uint32_t i=0, n=type.size(); i<n; ++i) {
+ _serialization50.push_back(type[i]);
+ }
+ _serialization50.push_back('\n');
+
+ for (uint32_t i=0, n=blob.size(); i<n; ++i) {
+ _serialization50.push_back(blob.data()[i]);
+ }
+ }
+}
+
+void
+StorageProtocolTest::testPut51()
+{
+ ScopedName test("testPut51");
+ PutCommand::SP cmd(new PutCommand(_bucket, _testDoc, 14));
+ cmd->setUpdateTimestamp(Timestamp(13));
+ cmd->setLoadType(_loadTypes["foo"]);
+ PutCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(*_testDoc, *cmd2->getDocument());
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("foo"), cmd2->getLoadType().getName());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(14), cmd2->getTimestamp());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(13), cmd2->getUpdateTimestamp());
+
+ PutReply::SP reply(new PutReply(*cmd2));
+ CPPUNIT_ASSERT(reply->hasDocument());
+ CPPUNIT_ASSERT_EQUAL(*_testDoc, *reply->getDocument());
+ PutReply::SP reply2(copyReply(reply));
+ CPPUNIT_ASSERT(reply2->hasDocument());
+ CPPUNIT_ASSERT_EQUAL(*_testDoc, *reply->getDocument());
+ CPPUNIT_ASSERT_EQUAL(_testDoc->getId(), reply2->getDocumentId());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(14), reply2->getTimestamp());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testUpdate51()
+{
+ ScopedName test("testUpdate51");
+ document::DocumentUpdate::SP update(
+ new document::DocumentUpdate(*_testDoc->getDataType(),
+ _testDoc->getId()));
+ std::shared_ptr<document::AssignValueUpdate> assignUpdate(
+ new document::AssignValueUpdate(document::IntFieldValue(17)));
+ document::FieldUpdate fieldUpdate(_testDoc->getField("headerval"));
+ fieldUpdate.addUpdate(*assignUpdate);
+ update->addUpdate(fieldUpdate);
+
+ update->addFieldPathUpdate(
+ document::FieldPathUpdate::CP(
+ new document::RemoveFieldPathUpdate(
+ _docMan.getTypeRepo(), *_testDoc->getDataType(),
+ "headerval", "testdoctype1.headerval > 0")));
+
+ UpdateCommand::SP cmd(new UpdateCommand(_bucket, update, 14));
+ CPPUNIT_ASSERT_EQUAL(Timestamp(0), cmd->getOldTimestamp());
+ cmd->setOldTimestamp(10);
+ UpdateCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(_testDocId, cmd2->getDocumentId());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(14), cmd2->getTimestamp());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(10), cmd2->getOldTimestamp());
+ CPPUNIT_ASSERT_EQUAL(*update, *cmd2->getUpdate());
+
+ UpdateReply::SP reply(new UpdateReply(*cmd2, 8));
+ UpdateReply::SP reply2(copyReply(reply));
+ CPPUNIT_ASSERT_EQUAL(_testDocId, reply2->getDocumentId());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(14), reply2->getTimestamp());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(8), reply->getOldTimestamp());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testGet51()
+{
+ ScopedName test("testGet51");
+ GetCommand::SP cmd(new GetCommand(_bucket, _testDocId, "foo,bar,vekterli", 123));
+ GetCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(_testDocId, cmd2->getDocumentId());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(123), cmd2->getBeforeTimestamp());
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("foo,bar,vekterli"), cmd2->getFieldSet());
+
+ GetReply::SP reply(new GetReply(*cmd2, _testDoc, 100));
+ GetReply::SP reply2(copyReply(reply));
+ CPPUNIT_ASSERT(reply2.get());
+ CPPUNIT_ASSERT(reply2->getDocument().get());
+ CPPUNIT_ASSERT_EQUAL(*_testDoc, *reply2->getDocument());
+ CPPUNIT_ASSERT_EQUAL(_testDoc->getId(), reply2->getDocumentId());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(123), reply2->getBeforeTimestamp());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(100), reply2->getLastModifiedTimestamp());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testRemove51()
+{
+ ScopedName test("testRemove51");
+ RemoveCommand::SP cmd(new RemoveCommand(_bucket, _testDocId, 159));
+ RemoveCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(_testDocId, cmd2->getDocumentId());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(159), cmd2->getTimestamp());
+
+ RemoveReply::SP reply(new RemoveReply(*cmd2, 48));
+ reply->setBucketInfo(BucketInfo(1,2,3,4,5, true, false, 48));
+
+ RemoveReply::SP reply2(copyReply(reply));
+ CPPUNIT_ASSERT_EQUAL(_testDocId, reply2->getDocumentId());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(159), reply2->getTimestamp());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(48), reply2->getOldTimestamp());
+ CPPUNIT_ASSERT_EQUAL(BucketInfo(1,2,3,4,5, true, false, 48),
+ reply2->getBucketInfo());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testRevert51()
+{
+ ScopedName test("testRevertCommand51");
+ std::vector<Timestamp> tokens;
+ tokens.push_back(59);
+ RevertCommand::SP cmd(new RevertCommand(_bucket, tokens));
+ RevertCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(tokens, cmd2->getRevertTokens());
+
+ RevertReply::SP reply(new RevertReply(*cmd2));
+ BucketInfo info(0x12345432, 101, 520);
+ reply->setBucketInfo(info);
+ RevertReply::SP reply2(copyReply(reply));
+
+ CPPUNIT_ASSERT_EQUAL(info, reply2->getBucketInfo());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testRequestBucketInfo51()
+{
+ ScopedName test("testRequestBucketInfo51");
+ {
+ std::vector<document::BucketId> ids;
+ ids.push_back(document::BucketId(3));
+ ids.push_back(document::BucketId(7));
+ RequestBucketInfoCommand::SP cmd(new RequestBucketInfoCommand(ids));
+ RequestBucketInfoCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(ids, cmd2->getBuckets());
+ CPPUNIT_ASSERT(!cmd2->hasSystemState());
+
+ recordOutput(*cmd2);
+ }
+ {
+ ClusterState state("distributor:3 .1.s:d");
+ RequestBucketInfoCommand::SP cmd(new RequestBucketInfoCommand(
+ 3, state, "14"));
+ RequestBucketInfoCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT(cmd2->hasSystemState());
+ CPPUNIT_ASSERT_EQUAL(uint16_t(3), cmd2->getDistributor());
+ CPPUNIT_ASSERT_EQUAL(state, cmd2->getSystemState());
+ CPPUNIT_ASSERT_EQUAL(size_t(0), cmd2->getBuckets().size());
+
+ RequestBucketInfoReply::SP reply(new RequestBucketInfoReply(*cmd));
+ RequestBucketInfoReply::Entry e;
+ e._bucketId = document::BucketId(4);
+ const uint64_t lastMod = 0x1337cafe98765432ULL;
+ e._info = BucketInfo(43, 24, 123, 44, 124, false, true, lastMod);
+ reply->getBucketInfo().push_back(e);
+ RequestBucketInfoReply::SP reply2(copyReply(reply));
+ CPPUNIT_ASSERT_EQUAL(size_t(1), reply2->getBucketInfo().size());
+ auto& entries(reply2->getBucketInfo());
+ CPPUNIT_ASSERT_EQUAL(e, entries[0]);
+ // "Last modified" not counted by operator== for some reason. Testing
+ // separately until we can figure out if this is by design or not.
+ CPPUNIT_ASSERT_EQUAL(lastMod, entries[0]._info.getLastModified());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+ }
+}
+
+void
+StorageProtocolTest::testNotifyBucketChange51()
+{
+ ScopedName test("testNotifyBucketChange51");
+ BucketInfo info(2, 3, 4);
+ NotifyBucketChangeCommand::SP cmd(new NotifyBucketChangeCommand(
+ document::BucketId(20, 1000), info));
+ NotifyBucketChangeCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(document::BucketId(20, 1000),
+ cmd2->getBucketId());
+ CPPUNIT_ASSERT_EQUAL(info, cmd2->getBucketInfo());
+
+ NotifyBucketChangeReply::SP reply(new NotifyBucketChangeReply(*cmd));
+ NotifyBucketChangeReply::SP reply2(copyReply(reply));
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testCreateBucket51()
+{
+ ScopedName test("testCreateBucket51");
+ document::BucketId id(623);
+
+ CreateBucketCommand::SP cmd(new CreateBucketCommand(id));
+ CreateBucketCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(id, cmd2->getBucketId());
+
+ CreateBucketReply::SP reply(new CreateBucketReply(*cmd));
+ CreateBucketReply::SP reply2(copyReply(reply));
+ CPPUNIT_ASSERT_EQUAL(id, reply2->getBucketId());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testDeleteBucket51()
+{
+ ScopedName test("testDeleteBucket51");
+ document::BucketId id(623);
+
+ DeleteBucketCommand::SP cmd(new DeleteBucketCommand(id));
+ BucketInfo info(0x100, 200, 300);
+ cmd->setBucketInfo(info);
+ DeleteBucketCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(id, cmd2->getBucketId());
+ CPPUNIT_ASSERT_EQUAL(info, cmd2->getBucketInfo());
+
+ DeleteBucketReply::SP reply(new DeleteBucketReply(*cmd));
+ // Not set automatically by constructor
+ reply->setBucketInfo(cmd2->getBucketInfo());
+ DeleteBucketReply::SP reply2(copyReply(reply));
+ CPPUNIT_ASSERT_EQUAL(id, reply2->getBucketId());
+ CPPUNIT_ASSERT_EQUAL(info, reply2->getBucketInfo());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testMergeBucket51()
+{
+ ScopedName test("testMergeBucket51");
+ document::BucketId id(623);
+
+ typedef api::MergeBucketCommand::Node Node;
+ std::vector<Node> nodes;
+ nodes.push_back(Node(4, false));
+ nodes.push_back(Node(13, true));
+ nodes.push_back(Node(26, true));
+
+ std::vector<uint16_t> chain;
+ // Not a valid chain wrt. the nodes, but just want to have unique values
+ chain.push_back(7);
+ chain.push_back(14);
+
+ MergeBucketCommand::SP cmd(
+ new MergeBucketCommand(id, nodes, Timestamp(1234), 567, chain));
+ MergeBucketCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(id, cmd2->getBucketId());
+ CPPUNIT_ASSERT_EQUAL(nodes, cmd2->getNodes());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(1234), cmd2->getMaxTimestamp());
+ CPPUNIT_ASSERT_EQUAL(uint32_t(567), cmd2->getClusterStateVersion());
+ CPPUNIT_ASSERT_EQUAL(chain, cmd2->getChain());
+
+ MergeBucketReply::SP reply(new MergeBucketReply(*cmd));
+ MergeBucketReply::SP reply2(copyReply(reply));
+ CPPUNIT_ASSERT_EQUAL(id, reply2->getBucketId());
+ CPPUNIT_ASSERT_EQUAL(nodes, reply2->getNodes());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(1234), reply2->getMaxTimestamp());
+ CPPUNIT_ASSERT_EQUAL(uint32_t(567), reply2->getClusterStateVersion());
+ CPPUNIT_ASSERT_EQUAL(chain, reply2->getChain());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testSplitBucket51()
+{
+ ScopedName test("testSplitBucket51");
+
+ document::BucketId bucket(16, 0);
+ SplitBucketCommand::SP cmd(new SplitBucketCommand(bucket));
+ CPPUNIT_ASSERT_EQUAL(0u, (uint32_t) cmd->getMinSplitBits());
+ CPPUNIT_ASSERT_EQUAL(58u, (uint32_t) cmd->getMaxSplitBits());
+ CPPUNIT_ASSERT_EQUAL(std::numeric_limits<uint32_t>().max(),
+ cmd->getMinByteSize());
+ CPPUNIT_ASSERT_EQUAL(std::numeric_limits<uint32_t>().max(),
+ cmd->getMinDocCount());
+ cmd->setMinByteSize(1000);
+ cmd->setMinDocCount(5);
+ cmd->setMaxSplitBits(40);
+ cmd->setMinSplitBits(20);
+ SplitBucketCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(20u, (uint32_t) cmd2->getMinSplitBits());
+ CPPUNIT_ASSERT_EQUAL(40u, (uint32_t) cmd2->getMaxSplitBits());
+ CPPUNIT_ASSERT_EQUAL(1000u, cmd2->getMinByteSize());
+ CPPUNIT_ASSERT_EQUAL(5u, cmd2->getMinDocCount());
+
+ SplitBucketReply::SP reply(new SplitBucketReply(*cmd2));
+ reply->getSplitInfo().push_back(SplitBucketReply::Entry(
+ document::BucketId(17, 0), BucketInfo(100, 1000, 10000, true, true)));
+ reply->getSplitInfo().push_back(SplitBucketReply::Entry(
+ document::BucketId(17, 1), BucketInfo(101, 1001, 10001, true, true)));
+ SplitBucketReply::SP reply2(copyReply(reply));
+
+ CPPUNIT_ASSERT_EQUAL(bucket, reply2->getBucketId());
+ CPPUNIT_ASSERT_EQUAL(size_t(2), reply2->getSplitInfo().size());
+ CPPUNIT_ASSERT_EQUAL(document::BucketId(17, 0),
+ reply2->getSplitInfo()[0].first);
+ CPPUNIT_ASSERT_EQUAL(document::BucketId(17, 1),
+ reply2->getSplitInfo()[1].first);
+ CPPUNIT_ASSERT_EQUAL(BucketInfo(100, 1000, 10000, true, true),
+ reply2->getSplitInfo()[0].second);
+ CPPUNIT_ASSERT_EQUAL(BucketInfo(101, 1001, 10001, true, true),
+ reply2->getSplitInfo()[1].second);
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testJoinBuckets51()
+{
+ ScopedName test("testJoinBuckets51");
+ document::BucketId bucket(16, 0);
+ std::vector<document::BucketId> sources;
+ sources.push_back(document::BucketId(17, 0));
+ sources.push_back(document::BucketId(17, 1));
+ JoinBucketsCommand::SP cmd(new JoinBucketsCommand(bucket));
+ cmd->getSourceBuckets() = sources;
+ cmd->setMinJoinBits(3);
+ JoinBucketsCommand::SP cmd2(copyCommand(cmd, _version5_1));
+
+ JoinBucketsReply::SP reply(new JoinBucketsReply(*cmd2));
+ reply->setBucketInfo(BucketInfo(3,4,5));
+ JoinBucketsReply::SP reply2(copyReply(reply));
+
+ CPPUNIT_ASSERT_EQUAL(sources, reply2->getSourceBuckets());
+ CPPUNIT_ASSERT_EQUAL(3, (int)cmd2->getMinJoinBits());
+ CPPUNIT_ASSERT_EQUAL(BucketInfo(3,4,5), reply2->getBucketInfo());
+ CPPUNIT_ASSERT_EQUAL(bucket, reply2->getBucketId());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+}
+
+void
+StorageProtocolTest::testDestroyVisitor51()
+{
+ ScopedName test("testDestroyVisitor51");
+
+ DestroyVisitorCommand::SP cmd(
+ new DestroyVisitorCommand("instance"));
+ DestroyVisitorCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(string("instance"), cmd2->getInstanceId());
+
+ DestroyVisitorReply::SP reply(new DestroyVisitorReply(*cmd2));
+ DestroyVisitorReply::SP reply2(copyReply(reply));
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testRemoveLocation51()
+{
+ ScopedName test("testRemoveLocation51");
+
+ RemoveLocationCommand::SP cmd(
+ new RemoveLocationCommand("id.group == \"mygroup\"", document::BucketId(16, 1234)));
+ RemoveLocationCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("id.group == \"mygroup\""), cmd2->getDocumentSelection());
+ CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1234), cmd2->getBucketId());
+
+ RemoveLocationReply::SP reply(new RemoveLocationReply(*cmd2));
+ RemoveLocationReply::SP reply2(copyReply(reply));
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testCreateVisitor51()
+{
+ ScopedName test("testCreateVisitor51");
+
+ std::vector<document::BucketId> buckets;
+ buckets.push_back(document::BucketId(16, 1));
+ buckets.push_back(document::BucketId(16, 2));
+
+ CreateVisitorCommand::SP cmd(
+ new CreateVisitorCommand("library", "id", "doc selection"));
+ cmd->setControlDestination("controldest");
+ cmd->setDataDestination("datadest");
+ cmd->setVisitorCmdId(1);
+ cmd->getParameters().set("one ring", "to rule them all");
+ cmd->getParameters().set("one ring to", "find them and");
+ cmd->getParameters().set("into darkness", "bind them");
+ cmd->setMaximumPendingReplyCount(2);
+ cmd->setFromTime(123);
+ cmd->setToTime(456);
+ cmd->getBuckets() = buckets;
+ cmd->setFieldSet("foo,bar,vekterli");
+ cmd->setVisitInconsistentBuckets();
+ cmd->setQueueTimeout(100);
+ cmd->setVisitorOrdering(document::OrderingSpecification::DESCENDING);
+ cmd->setPriority(149);
+ CreateVisitorCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(string("library"), cmd2->getLibraryName());
+ CPPUNIT_ASSERT_EQUAL(string("id"), cmd2->getInstanceId());
+ CPPUNIT_ASSERT_EQUAL(string("doc selection"),
+ cmd2->getDocumentSelection());
+ CPPUNIT_ASSERT_EQUAL(string("controldest"),
+ cmd2->getControlDestination());
+ CPPUNIT_ASSERT_EQUAL(string("datadest"), cmd2->getDataDestination());
+ CPPUNIT_ASSERT_EQUAL(api::Timestamp(123), cmd2->getFromTime());
+ CPPUNIT_ASSERT_EQUAL(api::Timestamp(456), cmd2->getToTime());
+ CPPUNIT_ASSERT_EQUAL(2u, cmd2->getMaximumPendingReplyCount());
+ CPPUNIT_ASSERT_EQUAL(buckets, cmd2->getBuckets());
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("foo,bar,vekterli"), cmd2->getFieldSet());
+ CPPUNIT_ASSERT(cmd2->visitInconsistentBuckets());
+ CPPUNIT_ASSERT_EQUAL(document::OrderingSpecification::DESCENDING, cmd2->getVisitorOrdering());
+ CPPUNIT_ASSERT_EQUAL(149, (int)cmd2->getPriority());
+
+ CreateVisitorReply::SP reply(new CreateVisitorReply(*cmd2));
+ CreateVisitorReply::SP reply2(copyReply(reply));
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testGetBucketDiff51()
+{
+ ScopedName test("testGetBucketDiff51");
+ document::BucketId id(623);
+
+ std::vector<api::MergeBucketCommand::Node> nodes;
+ nodes.push_back(4);
+ nodes.push_back(13);
+ std::vector<GetBucketDiffCommand::Entry> entries;
+ entries.push_back(GetBucketDiffCommand::Entry());
+ entries.back()._gid = document::GlobalId("1234567890abcdef");
+ entries.back()._timestamp = 123456;
+ entries.back()._headerSize = 100;
+ entries.back()._bodySize = 65536;
+ entries.back()._flags = 1;
+ entries.back()._hasMask = 3;
+
+ CPPUNIT_ASSERT_EQUAL(std::string(
+ "Entry(timestamp: 123456, gid(0x313233343536373839306162), "
+ "hasMask: 0x3,\n"
+ " header size: 100, body size: 65536, flags 0x1)"),
+ entries.back().toString(true));
+
+ GetBucketDiffCommand::SP cmd(new GetBucketDiffCommand(id, nodes, 1056));
+ cmd->getDiff() = entries;
+ GetBucketDiffCommand::SP cmd2(copyCommand(cmd, _version5_1));
+
+ GetBucketDiffReply::SP reply(new GetBucketDiffReply(*cmd2));
+ CPPUNIT_ASSERT_EQUAL(entries, reply->getDiff());
+ GetBucketDiffReply::SP reply2(copyReply(reply));
+
+ CPPUNIT_ASSERT_EQUAL(nodes, reply2->getNodes());
+ CPPUNIT_ASSERT_EQUAL(entries, reply2->getDiff());
+ CPPUNIT_ASSERT_EQUAL(Timestamp(1056), reply2->getMaxTimestamp());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testApplyBucketDiff51()
+{
+ ScopedName test("testApplyBucketDiff51");
+ document::BucketId id(16, 623);
+
+ std::vector<api::MergeBucketCommand::Node> nodes;
+ nodes.push_back(4);
+ nodes.push_back(13);
+ std::vector<ApplyBucketDiffCommand::Entry> entries;
+ entries.push_back(ApplyBucketDiffCommand::Entry());
+
+ ApplyBucketDiffCommand::SP cmd(new ApplyBucketDiffCommand(id, nodes, 1234));
+ cmd->getDiff() = entries;
+ ApplyBucketDiffCommand::SP cmd2(copyCommand(cmd, _version5_1));
+
+ ApplyBucketDiffReply::SP reply(new ApplyBucketDiffReply(*cmd2));
+ ApplyBucketDiffReply::SP reply2(copyReply(reply));
+
+ CPPUNIT_ASSERT_EQUAL(nodes, reply2->getNodes());
+ CPPUNIT_ASSERT_EQUAL(entries, reply2->getDiff());
+ CPPUNIT_ASSERT_EQUAL(1234u, reply2->getMaxBufferSize());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testMultiOperation51()
+{
+ ScopedName test("testMultiOperation51");
+
+ document::BucketId bucket(20, 0xf1f1f1f1f1ull);
+ DocumentTypeRepo::SP repo(new DocumentTypeRepo);
+ MultiOperationCommand::SP
+ cmd(new MultiOperationCommand(repo, bucket, 10000));
+ cmd->getOperations().addPut(*_testDoc);
+ MultiOperationCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(bucket, cmd2->getBucketId());
+ CPPUNIT_ASSERT_EQUAL(*_testDoc,
+ *cmd2->getOperations().begin()->getDocument());
+
+ MultiOperationReply::SP reply(new MultiOperationReply(*cmd2));
+ MultiOperationReply::SP reply2(copyReply(reply));
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+void
+StorageProtocolTest::testBatchPutRemove51()
+{
+ ScopedName test("testBatchPutRemove51");
+
+ document::BucketId bucket(20, 0xf1f1f1f1f1ull);
+ BatchPutRemoveCommand::SP cmd(new BatchPutRemoveCommand(bucket));
+ cmd->addPut(_testDoc, 100);
+ cmd->addHeaderUpdate(_testDoc, 101, 1234);
+ cmd->addRemove(_testDoc->getId(), 102);
+ cmd->forceMsgId(556677);
+ BatchPutRemoveCommand::SP cmd2(copyCommand(cmd, _version5_1));
+ CPPUNIT_ASSERT_EQUAL(bucket, cmd2->getBucketId());
+ CPPUNIT_ASSERT_EQUAL(3, (int)cmd2->getOperationCount());
+ CPPUNIT_ASSERT_EQUAL(*_testDoc, *(dynamic_cast<const BatchPutRemoveCommand::PutOperation&>(cmd2->getOperation(0)).document));
+ CPPUNIT_ASSERT_EQUAL((uint64_t)100, cmd2->getOperation(0).timestamp);
+ {
+ vespalib::nbostream header;
+ _testDoc->serializeHeader(header);
+ document::Document headerDoc(_docMan.getTypeRepo(), header);
+ CPPUNIT_ASSERT_EQUAL(
+ headerDoc,
+ *(dynamic_cast<const BatchPutRemoveCommand::HeaderUpdateOperation&>(
+ cmd2->getOperation(1)).document));
+ }
+ CPPUNIT_ASSERT_EQUAL((uint64_t)101, cmd2->getOperation(1).timestamp);
+ CPPUNIT_ASSERT_EQUAL(1234, (int)dynamic_cast<const BatchPutRemoveCommand::HeaderUpdateOperation&>(cmd2->getOperation(1)).timestampToUpdate);
+ CPPUNIT_ASSERT_EQUAL(_testDoc->getId(), dynamic_cast<const BatchPutRemoveCommand::RemoveOperation&>(cmd2->getOperation(2)).documentId);
+ CPPUNIT_ASSERT_EQUAL((uint64_t)102, cmd2->getOperation(2).timestamp);
+ CPPUNIT_ASSERT_EQUAL(uint64_t(556677), cmd2->getMsgId());
+
+ BatchPutRemoveReply::SP reply(new BatchPutRemoveReply(*cmd2));
+ reply->getDocumentsNotFound().push_back(document::DocumentId("userdoc:footype:1234:foo1"));
+ reply->getDocumentsNotFound().push_back(document::DocumentId("userdoc:footype:1234:foo2"));
+ reply->getDocumentsNotFound().push_back(document::DocumentId("userdoc:footype:1234:foo3"));
+
+ BatchPutRemoveReply::SP reply2(copyReply(reply));
+
+ CPPUNIT_ASSERT_EQUAL(3, (int)reply2->getDocumentsNotFound().size());
+ CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:footype:1234:foo1"), reply2->getDocumentsNotFound()[0]);
+ CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:footype:1234:foo2"), reply2->getDocumentsNotFound()[1]);
+ CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:footype:1234:foo3"), reply2->getDocumentsNotFound()[2]);
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+ recordSerialization50();
+}
+
+namespace {
+ struct MyCommand : public api::InternalCommand {
+ MyCommand() : InternalCommand(101) {}
+
+ api::StorageReply::UP makeReply();
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+ {
+ out << "MyCommand()";
+ if (verbose) {
+ out << " : ";
+ InternalCommand::print(out, verbose, indent);
+ }
+ }
+ };
+
+ struct MyReply : public api::InternalReply {
+ MyReply(const MyCommand& cmd) : InternalReply(102, cmd) {}
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+ {
+ out << "MyReply()";
+ if (verbose) {
+ out << " : ";
+ InternalReply::print(out, verbose, indent);
+ }
+ }
+ };
+
+ api::StorageReply::UP MyCommand::makeReply() {
+ return api::StorageReply::UP(new MyReply(*this));
+ }
+}
+
+void
+StorageProtocolTest::testInternalMessage()
+{
+ ScopedName test("testInternal51");
+ MyCommand cmd;
+ MyReply reply(cmd);
+
+ recordOutput(cmd);
+ recordOutput(reply);
+}
+
+void
+StorageProtocolTest::testSetBucketState51()
+{
+ ScopedName test("testSetBucketState51");
+ document::BucketId bucket(16, 0);
+ SetBucketStateCommand::SP cmd(
+ new SetBucketStateCommand(bucket, SetBucketStateCommand::ACTIVE));
+ SetBucketStateCommand::SP cmd2(copyCommand(cmd, _version5_1));
+
+ SetBucketStateReply::SP reply(new SetBucketStateReply(*cmd2));
+ SetBucketStateReply::SP reply2(copyReply(reply));
+
+ CPPUNIT_ASSERT_EQUAL(SetBucketStateCommand::ACTIVE, cmd2->getState());
+ CPPUNIT_ASSERT_EQUAL(bucket, cmd2->getBucketId());
+ CPPUNIT_ASSERT_EQUAL(bucket, reply2->getBucketId());
+
+ recordOutput(*cmd2);
+ recordOutput(*reply2);
+}
+
+void
+StorageProtocolTest::testPutCommand52()
+{
+ ScopedName test("testPutCommand52");
+
+ PutCommand::SP cmd(new PutCommand(_bucket, _testDoc, 14));
+ cmd->setCondition(TestAndSetCondition(CONDITION_STRING));
+
+ PutCommand::SP cmd2(copyCommand(cmd, _version5_2));
+ CPPUNIT_ASSERT_EQUAL(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection());
+}
+
+void
+StorageProtocolTest::testUpdateCommand52()
+{
+ ScopedName test("testUpdateCommand52");
+
+ document::DocumentUpdate::SP update(new document::DocumentUpdate(*_testDoc->getDataType(), _testDoc->getId()));
+ UpdateCommand::SP cmd(new UpdateCommand(_bucket, update, 14));
+ cmd->setCondition(TestAndSetCondition(CONDITION_STRING));
+
+ UpdateCommand::SP cmd2(copyCommand(cmd, _version5_2));
+ CPPUNIT_ASSERT_EQUAL(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection());
+}
+
+void
+StorageProtocolTest::testRemoveCommand52()
+{
+ ScopedName test("testRemoveCommand52");
+
+ RemoveCommand::SP cmd(new RemoveCommand(_bucket, _testDocId, 159));
+ cmd->setCondition(TestAndSetCondition(CONDITION_STRING));
+
+ RemoveCommand::SP cmd2(copyCommand(cmd, _version5_2));
+ CPPUNIT_ASSERT_EQUAL(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection());
+}
+
+void
+StorageProtocolTest::testStringOutputs()
+{
+ std::cerr << "\nNon verbose output:\n";
+ for (uint32_t i=0, n=_nonVerboseMessageStrings.size(); i<n; ++i) {
+ std::cerr << _nonVerboseMessageStrings[i] << "\n";
+ }
+ std::cerr << "\nVerbose output:\n";
+ for (uint32_t i=0, n=_verboseMessageStrings.size(); i<n; ++i) {
+ std::cerr << _verboseMessageStrings[i] << "\n";
+ }
+}
+
+void
+StorageProtocolTest::testWriteSerialization50()
+{
+ std::ofstream of("mbusprot/mbusprot.5.0.serialization.5.1");
+ of << std::hex << std::setfill('0');
+ for (uint32_t i=0, n=_serialization50.size(); i<n; ++i) {
+ char c = _serialization50[i];
+ if (c > 126 || (c < 32 && c != 10)) {
+ int32_t num = static_cast<int32_t>(c);
+ if (num < 0) num += 256;
+ of << '\\' << std::setw(2) << num;
+ } else if (c == '\\') {
+ of << "\\\\";
+ } else {
+ of << c;
+ }
+ }
+ of.close();
+}
+
+} // mbusprot
+} // storage
diff --git a/storageapi/src/tests/message/.gitignore b/storageapi/src/tests/message/.gitignore
new file mode 100644
index 00000000000..1d859456ef9
--- /dev/null
+++ b/storageapi/src/tests/message/.gitignore
@@ -0,0 +1,9 @@
+*.So
+*.core
+*.so
+.*.swp
+.config.log
+.depend
+Makefile
+test.vlog
+testrunner
diff --git a/storageapi/src/tests/messageapi/.gitignore b/storageapi/src/tests/messageapi/.gitignore
new file mode 100644
index 00000000000..1d859456ef9
--- /dev/null
+++ b/storageapi/src/tests/messageapi/.gitignore
@@ -0,0 +1,9 @@
+*.So
+*.core
+*.so
+.*.swp
+.config.log
+.depend
+Makefile
+test.vlog
+testrunner
diff --git a/storageapi/src/tests/messageapi/CMakeLists.txt b/storageapi/src/tests/messageapi/CMakeLists.txt
new file mode 100644
index 00000000000..b4b6669d3e5
--- /dev/null
+++ b/storageapi/src/tests/messageapi/CMakeLists.txt
@@ -0,0 +1,5 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(storageapi_testmessageapi INTERFACE
+ SOURCES
+ DEPENDS
+)
diff --git a/storageapi/src/tests/testrunner.cpp b/storageapi/src/tests/testrunner.cpp
new file mode 100644
index 00000000000..d6a42e57fb4
--- /dev/null
+++ b/storageapi/src/tests/testrunner.cpp
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/vdstestlib/cppunit/cppunittestrunner.h>
+
+LOG_SETUP("storageapicppunittestrunner");
+
+int
+main(int argc, char **argv)
+{
+ vdstestlib::CppUnitTestRunner testRunner;
+ return testRunner.run(argc, argv);
+}
diff --git a/storageapi/src/vespa/storageapi/.gitignore b/storageapi/src/vespa/storageapi/.gitignore
new file mode 100644
index 00000000000..43485abf58c
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/.gitignore
@@ -0,0 +1,5 @@
+.cvsignore
+.depend
+Makefile
+features.h
+/libstorageapi.so.5.1
diff --git a/storageapi/src/vespa/storageapi/CMakeLists.txt b/storageapi/src/vespa/storageapi/CMakeLists.txt
new file mode 100644
index 00000000000..32b02e73c81
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(storageapi
+ SOURCES
+ $<TARGET_OBJECTS:storageapi_message>
+ $<TARGET_OBJECTS:storageapi_buckets>
+ $<TARGET_OBJECTS:storageapi_messageapi>
+ $<TARGET_OBJECTS:storageapi_mbusprot>
+ INSTALL lib64
+ DEPENDS
+)
diff --git a/storageapi/src/vespa/storageapi/app/.gitignore b/storageapi/src/vespa/storageapi/app/.gitignore
new file mode 100644
index 00000000000..fa917bb5ae5
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/app/.gitignore
@@ -0,0 +1,14 @@
+*.lo
+.depend
+.depend.NEW
+.deps
+.libs
+Makefile
+dumpstatusfile
+getbucketid
+inspectfeedapistatus
+testdocstorefile
+testdocstorefile2
+testdocstoremem
+testresendctrl
+storageapi_getbucketid_app
diff --git a/storageapi/src/vespa/storageapi/app/CMakeLists.txt b/storageapi/src/vespa/storageapi/app/CMakeLists.txt
new file mode 100644
index 00000000000..45f48e4d1a0
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/app/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(storageapi_getbucketid_app
+ SOURCES
+ getbucketid.cpp
+ INSTALL bin
+ DEPENDS
+ storageapi
+)
diff --git a/storageapi/src/vespa/storageapi/app/getbucketid.cpp b/storageapi/src/vespa/storageapi/app/getbucketid.cpp
new file mode 100644
index 00000000000..49e6ecf6ec7
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/app/getbucketid.cpp
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/document/bucket/bucketidfactory.h>
+#include <vespa/document/base/documentid.h>
+
+int main(int argc, char** argv)
+{
+ if (argc != 2) {
+ std::cerr << "Usage: getbucketid <documentid>\n";
+ exit(1);
+ }
+ document::BucketIdFactory factory;
+ document::BucketId id = factory.getBucketId(document::DocumentId(argv[1]));
+
+ printf("%s has bucketid %s\n", argv[1], id.toString().c_str());
+}
+
+
diff --git a/storageapi/src/vespa/storageapi/buckets/.gitignore b/storageapi/src/vespa/storageapi/buckets/.gitignore
new file mode 100644
index 00000000000..7ad21cf1c5e
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/buckets/.gitignore
@@ -0,0 +1,9 @@
+*.lo
+.*.swp
+.depend
+.depend.NEW
+.deps
+.libs
+Makefile
+config-stor-bucketidgen.cpp
+config-stor-bucketidgen.h
diff --git a/storageapi/src/vespa/storageapi/buckets/CMakeLists.txt b/storageapi/src/vespa/storageapi/buckets/CMakeLists.txt
new file mode 100644
index 00000000000..9e4554b26a8
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/buckets/CMakeLists.txt
@@ -0,0 +1,6 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(storageapi_buckets OBJECT
+ SOURCES
+ bucketinfo.cpp
+ DEPENDS
+)
diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp
new file mode 100644
index 00000000000..e4c89b81f18
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp
@@ -0,0 +1,131 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/buckets/bucketinfo.h>
+
+namespace storage {
+namespace api {
+
+BucketInfo::BucketInfo()
+ : _lastModified(0),
+ _checksum(0),
+ _docCount(0),
+ _totDocSize(1),
+ _metaCount(0),
+ _usedFileSize(1),
+ _ready(false),
+ _active(false)
+{
+}
+
+BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount,
+ uint32_t totDocSize)
+ : _lastModified(0),
+ _checksum(checksum),
+ _docCount(docCount),
+ _totDocSize(totDocSize),
+ _metaCount(docCount),
+ _usedFileSize(totDocSize),
+ _ready(false),
+ _active(false)
+{
+}
+
+BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount,
+ uint32_t totDocSize, uint32_t metaCount,
+ uint32_t usedFileSize)
+ : _lastModified(0),
+ _checksum(checksum),
+ _docCount(docCount),
+ _totDocSize(totDocSize),
+ _metaCount(metaCount),
+ _usedFileSize(usedFileSize),
+ _ready(false),
+ _active(false)
+{
+}
+
+BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount,
+ uint32_t totDocSize, uint32_t metaCount,
+ uint32_t usedFileSize,
+ bool ready, bool active)
+ : _lastModified(0),
+ _checksum(checksum),
+ _docCount(docCount),
+ _totDocSize(totDocSize),
+ _metaCount(metaCount),
+ _usedFileSize(usedFileSize),
+ _ready(ready),
+ _active(active)
+{
+}
+
+BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount,
+ uint32_t totDocSize, uint32_t metaCount,
+ uint32_t usedFileSize,
+ bool ready, bool active, Timestamp lastModified)
+ : _lastModified(lastModified),
+ _checksum(checksum),
+ _docCount(docCount),
+ _totDocSize(totDocSize),
+ _metaCount(metaCount),
+ _usedFileSize(usedFileSize),
+ _ready(ready),
+ _active(active)
+{
+}
+
+bool
+BucketInfo::operator==(const BucketInfo& info) const
+{
+ return (_checksum == info._checksum &&
+ _docCount == info._docCount &&
+ _totDocSize == info._totDocSize &&
+ _metaCount == info._metaCount &&
+ _usedFileSize == info._usedFileSize &&
+ _ready == info._ready &&
+ _active == info._active);
+}
+
+// TODO: add ready/active to printing
+void
+BucketInfo::print(vespalib::asciistream& out, const PrintProperties&) const
+{
+ out << "BucketInfo(";
+ if (valid()) {
+ std::ostringstream ost;
+ ost << std::hex << _checksum;
+ out << "crc 0x" << ost.str()
+ << ", docCount " << _docCount
+ << ", totDocSize " << _totDocSize;
+ if (_totDocSize != _usedFileSize) {
+ out << ", metaCount " << _metaCount
+ << ", usedFileSize " << _usedFileSize;
+ }
+ out << ", ready " << (_ready ? "true" : "false")
+ << ", active " << (_active ? "true" : "false");
+
+ if (_lastModified != 0) {
+ out << ", last modified " << _lastModified;
+ }
+ } else {
+ out << "invalid";
+ }
+ out << ")";
+}
+
+void
+BucketInfo::printXml(vespalib::XmlOutputStream& xos) const
+{
+ using namespace vespalib::xml;
+ xos << XmlAttribute("checksum", _checksum, XmlAttribute::HEX)
+ << XmlAttribute("docs", _docCount)
+ << XmlAttribute("size", _totDocSize)
+ << XmlAttribute("metacount", _metaCount)
+ << XmlAttribute("usedfilesize", _usedFileSize)
+ << XmlAttribute("ready", _ready)
+ << XmlAttribute("active", _active)
+ << XmlAttribute("lastmodified", _lastModified);
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.h b/storageapi/src/vespa/storageapi/buckets/bucketinfo.h
new file mode 100644
index 00000000000..4b5a5a724d0
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/buckets/bucketinfo.h
@@ -0,0 +1,92 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class BucketInfo
+ * @ingroup bucket
+ *
+ * @brief Contains metadata about a bucket.
+ *
+ * This class contains metadata about a bucket. It is used to send metadata
+ * within storage nodes and to distributors.
+ *
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include <vespa/storageapi/defs.h>
+#include <vespa/vespalib/util/printable.h>
+#include <vespa/vespalib/util/xmlserializable.h>
+
+namespace storage {
+namespace api {
+
+class BucketInfo : public vespalib::AsciiPrintable
+{
+ Timestamp _lastModified;
+ uint32_t _checksum;
+ uint32_t _docCount;
+ uint32_t _totDocSize;
+ uint32_t _metaCount;
+ uint32_t _usedFileSize;
+ bool _ready;
+ bool _active;
+
+public:
+ BucketInfo();
+ BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize);
+ BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize,
+ uint32_t metaCount, uint32_t usedFileSize);
+ BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize,
+ uint32_t metaCount, uint32_t usedFileSize,
+ bool ready, bool active);
+ BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize,
+ uint32_t metaCount, uint32_t usedFileSize,
+ bool ready, bool active, Timestamp lastModified);
+
+ Timestamp getLastModified() const { return _lastModified; }
+ uint32_t getChecksum() const { return _checksum; }
+ uint32_t getDocumentCount() const { return _docCount; }
+ uint32_t getTotalDocumentSize() const { return _totDocSize; }
+ uint32_t getMetaCount() const { return _metaCount; }
+ uint32_t getUsedFileSize() const { return _usedFileSize; }
+ bool isReady() const { return _ready; }
+ bool isActive() const { return _active; }
+
+ void setChecksum(uint32_t crc) { _checksum = crc; }
+ void setDocumentCount(uint32_t count) { _docCount = count; }
+ void setTotalDocumentSize(uint32_t size) { _totDocSize = size; }
+ void setMetaCount(uint32_t count) { _metaCount = count; }
+ void setUsedFileSize(uint32_t size) { _usedFileSize = size; }
+ void setReady(bool ready = true) { _ready = ready; }
+ void setActive(bool active = true) { _active = active; }
+ void setLastModified(Timestamp lastModified) { _lastModified = lastModified; }
+
+ /**
+ * Only compare checksum, total document count and document
+ * size, not meta count or used file size.
+ */
+ bool equalDocumentInfo(const BucketInfo& other) const {
+ return (_checksum == other._checksum
+ && _docCount == other._docCount
+ && _totDocSize == other._totDocSize);
+ }
+
+ bool operator==(const BucketInfo& info) const;
+ bool valid() const { return (_docCount > 0 || _totDocSize == 0); }
+ bool empty() const {
+ return _metaCount == 0 && _usedFileSize == 0 && _checksum == 0;
+ }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+ {
+ vespalib::AsciiPrintable::print(out, verbose, indent);
+ }
+ virtual void print(vespalib::asciistream&, const PrintProperties&) const;
+
+ void printXml(vespalib::XmlOutputStream&) const;
+};
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/defs.h b/storageapi/src/vespa/storageapi/defs.h
new file mode 100644
index 00000000000..787e7bdfbee
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/defs.h
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \file defs.h
+ *
+ * \brief Common declarations for the storage API code.
+ */
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+
+namespace storage {
+namespace api {
+
+typedef uint64_t Timestamp;
+typedef uint32_t VisitorId;
+
+const Timestamp MAX_TIMESTAMP = (Timestamp)-1ll;
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/mbusprot/.gitignore b/storageapi/src/vespa/storageapi/mbusprot/.gitignore
new file mode 100644
index 00000000000..526f91c6668
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/.gitignore
@@ -0,0 +1,7 @@
+*.lo
+.*.swp
+.depend
+.depend.NEW
+.deps
+.libs
+Makefile
diff --git a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt
new file mode 100644
index 00000000000..3fea219677d
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt
@@ -0,0 +1,14 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(storageapi_mbusprot OBJECT
+ SOURCES
+ storagemessage.cpp
+ storagecommand.cpp
+ storagereply.cpp
+ protocolserialization.cpp
+ storageprotocol.cpp
+ protocolserialization4_2.cpp
+ protocolserialization5_0.cpp
+ protocolserialization5_1.cpp
+ protocolserialization5_2.cpp
+ DEPENDS
+)
diff --git a/storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h b/storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h
new file mode 100644
index 00000000000..9d2b13bff10
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \brief Maps return code values used between 4.2 and 5.0
+ */
+#pragma once
+
+namespace storage {
+namespace mbusprot {
+
+int getOldErrorCode(api::ReturnCode::Result newErrorCode) {
+ switch (newErrorCode) {
+ case api::ReturnCode::OK: return 1;
+ case api::ReturnCode::EXISTS: return 1001;
+ case api::ReturnCode::NOT_READY: return 2000;
+ case api::ReturnCode::WRONG_DISTRIBUTION: return 2001;
+ case api::ReturnCode::REJECTED: return 2002;
+ case api::ReturnCode::ABORTED: return 2003;
+ case api::ReturnCode::BUCKET_NOT_FOUND: return 2004;
+ case api::ReturnCode::BUCKET_DELETED: return 2004;
+ case api::ReturnCode::TIMESTAMP_EXIST: return 2005;
+ case api::ReturnCode::UNKNOWN_COMMAND: return 3000;
+ case api::ReturnCode::NOT_IMPLEMENTED: return 3001;
+ case api::ReturnCode::ILLEGAL_PARAMETERS: return 3002;
+ case api::ReturnCode::IGNORED: return 3003;
+ case api::ReturnCode::UNPARSEABLE: return 3004;
+ case api::ReturnCode::NOT_CONNECTED: return 4000;
+ case api::ReturnCode::TIMEOUT: return 4003;
+ case api::ReturnCode::BUSY: return 4004;
+ case api::ReturnCode::NO_SPACE: return 5000;
+ case api::ReturnCode::DISK_FAILURE: return 5001;
+ case api::ReturnCode::IO_FAILURE: return 5003;
+ case api::ReturnCode::INTERNAL_FAILURE: return 6000;
+ default: return 6001;
+ }
+}
+
+api::ReturnCode::Result getNewErrorCode(int oldErrorCode) {
+ switch (oldErrorCode) {
+ case 1: return api::ReturnCode::OK;
+ case 1000: return api::ReturnCode::OK; // NOT_FOUND
+ case 1001: return api::ReturnCode::EXISTS;
+ case 2000: return api::ReturnCode::NOT_READY;
+ case 2001: return api::ReturnCode::WRONG_DISTRIBUTION;
+ case 2002: return api::ReturnCode::REJECTED;
+ case 2003: return api::ReturnCode::ABORTED;
+ case 2004: return api::ReturnCode::BUCKET_NOT_FOUND;
+ case 2005: return api::ReturnCode::TIMESTAMP_EXIST;
+ case 3000: return api::ReturnCode::UNKNOWN_COMMAND;
+ case 3001: return api::ReturnCode::NOT_IMPLEMENTED;
+ case 3002: return api::ReturnCode::ILLEGAL_PARAMETERS;
+ case 3003: return api::ReturnCode::IGNORED;
+ case 3004: return api::ReturnCode::UNPARSEABLE;
+ case 4000: return api::ReturnCode::NOT_CONNECTED;
+ case 4001: return api::ReturnCode::BUSY; // OVERLOAD;
+ case 4002: return api::ReturnCode::NOT_READY; // REMOTE_DISABLED;
+ case 4003: return api::ReturnCode::TIMEOUT;
+ case 4004: return api::ReturnCode::BUSY;
+ case 5000: return api::ReturnCode::NO_SPACE;
+ case 5001: return api::ReturnCode::DISK_FAILURE;
+ case 5003: return api::ReturnCode::IO_FAILURE;
+ case 6000: return api::ReturnCode::INTERNAL_FAILURE;
+ default: return api::ReturnCode::INTERNAL_FAILURE;
+ }
+}
+
+} // mbusprot
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp
new file mode 100644
index 00000000000..1cbe0d09071
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp
@@ -0,0 +1,302 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/mbusprot/protocolserialization4_2.h>
+
+#include <vespa/log/log.h>
+#include <vespa/storageapi/messageapi/storagemessage.h>
+#include <vespa/storageapi/message/bucket.h>
+#include <vespa/storageapi/message/bucketsplitting.h>
+#include <vespa/storageapi/message/persistence.h>
+#include <vespa/storageapi/message/multioperation.h>
+#include <vespa/storageapi/message/batch.h>
+#include <vespa/storageapi/message/removelocation.h>
+#include <vespa/storageapi/mbusprot/serializationhelper.h>
+#include <vespa/storageapi/mbusprot/storagecommand.h>
+#include <vespa/storageapi/mbusprot/storagereply.h>
+#include <vespa/vespalib/util/growablebytebuffer.h>
+
+LOG_SETUP(".storage.api.mbusprot.serialization.base");
+
+namespace storage {
+namespace mbusprot {
+
+ProtocolSerialization::ProtocolSerialization(
+ const document::DocumentTypeRepo::SP& repo)
+ : _repo(repo)
+{
+}
+
+mbus::Blob
+ProtocolSerialization::encode(const api::StorageMessage& msg) const
+{
+ vespalib::GrowableByteBuffer buf;
+
+ buf.putInt(msg.getType().getId());
+ switch (msg.getType().getId()) {
+ case api::MessageType::PUT_ID:
+ onEncode(buf, static_cast<const api::PutCommand&>(msg));
+ break;
+ case api::MessageType::PUT_REPLY_ID:
+ onEncode(buf, static_cast<const api::PutReply&>(msg));
+ break;
+ case api::MessageType::UPDATE_ID:
+ onEncode(buf, static_cast<const api::UpdateCommand&>(msg));
+ break;
+ case api::MessageType::UPDATE_REPLY_ID:
+ onEncode(buf, static_cast<const api::UpdateReply&>(msg));
+ break;
+ case api::MessageType::GET_ID:
+ onEncode(buf, static_cast<const api::GetCommand&>(msg));
+ break;
+ case api::MessageType::GET_REPLY_ID:
+ onEncode(buf, static_cast<const api::GetReply&>(msg));
+ break;
+ case api::MessageType::REMOVE_ID:
+ onEncode(buf, static_cast<const api::RemoveCommand&>(msg));
+ break;
+ case api::MessageType::REMOVE_REPLY_ID:
+ onEncode(buf, static_cast<const api::RemoveReply&>(msg));
+ break;
+ case api::MessageType::REVERT_ID:
+ onEncode(buf, static_cast<const api::RevertCommand&>(msg));
+ break;
+ case api::MessageType::REVERT_REPLY_ID:
+ onEncode(buf, static_cast<const api::RevertReply&>(msg));
+ break;
+ case api::MessageType::DELETEBUCKET_ID:
+ onEncode(buf, static_cast<const api::DeleteBucketCommand&>(msg));
+ break;
+ case api::MessageType::DELETEBUCKET_REPLY_ID:
+ onEncode(buf, static_cast<const api::DeleteBucketReply&>(msg));
+ break;
+ case api::MessageType::CREATEBUCKET_ID:
+ onEncode(buf, static_cast<const api::CreateBucketCommand&>(msg));
+ break;
+ case api::MessageType::CREATEBUCKET_REPLY_ID:
+ onEncode(buf, static_cast<const api::CreateBucketReply&>(msg));
+ break;
+ case api::MessageType::MERGEBUCKET_ID:
+ onEncode(buf, static_cast<const api::MergeBucketCommand&>(msg));
+ break;
+ case api::MessageType::MERGEBUCKET_REPLY_ID:
+ onEncode(buf, static_cast<const api::MergeBucketReply&>(msg));
+ break;
+ case api::MessageType::GETBUCKETDIFF_ID:
+ onEncode(buf, static_cast<const api::GetBucketDiffCommand&>(msg));
+ break;
+ case api::MessageType::GETBUCKETDIFF_REPLY_ID:
+ onEncode(buf, static_cast<const api::GetBucketDiffReply&>(msg));
+ break;
+ case api::MessageType::APPLYBUCKETDIFF_ID:
+ onEncode(buf, static_cast<const api::ApplyBucketDiffCommand&>(msg));
+ break;
+ case api::MessageType::APPLYBUCKETDIFF_REPLY_ID:
+ onEncode(buf, static_cast<const api::ApplyBucketDiffReply&>(msg));
+ break;
+ case api::MessageType::REQUESTBUCKETINFO_ID:
+ onEncode(buf, static_cast<const api::RequestBucketInfoCommand&>(msg));
+ break;
+ case api::MessageType::REQUESTBUCKETINFO_REPLY_ID:
+ onEncode(buf, static_cast<const api::RequestBucketInfoReply&>(msg));
+ break;
+ case api::MessageType::NOTIFYBUCKETCHANGE_ID:
+ onEncode(buf, static_cast<const api::NotifyBucketChangeCommand&>(msg));
+ break;
+ case api::MessageType::NOTIFYBUCKETCHANGE_REPLY_ID:
+ onEncode(buf, static_cast<const api::NotifyBucketChangeReply&>(msg));
+ break;
+ case api::MessageType::SPLITBUCKET_ID:
+ onEncode(buf, static_cast<const api::SplitBucketCommand&>(msg));
+ break;
+ case api::MessageType::SPLITBUCKET_REPLY_ID:
+ onEncode(buf, static_cast<const api::SplitBucketReply&>(msg));
+ break;
+ case api::MessageType::JOINBUCKETS_ID:
+ onEncode(buf, static_cast<const api::JoinBucketsCommand&>(msg));
+ break;
+ case api::MessageType::JOINBUCKETS_REPLY_ID:
+ onEncode(buf, static_cast<const api::JoinBucketsReply&>(msg));
+ break;
+ case api::MessageType::MULTIOPERATION_ID:
+ onEncode(buf, static_cast<const api::MultiOperationCommand&>(msg));
+ break;
+ case api::MessageType::MULTIOPERATION_REPLY_ID:
+ onEncode(buf, static_cast<const api::MultiOperationReply&>(msg));
+ break;
+ case api::MessageType::VISITOR_CREATE_ID:
+ onEncode(buf, static_cast<const api::CreateVisitorCommand&>(msg));
+ break;
+ case api::MessageType::VISITOR_CREATE_REPLY_ID:
+ onEncode(buf, static_cast<const api::CreateVisitorReply&>(msg));
+ break;
+ case api::MessageType::VISITOR_DESTROY_ID:
+ onEncode(buf, static_cast<const api::DestroyVisitorCommand&>(msg));
+ break;
+ case api::MessageType::VISITOR_DESTROY_REPLY_ID:
+ onEncode(buf, static_cast<const api::DestroyVisitorReply&>(msg));
+ break;
+ case api::MessageType::REMOVELOCATION_ID:
+ onEncode(buf, static_cast<const api::RemoveLocationCommand&>(msg));
+ break;
+ case api::MessageType::REMOVELOCATION_REPLY_ID:
+ onEncode(buf, static_cast<const api::RemoveLocationReply&>(msg));
+ break;
+ case api::MessageType::BATCHPUTREMOVE_ID:
+ onEncode(buf, static_cast<const api::BatchPutRemoveCommand&>(msg));
+ break;
+ case api::MessageType::BATCHPUTREMOVE_REPLY_ID:
+ onEncode(buf, static_cast<const api::BatchPutRemoveReply&>(msg));
+ break;
+ case api::MessageType::SETBUCKETSTATE_ID:
+ onEncode(buf, static_cast<const api::SetBucketStateCommand&>(msg));
+ break;
+ case api::MessageType::SETBUCKETSTATE_REPLY_ID:
+ onEncode(buf, static_cast<const api::SetBucketStateReply&>(msg));
+ break;
+ default:
+ LOG(error, "Trying to encode unhandled type %s",
+ msg.getType().toString().c_str());
+ break;
+ }
+
+ mbus::Blob retVal(buf.position());
+ memcpy(retVal.data(), buf.getBuffer(), buf.position());
+ return retVal;
+}
+
+StorageCommand::UP
+ProtocolSerialization::decodeCommand(mbus::BlobRef data) const
+{
+ LOG(spam, "Decode %d bytes of data.", data.size());
+ if (data.size() < sizeof(int32_t)) {
+ std::ostringstream ost;
+ ost << "Request of size " << data.size() << " is not big enough to be "
+ "able to store a request.";
+ throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
+ }
+
+ document::ByteBuffer buf(data.data(), data.size());
+ int type;
+ buf.getIntNetwork(type);
+ SCmd::UP cmd;
+ switch (type) {
+ case api::MessageType::PUT_ID:
+ cmd = onDecodePutCommand(buf); break;
+ case api::MessageType::UPDATE_ID:
+ cmd = onDecodeUpdateCommand(buf); break;
+ case api::MessageType::GET_ID:
+ cmd = onDecodeGetCommand(buf); break;
+ case api::MessageType::REMOVE_ID:
+ cmd = onDecodeRemoveCommand(buf); break;
+ case api::MessageType::REVERT_ID:
+ cmd = onDecodeRevertCommand(buf); break;
+ case api::MessageType::CREATEBUCKET_ID:
+ cmd = onDecodeCreateBucketCommand(buf); break;
+ case api::MessageType::DELETEBUCKET_ID:
+ cmd = onDecodeDeleteBucketCommand(buf); break;
+ case api::MessageType::MERGEBUCKET_ID:
+ cmd = onDecodeMergeBucketCommand(buf); break;
+ case api::MessageType::GETBUCKETDIFF_ID:
+ cmd = onDecodeGetBucketDiffCommand(buf); break;
+ case api::MessageType::APPLYBUCKETDIFF_ID:
+ cmd = onDecodeApplyBucketDiffCommand(buf); break;
+ case api::MessageType::REQUESTBUCKETINFO_ID:
+ cmd = onDecodeRequestBucketInfoCommand(buf); break;
+ case api::MessageType::NOTIFYBUCKETCHANGE_ID:
+ cmd = onDecodeNotifyBucketChangeCommand(buf); break;
+ case api::MessageType::SPLITBUCKET_ID:
+ cmd = onDecodeSplitBucketCommand(buf); break;
+ case api::MessageType::JOINBUCKETS_ID:
+ cmd = onDecodeJoinBucketsCommand(buf); break;
+ case api::MessageType::MULTIOPERATION_ID:
+ cmd = onDecodeMultiOperationCommand(buf); break;
+ case api::MessageType::VISITOR_CREATE_ID:
+ cmd = onDecodeCreateVisitorCommand(buf); break;
+ case api::MessageType::VISITOR_DESTROY_ID:
+ cmd = onDecodeDestroyVisitorCommand(buf); break;
+ case api::MessageType::REMOVELOCATION_ID:
+ cmd = onDecodeRemoveLocationCommand(buf); break;
+ case api::MessageType::BATCHPUTREMOVE_ID:
+ cmd = onDecodeBatchPutRemoveCommand(buf); break;
+ case api::MessageType::SETBUCKETSTATE_ID:
+ cmd = onDecodeSetBucketStateCommand(buf); break;
+ default:
+ {
+ std::ostringstream ost;
+ ost << "Unknown storage command type " << type;
+ throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
+ }
+ }
+ return StorageCommand::UP(new StorageCommand(SCmd::SP(cmd.release())));
+}
+
+StorageReply::UP
+ProtocolSerialization::decodeReply(mbus::BlobRef data,
+ const api::StorageCommand& cmd) const
+{
+ LOG(spam, "Decode %d bytes of data.", data.size());
+ if (data.size() < sizeof(int32_t)) {
+ std::ostringstream ost;
+ ost << "Request of size " << data.size() << " is not big enough to be "
+ "able to store a request.";
+ throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
+ }
+
+ document::ByteBuffer buf(data.data(), data.size());
+ int type;
+ buf.getIntNetwork(type);
+ SRep::UP reply;
+ switch (type) {
+ case api::MessageType::PUT_REPLY_ID:
+ reply = onDecodePutReply(cmd, buf); break;
+ case api::MessageType::UPDATE_REPLY_ID:
+ reply = onDecodeUpdateReply(cmd, buf); break;
+ case api::MessageType::GET_REPLY_ID:
+ reply = onDecodeGetReply(cmd, buf); break;
+ case api::MessageType::REMOVE_REPLY_ID:
+ reply = onDecodeRemoveReply(cmd, buf); break;
+ case api::MessageType::REVERT_REPLY_ID:
+ reply = onDecodeRevertReply(cmd, buf); break;
+ case api::MessageType::CREATEBUCKET_REPLY_ID:
+ reply = onDecodeCreateBucketReply(cmd, buf); break;
+ case api::MessageType::DELETEBUCKET_REPLY_ID:
+ reply = onDecodeDeleteBucketReply(cmd, buf); break;
+ case api::MessageType::MERGEBUCKET_REPLY_ID:
+ reply = onDecodeMergeBucketReply(cmd, buf); break;
+ case api::MessageType::GETBUCKETDIFF_REPLY_ID:
+ reply = onDecodeGetBucketDiffReply(cmd, buf); break;
+ case api::MessageType::APPLYBUCKETDIFF_REPLY_ID:
+ reply = onDecodeApplyBucketDiffReply(cmd, buf); break;
+ case api::MessageType::REQUESTBUCKETINFO_REPLY_ID:
+ reply = onDecodeRequestBucketInfoReply(cmd, buf); break;
+ case api::MessageType::NOTIFYBUCKETCHANGE_REPLY_ID:
+ reply = onDecodeNotifyBucketChangeReply(cmd, buf); break;
+ case api::MessageType::SPLITBUCKET_REPLY_ID:
+ reply = onDecodeSplitBucketReply(cmd, buf); break;
+ case api::MessageType::JOINBUCKETS_REPLY_ID:
+ reply = onDecodeJoinBucketsReply(cmd, buf); break;
+ case api::MessageType::MULTIOPERATION_REPLY_ID:
+ reply = onDecodeMultiOperationReply(cmd, buf); break;
+ case api::MessageType::VISITOR_CREATE_REPLY_ID:
+ reply = onDecodeCreateVisitorReply(cmd, buf); break;
+ case api::MessageType::VISITOR_DESTROY_REPLY_ID:
+ reply = onDecodeDestroyVisitorReply(cmd, buf); break;
+ case api::MessageType::REMOVELOCATION_REPLY_ID:
+ reply = onDecodeRemoveLocationReply(cmd, buf); break;
+ case api::MessageType::BATCHPUTREMOVE_REPLY_ID:
+ reply = onDecodeBatchPutRemoveReply(cmd, buf); break;
+ case api::MessageType::SETBUCKETSTATE_REPLY_ID:
+ reply = onDecodeSetBucketStateReply(cmd, buf); break;
+ default:
+ {
+ std::ostringstream ost;
+ ost << "Unknown message type " << type;
+ throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
+ }
+ }
+ return StorageReply::UP(new StorageReply(SRep::SP(reply.release())));
+}
+
+} // mbusprot
+} // storage
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h
new file mode 100644
index 00000000000..c7740a84938
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h
@@ -0,0 +1,186 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/messagebus/routable.h>
+#include <vespa/storageapi/mbusprot/storagemessage.h>
+#include <vespa/storageapi/message/bucket.h>
+
+namespace document {
+ class ByteBuffer;
+}
+namespace mbus {
+ class Blob;
+ class BlobRef;
+}
+namespace vespalib {
+ class GrowableByteBuffer;
+}
+namespace storage {
+namespace api {
+class StorageCommand;
+class StorageReply;
+class PutCommand;
+class PutReply;
+class GetCommand;
+class GetReply;
+class RemoveCommand;
+class RemoveReply;
+class RevertCommand;
+class RevertReply;
+class DeleteBucketCommand;
+class DeleteBucketReply;
+class CreateBucketCommand;
+class CreateBucketReply;
+class MergeBucketCommand;
+class MergeBucketReply;
+class GetBucketDiffCommand;
+class GetBucketDiffReply;
+class ApplyBucketDiffCommand;
+class ApplyBucketDiffReply;
+class RequestBucketInfoCommand;
+class RequestBucketInfoReply;
+class NotifyBucketChangeCommand;
+class NotifyBucketChangeReply;
+class SplitBucketCommand;
+class SplitBucketReply;
+class JoinBucketsCommand;
+class JoinBucketsReply;
+class SetBucketStateCommand;
+class SetBucketStateReply;
+class MultiOperationCommand;
+class MultiOperationReply;
+class CreateVisitorCommand;
+class RemoveLocationCommand;
+class RemoveLocationReply;
+class BatchPutRemoveCommand;
+class BatchPutRemoveReply;
+class BatchDocumentUpdateCommand;
+class BatchDocumentUpdateReply;
+}
+
+namespace mbusprot {
+
+class SerializationHelper;
+class StorageCommand;
+class StorageReply;
+
+class ProtocolSerialization {
+ const document::DocumentTypeRepo::SP _repo;
+
+public:
+ virtual mbus::Blob encode(const api::StorageMessage&) const;
+ virtual std::unique_ptr<StorageCommand> decodeCommand(mbus::BlobRef) const;
+ virtual std::unique_ptr<StorageReply> decodeReply(
+ mbus::BlobRef, const api::StorageCommand&) const;
+
+protected:
+ const document::DocumentTypeRepo& getTypeRepo() const { return *_repo; }
+ const document::DocumentTypeRepo::SP getTypeRepoSp() const
+ { return _repo; }
+
+ ProtocolSerialization(const document::DocumentTypeRepo::SP &repo);
+ virtual ~ProtocolSerialization() {}
+
+ typedef api::StorageCommand SCmd;
+ typedef api::StorageReply SRep;
+ typedef document::ByteBuffer BBuf;
+ typedef vespalib::GrowableByteBuffer GBBuf;
+ typedef SerializationHelper SH;
+ typedef StorageMessage SM;
+
+ virtual void onEncode(GBBuf&, const api::PutCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::PutReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::UpdateCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::UpdateReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::GetCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::GetReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::RemoveCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::RemoveReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::RevertCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::RevertReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::DeleteBucketCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::DeleteBucketReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::CreateBucketCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::CreateBucketReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::MergeBucketCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::MergeBucketReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::GetBucketDiffCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::GetBucketDiffReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::ApplyBucketDiffCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::ApplyBucketDiffReply&) const = 0;
+ virtual void onEncode(GBBuf&,
+ const api::RequestBucketInfoCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::RequestBucketInfoReply&) const = 0;
+ virtual void onEncode(GBBuf&,
+ const api::NotifyBucketChangeCommand&) const = 0;
+ virtual void onEncode(GBBuf&,
+ const api::NotifyBucketChangeReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::SplitBucketCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::SplitBucketReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::JoinBucketsCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::JoinBucketsReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::SetBucketStateCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::SetBucketStateReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::MultiOperationCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::MultiOperationReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::CreateVisitorCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::CreateVisitorReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::DestroyVisitorCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::DestroyVisitorReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::RemoveLocationCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::RemoveLocationReply&) const = 0;
+ virtual void onEncode(GBBuf&, const api::BatchPutRemoveCommand&) const = 0;
+ virtual void onEncode(GBBuf&, const api::BatchPutRemoveReply&) const = 0;
+
+ virtual SCmd::UP onDecodePutCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodePutReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeUpdateCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeUpdateReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeGetCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeGetReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeRemoveCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeRemoveReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeRevertCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeRevertReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeDeleteBucketCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeDeleteBucketReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeCreateBucketCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeCreateBucketReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeMergeBucketCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeMergeBucketReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeGetBucketDiffCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeGetBucketDiffReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeApplyBucketDiffCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeApplyBucketDiffReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeRequestBucketInfoCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeRequestBucketInfoReply(const SCmd&,
+ BBuf&) const = 0;
+ virtual SCmd::UP onDecodeNotifyBucketChangeCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&,
+ BBuf&) const = 0;
+ virtual SCmd::UP onDecodeSplitBucketCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeSplitBucketReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeJoinBucketsCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeJoinBucketsReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeSetBucketStateCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeSetBucketStateReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeMultiOperationCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeMultiOperationReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeCreateVisitorReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeDestroyVisitorCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const = 0;
+ virtual SCmd::UP onDecodeBatchPutRemoveCommand(BBuf&) const = 0;
+ virtual SRep::UP onDecodeBatchPutRemoveReply(const SCmd&, BBuf&) const = 0;
+
+ virtual api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const = 0;
+ virtual void putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const = 0;
+
+};
+
+} // mbusprot
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp
new file mode 100644
index 00000000000..4965a6d7df9
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp
@@ -0,0 +1,705 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/mbusprot/protocolserialization4_2.h>
+
+#include <vespa/log/log.h>
+#include <vespa/messagebus/blob.h>
+#include <vespa/messagebus/blobref.h>
+#include <vespa/storageapi/messageapi/storagemessage.h>
+#include <vespa/storageapi/message/bucket.h>
+#include <vespa/storageapi/message/bucketsplitting.h>
+#include <vespa/storageapi/message/persistence.h>
+#include <vespa/storageapi/message/batch.h>
+#include <vespa/storageapi/message/multioperation.h>
+#include <vespa/storageapi/mbusprot/oldreturncodemapper.h>
+#include <vespa/storageapi/mbusprot/serializationhelper.h>
+#include <vespa/storageapi/mbusprot/storagecommand.h>
+#include <vespa/storageapi/mbusprot/storagereply.h>
+#include <vespa/storageapi/mbusprot/storageprotocol.h>
+#include <vespa/storageapi/message/removelocation.h>
+#include <vespa/vespalib/util/growablebytebuffer.h>
+
+LOG_SETUP(".storage.api.mbusprot.serialization.4_2");
+
+namespace storage {
+namespace mbusprot {
+
+ProtocolSerialization4_2::ProtocolSerialization4_2(
+ const document::DocumentTypeRepo::SP& repo)
+ : ProtocolSerialization(repo)
+{
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::MultiOperationCommand& msg) const
+{
+ uint64_t docBlockSize = msg.getOperations().spaceNeeded();
+ buf.putInt(docBlockSize);
+ char* pos = buf.allocate(docBlockSize);
+ vdslib::DocumentList copy(msg.getOperations(), pos, docBlockSize);
+ buf.putBoolean(msg.keepTimeStamps());
+ buf.putLong(msg.getBucketId().getRawId());
+ onEncodeBucketInfoCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeMultiOperationCommand(BBuf& buf) const
+{
+ uint32_t length = SH::getInt(buf);
+ std::vector<char> buffer(length);
+ buf.getBytes(&buffer[0], length);
+ bool keepTimestamps = SH::getBoolean(buf);
+ document::BucketId bucket(SH::getLong(buf));
+ api::MultiOperationCommand::UP msg(
+ new api::MultiOperationCommand(getTypeRepoSp(),
+ bucket, buffer, keepTimestamps));
+ onDecodeBucketInfoCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void
+ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::BatchPutRemoveCommand& msg) const
+{
+ // Serialization format - allow different types of serialization depending on source.
+ buf.putByte(0);
+ buf.putLong(msg.getBucketId().getRawId());
+ buf.putInt(msg.getOperationCount());
+
+ for (uint32_t i = 0; i < msg.getOperationCount(); i++) {
+ const api::BatchPutRemoveCommand::Operation& op = msg.getOperation(i);
+ buf.putByte((uint8_t)op.type);
+ buf.putLong(op.timestamp);
+
+ switch (op.type) {
+ case api::BatchPutRemoveCommand::Operation::REMOVE:
+ buf.putString(static_cast<const api::BatchPutRemoveCommand::RemoveOperation&>(op).documentId.toString());
+ break;
+ case api::BatchPutRemoveCommand::Operation::HEADERUPDATE:
+ {
+ buf.putLong(static_cast<const api::BatchPutRemoveCommand::HeaderUpdateOperation&>(op).timestampToUpdate);
+
+ vespalib::nbostream stream;
+ static_cast<const api::BatchPutRemoveCommand::HeaderUpdateOperation&>(op).document->serializeHeader(stream);
+ buf.putInt(stream.size());
+ buf.putBytes(stream.peek(), stream.size());
+ break;
+ }
+ case api::BatchPutRemoveCommand::Operation::PUT:
+ SH::putDocument(static_cast<const api::BatchPutRemoveCommand::PutOperation&>(op).document.get(), buf);
+ break;
+ }
+ }
+ onEncodeBucketInfoCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeBatchPutRemoveCommand(BBuf& buf) const
+{
+ SH::getByte(buf);
+ std::unique_ptr<api::BatchPutRemoveCommand> cmd(new api::BatchPutRemoveCommand(document::BucketId(SH::getLong(buf))));
+ int length = SH::getInt(buf);
+
+ for (int i = 0; i < length; i++) {
+ int type = SH::getByte(buf);
+ long timestamp = SH::getLong(buf);
+
+ switch (type) {
+ case api::BatchPutRemoveCommand::Operation::REMOVE:
+ cmd->addRemove(document::DocumentId(SH::getString(buf)), timestamp);
+ break;
+ case api::BatchPutRemoveCommand::Operation::HEADERUPDATE:
+ {
+ long newTimestamp = SH::getLong(buf);
+ cmd->addHeaderUpdate(document::Document::SP(
+ SH::getDocument(buf, getTypeRepo())),
+ timestamp, newTimestamp);
+ break;
+ }
+ case api::BatchPutRemoveCommand::Operation::PUT:
+ cmd->addPut(document::Document::SP(SH::getDocument(
+ buf, getTypeRepo())), timestamp);
+ break;
+ }
+ }
+
+ onDecodeBucketInfoCommand(buf, *cmd);
+
+ return api::StorageCommand::UP(cmd.release());
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::BatchPutRemoveReply& msg) const
+{
+ buf.putInt(msg.getDocumentsNotFound().size());
+ for (uint32_t i = 0; i < msg.getDocumentsNotFound().size(); i++) {
+ buf.putString(msg.getDocumentsNotFound()[i].toString());
+ }
+
+ onEncodeBucketInfoReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization4_2::onDecodeBatchPutRemoveReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::BatchPutRemoveReply::UP msg(new api::BatchPutRemoveReply(
+ static_cast<const api::BatchPutRemoveCommand&>(cmd)));
+ uint32_t count = SH::getInt(buf);
+ for (uint32_t i = 0; i < count; i++) {
+ msg->getDocumentsNotFound().push_back(document::DocumentId(SH::getString(buf)));
+ }
+
+ onDecodeBucketInfoReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::GetCommand& msg) const
+{
+ buf.putString(msg.getDocumentId().toString());
+ buf.putLong(msg.getBucketId().getRawId());
+ buf.putLong(msg.getBeforeTimestamp());
+ buf.putBoolean(msg.getFieldSet() == "[header]");
+ onEncodeCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeGetCommand(BBuf& buf) const
+{
+ document::DocumentId did(SH::getString(buf));
+ document::BucketId bucket(SH::getLong(buf));
+ api::Timestamp beforeTimestamp(SH::getLong(buf));
+ bool headerOnly(SH::getBoolean(buf));
+ api::GetCommand::UP msg(
+ new api::GetCommand(bucket, did, headerOnly ? "[header]" : "[all]", beforeTimestamp));
+ onDecodeCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::RemoveCommand& msg) const
+{
+ buf.putString(msg.getDocumentId().toString());
+ buf.putLong(msg.getBucketId().getRawId());
+ buf.putLong(msg.getTimestamp());
+ onEncodeBucketInfoCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeRemoveCommand(BBuf& buf) const
+{
+ document::DocumentId did(SH::getString(buf));
+ document::BucketId bucket(SH::getLong(buf));
+ api::Timestamp timestamp(SH::getLong(buf));
+ api::RemoveCommand::UP msg(new api::RemoveCommand(bucket, did, timestamp));
+ onDecodeBucketInfoCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::RevertCommand& msg) const
+{
+ buf.putLong(msg.getBucketId().getRawId());
+ buf.putInt(msg.getRevertTokens().size());
+ for (uint32_t i=0, n=msg.getRevertTokens().size(); i<n; ++i) {
+ buf.putLong(msg.getRevertTokens()[i]);
+ }
+ onEncodeBucketInfoCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeRevertCommand(BBuf& buf) const
+{
+ document::BucketId bid(SH::getLong(buf));
+ std::vector<api::Timestamp> tokens(SH::getInt(buf));
+ for (uint32_t i=0, n=tokens.size(); i<n; ++i) {
+ tokens[i] = SH::getLong(buf);
+ }
+ api::RevertCommand::UP msg(new api::RevertCommand(bid, tokens));
+ onDecodeBucketInfoCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::CreateBucketCommand& msg) const
+{
+ buf.putLong(msg.getBucketId().getRawId());
+ onEncodeBucketInfoCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeCreateBucketCommand(BBuf& buf) const
+{
+ document::BucketId bid(SH::getLong(buf));
+ api::CreateBucketCommand::UP msg(new api::CreateBucketCommand(bid));
+ onDecodeBucketInfoCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::MergeBucketCommand& msg) const
+{
+ buf.putLong(msg.getBucketId().getRawId());
+ const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes());
+ buf.putShort(nodes.size());
+ for (uint32_t i=0; i<nodes.size(); ++i) {
+ buf.putShort(nodes[i].index);
+ buf.putBoolean(nodes[i].sourceOnly);
+ }
+ buf.putLong(msg.getMaxTimestamp());
+ onEncodeCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeMergeBucketCommand(BBuf& buf) const
+{
+ typedef api::MergeBucketCommand::Node Node;
+ document::BucketId bid(SH::getLong(buf));
+ uint16_t nodeCount = SH::getShort(buf);
+ std::vector<Node> nodes;
+ nodes.reserve(nodeCount);
+ for (uint32_t i=0; i<nodeCount; ++i) {
+ uint16_t index(SH::getShort(buf));
+ bool sourceOnly = SH::getBoolean(buf);
+ nodes.push_back(Node(index, sourceOnly));
+ }
+ api::Timestamp timestamp(SH::getLong(buf));
+ api::MergeBucketCommand::UP msg(
+ new api::MergeBucketCommand(bid, nodes, timestamp));
+ onDecodeCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::GetBucketDiffCommand& msg) const
+{
+ buf.putLong(msg.getBucketId().getRawId());
+ const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes());
+ buf.putShort(nodes.size());
+ for (uint32_t i=0; i<nodes.size(); ++i) {
+ buf.putShort(nodes[i].index);
+ buf.putBoolean(nodes[i].sourceOnly);
+ }
+ buf.putLong(msg.getMaxTimestamp());
+ const std::vector<api::GetBucketDiffCommand::Entry>& entries(msg.getDiff());
+ buf.putInt(entries.size());
+ for (uint32_t i=0; i<entries.size(); ++i) {
+ onEncodeDiffEntry(buf, entries[i]);
+ }
+ onEncodeCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeGetBucketDiffCommand(BBuf& buf) const
+{
+ typedef api::MergeBucketCommand::Node Node;
+ document::BucketId bid(SH::getLong(buf));
+ uint16_t nodeCount = SH::getShort(buf);
+ std::vector<Node> nodes;
+ nodes.reserve(nodeCount);
+ for (uint32_t i=0; i<nodeCount; ++i) {
+ uint16_t index(SH::getShort(buf));
+ bool sourceOnly = SH::getBoolean(buf);
+ nodes.push_back(Node(index, sourceOnly));
+ }
+ api::Timestamp timestamp = SH::getLong(buf);
+ api::GetBucketDiffCommand::UP msg(
+ new api::GetBucketDiffCommand(bid, nodes, timestamp));
+ std::vector<api::GetBucketDiffCommand::Entry>& entries(msg->getDiff());
+ uint32_t entryCount = SH::getInt(buf);
+ if (entryCount > buf.getRemaining()) {
+ // Trigger out of bounds exception rather than out of memory error
+ buf.incPos(entryCount);
+ }
+ entries.resize(entryCount);
+ for (uint32_t i=0; i<entries.size(); ++i) {
+ onDecodeDiffEntry(buf, entries[i]);
+ }
+ onDecodeCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const
+{
+ buf.putLong(msg.getBucketId().getRawId());
+ const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes());
+ buf.putShort(nodes.size());
+ for (uint32_t i=0; i<nodes.size(); ++i) {
+ buf.putShort(nodes[i].index);
+ buf.putBoolean(nodes[i].sourceOnly);
+ }
+ buf.putInt(msg.getMaxBufferSize());
+ const std::vector<api::ApplyBucketDiffCommand::Entry>& entries(
+ msg.getDiff());
+ buf.putInt(entries.size());
+ for (uint32_t i=0; i<entries.size(); ++i) {
+ onEncodeDiffEntry(buf, entries[i]._entry);
+ buf.putString(entries[i]._docName);
+ buf.putInt(entries[i]._headerBlob.size());
+ buf.putBytes(&entries[i]._headerBlob[0],
+ entries[i]._headerBlob.size());
+ buf.putInt(entries[i]._bodyBlob.size());
+ buf.putBytes(&entries[i]._bodyBlob[0],
+ entries[i]._bodyBlob.size());
+ }
+ onEncodeBucketInfoCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeApplyBucketDiffCommand(BBuf& buf) const
+{
+ typedef api::MergeBucketCommand::Node Node;
+ document::BucketId bid(SH::getLong(buf));
+ uint16_t nodeCount = SH::getShort(buf);
+ std::vector<Node> nodes;
+ nodes.reserve(nodeCount);
+ for (uint32_t i=0; i<nodeCount; ++i) {
+ uint16_t index(SH::getShort(buf));
+ bool sourceOnly = SH::getBoolean(buf);
+ nodes.push_back(Node(index, sourceOnly));
+ }
+ uint32_t maxBufferSize(SH::getInt(buf));
+ api::ApplyBucketDiffCommand::UP msg(
+ new api::ApplyBucketDiffCommand(bid, nodes, maxBufferSize));
+ std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg->getDiff());
+ uint32_t entryCount = SH::getInt(buf);
+ if (entryCount > buf.getRemaining()) {
+ // Trigger out of bounds exception rather than out of memory error
+ buf.incPos(entryCount);
+ }
+ entries.resize(entryCount);
+ for (uint32_t i=0; i<entries.size(); ++i) {
+ onDecodeDiffEntry(buf, entries[i]._entry);
+ entries[i]._docName = SH::getString(buf);
+ uint32_t headerSize = SH::getInt(buf);
+ if (headerSize > buf.getRemaining()) {
+ buf.incPos(headerSize);
+ }
+ entries[i]._headerBlob.resize(headerSize);
+ buf.getBytes(&entries[i]._headerBlob[0],
+ entries[i]._headerBlob.size());
+ uint32_t bodySize = SH::getInt(buf);
+ if (bodySize > buf.getRemaining()) {
+ buf.incPos(bodySize);
+ }
+ entries[i]._bodyBlob.resize(bodySize);
+ buf.getBytes(&entries[i]._bodyBlob[0],
+ entries[i]._bodyBlob.size());
+ }
+ onDecodeBucketInfoCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void
+ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RequestBucketInfoReply& msg) const
+{
+ buf.putInt(msg.getBucketInfo().size());
+ for (const auto & entry : msg.getBucketInfo()) {
+ buf.putLong(entry._bucketId.getRawId());
+ putBucketInfo(entry._info, buf);
+ }
+ onEncodeReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::RequestBucketInfoReply::UP msg(new api::RequestBucketInfoReply(
+ static_cast<const api::RequestBucketInfoCommand&>(cmd)));
+ api::RequestBucketInfoReply::EntryVector & entries(msg->getBucketInfo());
+ uint32_t entryCount = SH::getInt(buf);
+ if (entryCount > buf.getRemaining()) {
+ // Trigger out of bounds exception rather than out of memory error
+ buf.incPos(entryCount);
+ }
+ entries.resize(entryCount);
+ for (auto & entry : entries) {
+ entry._bucketId = document::BucketId(SH::getLong(buf));
+ entry._info = getBucketInfo(buf);
+ }
+ onDecodeReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const
+{
+ buf.putLong(msg.getBucketId().getRawId());
+ putBucketInfo(msg.getBucketInfo(), buf);
+ onEncodeCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeNotifyBucketChangeCommand(BBuf& buf) const
+{
+ document::BucketId bucket(SH::getLong(buf));
+ api::BucketInfo info(getBucketInfo(buf));
+ api::NotifyBucketChangeCommand::UP msg(
+ new api::NotifyBucketChangeCommand(bucket, info));
+ onDecodeCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::NotifyBucketChangeReply& msg) const
+{
+ onEncodeReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization4_2::onDecodeNotifyBucketChangeReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::NotifyBucketChangeReply::UP msg(new api::NotifyBucketChangeReply(
+ static_cast<const api::NotifyBucketChangeCommand&>(cmd)));
+ onDecodeReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::SplitBucketCommand& msg) const
+{
+ buf.putLong(msg.getBucketId().getRawId());
+ buf.putByte(msg.getMinSplitBits());
+ buf.putByte(msg.getMaxSplitBits());
+ buf.putInt(msg.getMinByteSize());
+ buf.putInt(msg.getMinDocCount());
+ onEncodeCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeSplitBucketCommand(BBuf& buf) const
+{
+ document::BucketId id(SH::getLong(buf));
+ api::SplitBucketCommand::UP msg(new api::SplitBucketCommand(id));
+ msg->setMinSplitBits(SH::getByte(buf));
+ msg->setMaxSplitBits(SH::getByte(buf));
+ msg->setMinByteSize(SH::getInt(buf));
+ msg->setMinDocCount(SH::getInt(buf));
+ onDecodeCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf&, const api::SetBucketStateCommand&) const
+{
+ throw vespalib::IllegalStateException("Unsupported serialization",
+ VESPA_STRLOC);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeSetBucketStateCommand(BBuf&) const
+{
+ throw vespalib::IllegalStateException("Unsupported deserialization",
+ VESPA_STRLOC);
+}
+
+void ProtocolSerialization4_2::onEncode(
+ GBBuf&, const api::SetBucketStateReply&) const
+{
+ throw vespalib::IllegalStateException("Unsupported serialization",
+ VESPA_STRLOC);
+}
+
+api::StorageReply::UP
+ProtocolSerialization4_2::onDecodeSetBucketStateReply(const SCmd&,
+ BBuf&) const
+{
+ throw vespalib::IllegalStateException("Unsupported deserialization",
+ VESPA_STRLOC);
+}
+
+void
+ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::CreateVisitorCommand& msg) const
+{
+ buf.putString(msg.getLibraryName());
+ buf.putString(msg.getInstanceId());
+ buf.putString(msg.getDocumentSelection());
+ buf.putInt(msg.getVisitorCmdId());
+ buf.putString(msg.getControlDestination());
+ buf.putString(msg.getDataDestination());
+ buf.putInt(msg.getMaximumPendingReplyCount());
+ buf.putLong(msg.getFromTime());
+ buf.putLong(msg.getToTime());
+
+ buf.putInt(msg.getBuckets().size());
+ for (uint32_t i = 0; i < msg.getBuckets().size(); i++) {
+ buf.putLong(msg.getBuckets()[i].getRawId());
+ }
+
+ buf.putBoolean(msg.visitRemoves());
+ buf.putBoolean(msg.getFieldSet() == "[header]");
+ buf.putBoolean(msg.visitInconsistentBuckets());
+ buf.putInt(msg.getQueueTimeout());
+
+ uint32_t size = msg.getParameters().getSerializedSize();
+ char* docBuffer = buf.allocate(size);
+ document::ByteBuffer bbuf(docBuffer, size);
+ msg.getParameters().serialize(bbuf);
+
+ onEncodeCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const
+{
+ vespalib::stringref libraryName = SH::getString(buf);
+ vespalib::stringref instanceId = SH::getString(buf);
+ vespalib::stringref selection = SH::getString(buf);
+ api::CreateVisitorCommand::UP msg(
+ new api::CreateVisitorCommand(libraryName, instanceId, selection));
+ msg->setVisitorCmdId(SH::getInt(buf));
+ msg->setControlDestination(SH::getString(buf));
+ msg->setDataDestination(SH::getString(buf));
+ msg->setMaximumPendingReplyCount(SH::getInt(buf));
+
+ msg->setFromTime(SH::getLong(buf));
+ msg->setToTime(SH::getLong(buf));
+ uint32_t count = SH::getInt(buf);
+
+ if (count > buf.getRemaining()) {
+ // Trigger out of bounds exception rather than out of memory error
+ buf.incPos(count);
+ }
+
+ for (uint32_t i = 0; i < count; i++) {
+ msg->getBuckets().push_back(document::BucketId(SH::getLong(buf)));
+ }
+
+ if (SH::getBoolean(buf)) {
+ msg->setVisitRemoves();
+ }
+ if (SH::getBoolean(buf)) {
+ msg->setFieldSet("[header]");
+ }
+ if (SH::getBoolean(buf)) {
+ msg->setVisitInconsistentBuckets();
+ }
+ msg->setQueueTimeout(SH::getInt(buf));
+ msg->getParameters().deserialize(getTypeRepo(), buf);
+
+ onDecodeCommand(buf, *msg);
+ msg->setVisitorDispatcherVersion(42);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void
+ProtocolSerialization4_2::onEncode(
+ GBBuf& buf, const api::DestroyVisitorCommand& msg) const
+{
+ buf.putString(msg.getInstanceId());
+ onEncodeCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeDestroyVisitorCommand(BBuf& buf) const
+{
+ vespalib::stringref instanceId = SH::getString(buf);
+ api::DestroyVisitorCommand::UP msg(new api::DestroyVisitorCommand(instanceId));
+ onDecodeCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void
+ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::DestroyVisitorReply& msg) const
+{
+ onEncodeReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization4_2::onDecodeDestroyVisitorReply(const SCmd& cmd, BBuf& buf) const
+{
+ api::DestroyVisitorReply::UP msg(new api::DestroyVisitorReply(static_cast<const api::DestroyVisitorCommand&>(cmd)));
+ onDecodeReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void
+ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveLocationCommand& msg) const
+{
+ buf.putString(msg.getDocumentSelection());
+ buf.putLong(msg.getBucketId().getRawId());
+ onEncodeCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization4_2::onDecodeRemoveLocationCommand(BBuf& buf) const
+{
+ vespalib::stringref documentSelection = SH::getString(buf);
+ uint64_t bucketId = SH::getLong(buf);
+
+ api::RemoveLocationCommand::UP msg;
+ msg.reset(new api::RemoveLocationCommand(documentSelection, document::BucketId(bucketId)));
+ onDecodeCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void
+ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveLocationReply& msg) const
+{
+ onEncodeBucketInfoReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization4_2::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const
+{
+ api::RemoveLocationReply::UP msg(new api::RemoveLocationReply(static_cast<const api::RemoveLocationCommand&>(cmd)));
+ onDecodeBucketInfoReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+// Utility functions for serialization
+
+void
+ProtocolSerialization4_2::onEncodeBucketInfoCommand(
+ GBBuf& buf, const api::BucketInfoCommand& msg) const
+{
+ onEncodeCommand(buf, msg);
+}
+
+void
+ProtocolSerialization4_2::onDecodeBucketInfoCommand(
+ BBuf& buf, api::BucketInfoCommand& msg) const
+{
+ onDecodeCommand(buf, msg);
+}
+
+void
+ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::ReturnCode& rc) const
+{
+ // Convert error code to codes used in 4.2
+ buf.putInt(getOldErrorCode(rc.getResult()));
+ buf.putString(rc.getMessage());
+}
+
+void
+ProtocolSerialization4_2::onEncodeDiffEntry(
+ GBBuf& buf, const api::GetBucketDiffCommand::Entry& entry) const
+{
+ buf.putLong(entry._timestamp);
+ SH::putGlobalId(entry._gid, buf);
+ buf.putInt(entry._headerSize);
+ buf.putInt(entry._bodySize);
+ buf.putShort(entry._flags);
+ buf.putShort(entry._hasMask);
+}
+
+void
+ProtocolSerialization4_2::onDecodeDiffEntry(
+ BBuf& buf, api::GetBucketDiffCommand::Entry& entry) const
+{
+ entry._timestamp = SH::getLong(buf);
+ entry._gid = SH::getGlobalId(buf);
+ entry._headerSize = SH::getInt(buf);
+ entry._bodySize = SH::getInt(buf);
+ entry._flags = SH::getShort(buf);
+ entry._hasMask = SH::getShort(buf);
+}
+
+} // mbusprot
+} // storage
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h
new file mode 100644
index 00000000000..97a56b5da22
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/storageapi/mbusprot/protocolserialization.h>
+
+namespace storage {
+namespace mbusprot {
+
+class ProtocolSerialization4_2 : public ProtocolSerialization {
+public:
+ ProtocolSerialization4_2(const document::DocumentTypeRepo::SP&);
+
+protected:
+ virtual void onEncode(GBBuf&, const api::GetCommand&) const;
+ virtual void onEncode(GBBuf&, const api::RemoveCommand&) const;
+ virtual void onEncode(GBBuf&, const api::RevertCommand&) const;
+ virtual void onEncode(GBBuf&, const api::CreateBucketCommand&) const;
+ virtual void onEncode(GBBuf&, const api::MergeBucketCommand&) const;
+ virtual void onEncode(GBBuf&, const api::GetBucketDiffCommand&) const;
+ virtual void onEncode(GBBuf&, const api::ApplyBucketDiffCommand&) const;
+ virtual void onEncode(GBBuf&, const api::RequestBucketInfoReply&) const;
+ virtual void onEncode(GBBuf&, const api::NotifyBucketChangeCommand&) const;
+ virtual void onEncode(GBBuf&, const api::NotifyBucketChangeReply&) const;
+ virtual void onEncode(GBBuf&, const api::SplitBucketCommand&) const;
+ virtual void onEncode(GBBuf&, const api::MultiOperationCommand&) const;
+ virtual void onEncode(GBBuf&, const api::CreateVisitorCommand&) const;
+ virtual void onEncode(GBBuf&, const api::DestroyVisitorCommand&) const;
+ virtual void onEncode(GBBuf&, const api::DestroyVisitorReply&) const;
+ virtual void onEncode(GBBuf&, const api::RemoveLocationCommand&) const;
+ virtual void onEncode(GBBuf&, const api::RemoveLocationReply&) const;
+
+ // Not supported on 4.2, but implemented here for simplicity.
+ virtual void onEncode(GBBuf&, const api::BatchPutRemoveCommand&) const;
+ virtual void onEncode(GBBuf&, const api::BatchPutRemoveReply&) const;
+ virtual void onEncode(GBBuf&, const api::SetBucketStateCommand&) const;
+ virtual void onEncode(GBBuf&, const api::SetBucketStateReply&) const;
+
+ virtual void onEncodeBucketInfoCommand(GBBuf&, const api::BucketInfoCommand&) const;
+ virtual void onEncodeBucketInfoReply(GBBuf&, const api::BucketInfoReply&) const = 0;
+ virtual void onEncodeCommand(GBBuf&, const api::StorageCommand&) const = 0;
+ virtual void onEncodeReply(GBBuf&, const api::StorageReply&) const = 0;
+
+ virtual void onEncodeDiffEntry(GBBuf&, const api::GetBucketDiffCommand::Entry&) const;
+ virtual void onEncode(GBBuf&, const api::ReturnCode&) const;
+ virtual SCmd::UP onDecodeGetCommand(BBuf&) const;
+ virtual SCmd::UP onDecodeRemoveCommand(BBuf&) const;
+ virtual SCmd::UP onDecodeRevertCommand(BBuf&) const;
+ virtual SCmd::UP onDecodeCreateBucketCommand(BBuf&) const;
+ virtual SCmd::UP onDecodeMergeBucketCommand(BBuf&) const;
+ virtual SCmd::UP onDecodeGetBucketDiffCommand(BBuf&) const;
+ virtual SCmd::UP onDecodeApplyBucketDiffCommand(BBuf&) const;
+ virtual SRep::UP onDecodeRequestBucketInfoReply(const SCmd&, BBuf&) const;
+ virtual SCmd::UP onDecodeNotifyBucketChangeCommand(BBuf&) const;
+ virtual SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&, BBuf&) const;
+ virtual SCmd::UP onDecodeSplitBucketCommand(BBuf&) const;
+ virtual SCmd::UP onDecodeSetBucketStateCommand(BBuf&) const;
+ virtual SRep::UP onDecodeSetBucketStateReply(const SCmd&, BBuf&) const;
+ virtual SCmd::UP onDecodeMultiOperationCommand(BBuf&) const;
+ virtual SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const;
+ virtual SCmd::UP onDecodeDestroyVisitorCommand(BBuf&) const;
+ virtual SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const;
+ virtual SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const;
+ virtual SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const;
+
+ // Not supported on 4.2, but implemented here for simplicity.
+ virtual SCmd::UP onDecodeBatchPutRemoveCommand(BBuf&) const;
+ virtual SRep::UP onDecodeBatchPutRemoveReply(const SCmd&, BBuf&) const;
+
+ virtual void onDecodeBucketInfoCommand(BBuf&, api::BucketInfoCommand&) const;
+ virtual void onDecodeBucketInfoReply(BBuf&, api::BucketInfoReply&) const = 0;
+ virtual void onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const = 0;
+ virtual void onDecodeReply(BBuf&, api::StorageReply&) const = 0;
+
+ virtual void onDecodeDiffEntry(BBuf&, api::GetBucketDiffCommand::Entry&) const;
+};
+
+} // mbusprot
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp
new file mode 100644
index 00000000000..1b3427b5784
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp
@@ -0,0 +1,684 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/mbusprot/protocolserialization5_0.h>
+
+#include <vespa/log/log.h>
+#include <vespa/messagebus/blob.h>
+#include <vespa/messagebus/blobref.h>
+#include <vespa/storageapi/messageapi/storagemessage.h>
+#include <vespa/storageapi/message/bucket.h>
+#include <vespa/storageapi/message/bucketsplitting.h>
+#include <vespa/storageapi/message/persistence.h>
+#include <vespa/storageapi/message/multioperation.h>
+#include <vespa/storageapi/mbusprot/serializationhelper.h>
+#include <vespa/storageapi/mbusprot/storagecommand.h>
+#include <vespa/storageapi/mbusprot/storagereply.h>
+#include <vespa/storageapi/mbusprot/storageprotocol.h>
+#include <vespa/vespalib/util/growablebytebuffer.h>
+#include <vespa/document/select/orderingspecification.h>
+#include <vespa/storageapi/messageapi/returncode.h>
+
+LOG_SETUP(".storage.api.mbusprot.serialization.5_0");
+
+namespace storage {
+namespace mbusprot {
+
+api::BucketInfo
+ProtocolSerialization5_0::getBucketInfo(document::ByteBuffer& buf) const
+{
+ uint32_t crc(SH::getInt(buf));
+ uint32_t doccount(SH::getInt(buf));
+ uint32_t docsize(SH::getInt(buf));
+ uint32_t metacount(SH::getInt(buf));
+ uint32_t usedsize(SH::getInt(buf));
+ return api::BucketInfo(crc, doccount, docsize, metacount, usedsize);
+}
+
+void
+ProtocolSerialization5_0::putBucketInfo(
+ const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const
+{
+ buf.putInt(info.getChecksum());
+ buf.putInt(info.getDocumentCount());
+ buf.putInt(info.getTotalDocumentSize());
+ buf.putInt(info.getMetaCount());
+ buf.putInt(info.getUsedFileSize());
+}
+
+void
+ProtocolSerialization5_0::onEncodeReply(
+ GBBuf& buf, const api::StorageReply& msg) const
+{
+ SH::putReturnCode(msg.getResult(), buf);
+ buf.putLong(msg.getMsgId());
+ buf.putByte(msg.getPriority());
+}
+
+void
+ProtocolSerialization5_0::onDecodeReply(BBuf& buf,
+ api::StorageReply& msg) const
+{
+ msg.setResult(SH::getReturnCode(buf));
+ msg.forceMsgId(SH::getLong(buf));
+ msg.setPriority(SH::getByte(buf));
+}
+
+void
+ProtocolSerialization5_0::onEncodeCommand(
+ GBBuf& buf, const api::StorageCommand& msg) const
+{
+ buf.putLong(msg.getMsgId());
+ buf.putByte(msg.getPriority());
+ buf.putShort(msg.getSourceIndex());
+ buf.putInt(msg.getLoadType().getId());
+}
+
+void
+ProtocolSerialization5_0::onDecodeCommand(BBuf& buf,
+ api::StorageCommand& msg) const
+{
+ msg.forceMsgId(SH::getLong(buf));
+ uint8_t priority = SH::getByte(buf);
+ msg.setPriority(priority);
+ msg.setSourceIndex(SH::getShort(buf));
+ msg.setLoadType(_loadTypes[SH::getInt(buf)]);
+}
+
+
+ProtocolSerialization5_0::ProtocolSerialization5_0(
+ const document::DocumentTypeRepo::SP& repo,
+ const documentapi::LoadTypeSet& loadTypes)
+ : ProtocolSerialization4_2(repo),
+ _loadTypes(loadTypes)
+{
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::PutReply& msg) const
+{
+ buf.putBoolean(msg.wasFound());
+ onEncodeBucketInfoReply(buf, msg);
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::PutCommand& msg) const
+{
+ SH::putDocument(msg.getDocument().get(), buf);
+ buf.putLong(msg.getBucketId().getRawId());
+ buf.putLong(msg.getTimestamp());
+ buf.putLong(msg.getUpdateTimestamp());
+ onEncodeBucketInfoCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_0::onDecodePutCommand(BBuf& buf) const
+{
+ document::Document::SP doc(SH::getDocument(buf, getTypeRepo()));
+ document::BucketId id(SH::getLong(buf));
+ api::Timestamp ts(SH::getLong(buf));
+ api::PutCommand::UP msg(new api::PutCommand(id, doc, ts));
+ msg->setUpdateTimestamp(SH::getLong(buf));
+ onDecodeBucketInfoCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodePutReply(const SCmd& cmd, BBuf& buf) const
+{
+ bool wasFound = SH::getBoolean(buf);
+ api::PutReply::UP msg(new api::PutReply(
+ static_cast<const api::PutCommand&>(cmd), wasFound));
+ onDecodeBucketInfoReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::UpdateReply& msg) const
+{
+ buf.putLong(msg.getOldTimestamp());
+ onEncodeBucketInfoReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeUpdateReply(const SCmd& cmd, BBuf& buf) const
+{
+ api::Timestamp oldTimestamp(SH::getLong(buf));
+ api::UpdateReply::UP msg(new api::UpdateReply(
+ static_cast<const api::UpdateCommand&>(cmd), oldTimestamp));
+ onDecodeBucketInfoReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::GetReply& msg) const
+{
+ SH::putDocument(msg.getDocument().get(), buf);
+ buf.putLong(msg.getLastModifiedTimestamp());
+ onEncodeBucketInfoReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeGetReply(const SCmd& cmd, BBuf& buf) const
+{
+ try {
+ document::Document::SP doc(SH::getDocument(buf, getTypeRepo()));
+ api::Timestamp lastModified(SH::getLong(buf));
+ api::GetReply::UP msg(
+ new api::GetReply(
+ static_cast<const api::GetCommand&>(cmd),
+ doc,
+ lastModified));
+ onDecodeBucketInfoReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+ } catch (std::exception& e) {
+ api::GetReply::UP msg(
+ new api::GetReply(
+ static_cast<const api::GetCommand&>(cmd),
+ document::Document::SP(),
+ 0));
+ msg->setResult(api::ReturnCode(
+ api::ReturnCode::UNPARSEABLE,
+ e.what()));
+ return api::StorageReply::UP(msg.release());
+ }
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::RemoveReply& msg) const
+{
+ buf.putLong(msg.getOldTimestamp());
+ onEncodeBucketInfoReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeRemoveReply(const SCmd& cmd, BBuf& buf) const
+{
+ api::Timestamp oldTimestamp(SH::getLong(buf));
+ api::RemoveReply::UP msg(new api::RemoveReply(
+ static_cast<const api::RemoveCommand&>(cmd), oldTimestamp));
+ onDecodeBucketInfoReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::UpdateCommand& msg) const
+{
+ document::DocumentUpdate* update = msg.getUpdate().get();
+ if (update) {
+ vespalib::nbostream stream;
+ update->serializeHEAD(stream);
+ buf.putInt(stream.size());
+ buf.putBytes(stream.peek(), stream.size());
+ } else {
+ buf.putInt(0);
+ }
+
+ buf.putLong(msg.getBucketId().getRawId());
+ buf.putLong(msg.getTimestamp());
+ buf.putLong(msg.getOldTimestamp());
+ onEncodeBucketInfoCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_0::onDecodeUpdateCommand(BBuf& buf) const
+{
+ document::DocumentUpdate::SP update;
+
+ uint32_t size = SH::getInt(buf);
+ if (size != 0) {
+ document::ByteBuffer bbuf(buf.getBufferAtPos(), size);
+ buf.incPos(size);
+ update.reset(new document::DocumentUpdate(getTypeRepo(), bbuf,
+ document::DocumentUpdate::
+ SerializeVersion::
+ SERIALIZE_HEAD));
+ }
+
+ document::BucketId bucket(SH::getLong(buf));
+ api::Timestamp timestamp(SH::getLong(buf));
+ api::UpdateCommand::UP msg(
+ new api::UpdateCommand(bucket, update, timestamp));
+ msg->setOldTimestamp(SH::getLong(buf));
+ onDecodeBucketInfoCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::RevertReply& msg) const
+{
+ onEncodeBucketInfoReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeRevertReply(const SCmd& cmd, BBuf& buf) const
+{
+ api::RevertReply::UP msg(new api::RevertReply(
+ static_cast<const api::RevertCommand&>(cmd)));
+ onDecodeBucketInfoReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::CreateBucketReply& msg) const
+{
+ onEncodeBucketInfoReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeCreateBucketReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::CreateBucketReply::UP msg(new api::CreateBucketReply(
+ static_cast<const api::CreateBucketCommand&>(cmd)));
+ onDecodeBucketInfoReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void
+ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::DeleteBucketCommand& msg) const
+{
+ buf.putLong(msg.getBucketId().getRawId());
+ onEncodeBucketInfoCommand(buf, msg);
+ putBucketInfo(msg.getBucketInfo(), buf);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_0::onDecodeDeleteBucketCommand(BBuf& buf) const
+{
+ document::BucketId bid(SH::getLong(buf));
+ api::DeleteBucketCommand::UP msg(new api::DeleteBucketCommand(bid));
+ onDecodeBucketInfoCommand(buf, *msg);
+ if (buf.getRemaining() >= SH::BUCKET_INFO_SERIALIZED_SIZE) {
+ msg->setBucketInfo(getBucketInfo(buf));
+ }
+ return api::StorageCommand::UP(msg.release());
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::DeleteBucketReply& msg) const
+{
+ onEncodeBucketInfoReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeDeleteBucketReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::DeleteBucketReply::UP msg(new api::DeleteBucketReply(
+ static_cast<const api::DeleteBucketCommand&>(cmd)));
+ onDecodeBucketInfoReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::MergeBucketCommand& msg) const
+{
+ ProtocolSerialization4_2::onEncode(buf, msg);
+
+ buf.putInt(msg.getClusterStateVersion());
+ const std::vector<uint16_t>& chain(msg.getChain());
+ buf.putShort(chain.size());
+ for (std::size_t i = 0; i < chain.size(); ++i) {
+ buf.putShort(chain[i]);
+ }
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_0::onDecodeMergeBucketCommand(BBuf& buf) const
+{
+ api::StorageCommand::UP cmd
+ = ProtocolSerialization4_2::onDecodeMergeBucketCommand(buf);
+ uint32_t clusterStateVersion = SH::getInt(buf);
+ uint16_t chainSize = SH::getShort(buf);
+ std::vector<uint16_t> chain;
+ chain.reserve(chainSize);
+ for (std::size_t i = 0; i < chainSize; ++i) {
+ uint16_t index = SH::getShort(buf);
+ chain.push_back(index);
+ }
+ api::MergeBucketCommand& mergeCmd = static_cast<api::MergeBucketCommand&>(*cmd);
+ mergeCmd.setChain(chain);
+ mergeCmd.setClusterStateVersion(clusterStateVersion);
+ return cmd;
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::MergeBucketReply& msg) const
+{
+ onEncodeBucketReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeMergeBucketReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::MergeBucketReply::UP msg(new api::MergeBucketReply(
+ static_cast<const api::MergeBucketCommand&>(cmd)));
+ onDecodeBucketReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::GetBucketDiffReply& msg) const
+{
+ const std::vector<api::GetBucketDiffCommand::Entry>& entries(msg.getDiff());
+ buf.putInt(entries.size());
+ for (uint32_t i=0; i<entries.size(); ++i) {
+ onEncodeDiffEntry(buf, entries[i]);
+ }
+ onEncodeBucketReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeGetBucketDiffReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::GetBucketDiffReply::UP msg(new api::GetBucketDiffReply(
+ static_cast<const api::GetBucketDiffCommand&>(cmd)));
+ std::vector<api::GetBucketDiffCommand::Entry>& entries(msg->getDiff());
+ uint32_t entryCount = SH::getInt(buf);
+ if (entryCount > buf.getRemaining()) {
+ // Trigger out of bounds exception rather than out of memory error
+ buf.incPos(entryCount);
+ }
+ entries.resize(entryCount);
+ for (uint32_t i=0; i<entries.size(); ++i) {
+ onDecodeDiffEntry(buf, entries[i]);
+ }
+ onDecodeBucketReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::ApplyBucketDiffReply& msg) const
+{
+ const std::vector<api::ApplyBucketDiffCommand::Entry>& entries(
+ msg.getDiff());
+ buf.putInt(entries.size());
+ for (uint32_t i=0; i<entries.size(); ++i) {
+ onEncodeDiffEntry(buf, entries[i]._entry);
+ buf.putString(entries[i]._docName);
+ buf.putInt(entries[i]._headerBlob.size());
+ buf.putBytes(&entries[i]._headerBlob[0],
+ entries[i]._headerBlob.size());
+ buf.putInt(entries[i]._bodyBlob.size());
+ buf.putBytes(&entries[i]._bodyBlob[0],
+ entries[i]._bodyBlob.size());
+ }
+ onEncodeBucketInfoReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeApplyBucketDiffReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::ApplyBucketDiffReply::UP msg(new api::ApplyBucketDiffReply(
+ static_cast<const api::ApplyBucketDiffCommand&>(cmd)));
+ std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg->getDiff());
+ uint32_t entryCount = SH::getInt(buf);
+ if (entryCount > buf.getRemaining()) {
+ // Trigger out of bounds exception rather than out of memory error
+ buf.incPos(entryCount);
+ }
+ entries.resize(entryCount);
+ for (uint32_t i=0; i<entries.size(); ++i) {
+ onDecodeDiffEntry(buf, entries[i]._entry);
+ entries[i]._docName = SH::getString(buf);
+ uint32_t headerSize = SH::getInt(buf);
+ if (headerSize > buf.getRemaining()) {
+ buf.incPos(headerSize);
+ }
+ entries[i]._headerBlob.resize(headerSize);
+ buf.getBytes(&entries[i]._headerBlob[0],
+ entries[i]._headerBlob.size());
+ uint32_t bodySize = SH::getInt(buf);
+ if (bodySize > buf.getRemaining()) {
+ buf.incPos(bodySize);
+ }
+ entries[i]._bodyBlob.resize(bodySize);
+ buf.getBytes(&entries[i]._bodyBlob[0],
+ entries[i]._bodyBlob.size());
+ }
+ onDecodeBucketInfoReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::SplitBucketReply& msg) const
+{
+ const std::vector<api::SplitBucketReply::Entry>& entries(
+ msg.getSplitInfo());
+ buf.putInt(entries.size());
+ for (std::vector<api::SplitBucketReply::Entry>::const_iterator it
+ = entries.begin(); it != entries.end(); ++it)
+ {
+ buf.putLong(it->first.getRawId());
+ putBucketInfo(it->second, buf);
+ }
+ onEncodeBucketReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeSplitBucketReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::SplitBucketReply::UP msg(new api::SplitBucketReply(
+ static_cast<const api::SplitBucketCommand&>(cmd)));
+ std::vector<api::SplitBucketReply::Entry>& entries(msg->getSplitInfo());
+ uint32_t targetCount = SH::getInt(buf);
+ if (targetCount > buf.getRemaining()) {
+ // Trigger out of bounds exception rather than out of memory error
+ buf.incPos(targetCount);
+ }
+ entries.resize(targetCount);
+ for (std::vector<api::SplitBucketReply::Entry>::iterator it
+ = entries.begin(); it != entries.end(); ++it)
+ {
+ it->first = document::BucketId(SH::getLong(buf));
+ it->second = getBucketInfo(buf);
+ }
+ onDecodeBucketReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::MultiOperationReply& msg) const
+{
+ onEncodeBucketInfoReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeMultiOperationReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::MultiOperationReply::UP msg(new api::MultiOperationReply(
+ static_cast<const api::MultiOperationCommand&>(cmd)));
+ onDecodeBucketInfoReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void
+ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::JoinBucketsCommand& msg) const
+{
+ buf.putLong(msg.getBucketId().getRawId());
+ buf.putInt(msg.getSourceBuckets().size());
+ for (uint32_t i=0, n=msg.getSourceBuckets().size(); i<n; ++i) {
+ buf.putLong(msg.getSourceBuckets()[i].getRawId());
+ }
+ buf.putByte(msg.getMinJoinBits());
+ onEncodeCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_0::onDecodeJoinBucketsCommand(BBuf& buf) const
+{
+ document::BucketId bucket(SH::getLong(buf));
+ api::JoinBucketsCommand::UP msg(new api::JoinBucketsCommand(bucket));
+ uint32_t size = SH::getInt(buf);
+ if (size > buf.getRemaining()) {
+ // Trigger out of bounds exception rather than out of memory error
+ buf.incPos(size);
+ }
+ std::vector<document::BucketId>& entries(msg->getSourceBuckets());
+ for (uint32_t i=0; i<size; ++i) {
+ entries.push_back(document::BucketId(SH::getLong(buf)));
+ }
+ msg->setMinJoinBits(SH::getByte(buf));
+ onDecodeCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void
+ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::JoinBucketsReply& msg) const
+{
+ putBucketInfo(msg.getBucketInfo(), buf);
+ onEncodeBucketReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeJoinBucketsReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::JoinBucketsReply::UP msg(new api::JoinBucketsReply(
+ static_cast<const api::JoinBucketsCommand&>(cmd)));
+ msg->setBucketInfo(getBucketInfo(buf));
+ onDecodeBucketReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void
+ProtocolSerialization5_0::onEncodeBucketInfoReply(
+ GBBuf& buf, const api::BucketInfoReply& msg) const
+{
+ onEncodeBucketReply(buf, msg);
+ putBucketInfo(msg.getBucketInfo(), buf);
+}
+
+void
+ProtocolSerialization5_0::onDecodeBucketInfoReply(
+ BBuf& buf, api::BucketInfoReply& msg) const
+{
+ onDecodeBucketReply(buf, msg);
+ msg.setBucketInfo(getBucketInfo(buf));
+}
+
+void
+ProtocolSerialization5_0::onEncodeBucketReply(
+ GBBuf& buf, const api::BucketReply& msg) const
+{
+ onEncodeReply(buf, msg);
+ buf.putLong(msg.hasBeenRemapped() ? msg.getBucketId().getRawId() : 0);
+}
+
+void
+ProtocolSerialization5_0::onDecodeBucketReply(
+ BBuf& buf, api::BucketReply& msg) const
+{
+ onDecodeReply(buf, msg);
+ document::BucketId bucket(SH::getLong(buf));
+ if (bucket.getRawId() != 0) {
+ msg.remapBucketId(bucket);
+ }
+}
+
+void
+ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::CreateVisitorReply& msg) const
+{
+ onEncodeReply(buf, msg);
+ buf.putInt(msg.getVisitorStatistics().getBucketsVisited());
+ buf.putLong(msg.getVisitorStatistics().getDocumentsVisited());
+ buf.putLong(msg.getVisitorStatistics().getBytesVisited());
+ buf.putLong(msg.getVisitorStatistics().getDocumentsReturned());
+ buf.putLong(msg.getVisitorStatistics().getBytesReturned());
+ buf.putLong(msg.getVisitorStatistics().getSecondPassDocumentsReturned());
+ buf.putLong(msg.getVisitorStatistics().getSecondPassBytesReturned());
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::CreateVisitorReply::UP msg(new api::CreateVisitorReply(
+ static_cast<const api::CreateVisitorCommand&>(cmd)));
+ onDecodeReply(buf, *msg);
+
+ vdslib::VisitorStatistics vs;
+ vs.setBucketsVisited(SH::getInt(buf));
+ vs.setDocumentsVisited(SH::getLong(buf));
+ vs.setBytesVisited(SH::getLong(buf));
+ vs.setDocumentsReturned(SH::getLong(buf));
+ vs.setBytesReturned(SH::getLong(buf));
+ vs.setSecondPassDocumentsReturned(SH::getLong(buf));
+ vs.setSecondPassBytesReturned(SH::getLong(buf));
+ msg->setVisitorStatistics(vs);
+
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization5_0::onEncode(
+ GBBuf& buf, const api::RequestBucketInfoCommand& msg) const
+{
+ const std::vector<document::BucketId>& buckets(msg.getBuckets());
+ buf.putInt(buckets.size());
+ for (uint32_t i=0; i<buckets.size(); ++i) {
+ buf.putLong(buckets[i].getRawId());
+ }
+ if (buckets.size() == 0) {
+ buf.putShort(msg.getDistributor());
+ buf.putString(msg.getSystemState().toString());
+ buf.putString(msg.getDistributionHash());
+ }
+
+ onEncodeCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_0::onDecodeRequestBucketInfoCommand(BBuf& buf) const
+{
+ std::vector<document::BucketId> buckets(SH::getInt(buf));
+ for (uint32_t i=0; i<buckets.size(); ++i) {
+ buckets[i] = document::BucketId(SH::getLong(buf));
+ }
+ api::RequestBucketInfoCommand::UP msg;
+ if (buckets.size() != 0) {
+ msg.reset(new api::RequestBucketInfoCommand(buckets));
+ } else {
+ int distributor = SH::getShort(buf);
+ lib::ClusterState state(SH::getString(buf));
+ msg.reset(new api::RequestBucketInfoCommand(distributor, state, SH::getString(buf)));
+ }
+ onDecodeCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void
+ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::CreateVisitorCommand& cmd) const
+{
+ ProtocolSerialization4_2::onEncode(buf, cmd);
+
+ buf.putInt(cmd.getVisitorOrdering());
+ buf.putInt(cmd.getMaxBucketsPerVisitor());
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_0::onDecodeCreateVisitorCommand(BBuf& buf) const
+{
+ api::StorageCommand::UP cvc = ProtocolSerialization4_2::onDecodeCreateVisitorCommand(buf);
+
+ static_cast<api::CreateVisitorCommand*>(cvc.get())->setVisitorOrdering(
+ (document::OrderingSpecification::Order)SH::getInt(buf));
+
+ static_cast<api::CreateVisitorCommand*>(cvc.get())->setMaxBucketsPerVisitor(SH::getInt(buf));
+
+ static_cast<api::CreateVisitorCommand*>(cvc.get())->setVisitorDispatcherVersion(50);
+ return cvc;
+}
+
+} // mbusprot
+} // storage
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h
new file mode 100644
index 00000000000..61de1d97d37
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h
@@ -0,0 +1,80 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/storageapi/mbusprot/protocolserialization4_2.h>
+#include <vespa/documentapi/loadtypes/loadtypeset.h>
+
+namespace storage {
+namespace mbusprot {
+
+class ProtocolSerialization5_0 : public ProtocolSerialization4_2 {
+private:
+ const documentapi::LoadTypeSet& _loadTypes;
+
+public:
+ ProtocolSerialization5_0(const document::DocumentTypeRepo::SP&,
+ const documentapi::LoadTypeSet& loadTypes);
+
+ virtual api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const;
+ virtual void putBucketInfo(const api::BucketInfo& info,
+ vespalib::GrowableByteBuffer& buf) const;
+
+ virtual void onEncode(GBBuf&, const api::PutCommand&) const;
+ virtual void onEncode(GBBuf&, const api::PutReply&) const;
+ virtual void onEncode(GBBuf&, const api::UpdateCommand&) const;
+ virtual void onEncode(GBBuf&, const api::UpdateReply&) const;
+ virtual void onEncode(GBBuf&, const api::GetReply&) const;
+ virtual void onEncode(GBBuf&, const api::RemoveReply&) const;
+ virtual void onEncode(GBBuf&, const api::RevertReply&) const;
+ virtual void onEncode(GBBuf&, const api::CreateBucketReply&) const;
+ virtual void onEncode(GBBuf&, const api::DeleteBucketCommand&) const;
+ virtual void onEncode(GBBuf&, const api::DeleteBucketReply&) const;
+ virtual void onEncode(GBBuf&, const api::MergeBucketCommand&) const;
+ virtual void onEncode(GBBuf&, const api::MergeBucketReply&) const;
+ virtual void onEncode(GBBuf&, const api::GetBucketDiffReply&) const;
+ virtual void onEncode(GBBuf&, const api::ApplyBucketDiffReply&) const;
+ virtual void onEncode(GBBuf&, const api::SplitBucketReply&) const;
+ virtual void onEncode(GBBuf&, const api::MultiOperationReply&) const;
+ virtual void onEncode(GBBuf&, const api::JoinBucketsCommand&) const;
+ virtual void onEncode(GBBuf&, const api::JoinBucketsReply&) const;
+ virtual void onEncode(GBBuf&, const api::RequestBucketInfoCommand&) const;
+
+ virtual void onEncodeBucketInfoReply(GBBuf&, const api::BucketInfoReply&) const;
+ virtual void onEncodeBucketReply(GBBuf&, const api::BucketReply&) const;
+
+ virtual void onEncode(GBBuf&, const api::CreateVisitorCommand& msg) const;
+ virtual void onEncode(GBBuf&, const api::CreateVisitorReply& msg) const;
+ virtual void onEncodeCommand(GBBuf&, const api::StorageCommand&) const;
+ virtual void onEncodeReply(GBBuf&, const api::StorageReply&) const;
+
+ virtual SCmd::UP onDecodePutCommand(BBuf&) const;
+ virtual SRep::UP onDecodePutReply(const SCmd&, BBuf&) const;
+ virtual SCmd::UP onDecodeUpdateCommand(BBuf&) const;
+ virtual SRep::UP onDecodeUpdateReply(const SCmd&, BBuf&) const;
+ virtual SRep::UP onDecodeGetReply(const SCmd&, BBuf&) const;
+ virtual SRep::UP onDecodeRemoveReply(const SCmd&, BBuf&) const;
+ virtual SRep::UP onDecodeRevertReply(const SCmd&, BBuf&) const;
+ virtual SRep::UP onDecodeCreateBucketReply(const SCmd&, BBuf&) const;
+ virtual SCmd::UP onDecodeDeleteBucketCommand(BBuf&) const;
+ virtual SRep::UP onDecodeDeleteBucketReply(const SCmd&, BBuf&) const;
+ virtual SCmd::UP onDecodeMergeBucketCommand(BBuf&) const;
+ virtual SRep::UP onDecodeMergeBucketReply(const SCmd&, BBuf&) const;
+ virtual SRep::UP onDecodeGetBucketDiffReply(const SCmd&, BBuf&) const;
+ virtual SRep::UP onDecodeApplyBucketDiffReply(const SCmd&, BBuf&) const;
+ virtual SRep::UP onDecodeSplitBucketReply(const SCmd&, BBuf&) const;
+ virtual SRep::UP onDecodeMultiOperationReply(const SCmd&, BBuf&) const;
+ virtual SCmd::UP onDecodeJoinBucketsCommand(BBuf& buf) const;
+ virtual SRep::UP onDecodeJoinBucketsReply(const SCmd& cmd, BBuf& buf) const;
+ virtual SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const;
+ virtual SCmd::UP onDecodeRequestBucketInfoCommand(BBuf& buf) const;
+
+ virtual void onDecodeBucketInfoReply(BBuf&, api::BucketInfoReply&) const;
+ virtual void onDecodeBucketReply(BBuf&, api::BucketReply&) const;
+ virtual SRep::UP onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) const;
+ virtual void onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const;
+ virtual void onDecodeReply(BBuf&, api::StorageReply&) const;
+};
+
+} // mbusprot
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp
new file mode 100644
index 00000000000..9244189da1f
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp
@@ -0,0 +1,228 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/mbusprot/protocolserialization5_1.h>
+
+#include <vespa/log/log.h>
+#include <vespa/messagebus/blob.h>
+#include <vespa/messagebus/blobref.h>
+#include <vespa/storageapi/messageapi/storagemessage.h>
+#include <vespa/storageapi/message/bucket.h>
+#include <vespa/storageapi/message/bucketsplitting.h>
+#include <vespa/storageapi/message/persistence.h>
+#include <vespa/storageapi/message/multioperation.h>
+#include <vespa/storageapi/mbusprot/serializationhelper.h>
+#include <vespa/storageapi/mbusprot/storagecommand.h>
+#include <vespa/storageapi/mbusprot/storagereply.h>
+#include <vespa/storageapi/mbusprot/storageprotocol.h>
+#include <vespa/vespalib/util/growablebytebuffer.h>
+#include <vespa/document/select/orderingspecification.h>
+#include <vespa/storageapi/messageapi/returncode.h>
+
+LOG_SETUP(".storage.api.mbusprot.serialization.5_1");
+
+namespace storage {
+namespace mbusprot {
+
+api::BucketInfo
+ProtocolSerialization5_1::getBucketInfo(document::ByteBuffer& buf) const
+{
+ uint64_t lastModified(SH::getLong(buf));
+ uint32_t crc(SH::getInt(buf));
+ uint32_t doccount(SH::getInt(buf));
+ uint32_t docsize(SH::getInt(buf));
+ uint32_t metacount(SH::getInt(buf));
+ uint32_t usedsize(SH::getInt(buf));
+ uint8_t flags(SH::getByte(buf));
+ bool ready = (flags & BUCKET_READY) != 0;
+ bool active = (flags & BUCKET_ACTIVE) != 0;
+ return api::BucketInfo(crc, doccount, docsize,
+ metacount, usedsize,
+ ready, active, lastModified);
+}
+
+void
+ProtocolSerialization5_1::putBucketInfo(
+ const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const
+{
+ buf.putLong(info.getLastModified());
+ buf.putInt(info.getChecksum());
+ buf.putInt(info.getDocumentCount());
+ buf.putInt(info.getTotalDocumentSize());
+ buf.putInt(info.getMetaCount());
+ buf.putInt(info.getUsedFileSize());
+ uint8_t flags = (info.isReady() ? BUCKET_READY : 0) |
+ (info.isActive() ? BUCKET_ACTIVE : 0);
+ buf.putByte(flags);
+}
+
+ProtocolSerialization5_1::ProtocolSerialization5_1(
+ const document::DocumentTypeRepo::SP& repo,
+ const documentapi::LoadTypeSet& loadTypes)
+ : ProtocolSerialization5_0(repo, loadTypes)
+{
+}
+
+void ProtocolSerialization5_1::onEncode(
+ GBBuf& buf, const api::SetBucketStateCommand& msg) const
+{
+ buf.putLong(msg.getBucketId().getRawId());
+ buf.putByte(static_cast<uint8_t>(msg.getState()));
+ onEncodeCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_1::onDecodeSetBucketStateCommand(BBuf& buf) const
+{
+ document::BucketId bucket(SH::getLong(buf));
+ api::SetBucketStateCommand::BUCKET_STATE state(
+ static_cast<api::SetBucketStateCommand::BUCKET_STATE>(
+ SH::getByte(buf)));
+ api::SetBucketStateCommand::UP msg(
+ new api::SetBucketStateCommand(bucket, state));
+ onDecodeCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void ProtocolSerialization5_1::onEncode(
+ GBBuf& buf, const api::SetBucketStateReply& msg) const
+{
+ onEncodeBucketReply(buf, msg);
+}
+
+api::StorageReply::UP
+ProtocolSerialization5_1::onDecodeSetBucketStateReply(const SCmd& cmd,
+ BBuf& buf) const
+{
+ api::SetBucketStateReply::UP msg(new api::SetBucketStateReply(
+ static_cast<const api::SetBucketStateCommand&>(cmd)));
+ onDecodeBucketReply(buf, *msg);
+ return api::StorageReply::UP(msg.release());
+}
+
+void ProtocolSerialization5_1::onEncode(
+ GBBuf& buf, const api::GetCommand& msg) const
+{
+ buf.putString(msg.getDocumentId().toString());
+ buf.putLong(msg.getBucketId().getRawId());
+ buf.putLong(msg.getBeforeTimestamp());
+ buf.putString(msg.getFieldSet());
+ onEncodeCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_1::onDecodeGetCommand(BBuf& buf) const
+{
+ document::DocumentId did(SH::getString(buf));
+ document::BucketId bucket(SH::getLong(buf));
+ api::Timestamp beforeTimestamp(SH::getLong(buf));
+ std::string fieldSet(SH::getString(buf));
+ api::GetCommand::UP msg(
+ new api::GetCommand(bucket, did, fieldSet, beforeTimestamp));
+ onDecodeCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void
+ProtocolSerialization5_1::onEncode(
+ GBBuf& buf, const api::CreateVisitorCommand& msg) const
+{
+ buf.putString(msg.getLibraryName());
+ buf.putString(msg.getInstanceId());
+ buf.putString(msg.getDocumentSelection());
+ buf.putInt(msg.getVisitorCmdId());
+ buf.putString(msg.getControlDestination());
+ buf.putString(msg.getDataDestination());
+ buf.putInt(msg.getMaximumPendingReplyCount());
+ buf.putLong(msg.getFromTime());
+ buf.putLong(msg.getToTime());
+
+ buf.putInt(msg.getBuckets().size());
+ for (uint32_t i = 0; i < msg.getBuckets().size(); i++) {
+ buf.putLong(msg.getBuckets()[i].getRawId());
+ }
+
+ buf.putBoolean(msg.visitRemoves());
+ buf.putString(msg.getFieldSet());
+ buf.putBoolean(msg.visitInconsistentBuckets());
+ buf.putInt(msg.getQueueTimeout());
+
+ uint32_t size = msg.getParameters().getSerializedSize();
+ char* docBuffer = buf.allocate(size);
+ document::ByteBuffer bbuf(docBuffer, size);
+ msg.getParameters().serialize(bbuf);
+
+ onEncodeCommand(buf, msg);
+
+ buf.putInt(msg.getVisitorOrdering());
+ buf.putInt(msg.getMaxBucketsPerVisitor());
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_1::onDecodeCreateVisitorCommand(BBuf& buf) const
+{
+ vespalib::stringref libraryName = SH::getString(buf);
+ vespalib::stringref instanceId = SH::getString(buf);
+ vespalib::stringref selection = SH::getString(buf);
+ api::CreateVisitorCommand::UP msg(
+ new api::CreateVisitorCommand(libraryName, instanceId, selection));
+ msg->setVisitorCmdId(SH::getInt(buf));
+ msg->setControlDestination(SH::getString(buf));
+ msg->setDataDestination(SH::getString(buf));
+ msg->setMaximumPendingReplyCount(SH::getInt(buf));
+
+ msg->setFromTime(SH::getLong(buf));
+ msg->setToTime(SH::getLong(buf));
+ uint32_t count = SH::getInt(buf);
+
+ if (count > buf.getRemaining()) {
+ // Trigger out of bounds exception rather than out of memory error
+ buf.incPos(count);
+ }
+
+ for (uint32_t i = 0; i < count; i++) {
+ msg->getBuckets().push_back(document::BucketId(SH::getLong(buf)));
+ }
+
+ if (SH::getBoolean(buf)) {
+ msg->setVisitRemoves();
+ }
+
+ msg->setFieldSet(SH::getString(buf));
+
+ if (SH::getBoolean(buf)) {
+ msg->setVisitInconsistentBuckets();
+ }
+ msg->setQueueTimeout(SH::getInt(buf));
+ msg->getParameters().deserialize(getTypeRepo(), buf);
+
+ onDecodeCommand(buf, *msg);
+ msg->setVisitorOrdering(
+ (document::OrderingSpecification::Order)SH::getInt(buf));
+ msg->setMaxBucketsPerVisitor(SH::getInt(buf));
+ msg->setVisitorDispatcherVersion(50);
+ return api::StorageCommand::UP(msg.release());
+}
+
+void ProtocolSerialization5_1::onEncode(
+ GBBuf& buf, const api::CreateBucketCommand& msg) const
+{
+ buf.putLong(msg.getBucketId().getRawId());
+ buf.putBoolean(msg.getActive());
+ onEncodeBucketInfoCommand(buf, msg);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_1::onDecodeCreateBucketCommand(BBuf& buf) const
+{
+ document::BucketId bid(SH::getLong(buf));
+ bool setActive = SH::getBoolean(buf);
+ api::CreateBucketCommand::UP msg(new api::CreateBucketCommand(bid));
+ msg->setActive(setActive);
+ onDecodeBucketInfoCommand(buf, *msg);
+ return api::StorageCommand::UP(msg.release());
+}
+
+
+} // mbusprot
+} // storage
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h
new file mode 100644
index 00000000000..491dd5e02eb
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/storageapi/mbusprot/protocolserialization5_0.h>
+#include <vespa/documentapi/loadtypes/loadtypeset.h>
+
+namespace storage {
+namespace mbusprot {
+
+class ProtocolSerialization5_1 : public ProtocolSerialization5_0
+{
+ enum BucketState {
+ BUCKET_READY = 0x1,
+ BUCKET_ACTIVE = 0x2,
+ };
+public:
+ ProtocolSerialization5_1(const document::DocumentTypeRepo::SP&,
+ const documentapi::LoadTypeSet& loadTypes);
+
+ virtual api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const;
+ virtual void putBucketInfo(const api::BucketInfo& info,
+ vespalib::GrowableByteBuffer& buf) const;
+
+protected:
+ virtual void onEncode(GBBuf&, const api::SetBucketStateCommand&) const;
+ virtual void onEncode(GBBuf&, const api::SetBucketStateReply&) const;
+ virtual void onEncode(GBBuf&, const api::GetCommand&) const;
+ virtual void onEncode(GBBuf&, const api::CreateVisitorCommand&) const;
+ virtual void onEncode(GBBuf&, const api::CreateBucketCommand&) const;
+
+ virtual SCmd::UP onDecodeSetBucketStateCommand(BBuf&) const;
+ virtual SRep::UP onDecodeSetBucketStateReply(const SCmd&, BBuf&) const;
+ virtual SCmd::UP onDecodeGetCommand(BBuf&) const;
+ virtual SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const;
+ virtual SCmd::UP onDecodeCreateBucketCommand(BBuf&) const;
+};
+
+} // mbusprot
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp
new file mode 100644
index 00000000000..68b8449cf4c
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// @author Vegard Sjonfjell
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/mbusprot/protocolserialization5_2.h>
+#include <vespa/storageapi/mbusprot/storagecommand.h>
+#include <vespa/storageapi/mbusprot/serializationhelper.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP(".storage.api.mbusprot.serialization.5_2");
+
+namespace storage {
+namespace mbusprot {
+
+using documentapi::TestAndSetCondition;
+
+void ProtocolSerialization5_2::onEncode(GBBuf & buf, const api::PutCommand & cmd) const
+{
+ ProtocolSerialization5_0::onEncode(buf, cmd);
+ encodeTasCondition(buf, cmd);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_2::onDecodePutCommand(BBuf & buf) const
+{
+ auto cmd = ProtocolSerialization5_0::onDecodePutCommand(buf);
+ decodeTasCondition(*cmd, buf);
+ return cmd;
+}
+
+void ProtocolSerialization5_2::onEncode(GBBuf & buf, const api::RemoveCommand & cmd) const
+{
+ ProtocolSerialization4_2::onEncode(buf, cmd);
+ encodeTasCondition(buf, cmd);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_2::onDecodeRemoveCommand(BBuf & buf) const
+{
+ auto cmd = ProtocolSerialization4_2::onDecodeRemoveCommand(buf);
+ decodeTasCondition(*cmd, buf);
+ return cmd;
+}
+
+void ProtocolSerialization5_2::onEncode(GBBuf & buf, const api::UpdateCommand & cmd) const
+{
+ ProtocolSerialization5_0::onEncode(buf, cmd);
+ encodeTasCondition(buf, cmd);
+}
+
+api::StorageCommand::UP
+ProtocolSerialization5_2::onDecodeUpdateCommand(BBuf & buf) const
+{
+ auto cmd = ProtocolSerialization5_0::onDecodeUpdateCommand(buf);
+ decodeTasCondition(*cmd, buf);
+ return cmd;
+}
+
+void ProtocolSerialization5_2::decodeTasCondition(api::StorageCommand & storageCmd, BBuf & buf) {
+ auto & cmd = static_cast<api::TestAndSetCommand &>(storageCmd);
+ cmd.setCondition(TestAndSetCondition(SH::getString(buf)));
+}
+
+void ProtocolSerialization5_2::encodeTasCondition(GBBuf & buf, const api::StorageCommand & storageCmd) {
+ auto & cmd = static_cast<const api::TestAndSetCommand &>(storageCmd);
+ buf.putString(cmd.getCondition().getSelection());
+}
+
+} // mbusprot
+} // storage
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h
new file mode 100644
index 00000000000..35af05d192b
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// @author Vegard Sjonfjell
+
+#pragma once
+
+#include <vespa/vespalib/util/growablebytebuffer.h>
+#include <vespa/documentapi/loadtypes/loadtypeset.h>
+#include <vespa/storageapi/mbusprot/protocolserialization5_1.h>
+#include <vespa/storageapi/message/persistence.h>
+
+namespace storage {
+namespace mbusprot {
+
+class ProtocolSerialization5_2 : public ProtocolSerialization5_1
+{
+public:
+ ProtocolSerialization5_2(
+ const document::DocumentTypeRepo::SP& repo,
+ const documentapi::LoadTypeSet & loadTypes)
+ : ProtocolSerialization5_1(repo, loadTypes)
+ {}
+
+protected:
+ virtual void onEncode(GBBuf &, const api::PutCommand &) const override;
+ virtual void onEncode(GBBuf &, const api::RemoveCommand &) const override;
+ virtual void onEncode(GBBuf &, const api::UpdateCommand &) const override;
+
+ virtual SCmd::UP onDecodePutCommand(BBuf &) const override;
+ virtual SCmd::UP onDecodeRemoveCommand(BBuf &) const override;
+ virtual SCmd::UP onDecodeUpdateCommand(BBuf &) const override;
+
+ static void decodeTasCondition(api::StorageCommand & cmd, BBuf & buf);
+ static void encodeTasCondition(GBBuf & buf, const api::StorageCommand & cmd);
+};
+
+} // mbusprot
+} // storage
diff --git a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h
new file mode 100644
index 00000000000..dea5ca70e9a
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h
@@ -0,0 +1,151 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/fastos/types.h>
+#include <vespa/document/base/globalid.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/util/growablebytebuffer.h>
+
+namespace storage {
+namespace mbusprot {
+
+class SerializationHelper
+{
+public:
+ static int64_t getLong(document::ByteBuffer& buf) {
+ int64_t tmp;
+ buf.getLongNetwork(tmp);
+ return tmp;
+ }
+
+ static int32_t getInt(document::ByteBuffer& buf) {
+ int32_t tmp;
+ buf.getIntNetwork(tmp);
+ return tmp;
+ }
+
+ static int16_t getShort(document::ByteBuffer& buf) {
+ int16_t tmp;
+ buf.getShortNetwork(tmp);
+ return tmp;
+ }
+
+ static uint8_t getByte(document::ByteBuffer& buf) {
+ uint8_t tmp;
+ buf.getByte(tmp);
+ return tmp;
+ }
+
+ static vespalib::stringref getString(document::ByteBuffer& buf) {
+ uint32_t tmp;
+ buf.getIntNetwork((int32_t&) tmp);
+ const char * p = buf.getBufferAtPos();
+ buf.incPos(tmp);
+ vespalib::stringref s(p, tmp);
+ return s;
+ }
+
+ static bool getBoolean(document::ByteBuffer& buf) {
+ uint8_t tmp;
+ buf.getByte(tmp);
+ return (tmp == 1);
+ }
+
+ static api::ReturnCode getReturnCode(document::ByteBuffer& buf) {
+ api::ReturnCode::Result result = (api::ReturnCode::Result) getInt(buf);
+ vespalib::stringref message = getString(buf);
+ return api::ReturnCode(result, message);
+ }
+
+ static void putReturnCode(const api::ReturnCode& code,
+ vespalib::GrowableByteBuffer& buf)
+ {
+ buf.putInt(code.getResult());
+ buf.putString(code.getMessage());
+ }
+
+ static const uint32_t BUCKET_INFO_SERIALIZED_SIZE = sizeof(uint32_t) * 3;
+
+ static document::GlobalId getGlobalId(document::ByteBuffer& buf) {
+ std::vector<char> buffer(getShort(buf));
+ for (uint32_t i=0; i<buffer.size(); ++i) {
+ buffer[i] = getByte(buf);
+ }
+ return document::GlobalId(&buffer[0]);
+ }
+
+ static void putGlobalId(const document::GlobalId& gid,
+ vespalib::GrowableByteBuffer& buf)
+ {
+ buf.putShort(document::GlobalId::LENGTH);
+ for (uint32_t i=0; i<document::GlobalId::LENGTH; ++i) {
+ buf.putByte(gid.get()[i]);
+ }
+ }
+
+ static document::Document::UP getDocument(
+ document::ByteBuffer& buf,
+ const document::DocumentTypeRepo& repo)
+ {
+ uint32_t size = getInt(buf);
+ if (size == 0) {
+ return document::Document::UP();
+ } else {
+ document::ByteBuffer bbuf(buf.getBufferAtPos(), size);
+ buf.incPos(size);
+ return document::Document::UP(new document::Document(repo, bbuf));
+ }
+ }
+
+ static document::DocumentUpdate::UP getUpdate(
+ document::ByteBuffer& buf,
+ const document::DocumentTypeRepo& repo)
+ {
+ uint32_t size = getInt(buf);
+ if (size == 0) {
+ return document::DocumentUpdate::UP();
+ } else {
+ document::ByteBuffer bbuf(buf.getBufferAtPos(), size);
+ buf.incPos(size);
+ return document::DocumentUpdate::UP(
+ new document::DocumentUpdate(repo, bbuf,
+ document::DocumentUpdate::
+ SerializeVersion::
+ SERIALIZE_42));
+ }
+ }
+
+ static void putDocument(document::Document* doc,
+ vespalib::GrowableByteBuffer& buf)
+ {
+ if (doc) {
+ vespalib::nbostream stream;
+ doc->serialize(stream);
+ buf.putInt(stream.size());
+ buf.putBytes(stream.peek(), stream.size());
+ } else {
+ buf.putInt(0);
+ }
+ }
+
+ static void putUpdate(document::DocumentUpdate* update,
+ vespalib::GrowableByteBuffer& buf)
+ {
+ if (update) {
+ vespalib::nbostream stream;
+ update->serialize42(stream);
+ buf.putInt(stream.size());
+ buf.putBytes(stream.peek(), stream.size());
+ } else {
+ buf.putInt(0);
+ }
+ }
+
+};
+
+} // mbusprot
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp
new file mode 100644
index 00000000000..04d92b8fa9e
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/mbusprot/storagecommand.h>
+
+namespace storage {
+namespace mbusprot {
+
+StorageCommand::StorageCommand(const storage::api::StorageCommand::SP& cmd)
+ : mbus::Message(),
+ _cmd(cmd)
+{
+}
+
+} // mbusprot
+} // storage
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h b/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h
new file mode 100644
index 00000000000..11ef23a1254
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/messagebus/message.h>
+#include <vespa/storageapi/mbusprot/storagemessage.h>
+#include <vespa/storageapi/mbusprot/storageprotocol.h>
+#include <vespa/storageapi/messageapi/storagecommand.h>
+
+namespace storage {
+namespace mbusprot {
+
+class StorageCommand : public mbus::Message, public StorageMessage {
+public:
+ typedef std::unique_ptr<StorageCommand> UP;
+
+ StorageCommand(const storage::api::StorageCommand::SP&);
+
+ const mbus::string & getProtocol() const { return StorageProtocol::NAME; }
+
+ uint32_t getType() const { return _cmd->getType().getId(); }
+
+ const api::StorageCommand::SP& getCommand() { return _cmd; }
+ api::StorageCommand::CSP getCommand() const { return _cmd; }
+ virtual api::StorageMessage::SP getInternalMessage() { return _cmd; }
+ virtual api::StorageMessage::CSP getInternalMessage() const { return _cmd; }
+
+ virtual uint8_t priority() const {
+ return ((getInternalMessage()->getPriority()) / 255) * 16;
+ }
+
+private:
+ api::StorageCommand::SP _cmd;
+};
+
+} // mbusprot
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagemessage.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagemessage.cpp
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/storagemessage.cpp
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagemessage.h b/storageapi/src/vespa/storageapi/mbusprot/storagemessage.h
new file mode 100644
index 00000000000..8424712b7c9
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/storagemessage.h
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/storageapi/messageapi/storagemessage.h>
+
+namespace storage {
+namespace mbusprot {
+
+class StorageMessage {
+public:
+ typedef std::unique_ptr<StorageMessage> UP;
+
+ virtual ~StorageMessage() {}
+
+ virtual api::StorageMessage::SP getInternalMessage() = 0;
+ virtual storage::api::StorageMessage::CSP getInternalMessage() const = 0;
+
+};
+
+} // protocol
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp
new file mode 100644
index 00000000000..7e1e3815dc5
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp
@@ -0,0 +1,173 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/mbusprot/storageprotocol.h>
+
+#include <vespa/log/log.h>
+#include <vespa/storageapi/mbusprot/protocolserialization.h>
+#include <vespa/storageapi/mbusprot/serializationhelper.h>
+#include <vespa/storageapi/mbusprot/storagecommand.h>
+#include <vespa/storageapi/mbusprot/storagereply.h>
+#include <vespa/vespalib/util/exceptions.h>
+
+LOG_SETUP(".storage.api.mbusprot.protocol");
+
+namespace storage {
+namespace mbusprot {
+
+mbus::string StorageProtocol::NAME = "StorageProtocol";
+
+StorageProtocol::StorageProtocol(const document::DocumentTypeRepo::SP repo,
+ const documentapi::LoadTypeSet& loadTypes)
+ : _serializer5_0(repo, loadTypes),
+ _serializer5_1(repo, loadTypes),
+ _serializer5_2(repo, loadTypes)
+{
+}
+
+mbus::IRoutingPolicy::UP
+StorageProtocol::createPolicy(const mbus::string&, const mbus::string&) const
+{
+ return mbus::IRoutingPolicy::UP();
+}
+
+namespace {
+ vespalib::Version version5_2(5, 93, 30);
+ vespalib::Version version5_1(5, 1, 0);
+ vespalib::Version version5_0(5, 0, 12);
+ vespalib::Version version5_0beta(4, 3, 0);
+}
+
+static mbus::Blob
+encodeMessage(const ProtocolSerialization & serializer,
+ const mbus::Routable & routable,
+ const StorageMessage & message,
+ const vespalib::Version & serializerVersion,
+ const vespalib::Version & actualVersion)
+{
+ mbus::Blob blob(serializer.encode(*message.getInternalMessage()));
+
+ if (LOG_WOULD_LOG(spam)) {
+ std::ostringstream messageStream;
+ document::StringUtil::printAsHex(messageStream, blob.data(), blob.size());
+
+ LOG(spam, "Encoded message of protocol %s type %s using "
+ "%s serialization as version is %s:\n%s",
+ routable.getProtocol().c_str(),
+ message.getInternalMessage()->getType().toString().c_str(),
+ serializerVersion.toString().c_str(),
+ actualVersion.toString().c_str(),
+ messageStream.str().c_str());
+ }
+
+ return blob;
+}
+
+
+mbus::Blob
+StorageProtocol::encode(const vespalib::Version& version,
+ const mbus::Routable& routable) const
+{
+ const StorageMessage & message(dynamic_cast<const StorageMessage &>(routable));
+
+ try {
+ if (message.getInternalMessage().get() == 0) {
+ throw vespalib::IllegalArgumentException(
+ "Given storage message wrapper does not contain a "
+ "storage message.",
+ VESPA_STRLOC);
+ }
+
+ if (version < version5_1) {
+ if (version < version5_0beta) {
+ LOGBP(warning,
+ "No support for using messagebus for version %s."
+ "Minimum version is %s. Thus we cannot serialize %s.",
+ version.toString().c_str(),
+ version5_0beta.toString().c_str(),
+ message.getInternalMessage()->toString().c_str());
+
+ return mbus::Blob(0);
+ } else {
+ return encodeMessage(_serializer5_0, routable, message, version5_0, version);
+ }
+ } else if (version < version5_2) {
+ return encodeMessage(_serializer5_1, routable, message, version5_1, version);
+ } else {
+ return encodeMessage(_serializer5_2, routable, message, version5_2, version);
+ }
+
+ } catch (std::exception & e) {
+ LOGBP(warning, "Failed to encode %s storage protocol message %s: %s",
+ version.toString().c_str(),
+ message.getInternalMessage()->toString().c_str(),
+ e.what());
+ }
+
+ return mbus::Blob(0);
+}
+
+static mbus::Routable::UP
+decodeMessage(const ProtocolSerialization & serializer,
+ mbus::BlobRef data,
+ const api::MessageType & type,
+ const vespalib::Version & serializerVersion,
+ const vespalib::Version & actualVersion)
+{
+ if (LOG_WOULD_LOG(spam)) {
+ std::ostringstream messageStream;
+ document::StringUtil::printAsHex(messageStream, data.data(), data.size());
+
+ LOG(spam,
+ "Decoding %s of version %s "
+ "using %s decoder from:\n%s",
+ type.toString().c_str(),
+ actualVersion.toString().c_str(),
+ serializerVersion.toString().c_str(),
+ messageStream.str().c_str());
+ }
+
+ if (type.isReply()) {
+ return std::make_unique<StorageReply>(data, serializer);
+ } else {
+ return mbus::Routable::UP(serializer.decodeCommand(data).release());
+ }
+}
+
+mbus::Routable::UP
+StorageProtocol::decode(const vespalib::Version & version,
+ mbus::BlobRef data) const
+{
+ try {
+ document::ByteBuffer buf(data.data(), data.size());
+ auto & type = api::MessageType::get(
+ static_cast<api::MessageType::Id>(SerializationHelper::getInt(buf)));
+
+ StorageMessage::UP message;
+ if (version < version5_1) {
+ if (version < version5_0beta) {
+ LOGBP(error,
+ "No support for using messagebus for version %s."
+ "Minimum version is %s.",
+ version.toString().c_str(),
+ version5_0beta.toString().c_str());
+ } else {
+ return decodeMessage(_serializer5_0, data, type, version5_0, version);
+ }
+ } else if (version < version5_2) {
+ return decodeMessage(_serializer5_1, data, type, version5_1, version);
+ } else {
+ return decodeMessage(_serializer5_2, data, type, version5_2, version);
+ }
+ } catch (std::exception & e) {
+ std::ostringstream ost;
+ ost << "Failed to decode " << version.toString() << " messagebus "
+ << "storage protocol message: " << e.what() << "\n";
+ document::StringUtil::printAsHex(ost, data.data(), data.size());
+ LOGBP(warning, "%s", ost.str().c_str());
+ }
+
+ return mbus::Routable::UP();
+}
+
+} // mbusprot
+} // storage
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h
new file mode 100644
index 00000000000..c490a17555a
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/messagebus/iprotocol.h>
+#include <string>
+#include <vespa/storageapi/mbusprot/protocolserialization5_0.h>
+#include <vespa/storageapi/mbusprot/protocolserialization5_1.h>
+#include <vespa/storageapi/mbusprot/protocolserialization5_2.h>
+#include <vespa/documentapi/loadtypes/loadtypeset.h>
+
+namespace storage {
+namespace mbusprot {
+
+class StorageProtocol : public mbus::IProtocol
+{
+public:
+ typedef std::shared_ptr<StorageProtocol> SP;
+
+ static mbus::string NAME;
+
+ StorageProtocol(const document::DocumentTypeRepo::SP,
+ const documentapi::LoadTypeSet& loadTypes);
+
+ // Implements IProtocol.
+ const mbus::string& getName() const { return NAME; }
+
+ // Implements IProtocol.
+ mbus::IRoutingPolicy::UP createPolicy(const mbus::string& name,
+ const mbus::string& param) const;
+
+ // Implements IProtocol.
+ mbus::Blob encode(const vespalib::Version&, const mbus::Routable&) const;
+
+ // Implements IProtocol.
+ mbus::Routable::UP decode(const vespalib::Version&, mbus::BlobRef) const;
+
+private:
+ ProtocolSerialization5_0 _serializer5_0;
+ ProtocolSerialization5_1 _serializer5_1;
+ ProtocolSerialization5_2 _serializer5_2;
+};
+
+} // mbusprot
+} // storage
+
+
+
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp
new file mode 100644
index 00000000000..b9fde51eaef
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/mbusprot/storagereply.h>
+
+#include <vespa/storageapi/mbusprot/storagecommand.h>
+
+namespace storage {
+namespace mbusprot {
+
+StorageReply::StorageReply(const mbus::BlobRef& data,
+ const ProtocolSerialization& serializer)
+ : _serializer(&serializer),
+ _buffer(data.size()),
+ _mbusType(0),
+ _reply()
+{
+ memcpy(_buffer.get(), data.data(), _buffer.size());
+ document::ByteBuffer buf(data.data(), data.size());
+ buf.getIntNetwork(reinterpret_cast<int32_t&>(_mbusType));
+}
+
+StorageReply::StorageReply(const api::StorageReply::SP& reply)
+ : _serializer(0),
+ _buffer(),
+ _mbusType(reply->getType().getId()),
+ _reply(reply)
+{
+}
+
+StorageReply::~StorageReply()
+{
+}
+
+void
+StorageReply::deserialize() const
+{
+ if (_reply.get()) return;
+ StorageReply& reply(const_cast<StorageReply&>(*this));
+ mbus::Message::UP msg(reply.getMessage());
+ if (msg.get() == 0) {
+ throw vespalib::IllegalStateException(
+ "Cannot deserialize storage reply before message have been set",
+ VESPA_STRLOC);
+ }
+ const StorageCommand* cmd(dynamic_cast<const StorageCommand*>(msg.get()));
+ reply.setMessage(std::move(msg));
+ if (cmd == 0) {
+ throw vespalib::IllegalStateException(
+ "Storage reply get message did not return a storage command",
+ VESPA_STRLOC);
+ }
+ mbus::BlobRef blobRef(static_cast<char *>(_buffer.get()), _buffer.size());
+ _reply = _serializer->decodeReply(blobRef, *cmd->getCommand())->getReply();
+ vespalib::DefaultAlloc().swap(_buffer);
+}
+
+} // mbusprot
+} // storage
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagereply.h b/storageapi/src/vespa/storageapi/mbusprot/storagereply.h
new file mode 100644
index 00000000000..52f7efc50cc
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/mbusprot/storagereply.h
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/messagebus/reply.h>
+#include <vespa/storageapi/mbusprot/storagemessage.h>
+#include <vespa/storageapi/mbusprot/storageprotocol.h>
+#include <vespa/storageapi/messageapi/storagereply.h>
+
+namespace storage {
+namespace mbusprot {
+
+class StorageReply : public mbus::Reply, public StorageMessage {
+ const ProtocolSerialization* _serializer;
+ mutable vespalib::DefaultAlloc _buffer;
+ uint32_t _mbusType;
+ mutable api::StorageReply::SP _reply;
+
+public:
+ typedef std::unique_ptr<StorageReply> UP;
+
+ StorageReply(const mbus::BlobRef& data, const ProtocolSerialization&);
+ StorageReply(const api::StorageReply::SP& reply);
+ virtual ~StorageReply();
+
+ virtual const mbus::string& getProtocol() const
+ { return StorageProtocol::NAME; }
+
+ uint32_t getType() const { return _mbusType; }
+
+ const api::StorageReply::SP& getReply() { deserialize(); return _reply; }
+ api::StorageReply::CSP getReply() const { deserialize(); return _reply; }
+ virtual api::StorageMessage::SP getInternalMessage()
+ { deserialize(); return _reply; }
+ virtual api::StorageMessage::CSP getInternalMessage() const
+ { deserialize(); return _reply; }
+
+ virtual uint8_t priority() const {
+ if (_reply.get()) {
+ return _reply->getPriority();
+ }
+ return 0;
+ }
+
+private:
+
+ void deserialize() const;
+};
+
+} // mbusprot
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/message/.gitignore b/storageapi/src/vespa/storageapi/message/.gitignore
new file mode 100644
index 00000000000..526f91c6668
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/.gitignore
@@ -0,0 +1,7 @@
+*.lo
+.*.swp
+.depend
+.depend.NEW
+.deps
+.libs
+Makefile
diff --git a/storageapi/src/vespa/storageapi/message/CMakeLists.txt b/storageapi/src/vespa/storageapi/message/CMakeLists.txt
new file mode 100644
index 00000000000..a263f495a1d
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/CMakeLists.txt
@@ -0,0 +1,18 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(storageapi_message OBJECT
+ SOURCES
+ datagram.cpp
+ persistence.cpp
+ bucket.cpp
+ visitor.cpp
+ state.cpp
+ searchresult.cpp
+ bucketsplitting.cpp
+ multioperation.cpp
+ documentsummary.cpp
+ stat.cpp
+ removelocation.cpp
+ queryresult.cpp
+ batch.cpp
+ DEPENDS
+)
diff --git a/storageapi/src/vespa/storageapi/message/batch.cpp b/storageapi/src/vespa/storageapi/message/batch.cpp
new file mode 100644
index 00000000000..6da213efaf5
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/batch.cpp
@@ -0,0 +1,171 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <algorithm>
+#include <vespa/storageapi/message/batch.h>
+#include <vespa/document/bucket/bucketidfactory.h>
+
+using namespace storage::api;
+
+IMPLEMENT_COMMAND(BatchPutRemoveCommand, BatchPutRemoveReply)
+IMPLEMENT_REPLY(BatchPutRemoveReply)
+IMPLEMENT_COMMAND(BatchDocumentUpdateCommand, BatchDocumentUpdateReply)
+IMPLEMENT_REPLY(BatchDocumentUpdateReply)
+
+
+BatchPutRemoveCommand::Operation::Operation(uint64_t ts, Type tp)
+ : timestamp(ts),
+ type(tp)
+{
+}
+
+BatchPutRemoveCommand::PutOperation::PutOperation(document::Document::SP doc, uint64_t ts)
+ : Operation(ts, PUT),
+ document(doc)
+{
+}
+
+BatchPutRemoveCommand::HeaderUpdateOperation::HeaderUpdateOperation(document::Document::SP doc, uint64_t newTimestamp, uint64_t timestampToUpdate_)
+ : Operation(newTimestamp, HEADERUPDATE),
+ document(doc),
+ timestampToUpdate(timestampToUpdate_)
+{
+}
+
+BatchPutRemoveCommand::RemoveOperation::RemoveOperation(const document::DocumentId& docId, uint64_t ts)
+ : Operation(ts, REMOVE),
+ documentId(docId)
+{
+}
+
+BatchPutRemoveCommand::BatchPutRemoveCommand(const document::BucketId& bucketId)
+ : BucketInfoCommand(MessageType::BATCHPUTREMOVE, bucketId),
+ _approxSize(0)
+{
+}
+
+void
+BatchPutRemoveCommand::addPut(document::Document::SP document, uint64_t ts)
+{
+ _operations.push_back(vespalib::LinkedPtr<Operation>(new PutOperation(document, ts)));
+ _approxSize += document->serialize()->getLength();
+}
+
+void
+BatchPutRemoveCommand::addHeaderUpdate(document::Document::SP document, uint64_t ts, uint64_t timestampToUpdate)
+{
+ _operations.push_back(vespalib::LinkedPtr<Operation>(new HeaderUpdateOperation(document, ts, timestampToUpdate)));
+ _approxSize += document->serialize()->getLength();
+}
+
+void
+BatchPutRemoveCommand::addRemove(const document::DocumentId& docId, uint64_t ts)
+{
+ _operations.push_back(vespalib::LinkedPtr<Operation>(new RemoveOperation(docId, ts)));
+ _approxSize += docId.toString().length();
+}
+
+void
+BatchPutRemoveCommand::addOperation(const Operation& op, bool cloneDocument)
+{
+ switch (op.type) {
+ case Operation::PUT:
+ {
+ document::Document::SP doc;
+ if (!cloneDocument) {
+ doc = static_cast<const PutOperation&>(op).document;
+ } else {
+ doc.reset(static_cast<const PutOperation&>(op).document->clone());
+ }
+ addPut(doc, op.timestamp);
+ break;
+ }
+ case Operation::REMOVE:
+ addRemove(static_cast<const RemoveOperation&>(op).documentId, op.timestamp);
+ break;
+ case Operation::HEADERUPDATE:
+ {
+ const HeaderUpdateOperation& hup = static_cast<const HeaderUpdateOperation&>(op);
+ document::Document::SP doc;
+ if (!cloneDocument) {
+ doc = hup.document;
+ } else {
+ doc.reset(hup.document->clone());
+ }
+ addHeaderUpdate(doc, op.timestamp, hup.timestampToUpdate);
+ break;
+ }
+ }
+}
+
+void
+BatchPutRemoveCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const {
+ out << "BatchPutRemove(" << getBucketId() << ", " << _operations.size() << " operations)";
+
+ if (verbose) {
+ out << " : ";
+ BucketInfoCommand::print(out, verbose, indent);
+ }
+}
+
+BatchPutRemoveReply::BatchPutRemoveReply(const BatchPutRemoveCommand& cmd)
+ : BucketInfoReply(cmd)
+{
+}
+
+void
+BatchPutRemoveReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const {
+ out << "BatchPutRemoveReply(";
+ out << _documentsNotFound.size() << " documents not found)";
+
+ if (verbose) {
+ out << " {";
+ for (std::vector<document::DocumentId>::const_iterator it =
+ _documentsNotFound.begin();
+ it != _documentsNotFound.end(); ++it)
+ {
+ out << "\n" << indent << " " << (*it);
+ }
+ out << "\n" << indent << "} : ";
+ BucketInfoReply::print(out, verbose, indent);
+ }
+}
+
+BatchDocumentUpdateCommand::BatchDocumentUpdateCommand(const UpdateList& updates)
+ : StorageCommand(MessageType::BATCHDOCUMENTUPDATE),
+ _updates(updates)
+{
+ document::BucketIdFactory factory;
+ _bucketId = factory.getBucketId(updates[0]->getId());
+}
+
+void
+BatchDocumentUpdateCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const {
+ out << "BatchDocumentUpdate(" << _updates.size() << " operations)";
+
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+BatchDocumentUpdateReply::BatchDocumentUpdateReply(const BatchDocumentUpdateCommand& cmd)
+ : StorageReply(cmd),
+ _documentsNotFound()
+{
+}
+
+void
+BatchDocumentUpdateReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const {
+ out << "BatchDocumentUpdateReply("
+ << std::count(_documentsNotFound.begin(), _documentsNotFound.end(), true)
+ << " not found)";
+
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
diff --git a/storageapi/src/vespa/storageapi/message/batch.h b/storageapi/src/vespa/storageapi/message/batch.h
new file mode 100644
index 00000000000..a1b92fa1f0e
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/batch.h
@@ -0,0 +1,204 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/update/documentupdate.h>
+#include <vespa/storageapi/messageapi/bucketinfocommand.h>
+#include <vespa/storageapi/messageapi/bucketinforeply.h>
+
+namespace storage {
+namespace api {
+
+/**
+ * @class BatchPutRemoveCommand
+ * @ingroup message
+ *
+ * @brief Sends a batch of puts and removes
+ */
+class BatchPutRemoveCommand : public BucketInfoCommand {
+public:
+ class Operation {
+ public:
+ enum Type {
+ REMOVE, // Removes a document
+ HEADERUPDATE, // Updates the header of a document, if it already exists.
+ PUT // Inserts a new document.
+ };
+
+ Operation(uint64_t ts, Type type);
+ virtual ~Operation() {};
+
+ uint64_t timestamp;
+ Type type;
+
+ virtual const document::DocumentId& getDocumentId() const = 0;
+ };
+
+ explicit BatchPutRemoveCommand(const document::BucketId& id);
+
+ class PutOperation : public Operation {
+ public:
+ PutOperation(document::Document::SP document, uint64_t timestamp);
+
+ document::Document::SP document;
+
+ const document::DocumentId& getDocumentId() const {
+ return document->getId();
+ }
+ };
+
+ class HeaderUpdateOperation : public Operation {
+ public:
+ HeaderUpdateOperation(document::Document::SP document, uint64_t newTimestamp, uint64_t timestampToUpdate);
+
+ document::Document::SP document;
+ uint64_t timestampToUpdate;
+
+ const document::DocumentId& getDocumentId() const {
+ return document->getId();
+ }
+ };
+
+ class RemoveOperation : public Operation {
+ public:
+ RemoveOperation(const document::DocumentId& docId, uint64_t timestamp);
+
+ document::DocumentId documentId;
+
+ const document::DocumentId& getDocumentId() const {
+ return documentId;
+ }
+ };
+
+ /**
+ Adds a PUT operation to be performed.
+ */
+ void addPut(document::Document::SP document, uint64_t timestamp);
+
+ /**
+ Adds a PUT operation to be performed.
+ */
+ void addHeaderUpdate(document::Document::SP document, uint64_t newTimestamp, uint64_t timestampToUpdate);
+
+ /**
+ Adds a REMOVE operation to be performed.
+ */
+ void addRemove(const document::DocumentId& docId, uint64_t timestamp);
+
+ /**
+ * Adds an operation to be performed. Optionally deep-clones the
+ * operation's document.
+ */
+ void addOperation(const Operation& op, bool cloneDocument = false);
+
+ /**
+ Returns the number of operations in this batch.
+ */
+ uint32_t getOperationCount() const { return _operations.size(); }
+
+ /**
+ Returns the nth operation in this batch.
+ */
+ const Operation& getOperation(uint32_t index) const { return *_operations[index]; }
+
+ /**
+ Returns the nth operation in this batch.
+ */
+ Operation& getOperation(uint32_t index) { return *_operations[index]; }
+
+ /**
+ Returns an approximate size of this message.
+ */
+ uint32_t getMemoryFootprint() const { return _approxSize + 20; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(BatchPutRemoveCommand, onBatchPutRemove)
+
+private:
+ std::vector<vespalib::LinkedPtr<Operation> > _operations;
+ uint32_t _approxSize;
+};
+
+/**
+ * @class BatchPutRemoveReply
+ * @ingroup message
+ *
+ * @brief Confirm that a given docoperations have been received.
+ */
+class BatchPutRemoveReply : public BucketInfoReply {
+private:
+ std::vector<document::DocumentId> _documentsNotFound;
+
+public:
+ explicit BatchPutRemoveReply(const BatchPutRemoveCommand&);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ const std::vector<document::DocumentId>& getDocumentsNotFound() const { return _documentsNotFound; }
+ std::vector<document::DocumentId>& getDocumentsNotFound() { return _documentsNotFound; }
+
+ DECLARE_STORAGEREPLY(BatchPutRemoveReply, onBatchPutRemoveReply)
+};
+
+class BatchDocumentUpdateCommand : public StorageCommand
+{
+public:
+ typedef std::vector<document::DocumentUpdate::SP > UpdateList;
+
+ /**
+ Creates a batch update message containing the given updates.
+ */
+ BatchDocumentUpdateCommand(const UpdateList& updates);
+
+ /**
+ @return Returns a list of the updates to be performed.
+ */
+ const UpdateList& getUpdates() const { return _updates; };
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ /**
+ Returns a bucket id suitable for routing this message.
+ */
+ document::BucketId getBucketId() const { return _bucketId; }
+ virtual bool hasSingleBucketId() const { return true; }
+
+ DECLARE_STORAGECOMMAND(BatchDocumentUpdateCommand, onBatchDocumentUpdate)
+
+private:
+ UpdateList _updates;
+ document::BucketId _bucketId;
+};
+
+/**
+ * @class BatchDocumentUpdateReply
+ * @ingroup message
+ *
+ * @brief Confirm that a given docoperations have been received.
+ */
+class BatchDocumentUpdateReply : public StorageReply {
+ // 1-1 mapping of found/not found state for documents
+ std::vector<bool> _documentsNotFound;
+public:
+ explicit BatchDocumentUpdateReply(const BatchDocumentUpdateCommand&);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ const std::vector<bool>& getDocumentsNotFound() const { return _documentsNotFound; }
+ std::vector<bool>& getDocumentsNotFound() { return _documentsNotFound; }
+
+ DECLARE_STORAGEREPLY(BatchDocumentUpdateReply, onBatchDocumentUpdateReply)
+};
+
+
+
+}
+}
+
+
diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp
new file mode 100644
index 00000000000..e2450ca17ca
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/bucket.cpp
@@ -0,0 +1,615 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/message/bucket.h>
+
+#include <vespa/document/fieldvalue/document.h>
+
+namespace storage {
+namespace api {
+
+IMPLEMENT_COMMAND(CreateBucketCommand, CreateBucketReply)
+IMPLEMENT_REPLY(CreateBucketReply)
+IMPLEMENT_COMMAND(DeleteBucketCommand, DeleteBucketReply)
+IMPLEMENT_REPLY(DeleteBucketReply)
+IMPLEMENT_COMMAND(MergeBucketCommand, MergeBucketReply)
+IMPLEMENT_REPLY(MergeBucketReply)
+IMPLEMENT_COMMAND(GetBucketDiffCommand, GetBucketDiffReply)
+IMPLEMENT_REPLY(GetBucketDiffReply)
+IMPLEMENT_COMMAND(ApplyBucketDiffCommand, ApplyBucketDiffReply)
+IMPLEMENT_REPLY(ApplyBucketDiffReply)
+IMPLEMENT_COMMAND(RequestBucketInfoCommand, RequestBucketInfoReply)
+IMPLEMENT_REPLY(RequestBucketInfoReply)
+IMPLEMENT_COMMAND(NotifyBucketChangeCommand, NotifyBucketChangeReply)
+IMPLEMENT_REPLY(NotifyBucketChangeReply)
+IMPLEMENT_COMMAND(SetBucketStateCommand, SetBucketStateReply)
+IMPLEMENT_REPLY(SetBucketStateReply)
+
+CreateBucketCommand::CreateBucketCommand(const document::BucketId& id)
+ : MaintenanceCommand(MessageType::CREATEBUCKET, id),
+ _active(false)
+{
+}
+
+void
+CreateBucketCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "CreateBucketCommand(" << getBucketId();
+ if (_active) {
+ out << ", active";
+ } else {
+ out << ", inactive";
+ }
+ out << ")";
+ out << " Reasons to start: " << _reason;
+ if (verbose) {
+ out << " : ";
+ MaintenanceCommand::print(out, verbose, indent);
+ }
+}
+
+CreateBucketReply::CreateBucketReply(const CreateBucketCommand& cmd)
+ : BucketInfoReply(cmd)
+{
+}
+
+void
+CreateBucketReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "CreateBucketReply(" << getBucketId() << ")";
+ if (verbose) {
+ out << " : ";
+ BucketInfoReply::print(out, verbose, indent);
+ }
+}
+
+DeleteBucketCommand::DeleteBucketCommand(const document::BucketId& id)
+ : MaintenanceCommand(MessageType::DELETEBUCKET, id)
+{
+}
+
+void
+DeleteBucketCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "DeleteBucketCommand(" << getBucketId() << ")";
+ out << " Reasons to start: " << _reason;
+ if (verbose) {
+ out << " : ";
+ MaintenanceCommand::print(out, verbose, indent);
+ }
+}
+
+DeleteBucketReply::DeleteBucketReply(const DeleteBucketCommand& cmd)
+ : BucketInfoReply(cmd)
+{
+}
+
+void
+DeleteBucketReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "DeleteBucketReply(" << getBucketId() << ")";
+ if (verbose) {
+ out << " : ";
+ BucketInfoReply::print(out, verbose, indent);
+ }
+}
+
+MergeBucketCommand::MergeBucketCommand(
+ const document::BucketId& id, const std::vector<Node>& nodes,
+ Timestamp maxTimestamp, uint32_t clusterStateVersion,
+ const std::vector<uint16_t>& chain)
+ : MaintenanceCommand(MessageType::MERGEBUCKET, id),
+ _nodes(nodes),
+ _maxTimestamp(maxTimestamp),
+ _clusterStateVersion(clusterStateVersion),
+ _chain(chain)
+{
+}
+
+void
+MergeBucketCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "MergeBucketCommand(" << getBucketId() << ", to time "
+ << _maxTimestamp << ", cluster state version: "
+ << _clusterStateVersion << ", nodes: [";
+ for (uint32_t i=0; i<_nodes.size(); ++i) {
+ if (i != 0) out << ", ";
+ out << _nodes[i];
+ }
+ out << "], chain: [";
+ for (uint32_t i = 0; i < _chain.size(); ++i) {
+ if (i != 0) out << ", ";
+ out << _chain[i];
+ }
+ out << "]";
+ out << ", reasons to start: " << _reason;
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ BucketCommand::print(out, verbose, indent);
+ }
+}
+
+MergeBucketReply::MergeBucketReply(const MergeBucketCommand& cmd)
+ : BucketReply(cmd),
+ _nodes(cmd.getNodes()),
+ _maxTimestamp(cmd.getMaxTimestamp()),
+ _clusterStateVersion(cmd.getClusterStateVersion()),
+ _chain(cmd.getChain())
+{
+}
+
+void
+MergeBucketReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "MergeBucketReply(" << getBucketId() << ", to time "
+ << _maxTimestamp << ", cluster state version: "
+ << _clusterStateVersion << ", nodes: ";
+ for (uint32_t i=0; i<_nodes.size(); ++i) {
+ if (i != 0) out << ", ";
+ out << _nodes[i];
+ }
+ out << "], chain: [";
+ for (uint32_t i = 0; i < _chain.size(); ++i) {
+ if (i != 0) out << ", ";
+ out << _chain[i];
+ }
+ out << "])";
+ if (verbose) {
+ out << " : ";
+ BucketReply::print(out, verbose, indent);
+ }
+}
+
+GetBucketDiffCommand::Entry::Entry()
+ : _timestamp(0),
+ _gid(),
+ _headerSize(0),
+ _bodySize(0),
+ _flags(0),
+ _hasMask(0)
+{
+}
+
+void GetBucketDiffCommand::Entry::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "Entry(timestamp: " << _timestamp
+ << ", " << _gid << ", hasMask: 0x" << _hasMask;
+ if (verbose) {
+ out << ",\n" << indent << " " << "header size: "
+ << std::dec << _headerSize << ", body size: " << _bodySize
+ << ", flags 0x" << std::hex << _flags << std::dec;
+ }
+ out << ")";
+}
+
+bool GetBucketDiffCommand::Entry::operator==(const Entry& e) const
+{
+ return (_timestamp == e._timestamp &&
+ _headerSize == e._headerSize &&
+ _bodySize == e._bodySize &&
+ _gid == e._gid &&
+ _flags == e._flags);
+}
+
+GetBucketDiffCommand::GetBucketDiffCommand(
+ const document::BucketId& id, const std::vector<Node>& nodes,
+ Timestamp maxTimestamp)
+ : BucketCommand(MessageType::GETBUCKETDIFF, id),
+ _nodes(nodes),
+ _maxTimestamp(maxTimestamp)
+{
+}
+
+void
+GetBucketDiffCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "GetBucketDiffCommand(" << getBucketId() << ", to time "
+ << _maxTimestamp << ", nodes: ";
+ for (uint32_t i=0; i<_nodes.size(); ++i) {
+ if (i != 0) out << ", ";
+ out << _nodes[i];
+ }
+ if (_diff.empty()) {
+ out << ", no entries";
+ } else if (verbose) {
+ out << ",";
+ for (uint32_t i=0; i<_diff.size(); ++i) {
+ out << "\n" << indent << " ";
+ _diff[i].print(out, verbose, indent + " ");
+ }
+ } else {
+ out << ", " << _diff.size() << " entries";
+ out << ", id " << _msgId;
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ BucketCommand::print(out, verbose, indent);
+ }
+}
+
+GetBucketDiffReply::GetBucketDiffReply(const GetBucketDiffCommand& cmd)
+ : BucketReply(cmd),
+ _nodes(cmd.getNodes()),
+ _maxTimestamp(cmd.getMaxTimestamp()),
+ _diff(cmd.getDiff())
+{
+}
+
+void
+GetBucketDiffReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "GetBucketDiffReply(" << getBucketId() << ", to time "
+ << _maxTimestamp << ", nodes: ";
+ for (uint32_t i=0; i<_nodes.size(); ++i) {
+ if (i != 0) out << ", ";
+ out << _nodes[i];
+ }
+ if (_diff.empty()) {
+ out << ", no entries";
+ } else if (verbose) {
+ out << ",";
+ for (uint32_t i=0; i<_diff.size(); ++i) {
+ out << "\n" << indent << " ";
+ _diff[i].print(out, verbose, indent + " ");
+ }
+ } else {
+ out << ", " << _diff.size() << " entries";
+ out << ", id " << _msgId;
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ BucketReply::print(out, verbose, indent);
+ }
+}
+
+ApplyBucketDiffCommand::Entry::Entry()
+ : _entry(),
+ _docName(),
+ _headerBlob(),
+ _bodyBlob(),
+ _repo()
+{
+}
+
+ApplyBucketDiffCommand::Entry::Entry(const GetBucketDiffCommand::Entry& e)
+ : _entry(e),
+ _docName(),
+ _headerBlob(),
+ _bodyBlob(),
+ _repo()
+{
+}
+
+bool ApplyBucketDiffCommand::Entry::filled() const
+{
+ return ((_headerBlob.size() > 0 ||
+ (_entry._headerSize == 0 && !_docName.empty())) &&
+ (_bodyBlob.size() > 0 ||
+ _entry._bodySize == 0));
+}
+
+void ApplyBucketDiffCommand::Entry::print(
+ std::ostream& out, bool verbose, const std::string& indent) const
+{
+ out << "ApplyEntry(";
+ _entry.print(out, verbose, indent + " ");
+ out << ",\n" << indent << " name(" << _docName
+ << "), headerBlob(" << _headerBlob.size()
+ << "), bodyBlob(" << _bodyBlob.size() << ")";
+ if (_headerBlob.size() > 0) {
+ document::ByteBuffer buf(&_headerBlob[0],
+ _headerBlob.size());
+ if (_repo) {
+ document::Document doc(*_repo, buf);
+ out << ",\n" << indent << " " << doc.getId().getGlobalId();
+ } else {
+ out << ",\n" << indent << " unknown global id. (repo missing)";
+ }
+ }
+ out << ")";
+}
+
+bool ApplyBucketDiffCommand::Entry::operator==(const Entry& e) const
+{
+ return (_entry == e._entry &&
+ _headerBlob == e._headerBlob &&
+ _bodyBlob == e._bodyBlob);
+}
+
+ApplyBucketDiffCommand::ApplyBucketDiffCommand(
+ const document::BucketId& id, const std::vector<Node>& nodes,
+ uint32_t maxBufferSize)
+ : BucketInfoCommand(MessageType::APPLYBUCKETDIFF, id),
+ _nodes(nodes),
+ _diff(),
+ _maxBufferSize(maxBufferSize)
+{
+}
+
+void
+ApplyBucketDiffCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ uint32_t totalSize = 0;
+ uint32_t filled = 0;
+ for (std::vector<Entry>::const_iterator it = _diff.begin();
+ it != _diff.end(); ++it)
+ {
+ totalSize += it->_headerBlob.size();
+ totalSize += it->_bodyBlob.size();
+ if (it->filled()) ++filled;
+ }
+ out << "ApplyBucketDiffCommand(" << getBucketId() << ", nodes: ";
+ for (uint32_t i=0; i<_nodes.size(); ++i) {
+ if (i != 0) out << ", ";
+ out << _nodes[i];
+ }
+ out << ", max buffer size " << _maxBufferSize << " bytes"
+ << ", " << _diff.size() << " entries of " << totalSize << " bytes, "
+ << (100.0 * filled / _diff.size()) << " \% filled)";
+ if (_diff.empty()) {
+ out << ", no entries";
+ } else if (verbose) {
+ out << ",";
+ for (uint32_t i=0; i<_diff.size(); ++i) {
+ out << "\n" << indent << " ";
+ _diff[i].print(out, verbose, indent + " ");
+ }
+ } else {
+ out << ", " << _diff.size() << " entries";
+ out << ", id " << _msgId;
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ BucketCommand::print(out, verbose, indent);
+ }
+}
+
+ApplyBucketDiffReply::ApplyBucketDiffReply(const ApplyBucketDiffCommand& cmd)
+ : BucketInfoReply(cmd),
+ _nodes(cmd.getNodes()),
+ _diff(cmd.getDiff()),
+ _maxBufferSize(cmd.getMaxBufferSize())
+{
+}
+
+void
+ApplyBucketDiffReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ uint32_t totalSize = 0;
+ uint32_t filled = 0;
+ for (std::vector<Entry>::const_iterator it = _diff.begin();
+ it != _diff.end(); ++it)
+ {
+ totalSize += it->_headerBlob.size();
+ totalSize += it->_bodyBlob.size();
+ if (it->filled()) ++filled;
+ }
+ out << "ApplyBucketDiffReply(" << getBucketId() << ", nodes: ";
+ for (uint32_t i=0; i<_nodes.size(); ++i) {
+ if (i != 0) out << ", ";
+ out << _nodes[i];
+ }
+ out << ", max buffer size " << _maxBufferSize << " bytes"
+ << ", " << _diff.size() << " entries of " << totalSize << " bytes, "
+ << (100.0 * filled / _diff.size()) << " \% filled)";
+ if (_diff.empty()) {
+ out << ", no entries";
+ } else if (verbose) {
+ out << ",";
+ for (uint32_t i=0; i<_diff.size(); ++i) {
+ out << "\n" << indent << " ";
+ _diff[i].print(out, verbose, indent + " ");
+ }
+ } else {
+ out << ", " << _diff.size() << " entries";
+ out << ", id " << _msgId;
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ BucketInfoReply::print(out, verbose, indent);
+ }
+}
+
+RequestBucketInfoCommand::RequestBucketInfoCommand(
+ const std::vector<document::BucketId>& buckets)
+ : StorageCommand(MessageType::REQUESTBUCKETINFO),
+ _buckets(buckets),
+ _state(),
+ _distributor(0xFFFF)
+{
+}
+
+RequestBucketInfoCommand::RequestBucketInfoCommand(
+ uint16_t distributor, const lib::ClusterState& state,
+ const vespalib::stringref & distributionHash)
+ : StorageCommand(MessageType::REQUESTBUCKETINFO),
+ _buckets(),
+ _state(new lib::ClusterState(state)),
+ _distributor(distributor),
+ _distributionHash(distributionHash)
+{
+}
+
+RequestBucketInfoCommand::RequestBucketInfoCommand(
+ uint16_t distributor, const lib::ClusterState& state)
+ : StorageCommand(MessageType::REQUESTBUCKETINFO),
+ _buckets(),
+ _state(new lib::ClusterState(state)),
+ _distributor(distributor),
+ _distributionHash("")
+{
+}
+
+void
+RequestBucketInfoCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "RequestBucketInfoCommand(";
+ if (_buckets.size() != 0) {
+ out << _buckets.size() << " buckets";
+ }
+ if (hasSystemState()) {
+ out << "distributor " << _distributor << " in ";
+ _state->print(out, verbose, indent + " ");
+ }
+ if (verbose && _buckets.size() > 0) {
+ out << "\n" << indent << " Specified buckets:\n" << indent << " ";
+ std::copy(_buckets.begin(), _buckets.end(),
+ std::ostream_iterator<document::BucketId>(
+ out, ("\n" + indent + " ").c_str()));
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+std::ostream& operator<<(std::ostream& out,
+ const RequestBucketInfoReply::Entry& e)
+{
+ return out << e._bucketId << " - " << e._info;
+}
+
+
+RequestBucketInfoReply::RequestBucketInfoReply(
+ const RequestBucketInfoCommand& cmd)
+ : StorageReply(cmd),
+ _buckets()
+{
+}
+
+uint32_t
+RequestBucketInfoReply::getMemoryFootprint() const
+{
+ return sizeof(Entry) * _buckets.capacity();
+}
+
+void
+RequestBucketInfoReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "RequestBucketInfoReply(" << _buckets.size();
+ if (verbose) {
+ out << "\n" << indent << " ";
+ std::copy(_buckets.begin(), _buckets.end(),
+ std::ostream_iterator<Entry>(out,
+ ("\n" + indent + " ").c_str()));
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+NotifyBucketChangeCommand::NotifyBucketChangeCommand(
+ const document::BucketId& id, const BucketInfo& info)
+ : BucketCommand(MessageType::NOTIFYBUCKETCHANGE, id),
+ _info(info)
+{
+}
+
+void
+NotifyBucketChangeCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "NotifyBucketChangeCommand(" << getBucketId() << ", ";
+ _info.print(out, verbose, indent);
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+NotifyBucketChangeReply::NotifyBucketChangeReply(
+ const NotifyBucketChangeCommand& cmd)
+ : BucketReply(cmd)
+{
+}
+
+void
+NotifyBucketChangeReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "NotifyBucketChangeReply(" << getBucketId() << ")";
+ if (verbose) {
+ out << " : ";
+ BucketReply::print(out, verbose, indent);
+ }
+}
+
+SetBucketStateCommand::SetBucketStateCommand(
+ const document::BucketId& bucket,
+ BUCKET_STATE state)
+ : MaintenanceCommand(MessageType::SETBUCKETSTATE, bucket),
+ _state(state)
+{
+}
+
+void
+SetBucketStateCommand::print(std::ostream& out,
+ bool verbose,
+ const std::string& indent) const
+{
+ out << "SetBucketStateCommand(" << getBucketId() << ", ";
+ switch (_state) {
+ case INACTIVE:
+ out << "INACTIVE";
+ break;
+ case ACTIVE:
+ out << "ACTIVE";
+ break;
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ MaintenanceCommand::print(out, verbose, indent);
+ }
+}
+
+vespalib::string
+SetBucketStateCommand::getSummary() const
+{
+ vespalib::asciistream stream;
+ stream << "SetBucketStateCommand(" << getBucketId().toString() << ", "
+ << ((_state == ACTIVE) ? "ACTIVE" : "INACTIVE") << ")";
+ return stream.str();
+}
+
+SetBucketStateReply::SetBucketStateReply(
+ const SetBucketStateCommand& cmd)
+ : BucketInfoReply(cmd)
+{
+}
+
+void
+SetBucketStateReply::print(std::ostream& out,
+ bool verbose,
+ const std::string& indent) const
+{
+ out << "SetBucketStateReply(" << getBucketId() << ")";
+ if (verbose) {
+ out << " : ";
+ BucketInfoReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storageapi/src/vespa/storageapi/message/bucket.h
new file mode 100644
index 00000000000..28b3ba36f30
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/bucket.h
@@ -0,0 +1,528 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @file bucketinfo.h
+ *
+ * Bucket related commands.
+ */
+
+#pragma once
+
+#include <vespa/document/base/globalid.h>
+#include <vespa/vdslib/state/clusterstate.h>
+#include <vespa/storageapi/defs.h>
+#include <vespa/storageapi/messageapi/bucketcommand.h>
+#include <vespa/storageapi/messageapi/bucketreply.h>
+#include <vespa/storageapi/messageapi/bucketinfocommand.h>
+#include <vespa/storageapi/messageapi/bucketinforeply.h>
+#include <vespa/storageapi/messageapi/maintenancecommand.h>
+
+namespace document { class DocumentTypeRepo; }
+
+namespace storage {
+namespace api {
+
+/**
+ * @class CreateBucketCommand
+ * @ingroup message
+ *
+ * @brief Command for creating a new bucket on a storage node.
+ */
+class CreateBucketCommand : public MaintenanceCommand {
+ bool _active;
+
+public:
+ explicit CreateBucketCommand(const document::BucketId& id);
+
+ void setActive(bool active) { _active = active; }
+ bool getActive() const { return _active; }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(CreateBucketCommand, onCreateBucket)
+};
+
+/**
+ * @class CreateBucketReply
+ * @ingroup message
+ *
+ * @brief Reply of a create bucket command.
+ */
+class CreateBucketReply : public BucketInfoReply {
+public:
+ explicit CreateBucketReply(const CreateBucketCommand& cmd);
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(CreateBucketReply, onCreateBucketReply);
+};
+
+/**
+ * @class DeleteBucketCommand
+ * @ingroup message
+ *
+ * @brief Command for deleting a bucket from one or more storage nodes.
+ */
+class DeleteBucketCommand : public MaintenanceCommand {
+ BucketInfo _info;
+public:
+ explicit DeleteBucketCommand(const document::BucketId& id);
+
+ const BucketInfo& getBucketInfo() const { return _info; }
+ void setBucketInfo(const BucketInfo& info) { _info = info; }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(DeleteBucketCommand, onDeleteBucket)
+};
+
+/**
+ * @class DeleteBucketReply
+ * @ingroup message
+ *
+ * @brief Reply of a delete bucket command.
+ */
+class DeleteBucketReply : public BucketInfoReply {
+public:
+ explicit DeleteBucketReply(const DeleteBucketCommand& cmd);
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(DeleteBucketReply, onDeleteBucketReply)
+};
+
+
+/**
+ * @class MergeBucketCommand
+ * @ingroup message
+ *
+ * @brief Merge a bucket
+ *
+ * Merges given bucket copies, held on the given node list. A maximum timestamp
+ * should be given, such that the buckets may be used during merge. If not
+ * given, storage will set current time for it, but distributors should really
+ * set it, as they have the reference clock for a bucket.
+ *
+ * An optional "only for source" node list can be provided. In this case, the
+ * nodes in that list are only used for sources in the merge, and never as
+ * targets, even if they are missing documents from the other nodes.
+ *
+ */
+class MergeBucketCommand : public MaintenanceCommand {
+public:
+ struct Node {
+ uint16_t index;
+ bool sourceOnly;
+
+ Node(uint16_t index_, bool sourceOnly_ = false)
+ : index(index_), sourceOnly(sourceOnly_) {}
+
+ bool operator==(const Node& n) const
+ { return (index == n.index && sourceOnly == n.sourceOnly); }
+ };
+
+private:
+ std::vector<Node> _nodes;
+ Timestamp _maxTimestamp;
+ uint32_t _clusterStateVersion;
+ std::vector<uint16_t> _chain;
+
+public:
+ MergeBucketCommand(const document::BucketId&,
+ const std::vector<Node>&,
+ Timestamp maxTimestamp,
+ uint32_t clusterStateVersion = 0,
+ const std::vector<uint16_t>& chain = std::vector<uint16_t>());
+
+ const std::vector<Node>& getNodes() const { return _nodes; }
+ Timestamp getMaxTimestamp() const { return _maxTimestamp; }
+ const std::vector<uint16_t>& getChain() const { return _chain; }
+
+ uint32_t getClusterStateVersion() const { return _clusterStateVersion; }
+
+ void setClusterStateVersion(uint32_t version) { _clusterStateVersion = version; }
+ void setChain(const std::vector<uint16_t>& chain) { _chain = chain; }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(MergeBucketCommand, onMergeBucket)
+};
+
+inline std::ostream&
+operator<<(std::ostream& out, const MergeBucketCommand::Node& n) {
+ return out << n.index << (n.sourceOnly ? " (source only)" : "");
+}
+
+/**
+ * @class MergeBucketReply
+ * @ingroup message
+ *
+ * @brief Reply of a merge bucket command.
+ */
+class MergeBucketReply : public BucketReply {
+public:
+ typedef MergeBucketCommand::Node Node;
+
+private:
+ std::vector<Node> _nodes;
+ Timestamp _maxTimestamp;
+ uint32_t _clusterStateVersion;
+ std::vector<uint16_t> _chain;
+
+public:
+ explicit MergeBucketReply(const MergeBucketCommand& cmd);
+
+ const std::vector<Node>& getNodes() const { return _nodes; }
+ Timestamp getMaxTimestamp() const { return _maxTimestamp; }
+ const std::vector<uint16_t>& getChain() const { return _chain; }
+ uint32_t getClusterStateVersion() const { return _clusterStateVersion; }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(MergeBucketReply, onMergeBucketReply)
+};
+
+/**
+ * @class GetBucketDiff
+ * @ingroup message
+ *
+ * @brief Message sent between storage nodes as the first step of merge.
+ */
+class GetBucketDiffCommand : public BucketCommand {
+public:
+ typedef MergeBucketCommand::Node Node;
+
+ struct Entry : public document::Printable {
+ Timestamp _timestamp;
+ document::GlobalId _gid;
+ uint32_t _headerSize;
+ uint32_t _bodySize;
+ uint16_t _flags;
+ uint16_t _hasMask;
+
+ Entry();
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+ bool operator==(const Entry&) const;
+ bool operator<(const Entry& e) const
+ { return (_timestamp < e._timestamp); }
+ };
+private:
+ std::vector<Node> _nodes;
+ Timestamp _maxTimestamp;
+ std::vector<Entry> _diff;
+
+public:
+ GetBucketDiffCommand(const document::BucketId&,
+ const std::vector<Node>&,
+ Timestamp maxTimestamp);
+
+ const std::vector<Node>& getNodes() const { return _nodes; }
+ Timestamp getMaxTimestamp() const { return _maxTimestamp; }
+ const std::vector<Entry>& getDiff() const { return _diff; }
+ std::vector<Entry>& getDiff() { return _diff; }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(GetBucketDiffCommand, onGetBucketDiff)
+};
+
+/**
+ * @class GetBucketDiffReply
+ * @ingroup message
+ *
+ * @brief Reply of GetBucketDiffCommand
+ */
+class GetBucketDiffReply : public BucketReply {
+public:
+ typedef MergeBucketCommand::Node Node;
+ typedef GetBucketDiffCommand::Entry Entry;
+
+private:
+ std::vector<Node> _nodes;
+ Timestamp _maxTimestamp;
+ std::vector<Entry> _diff;
+
+public:
+ explicit GetBucketDiffReply(const GetBucketDiffCommand& cmd);
+
+ const std::vector<Node>& getNodes() const { return _nodes; }
+ Timestamp getMaxTimestamp() const { return _maxTimestamp; }
+ const std::vector<Entry>& getDiff() const { return _diff; }
+ std::vector<Entry>& getDiff() { return _diff; }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(GetBucketDiffReply, onGetBucketDiffReply)
+};
+
+/**
+ * @class ApplyBucketDiff
+ * @ingroup message
+ *
+ * @brief Sends a chunk of document entries, which the bucket copies can use
+ * to update themselves.
+ */
+class ApplyBucketDiffCommand : public BucketInfoCommand {
+public:
+ typedef MergeBucketCommand::Node Node;
+ struct Entry : public document::Printable {
+ GetBucketDiffCommand::Entry _entry;
+ vespalib::string _docName;
+ std::vector<char> _headerBlob;
+ std::vector<char> _bodyBlob;
+ const document::DocumentTypeRepo *_repo;
+
+ Entry();
+ Entry(const GetBucketDiffCommand::Entry&);
+
+ bool filled() const;
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+ bool operator==(const Entry&) const;
+ };
+private:
+ std::vector<Node> _nodes;
+ std::vector<Entry> _diff;
+ // We may send more metadata entries that should fit in one apply bucket
+ // diff command, to let node pick which ones it wants to fill in first.
+ // Nodes should verify that they don't fill a command up with more than
+ // this number of bytes.
+ uint32_t _maxBufferSize;
+
+public:
+ ApplyBucketDiffCommand(const document::BucketId& id,
+ const std::vector<Node>& nodes,
+ uint32_t maxBufferSize);
+
+ const std::vector<Node>& getNodes() const { return _nodes; }
+ const std::vector<Entry>& getDiff() const { return _diff; }
+ std::vector<Entry>& getDiff() { return _diff; }
+ uint32_t getMaxBufferSize() const { return _maxBufferSize; }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(ApplyBucketDiffCommand, onApplyBucketDiff)
+};
+
+/**
+ * @class ApplyBucketDiffReply
+ * @ingroup message
+ *
+ * @brief Reply of ApplyBucketDiffCommand
+ */
+class ApplyBucketDiffReply : public BucketInfoReply {
+public:
+ typedef MergeBucketCommand::Node Node;
+ typedef ApplyBucketDiffCommand::Entry Entry;
+
+private:
+ std::vector<Node> _nodes;
+ std::vector<Entry> _diff;
+ uint32_t _maxBufferSize;
+
+public:
+ explicit ApplyBucketDiffReply(const ApplyBucketDiffCommand& cmd);
+
+ const std::vector<Node>& getNodes() const { return _nodes; }
+ const std::vector<Entry>& getDiff() const { return _diff; }
+ std::vector<Entry>& getDiff() { return _diff; }
+ uint32_t getMaxBufferSize() const { return _maxBufferSize; }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(ApplyBucketDiffReply, onApplyBucketDiffReply)
+};
+
+/**
+ * @class RequestBucketInfoCommand
+ * @ingroup message
+ *
+ * @brief Command for getting bucket info.
+ *
+ * Used to get checksums of buckets from a storage node.
+ * If list of buckets for which to retrieve info is given. If it is empty,
+ * it means all buckets.
+ * A system state and a distributor index may be given. If given, only info for
+ * the buckets that belong to the given distributor should be returned.
+ */
+class RequestBucketInfoCommand : public StorageCommand {
+ std::vector<document::BucketId> _buckets;
+ std::unique_ptr<lib::ClusterState> _state;
+ uint16_t _distributor;
+ vespalib::string _distributionHash;
+
+public:
+ explicit RequestBucketInfoCommand(
+ const std::vector<document::BucketId>& buckets);
+ RequestBucketInfoCommand(uint16_t distributor,
+ const lib::ClusterState& state,
+ const vespalib::stringref & _distributionHash);
+
+ RequestBucketInfoCommand(uint16_t distributor,
+ const lib::ClusterState& state);
+
+ const std::vector<document::BucketId>& getBuckets() const
+ { return _buckets; }
+
+ bool hasSystemState() const { return (_state.get() != 0); }
+ uint16_t getDistributor() const { return _distributor; }
+ const lib::ClusterState& getSystemState() const
+ { return *_state; }
+
+ const vespalib::string& getDistributionHash() const { return _distributionHash; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(RequestBucketInfoCommand, onRequestBucketInfo)
+};
+
+
+/**
+ * @class RequestBucketInfoReply
+ * @ingroup message
+ *
+ * @brief Answer of a bucket info command.
+ */
+class RequestBucketInfoReply : public StorageReply {
+public:
+ struct Entry {
+ document::BucketId _bucketId;
+ BucketInfo _info;
+
+ bool operator==(const Entry& e) const
+ { return (_bucketId == e._bucketId && _info == e._info); }
+ Entry() : _bucketId(), _info() {}
+ Entry(const document::BucketId& id, const BucketInfo& info)
+ : _bucketId(id), _info(info) {}
+ friend std::ostream& operator<<(std::ostream& os, const Entry&);
+ };
+ typedef vespalib::Array<Entry, vespalib::DefaultAlloc> EntryVector;
+private:
+ EntryVector _buckets;
+
+public:
+
+ explicit RequestBucketInfoReply(const RequestBucketInfoCommand& cmd);
+
+ const EntryVector & getBucketInfo() const { return _buckets; }
+ EntryVector & getBucketInfo() { return _buckets; }
+
+ uint32_t getMemoryFootprint() const;
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(RequestBucketInfoReply, onRequestBucketInfoReply)
+};
+
+/**
+ * @class NotifyBucketChangeCommand
+ * @ingroup message
+ *
+ * @brief Command for letting others know a bucket have been altered.
+ *
+ * When the persistence layer notices a bucket has been corrupted, such that
+ * it needs to be repaired, this message will be sent to notify others
+ * of change. Others being bucket database on storage node, and possibly
+ * distributor.
+ */
+class NotifyBucketChangeCommand : public BucketCommand {
+ BucketInfo _info;
+
+public:
+ NotifyBucketChangeCommand(const document::BucketId& bucket,
+ const BucketInfo& bucketInfo);
+
+ const BucketInfo& getBucketInfo() const { return _info; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(NotifyBucketChangeCommand, onNotifyBucketChange)
+};
+
+
+/**
+ * @class NotifyBucketChangeReply
+ * @ingroup message
+ *
+ * @brief Answer of notify bucket command.
+ *
+ * Noone will resend these messages, and they're not needed, but all commands
+ * need to have a reply.
+ */
+class NotifyBucketChangeReply : public BucketReply {
+public:
+ explicit NotifyBucketChangeReply(const NotifyBucketChangeCommand& cmd);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(NotifyBucketChangeReply, onNotifyBucketChangeReply)
+};
+
+/**
+ * @class SetBucketStateCommand
+ * @ingroup message
+ *
+ * @brief Sent by distributor to set the ready/active state of a bucket.
+ */
+class SetBucketStateCommand : public MaintenanceCommand
+{
+public:
+ enum BUCKET_STATE
+ {
+ INACTIVE,
+ ACTIVE
+ };
+private:
+ BUCKET_STATE _state;
+public:
+ SetBucketStateCommand(const document::BucketId& bucket,
+ BUCKET_STATE state);
+
+ void print(std::ostream& out,
+ bool verbose,
+ const std::string& indent) const;
+
+ BUCKET_STATE getState() const { return _state; }
+
+ DECLARE_STORAGECOMMAND(SetBucketStateCommand, onSetBucketState)
+private:
+ virtual vespalib::string getSummary() const;
+};
+
+/**
+ * @class SetBucketStateReply
+ * @ingroup message
+ *
+ * @brief Answer to SetBucketStateCommand.
+ */
+class SetBucketStateReply : public BucketInfoReply
+{
+public:
+ explicit SetBucketStateReply(const SetBucketStateCommand&);
+
+ void print(std::ostream& out,
+ bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(SetBucketStateReply, onSetBucketStateReply)
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/message/bucketsplitting.cpp b/storageapi/src/vespa/storageapi/message/bucketsplitting.cpp
new file mode 100644
index 00000000000..aa692ae2d57
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/bucketsplitting.cpp
@@ -0,0 +1,135 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/message/bucketsplitting.h>
+
+namespace storage {
+namespace api {
+
+IMPLEMENT_COMMAND(SplitBucketCommand, SplitBucketReply)
+IMPLEMENT_REPLY(SplitBucketReply)
+IMPLEMENT_COMMAND(JoinBucketsCommand, JoinBucketsReply)
+IMPLEMENT_REPLY(JoinBucketsReply)
+
+SplitBucketCommand::SplitBucketCommand(const document::BucketId& id)
+ : MaintenanceCommand(MessageType::SPLITBUCKET, id),
+ _minSplitBits(0),
+ _maxSplitBits(58),
+ _minByteSize(std::numeric_limits<uint32_t>::max()),
+ _minDocCount(std::numeric_limits<uint32_t>::max())
+{
+ // By default, set very large sizes, to ensure we trigger 'already big
+ // enough' behaviour, only splitting one step by default. The distributor
+ // should always overwrite one of these values to get correct behaviour.
+}
+
+void
+SplitBucketCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "SplitBucketCommand(" << getBucketId();
+ if (_minDocCount != std::numeric_limits<uint32_t>::max()
+ || _minByteSize != std::numeric_limits<uint32_t>::max())
+ {
+ out << "Max doc count: " << _minDocCount
+ << ", Max total doc size: " << _minByteSize;
+ } else if (_maxSplitBits != 58) {
+ out << "Max split bits to use: " << _maxSplitBits;
+ }
+ out << ")";
+ out << " Reasons to start: " << _reason;
+ if (verbose) {
+ out << " : ";
+ BucketCommand::print(out, verbose, indent);
+ }
+}
+
+SplitBucketReply::SplitBucketReply(const SplitBucketCommand& cmd)
+ : BucketReply(cmd)
+{
+}
+
+void
+SplitBucketReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "SplitBucketReply(" << getBucketId();
+ if (_result.empty()) {
+ out << " - No target files created.";
+ } else {
+ out << " ->";
+ for (uint32_t i=0; i<_result.size(); ++i) {
+ out << "\n" << indent << " " << _result[i].first << ": "
+ << _result[i].second;
+ }
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ BucketReply::print(out, verbose, indent);
+ }
+}
+
+JoinBucketsCommand::JoinBucketsCommand(const document::BucketId& target)
+ : MaintenanceCommand(MessageType::JOINBUCKETS, target),
+ _minJoinBits(0)
+{
+}
+
+void
+JoinBucketsCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "JoinBucketsCommand(" << getBucketId();
+ if (_sources.empty()) {
+ out << " - No files to join.";
+ } else {
+ out << " <-";
+ for (uint32_t i=0; i<_sources.size(); ++i) {
+ out << " " << _sources[i];
+ }
+ }
+ out << ")";
+ out << " Reasons to start: " << _reason;
+ if (verbose) {
+ out << " : ";
+ BucketCommand::print(out, verbose, indent);
+ }
+}
+
+
+JoinBucketsReply::JoinBucketsReply(const JoinBucketsCommand& cmd)
+ : BucketInfoReply(cmd),
+ _sources(cmd.getSourceBuckets())
+{
+}
+
+JoinBucketsReply::JoinBucketsReply(const JoinBucketsCommand& cmd, const BucketInfo& bucketInfo)
+ : BucketInfoReply(cmd),
+ _sources(cmd.getSourceBuckets())
+{
+ setBucketInfo(bucketInfo);
+}
+
+void
+JoinBucketsReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "JoinBucketsReply(" << getBucketId();
+ if (_sources.empty()) {
+ out << " - No files to join.";
+ } else {
+ out << " <-";
+ for (uint32_t i=0; i<_sources.size(); ++i) {
+ out << " " << _sources[i];
+ }
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ BucketReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/message/bucketsplitting.h b/storageapi/src/vespa/storageapi/message/bucketsplitting.h
new file mode 100644
index 00000000000..cbddfc8eaf6
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/bucketsplitting.h
@@ -0,0 +1,144 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/storageapi/buckets/bucketinfo.h>
+#include <vespa/storageapi/messageapi/bucketcommand.h>
+#include <vespa/storageapi/messageapi/bucketinforeply.h>
+#include <vespa/storageapi/messageapi/maintenancecommand.h>
+
+namespace storage {
+namespace api {
+
+/**
+ * @class SplitBucketCommand
+ * @ingroup message
+ *
+ * @brief Split a bucket
+ *
+ * Splits a bucket into two parts using the next split bit that is unused.
+ *
+ * Distributors can issue splits for multiple reasons:
+ * - Inconsistent buckets, so we need to split buckets containing others until
+ * they are either split equally, or no longer contains others.
+ * - Buckets that are too large are split to reduce file size.
+ * - Buckets with too many entries are split to reduce amount of metadata.
+ *
+ * In the first case, min and max split bits can be set. This will make storage
+ * able to split several bits at a time, but know where to stop.
+ *
+ * In the second case, min byte size can be set, to ensure that we don't split
+ * bucket more one step if the copy at the time of processing is
+ * actually smaller. Since removes can happen in the meantime, the min byte size
+ * should be smaller than the limit we use for splitting. Suggesting half.
+ *
+ * Similarily we can do as the second case in the third case too, just using
+ * min doc count as limiter instead.
+ *
+ * If neither are specified, min/max split bits limits nothing, but the sizes
+ * are set to max, which ensures that only one split step is taken.
+ */
+class SplitBucketCommand : public MaintenanceCommand {
+private:
+ uint8_t _minSplitBits;
+ uint8_t _maxSplitBits;
+ uint32_t _minByteSize;
+ uint32_t _minDocCount;
+
+public:
+ SplitBucketCommand(const document::BucketId& id);
+
+ uint8_t getMinSplitBits() const { return _minSplitBits; }
+ uint8_t getMaxSplitBits() const { return _maxSplitBits; }
+ uint32_t getMinByteSize() const { return _minByteSize; }
+ uint32_t getMinDocCount() const { return _minDocCount; }
+
+ void setMinSplitBits(uint8_t v) { _minSplitBits = v; }
+ void setMaxSplitBits(uint8_t v) { _maxSplitBits = v; }
+ void setMinByteSize(uint32_t v) { _minByteSize = v; }
+ void setMinDocCount(uint32_t v) { _minDocCount = v; }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(SplitBucketCommand, onSplitBucket)
+};
+
+/**
+ * @class SplitBucketReply
+ * @ingroup message
+ *
+ * @brief Reply of a split bucket command.
+ */
+class SplitBucketReply : public BucketReply {
+public:
+ typedef std::pair<document::BucketId, BucketInfo> Entry;
+
+private:
+ std::vector<Entry> _result;
+
+public:
+ explicit SplitBucketReply(const SplitBucketCommand& cmd);
+
+ std::vector<Entry>& getSplitInfo() { return _result; }
+ const std::vector<Entry>& getSplitInfo() const { return _result; }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(SplitBucketReply, onSplitBucketReply)
+};
+
+/**
+ * @class JoinBucketCommand
+ * @ingroup message
+ *
+ * @brief Join two buckets
+ *
+ * Joins two buckets on the same node into a bucket with one fewer split bit.
+ */
+class JoinBucketsCommand : public MaintenanceCommand {
+ std::vector<document::BucketId> _sources;
+ uint8_t _minJoinBits;
+
+public:
+ explicit JoinBucketsCommand(const document::BucketId& target);
+
+ std::vector<document::BucketId>& getSourceBuckets() { return _sources; }
+ const std::vector<document::BucketId>& getSourceBuckets() const
+ { return _sources; }
+
+ void setMinJoinBits(uint8_t minJoinBits) { _minJoinBits = minJoinBits; }
+ uint8_t getMinJoinBits() const { return _minJoinBits; }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(JoinBucketsCommand, onJoinBuckets)
+};
+
+/**
+ * @class JoinBucketsReply
+ * @ingroup message
+ *
+ * @brief Reply of a join bucket command.
+ */
+class JoinBucketsReply : public BucketInfoReply {
+ std::vector<document::BucketId> _sources;
+
+public:
+ explicit JoinBucketsReply(const JoinBucketsCommand& cmd);
+
+ JoinBucketsReply(const JoinBucketsCommand& cmd, const BucketInfo& bucketInfo);
+
+ const std::vector<document::BucketId>& getSourceBuckets() const
+ { return _sources; }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(JoinBucketsReply, onJoinBucketsReply)
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/message/datagram.cpp b/storageapi/src/vespa/storageapi/message/datagram.cpp
new file mode 100644
index 00000000000..bc0dadc3fb3
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/datagram.cpp
@@ -0,0 +1,189 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <cassert>
+#include <vespa/storageapi/message/datagram.h>
+
+namespace storage {
+namespace api {
+
+IMPLEMENT_COMMAND(DocBlockCommand, DocBlockReply)
+IMPLEMENT_REPLY(DocBlockReply)
+IMPLEMENT_COMMAND(MapVisitorCommand, MapVisitorReply)
+IMPLEMENT_REPLY(MapVisitorReply)
+IMPLEMENT_COMMAND(DocumentListCommand, DocumentListReply)
+IMPLEMENT_REPLY(DocumentListReply)
+IMPLEMENT_COMMAND(EmptyBucketsCommand, EmptyBucketsReply)
+IMPLEMENT_REPLY(EmptyBucketsReply)
+
+DocBlockCommand::DocBlockCommand(const document::BucketId& bucketId,
+ const vdslib::DocumentList& block,
+ const std::shared_ptr<void>& buffer)
+ : StorageCommand(MessageType::DOCBLOCK),
+ _bucketId(bucketId),
+ _docBlock(block),
+ _buffer(buffer),
+ _keepTimeStamps(false)
+{
+}
+
+void
+DocBlockCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "DocBlockCommand("
+ << "size " << _docBlock.getBufferSize() << ", used space "
+ << (_docBlock.getBufferSize() - _docBlock.countFree()) << ", doccount "
+ << _docBlock.size() << ")";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+DocBlockReply::DocBlockReply(const DocBlockCommand& cmd)
+ : StorageReply(cmd)
+{
+}
+
+void
+DocBlockReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "DocBlockReply()";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+MapVisitorCommand::MapVisitorCommand()
+ : StorageCommand(MessageType::MAPVISITOR)
+{
+}
+
+void
+MapVisitorCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "MapVisitor(" << _statistics.size() << " entries";
+ if (verbose) {
+ for (vdslib::Parameters::ParametersMap::const_iterator it
+ = _statistics.begin(); it != _statistics.end(); ++it)
+ {
+ out << ",\n" << indent << " " << it->first << ": "
+ << vespalib::stringref(it->second.c_str(), it->second.length());
+ }
+ out << ") : ";
+ StorageCommand::print(out, verbose, indent);
+ } else {
+ out << ")";
+ }
+}
+
+MapVisitorReply::MapVisitorReply(const MapVisitorCommand& cmd)
+ : StorageReply(cmd)
+{
+}
+
+void
+MapVisitorReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "MapVisitorReply()";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+DocumentListCommand::DocumentListCommand(const document::BucketId& bid)
+ : StorageCommand(MessageType::DOCUMENTLIST),
+ _bucketId(bid),
+ _documents()
+{
+}
+
+void
+DocumentListCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "DocumentList(" << _bucketId;
+ if (_documents.empty()) {
+ out << ", empty";
+ } else if (verbose) {
+ out << ",";
+ for (uint32_t i=0; i<_documents.size(); ++i) {
+ out << "\n" << indent << " ";
+ out << ":" << _documents[i];
+ }
+ } else {
+ out << ", " << _documents.size() << " documents";
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+DocumentListReply::DocumentListReply(const DocumentListCommand& cmd)
+ : StorageReply(cmd)
+{
+}
+
+void
+DocumentListReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "DocumentListReply()";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+EmptyBucketsCommand::EmptyBucketsCommand(
+ const std::vector<document::BucketId>& buckets)
+ : StorageCommand(MessageType::EMPTYBUCKETS),
+ _buckets(buckets)
+{
+}
+
+void
+EmptyBucketsCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "EmptyBuckets(";
+ if (verbose) {
+ for (uint32_t i=0; i<_buckets.size(); ++i) {
+ out << "\n" << indent << " ";
+ out << _buckets[i];
+ }
+ } else {
+ out << _buckets.size() << " buckets";
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+EmptyBucketsReply::EmptyBucketsReply(const EmptyBucketsCommand& cmd)
+ : StorageReply(cmd)
+{
+}
+
+void
+EmptyBucketsReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "EmptyBucketsReply()";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/message/datagram.h b/storageapi/src/vespa/storageapi/message/datagram.h
new file mode 100644
index 00000000000..69563bb32f8
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/datagram.h
@@ -0,0 +1,226 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/document.h>
+#include <vespa/vdslib/container/documentlist.h>
+#include <vespa/storageapi/messageapi/storagecommand.h>
+#include <vespa/storageapi/messageapi/storagereply.h>
+#include <vespa/storageapi/defs.h>
+#include <vespa/storageapi/message/visitor.h>
+
+namespace storage {
+namespace api {
+
+/**
+ * @class DocBlockCommand
+ * @ingroup message
+ *
+ * @brief Sends a docblock to a visitor or subscriber.
+ */
+class DocBlockCommand : public StorageCommand {
+ document::BucketId _bucketId;
+ vdslib::DocumentList _docBlock;
+ std::shared_ptr<void> _buffer; // Owns data in docblock
+ bool _keepTimeStamps; // Used for recovery/synchronization where we want to
+ // keep the timestamps of the origin.
+
+public:
+ DocBlockCommand(const document::BucketId& bucketId,
+ const vdslib::DocumentList& block,
+ const std::shared_ptr<void>& buffer);
+
+ vdslib::DocumentList& getDocumentBlock()
+ { assert(_docBlock.getBufferSize() > 0); return _docBlock; }
+ const vdslib::DocumentList& getDocumentBlock() const
+ { assert(_docBlock.getBufferSize() > 0); return _docBlock; }
+
+ void setDocumentBlock(vdslib::DocumentList& block) {
+ _docBlock = block;
+ }
+
+ document::BucketId getBucketId() const { return _bucketId; }
+ virtual bool hasSingleBucketId() const { return true; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ bool keepTimeStamps() const { return _keepTimeStamps; }
+ void keepTimeStamps(bool keepTime) { _keepTimeStamps = keepTime; }
+
+ DECLARE_STORAGECOMMAND(DocBlockCommand, onDocBlock)
+};
+
+/**
+ * @class DocBlockReply
+ * @ingroup message
+ *
+ * @brief Confirm that a given docblock have been received.
+ */
+class DocBlockReply : public StorageReply {
+public:
+ explicit DocBlockReply(const DocBlockCommand&);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(DocBlockReply, onDocBlockReply)
+};
+
+/**
+ * @class MapStorageCommand
+ * @ingroup message
+ *
+ * @brief Sends a map of data to a visitor.
+ *
+ * This is a generic way to transfer data to the visitor data handler.
+ * It is for instance used when doing a specialized visitor to gather statistics
+ * on usage of document types and namespaces.
+ */
+class MapVisitorCommand : public StorageCommand {
+ vdslib::Parameters _statistics;
+
+public:
+ MapVisitorCommand();
+
+ vdslib::Parameters& getData() { return _statistics; };
+ const vdslib::Parameters& getData() const { return _statistics; };
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(MapVisitorCommand, onMapVisitor)
+};
+
+/**
+ * @class MapStorageReply
+ * @ingroup message
+ *
+ * @brief Confirm that a given map visitor command has been received.
+ */
+class MapVisitorReply : public StorageReply {
+public:
+ explicit MapVisitorReply(const MapVisitorCommand&);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(MapVisitorReply, onMapVisitorReply)
+};
+
+/**
+ * @class DocumentListCommand
+ * @ingroup message
+ *
+ * @brief Sends a list of documents to the visitor data handler.
+ *
+ * This is used in synchronization in order to transfer minimal amount of data
+ * to the synchronization agent.
+ */
+class DocumentListCommand : public StorageCommand {
+public:
+ struct Entry {
+ document::Document::SP _doc;
+ int64_t _lastModified;
+ bool _removeEntry;
+
+ Entry() : _doc(), _lastModified(0), _removeEntry(false) {}
+
+ Entry(const document::Document::SP& doc, int64_t lastModified,
+ bool removeEntry)
+ : _doc(doc),
+ _lastModified(lastModified),
+ _removeEntry(removeEntry)
+ {
+ }
+ };
+
+private:
+ document::BucketId _bucketId;
+ std::vector<Entry> _documents;
+
+public:
+ DocumentListCommand(const document::BucketId& bid);
+
+ const document::BucketId& getBucketId() { return _bucketId; }
+ std::vector<Entry>& getDocuments() { return _documents; }
+ const std::vector<Entry>& getDocuments() const { return _documents; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(DocumentListCommand, onDocumentList)
+
+};
+
+inline std::ostream& operator<<(std::ostream& out,
+ const DocumentListCommand::Entry& e)
+{
+ out << e._doc->getId();
+ if (e._removeEntry) out << " - removed";
+ out << ", last modified at " << e._lastModified;
+ return out;
+}
+
+/**
+ * @class DocumentListReply
+ * @ingroup message
+ *
+ * @brief Confirm that a given visitorstatisticscommand has been received.
+ */
+class DocumentListReply : public StorageReply {
+public:
+ explicit DocumentListReply(const DocumentListCommand&);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(DocumentListReply, onDocumentListReply)
+};
+
+/**
+ * @class EmptyBucketsCommand
+ * @ingroup message
+ *
+ * @brief Sends a vector of bucket ids to a visitor.
+ *
+ * This message is used in synchronization to tell the synchronization client
+ * that a bucket contains no data at all. This is needed to let the follower be
+ * able to delete documents from these buckets, as they would otherwise be
+ * ignored by the synch agent.
+ */
+class EmptyBucketsCommand : public StorageCommand {
+ std::vector<document::BucketId> _buckets;
+
+public:
+ EmptyBucketsCommand(const std::vector<document::BucketId>&);
+
+ const std::vector<document::BucketId>& getBuckets() const
+ { return _buckets; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(EmptyBucketsCommand, onEmptyBuckets)
+
+private:
+};
+
+/**
+ * @class EmptyBucketsReply
+ * @ingroup message
+ *
+ * @brief Confirm that a given emptybucketscommad has been received.
+ */
+class EmptyBucketsReply : public StorageReply {
+public:
+ explicit EmptyBucketsReply(const EmptyBucketsCommand&);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(EmptyBucketsReply, onEmptyBucketsReply)
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/message/documentsummary.cpp b/storageapi/src/vespa/storageapi/message/documentsummary.cpp
new file mode 100644
index 00000000000..cfb56489459
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/documentsummary.cpp
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/message/documentsummary.h>
+
+namespace storage {
+namespace api {
+
+IMPLEMENT_COMMAND(DocumentSummaryCommand, DocumentSummaryReply)
+IMPLEMENT_REPLY(DocumentSummaryReply)
+
+DocumentSummaryCommand::DocumentSummaryCommand()
+ : StorageCommand(MessageType::DOCUMENTSUMMARY),
+ DocumentSummary()
+{
+}
+
+void
+DocumentSummaryCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "DocumentSummary(" << getSummaryCount() << " summaries)";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+DocumentSummaryReply::DocumentSummaryReply(const DocumentSummaryCommand& cmd)
+ : StorageReply(cmd)
+{
+}
+
+void
+DocumentSummaryReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "DocumentSummaryReply()";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/message/documentsummary.h b/storageapi/src/vespa/storageapi/message/documentsummary.h
new file mode 100644
index 00000000000..2aa3c4765c5
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/documentsummary.h
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vdslib/container/documentsummary.h>
+#include <vespa/storageapi/message/visitor.h>
+
+namespace storage {
+namespace api {
+
+/**
+ * @class DocumentSummaryCommand
+ * @ingroup message
+ *
+ * @brief The result of a searchvisitor.
+ */
+class DocumentSummaryCommand : public StorageCommand,
+ public vdslib::DocumentSummary
+{
+public:
+ explicit DocumentSummaryCommand();
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ uint32_t getMemoryFootprint() const {
+ return getSerializedSize();
+ }
+
+ DECLARE_STORAGECOMMAND(DocumentSummaryCommand, onDocumentSummary)
+};
+
+/**
+ * @class DocumentSummaryReply
+ * @ingroup message
+ *
+ * @brief Response to a document summary command.
+ */
+class DocumentSummaryReply : public StorageReply {
+public:
+ explicit DocumentSummaryReply(const DocumentSummaryCommand& command);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(DocumentSummaryReply, onDocumentSummaryReply)
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/message/internal.h b/storageapi/src/vespa/storageapi/message/internal.h
new file mode 100644
index 00000000000..9b92d956919
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/internal.h
@@ -0,0 +1,110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @file internal.h
+ *
+ * Internal commands, used in storage. These are commands that don't need to
+ * be serialized as they never leave a node, implemented within storage itself
+ * to be able to include storage types not defined in the API.
+ *
+ * Historically these messages also existed so we could alter internal messages
+ * without recompiling clients, but currently no clients use storage API for
+ * communication anymore so this is no longer an issue.
+ */
+#pragma once
+
+#include <vespa/storageapi/messageapi/storagecommand.h>
+#include <vespa/storageapi/messageapi/storagereply.h>
+
+namespace storage {
+namespace api {
+
+/**
+ * @class InternalCommand
+ * @ingroup message
+ *
+ * @brief Base class for commands local to a VDS node.
+ *
+ * This is the base class for internal server commands. They can not be
+ * serialized, so any attempt of sending such a command away from a storage
+ * node will fail.
+ */
+class InternalCommand : public StorageCommand {
+ uint32_t _type;
+
+public:
+ InternalCommand(uint32_t type)
+ : StorageCommand(MessageType::INTERNAL), _type(type) {}
+
+ uint32_t getType() const { return _type; }
+
+ virtual bool callHandler(MessageHandler& h,
+ const std::shared_ptr<StorageMessage> & m) const
+ {
+ return h.onInternal(std::static_pointer_cast<InternalCommand>(m));
+ }
+
+ /**
+ * Enforcing that subclasses implement print such that we can see what kind
+ * of message it is when debugging.
+ */
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const = 0;
+};
+
+/**
+ * @class InternalReply
+ * @ingroup message
+ *
+ * @brief Response of an internal command.
+ */
+class InternalReply : public StorageReply {
+ uint32_t _type;
+
+public:
+ InternalReply(uint32_t type, const InternalCommand& cmd)
+ : StorageReply(cmd),
+ _type(type)
+ {
+ }
+
+ uint32_t getType() const { return _type; }
+
+ virtual bool callHandler(MessageHandler& h,
+ const std::shared_ptr<StorageMessage> & m) const
+ {
+ return h.onInternalReply(std::static_pointer_cast<InternalReply>(m));
+ }
+
+ /**
+ * Enforcing that subclasses implement print such that we can see what kind
+ * of message it is when debugging.
+ */
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const = 0;
+};
+
+inline void
+InternalCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "InternalCommand(" << _type << ")";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+inline void
+InternalReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "InternalReply(" << _type << ")";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/message/multioperation.cpp b/storageapi/src/vespa/storageapi/message/multioperation.cpp
new file mode 100644
index 00000000000..35e4075e3ab
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/multioperation.cpp
@@ -0,0 +1,108 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/message/multioperation.h>
+
+using document::DocumentTypeRepo;
+
+namespace storage {
+namespace api {
+
+IMPLEMENT_COMMAND(MultiOperationCommand, MultiOperationReply)
+IMPLEMENT_REPLY(MultiOperationReply)
+
+MultiOperationCommand::MultiOperationCommand(const DocumentTypeRepo::SP &repo,
+ const document::BucketId& id,
+ int bufferSize,
+ bool keepTimeStamps_)
+ : BucketInfoCommand(MessageType::MULTIOPERATION, id),
+ _buffer(),
+ _operations(repo, 0, 0),
+ _keepTimeStamps(keepTimeStamps_)
+{
+ _buffer.resize(bufferSize);
+ if (_buffer.size() > 0) {
+ _operations = vdslib::WritableDocumentList(_operations.getTypeRepo(),
+ &_buffer[0], _buffer.size(), false);
+ }
+}
+
+MultiOperationCommand::MultiOperationCommand(const DocumentTypeRepo::SP &repo,
+ const document::BucketId& id,
+ const std::vector<char>& buffer,
+ bool keepTimeStamps_)
+ : BucketInfoCommand(MessageType::MULTIOPERATION, id),
+ _buffer(buffer),
+ _operations(repo, 0, 0),
+ _keepTimeStamps(keepTimeStamps_)
+{
+ if (_buffer.size() > 0) {
+ _operations = vdslib::WritableDocumentList(_operations.getTypeRepo(),
+ &_buffer[0], _buffer.size(), true);
+ }
+}
+
+MultiOperationCommand::MultiOperationCommand(const MultiOperationCommand& o)
+ : BucketInfoCommand(MessageType::MULTIOPERATION, o.getBucketId()),
+ _buffer(o._buffer),
+ _operations(o._operations.getTypeRepo(),0, 0),
+ _keepTimeStamps(o._keepTimeStamps)
+{
+ setTimeout(o.getTimeout());
+ setSourceIndex(o.getSourceIndex());
+ setPriority(o.getPriority());
+ if (_buffer.size() > 0) {
+ _operations = vdslib::WritableDocumentList(_operations.getTypeRepo(),
+ &_buffer[0], _buffer.size(), true);
+ }
+}
+
+void
+MultiOperationCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "MultiOperationCommand(" << getBucketId()
+ << ", size " << _operations.getBufferSize() << ", used space "
+ << (_operations.getBufferSize() - _operations.countFree())
+ << ", doccount " << _operations.size() << ", keepTimeStamps "
+ << _keepTimeStamps << ")";
+ if (verbose) {
+ out << " {";
+ bool first = true;
+ for(vdslib::DocumentList::const_iterator it = _operations.begin();
+ it != _operations.end(); ++it)
+ {
+ if (!first) { out << ","; } else { first = false; }
+ out << "\n" << indent << " ";
+ if (it->isRemoveEntry()) {
+ out << "Remove(" << it->getDocumentId() << ")";
+ } else if (it->isUpdateEntry()) {
+ out << "Update(" << it->getDocumentId() << ")";
+ } else {
+ out << "Put(" << it->getDocumentId() << ")";
+ }
+ }
+ out << "\n" << indent << "} : ";
+ BucketInfoCommand::print(out, verbose, indent);
+ }
+}
+
+MultiOperationReply::MultiOperationReply(const MultiOperationCommand& cmd)
+ : BucketInfoReply(cmd),
+ _highestModificationTimestamp(0)
+{
+}
+
+void
+MultiOperationReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "MultiOperationReply(" << getBucketId() << ")";
+ if (verbose) {
+ out << " : ";
+ BucketInfoReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/message/multioperation.h b/storageapi/src/vespa/storageapi/message/multioperation.h
new file mode 100644
index 00000000000..31db78458f9
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/multioperation.h
@@ -0,0 +1,93 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/document/fieldvalue/document.h>
+#include <vespa/storageapi/messageapi/bucketinfocommand.h>
+#include <vespa/storageapi/messageapi/bucketinforeply.h>
+#include <vespa/storageapi/defs.h>
+#include <vespa/storageapi/message/visitor.h>
+#include <vespa/vdslib/container/writabledocumentlist.h>
+
+namespace storage {
+namespace api {
+
+/**
+ * @class MultiOperationCommand
+ * @ingroup message
+ *
+ * @brief Sends a documentlist
+ */
+class MultiOperationCommand : public BucketInfoCommand {
+private:
+ std::vector<char> _buffer; // Used to hold data refered to by document list
+ // if message is to own its data.
+ vdslib::WritableDocumentList _operations;
+ bool _keepTimeStamps;
+
+public:
+ explicit MultiOperationCommand(const document::DocumentTypeRepo::SP &repo,
+ const document::BucketId& id,
+ int bufferSize,
+ bool keepTimeStamps = false);
+ explicit MultiOperationCommand(const document::DocumentTypeRepo::SP &repo,
+ const document::BucketId& id,
+ const std::vector<char>& buffer,
+ bool keepTimeStamps = false);
+ explicit MultiOperationCommand(const MultiOperationCommand& template_);
+
+ std::vector<char>& getBuffer() { return _buffer; };
+ const std::vector<char>& getBuffer() const { return _buffer; };
+
+ vdslib::WritableDocumentList& getOperations()
+ { assert(_operations.getBufferSize() > 0); return _operations; }
+ const vdslib::WritableDocumentList& getOperations() const
+ { assert(_operations.getBufferSize() > 0); return _operations; }
+
+ void setOperations(vdslib::WritableDocumentList& operations) {
+ _buffer.clear();
+ _operations = operations;
+ }
+
+ uint32_t getMemoryFootprint() const {
+ return _buffer.size() + 20;
+ }
+
+ bool keepTimeStamps() const { return _keepTimeStamps; }
+ void keepTimeStamps(bool keepTime) { _keepTimeStamps = keepTime; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(MultiOperationCommand, onMultiOperation)
+};
+
+/**
+ * @class MultiOperationReply
+ * @ingroup message
+ *
+ * @brief Confirm that a given docoperations have been received.
+ */
+class MultiOperationReply : public BucketInfoReply {
+private:
+ // No need to serialize this, as it's only used internally in the distributor.
+ uint64_t _highestModificationTimestamp;
+
+public:
+ explicit MultiOperationReply(const MultiOperationCommand&);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ void setHighestModificationTimestamp(uint64_t highestModificationTimestamp) {
+ _highestModificationTimestamp = highestModificationTimestamp;
+ }
+ uint64_t getHighestModificationTimestamp() const { return _highestModificationTimestamp; }
+
+ DECLARE_STORAGEREPLY(MultiOperationReply, onMultiOperationReply)
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/message/persistence.cpp b/storageapi/src/vespa/storageapi/message/persistence.cpp
new file mode 100644
index 00000000000..e39c03cf440
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/persistence.cpp
@@ -0,0 +1,358 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/message/persistence.h>
+
+#include <vespa/vespalib/util/exceptions.h>
+
+namespace storage {
+namespace api {
+
+IMPLEMENT_COMMAND(PutCommand, PutReply)
+IMPLEMENT_REPLY(PutReply)
+IMPLEMENT_COMMAND(UpdateCommand, UpdateReply)
+IMPLEMENT_REPLY(UpdateReply)
+IMPLEMENT_COMMAND(GetCommand, GetReply)
+IMPLEMENT_REPLY(GetReply)
+IMPLEMENT_COMMAND(RemoveCommand, RemoveReply)
+IMPLEMENT_REPLY(RemoveReply)
+IMPLEMENT_COMMAND(RevertCommand, RevertReply)
+IMPLEMENT_REPLY(RevertReply)
+
+PutCommand::PutCommand(const document::BucketId& id,
+ const document::Document::SP& doc, Timestamp time)
+ : TestAndSetCommand(MessageType::PUT, id),
+ _doc(doc),
+ _timestamp(time),
+ _updateTimestamp(0)
+{
+ if (_doc.get() == 0) {
+ throw vespalib::IllegalArgumentException(
+ "Cannot put a null document", VESPA_STRLOC);
+ }
+}
+
+StorageCommand::UP
+PutCommand::createCopyToForward(
+ const document::BucketId& bucket, uint64_t timestamp) const
+{
+ StorageCommand::UP cmd(new PutCommand(bucket, _doc, timestamp));
+ static_cast<PutCommand&>(*cmd).setUpdateTimestamp(_updateTimestamp);
+ return cmd;
+}
+
+vespalib::string
+PutCommand::getSummary() const
+{
+ vespalib::asciistream stream;
+ stream << "Put(BucketId(0x" << vespalib::hex << getBucketId().getId() << "), "
+ << _doc->getId().toString()
+ << ", timestamp " << vespalib::dec << _timestamp
+ << ')';
+
+ return stream.str();
+}
+
+void
+PutCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "Put(" << getBucketId() << ", " << _doc->getId()
+ << ", timestamp " << _timestamp << ", size "
+ << _doc->serialize()->getLength() << ")";
+ if (verbose) {
+ out << " {\n" << indent << " ";
+ _doc->print(out, verbose, indent + " ");
+ out << "\n" << indent << "}" << " : ";
+ BucketInfoCommand::print(out, verbose, indent);
+ }
+}
+
+PutReply::PutReply(const PutCommand& cmd, bool wasFoundFlag)
+ : BucketInfoReply(cmd),
+ _docId(cmd.getDocumentId()),
+ _document(cmd.getDocument()),
+ _timestamp(cmd.getTimestamp()),
+ _updateTimestamp(cmd.getUpdateTimestamp()),
+ _wasFound(wasFoundFlag)
+{
+}
+
+void
+PutReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "PutReply("
+ << _docId.toString() << ", " << getBucketId() << ", timestamp "
+ << _timestamp;
+
+ if (hasBeenRemapped()) {
+ out << " (was remapped)";
+ }
+
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ BucketInfoReply::print(out, verbose, indent);
+ }
+}
+
+UpdateCommand::UpdateCommand(const document::BucketId& id,
+ const document::DocumentUpdate::SP& update,
+ Timestamp time)
+ : TestAndSetCommand(MessageType::UPDATE, id),
+ _update(update),
+ _timestamp(time),
+ _oldTimestamp(0)
+{
+ if (_update.get() == 0) {
+ throw vespalib::IllegalArgumentException(
+ "Cannot update a null update", VESPA_STRLOC);
+ }
+}
+
+vespalib::string
+UpdateCommand::getSummary() const {
+ vespalib::asciistream stream;
+ stream << "Update(BucketId(0x" << vespalib::hex << getBucketId().getId() << "), "
+ << _update->getId().toString()
+ << ", timestamp " << vespalib::dec << _timestamp;
+ if (_oldTimestamp != 0) {
+ stream << ", old timestamp " << _oldTimestamp;
+ }
+ stream << ')';
+
+ return stream.str();
+}
+
+void
+UpdateCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "Update(" << getBucketId() << ", " << _update->getId()
+ << ", timestamp " << _timestamp;
+ if (_oldTimestamp != 0) {
+ out << ", old timestamp " << _oldTimestamp;
+ }
+ out << ")";
+ if (verbose) {
+ out << " {\n" << indent << " ";
+ _update->print(out, verbose, indent + " ");
+ out << "\n" << indent << "} : ";
+ BucketInfoCommand::print(out, verbose, indent);
+ }
+}
+
+StorageCommand::UP
+UpdateCommand::createCopyToForward(
+ const document::BucketId& bucket, uint64_t timestamp) const
+{
+ StorageCommand::UP cmd(new UpdateCommand(bucket, _update, timestamp));
+ static_cast<UpdateCommand&>(*cmd).setOldTimestamp(_oldTimestamp);
+ return cmd;
+}
+
+UpdateReply::UpdateReply(const UpdateCommand& cmd, Timestamp oldTimestamp)
+ : BucketInfoReply(cmd),
+ _docId(cmd.getDocumentId()),
+ _timestamp(cmd.getTimestamp()),
+ _oldTimestamp(oldTimestamp),
+ _consistentNode((uint16_t)-1)
+{
+}
+
+void
+UpdateReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "UpdateReply("
+ << _docId.toString() << ", " << getBucketId() << ", timestamp "
+ << _timestamp << ", timestamp of updated doc: " << _oldTimestamp;
+
+ if (_consistentNode != (uint16_t)-1) {
+ out << " Was inconsistent (best node " << _consistentNode << ")";
+ }
+
+ out << ")";
+
+ if (verbose) {
+ out << " : ";
+ BucketInfoReply::print(out, verbose, indent);
+ }
+}
+
+GetCommand::GetCommand(const document::BucketId& bid,
+ const document::DocumentId& docId,
+ const vespalib::stringref & fieldSet, Timestamp before)
+ : BucketInfoCommand(MessageType::GET, bid),
+ _docId(docId),
+ _beforeTimestamp(before),
+ _fieldSet(fieldSet)
+{
+}
+
+StorageCommand::UP
+GetCommand::createCopyToForward(
+ const document::BucketId& bucket, uint64_t timestamp) const
+{
+ (void) timestamp;
+ StorageCommand::UP cmd(new GetCommand(
+ bucket, _docId, _fieldSet, _beforeTimestamp));
+ return cmd;
+}
+
+vespalib::string
+GetCommand::getSummary() const
+{
+ vespalib::asciistream stream;
+ stream << "Get(BucketId(" << vespalib::hex << getBucketId().getId() << "), "
+ << _docId.toString()
+ << ", beforetimestamp " << vespalib::dec << _beforeTimestamp
+ << ')';
+
+ return stream.str();
+}
+
+
+void
+GetCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "Get(" << getBucketId() << ", " << _docId << ")";
+ if (verbose) {
+ out << " : ";
+ BucketCommand::print(out, verbose, indent);
+ }
+}
+
+GetReply::GetReply(const GetCommand& cmd,
+ const document::Document::SP& doc,
+ Timestamp lastModified)
+ : BucketInfoReply(cmd),
+ _docId(cmd.getDocumentId()),
+ _fieldSet(cmd.getFieldSet()),
+ _doc(doc),
+ _beforeTimestamp(cmd.getBeforeTimestamp()),
+ _lastModifiedTime(lastModified)
+{
+}
+
+void
+GetReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "GetReply(" << getBucketId() << ", " << _docId
+ << ", timestamp " << _lastModifiedTime << ")";
+ if (verbose) {
+ out << " : ";
+ BucketReply::print(out, verbose, indent);
+ }
+}
+
+RemoveCommand::RemoveCommand(const document::BucketId& bid,
+ const document::DocumentId& docId,
+ Timestamp timestamp)
+ : TestAndSetCommand(MessageType::REMOVE, bid),
+ _docId(docId),
+ _timestamp(timestamp)
+{
+}
+
+StorageCommand::UP
+RemoveCommand::createCopyToForward(
+ const document::BucketId& bucket, uint64_t timestamp) const
+{
+ StorageCommand::UP cmd(new RemoveCommand(bucket, _docId, timestamp));
+ return cmd;
+}
+
+vespalib::string
+RemoveCommand::getSummary() const {
+ vespalib::asciistream stream;
+ stream << "Remove(BucketId(0x" << vespalib::hex << getBucketId().getId() << "), "
+ << _docId.toString() << ", timestamp " << vespalib::dec << _timestamp << ')';
+
+ return stream.str();
+}
+void
+RemoveCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "Remove(" << getBucketId() << ", " << _docId
+ << ", timestamp " << _timestamp << ")";
+ if (verbose) {
+ out << " : ";
+ BucketInfoCommand::print(out, verbose, indent);
+ }
+}
+
+RemoveReply::RemoveReply(const RemoveCommand& cmd, Timestamp oldTimestamp)
+ : BucketInfoReply(cmd),
+ _docId(cmd.getDocumentId()),
+ _timestamp(cmd.getTimestamp()),
+ _oldTimestamp(oldTimestamp)
+{
+}
+
+void
+RemoveReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "RemoveReply(" << getBucketId() << ", " << _docId
+ << ", timestamp " << _timestamp;
+ if (_oldTimestamp != 0) {
+ out << ", removed doc from " << _oldTimestamp;
+ } else {
+ out << ", not found";
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ BucketInfoReply::print(out, verbose, indent);
+ }
+}
+
+RevertCommand::RevertCommand(const document::BucketId& id,
+ const std::vector<Timestamp>& revertTokens)
+ : BucketInfoCommand(MessageType::REVERT, id),
+ _tokens(revertTokens)
+{
+}
+
+void
+RevertCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "Revert(" << getBucketId();
+ if (verbose) {
+ out << ",";
+ for (uint32_t i=0; i<_tokens.size(); ++i) {
+ out << "\n" << indent << " " << _tokens[i];
+ }
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ BucketInfoCommand::print(out, verbose, indent);
+ }
+}
+
+RevertReply::RevertReply(const RevertCommand& cmd)
+ : BucketInfoReply(cmd),
+ _tokens(cmd.getRevertTokens())
+{
+}
+
+void
+RevertReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "RevertReply(" << getBucketId() << ")";
+ if (verbose) {
+ out << " : ";
+ BucketInfoReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/message/persistence.h b/storageapi/src/vespa/storageapi/message/persistence.h
new file mode 100644
index 00000000000..12d371bf561
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/persistence.h
@@ -0,0 +1,363 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @file persistence.h
+ *
+ * Persistence related commands, like put, get & remove
+ */
+#pragma once
+
+#include <vespa/document/document.h>
+#include <vespa/storageapi/messageapi/bucketinforeply.h>
+#include <vespa/storageapi/defs.h>
+#include <vespa/documentapi/messagebus/messages/testandsetcondition.h>
+
+namespace storage {
+namespace api {
+
+using documentapi::TestAndSetCondition;
+
+class TestAndSetCommand : public BucketInfoCommand {
+ TestAndSetCondition _condition;
+
+public:
+ TestAndSetCommand(const MessageType & messageType,
+ const document::BucketId & id)
+ : BucketInfoCommand(messageType, id) {}
+
+ void setCondition(const TestAndSetCondition & condition) { _condition = condition; }
+ const TestAndSetCondition & getCondition() const { return _condition; }
+
+ /**
+ * Uniform interface to get document id
+ * Used by test and set to retrieve already existing document
+ */
+ virtual const document::DocumentId & getDocumentId() const = 0;
+};
+
+/**
+ * @class PutCommand
+ * @ingroup message
+ *
+ * @brief Command for adding a document to the storage system.
+ */
+class PutCommand : public TestAndSetCommand {
+ document::Document::SP _doc;
+ Timestamp _timestamp;
+ Timestamp _updateTimestamp;
+
+public:
+ PutCommand(const document::BucketId&, const document::Document::SP&,
+ Timestamp);
+
+ void setTimestamp(Timestamp ts) { _timestamp = ts; }
+
+ /**
+ * If set, this PUT will only update the header of an existing document,
+ * rather than writing an entire new PUT. It will only perform the write if
+ * there exists a document already with the given timestamp.
+ */
+ void setUpdateTimestamp(Timestamp ts) { _updateTimestamp = ts; }
+ Timestamp getUpdateTimestamp() const { return _updateTimestamp; }
+
+ const document::Document::SP& getDocument() const { return _doc; }
+ const document::DocumentId& getDocumentId() const override { return _doc->getId(); }
+ Timestamp getTimestamp() const { return _timestamp; }
+
+ uint32_t getMemoryFootprint() const {
+ return (_doc.get() ? 4096 : 0) + 20;
+ }
+
+ vespalib::string getSummary() const override;
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ virtual StorageCommand::UP createCopyToForward(
+ const document::BucketId& bucket, uint64_t timestamp) const;
+
+ DECLARE_STORAGECOMMAND(PutCommand, onPut);
+};
+
+/**
+ * @class PutReply
+ * @ingroup message
+ *
+ * @brief Reply of a put command.
+ */
+class PutReply : public BucketInfoReply {
+ document::DocumentId _docId;
+ document::Document::SP _document; // Not serialized
+ Timestamp _timestamp;
+ Timestamp _updateTimestamp;
+ bool _wasFound;
+
+public:
+ explicit PutReply(const PutCommand& cmd, bool wasFound = true);
+
+ const document::DocumentId& getDocumentId() const { return _docId; }
+ bool hasDocument() const { return _document.get(); }
+ const document::Document::SP& getDocument() const { return _document; }
+ Timestamp getTimestamp() const { return _timestamp; };
+ Timestamp getUpdateTimestamp() const { return _updateTimestamp; }
+
+ bool isHeadersOnlyPut() const { return (_updateTimestamp != 0); }
+ bool wasFound() const { return _wasFound; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(PutReply, onPutReply)
+};
+
+/**
+ * @class UpdateCommand
+ * @ingroup message
+ *
+ * @brief Command for updating a document to the storage system.
+ */
+class UpdateCommand : public TestAndSetCommand {
+ document::DocumentUpdate::SP _update;
+ Timestamp _timestamp;
+ Timestamp _oldTimestamp;
+
+public:
+ UpdateCommand(const document::BucketId&,
+ const document::DocumentUpdate::SP&, Timestamp);
+
+ void setTimestamp(Timestamp ts) { _timestamp = ts; }
+ void setOldTimestamp(Timestamp ts) { _oldTimestamp = ts; }
+
+ const document::DocumentUpdate::SP& getUpdate() const { return _update; }
+ const document::DocumentId& getDocumentId() const override
+ { return _update->getId(); }
+ Timestamp getTimestamp() const { return _timestamp; }
+ Timestamp getOldTimestamp() const { return _oldTimestamp; }
+
+ uint32_t getMemoryFootprint() const {
+ return (_update.get() ? 1024 : 0) + 30;
+ }
+
+ vespalib::string getSummary() const override;
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ virtual StorageCommand::UP createCopyToForward(
+ const document::BucketId& bucket, uint64_t timestamp) const;
+
+ DECLARE_STORAGECOMMAND(UpdateCommand, onUpdate);
+};
+
+/**
+ * @class UpdateReply
+ * @ingroup message
+ *
+ * @brief Reply of a update command.
+ */
+class UpdateReply : public BucketInfoReply {
+ document::DocumentId _docId;
+ Timestamp _timestamp;
+ Timestamp _oldTimestamp;
+ uint16_t _consistentNode;
+
+public:
+ UpdateReply(const UpdateCommand& cmd, Timestamp oldTimestamp = 0);
+
+ void setOldTimestamp(Timestamp ts) { _oldTimestamp = ts; }
+
+ const document::DocumentId& getDocumentId() const { return _docId; }
+ Timestamp getTimestamp() const { return _timestamp; }
+ Timestamp getOldTimestamp() const { return _oldTimestamp; }
+
+ bool wasFound() const { return (_oldTimestamp != 0); }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ /**
+ * If this update was inconsistent (multiple different timestamps returned),
+ * set the "best" node.
+ */
+ void setNodeWithNewestTimestamp(uint16_t node) { _consistentNode = node; }
+
+ uint16_t getNodeWithNewestTimestamp() { return _consistentNode; }
+
+ DECLARE_STORAGEREPLY(UpdateReply, onUpdateReply)
+};
+
+
+/**
+ * @class GetCommand
+ * @ingroup message
+ *
+ * @brief Command for returning a single document.
+ *
+ * Normally, the newest version of a document is retrieved. The timestamp can
+ * be used to retrieve the newest copy, which is not newer than the given
+ * timestamp.
+ */
+class GetCommand : public BucketInfoCommand {
+ document::DocumentId _docId;
+ Timestamp _beforeTimestamp;
+ vespalib::string _fieldSet;
+
+public:
+ GetCommand(const document::BucketId&, const document::DocumentId&,
+ const vespalib::stringref & fieldSet, Timestamp before = MAX_TIMESTAMP);
+
+ void setBeforeTimestamp(Timestamp ts) { _beforeTimestamp = ts; }
+
+ const document::DocumentId& getDocumentId() const { return _docId; }
+ Timestamp getBeforeTimestamp() const { return _beforeTimestamp; }
+ const vespalib::string& getFieldSet() const { return _fieldSet; }
+ void setFieldSet(const vespalib::stringref & fieldSet) { _fieldSet = fieldSet; }
+
+ vespalib::string getSummary() const override;
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ virtual StorageCommand::UP createCopyToForward(
+ const document::BucketId& bucket, uint64_t timestamp) const;
+
+ DECLARE_STORAGECOMMAND(GetCommand, onGet)
+};
+
+/**
+ * @class GetReply
+ * @ingroup message
+ *
+ * @brief Reply for a get command.
+ */
+class GetReply : public BucketInfoReply {
+ document::DocumentId _docId; // In case of not found, we want id still
+ vespalib::string _fieldSet;
+ document::Document::SP _doc; // Null pointer if not found
+ Timestamp _beforeTimestamp;
+ Timestamp _lastModifiedTime;
+
+public:
+ GetReply(const GetCommand& cmd,
+ const document::Document::SP& doc = document::Document::SP(),
+ Timestamp lastModified = 0);
+
+ const document::Document::SP& getDocument() const { return _doc; }
+ const document::DocumentId& getDocumentId() const { return _docId; }
+ const vespalib::string& getFieldSet() const { return _fieldSet; }
+
+ Timestamp getLastModifiedTimestamp() const { return _lastModifiedTime; }
+ Timestamp getBeforeTimestamp() const { return _beforeTimestamp; }
+
+ bool wasFound() const { return (_doc.get() != 0); }
+
+ uint32_t getMemoryFootprint() const {
+ return (_doc.get() ? 4096 : 0) + 30;
+ }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(GetReply, onGetReply)
+};
+
+/**
+ * @class RemoveCommand
+ * @ingroup message
+ *
+ * @brief Command for removing a document.
+ */
+class RemoveCommand : public TestAndSetCommand {
+ document::DocumentId _docId;
+ Timestamp _timestamp;
+
+public:
+ RemoveCommand(const document::BucketId&, const document::DocumentId& docId,
+ Timestamp timestamp);
+
+ void setTimestamp(Timestamp ts) { _timestamp = ts; }
+
+ const document::DocumentId& getDocumentId() const override { return _docId; }
+ Timestamp getTimestamp() const { return _timestamp; }
+
+ vespalib::string getSummary() const override;
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ virtual StorageCommand::UP createCopyToForward(
+ const document::BucketId& bucket, uint64_t timestamp) const;
+
+ DECLARE_STORAGECOMMAND(RemoveCommand, onRemove)
+};
+
+/**
+ * @class RemoveReply
+ * @ingroup message
+ *
+ * @brief Reply for a remove command.
+ */
+class RemoveReply : public BucketInfoReply {
+ document::DocumentId _docId;
+ Timestamp _timestamp;
+ Timestamp _oldTimestamp;
+
+public:
+ explicit RemoveReply(const RemoveCommand& cmd, Timestamp oldTimestamp = 0);
+
+ const document::DocumentId& getDocumentId() const { return _docId; }
+ Timestamp getTimestamp() { return _timestamp; };
+ Timestamp getOldTimestamp() const { return _oldTimestamp; }
+
+ void setOldTimestamp(Timestamp oldTimestamp) { _oldTimestamp = oldTimestamp; }
+
+ bool wasFound() const { return (_oldTimestamp != 0); }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(RemoveReply, onRemoveReply)
+};
+
+/**
+ * @class RevertCommand
+ * @ingroup message
+ *
+ * @brief Command for reverting a write or remove operation.
+ */
+class RevertCommand : public BucketInfoCommand {
+ std::vector<Timestamp> _tokens;
+
+public:
+ RevertCommand(const document::BucketId& bucket,
+ const std::vector<Timestamp>& revertTokens);
+
+ const std::vector<Timestamp>& getRevertTokens() const { return _tokens; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(RevertCommand, onRevert)
+};
+
+/**
+ * @class RevertReply
+ * @ingroup message
+ *
+ * @brief Reply for a revert command.
+ */
+class RevertReply : public BucketInfoReply {
+ std::vector<Timestamp> _tokens;
+
+public:
+ explicit RevertReply(const RevertCommand& cmd);
+
+ const std::vector<Timestamp>& getRevertTokens() const { return _tokens; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(RevertReply, onRevertReply)
+};
+
+} // api
+} // storage
+
+
diff --git a/storageapi/src/vespa/storageapi/message/queryresult.cpp b/storageapi/src/vespa/storageapi/message/queryresult.cpp
new file mode 100644
index 00000000000..0bef270b6ce
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/queryresult.cpp
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/message/queryresult.h>
+
+namespace storage {
+namespace api {
+
+IMPLEMENT_COMMAND(QueryResultCommand, QueryResultReply)
+IMPLEMENT_REPLY(QueryResultReply)
+
+QueryResultCommand::QueryResultCommand()
+ : StorageCommand(MessageType::QUERYRESULT),
+ _searchResult(),
+ _summary()
+{
+}
+
+void
+QueryResultCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "QueryResultCommand(" << _searchResult.getHitCount() << " hits)";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+QueryResultReply::QueryResultReply(const QueryResultCommand& cmd)
+ : StorageReply(cmd)
+{
+}
+
+void
+QueryResultReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "QueryResultReply()";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/message/queryresult.h b/storageapi/src/vespa/storageapi/message/queryresult.h
new file mode 100644
index 00000000000..c0bed1eee54
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/queryresult.h
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vdslib/container/searchresult.h>
+#include <vespa/vdslib/container/documentsummary.h>
+#include <vespa/storageapi/message/visitor.h>
+
+namespace storage {
+namespace api {
+
+/**
+ * @class QueryResultCommand
+ * @ingroup message
+ *
+ * @brief The result of a searchvisitor.
+ */
+class QueryResultCommand : public StorageCommand {
+public:
+ QueryResultCommand();
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ uint32_t getMemoryFootprint() const {
+ return getSearchResult().getSerializedSize() + getDocumentSummary().getSerializedSize();
+ }
+ const vdslib::SearchResult & getSearchResult() const { return _searchResult; }
+ vdslib::SearchResult & getSearchResult() { return _searchResult; }
+ const vdslib::DocumentSummary & getDocumentSummary() const { return _summary; }
+ vdslib::DocumentSummary & getDocumentSummary() { return _summary; }
+
+ DECLARE_STORAGECOMMAND(QueryResultCommand, onQueryResult)
+private:
+ vdslib::SearchResult _searchResult;
+ vdslib::DocumentSummary _summary;
+};
+
+/**
+ * @class QueryResultReply
+ * @ingroup message
+ *
+ * @brief Response to a search result command.
+ */
+class QueryResultReply : public StorageReply {
+public:
+ explicit QueryResultReply(const QueryResultCommand& command);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(QueryResultReply, onQueryResultReply)
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/message/removelocation.cpp b/storageapi/src/vespa/storageapi/message/removelocation.cpp
new file mode 100644
index 00000000000..e0c5d438b6e
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/removelocation.cpp
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/message/removelocation.h>
+
+namespace storage {
+namespace api {
+
+IMPLEMENT_COMMAND(RemoveLocationCommand, RemoveLocationReply)
+IMPLEMENT_REPLY(RemoveLocationReply)
+
+RemoveLocationCommand::RemoveLocationCommand(
+ const vespalib::stringref & documentSelection, const document::BucketId& id)
+ : BucketInfoCommand(MessageType::REMOVELOCATION, id),
+ _documentSelection(documentSelection)
+{
+}
+
+void
+RemoveLocationCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ if (_documentSelection.length()) {
+ out << "Remove selection(" << _documentSelection << "): ";
+ }
+ BucketInfoCommand::print(out, verbose, indent);
+}
+
+RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd)
+ : BucketInfoReply(cmd)
+{
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/message/removelocation.h b/storageapi/src/vespa/storageapi/message/removelocation.h
new file mode 100644
index 00000000000..77e8d42b052
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/removelocation.h
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/storageapi/defs.h>
+#include <vespa/storageapi/messageapi/storagecommand.h>
+#include <vespa/storageapi/messageapi/bucketinforeply.h>
+
+
+namespace storage {
+namespace api {
+
+class RemoveLocationCommand : public BucketInfoCommand
+{
+public:
+ RemoveLocationCommand(const vespalib::stringref & documentSelection,
+ const document::BucketId&);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ uint32_t getMemoryFootprint() const {
+ return _documentSelection.length();
+ }
+
+ const vespalib::string& getDocumentSelection() const { return _documentSelection; }
+
+ DECLARE_STORAGECOMMAND(RemoveLocationCommand, onRemoveLocation);
+
+private:
+ vespalib::string _documentSelection;
+};
+
+class RemoveLocationReply : public BucketInfoReply
+{
+public:
+ RemoveLocationReply(const RemoveLocationCommand& cmd);
+
+ DECLARE_STORAGEREPLY(RemoveLocationReply, onRemoveLocationReply)
+};
+
+}
+}
+
diff --git a/storageapi/src/vespa/storageapi/message/searchresult.cpp b/storageapi/src/vespa/storageapi/message/searchresult.cpp
new file mode 100644
index 00000000000..b344ce8566a
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/searchresult.cpp
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/message/searchresult.h>
+
+using vdslib::SearchResult;
+
+namespace storage {
+namespace api {
+
+IMPLEMENT_COMMAND(SearchResultCommand, SearchResultReply)
+IMPLEMENT_REPLY(SearchResultReply)
+
+SearchResultCommand::SearchResultCommand()
+ : StorageCommand(MessageType::SEARCHRESULT),
+ SearchResult()
+{
+}
+
+void
+SearchResultCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "SearchResultCommand(" << getHitCount() << " hits)";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+SearchResultReply::SearchResultReply(const SearchResultCommand& cmd)
+ : StorageReply(cmd)
+{
+}
+
+void
+SearchResultReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "SearchResultReply()";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/message/searchresult.h b/storageapi/src/vespa/storageapi/message/searchresult.h
new file mode 100644
index 00000000000..ea5a47cf37c
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/searchresult.h
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vdslib/container/searchresult.h>
+#include <vespa/storageapi/message/visitor.h>
+
+namespace storage {
+namespace api {
+
+/**
+ * @class SearchResultCommand
+ * @ingroup message
+ *
+ * @brief The result of a searchvisitor.
+ */
+class SearchResultCommand : public StorageCommand, public vdslib::SearchResult {
+public:
+ SearchResultCommand();
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ uint32_t getMemoryFootprint() const {
+ return getSerializedSize();
+ }
+
+ DECLARE_STORAGECOMMAND(SearchResultCommand, onSearchResult)
+};
+
+/**
+ * @class SearchResultReply
+ * @ingroup message
+ *
+ * @brief Response to a search result command.
+ */
+class SearchResultReply : public StorageReply {
+public:
+ explicit SearchResultReply(const SearchResultCommand& command);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(SearchResultReply, onSearchResultReply)
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/message/stat.cpp b/storageapi/src/vespa/storageapi/message/stat.cpp
new file mode 100644
index 00000000000..e006b95ed16
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/stat.cpp
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/message/stat.h>
+
+namespace storage {
+namespace api {
+
+IMPLEMENT_COMMAND(StatBucketCommand, StatBucketReply)
+IMPLEMENT_REPLY(StatBucketReply)
+IMPLEMENT_COMMAND(GetBucketListCommand, GetBucketListReply)
+IMPLEMENT_REPLY(GetBucketListReply)
+
+StatBucketCommand::StatBucketCommand(const document::BucketId& id,
+ const vespalib::stringref & documentSelection)
+ : BucketCommand(MessageType::STATBUCKET, id),
+ _docSelection(documentSelection)
+{
+}
+
+void
+StatBucketCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "StatBucketCommand(" << getBucketId()
+ << ", selection: " << _docSelection << ")";
+ if (verbose) {
+ out << " : ";
+ BucketCommand::print(out, verbose, indent);
+ }
+}
+
+StorageCommand::UP
+StatBucketCommand::createCopyToForward(
+ const document::BucketId& bucket, uint64_t) const
+{
+ StatBucketCommand::UP cmd(new StatBucketCommand(bucket, _docSelection));
+ return std::move(cmd);
+}
+
+StatBucketReply::StatBucketReply(const StatBucketCommand& cmd,
+ const vespalib::stringref & results)
+ : BucketReply(cmd),
+ _results(results)
+{
+}
+
+void
+StatBucketReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "StatBucketReply(" << getBucketId();
+ if (verbose) {
+ out << ", result: " << _results << ") : ";
+ BucketReply::print(out, verbose, indent);
+ } else {
+ vespalib::string::size_type pos = _results.find('\n');
+ vespalib::string overview;
+ if (pos != vespalib::string::npos) {
+ overview = _results.substr(0, pos) + " ...";
+ } else {
+ overview = _results;
+ }
+ out << ", result: " << overview << ")";
+ }
+}
+
+GetBucketListCommand::GetBucketListCommand(const document::BucketId& id)
+ : BucketCommand(MessageType::GETBUCKETLIST, id)
+{
+}
+
+void
+GetBucketListCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "GetBucketList(" << getBucketId() << ")";
+ if (verbose) {
+ out << " : ";
+ BucketCommand::print(out, verbose, indent);
+ }
+}
+
+GetBucketListReply::GetBucketListReply(const GetBucketListCommand& cmd)
+ : BucketReply(cmd),
+ _buckets()
+{
+}
+
+void
+GetBucketListReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "GetBucketListReply(" << getBucketId() << ", Info on "
+ << _buckets.size() << " buckets)";
+ if (verbose) {
+ out << " : ";
+ BucketReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/message/stat.h b/storageapi/src/vespa/storageapi/message/stat.h
new file mode 100644
index 00000000000..dbad3e9cd7b
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/stat.h
@@ -0,0 +1,118 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/storageapi/messageapi/bucketcommand.h>
+#include <vespa/storageapi/messageapi/bucketreply.h>
+
+namespace storage {
+
+namespace api {
+
+/**
+ * \class StatBucketCommand
+ * \ingroup stat
+ *
+ * \brief Command used to get information about a given bucket..
+ *
+ * Command used by stat to get detailed information about a bucket.
+ */
+class StatBucketCommand : public BucketCommand {
+private:
+ vespalib::string _docSelection;
+
+public:
+ StatBucketCommand(const document::BucketId& bucket,
+ const vespalib::stringref & documentSelection);
+
+ const vespalib::string& getDocumentSelection() const { return _docSelection; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ virtual StorageCommand::UP createCopyToForward(
+ const document::BucketId&, uint64_t timestamp) const;
+
+ DECLARE_STORAGECOMMAND(StatBucketCommand, onStatBucket);
+};
+
+class StatBucketReply : public BucketReply {
+ vespalib::string _results;
+
+public:
+ StatBucketReply(const StatBucketCommand&, const vespalib::stringref & results = "");
+
+ const vespalib::string& getResults() { return _results; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(StatBucketReply, onStatBucketReply)
+};
+
+/**
+ * \class GetBucketListCommand
+ * \ingroup stat
+ *
+ * \brief Command used to find actual buckets related to a given one.
+ *
+ * Command used by stat to query distributor to find actual buckets contained
+ * by the given bucket, or buckets that contain the given bucket. (getAll() call
+ * on bucket database)
+ */
+class GetBucketListCommand : public BucketCommand {
+public:
+ GetBucketListCommand(const document::BucketId& bucket);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(GetBucketListCommand, onGetBucketList);
+};
+
+class GetBucketListReply : public BucketReply {
+public:
+ struct BucketInfo {
+ document::BucketId _bucket;
+ vespalib::string _bucketInformation;
+
+ BucketInfo(const document::BucketId& id,
+ const vespalib::stringref & bucketInformation)
+ : _bucket(id),
+ _bucketInformation(bucketInformation)
+ {
+ }
+
+ bool operator==(const BucketInfo& other) const {
+ return (_bucket == other._bucket
+ && _bucketInformation == other._bucketInformation);
+ }
+ };
+
+private:
+ std::vector<BucketInfo> _buckets;
+
+public:
+ GetBucketListReply(const GetBucketListCommand&);
+
+ std::vector<BucketInfo>& getBuckets() { return _buckets; }
+ const std::vector<BucketInfo>& getBuckets() const { return _buckets; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(GetBucketListReply, onGetBucketListReply)
+
+};
+
+inline std::ostream& operator<<(std::ostream& out,
+ const GetBucketListReply::BucketInfo& instance)
+{
+ out << "BucketInfo(" << instance._bucket << ": "
+ << instance._bucketInformation << ")";
+ return out;
+}
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/message/state.cpp b/storageapi/src/vespa/storageapi/message/state.cpp
new file mode 100644
index 00000000000..5b81ee72aa4
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/state.cpp
@@ -0,0 +1,99 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/message/state.h>
+#include <vespa/storageapi/messageapi/storagemessage.h>
+
+namespace storage {
+namespace api {
+
+IMPLEMENT_COMMAND(GetNodeStateCommand, GetNodeStateReply)
+IMPLEMENT_REPLY(GetNodeStateReply)
+IMPLEMENT_COMMAND(SetSystemStateCommand, SetSystemStateReply)
+IMPLEMENT_REPLY(SetSystemStateReply)
+
+GetNodeStateCommand::GetNodeStateCommand(lib::NodeState::UP expectedState)
+ : StorageCommand(MessageType::GETNODESTATE),
+ _expectedState(std::move(expectedState))
+{
+}
+
+void
+GetNodeStateCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "GetNodeStateCommand(";
+ if (_expectedState.get() != 0) {
+ out << "Expected state: " << *_expectedState;
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+GetNodeStateReply::GetNodeStateReply(const GetNodeStateCommand& cmd)
+ : StorageReply(cmd),
+ _state()
+{
+}
+
+GetNodeStateReply::GetNodeStateReply(const GetNodeStateCommand& cmd,
+ const lib::NodeState& state)
+ : StorageReply(cmd),
+ _state(new lib::NodeState(state))
+{
+}
+
+void
+GetNodeStateReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "GetNodeStateReply(";
+ if (_state.get()) {
+ out << "State: " << *_state;
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+SetSystemStateCommand::SetSystemStateCommand(const lib::ClusterState& state)
+ : StorageCommand(MessageType::SETSYSTEMSTATE),
+ _state(state)
+{
+}
+
+void
+SetSystemStateCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "SetSystemStateCommand(" << _state << ")";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+SetSystemStateReply::SetSystemStateReply(const SetSystemStateCommand& cmd)
+ : StorageReply(cmd),
+ _state(cmd.getSystemState())
+{
+}
+
+void
+SetSystemStateReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "SetSystemStateReply()";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/message/state.h b/storageapi/src/vespa/storageapi/message/state.h
new file mode 100644
index 00000000000..d3120943619
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/state.h
@@ -0,0 +1,108 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @file state.h
+ *
+ * State commands.
+ */
+#pragma once
+
+#include <vespa/storageapi/messageapi/storagecommand.h>
+#include <vespa/storageapi/messageapi/storagereply.h>
+#include <vespa/vdslib/state/clusterstate.h>
+
+namespace storage {
+namespace api {
+
+/**
+ * @class GetNodeStateCommand
+ * @ingroup message
+ *
+ * @brief Command for setting node state. No payload
+ */
+class GetNodeStateCommand : public StorageCommand {
+ lib::NodeState::UP _expectedState;
+
+public:
+ explicit GetNodeStateCommand(lib::NodeState::UP expectedState);
+
+ const lib::NodeState* getExpectedState() const
+ { return _expectedState.get(); }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(GetNodeStateCommand, onGetNodeState)
+};
+
+/**
+ * @class GetNodeStateReply
+ * @ingroup message
+ *
+ * @brief Reply to GetNodeStateCommand
+ */
+class GetNodeStateReply : public StorageReply {
+ lib::NodeState::UP _state;
+ std::string _nodeInfo;
+
+public:
+ GetNodeStateReply(const GetNodeStateCommand&); // Only used on makeReply()
+ GetNodeStateReply(const GetNodeStateCommand&, const lib::NodeState&);
+
+ bool hasNodeState() const { return (_state.get() != 0); }
+ const lib::NodeState& getNodeState() const { return *_state; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ void setNodeInfo(const std::string& info) { _nodeInfo = info; }
+ const std::string& getNodeInfo() const { return _nodeInfo; }
+
+ DECLARE_STORAGEREPLY(GetNodeStateReply, onGetNodeStateReply)
+};
+
+/**
+ * @class SetSystemStateCommand
+ * @ingroup message
+ *
+ * @brief Command for telling a node about the system state - state of each node
+ * in the system and state of the system (all ok, no merging, block
+ * put/get/remove etx)
+ */
+class SetSystemStateCommand : public StorageCommand {
+ lib::ClusterState _state;
+
+public:
+ explicit SetSystemStateCommand(const lib::ClusterState&);
+
+ const lib::ClusterState& getSystemState() const { return _state; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(SetSystemStateCommand, onSetSystemState)
+};
+
+/**
+ * @class SetSystemStateReply
+ * @ingroup message
+ *
+ * @brief Reply received after a SetSystemStateCommand.
+ */
+class SetSystemStateReply : public StorageReply {
+ lib::ClusterState _state;
+
+public:
+ explicit SetSystemStateReply(const SetSystemStateCommand& cmd);
+
+ // Not serialized. Available locally
+ const lib::ClusterState& getSystemState() const { return _state; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(SetSystemStateReply, onSetSystemStateReply)
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp
new file mode 100644
index 00000000000..f2ffab76326
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/visitor.cpp
@@ -0,0 +1,227 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/* $Id$ */
+
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/message/visitor.h>
+
+namespace storage {
+namespace api {
+
+IMPLEMENT_COMMAND(CreateVisitorCommand, CreateVisitorReply)
+IMPLEMENT_REPLY(CreateVisitorReply)
+IMPLEMENT_COMMAND(DestroyVisitorCommand, DestroyVisitorReply)
+IMPLEMENT_REPLY(DestroyVisitorReply)
+IMPLEMENT_COMMAND(VisitorInfoCommand, VisitorInfoReply)
+IMPLEMENT_REPLY(VisitorInfoReply)
+
+CreateVisitorCommand::CreateVisitorCommand(const vespalib::stringref & libraryName,
+ const vespalib::stringref & instanceId,
+ const vespalib::stringref & docSelection)
+ : StorageCommand(MessageType::VISITOR_CREATE),
+ _libName(libraryName),
+ _params(),
+ _controlDestination(),
+ _dataDestination(),
+ _docSelection(docSelection),
+ _buckets(),
+ _fromTime(0),
+ _toTime(api::MAX_TIMESTAMP),
+ _visitorCmdId(getMsgId()),
+ _instanceId(instanceId),
+ _visitorId(0),
+ _visitRemoves(false),
+ _fieldSet("[all]"),
+ _visitInconsistentBuckets(false),
+ _queueTimeout(2000),
+ _maxPendingReplyCount(2),
+ _version(50),
+ _ordering(document::OrderingSpecification::ASCENDING),
+ _maxBucketsPerVisitor(1)
+{
+}
+
+CreateVisitorCommand::CreateVisitorCommand(const CreateVisitorCommand& o)
+ : StorageCommand(o),
+ _libName(o._libName),
+ _params(o._params),
+ _controlDestination(o._controlDestination),
+ _dataDestination(o._dataDestination),
+ _docSelection(o._docSelection),
+ _buckets(o._buckets),
+ _fromTime(o._fromTime),
+ _toTime(o._toTime),
+ _visitorCmdId(getMsgId()),
+ _instanceId(o._instanceId),
+ _visitorId(o._visitorId),
+ _visitRemoves(o._visitRemoves),
+ _fieldSet(o._fieldSet),
+ _visitInconsistentBuckets(o._visitInconsistentBuckets),
+ _queueTimeout(o._queueTimeout),
+ _maxPendingReplyCount(o._maxPendingReplyCount),
+ _version(o._version),
+ _ordering(o._ordering),
+ _maxBucketsPerVisitor(o._maxBucketsPerVisitor)
+{
+}
+
+StorageCommand::UP
+CreateVisitorCommand::createCopyToForward(
+ const document::BucketId& bucket, uint64_t) const
+{
+ CreateVisitorCommand::UP cmd(new CreateVisitorCommand(*this));
+ cmd->_buckets.clear();
+ cmd->_buckets.push_back(bucket);
+ return std::move(cmd);
+}
+
+void
+CreateVisitorCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "CreateVisitorCommand(" << _libName << ", " << _docSelection;
+ if (verbose) {
+ out << ") {";
+ out << "\n" << indent << " Library name: '" << _libName << "'";
+ out << "\n" << indent << " Instance Id: '" << _instanceId << "'";
+ out << "\n" << indent << " Control Destination: '"
+ << _controlDestination << "'";
+ out << "\n" << indent << " Data Destination: '"
+ << _dataDestination << "'";
+ out << "\n" << indent << " Doc Selection: '" << _docSelection << "'";
+ out << "\n" << indent << " Max pending: '"
+ << _maxPendingReplyCount << "'";
+ out << "\n" << indent << " Timeout: " << getTimeout();
+ out << "\n" << indent << " Queue timeout: " << _queueTimeout << " ms";
+ out << "\n" << indent << " VisitorDispatcher version: '" << _version << "'";
+ if (visitRemoves()) {
+ out << "\n" << indent << " Visiting remove entries too";
+ }
+
+ out << "\n" << indent << " Returning fields: " << _fieldSet;
+
+ if (visitInconsistentBuckets()) {
+ out << "\n" << indent << " Visiting inconsistent buckets";
+ }
+ out << "\n" << indent << " From " << _fromTime << " to " << _toTime;
+ for (std::vector<document::BucketId>::const_iterator it
+ = _buckets.begin(); it != _buckets.end(); ++it)
+ {
+ out << "\n" << indent << " " << (*it);
+ }
+ out << "\n" << indent << " ";
+ _params.print(out, verbose, indent + " ");
+ out << "\n" << indent << " " << "ordering(" << _ordering << ")";
+ out << "\n" << indent << " Max buckets: '"
+ << _maxBucketsPerVisitor << "'";
+ out << "\n" << indent << "} : ";
+ StorageCommand::print(out, verbose, indent);
+ } else if (_buckets.size() == 2) {
+ out << ", top " << _buckets[0] << ", progress " << _buckets[1] << ")";
+ } else {
+ out << ", " << _buckets.size() << " buckets)";
+ }
+
+}
+
+CreateVisitorReply::CreateVisitorReply(const CreateVisitorCommand& cmd)
+ : StorageReply(cmd),
+ _lastBucket(document::BucketId(INT_MAX))
+{
+}
+
+void
+CreateVisitorReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "CreateVisitorReply(last=" << _lastBucket << ")";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+DestroyVisitorCommand::DestroyVisitorCommand(const vespalib::stringref & instanceId)
+ : StorageCommand(MessageType::VISITOR_DESTROY),
+ _instanceId(instanceId)
+{
+}
+
+void
+DestroyVisitorCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "DestroyVisitorCommand(" << _instanceId << ")";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+DestroyVisitorReply::DestroyVisitorReply(const DestroyVisitorCommand& cmd)
+ : StorageReply(cmd)
+{
+}
+
+void
+DestroyVisitorReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "DestroyVisitorReply()";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+VisitorInfoCommand::VisitorInfoCommand()
+ : StorageCommand(MessageType::VISITOR_INFO),
+ _completed(false),
+ _bucketsCompleted(),
+ _error(ReturnCode::OK)
+{
+}
+
+void
+VisitorInfoCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "VisitorInfoCommand(";
+ if (_completed) { out << "completed"; }
+ if (_error.failed()) {
+ out << _error;
+ }
+ if (verbose) {
+ out << ") : ";
+ StorageCommand::print(out, verbose, indent);
+ } else {
+ if (!_bucketsCompleted.empty()) {
+ out << _bucketsCompleted.size() << " buckets completed";
+ }
+ out << ")";
+ }
+}
+
+VisitorInfoReply::VisitorInfoReply(const VisitorInfoCommand& cmd)
+ : StorageReply(cmd),
+ _completed(cmd.visitorCompleted())
+{
+}
+
+void
+VisitorInfoReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "VisitorInfoReply(";
+ if (_completed) { out << "completed"; }
+ if (verbose) {
+ out << ") : ";
+ StorageReply::print(out, verbose, indent);
+ } else {
+ out << ")";
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/message/visitor.h b/storageapi/src/vespa/storageapi/message/visitor.h
new file mode 100644
index 00000000000..10e7aa65e40
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/message/visitor.h
@@ -0,0 +1,269 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @file storageapi/message/visitor.h
+ *
+ * Messages related to visitors, used by the visitor manager.
+ */
+
+#pragma once
+
+#include <vespa/storageapi/defs.h>
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/vdslib/container/parameters.h>
+#include <vespa/vdslib/container/visitorstatistics.h>
+#include <vespa/storageapi/messageapi/storagecommand.h>
+#include <vespa/storageapi/messageapi/storagereply.h>
+#include <vespa/document/select/orderingspecification.h>
+
+namespace storage {
+namespace api {
+
+/**
+ * @class CreateVisitorCommand
+ * @ingroup message
+ *
+ * @brief Command for creating a visitor.
+ */
+class CreateVisitorCommand : public StorageCommand {
+private:
+ vespalib::string _libName; // Name of visitor library to use, ie. DumpVisitor.so
+ vdslib::Parameters _params;
+
+ vespalib::string _controlDestination;
+ vespalib::string _dataDestination;
+
+ vespalib::string _docSelection;
+ std::vector<document::BucketId> _buckets;
+ Timestamp _fromTime;
+ Timestamp _toTime;
+
+ uint32_t _visitorCmdId;
+ vespalib::string _instanceId;
+ VisitorId _visitorId; // Set on storage node
+
+ bool _visitRemoves;
+ vespalib::string _fieldSet;
+ bool _visitInconsistentBuckets;
+
+ uint32_t _queueTimeout;
+ uint32_t _maxPendingReplyCount;
+ uint32_t _version;
+
+ document::OrderingSpecification::Order _ordering;
+ uint32_t _maxBucketsPerVisitor;
+
+public:
+ CreateVisitorCommand(const vespalib::stringref & libraryName,
+ const vespalib::stringref & instanceId,
+ const vespalib::stringref & docSelection);
+
+ /** Create another command with similar visitor settings. */
+ CreateVisitorCommand(const CreateVisitorCommand& template_);
+
+ void setVisitorCmdId(uint32_t id) { _visitorCmdId = id; }
+ void setControlDestination(const vespalib::stringref & d)
+ { _controlDestination = d; }
+ void setDataDestination(const vespalib::stringref & d) { _dataDestination = d; }
+ void setParameters(const vdslib::Parameters& params) { _params = params; }
+ void setMaximumPendingReplyCount(uint32_t count)
+ { _maxPendingReplyCount = count; }
+ void setFieldSet(const vespalib::stringref& fieldSet)
+ { _fieldSet = fieldSet; }
+ void setVisitRemoves(bool value = true) { _visitRemoves = value; }
+ void setVisitInconsistentBuckets(bool visitInconsistent = true)
+ { _visitInconsistentBuckets = visitInconsistent; }
+ void addBucketToBeVisited(const document::BucketId& id)
+ { _buckets.push_back(id); }
+ void setVisitorId(const VisitorId id) { _visitorId = id; }
+ void setInstanceId(const vespalib::stringref & id) { _instanceId = id; }
+ void setQueueTimeout(uint32_t milliSecs) { _queueTimeout = milliSecs; }
+ void setFromTime(Timestamp ts) { _fromTime = ts; }
+ void setToTime(Timestamp ts) { _toTime = ts; }
+
+ VisitorId getVisitorId() const { return _visitorId; }
+ uint32_t getVisitorCmdId() const { return _visitorCmdId; }
+ const vespalib::string & getLibraryName() const { return _libName; }
+ const vespalib::string & getInstanceId() const { return _instanceId; }
+ const vespalib::string & getControlDestination() const
+ { return _controlDestination; }
+ const vespalib::string & getDataDestination() const { return _dataDestination; }
+ const vespalib::string & getDocumentSelection() const { return _docSelection; }
+ const vdslib::Parameters& getParameters() const { return _params; }
+ vdslib::Parameters& getParameters() { return _params; }
+ uint32_t getMaximumPendingReplyCount() const
+ { return _maxPendingReplyCount; }
+ const std::vector<document::BucketId>& getBuckets() const
+ { return _buckets; }
+ Timestamp getFromTime() const { return _fromTime; }
+ Timestamp getToTime() const { return _toTime; }
+ std::vector<document::BucketId>& getBuckets() { return _buckets; }
+ bool visitRemoves() const { return _visitRemoves; }
+ const vespalib::string& getFieldSet() const { return _fieldSet; }
+ bool visitInconsistentBuckets() const { return _visitInconsistentBuckets; }
+ // In millisec
+ uint32_t getQueueTimeout() const { return _queueTimeout; }
+
+ void setVisitorDispatcherVersion(uint32_t version) { _version = version; }
+ uint32_t getVisitorDispatcherVersion() const { return _version; }
+
+ void setVisitorOrdering(document::OrderingSpecification::Order ordering) { _ordering = ordering; }
+ document::OrderingSpecification::Order getVisitorOrdering() const { return _ordering; }
+ void setMaxBucketsPerVisitor(uint32_t max)
+ { _maxBucketsPerVisitor = max; }
+ uint32_t getMaxBucketsPerVisitor() const
+ { return _maxBucketsPerVisitor; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ virtual StorageCommand::UP createCopyToForward(
+ const document::BucketId&, uint64_t timestamp) const;
+
+ DECLARE_STORAGECOMMAND(CreateVisitorCommand, onCreateVisitor)
+};
+
+/**
+ * @class CreateVisitorReply
+ * @ingroup message
+ *
+ * @brief Response to a create visitor command.
+ */
+class CreateVisitorReply : public StorageReply {
+private:
+ document::BucketId _lastBucket;
+ vdslib::VisitorStatistics _visitorStatistics;
+
+public:
+ explicit CreateVisitorReply(const CreateVisitorCommand& cmd);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ void setLastBucket(const document::BucketId& lastBucket) { _lastBucket = lastBucket; }
+
+ const document::BucketId& getLastBucket() const { return _lastBucket; }
+
+ void setVisitorStatistics(const vdslib::VisitorStatistics& stats) { _visitorStatistics = stats; }
+
+ const vdslib::VisitorStatistics& getVisitorStatistics() const { return _visitorStatistics; }
+
+ DECLARE_STORAGEREPLY(CreateVisitorReply, onCreateVisitorReply)
+};
+
+/**
+ * @class DestroyVisitorCommand
+ * @ingroup message
+ *
+ * @brief Command for removing a visitor.
+ */
+class DestroyVisitorCommand : public StorageCommand {
+private:
+ vespalib::string _instanceId;
+
+public:
+ explicit DestroyVisitorCommand(const vespalib::stringref & instanceId);
+
+ const vespalib::string & getInstanceId() const { return _instanceId; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(DestroyVisitorCommand, onDestroyVisitor)
+};
+
+/**
+ * @class DestroyVisitorReply
+ * @ingroup message
+ *
+ * @brief Response to a destroy visitor command.
+ */
+class DestroyVisitorReply : public StorageReply {
+public:
+ explicit DestroyVisitorReply(const DestroyVisitorCommand& cmd);
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(DestroyVisitorReply, onDestroyVisitorReply)
+};
+
+/**
+ * @class VisitorInfoCommand
+ * @ingroup message
+ *
+ * @brief Sends status information of an ongoing visitor.
+ *
+ * Includes three different kinds of data.
+ * - Notification when visiting is complete.
+ * - Notification when individual buckets have been completely visited.
+ * (Including the timestamp of the newest document visited)
+ * - Notification that some error condition arose during visiting.
+ */
+class VisitorInfoCommand : public StorageCommand {
+public:
+ struct BucketTimestampPair {
+ document::BucketId bucketId;
+ Timestamp timestamp;
+
+ BucketTimestampPair() : bucketId(), timestamp(0) {}
+ BucketTimestampPair(const document::BucketId& bucket,
+ const Timestamp& ts)
+ : bucketId(bucket), timestamp(ts) {}
+
+ bool operator==(const BucketTimestampPair& other) const {
+ return (bucketId == other.bucketId && timestamp && other.timestamp);
+ }
+ };
+
+private:
+ bool _completed;
+ std::vector<BucketTimestampPair> _bucketsCompleted;
+ ReturnCode _error;
+
+public:
+ VisitorInfoCommand();
+
+ void setErrorCode(const ReturnCode& code) { _error = code; }
+ void setCompleted() { _completed = true; }
+ void setBucketCompleted(const document::BucketId& id, Timestamp lastVisited)
+ {
+ _bucketsCompleted.push_back(BucketTimestampPair(id, lastVisited));
+ }
+ void setBucketsCompleted(const std::vector<BucketTimestampPair>& bc)
+ { _bucketsCompleted = bc; }
+
+ const ReturnCode& getErrorCode() const { return _error; }
+ const std::vector<BucketTimestampPair>& getCompletedBucketsList() const
+ { return _bucketsCompleted; }
+ bool visitorCompleted() const { return _completed; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGECOMMAND(VisitorInfoCommand, onVisitorInfo)
+};
+
+inline std::ostream&
+operator<<(std::ostream& out,
+ const VisitorInfoCommand::BucketTimestampPair& pair)
+{
+ return out << pair.bucketId << " - " << pair.timestamp;
+}
+
+class VisitorInfoReply : public StorageReply {
+ bool _completed;
+
+public:
+ VisitorInfoReply(const VisitorInfoCommand& cmd);
+
+ bool visitorCompleted() const { return _completed; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ DECLARE_STORAGEREPLY(VisitorInfoReply, onVisitorInfoReply)
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/messageapi/.gitignore b/storageapi/src/vespa/storageapi/messageapi/.gitignore
new file mode 100644
index 00000000000..6e555686642
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/.gitignore
@@ -0,0 +1,6 @@
+*.lo
+.depend
+.depend.NEW
+.deps
+.libs
+Makefile
diff --git a/storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt b/storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt
new file mode 100644
index 00000000000..7b272d672ed
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(storageapi_messageapi OBJECT
+ SOURCES
+ returncode.cpp
+ storagemessage.cpp
+ storagecommand.cpp
+ storagereply.cpp
+ bucketcommand.cpp
+ bucketreply.cpp
+ bucketinfocommand.cpp
+ bucketinforeply.cpp
+ DEPENDS
+)
diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp
new file mode 100644
index 00000000000..465833a7abf
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/messageapi/bucketcommand.h>
+
+namespace storage {
+namespace api {
+
+void
+BucketCommand::print(std::ostream& out,
+ bool verbose, const std::string& indent) const
+{
+ out << "BucketCommand(" << _bucket;
+ if (hasBeenRemapped()) {
+ out << " <- " << _originalBucket;
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ StorageCommand::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketcommand.h b/storageapi/src/vespa/storageapi/messageapi/bucketcommand.h
new file mode 100644
index 00000000000..f4f5909fccd
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/bucketcommand.h
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class storage::api::BucketCommand
+ * @ingroup messageapi
+ *
+ * @brief Superclass for storage commands that operate towards a single bucket.
+ */
+
+#pragma once
+
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/storageapi/messageapi/storagecommand.h>
+
+namespace storage {
+namespace api {
+
+class BucketCommand : public StorageCommand {
+ document::BucketId _bucket;
+ document::BucketId _originalBucket;
+
+protected:
+ BucketCommand(const MessageType& type, const document::BucketId& id)
+ : StorageCommand(type), _bucket(id), _originalBucket() {}
+
+public:
+ DECLARE_POINTER_TYPEDEFS(BucketCommand);
+
+ void remapBucketId(const document::BucketId& bucket) {
+ if (_originalBucket.getRawId() == 0) _originalBucket = _bucket;
+ _bucket = bucket;
+ }
+
+ document::BucketId getBucketId() const { return _bucket; }
+ bool hasBeenRemapped() const { return (_originalBucket.getRawId() != 0); }
+ const document::BucketId& getOriginalBucketId() const
+ { return _originalBucket; }
+
+ virtual void print(std::ostream& out,
+ bool verbose, const std::string& indent) const;
+
+ virtual bool hasSingleBucketId() const { return true; }
+
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp
new file mode 100644
index 00000000000..feb03e58c71
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/messageapi/bucketinfocommand.h>
+
+#include <vespa/storageapi/message/persistence.h>
+
+namespace storage {
+namespace api {
+
+void
+BucketInfoCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "BucketInfoCommand()";
+ if (verbose) {
+ out << " : ";
+ BucketCommand::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h b/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h
new file mode 100644
index 00000000000..4579d83cfed
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class storage::api::BucketInfoCommand
+ * @ingroup messageapi
+ *
+ * @brief Superclass for storage commands that returns bucket info.
+ *
+ * This class doesn't add any functionality now, other than being able to check
+ * if a message is an instance of this class. But we want commands and replies
+ * to be in the same inheritance structure, and the reply adds functionality.
+ */
+
+#pragma once
+
+#include <vespa/storageapi/messageapi/bucketcommand.h>
+
+namespace storage {
+namespace api {
+
+class BucketInfoCommand : public BucketCommand {
+protected:
+ BucketInfoCommand(const MessageType& type, const document::BucketId& id)
+ : BucketCommand(type, id) {}
+
+public:
+ DECLARE_POINTER_TYPEDEFS(BucketInfoCommand);
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp
new file mode 100644
index 00000000000..99736e7eaef
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/messageapi/bucketinforeply.h>
+
+namespace storage {
+namespace api {
+
+BucketInfoReply::BucketInfoReply(const BucketInfoCommand& cmd,
+ const ReturnCode& code)
+ : BucketReply(cmd, code),
+ _result()
+{
+}
+
+void
+BucketInfoReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "BucketInfoReply(" << _result << ")";
+ if (verbose) {
+ out << " : ";
+ BucketReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h b/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h
new file mode 100644
index 00000000000..0a5c435ade3
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class storage::api::BucketInfoReply
+ * @ingroup messageapi
+ *
+ * @brief Superclass for storage replies which returns bucket info in reply.
+ *
+ * A bucket info reply contains information about the state of a bucket. This
+ * can be altered from before the operation if this was a write operation or if
+ * the bucket was repaired in the process.
+ */
+
+#pragma once
+
+#include <vespa/storageapi/buckets/bucketinfo.h>
+#include <vespa/storageapi/messageapi/bucketreply.h>
+#include <vespa/storageapi/messageapi/bucketinfocommand.h>
+
+namespace storage {
+namespace api {
+
+class BucketInfoReply : public BucketReply {
+ BucketInfo _result;
+
+protected:
+ BucketInfoReply(const BucketInfoCommand& cmd,
+ const ReturnCode& code = ReturnCode(ReturnCode::OK));
+
+public:
+ DECLARE_POINTER_TYPEDEFS(BucketInfoReply);
+
+ const BucketInfo& getBucketInfo() const { return _result; };
+ void setBucketInfo(const BucketInfo& info) { _result = info; }
+
+ /** Overload this to get more descriptive message output. */
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp b/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp
new file mode 100644
index 00000000000..573d0e9fbbb
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/messageapi/bucketreply.h>
+
+#include <vespa/storageapi/messageapi/bucketcommand.h>
+
+namespace storage {
+namespace api {
+
+BucketReply::BucketReply(const BucketCommand& cmd,
+ const ReturnCode& code)
+ : StorageReply(cmd, code),
+ _bucket(cmd.getBucketId()),
+ _originalBucket(cmd.getOriginalBucketId())
+{
+}
+
+void
+BucketReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ out << "BucketReply(" << _bucket;
+ if (hasBeenRemapped()) {
+ out << " <- " << _originalBucket;
+ }
+ out << ")";
+ if (verbose) {
+ out << " : ";
+ StorageReply::print(out, verbose, indent);
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketreply.h b/storageapi/src/vespa/storageapi/messageapi/bucketreply.h
new file mode 100644
index 00000000000..d2605604568
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/bucketreply.h
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class storage::api::BucketReply
+ * @ingroup messageapi
+ *
+ * @brief Superclass for storage replies which operates on single bucket.
+ */
+
+#pragma once
+
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/storageapi/messageapi/storagereply.h>
+
+namespace storage {
+namespace api {
+
+class BucketCommand;
+
+class BucketReply : public StorageReply {
+ document::BucketId _bucket;
+ document::BucketId _originalBucket;
+
+protected:
+ BucketReply(const BucketCommand& cmd,
+ const ReturnCode& code = ReturnCode(ReturnCode::OK));
+
+public:
+ DECLARE_POINTER_TYPEDEFS(BucketReply);
+
+ document::BucketId getBucketId() const { return _bucket; }
+ virtual bool hasSingleBucketId() const { return true; }
+
+ bool hasBeenRemapped() const { return (_originalBucket.getRawId() != 0); }
+ const document::BucketId& getOriginalBucketId() const
+ { return _originalBucket; }
+
+ /** The deserialization code need access to set the remapping. */
+ void remapBucketId(const document::BucketId& bucket) {
+ if (_originalBucket.getRawId() == 0) _originalBucket = _bucket;
+ _bucket = bucket;
+ }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h b/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h
new file mode 100644
index 00000000000..0687200f9fb
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/storageapi/messageapi/bucketinfocommand.h>
+
+namespace storage {
+namespace api {
+
+class MaintenanceCommand : public BucketInfoCommand
+{
+public:
+ MaintenanceCommand(const MessageType& type, const document::BucketId& id)
+ : BucketInfoCommand(type, id) {}
+
+ const vespalib::string& getReason() const { return _reason; };
+
+ void setReason(const vespalib::stringref & reason) { _reason = reason; };
+
+protected:
+ vespalib::string _reason;
+};
+
+}
+
+}
+
diff --git a/storageapi/src/vespa/storageapi/messageapi/messagehandler.h b/storageapi/src/vespa/storageapi/messageapi/messagehandler.h
new file mode 100644
index 00000000000..66873dfba27
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/messagehandler.h
@@ -0,0 +1,367 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class storage::api::MessageHandler
+ * @ingroup messageapi
+ *
+ * @brief Class to prevent manual casting and switches of message types.
+ *
+ * MessageHandler defines an interface for processing StorageMessage objects
+ * of various subclasses.
+ *
+ * @version $Id$
+ */
+
+#pragma once
+
+
+namespace storage {
+namespace api {
+
+// Commands
+
+class GetCommand; // Retrieve document
+class PutCommand; // Add document
+class UpdateCommand; // Update document
+class RemoveCommand; // Remove document
+class RevertCommand; // Revert put/remove operation
+class MultiOperationCommand; // Multi put/remove/update operation
+class BatchPutRemoveCommand;
+class BatchDocumentUpdateCommand;
+
+class CreateVisitorCommand; // Create a new visitor
+class DestroyVisitorCommand; // Destroy a running visitor
+class VisitorInfoCommand; // Sends visitor info to visitor controller
+class DocBlockCommand; // A block of documents visited
+class MapVisitorCommand;
+class SearchResultCommand;
+class DocumentSummaryCommand;
+class QueryResultCommand;
+
+class InternalCommand;
+
+class CreateBucketCommand;
+class DeleteBucketCommand;
+class MergeBucketCommand;
+class GetBucketDiffCommand;
+class ApplyBucketDiffCommand;
+class SplitBucketCommand;
+class JoinBucketsCommand;
+class SetBucketStateCommand;
+
+class RequestBucketInfoCommand;
+class NotifyBucketChangeCommand;
+class SetNodeStateCommand;
+class GetNodeStateCommand;
+class SetSystemStateCommand;
+class GetSystemStateCommand;
+class GetBucketNodesCommand;
+class BucketsAddedCommand;
+class BucketsRemovedCommand;
+
+// Replies
+
+class GetReply;
+class PutReply;
+class UpdateReply;
+class RemoveReply;
+class RevertReply;
+class MultiOperationReply;
+class BatchPutRemoveReply;
+class BatchDocumentUpdateReply;
+
+class CreateVisitorReply;
+class DestroyVisitorReply;
+class VisitorInfoReply;
+class DocBlockReply;
+class MapVisitorReply;
+class SearchResultReply;
+class DocumentSummaryReply;
+class QueryResultReply;
+
+class InternalReply;
+
+class CreateBucketReply;
+class DeleteBucketReply;
+class MergeBucketReply;
+class GetBucketDiffReply;
+class ApplyBucketDiffReply;
+class SplitBucketReply;
+class JoinBucketsReply;
+class SetBucketStateReply;
+
+class RequestBucketInfoReply;
+class NotifyBucketChangeReply;
+class SetNodeStateReply;
+class GetNodeStateReply;
+class SetSystemStateReply;
+class GetSystemStateReply;
+class GetBucketNodesReply;
+class BucketsAddedReply;
+class BucketsRemovedReply;
+
+class StatBucketCommand;
+class StatBucketReply;
+class GetBucketListCommand;
+class GetBucketListReply;
+
+class DocumentListCommand;
+class DocumentListReply;
+
+class EmptyBucketsCommand;
+class EmptyBucketsReply;
+
+class RemoveLocationCommand;
+class RemoveLocationReply;
+
+#define _INTERNAL_DEF_ON_MC(m, c) virtual bool m(const std::shared_ptr<storage::api::c> & )
+#define _INTERNAL_DEF_IMPL_ON_MC(m, c) virtual bool m(const std::shared_ptr<storage::api::c> & ) { return false; }
+#define _INTERNAL_IMPL_ON_MC(cl, m, c, p) bool cl::m(const std::shared_ptr<storage::api::c> & p)
+#define DEF_IMPL_MSG_COMMAND_H(m) _INTERNAL_DEF_IMPL_ON_MC(on##m, m##Command)
+#define DEF_IMPL_MSG_REPLY_H(m) _INTERNAL_DEF_IMPL_ON_MC(on##m##Reply, m##Reply)
+#define DEF_MSG_COMMAND_H(m) _INTERNAL_DEF_ON_MC(on##m, m##Command)
+#define DEF_MSG_REPLY_H(m) _INTERNAL_DEF_ON_MC(on##m##Reply, m##Reply)
+#define IMPL_MSG_COMMAND_ARG_H(cl, m, p) _INTERNAL_IMPL_ON_MC(cl, on##m, m##Command, p)
+#define IMPL_MSG_REPLY_ARG_H(cl, m, p) _INTERNAL_IMPL_ON_MC(cl, on##m##Reply, m##Reply, p)
+#define IMPL_MSG_COMMAND_H(cl, m) IMPL_MSG_COMMAND_ARG_H(cl, m, cmd)
+#define IMPL_MSG_REPLY_H(cl, m) IMPL_MSG_REPLY_ARG_H(cl, m, reply)
+#define ON_M(m) DEF_IMPL_MSG_COMMAND_H(m); DEF_IMPL_MSG_REPLY_H(m)
+
+class MessageHandler {
+public:
+ // Basic operations
+ virtual bool onGet(const std::shared_ptr<api::GetCommand>&)
+ { return false; }
+ virtual bool onGetReply(const std::shared_ptr<api::GetReply>&)
+ { return false; }
+ virtual bool onPut(const std::shared_ptr<api::PutCommand>&)
+ { return false; }
+ virtual bool onPutReply(const std::shared_ptr<api::PutReply>&)
+ { return false; }
+ virtual bool onUpdate(const std::shared_ptr<api::UpdateCommand>&)
+ { return false; }
+ virtual bool onUpdateReply(const std::shared_ptr<api::UpdateReply>&)
+ { return false; }
+ virtual bool onRemove(const std::shared_ptr<api::RemoveCommand>&)
+ { return false; }
+ virtual bool onRemoveReply(const std::shared_ptr<api::RemoveReply>&)
+ { return false; }
+ virtual bool onRevert(const std::shared_ptr<api::RevertCommand>&)
+ { return false; }
+ virtual bool onRevertReply(const std::shared_ptr<api::RevertReply>&)
+ { return false; }
+ virtual bool onMultiOperation(
+ const std::shared_ptr<api::MultiOperationCommand>&)
+ { return false; }
+ virtual bool onMultiOperationReply(
+ const std::shared_ptr<api::MultiOperationReply>&)
+ { return false; }
+ virtual bool onBatchPutRemove(
+ const std::shared_ptr<api::BatchPutRemoveCommand>&)
+ { return false; }
+ virtual bool onBatchPutRemoveReply(
+ const std::shared_ptr<api::BatchPutRemoveReply>&)
+ { return false; }
+ virtual bool onBatchDocumentUpdate(
+ const std::shared_ptr<api::BatchDocumentUpdateCommand>&)
+ { return false; }
+ virtual bool onBatchDocumentUpdateReply(
+ const std::shared_ptr<api::BatchDocumentUpdateReply>&)
+ { return false; }
+
+ // Visiting
+ virtual bool onCreateVisitor(
+ const std::shared_ptr<api::CreateVisitorCommand>&)
+ { return false; }
+ virtual bool onCreateVisitorReply(
+ const std::shared_ptr<api::CreateVisitorReply>&)
+ { return false; }
+ virtual bool onDestroyVisitor(
+ const std::shared_ptr<api::DestroyVisitorCommand>&)
+ { return false; }
+ virtual bool onDestroyVisitorReply(
+ const std::shared_ptr<api::DestroyVisitorReply>&)
+ { return false; }
+ virtual bool onVisitorInfo(
+ const std::shared_ptr<api::VisitorInfoCommand>&)
+ { return false; }
+ virtual bool onVisitorInfoReply(
+ const std::shared_ptr<api::VisitorInfoReply>&)
+ { return false; }
+ virtual bool onDocBlock(
+ const std::shared_ptr<api::DocBlockCommand>&)
+ { return false; }
+ virtual bool onDocBlockReply(
+ const std::shared_ptr<api::DocBlockReply>&)
+ { return false; }
+ virtual bool onMapVisitor(
+ const std::shared_ptr<api::MapVisitorCommand>&)
+ { return false; }
+ virtual bool onMapVisitorReply(
+ const std::shared_ptr<api::MapVisitorReply>&)
+ { return false; }
+ virtual bool onSearchResult(
+ const std::shared_ptr<api::SearchResultCommand>&)
+ { return false; }
+ virtual bool onSearchResultReply(
+ const std::shared_ptr<api::SearchResultReply>&)
+ { return false; }
+ virtual bool onQueryResult(
+ const std::shared_ptr<api::QueryResultCommand>&)
+ { return false; }
+ virtual bool onQueryResultReply(
+ const std::shared_ptr<api::QueryResultReply>&)
+ { return false; }
+ virtual bool onDocumentSummary(
+ const std::shared_ptr<api::DocumentSummaryCommand>&)
+ { return false; }
+ virtual bool onDocumentSummaryReply(
+ const std::shared_ptr<api::DocumentSummaryReply>&)
+ { return false; }
+ virtual bool onDocumentList(
+ const std::shared_ptr<api::DocumentListCommand>&)
+ { return false; }
+ virtual bool onDocumentListReply(
+ const std::shared_ptr<api::DocumentListReply>&)
+ { return false; }
+ virtual bool onEmptyBuckets(
+ const std::shared_ptr<api::EmptyBucketsCommand>&)
+ { return false; }
+ virtual bool onEmptyBucketsReply(
+ const std::shared_ptr<api::EmptyBucketsReply>&)
+ { return false; }
+
+
+ virtual bool onInternal(const std::shared_ptr<api::InternalCommand>&)
+ { return false; }
+ virtual bool onInternalReply(
+ const std::shared_ptr<api::InternalReply>&)
+ { return false; }
+
+ virtual bool onCreateBucket(
+ const std::shared_ptr<api::CreateBucketCommand>&)
+ { return false; }
+ virtual bool onCreateBucketReply(
+ const std::shared_ptr<api::CreateBucketReply>&)
+ { return false; }
+ virtual bool onDeleteBucket(
+ const std::shared_ptr<api::DeleteBucketCommand>&)
+ { return false; }
+ virtual bool onDeleteBucketReply(
+ const std::shared_ptr<api::DeleteBucketReply>&)
+ { return false; }
+ virtual bool onMergeBucket(
+ const std::shared_ptr<api::MergeBucketCommand>&)
+ { return false; }
+ virtual bool onMergeBucketReply(
+ const std::shared_ptr<api::MergeBucketReply>&)
+ { return false; }
+ virtual bool onGetBucketDiff(
+ const std::shared_ptr<api::GetBucketDiffCommand>&)
+ { return false; }
+ virtual bool onGetBucketDiffReply(
+ const std::shared_ptr<api::GetBucketDiffReply>&)
+ { return false; }
+ virtual bool onApplyBucketDiff(
+ const std::shared_ptr<api::ApplyBucketDiffCommand>&)
+ { return false; }
+ virtual bool onApplyBucketDiffReply(
+ const std::shared_ptr<api::ApplyBucketDiffReply>&)
+ { return false; }
+ virtual bool onSplitBucket(
+ const std::shared_ptr<api::SplitBucketCommand>&)
+ { return false; }
+ virtual bool onSplitBucketReply(
+ const std::shared_ptr<api::SplitBucketReply>&)
+ { return false; }
+ virtual bool onJoinBuckets(
+ const std::shared_ptr<api::JoinBucketsCommand>&)
+ { return false; }
+ virtual bool onJoinBucketsReply(
+ const std::shared_ptr<api::JoinBucketsReply>&)
+ { return false; }
+ virtual bool onSetBucketState(
+ const std::shared_ptr<api::SetBucketStateCommand>&)
+ { return false; }
+ virtual bool onSetBucketStateReply(
+ const std::shared_ptr<api::SetBucketStateReply>&)
+ { return false; }
+
+ virtual bool onRequestBucketInfo(
+ const std::shared_ptr<api::RequestBucketInfoCommand>&)
+ { return false; }
+ virtual bool onRequestBucketInfoReply(
+ const std::shared_ptr<api::RequestBucketInfoReply>&)
+ { return false; }
+ virtual bool onNotifyBucketChange(
+ const std::shared_ptr<api::NotifyBucketChangeCommand>&)
+ { return false; }
+ virtual bool onNotifyBucketChangeReply(
+ const std::shared_ptr<api::NotifyBucketChangeReply>&)
+ { return false; }
+ virtual bool onSetNodeState(
+ const std::shared_ptr<api::SetNodeStateCommand>&)
+ { return false; }
+ virtual bool onSetNodeStateReply(
+ const std::shared_ptr<api::SetNodeStateReply>&)
+ { return false; }
+ virtual bool onGetNodeState(
+ const std::shared_ptr<api::GetNodeStateCommand>&)
+ { return false; }
+ virtual bool onGetNodeStateReply(
+ const std::shared_ptr<api::GetNodeStateReply>&)
+ { return false; }
+ virtual bool onSetSystemState(
+ const std::shared_ptr<api::SetSystemStateCommand>&)
+ { return false; }
+ virtual bool onSetSystemStateReply(
+ const std::shared_ptr<api::SetSystemStateReply>&)
+ { return false; }
+ virtual bool onGetSystemState(
+ const std::shared_ptr<api::GetSystemStateCommand>&)
+ { return false; }
+ virtual bool onGetSystemStateReply(
+ const std::shared_ptr<api::GetSystemStateReply>&)
+ { return false; }
+ virtual bool onBucketsAdded(
+ const std::shared_ptr<api::BucketsAddedCommand>&)
+ { return false; }
+ virtual bool onBucketsAddedReply(
+ const std::shared_ptr<api::BucketsAddedReply>&)
+ { return false; }
+ virtual bool onBucketsRemoved(
+ const std::shared_ptr<api::BucketsRemovedCommand>&)
+ { return false; }
+ virtual bool onBucketsRemovedReply(
+ const std::shared_ptr<api::BucketsRemovedReply>&)
+ { return false; }
+
+ virtual bool onStatBucket(
+ const std::shared_ptr<api::StatBucketCommand>&)
+ { return false; }
+ virtual bool onStatBucketReply(
+ const std::shared_ptr<api::StatBucketReply>&)
+ { return false; }
+
+ virtual bool onGetBucketList(
+ const std::shared_ptr<api::GetBucketListCommand>&)
+ { return false; }
+ virtual bool onGetBucketListReply(
+ const std::shared_ptr<api::GetBucketListReply>&)
+ { return false; }
+
+ virtual bool onRemoveLocation(
+ const std::shared_ptr<api::RemoveLocationCommand>&)
+ { return false; }
+
+ virtual bool onRemoveLocationReply(
+ const std::shared_ptr<api::RemoveLocationReply>&)
+ { return false; }
+
+ virtual ~MessageHandler() {}
+};
+
+#undef ON_M
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp
new file mode 100644
index 00000000000..6b2a80563c8
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp
@@ -0,0 +1,181 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/messageapi/returncode.h>
+#include <vespa/messagebus/errorcode.h>
+
+#include <vespa/document/util/bytebuffer.h>
+#include <vespa/fnet/frt/error.h>
+#include <iostream>
+#include <vector>
+
+namespace storage {
+namespace api {
+
+ReturnCode::ReturnCode()
+ : _result(OK),
+ _message()
+{
+}
+
+ReturnCode::ReturnCode(Result result, const vespalib::stringref & msg)
+ : _result(result),
+ _message(msg)
+{
+}
+
+ReturnCode::ReturnCode(const document::DocumentTypeRepo &repo,
+ document::ByteBuffer& buffer)
+ : _result(OK),
+ _message()
+{
+ deserialize(repo, buffer);
+}
+
+void ReturnCode::
+onDeserialize(const document::DocumentTypeRepo &, document::ByteBuffer& buffer)
+{
+ int32_t result;
+ buffer.getInt(result);
+ _result = static_cast<Result>(result);
+ int32_t size;
+ buffer.getInt(size);
+ const char * p = buffer.getBufferAtPos();
+ buffer.incPos(size);
+ _message.assign(p, size);
+}
+
+void ReturnCode::onSerialize(document::ByteBuffer& buffer) const
+{
+ buffer.putInt(_result);
+ buffer.putInt(_message.size());
+ buffer.putBytes(_message.c_str(), _message.size());
+}
+
+size_t ReturnCode::getSerializedSize() const
+{
+ return 2 * sizeof(int32_t) + _message.size();
+}
+
+void
+ReturnCode::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ (void) verbose; (void) indent;
+ out << "ReturnCode(" << ReturnCode::getResultString(getResult());
+ if (getMessage().size() > 0) out << ", " << getMessage();
+ out << ")";
+}
+
+vespalib::string ReturnCode::getResultString(Result result) {
+ return documentapi::DocumentProtocol::getErrorName(result);
+}
+
+bool
+ReturnCode::isBusy() const
+{
+ // Casting to suppress -Wswitch since we're comparing against enum values
+ // not present in the ReturnCode::Result enum.
+ switch (static_cast<uint32_t>(_result)) {
+ case mbus::ErrorCode::SEND_QUEUE_FULL:
+ case mbus::ErrorCode::SESSION_BUSY:
+ case mbus::ErrorCode::TIMEOUT:
+ case Protocol::ERROR_BUSY:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool
+ReturnCode::isNodeDownOrNetwork() const
+{
+ switch (static_cast<uint32_t>(_result)) {
+ case mbus::ErrorCode::NO_ADDRESS_FOR_SERVICE:
+ case mbus::ErrorCode::CONNECTION_ERROR:
+ case mbus::ErrorCode::UNKNOWN_SESSION:
+ case mbus::ErrorCode::HANDSHAKE_FAILED:
+ case mbus::ErrorCode::NO_SERVICES_FOR_ROUTE:
+ case mbus::ErrorCode::SERVICE_OOS:
+ case mbus::ErrorCode::NETWORK_ERROR:
+ case mbus::ErrorCode::UNKNOWN_PROTOCOL:
+ case Protocol::ERROR_NODE_NOT_READY:
+ case Protocol::ERROR_NOT_CONNECTED:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool
+ReturnCode::isCriticalForMaintenance() const
+{
+ if (_result >= static_cast<uint32_t>(mbus::ErrorCode::FATAL_ERROR)) {
+ return true;
+ }
+
+ switch (static_cast<uint32_t>(_result)) {
+ case Protocol::ERROR_INTERNAL_FAILURE:
+ case Protocol::ERROR_NO_SPACE:
+ case Protocol::ERROR_UNPARSEABLE:
+ case Protocol::ERROR_ILLEGAL_PARAMETERS:
+ case Protocol::ERROR_NOT_IMPLEMENTED:
+ case Protocol::ERROR_UNKNOWN_COMMAND:
+ case Protocol::ERROR_PROCESSING_FAILURE:
+ case Protocol::ERROR_IGNORED:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool
+ReturnCode::isCriticalForVisitor() const
+{
+ return isCriticalForMaintenance();
+}
+
+bool
+ReturnCode::isCriticalForVisitorDispatcher() const
+{
+ return isCriticalForMaintenance();
+}
+
+bool
+ReturnCode::isNonCriticalForIntegrityChecker() const
+{
+ switch (static_cast<uint32_t>(_result)) {
+ case Protocol::ERROR_ABORTED:
+ case Protocol::ERROR_BUCKET_DELETED:
+ case Protocol::ERROR_BUCKET_NOT_FOUND:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool
+ReturnCode::isShutdownRelated() const
+{
+ switch (static_cast<uint32_t>(_result)) {
+ case Protocol::ERROR_ABORTED:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool
+ReturnCode::isBucketDisappearance() const
+{
+ switch (static_cast<uint32_t>(_result)) {
+ case Protocol::ERROR_BUCKET_NOT_FOUND:
+ case Protocol::ERROR_BUCKET_DELETED:
+ return true;
+ default:
+ return false;
+ }
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.h b/storageapi/src/vespa/storageapi/messageapi/returncode.h
new file mode 100644
index 00000000000..997487853e8
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/returncode.h
@@ -0,0 +1,127 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class storage::api::ReturnCode
+ * @ingroup messageapi
+ *
+ * @brief Class for representing return values from the processing chain
+ *
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <vespa/vespalib/util/printable.h>
+#include <vespa/document/util/serializable.h>
+#include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <iosfwd>
+#include <string>
+
+namespace document {
+ class ByteBuffer;
+}
+
+namespace storage {
+namespace api {
+
+class ReturnCode : public document::Deserializable,
+ public vespalib::Printable {
+public:
+ typedef documentapi::DocumentProtocol Protocol;
+
+ /** Return status codes */
+ enum Result {
+ OK = mbus::ErrorCode::NONE,
+
+ EXISTS = Protocol::ERROR_EXISTS,
+
+ NOT_READY = Protocol::ERROR_NODE_NOT_READY,
+ WRONG_DISTRIBUTION = Protocol::ERROR_WRONG_DISTRIBUTION,
+ REJECTED = Protocol::ERROR_REJECTED,
+ ABORTED = Protocol::ERROR_ABORTED,
+ BUCKET_NOT_FOUND = Protocol::ERROR_BUCKET_NOT_FOUND,
+ BUCKET_DELETED = Protocol::ERROR_BUCKET_DELETED,
+ TIMESTAMP_EXIST = Protocol::ERROR_TIMESTAMP_EXIST,
+ STALE_TIMESTAMP = Protocol::ERROR_STALE_TIMESTAMP,
+ TEST_AND_SET_CONDITION_FAILED = Protocol::ERROR_TEST_AND_SET_CONDITION_FAILED,
+
+ // Wrong use
+ UNKNOWN_COMMAND = Protocol::ERROR_UNKNOWN_COMMAND,
+ NOT_IMPLEMENTED = Protocol::ERROR_NOT_IMPLEMENTED,
+ ILLEGAL_PARAMETERS = Protocol::ERROR_ILLEGAL_PARAMETERS,
+ IGNORED = Protocol::ERROR_IGNORED,
+ UNPARSEABLE = Protocol::ERROR_UNPARSEABLE,
+
+ // Network failure
+ NOT_CONNECTED = Protocol::ERROR_NOT_CONNECTED,
+ TIMEOUT = mbus::ErrorCode::TIMEOUT,
+ BUSY = Protocol::ERROR_BUSY,
+
+ // Disk operations
+ NO_SPACE = Protocol::ERROR_NO_SPACE,
+ DISK_FAILURE = Protocol::ERROR_DISK_FAILURE,
+ IO_FAILURE = Protocol::ERROR_IO_FAILURE,
+
+ // Don't know what happened (catch-all)
+ INTERNAL_FAILURE = Protocol::ERROR_INTERNAL_FAILURE
+ };
+
+private:
+ Result _result;
+ vespalib::string _message;
+
+ void onDeserialize(const document::DocumentTypeRepo &repo,
+ document::ByteBuffer& buffer);
+ void onSerialize(document::ByteBuffer& buffer) const;
+
+public:
+ ReturnCode();
+ explicit ReturnCode(Result result, const vespalib::stringref & msg = "");
+ ReturnCode(const document::DocumentTypeRepo &repo,
+ document::ByteBuffer& buffer);
+
+ ReturnCode* clone() const { return new ReturnCode(*this); }
+
+ size_t getSerializedSize() const;
+
+ const vespalib::string& getMessage() const { return _message; }
+ void setMessage(const vespalib::stringref & message) { _message = message; }
+
+ Result getResult() const { return _result; }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ /**
+ * Translate from status code to human-readable string
+ * @param result Status code returned from getResult()
+ */
+ static vespalib::string getResultString(Result result);
+
+ bool failed() const { return (_result != OK); }
+ bool success() const { return (_result == OK); }
+
+ bool operator==(Result res) const { return _result == res; }
+ bool operator!=(Result res) const { return _result != res; }
+ bool operator==(const ReturnCode& code) const
+ { return _result == code._result && _message == code._message; }
+ bool operator!=(const ReturnCode& code) const
+ { return _result != code._result || _message != code._message; }
+
+ // To avoid lots of code matching various return codes in storage, we define
+ // some functions they can use to match those codes that corresponds to what
+ // they want to match.
+
+ bool isBusy() const;
+ bool isNodeDownOrNetwork() const;
+ bool isCriticalForMaintenance() const;
+ bool isCriticalForVisitor() const;
+ bool isCriticalForVisitorDispatcher() const;
+ bool isShutdownRelated() const;
+ bool isBucketDisappearance() const;
+ bool isNonCriticalForIntegrityChecker() const;
+};
+
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp b/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp
new file mode 100644
index 00000000000..9578c347b92
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <limits>
+#include <vespa/storageapi/messageapi/storagecommand.h>
+
+namespace storage {
+namespace api {
+
+StorageCommand::StorageCommand(const StorageCommand& other)
+ : StorageMessage(other, generateMsgId()),
+ _timeout(other._timeout),
+ _sourceIndex(other._sourceIndex)
+{
+ setTrace(other.getTrace());
+}
+
+StorageCommand::StorageCommand(const MessageType& type, Priority p)
+ : StorageMessage(type, generateMsgId()),
+ // Default timeout is unlimited. Set from mbus message. Some internal
+ // use want unlimited timeout, (such as readbucketinfo, repair bucket
+ // etc)
+ _timeout(std::numeric_limits<uint32_t>().max()),
+ _sourceIndex(0xFFFF)
+{
+ setPriority(p);
+}
+
+void
+StorageCommand::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ (void) verbose; (void) indent;
+ out << "StorageCommand(" << _type.getName();
+ if (_priority != NORMAL) out << ", priority = " << static_cast<int>(_priority);
+ if (_sourceIndex != 0xFFFF) out << ", source = " << _sourceIndex;
+ out << ", timeout = " << _timeout << " ms";
+ out << ")";
+}
+
+StorageCommand::UP
+StorageCommand::createCopyToForward(const document::BucketId&, uint64_t) const
+{
+ throw vespalib::IllegalStateException(
+ "Command " + _type.getName() + " does not support forwarding.",
+ VESPA_STRLOC);
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagecommand.h b/storageapi/src/vespa/storageapi/messageapi/storagecommand.h
new file mode 100644
index 00000000000..c71a5097bc4
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/storagecommand.h
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class storage::api::StorageCommand
+ * @ingroup messageapi
+ *
+ * @brief Superclass for all storage commands.
+ *
+ * A storage command is a storage message you will get a storage reply for.
+ *
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <vespa/storageapi/messageapi/storagemessage.h>
+
+namespace storage {
+namespace api {
+
+class StorageReply;
+
+class StorageCommand : public StorageMessage {
+ uint32_t _timeout; /** Timeout of command in milliseconds */
+ /** Sets what node this message origins from. 0xFFFF is unset. */
+ uint16_t _sourceIndex;
+
+protected:
+ explicit StorageCommand(const StorageCommand& other);
+ explicit StorageCommand(const MessageType& type, Priority p = NORMAL);
+
+public:
+ DECLARE_POINTER_TYPEDEFS(StorageCommand);
+
+ virtual ~StorageCommand() {}
+
+ bool sourceIndexSet() const { return (_sourceIndex != 0xffff); }
+ void setSourceIndex(uint16_t sourceIndex) { _sourceIndex = sourceIndex; }
+ uint16_t getSourceIndex() const { return _sourceIndex; }
+
+ /** Set timeout in milliseconds. */
+ void setTimeout(uint32_t milliseconds) { _timeout = milliseconds; }
+ /** Get timeout in milliseconds. */
+ uint32_t getTimeout() const { return _timeout; }
+
+ /** Used to set a new id so the message can be resent. */
+ void setNewId() { StorageMessage::setNewMsgId(); }
+
+ /** Overload this to get more descriptive message output. */
+ virtual void print(std::ostream& out,
+ bool verbose,
+ const std::string& indent) const;
+
+ /**
+ * A way for someone to make a reply to a storage message without
+ * knowing the type of the message. Should just call reply constructor
+ * taking command as input.
+ */
+ virtual std::unique_ptr<StorageReply> makeReply() = 0;
+
+ /**
+ * Distributors need a way to create copies of messages to send to forward
+ * to different nodes. Only messages sent through the distributor needs to
+ * have an actual implementation of this.
+ */
+ virtual StorageCommand::UP createCopyToForward(
+ const document::BucketId& bucket, uint64_t timestamp) const;
+
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp
new file mode 100644
index 00000000000..80b25111488
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp
@@ -0,0 +1,334 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/messageapi/storagemessage.h>
+
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/messagebus/routing/verbatimdirective.h>
+
+namespace storage {
+namespace api {
+
+static const vespalib::string STORAGEADDRESS_PREFIX = "storage/cluster.";
+
+const char*
+StorageMessage::getPriorityString(Priority p) {
+ switch (p) {
+ case LOW: return "LOW";
+ case NORMAL: return "NORMAL";
+ case HIGH: return "HIGH";
+ case VERYHIGH: return "VERYHIGH";
+ default: return "UNKNOWN";
+ }
+}
+
+std::map<MessageType::Id, MessageType*> MessageType::_codes;
+
+const MessageType MessageType::DOCBLOCK("DocBlock", DOCBLOCK_ID);
+const MessageType MessageType::DOCBLOCK_REPLY(
+ "DocBlock Reply", DOCBLOCK_REPLY_ID, &MessageType::DOCBLOCK);
+const MessageType MessageType::GET("Get", GET_ID);
+const MessageType MessageType::GET_REPLY(
+ "Get Reply", GET_REPLY_ID, &MessageType::GET);
+const MessageType MessageType::INTERNAL("Internal", INTERNAL_ID);
+const MessageType MessageType::INTERNAL_REPLY(
+ "Internal Reply", INTERNAL_REPLY_ID, &MessageType::INTERNAL);
+const MessageType MessageType::PUT("Put", PUT_ID);
+const MessageType MessageType::PUT_REPLY(
+ "Put Reply", PUT_REPLY_ID, &MessageType::PUT);
+const MessageType MessageType::UPDATE("Update", UPDATE_ID);
+const MessageType MessageType::UPDATE_REPLY(
+ "Update Reply", UPDATE_REPLY_ID, &MessageType::UPDATE);
+const MessageType MessageType::REMOVE("Remove", REMOVE_ID);
+const MessageType MessageType::REMOVE_REPLY(
+ "Remove Reply", REMOVE_REPLY_ID, &MessageType::REMOVE);
+const MessageType MessageType::REVERT("Revert", REVERT_ID);
+const MessageType MessageType::REVERT_REPLY(
+ "Revert Reply", REVERT_REPLY_ID, &MessageType::REVERT);
+const MessageType MessageType::VISITOR_CREATE(
+ "Visitor Create", VISITOR_CREATE_ID);
+const MessageType MessageType::VISITOR_CREATE_REPLY(
+ "Visitor Create Reply", VISITOR_CREATE_REPLY_ID,
+ &MessageType::VISITOR_CREATE);
+const MessageType MessageType::VISITOR_DESTROY(
+ "Visitor Destroy", VISITOR_DESTROY_ID);
+const MessageType MessageType::VISITOR_DESTROY_REPLY(
+ "Visitor Destroy Reply", VISITOR_DESTROY_REPLY_ID,
+ &MessageType::VISITOR_DESTROY);
+const MessageType MessageType::REQUESTBUCKETINFO("Request bucket info",
+ REQUESTBUCKETINFO_ID);
+const MessageType MessageType::REQUESTBUCKETINFO_REPLY(
+ "Request bucket info reply", REQUESTBUCKETINFO_REPLY_ID,
+ &MessageType::REQUESTBUCKETINFO);
+const MessageType MessageType::NOTIFYBUCKETCHANGE("Notify bucket change",
+ NOTIFYBUCKETCHANGE_ID);
+const MessageType MessageType::NOTIFYBUCKETCHANGE_REPLY(
+ "Notify bucket change reply", NOTIFYBUCKETCHANGE_REPLY_ID,
+ &MessageType::NOTIFYBUCKETCHANGE);
+const MessageType MessageType::CREATEBUCKET("Create bucket", CREATEBUCKET_ID);
+const MessageType MessageType::CREATEBUCKET_REPLY(
+ "Create bucket reply", CREATEBUCKET_REPLY_ID,
+ &MessageType::CREATEBUCKET);
+const MessageType MessageType::MERGEBUCKET("Merge bucket", MERGEBUCKET_ID);
+const MessageType MessageType::MERGEBUCKET_REPLY(
+ "Merge bucket reply", MERGEBUCKET_REPLY_ID,
+ &MessageType::MERGEBUCKET);
+const MessageType MessageType::DELETEBUCKET("Delete bucket", DELETEBUCKET_ID);
+const MessageType MessageType::DELETEBUCKET_REPLY(
+ "Delete bucket reply", DELETEBUCKET_REPLY_ID,
+ &MessageType::DELETEBUCKET);
+const MessageType MessageType::SETNODESTATE("Set node state", SETNODESTATE_ID);
+const MessageType MessageType::SETNODESTATE_REPLY(
+ "Set node state reply", SETNODESTATE_REPLY_ID,
+ &MessageType::SETNODESTATE);
+const MessageType MessageType::GETNODESTATE("Get node state", GETNODESTATE_ID);
+const MessageType MessageType::GETNODESTATE_REPLY(
+ "Get node state reply", GETNODESTATE_REPLY_ID,
+ &MessageType::GETNODESTATE);
+const MessageType MessageType::SETSYSTEMSTATE("Set system state", SETSYSTEMSTATE_ID);
+const MessageType MessageType::SETSYSTEMSTATE_REPLY(
+ "Set system state reply", SETSYSTEMSTATE_REPLY_ID,
+ &MessageType::SETSYSTEMSTATE);
+const MessageType MessageType::GETSYSTEMSTATE("Get system state", GETSYSTEMSTATE_ID);
+const MessageType MessageType::GETSYSTEMSTATE_REPLY(
+ "get system state reply", GETSYSTEMSTATE_REPLY_ID,
+ &MessageType::GETSYSTEMSTATE);
+const MessageType MessageType::GETBUCKETDIFF("GetBucketDiff", GETBUCKETDIFF_ID);
+const MessageType MessageType::GETBUCKETDIFF_REPLY(
+ "GetBucketDiff reply", GETBUCKETDIFF_REPLY_ID,
+ &MessageType::GETBUCKETDIFF);
+const MessageType MessageType::APPLYBUCKETDIFF("ApplyBucketDiff",
+ APPLYBUCKETDIFF_ID);
+const MessageType MessageType::APPLYBUCKETDIFF_REPLY(
+ "ApplyBucketDiff reply", APPLYBUCKETDIFF_REPLY_ID,
+ &MessageType::APPLYBUCKETDIFF);
+const MessageType MessageType::VISITOR_INFO("VisitorInfo",
+ VISITOR_INFO_ID);
+const MessageType MessageType::VISITOR_INFO_REPLY(
+ "VisitorInfo reply", VISITOR_INFO_REPLY_ID,
+ &MessageType::VISITOR_INFO);
+const MessageType MessageType::SEARCHRESULT("SearchResult", SEARCHRESULT_ID);
+const MessageType MessageType::SEARCHRESULT_REPLY(
+ "SearchResult reply", SEARCHRESULT_REPLY_ID,
+ &MessageType::SEARCHRESULT);
+const MessageType MessageType::DOCUMENTSUMMARY("DocumentSummary", DOCUMENTSUMMARY_ID);
+const MessageType MessageType::DOCUMENTSUMMARY_REPLY(
+ "DocumentSummary reply", DOCUMENTSUMMARY_REPLY_ID,
+ &MessageType::DOCUMENTSUMMARY);
+const MessageType MessageType::MAPVISITOR("Mapvisitor", MAPVISITOR_ID);
+const MessageType MessageType::MAPVISITOR_REPLY(
+ "Mapvisitor reply", MAPVISITOR_REPLY_ID,
+ &MessageType::MAPVISITOR);
+const MessageType MessageType::SPLITBUCKET("SplitBucket", SPLITBUCKET_ID);
+const MessageType MessageType::SPLITBUCKET_REPLY(
+ "SplitBucket reply", SPLITBUCKET_REPLY_ID,
+ &MessageType::SPLITBUCKET);
+const MessageType MessageType::JOINBUCKETS("Joinbuckets", JOINBUCKETS_ID);
+const MessageType MessageType::JOINBUCKETS_REPLY(
+ "Joinbuckets reply", JOINBUCKETS_REPLY_ID,
+ &MessageType::JOINBUCKETS);
+const MessageType MessageType::MULTIOPERATION("MultiOperation", MULTIOPERATION_ID);
+const MessageType MessageType::MULTIOPERATION_REPLY(
+ "MultiOperation Reply", MULTIOPERATION_REPLY_ID, &MessageType::MULTIOPERATION);
+const MessageType MessageType::STATBUCKET("Statbucket", STATBUCKET_ID);
+const MessageType MessageType::STATBUCKET_REPLY(
+ "Statbucket Reply", STATBUCKET_REPLY_ID, &MessageType::STATBUCKET);
+const MessageType MessageType::GETBUCKETLIST("Getbucketlist", GETBUCKETLIST_ID);
+const MessageType MessageType::GETBUCKETLIST_REPLY(
+ "Getbucketlist Reply", GETBUCKETLIST_REPLY_ID, &MessageType::GETBUCKETLIST);
+const MessageType MessageType::DOCUMENTLIST("documentlist", DOCUMENTLIST_ID);
+const MessageType MessageType::DOCUMENTLIST_REPLY(
+ "documentlist Reply", DOCUMENTLIST_REPLY_ID, &MessageType::DOCUMENTLIST);
+const MessageType MessageType::EMPTYBUCKETS("Emptybuckets", EMPTYBUCKETS_ID);
+const MessageType MessageType::EMPTYBUCKETS_REPLY(
+ "Emptybuckets Reply", EMPTYBUCKETS_REPLY_ID, &MessageType::EMPTYBUCKETS);
+const MessageType MessageType::REMOVELOCATION("Removelocation", REMOVELOCATION_ID);
+const MessageType MessageType::REMOVELOCATION_REPLY(
+ "Removelocation Reply", REMOVELOCATION_REPLY_ID, &MessageType::REMOVELOCATION);
+const MessageType MessageType::QUERYRESULT("QueryResult", QUERYRESULT_ID);
+const MessageType MessageType::QUERYRESULT_REPLY(
+ "QueryResult reply", QUERYRESULT_REPLY_ID,
+ &MessageType::QUERYRESULT);
+const MessageType MessageType::BATCHPUTREMOVE("BatchPutRemove", BATCHPUTREMOVE_ID);
+const MessageType MessageType::BATCHPUTREMOVE_REPLY(
+ "BatchPutRemove reply", BATCHPUTREMOVE_REPLY_ID,
+ &MessageType::BATCHPUTREMOVE);
+const MessageType MessageType::BATCHDOCUMENTUPDATE("BatchDocumentUpdate", BATCHDOCUMENTUPDATE_ID);
+const MessageType MessageType::BATCHDOCUMENTUPDATE_REPLY(
+ "BatchDocumentUpdate reply", BATCHDOCUMENTUPDATE_REPLY_ID,
+ &MessageType::BATCHDOCUMENTUPDATE);
+const MessageType MessageType::SETBUCKETSTATE(
+ "SetBucketState",
+ SETBUCKETSTATE_ID);
+const MessageType MessageType::SETBUCKETSTATE_REPLY(
+ "SetBucketStateReply",
+ SETBUCKETSTATE_REPLY_ID,
+ &MessageType::SETBUCKETSTATE);
+
+const MessageType& MessageType::get(Id id)
+{
+ std::map<Id, MessageType*>::const_iterator it = _codes.find(id);
+ if (it == _codes.end()) {
+ std::ostringstream ost;
+ ost << "No message type with id " << id << ".";
+ throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
+ }
+ return *it->second;
+}
+
+StorageMessageAddress::StorageMessageAddress(const mbus::Route& route)
+ : _route(route),
+ _retryEnabled(false),
+ _protocol(DOCUMENT),
+ _cluster(""),
+ _type(0),
+ _index(0xFFFF)
+{
+}
+
+static vespalib::string
+createAddress(const vespalib::stringref & cluster, const lib::NodeType& type,
+ uint16_t index)
+{
+ vespalib::asciistream os;
+ os << STORAGEADDRESS_PREFIX << cluster << '/' << type.toString() << '/' << index << "/default";
+ return os.str();
+}
+
+StorageMessageAddress::StorageMessageAddress(
+ const vespalib::stringref & cluster, const lib::NodeType& type,
+ uint16_t index, Protocol protocol)
+ : _retryEnabled(false),
+ _protocol(protocol),
+ _cluster(cluster),
+ _type(&type),
+ _index(index)
+{
+ mbus::IHopDirective::SP directive(new mbus::VerbatimDirective(
+ createAddress(cluster, type, index)));
+
+ std::vector<mbus::IHopDirective::SP> directives;
+ directives.push_back(directive);
+ mbus::Hop hop(directives, false);
+ _route.addHop(hop);
+}
+
+uint16_t
+StorageMessageAddress::getIndex() const
+{
+ if (_type == 0) {
+ throw vespalib::IllegalStateException(
+ "Cannot retrieve node index out of external address",
+ VESPA_STRLOC);
+ }
+ return _index;
+}
+
+const lib::NodeType&
+StorageMessageAddress::getNodeType() const
+{
+ if (_type == 0) {
+ throw vespalib::IllegalStateException(
+ "Cannot retrieve node type out of external address",
+ VESPA_STRLOC);
+ }
+ return *_type;
+}
+
+const vespalib::string&
+StorageMessageAddress::getCluster() const
+{
+ if (_type == 0) {
+ throw vespalib::IllegalStateException(
+ "Cannot retrieve cluster out of external address",
+ VESPA_STRLOC);
+ }
+ return _cluster;
+}
+
+bool
+StorageMessageAddress::operator==(const StorageMessageAddress& other) const
+{
+ if (_protocol != other._protocol) return false;
+ if (_retryEnabled != other._retryEnabled) return false;
+ if (_type != other._type) return false;
+ if (_type != 0) {
+ if (_cluster != other._cluster) return false;
+ if (_index != other._index) return false;
+ if (_type != other._type) return false;
+ }
+ return true;
+}
+
+vespalib::string
+StorageMessageAddress::toString() const
+{
+ vespalib::asciistream os;
+ print(os);
+ return os.str();
+}
+
+void
+StorageMessageAddress::print(vespalib::asciistream & out) const
+{
+ out << "StorageMessageAddress(";
+ if (_protocol == STORAGE) {
+ out << "Storage protocol";
+ } else {
+ out << "Document protocol";
+ }
+ if (_retryEnabled) {
+ out << ", retry enabled";
+ }
+ if (_type == 0) {
+ out << ", " << _route.toString() << ")";
+ } else {
+ out << ", cluster " << _cluster << ", nodetype " << *_type
+ << ", index " << _index << ")";
+ }
+}
+
+TransportContext::~TransportContext()
+{
+}
+
+vespalib::Lock StorageMessage::_msgIdLock;
+StorageMessage::Id StorageMessage::_lastMsgId = 1000;
+
+StorageMessage::Id
+StorageMessage::generateMsgId()
+{
+ vespalib::LockGuard sync(_msgIdLock);
+ Id msgId = _lastMsgId++;
+ _lastMsgId &= ((Id(-1) << 8) >> 8);
+ return msgId;
+}
+
+StorageMessage::StorageMessage(const MessageType& type, Id id)
+ : _type(type),
+ _msgId(id),
+ _priority(NORMAL),
+ _address(),
+ _loadType(documentapi::LoadType::DEFAULT)
+{
+}
+
+StorageMessage::StorageMessage(const StorageMessage& other, Id id)
+ : _type(other._type),
+ _msgId(id),
+ _priority(other._priority),
+ _address(),
+ _loadType(other._loadType)
+{
+
+}
+
+void StorageMessage::setNewMsgId()
+{
+ vespalib::LockGuard sync(_msgIdLock);
+ _msgId = _lastMsgId++;
+ _lastMsgId &= ((Id(-1) << 8) >> 8);
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
new file mode 100644
index 00000000000..85cb53c3681
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
@@ -0,0 +1,458 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/**
+ * @class storage::api::StorageMessage
+ * @ingroup messageapi
+ *
+ * @brief Superclass for all storage messages.
+ *
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <map>
+#include <vespa/vespalib/util/printable.h>
+#include <vespa/messagebus/routing/route.h>
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/storageapi/messageapi/messagehandler.h>
+#include <vespa/vdslib/state/nodetype.h>
+#include <vespa/messagebus/trace.h>
+#include <vespa/documentapi/loadtypes/loadtype.h>
+#include <vespa/storageframework/generic/memory/memorytoken.h>
+#include <vespa/document/bucket/bucketid.h>
+
+// The following macros are provided as a way to write storage messages simply.
+// They implement the parts of the code that can easily be automaticly
+// generated.
+
+/**
+ * Adds a messagehandler callback and some utilities
+ */
+#define DECLARE_POINTER_TYPEDEFS(message) \
+ typedef std::unique_ptr<message> UP; \
+ typedef std::shared_ptr<message> SP; \
+ typedef std::shared_ptr<const message> CSP;
+
+#define DECLARE_STORAGEREPLY(reply, callback) \
+public: \
+ DECLARE_POINTER_TYPEDEFS(reply) \
+private: \
+ virtual bool callHandler( \
+ MessageHandler& h, \
+ const std::shared_ptr<StorageMessage>& m) const \
+ { \
+ return h.callback(std::static_pointer_cast<reply>(m)); \
+ }
+
+/** Commands also has a command to implement to create the reply. */
+#define DECLARE_STORAGECOMMAND(command, callback) \
+public: \
+ std::unique_ptr<StorageReply> makeReply(); \
+ DECLARE_STORAGEREPLY(command, callback)
+
+/** This macro implements common stuff for all storage messages. */
+#define IMPLEMENT_COMMON(message) \
+
+/** This macro is used to implement common storage reply functionality. */
+#define IMPLEMENT_REPLY(reply) \
+ IMPLEMENT_COMMON(reply) \
+
+/** This macro is used to implement common storage command functionality. */
+#define IMPLEMENT_COMMAND(command, reply) \
+ IMPLEMENT_COMMON(command) \
+ std::unique_ptr<storage::api::StorageReply> \
+ storage::api::command::makeReply() \
+ { \
+ return std::unique_ptr<storage::api::StorageReply>(new reply(*this)); \
+ }
+
+
+namespace storage {
+
+namespace api {
+
+/**
+ * @class MessageType
+ * @ingroup messageapi
+ *
+ * @brief This class defines the different message types we have.
+ *
+ * This is used to be able to deserialize messages of various classes.
+ */
+class MessageType : public vespalib::Printable {
+private:
+ MessageType(const MessageType &);
+ MessageType& operator=(const MessageType &);
+
+public:
+ enum Id {
+ GET_ID = 4,
+ GET_REPLY_ID = 5,
+ INTERNAL_ID = 6,
+ INTERNAL_REPLY_ID = 7,
+ PUT_ID = 10,
+ PUT_REPLY_ID = 11,
+ REMOVE_ID = 12,
+ REMOVE_REPLY_ID = 13,
+ REVERT_ID = 14,
+ REVERT_REPLY_ID = 15,
+ STAT_ID = 16,
+ STAT_REPLY_ID = 17,
+ VISITOR_CREATE_ID = 18,
+ VISITOR_CREATE_REPLY_ID = 19,
+ VISITOR_DESTROY_ID = 20,
+ VISITOR_DESTROY_REPLY_ID = 21,
+ CREATEBUCKET_ID = 26,
+ CREATEBUCKET_REPLY_ID = 27,
+ MERGEBUCKET_ID = 32,
+ MERGEBUCKET_REPLY_ID = 33,
+ DELETEBUCKET_ID = 34,
+ DELETEBUCKET_REPLY_ID = 35,
+ SETNODESTATE_ID = 36,
+ SETNODESTATE_REPLY_ID = 37,
+ GETNODESTATE_ID = 38,
+ GETNODESTATE_REPLY_ID = 39,
+ SETSYSTEMSTATE_ID = 40,
+ SETSYSTEMSTATE_REPLY_ID = 41,
+ GETSYSTEMSTATE_ID = 42,
+ GETSYSTEMSTATE_REPLY_ID = 43,
+ GETBUCKETDIFF_ID = 50,
+ GETBUCKETDIFF_REPLY_ID = 51,
+ APPLYBUCKETDIFF_ID = 52,
+ APPLYBUCKETDIFF_REPLY_ID = 53,
+ REQUESTBUCKETINFO_ID = 54,
+ REQUESTBUCKETINFO_REPLY_ID = 55,
+ NOTIFYBUCKETCHANGE_ID = 56,
+ NOTIFYBUCKETCHANGE_REPLY_ID = 57,
+ DOCBLOCK_ID = 58,
+ DOCBLOCK_REPLY_ID = 59,
+ VISITOR_INFO_ID = 60,
+ VISITOR_INFO_REPLY_ID = 61,
+ SEARCHRESULT_ID = 64,
+ SEARCHRESULT_REPLY_ID = 65,
+ SPLITBUCKET_ID = 66,
+ SPLITBUCKET_REPLY_ID = 67,
+ JOINBUCKETS_ID = 68,
+ JOINBUCKETS_REPLY_ID = 69,
+ MULTIOPERATION_ID = 70,
+ MULTIOPERATION_REPLY_ID = 71,
+ DOCUMENTSUMMARY_ID = 72,
+ DOCUMENTSUMMARY_REPLY_ID = 73,
+ MAPVISITOR_ID = 74,
+ MAPVISITOR_REPLY_ID = 75,
+ STATBUCKET_ID = 76,
+ STATBUCKET_REPLY_ID = 77,
+ GETBUCKETLIST_ID = 78,
+ GETBUCKETLIST_REPLY_ID = 79,
+ DOCUMENTLIST_ID = 80,
+ DOCUMENTLIST_REPLY_ID = 81,
+ UPDATE_ID = 82,
+ UPDATE_REPLY_ID = 83,
+ EMPTYBUCKETS_ID = 84,
+ EMPTYBUCKETS_REPLY_ID = 85,
+ REMOVELOCATION_ID = 86,
+ REMOVELOCATION_REPLY_ID = 87,
+ QUERYRESULT_ID = 88,
+ QUERYRESULT_REPLY_ID = 89,
+ BATCHPUTREMOVE_ID = 90,
+ BATCHPUTREMOVE_REPLY_ID = 91,
+ BATCHDOCUMENTUPDATE_ID = 92,
+ BATCHDOCUMENTUPDATE_REPLY_ID = 93,
+ SETBUCKETSTATE_ID = 94,
+ SETBUCKETSTATE_REPLY_ID = 95,
+ MESSAGETYPE_MAX_ID
+ };
+
+private:
+ static std::map<Id, MessageType*> _codes;
+ const vespalib::string _name;
+ Id _id;
+ MessageType *_reply;
+ const MessageType *_replyOf;
+
+ MessageType(const vespalib::stringref & name, Id id,
+ const MessageType* replyOf = 0)
+ : _name(name), _id(id), _reply(NULL), _replyOf(replyOf)
+ {
+ _codes[id] = this;
+ if (_replyOf != 0) {
+ assert(_replyOf->_reply == 0);
+ // Ugly cast to let initialization work
+ MessageType& type = const_cast<MessageType&>(*_replyOf);
+ type._reply = this;
+ }
+ }
+public:
+ static const MessageType DOCBLOCK;
+ static const MessageType DOCBLOCK_REPLY;
+ static const MessageType GET;
+ static const MessageType GET_REPLY;
+ static const MessageType INTERNAL;
+ static const MessageType INTERNAL_REPLY;
+ static const MessageType PUT;
+ static const MessageType PUT_REPLY;
+ static const MessageType REMOVE;
+ static const MessageType REMOVE_REPLY;
+ static const MessageType REVERT;
+ static const MessageType REVERT_REPLY;
+ static const MessageType VISITOR_CREATE;
+ static const MessageType VISITOR_CREATE_REPLY;
+ static const MessageType VISITOR_DESTROY;
+ static const MessageType VISITOR_DESTROY_REPLY;
+ static const MessageType REQUESTBUCKETINFO;
+ static const MessageType REQUESTBUCKETINFO_REPLY;
+ static const MessageType NOTIFYBUCKETCHANGE;
+ static const MessageType NOTIFYBUCKETCHANGE_REPLY;
+ static const MessageType CREATEBUCKET;
+ static const MessageType CREATEBUCKET_REPLY;
+ static const MessageType MERGEBUCKET;
+ static const MessageType MERGEBUCKET_REPLY;
+ static const MessageType DELETEBUCKET;
+ static const MessageType DELETEBUCKET_REPLY;
+ static const MessageType SETNODESTATE;
+ static const MessageType SETNODESTATE_REPLY;
+ static const MessageType GETNODESTATE;
+ static const MessageType GETNODESTATE_REPLY;
+ static const MessageType SETSYSTEMSTATE;
+ static const MessageType SETSYSTEMSTATE_REPLY;
+ static const MessageType GETSYSTEMSTATE;
+ static const MessageType GETSYSTEMSTATE_REPLY;
+ static const MessageType BUCKETSADDED;
+ static const MessageType BUCKETSADDED_REPLY;
+ static const MessageType BUCKETSREMOVED;
+ static const MessageType BUCKETSREMOVED_REPLY;
+ static const MessageType GETBUCKETDIFF;
+ static const MessageType GETBUCKETDIFF_REPLY;
+ static const MessageType APPLYBUCKETDIFF;
+ static const MessageType APPLYBUCKETDIFF_REPLY;
+ static const MessageType VISITOR_INFO;
+ static const MessageType VISITOR_INFO_REPLY;
+ static const MessageType SEARCHRESULT;
+ static const MessageType SEARCHRESULT_REPLY;
+ static const MessageType SPLITBUCKET;
+ static const MessageType SPLITBUCKET_REPLY;
+ static const MessageType JOINBUCKETS;
+ static const MessageType JOINBUCKETS_REPLY;
+ static const MessageType MULTIOPERATION;
+ static const MessageType MULTIOPERATION_REPLY;
+ static const MessageType DOCUMENTSUMMARY;
+ static const MessageType DOCUMENTSUMMARY_REPLY;
+ static const MessageType MAPVISITOR;
+ static const MessageType MAPVISITOR_REPLY;
+ static const MessageType STATBUCKET;
+ static const MessageType STATBUCKET_REPLY;
+ static const MessageType GETBUCKETLIST;
+ static const MessageType GETBUCKETLIST_REPLY;
+ static const MessageType DOCUMENTLIST;
+ static const MessageType DOCUMENTLIST_REPLY;
+ static const MessageType UPDATE;
+ static const MessageType UPDATE_REPLY;
+ static const MessageType EMPTYBUCKETS;
+ static const MessageType EMPTYBUCKETS_REPLY;
+ static const MessageType REMOVELOCATION;
+ static const MessageType REMOVELOCATION_REPLY;
+ static const MessageType QUERYRESULT;
+ static const MessageType QUERYRESULT_REPLY;
+ static const MessageType BATCHPUTREMOVE;
+ static const MessageType BATCHPUTREMOVE_REPLY;
+ static const MessageType BATCHDOCUMENTUPDATE;
+ static const MessageType BATCHDOCUMENTUPDATE_REPLY;
+ static const MessageType SETBUCKETSTATE;
+ static const MessageType SETBUCKETSTATE_REPLY;
+
+ static const MessageType& get(Id id);
+
+ Id getId() const { return _id; }
+ static Id getMaxId() { return MESSAGETYPE_MAX_ID; }
+ const vespalib::string& getName() const { return _name; }
+ bool isReply() const { return (_replyOf != 0); }
+ /** Only valid to call on replies. */
+ const MessageType& getCommandType() const
+ { assert(_replyOf); return *_replyOf; }
+ /** Only valid to call on commands. */
+ const MessageType& getReplyType() const
+ { assert(_reply); return *_reply; }
+ bool operator==(const MessageType& type) const { return (_id == type._id); }
+ bool operator!=(const MessageType& type) const { return (_id != type._id); }
+
+ void print(std::ostream& out, bool verbose, const std::string& indent) const
+ {
+ (void) verbose; (void) indent;
+ out << "MessageType(" << _id << ", " << _name;
+ if (_replyOf) {
+ out << ", reply of " << _replyOf->getName();
+ }
+ out << ")";
+ }
+};
+
+/**
+ * Represent an address we can send a storage message to.
+ * We have two kinds of addresses:
+ * - A VDS address used to send to a single VDS node.
+ * - An external mbus route, used to send to an external source.
+ */
+class StorageMessageAddress {
+public:
+ enum Protocol { STORAGE, DOCUMENT };
+
+private:
+ mbus::Route _route;
+ bool _retryEnabled;
+ Protocol _protocol;
+ // Used for internal VDS addresses only
+ vespalib::string _cluster;
+ const lib::NodeType* _type;
+ uint16_t _index;
+
+public:
+ StorageMessageAddress(const mbus::Route& route);
+ StorageMessageAddress(const vespalib::stringref & clusterName,
+ const lib::NodeType& type, uint16_t index,
+ Protocol protocol = STORAGE);
+
+ void setProtocol(Protocol p) { _protocol = p; }
+ void enableRetry(bool enable = true) { _retryEnabled = enable; }
+
+ const mbus::Route& getRoute() const { return _route; }
+ bool retryEnabled() const { return _retryEnabled; }
+ Protocol getProtocol() const { return _protocol; }
+ uint16_t getIndex() const;
+ const lib::NodeType& getNodeType() const;
+ const vespalib::string& getCluster() const;
+
+ bool operator==(const StorageMessageAddress& other) const;
+ vespalib::string toString() const;
+ friend std::ostream & operator <<
+ (std::ostream & os, const StorageMessageAddress & addr) {
+ return os << addr.toString();
+ }
+
+private:
+ void print(vespalib::asciistream & out) const;
+};
+
+struct TransportContext {
+ virtual ~TransportContext() = 0;
+};
+
+class StorageMessage : public vespalib::Printable
+{
+ friend class StorageMessageTest; // Used for testing only
+public:
+ DECLARE_POINTER_TYPEDEFS(StorageMessage);
+ typedef uint64_t Id;
+ typedef uint8_t Priority;
+
+ enum LegacyPriorityValues {
+ LOW = 225,
+ NORMAL = 127,
+ HIGH = 50,
+ VERYHIGH = 0
+ }; // FIXME
+ //static const unsigned int NUM_PRIORITIES = UINT8_MAX;
+ static const char* getPriorityString(Priority);
+
+private:
+ static vespalib::Lock _msgIdLock;
+ static Id _lastMsgId;
+
+ StorageMessage& operator=(const StorageMessage&);
+ StorageMessage(const StorageMessage&);
+
+ mutable std::unique_ptr<TransportContext> _transportContext;
+ std::unique_ptr<framework::MemoryToken> _memoryToken;
+
+protected:
+ static Id generateMsgId();
+
+ const MessageType& _type;
+ Id _msgId;
+ Priority _priority;
+ std::unique_ptr<StorageMessageAddress> _address;
+ documentapi::LoadType _loadType;
+ mbus::Trace _trace;
+
+ StorageMessage(const MessageType& code, Id id);
+ StorageMessage(const StorageMessage&, Id id);
+
+public:
+ virtual ~StorageMessage() {}
+
+ Id getMsgId() const { return _msgId; }
+
+ /** Method used by storage commands to set a new id. */
+ void setNewMsgId();
+
+ /**
+ * Set the id of this message. Typically used to set the id to a
+ * unique value previously generated with the generateMsgId method.
+ **/
+ void forceMsgId(Id msgId) { _msgId = msgId; }
+
+ const MessageType& getType() const { return _type; }
+
+ void setPriority(Priority p) { _priority = p; }
+ Priority getPriority() const { return _priority; }
+
+ const StorageMessageAddress* getAddress() const { return _address.get(); }
+
+ void setAddress(const StorageMessageAddress& address)
+ { _address.reset(new StorageMessageAddress(address)); }
+
+ void setMemoryToken(std::unique_ptr<framework::MemoryToken> token) {
+ _memoryToken = std::move(token);
+ }
+
+ /**
+ Returns the approximate memory footprint of a storage message.
+ By default, returns 50 bytes. This only needs to be overriden if the
+ message potentially can be much larger than this.
+ */
+ virtual uint32_t getMemoryFootprint() const {
+ return 50;
+ }
+
+ /**
+ * Used by storage to remember the context in which this message was
+ * created, whether it was a storageprotocol message, a documentprotocol
+ * message, or an RPC call.
+ */
+ void setTransportContext(std::unique_ptr<TransportContext> context)
+ { _transportContext = std::move(context); }
+
+ std::unique_ptr<TransportContext> getTransportContext() const
+ { return std::move(_transportContext); }
+
+ /**
+ * This method is overloaded in subclasses and will call the correct
+ * method in the MessageHandler interface.
+ */
+ virtual bool callHandler(MessageHandler&,
+ const StorageMessage::SP&) const = 0;
+
+ const documentapi::LoadType& getLoadType() const { return _loadType; }
+ void setLoadType(const documentapi::LoadType& type) { _loadType = type; }
+
+ mbus::Trace& getTrace() { return _trace; }
+ const mbus::Trace& getTrace() const { return _trace; }
+
+ /**
+ Sets the trace object for this message.
+ */
+ void setTrace(const mbus::Trace &trace) { _trace = trace; }
+
+ /**
+ * Cheap version of tostring().
+ */
+ virtual vespalib::string getSummary() const { return toString(); };
+
+ virtual document::BucketId getBucketId() const { return document::BucketId(); }
+ virtual bool hasSingleBucketId() const { return false; }
+};
+
+} // api
+} // storage
+
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp
new file mode 100644
index 00000000000..aa473cd90a1
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/storageapi/messageapi/storagereply.h>
+
+#include <vespa/storageapi/messageapi/storagecommand.h>
+
+namespace storage {
+namespace api {
+
+StorageReply::StorageReply(const StorageCommand& cmd, ReturnCode code)
+ : StorageMessage(cmd.getType().getReplyType(), cmd.getMsgId()),
+ _result(code)
+{
+ setPriority(cmd.getPriority());
+ if (cmd.getAddress()) {
+ setAddress(*cmd.getAddress());
+ }
+ setTrace(cmd.getTrace());
+ setTransportContext(cmd.getTransportContext());
+}
+
+void
+StorageReply::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ (void) verbose; (void) indent;
+ out << "StorageReply(" << _type.getName() << ", "
+ << _result.toString() << ")";
+}
+
+} // api
+} // storage
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.h b/storageapi/src/vespa/storageapi/messageapi/storagereply.h
new file mode 100644
index 00000000000..65475335519
--- /dev/null
+++ b/storageapi/src/vespa/storageapi/messageapi/storagereply.h
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class storage::api::StorageReply
+ * @ingroup messageapi
+ *
+ * @brief Superclass for all storage replies.
+ *
+ * A storage reply is a storage message sent in reply to a storage command.
+ *
+ * @version $Id$
+ */
+
+#pragma once
+
+#include <vespa/storageapi/messageapi/returncode.h>
+#include <vespa/storageapi/messageapi/storagemessage.h>
+
+namespace storage {
+namespace api {
+
+class StorageCommand;
+
+class StorageReply : public StorageMessage {
+ ReturnCode _result;
+
+protected:
+ explicit StorageReply(const StorageCommand& cmd,
+ ReturnCode code = ReturnCode(ReturnCode::OK));
+
+public:
+ DECLARE_POINTER_TYPEDEFS(StorageReply);
+
+ void setResult(const ReturnCode& r) { _result = r; }
+ void setResult(ReturnCode::Result r) { _result = ReturnCode(r); }
+ const ReturnCode& getResult() const { return _result; }
+
+ /** Overload this to get more descriptive output. */
+ virtual void print(std::ostream& out,
+ bool verbose,
+ const std::string& indent) const;
+};
+
+} // api
+} // storage
+