summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@oath.com>2018-09-28 12:01:47 +0200
committerJon Bratseth <bratseth@oath.com>2018-09-28 12:01:47 +0200
commita0494eede7b7fa9e9c0abbef1b263d6a11de5c64 (patch)
treec0f390c31de0cd2ddb27e181a3ea100a36fe9940
parent2aa1a397cf63b7db49866056341311a8e501f270 (diff)
parent157c8b77cfe432d0476f91eb87780e6b1fcd499f (diff)
Merge with master
-rw-r--r--config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java100
-rw-r--r--config-application-package/src/main/java/com/yahoo/config/application/PreProcessor.java7
-rw-r--r--config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java5
-rw-r--r--config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java3
-rw-r--r--config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java1
-rw-r--r--config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java28
-rw-r--r--config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java66
-rw-r--r--config-application-package/src/test/resources/multienvapp/services.xml11
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java9
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java5
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java1
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java6
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java6
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java1
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java10
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java25
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java4
-rw-r--r--config-model/src/test/examples/duplicatenamesindoc.sd12
-rw-r--r--config-model/src/test/examples/duplicatenamesinsearchdifferenttype.sd12
-rw-r--r--config-model/src/test/examples/incorrectsummarytypes.sd19
-rw-r--r--config-model/src/test/examples/name-check.sd20
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java13
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java36
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java8
-rw-r--r--container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java25
-rw-r--r--container-dev/pom.xml4
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java1
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java1
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java15
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java11
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java33
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java13
-rw-r--r--docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java1
-rw-r--r--document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java9
-rw-r--r--document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java6
-rwxr-xr-xdocumentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java85
-rwxr-xr-xdocumentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java15
-rwxr-xr-xdocumentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories50.java917
-rwxr-xr-xdocumentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories51.java137
-rw-r--r--documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java916
-rw-r--r--documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolTest.java36
-rw-r--r--documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java904
-rw-r--r--documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java121
-rw-r--r--documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java850
-rwxr-xr-xdocumentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java6
-rw-r--r--documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java2
-rw-r--r--documentapi/src/tests/messagebus/messagebus_test.cpp5
-rw-r--r--documentapi/src/tests/messages/CMakeLists.txt19
-rw-r--r--documentapi/src/tests/messages/messages50app.cpp5
-rw-r--r--documentapi/src/tests/messages/messages50test.cpp198
-rw-r--r--documentapi/src/tests/messages/messages50test.h2
-rw-r--r--documentapi/src/tests/messages/messages51app.cpp5
-rw-r--r--documentapi/src/tests/messages/messages52test.cpp1
-rw-r--r--documentapi/src/tests/policies/policies_test.cpp1
-rw-r--r--documentapi/src/vespa/documentapi/documentapi.h2
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp85
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/documentprotocol.h2
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt2
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp82
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h81
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.cpp14
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h30
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp12
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp5
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp64
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/routablefactories50.h13
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/routablefactories51.h2
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateMessage.datbin207 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateReply.datbin20 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-BatchMessage.datbin43 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-BatchReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorMessage.datbin173 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorReply.datbin65 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorMessage.datbin22 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-DocumentListMessage.datbin68 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-DocumentListReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsMessage.datbin117 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-GetBucketListMessage.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-GetBucketListReply.datbin56 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateMessage.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateReply.datbin53 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-GetDocumentMessage.datbin25 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-GetDocumentReply.datbin43 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-MapVisitorMessage.datbin37 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-MapVisitorReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-MultiOperationMessage.datbin90 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-MultiOperationReply.datbin13 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-PutDocumentMessage.datbin46 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-PutDocumentReply.datbin13 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-QueryResultReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentMessage.datbin21 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentReply.datbin14 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessage.datbin34 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageGroup.datbin24 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageUser.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-SearchResultReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-StatBucketMessage.datbin32 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-StatBucketReply.datbin50 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentMessage.datbin94 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentReply.datbin14 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoMessage.datbin72 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-cpp-WrongDistributionReply.datbin32 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateMessage.datbin207 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateReply.datbin20 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-BatchMessage.datbin43 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-BatchReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-CreateVisitorMessage.datbin173 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-CreateVisitorReply.datbin65 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-DestroyVisitorMessage.datbin22 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-DestroyVisitorReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-DocumentListMessage.datbin68 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-DocumentListReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-DocumentSummaryReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-EmptyBucketsMessage.datbin117 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-EmptyBucketsReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-GetBucketListMessage.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-GetBucketListReply.datbin56 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-GetBucketStateMessage.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-GetBucketStateReply.datbin53 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-GetDocumentMessage.datbin25 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-GetDocumentReply.datbin43 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-MapVisitorMessage.datbin37 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-MapVisitorReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-MultiOperationMessage.datbin90 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-MultiOperationReply.datbin13 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-PutDocumentMessage.datbin46 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-PutDocumentReply.datbin13 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-QueryResultReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-RemoveDocumentMessage.datbin21 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-RemoveDocumentReply.datbin14 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessage.datbin34 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageGroup.datbin24 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageUser.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-RemoveLocationReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-SearchResultReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-StatBucketMessage.datbin32 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-StatBucketReply.datbin50 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-UpdateDocumentMessage.datbin94 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-UpdateDocumentReply.datbin14 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-VisitorInfoMessage.datbin72 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-VisitorInfoReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5-java-WrongDistributionReply.datbin32 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateMessage.datbin207 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateReply.datbin20 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-BatchMessage.datbin43 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-BatchReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorMessage.datbin183 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorReply.datbin65 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorMessage.datbin22 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-DocumentIgnoredReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListMessage.datbin68 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-1.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-2.datbin57 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-3.datbin57 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsMessage.datbin117 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListMessage.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListReply.datbin56 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateMessage.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateReply.datbin53 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentMessage.datbin32 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentReply.datbin43 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorMessage.datbin37 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationMessage.datbin90 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationReply.datbin13 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentMessage.datbin46 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentReply.datbin13 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-1.datbin37 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-2.datbin68 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-3.datbin68 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-4.datbin121 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-5.datbin121 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentMessage.datbin21 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentReply.datbin14 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessage.datbin34 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageGroup.datbin24 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageUser.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-1.datbin29 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-2.datbin60 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-3.datbin60 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-4.datbin113 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-5.datbin113 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketMessage.datbin32 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketReply.datbin50 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentMessage.datbin94 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentReply.datbin14 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoMessage.datbin72 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-cpp-WrongDistributionReply.datbin32 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateMessage.datbin207 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateReply.datbin20 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-BatchMessage.datbin43 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-BatchReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorMessage.datbin183 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorReply.datbin65 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorMessage.datbin22 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-DocumentIgnoredReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-DocumentListMessage.datbin68 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-DocumentListReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-DocumentSummaryReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsMessage.datbin117 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-GetBucketListMessage.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-GetBucketListReply.datbin56 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateMessage.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateReply.datbin53 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-GetDocumentMessage.datbin32 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-GetDocumentReply.datbin43 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-MapVisitorMessage.datbin37 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-MapVisitorReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-MultiOperationMessage.datbin90 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-MultiOperationReply.datbin13 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-PutDocumentMessage.datbin46 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-PutDocumentReply.datbin13 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-QueryResultReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentMessage.datbin21 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentReply.datbin14 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessage.datbin34 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageGroup.datbin24 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageUser.datbin17 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-SearchResultReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-StatBucketMessage.datbin32 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-StatBucketReply.datbin50 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentMessage.datbin94 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentReply.datbin14 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoMessage.datbin72 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoReply.datbin5 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.1-java-WrongDistributionReply.datbin32 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateMessage.datbin207 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateReply.datbin20 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateMessage.datbin207 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateReply.datbin20 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateMessage.datbin207 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateReply.datbin20 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-1.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-1.dat)bin17 -> 17 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-2.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-2.dat)bin57 -> 57 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-3.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-3.dat)bin57 -> 57 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-1.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-1.dat)bin37 -> 37 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-2.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-2.dat)bin68 -> 68 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-3.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-3.dat)bin68 -> 68 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-4.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-4.dat)bin121 -> 121 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-5.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-5.dat)bin121 -> 121 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-1.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-1.dat)bin29 -> 29 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-2.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-2.dat)bin60 -> 60 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-3.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-3.dat)bin60 -> 60 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-4.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-4.dat)bin113 -> 113 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-5.dat (renamed from documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-5.dat)bin113 -> 113 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateMessage.datbin207 -> 0 bytes
-rw-r--r--documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateReply.datbin20 -> 0 bytes
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java32
-rw-r--r--jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java58
-rw-r--r--jdisc_core/pom.xml1
-rw-r--r--jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java47
-rw-r--r--jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java6
-rw-r--r--jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java4
-rw-r--r--jdisc_core_test/test_bundles/app-g-act/pom.xml6
-rw-r--r--jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/ApplicationG.java (renamed from jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/ApplicationG.java)2
-rw-r--r--jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/MyBundleActivator.java (renamed from jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/MyBundleActivator.java)2
-rw-r--r--jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/MyService.java (renamed from jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/MyService.java)2
-rw-r--r--jdisc_core_test/test_bundles/my-bundle-activator/pom.xml4
-rw-r--r--jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/my_act/MyBundleActivator.java (renamed from jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/MyBundleActivator.java)2
-rw-r--r--jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/my_act/MyService.java (renamed from jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/MyService.java)2
-rw-r--r--jrt/src/com/yahoo/jrt/CryptoEngine.java24
-rw-r--r--jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java12
-rw-r--r--jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java12
-rw-r--r--jrt/tests/com/yahoo/jrt/TlsDetectionTest.java17
-rw-r--r--metrics/src/vespa/metrics/countmetric.h5
-rw-r--r--metrics/src/vespa/metrics/countmetric.hpp16
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerNetworking.java2
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java19
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java26
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java9
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java59
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java4
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java4
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java43
-rw-r--r--parent/pom.xml8
-rw-r--r--storage/src/tests/distributor/twophaseupdateoperationtest.cpp1
-rw-r--r--storage/src/tests/persistence/filestorage/filestormanagertest.cpp2
-rw-r--r--storage/src/tests/storageserver/documentapiconvertertest.cpp56
-rw-r--r--storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp2
-rw-r--r--storage/src/vespa/storage/common/messagebucket.cpp3
-rw-r--r--storage/src/vespa/storage/distributor/distributor.cpp2
-rw-r--r--storage/src/vespa/storage/distributor/externaloperationhandler.cpp11
-rw-r--r--storage/src/vespa/storage/distributor/operations/external/getoperation.cpp12
-rw-r--r--storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp2
-rw-r--r--storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h1
-rw-r--r--storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp4
-rw-r--r--storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp14
-rw-r--r--storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp4
-rw-r--r--storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp11
-rw-r--r--storage/src/vespa/storage/persistence/filestorage/filestormanager.h1
-rw-r--r--storage/src/vespa/storage/persistence/mergehandler.cpp4
-rw-r--r--storage/src/vespa/storage/persistence/persistencethread.cpp10
-rw-r--r--storage/src/vespa/storage/persistence/persistenceutil.cpp4
-rw-r--r--storage/src/vespa/storage/storageserver/documentapiconverter.cpp12
-rw-r--r--storage/src/vespa/storage/storageserver/mergethrottler.cpp20
-rw-r--r--storageapi/src/tests/mbusprot/storageprotocoltest.cpp52
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp11
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h8
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp99
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h6
-rw-r--r--storageapi/src/vespa/storageapi/message/CMakeLists.txt1
-rw-r--r--storageapi/src/vespa/storageapi/message/batch.cpp173
-rw-r--r--storageapi/src/vespa/storageapi/message/batch.h186
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/messagehandler.h16
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp4
-rw-r--r--storageapi/src/vespa/storageapi/messageapi/storagemessage.h8
-rwxr-xr-xvespabase/src/start-cbinaries.sh2
-rw-r--r--vespajlib/pom.xml4
-rw-r--r--vespajlib/src/main/java/com/yahoo/io/NativeIO.java104
-rw-r--r--vespajlib/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java66
-rw-r--r--vespajlib/src/test/java/com/yahoo/io/NativeIOTestCase.java35
-rw-r--r--vespalib/CMakeLists.txt3
-rw-r--r--vespalib/src/apps/vespa-drop-file-from-cache/.gitignore1
-rw-r--r--vespalib/src/apps/vespa-drop-file-from-cache/CMakeLists.txt9
-rw-r--r--vespalib/src/apps/vespa-drop-file-from-cache/drop_file_from_cache.cpp32
-rw-r--r--vespalib/src/tests/drop-file-from-cache/CMakeLists.txt8
-rw-r--r--vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp25
-rw-r--r--vespalib/src/tests/net/tls/protocol_snooping/CMakeLists.txt9
-rw-r--r--vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp78
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt1
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h6
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/protocol_snooping.cpp127
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/protocol_snooping.h32
-rw-r--r--vespalib/src/vespa/vespalib/util/slaveproc.h15
342 files changed, 3019 insertions, 3859 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java
index d68b36e063c..16accb368fd 100644
--- a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java
+++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.config.application;
+import com.google.common.collect.ImmutableSet;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.RegionName;
import com.yahoo.log.LogLevel;
@@ -10,8 +11,16 @@ import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import javax.xml.transform.TransformerException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.logging.Logger;
+import java.util.stream.Collectors;
/**
* Handles overrides in a XML document according to the rules defined for multi environment application packages.
@@ -23,8 +32,7 @@ import java.util.logging.Logger;
* 3. When multiple XML elements with the same name is specified (i.e. when specifying search or docproc chains),
* the id attribute of the element is used together with the element name when applying directives
*
- * @author lulf
- * @since 5.22
+ * @author Ulf Lilleengen
*/
class OverrideProcessor implements PreProcessor {
@@ -71,15 +79,13 @@ class OverrideProcessor implements PreProcessor {
}
private Context getParentContext(Element parent, Context context) {
- Optional<Environment> environment = context.environment;
- RegionName region = context.region;
- if ( ! environment.isPresent()) {
- environment = getEnvironment(parent);
- }
- if (region.isDefault()) {
- region = getRegion(parent);
- }
- return Context.create(environment, region);
+ Set<Environment> environments = context.environments;
+ Set<RegionName> regions = context.regions;
+ if (environments.isEmpty())
+ environments = getEnvironments(parent);
+ if (regions.isEmpty())
+ regions = getRegions(parent);
+ return Context.create(environments, regions);
}
/**
@@ -100,13 +106,13 @@ class OverrideProcessor implements PreProcessor {
*/
private void checkConsistentInheritance(List<Element> children, Context context) {
for (Element child : children) {
- Optional<Environment> env = getEnvironment(child);
- RegionName reg = getRegion(child);
- if (env.isPresent() && context.environment.isPresent() && !env.equals(context.environment)) {
- throw new IllegalArgumentException("Environment in child (" + env.get() + ") differs from that inherited from parent (" + context.environment + ") at " + child);
+ Set<Environment> environments = getEnvironments(child);
+ Set<RegionName> regions = getRegions(child);
+ if ( ! environments.isEmpty() && ! context.environments.isEmpty() && !environments.equals(context.environments)) {
+ throw new IllegalArgumentException("Environments in child (" + environments + ") differs from that inherited from parent (" + context.environments + ") at " + child);
}
- if (!reg.isDefault() && !context.region.isDefault() && !reg.equals(context.region)) {
- throw new IllegalArgumentException("Region in child (" + reg + ") differs from that inherited from parent (" + context.region + ") at " + child);
+ if ( ! regions.isEmpty() && ! context.regions.isEmpty() && ! regions.equals(context.regions)) {
+ throw new IllegalArgumentException("Regions in child (" + regions + ") differs from that inherited from parent (" + context.regions + ") at " + child);
}
}
}
@@ -118,22 +124,24 @@ class OverrideProcessor implements PreProcessor {
Iterator<Element> elemIt = children.iterator();
while (elemIt.hasNext()) {
Element child = elemIt.next();
- if ( ! matches(getEnvironment(child), getRegion(child))) {
+ if ( ! matches(getEnvironments(child), getRegions(child))) {
parent.removeChild(child);
elemIt.remove();
}
}
}
- private boolean matches(Optional<Environment> elementEnvironment, RegionName elementRegion) {
- if (elementEnvironment.isPresent()) { // match environment
- if (! environment.equals(elementEnvironment.get())) return false;
+ private boolean matches(Set<Environment> elementEnvironments, Set<RegionName> elementRegions) {
+ if ( ! elementEnvironments.isEmpty()) { // match environment
+ if ( ! elementEnvironments.contains(environment)) return false;
}
- if ( ! elementRegion.isDefault()) { // match region
- if ( ! region.equals(elementRegion)) return false;
- // match region but no environment in prod only to avoid a region attribute overriding capacity policies outside prod
- if ( ! elementEnvironment.isPresent() && ! environment.equals(Environment.prod)) return false;
+ if ( ! elementRegions.isEmpty()) { // match region
+ // match region in prod only
+ if ( environment.equals(Environment.prod) && ! elementRegions.contains(region)) return false;
+
+ // explicit region implies prod
+ if ( ! environment.equals(Environment.prod) && elementEnvironments.isEmpty() ) return false;
}
return true;
@@ -174,11 +182,11 @@ class OverrideProcessor implements PreProcessor {
private int getNumberOfOverrides(Element child, Context context) {
int currentMatch = 0;
- Optional<Environment> elementEnvironment = hasEnvironment(child) ? getEnvironment(child) : context.environment;
- RegionName elementRegion = hasRegion(child) ? getRegion(child) : context.region;
- if (elementEnvironment.isPresent() && elementEnvironment.get().equals(environment))
+ Set<Environment> elementEnvironments = hasEnvironment(child) ? getEnvironments(child) : context.environments;
+ Set<RegionName> elementRegions = hasRegion(child) ? getRegions(child) : context.regions;
+ if ( ! elementEnvironments.isEmpty() && elementEnvironments.contains(environment))
currentMatch++;
- if ( ! elementRegion.isDefault() && elementRegion.equals(region))
+ if ( ! elementRegions.isEmpty() && elementRegions.contains(region))
currentMatch++;
return currentMatch;
}
@@ -220,20 +228,16 @@ class OverrideProcessor implements PreProcessor {
return element.hasAttributeNS(XmlPreProcessor.deployNamespaceUri, ATTR_ENV);
}
- private Optional<Environment> getEnvironment(Element element) {
+ private Set<Environment> getEnvironments(Element element) {
String env = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, ATTR_ENV);
- if (env == null || env.isEmpty()) {
- return Optional.empty();
- }
- return Optional.of(Environment.from(env));
+ if (env == null || env.isEmpty()) return Collections.emptySet();
+ return Arrays.stream(env.split(" ")).map(Environment::from).collect(Collectors.toSet());
}
- private RegionName getRegion(Element element) {
+ private Set<RegionName> getRegions(Element element) {
String reg = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, ATTR_REG);
- if (reg == null || reg.isEmpty()) {
- return RegionName.defaultName();
- }
- return RegionName.from(reg);
+ if (reg == null || reg.isEmpty()) return Collections.emptySet();
+ return Arrays.stream(reg.split(" ")).map(RegionName::from).collect(Collectors.toSet());
}
private Map<String, List<Element>> elementsByTagNameAndId(List<Element> children) {
@@ -287,21 +291,21 @@ class OverrideProcessor implements PreProcessor {
*/
private static final class Context {
- final Optional<Environment> environment;
+ final ImmutableSet<Environment> environments;
- final RegionName region;
+ final ImmutableSet<RegionName> regions;
- private Context(Optional<Environment> environment, RegionName region) {
- this.environment = environment;
- this.region = region;
+ private Context(Set<Environment> environments, Set<RegionName> regions) {
+ this.environments = ImmutableSet.copyOf(environments);
+ this.regions = ImmutableSet.copyOf(regions);
}
static Context empty() {
- return new Context(Optional.empty(), RegionName.defaultName());
+ return new Context(ImmutableSet.of(), ImmutableSet.of());
}
- public static Context create(Optional<Environment> environment, RegionName region) {
- return new Context(environment, region);
+ public static Context create(Set<Environment> environments, Set<RegionName> regions) {
+ return new Context(environments, regions);
}
}
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/PreProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/PreProcessor.java
index d331c4432bc..f207a07d3be 100644
--- a/config-application-package/src/main/java/com/yahoo/config/application/PreProcessor.java
+++ b/config-application-package/src/main/java/com/yahoo/config/application/PreProcessor.java
@@ -9,9 +9,10 @@ import java.io.IOException;
/**
* Performs pre-processing of XML document and returns new document that has been processed.
*
- * @author lulf
- * @since 5.21
+ * @author Ulf Lilleengen
*/
public interface PreProcessor {
- public Document process(Document input) throws IOException, TransformerException;
+
+ Document process(Document input) throws IOException, TransformerException;
+
}
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java
index 6e45460a4c4..65120c3677c 100644
--- a/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java
+++ b/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java
@@ -16,7 +16,6 @@ import java.util.logging.Logger;
* Handles getting properties from services.xml and replacing references to properties with their real values
*
* @author hmusum
- * @since 5.22
*/
class PropertiesProcessor implements PreProcessor {
private final static Logger log = Logger.getLogger(PropertiesProcessor.class.getName());
@@ -82,8 +81,8 @@ class PropertiesProcessor implements PreProcessor {
}
private String replaceValue(String propertyValue) {
- /* Use a list with keys sorted by length (longest key first)
- Needed for replacing values where you have overlapping keys */
+ // Use a list with keys sorted by length (longest key first)
+ // Needed for replacing values where you have overlapping keys
ArrayList<String> keys = new ArrayList<>(properties.keySet());
Collections.sort(keys, Collections.reverseOrder(Comparator.comparing(String::length)));
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java
index ceef0a6730e..0bb160319c0 100644
--- a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java
+++ b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java
@@ -22,7 +22,6 @@ import java.util.List;
* and create a new Document which is based on the supplied environment and region
*
* @author hmusum
- * @since 5.22
*/
public class XmlPreProcessor {
@@ -51,7 +50,7 @@ public class XmlPreProcessor {
public Document run() throws ParserConfigurationException, IOException, SAXException, TransformerException {
DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder();
- final Document document = docBuilder.parse(new InputSource(xmlInput));
+ Document document = docBuilder.parse(new InputSource(xmlInput));
return execute(document);
}
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
index 7ca9bcf48f3..1724bad765f 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
@@ -305,6 +305,7 @@ public class FilesApplicationPackage implements ApplicationPackage {
/**
* Verify that two sets of search definitions are disjoint (TODO: everything except error message is very generic).
+ *
* @param fileSds Set of search definitions from file
* @param bundleSds Set of search definitions from bundles
*/
diff --git a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java
index d3c2b672ee5..a456924673d 100644
--- a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java
+++ b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java
@@ -13,44 +13,48 @@ import java.io.*;
import java.nio.file.NoSuchFileException;
/**
- * @author lulf
- * @since 5.22
+ * @author Ulf Lilleengen
*/
public class IncludeProcessorTest {
@Test
- public void testInclude() throws IOException, SAXException, XMLStreamException, ParserConfigurationException, TransformerException {
+ public void testInclude() throws IOException, SAXException, ParserConfigurationException, TransformerException {
File app = new File("src/test/resources/multienvapp");
DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder();
- String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" +
+ String expected =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+ "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" +
" <preprocess:properties>\n" +
" <qrs.port>4099</qrs.port>\n" +
" <qrs.port>5000</qrs.port>\n" +
" </preprocess:properties>\n" +
" <preprocess:properties deploy:environment='prod'>\n" +
" <qrs.port deploy:region='us-west'>5001</qrs.port>" +
- " <qrs.port deploy:region='us-east'>5002</qrs.port>" +
+ " <qrs.port deploy:region='us-east us-central'>5002</qrs.port>" +
" </preprocess:properties>\n" +
" <admin version=\"2.0\">\n" +
" <adminserver hostalias=\"node0\"/>\n" +
" </admin>\n" +
- " <admin deploy:environment=\"prod\" version=\"2.0\">\n" +
+ " <admin deploy:environment=\"staging prod\" deploy:region=\"us-east us-central\" version=\"2.0\">\n" +
" <adminserver hostalias=\"node1\"/>\n" +
" </admin>\n" +
" <content id=\"foo\" version=\"1.0\">\n" +
" <redundancy>1</redundancy><documents>\n" +
" <document mode=\"index\" type=\"music.sd\"/>\n" +
- "</documents><nodes>\n" +
+ " </documents><nodes>\n" +
" <node distribution-key=\"0\" hostalias=\"node0\"/>\n" +
- "</nodes><nodes deploy:environment=\"prod\">\n" +
+ " </nodes>" +
+ " <nodes deploy:environment=\"prod\">\n" +
" <node distribution-key=\"0\" hostalias=\"node0\"/>\n" +
" <node distribution-key=\"1\" hostalias=\"node1\"/>\n" +
- "</nodes><nodes deploy:environment=\"prod\" deploy:region=\"us-west\">\n" +
+ " </nodes>" +
+ " <nodes deploy:environment=\"prod\" deploy:region=\"us-west\">\n" +
" <node distribution-key=\"0\" hostalias=\"node0\"/>\n" +
" <node distribution-key=\"1\" hostalias=\"node1\"/>\n" +
" <node distribution-key=\"2\" hostalias=\"node2\"/>\n" +
- "</nodes></content>\n" +
+ " </nodes>" +
+ "</content>\n" +
"<jdisc id=\"stateless\" version=\"1.0\">\n" +
" <search deploy:environment=\"prod\">\n" +
" <chain id=\"common\">\n" +
@@ -68,7 +72,7 @@ public class IncludeProcessorTest {
" </nodes>\n" +
"</jdisc></services>";
- Document doc = (new IncludeProcessor(app)).process(docBuilder.parse(Xml.getServices(app)));
+ Document doc = new IncludeProcessor(app).process(docBuilder.parse(Xml.getServices(app)));
// System.out.println(Xml.documentAsString(doc));
TestBase.assertDocument(expected, doc);
}
@@ -77,7 +81,7 @@ public class IncludeProcessorTest {
public void testRequiredIncludeIsDefault() throws ParserConfigurationException, IOException, SAXException, TransformerException {
File app = new File("src/test/resources/multienvapp_failrequired");
DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder();
- (new IncludeProcessor(app)).process(docBuilder.parse(Xml.getServices(app)));
+ new IncludeProcessor(app).process(docBuilder.parse(Xml.getServices(app)));
}
}
diff --git a/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java
index b20437bc259..3827fe2ad42 100644
--- a/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java
+++ b/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java
@@ -23,8 +23,10 @@ public class XmlPreprocessorTest {
private static final File services = new File(appDir, "services.xml");
@Test
- public void testPreProcessing() throws IOException, SAXException, XMLStreamException, ParserConfigurationException, TransformerException {
- String expectedDev = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" +
+ public void testPreProcessing() throws IOException, SAXException, ParserConfigurationException, TransformerException {
+ String expectedDev =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+ "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" +
" <admin version=\"2.0\">\n" +
" <adminserver hostalias=\"node0\"/>\n" +
" </admin>\n" +
@@ -46,14 +48,40 @@ public class XmlPreprocessorTest {
" </nodes>\n" +
" </jdisc>\n" +
"</services>";
+ TestBase.assertDocument(expectedDev, new XmlPreProcessor(appDir, services, Environment.dev, RegionName.from("default")).run());
- Document docDev = (new XmlPreProcessor(appDir, services, Environment.dev, RegionName.from("default")).run());
- TestBase.assertDocument(expectedDev, docDev);
-
+ String expectedStaging =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+ "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" +
+ " <admin version=\"2.0\">\n" +
+ " <adminserver hostalias=\"node1\"/>\n" + // Difference from dev: node1
+ " </admin>\n" +
+ " <content id=\"foo\" version=\"1.0\">\n" +
+ " <redundancy>1</redundancy>\n" +
+ " <documents>\n" +
+ " <document mode=\"index\" type=\"music.sd\"/>\n" +
+ " </documents>\n" +
+ " <nodes>\n" +
+ " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" +
+ " </nodes>\n" +
+ " </content>\n" +
+ " <jdisc id=\"stateless\" version=\"1.0\">\n" +
+ " <search/>\n" +
+ " <component bundle=\"foobundle\" class=\"MyFoo\" id=\"foo\"/>\n" +
+ "" + // Difference from dev: no TestBar
+ " <nodes>\n" +
+ " <node hostalias=\"node0\" baseport=\"5000\"/>\n" +
+ " </nodes>\n" +
+ " </jdisc>\n" +
+ "</services>";
+ // System.out.println(Xml.documentAsString(new XmlPreProcessor(appDir, services, Environment.staging, RegionName.from("default")).run()));
+ TestBase.assertDocument(expectedStaging, new XmlPreProcessor(appDir, services, Environment.staging, RegionName.from("default")).run());
- String expectedUsWest = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" +
+ String expectedUsWest =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+ "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" +
" <admin version=\"2.0\">\n" +
- " <adminserver hostalias=\"node1\"/>\n" +
+ " <adminserver hostalias=\"node0\"/>\n" +
" </admin>\n" +
" <content id=\"foo\" version=\"1.0\">\n" +
" <redundancy>1</redundancy>\n" +
@@ -81,12 +109,11 @@ public class XmlPreprocessorTest {
" </nodes>\n" +
" </jdisc>\n" +
"</services>";
+ TestBase.assertDocument(expectedUsWest, new XmlPreProcessor(appDir, services, Environment.prod, RegionName.from("us-west")).run());
- Document docUsWest = (new XmlPreProcessor(appDir, services, Environment.prod, RegionName.from("us-west"))).run();
- // System.out.println(Xml.documentAsString(docUsWest));
- TestBase.assertDocument(expectedUsWest, docUsWest);
-
- String expectedUsEast = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" +
+ String expectedUsEastAndCentral =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+ "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" +
" <admin version=\"2.0\">\n" +
" <adminserver hostalias=\"node1\"/>\n" +
" </admin>\n" +
@@ -115,14 +142,16 @@ public class XmlPreprocessorTest {
" </nodes>\n" +
" </jdisc>\n" +
"</services>";
-
- Document docUsEast = (new XmlPreProcessor(appDir, services, Environment.prod, RegionName.from("us-east"))).run();
- TestBase.assertDocument(expectedUsEast, docUsEast);
+ TestBase.assertDocument(expectedUsEastAndCentral,
+ new XmlPreProcessor(appDir, services, Environment.prod, RegionName.from("us-east")).run());
+ TestBase.assertDocument(expectedUsEastAndCentral,
+ new XmlPreProcessor(appDir, services, Environment.prod, RegionName.from("us-central")).run());
}
@Test
- public void testPropertiesWithOverlappingNames() throws IOException, SAXException, XMLStreamException, ParserConfigurationException, TransformerException {
- String input = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+ public void testPropertiesWithOverlappingNames() throws IOException, SAXException, ParserConfigurationException, TransformerException {
+ String input =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
"<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" +
" <preprocess:properties>" +
" <sherpa.host>gamma-usnc1.dht.yahoo.com</sherpa.host>" +
@@ -146,7 +175,8 @@ public class XmlPreprocessorTest {
" </admin>" +
"</services>";
- String expectedProd = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+ String expectedProd =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
"<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" +
" <config name='a'>" +
" <a>36000</a>" +
diff --git a/config-application-package/src/test/resources/multienvapp/services.xml b/config-application-package/src/test/resources/multienvapp/services.xml
index 3d4a2087c57..d0f43f0b025 100644
--- a/config-application-package/src/test/resources/multienvapp/services.xml
+++ b/config-application-package/src/test/resources/multienvapp/services.xml
@@ -1,5 +1,6 @@
<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
<services version='1.0' xmlns:deploy="vespa" xmlns:preprocess="properties">
+
<preprocess:properties>
<qrs.port>4099</qrs.port>
<qrs.port>5000</qrs.port>
@@ -7,17 +8,23 @@
<preprocess:properties deploy:environment='prod'>
<qrs.port deploy:region='us-west'>5001</qrs.port>
- <qrs.port deploy:region='us-east'>5002</qrs.port>
+ <qrs.port deploy:region='us-east us-central'>5002</qrs.port>
</preprocess:properties>
+
<admin version='2.0'>
<adminserver hostalias='node0'/>
</admin>
- <admin version='2.0' deploy:environment='prod'>
+
+ <admin version='2.0' deploy:environment='staging prod' deploy:region='us-east us-central'>
<adminserver hostalias='node1'/>
</admin>
+
<preprocess:include file='jdisc.xml'/>
+
<content version='1.0' id='foo'>
<preprocess:include file='content/content_foo.xml'/>
</content>
+
<preprocess:include file='doesnotexist.xml' required='false' />
+
</services>
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
index 6c7396d607b..dc6c17e425e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
@@ -41,17 +41,12 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
derive(search);
}
- private boolean hasFullIndexingDocprocRights(ImmutableSDField field) {
- Attribute self = field.getAttributes().get(field.getName());
- return (!field.isExtraField() || ((self != null) && self.isMutable()));
- }
-
@Override
protected void derive(ImmutableSDField field, Search search) {
if (field.isImportedField()) {
return;
}
- if (hasFullIndexingDocprocRights(field)) {
+ if (field.hasFullIndexingDocprocRights()) {
docFields.add(field.getName());
}
if (field.usesStructOrMap() &&
@@ -93,7 +88,7 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
if (modifiesSelf(exp)) {
later.add(exp);
} else {
- ilscriptBuilder.content(exp.toString());
+ ilscriptBuilder.content(exp.toString());
}
fieldFetcher.visit(exp);
touchedFields.addAll(fieldFetcher.touchedFields());
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java
index 125e6b976b5..ff11c2fdf22 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java
@@ -169,4 +169,9 @@ public class ImmutableImportedSDField implements ImmutableSDField {
return new UnsupportedOperationException("'" + aspect + "' is not meaningful or relevant for an imported field.");
}
+ @Override
+ public boolean hasFullIndexingDocprocRights() {
+ return importedField.targetField().hasFullIndexingDocprocRights();
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java
index 70553d4b57c..6fe8a4da92b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java
@@ -81,4 +81,5 @@ public interface ImmutableSDField {
*/
Field asField();
+ boolean hasFullIndexingDocprocRights();
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
index 0590027a402..e310b6b6858 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
@@ -799,4 +799,10 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
return aliasToName;
}
+ @Override
+ public boolean hasFullIndexingDocprocRights() {
+ Attribute self = getAttributes().get(getName());
+ return (!isExtraField() || ((self != null) && self.isMutable()));
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
index aeab2bb6638..b0129c0a836 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
@@ -97,13 +97,11 @@ public class IndexingInputs extends Processor {
@Override
protected void doVisit(Expression exp) {
if ( ! (exp instanceof InputExpression)) return;
-
- SDDocumentType docType = search.getDocument();
String inputField = ((InputExpression)exp).getFieldName();
- if (docType.getField(inputField) != null) return;
+ if (search.getField(inputField).hasFullIndexingDocprocRights()) return;
fail(search, field, "Indexing script refers to field '" + inputField + "' which does not exist " +
- "in document type '" + docType.getName() + "'.");
+ "in document type '" + search.getDocument().getName() + "', and is not a mutable attribute.");
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
index 15d295736c1..1af2a979cb4 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
@@ -31,6 +31,7 @@ public class Processing {
UriHack::new,
LiteralBoost::new,
TagType::new,
+ ValidateFieldTypesDocumentsOnly::new,
IndexingInputs::new,
OptimizeIlscript::new,
ValidateFieldWithIndexSettingsCreatesIndex::new,
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
index 21b7f1d2675..1cf3d61d8f1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
@@ -29,16 +29,24 @@ public class ValidateFieldTypes extends Processor {
@Override
public void process(boolean validate, boolean documentsOnly) {
- if ( ! validate) return;
+ if (!validate) return;
String searchName = search.getName();
Map<String, DataType> seenFields = new HashMap<>();
+ verifySearchAndDocFields(searchName, seenFields);
+ verifySummaryFields(searchName, seenFields);
+ }
+
+ final protected void verifySearchAndDocFields(String searchName, Map<String, DataType> seenFields) {
search.allFields().forEach(field -> {
checkFieldType(searchName, "index field", field.getName(), field.getDataType(), seenFields);
for (Map.Entry<String, Attribute> entry : field.getAttributes().entrySet()) {
checkFieldType(searchName, "attribute", entry.getKey(), entry.getValue().getDataType(), seenFields);
}
});
+
+ }
+ final protected void verifySummaryFields(String searchName, Map<String, DataType> seenFields) {
for (DocumentSummary summary : search.getSummaries().values()) {
for (SummaryField field : summary.getSummaryFields()) {
checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), seenFields);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java
new file mode 100644
index 00000000000..ff3e638fff1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java
@@ -0,0 +1,25 @@
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ValidateFieldTypesDocumentsOnly extends ValidateFieldTypes {
+ public ValidateFieldTypesDocumentsOnly(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+ super(search, deployLogger, rankProfileRegistry, queryProfiles);
+ }
+
+ @Override
+ public void process(boolean validate, boolean documentsOnly) {
+ if ( ! validate) return;
+
+ String searchName = search.getName();
+ Map<String, DataType> seenFields = new HashMap<>();
+ verifySearchAndDocFields(searchName, seenFields);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
index 73d77406700..8e7c20d4234 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
@@ -349,6 +349,7 @@ public class VespaMetricSet {
metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.createiterator.sum.count.rate"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.visit.sum.count.rate"));
metrics.add(new Metric("vds.filestor.alldisks.queuesize.average","diskqueuesize"));
metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average","diskqueuewait"));
@@ -356,11 +357,14 @@ public class VespaMetricSet {
metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average"));
metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average"));
metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate","visit"));
+ metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.average"));
+ metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.average"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.average"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.average"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.average"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.average"));
+ metrics.add(new Metric("vds.filestor.alldisks.allthreads.createiterator.sum.latency.average"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.visit.sum.latency.average"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.splitbuckets.count.rate"));
metrics.add(new Metric("vds.filestor.alldisks.allthreads.joinbuckets.count.rate"));
diff --git a/config-model/src/test/examples/duplicatenamesindoc.sd b/config-model/src/test/examples/duplicatenamesindoc.sd
deleted file mode 100644
index 3c55efe953d..00000000000
--- a/config-model/src/test/examples/duplicatenamesindoc.sd
+++ /dev/null
@@ -1,12 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-search duplicatenamesindoc {
-document {
- field foo type int {
- indexing: attribute
- }
-
- field fOo type string {
- indexing: index
- }
-}
-}
diff --git a/config-model/src/test/examples/duplicatenamesinsearchdifferenttype.sd b/config-model/src/test/examples/duplicatenamesinsearchdifferenttype.sd
deleted file mode 100644
index 502a4bb7aff..00000000000
--- a/config-model/src/test/examples/duplicatenamesinsearchdifferenttype.sd
+++ /dev/null
@@ -1,12 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-search duplicatenamesinsearch {
-document {
- field grpphotoids64 type string { }
-}
-
-field grpphotoids64 type array<long> {
- indexing: input grpphotoids64 | split " " | for_each {
- base64decode } | attribute
- }
-
-}
diff --git a/config-model/src/test/examples/incorrectsummarytypes.sd b/config-model/src/test/examples/incorrectsummarytypes.sd
deleted file mode 100644
index 9304ad5ca20..00000000000
--- a/config-model/src/test/examples/incorrectsummarytypes.sd
+++ /dev/null
@@ -1,19 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-search incorrectsummarytypes {
-
- document incorrectsummarytypes {
-
- field somestring type string {
- indexing: summary
- }
-
- }
-
- document-summary incorrect {
-
- summary somestring type int {
- }
-
- }
-
-}
diff --git a/config-model/src/test/examples/name-check.sd b/config-model/src/test/examples/name-check.sd
deleted file mode 100644
index bd7c0713e3a..00000000000
--- a/config-model/src/test/examples/name-check.sd
+++ /dev/null
@@ -1,20 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-# An entry-level configuration.
-# You can get a reasonable configuration by only configuring
-# a document
-search simple {
-
- document name-check {
-
- field title type string {
- indexing: summary | index
- }
-
- # reserved name, should trigger error
- field sddocname type string {
- indexing: index
- }
-
- }
-
-}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
index cadb2cda324..71db60065aa 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
@@ -165,7 +165,7 @@ public class AttributeSettingsTestCase extends SearchDefinitionTestCase {
" }\n" +
" }\n" +
" field m type long {\n" +
- " indexing: 0 | to_long | attribute\n" +
+ " indexing: attribute\n" +
" attribute: mutable\n" +
" }\n" +
" field f type long {\n" +
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java
index f11799495a2..627394bb6ea 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java
@@ -17,7 +17,18 @@ public class IncorrectSummaryTypesTestCase extends SearchDefinitionTestCase {
@Test
public void testImportingIncorrect() throws IOException, ParseException {
try {
- SearchBuilder.buildFromFile("src/test/examples/incorrectsummarytypes.sd");
+ SearchBuilder.createFromString(
+ "search incorrectsummarytypes {\n" +
+ " document incorrectsummarytypes {\n" +
+ " field somestring type string {\n" +
+ " indexing: summary\n" +
+ " }\n" +
+ " }\n" +
+ " document-summary incorrect {\n" +
+ " summary somestring type int {\n" +
+ " }\n" +
+ " }\n" +
+ "}\n");
fail("processing should have failed");
} catch (RuntimeException e) {
assertEquals("'summary somestring type string' in 'destinations(default )' is inconsistent with 'summary somestring type int' in 'destinations(incorrect )': All declarations of the same summary field must have the same type", e.getMessage());
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java
index c19244a4bf2..b539c65150d 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java
@@ -22,7 +22,18 @@ public class NameFieldCheckTestCase extends SearchDefinitionTestCase {
@Test
public void testNameField() throws IOException, ParseException {
try {
- SearchBuilder.buildFromFile("src/test/examples/name-check.sd");
+ SearchBuilder.createFromString(
+ "search simple {\n" +
+ " document name-check {\n" +
+ " field title type string {\n" +
+ " indexing: summary | index\n" +
+ " }\n" +
+ " # reserved name, should trigger error\n" +
+ " field sddocname type string {\n" +
+ " indexing: index\n" +
+ " }\n" +
+ " }\n" +
+ "}");
fail("Should throw exception.");
} catch (Exception expected) {
// Success
@@ -32,7 +43,16 @@ public class NameFieldCheckTestCase extends SearchDefinitionTestCase {
@Test
public void testDuplicateNamesInSearchDifferentType() {
try {
- SearchBuilder.buildFromFile("src/test/examples/duplicatenamesinsearchdifferenttype.sd");
+ SearchBuilder.createFromString(
+ "search duplicatenamesinsearch {\n" +
+ " document {\n" +
+ " field grpphotoids64 type string { }\n" +
+ " }\n" +
+ " field grpphotoids64 type array<long> {\n" +
+ " indexing: input grpphotoids64 | split \" \" | for_each {\n" +
+ " base64decode } | attribute\n" +
+ " }\n" +
+ "}");
fail("Should throw exception.");
} catch (Exception e) {
assertEquals("For search 'duplicatenamesinsearch', field 'grpphotoids64': Incompatible types. Expected Array<long> for index field 'grpphotoids64', got string.", e.getMessage());
@@ -42,7 +62,17 @@ public class NameFieldCheckTestCase extends SearchDefinitionTestCase {
@Test
public void testDuplicateNamesInDoc() {
try {
- SearchBuilder.buildFromFile("src/test/examples/duplicatenamesindoc.sd");
+ SearchBuilder.createFromString(
+ "search duplicatenamesindoc {\n" +
+ " document {\n" +
+ " field foo type int {\n" +
+ " indexing: attribute\n" +
+ " }\n" +
+ " field fOo type string {\n" +
+ " indexing: index\n" +
+ " }\n" +
+ " }\n" +
+ "}");
fail("Should throw exception.");
} catch (Exception e) {
assertTrue(e.getMessage().matches(".*Duplicate.*"));
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java
index 8fb40a73f0a..582f5b08517 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java
@@ -18,27 +18,27 @@ public class IndexingInputsTestCase {
assertBuildFails("src/test/examples/indexing_extra_field_input_extra_field.sd",
"For search 'indexing_extra_field_input_extra_field', field 'bar': Indexing script refers " +
"to field 'bar' which does not exist in document type " +
- "'indexing_extra_field_input_extra_field'.");
+ "'indexing_extra_field_input_extra_field', and is not a mutable attribute.");
}
@Test
public void requireThatExtraFieldInputImplicitThrows() throws IOException, ParseException {
assertBuildFails("src/test/examples/indexing_extra_field_input_implicit.sd",
"For search 'indexing_extra_field_input_implicit', field 'foo': Indexing script refers to " +
- "field 'foo' which does not exist in document type 'indexing_extra_field_input_implicit'.");
+ "field 'foo' which does not exist in document type 'indexing_extra_field_input_implicit', and is not a mutable attribute.");
}
@Test
public void requireThatExtraFieldInputNullThrows() throws IOException, ParseException {
assertBuildFails("src/test/examples/indexing_extra_field_input_null.sd",
"For search 'indexing_extra_field_input_null', field 'foo': Indexing script refers to field " +
- "'foo' which does not exist in document type 'indexing_extra_field_input_null'.");
+ "'foo' which does not exist in document type 'indexing_extra_field_input_null', and is not a mutable attribute.");
}
@Test
public void requireThatExtraFieldInputSelfThrows() throws IOException, ParseException {
assertBuildFails("src/test/examples/indexing_extra_field_input_self.sd",
"For search 'indexing_extra_field_input_self', field 'foo': Indexing script refers to field " +
- "'foo' which does not exist in document type 'indexing_extra_field_input_self'.");
+ "'foo' which does not exist in document type 'indexing_extra_field_input_self', and is not a mutable attribute.");
}
}
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
index d729b092670..805fa52c105 100644
--- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
+++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
@@ -2,6 +2,7 @@
package com.yahoo.container.logging;
import com.yahoo.container.core.AccessLogConfig;
+import com.yahoo.io.NativeIO;
import com.yahoo.log.LogFileDb;
import java.io.File;
@@ -263,15 +264,31 @@ public class LogFileHandler extends StreamHandler {
numberOfRecords = 0;
lastRotationTime = now;
nextRotationTime = 0; //figure it out later (lazy evaluation)
- if (compressOnRotation && (oldFileName != null)) {
- triggerCompression(oldFileName);
+ if ((oldFileName != null)) {
+ File oldFile = new File(oldFileName);
+ if (oldFile.exists()) {
+ if (compressOnRotation) {
+ triggerCompression(oldFile);
+ } else {
+ NativeIO nativeIO = new NativeIO();
+ nativeIO.dropFileFromCache(oldFile);
+ }
+ }
}
}
- private void triggerCompression(String oldFileName) {
+ private void triggerCompression(File oldFile) {
try {
+ String oldFileName = oldFile.getPath();
+ String gzippedFileName = oldFileName + ".gz";
Runtime r = Runtime.getRuntime();
- Process p = r.exec(new String[] { "gzip", oldFileName });
+ StringBuilder cmd = new StringBuilder("gzip");
+ cmd.append(" < "). append(oldFileName).append(" > ").append(gzippedFileName);
+ Process p = r.exec(cmd.toString());
+ NativeIO nativeIO = new NativeIO();
+ nativeIO.dropFileFromCache(oldFile); // Drop from cache in case somebody else has a reference to it preventing from dying quickly.
+ oldFile.delete();
+ nativeIO.dropFileFromCache(new File(gzippedFileName));
// Detonator pattern: Think of all the fun we can have if gzip isn't what we
// think it is, if it doesn't return, etc, etc
} catch (IOException e) {
diff --git a/container-dev/pom.xml b/container-dev/pom.xml
index c4c93563026..1b0f36e3adb 100644
--- a/container-dev/pom.xml
+++ b/container-dev/pom.xml
@@ -142,6 +142,10 @@
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java
index a238facc983..713fd7e273c 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java
@@ -25,6 +25,7 @@ public enum JobType {
productionEuWest1 ("production-eu-west-1" , ZoneId.from("prod" , "eu-west-1") , null ),
productionAwsUsEast1a ("production-aws-us-east-1a" , ZoneId.from("prod" , "aws-us-east-1a") , null ),
productionAwsUsWest1b ("production-aws-us-west-1b" , ZoneId.from("prod" , "aws-us-west-1b") , null ),
+ productionAwsUsEast1b ("production-aws-us-east-1b" , ZoneId.from("prod" , "aws-us-east-1b") , null ),
productionCdAwsUsEast1a("production-cd-aws-us-east-1a", null , ZoneId.from("prod" , "cd-aws-us-east-1a")),
productionCdUsCentral1 ("production-cd-us-central-1" , null , ZoneId.from("prod" , "cd-us-central-1") ),
productionCdUsCentral2 ("production-cd-us-central-2" , null , ZoneId.from("prod" , "cd-us-central-2") );
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index a6c3f11470d..cd5178d80ac 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -199,6 +199,7 @@ public class Application {
// Rotation status only contains VIP host names, one per zone in the system. The only way to map VIP hostname to
// this deployment, and thereby determine rotation status, is to check if VIP hostname contains the
// deployment's environment and region.
+ // TODO: change this map to be indexed by zones, then?
return rotationStatus.entrySet().stream()
.filter(kv -> kv.getKey().value().contains(deployment.zone().value()))
.map(Map.Entry::getValue)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 2dd3b1dda23..b3158102182 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -44,6 +44,7 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus;
import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun;
import com.yahoo.vespa.hosted.controller.application.SystemApplication;
import com.yahoo.vespa.hosted.controller.concurrent.Once;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
import com.yahoo.vespa.hosted.controller.rotation.Rotation;
@@ -330,13 +331,9 @@ public class ApplicationController {
validateRun(application.get(), zone, platformVersion, applicationVersion);
}
- validate(applicationPackage.deploymentSpec());
-
// Update application with information from application package
- if ( ! preferOldestVersion && ! application.get().deploymentJobs().builtInternally()) {
- application = withUpdatedConfig(application, applicationPackage);
- store(application); // store missing information even if we fail deployment below
- }
+ if ( ! preferOldestVersion && ! application.get().deploymentJobs().builtInternally())
+ application = storeWithUpdatedConfig(application, applicationPackage);
// Assign global rotation
application = withRotation(application, zone);
@@ -359,8 +356,9 @@ public class ApplicationController {
}
/** Stores the deployment spec and validation overrides from the application package, and runs cleanup. */
- public LockedApplication withUpdatedConfig(LockedApplication application, ApplicationPackage applicationPackage) {
- // Store information about application package
+ public LockedApplication storeWithUpdatedConfig(LockedApplication application, ApplicationPackage applicationPackage) {
+ validate(applicationPackage.deploymentSpec());
+
application = application.with(applicationPackage.deploymentSpec());
application = application.with(applicationPackage.validationOverrides());
@@ -649,6 +647,7 @@ public class ApplicationController {
/** Verify that each of the production zones listed in the deployment spec exist in this system. */
private void validate(DeploymentSpec deploymentSpec) {
+ new DeploymentSteps(deploymentSpec, controller::system).jobs();
deploymentSpec.zones().stream()
.filter(zone -> zone.environment() == Environment.prod)
.forEach(zone -> {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 2bb878366c5..71b93731a44 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -101,7 +101,7 @@ public class InternalStepRunner implements StepRunner {
case endTests: return endTests(id, logger);
case deactivateReal: return deactivateReal(id, logger);
case deactivateTester: return deactivateTester(id, logger);
- case report: return report(id);
+ case report: return report(id, logger);
default: throw new AssertionError("Unknown step '" + step + "'!");
}
}
@@ -398,8 +398,13 @@ public class InternalStepRunner implements StepRunner {
return Optional.of(running);
}
- private Optional<RunStatus> report(RunId id) {
- controller.jobController().active(id).ifPresent(run -> controller.applications().deploymentTrigger().notifyOfCompletion(report(run)));
+ private Optional<RunStatus> report(RunId id, DualLogger logger) {
+ try {
+ controller.jobController().active(id).ifPresent(run -> controller.applications().deploymentTrigger().notifyOfCompletion(report(run)));
+ }
+ catch (IllegalStateException e) {
+ logger.log(INFO, "Job '" + id.type() + "'no longer supposed to run?:", e);
+ }
return Optional.of(running);
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index e9b0267a443..3cff6ac2430 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -203,32 +203,31 @@ public class JobController {
* Accepts and stores a new application package and test jar pair under a generated application version key.
*/
public ApplicationVersion submit(ApplicationId id, SourceRevision revision,
- byte[] applicationPackage, byte[] applicationTestPackage) {
+ byte[] packageBytes, byte[] testPackageBytes) {
AtomicReference<ApplicationVersion> version = new AtomicReference<>();
controller.applications().lockOrThrow(id, application -> {
+ if ( ! application.get().deploymentJobs().builtInternally()) {
+ // Copy all current packages to the new application store
+ application.get().deployments().values().stream()
+ .map(Deployment::applicationVersion)
+ .distinct()
+ .forEach(appVersion -> {
+ byte[] content = controller.applications().artifacts().getApplicationPackage(application.get().id(), appVersion.id());
+ controller.applications().applicationStore().putApplicationPackage(application.get().id(), appVersion.id(), content);
+ });
+ }
+
long run = nextBuild(id);
version.set(ApplicationVersion.from(revision, run));
controller.applications().applicationStore().putApplicationPackage(id,
- version.get().id(),
- applicationPackage);
+ version.get().id(),
+ packageBytes);
controller.applications().applicationStore().putTesterPackage(testerOf(id),
version.get().id(),
- applicationTestPackage);
-
- if (!application.get().deploymentJobs().builtInternally()) {
- // Copy all current packages to the new application store
- application.get().deployments().values().stream()
- .map(Deployment::applicationVersion)
- .distinct()
- .forEach(appVersion -> {
- byte[] content = controller.applications().artifacts().getApplicationPackage(application.get().id(), appVersion.id());
- controller.applications().applicationStore().putApplicationPackage(application.get().id(), appVersion.id(), content);
- });
- }
+ testPackageBytes);
- controller.applications().store(controller.applications().withUpdatedConfig(application.withBuiltInternally(true),
- new ApplicationPackage(applicationPackage)));
+ controller.applications().storeWithUpdatedConfig(application.withBuiltInternally(true), new ApplicationPackage(packageBytes));
notifyOfNewSubmission(id, revision, run);
});
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 635e0c1fb26..a9353af1b20 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -147,6 +147,19 @@ public class ControllerTest {
assertEquals(5, applications.get(app1.id()).get().deploymentJobs().jobStatus().size());
+ // Production zone for which there is no JobType is not allowed.
+ applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("deep-space-9")
+ .build();
+ try {
+ tester.controller().jobController().submit(app1.id(), BuildJob.defaultSourceRevision, applicationPackage.zippedContent(), new byte[0]);
+ fail("Expected exception due to illegal deployment spec.");
+ }
+ catch (IllegalArgumentException e) {
+ assertEquals("No job is known for zone prod.deep-space-9 in default.", e.getMessage());
+ }
+
// prod zone removal is not allowed
applicationPackage = new ApplicationPackageBuilder()
.environment(Environment.prod)
diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java
index 10e3ce561d7..a2321e912e1 100644
--- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java
+++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java
@@ -9,7 +9,6 @@ import com.yahoo.document.datatypes.Array;
import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.document.datatypes.MapFieldValue;
import com.yahoo.document.datatypes.StringFieldValue;
-import com.yahoo.document.datatypes.Struct;
import com.yahoo.document.datatypes.StructuredFieldValue;
import com.yahoo.document.datatypes.WeightedSet;
import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
index e486e2647cf..8e934001381 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
@@ -116,11 +116,10 @@ public class JsonSerializationHelper {
}
public static void serializeStringField(JsonGenerator generator, FieldBase field, StringFieldValue value) {
- // Hide empty strings
- if (value.getString().length() == 0) {
+ // Hide fields which only contains an empty string
+ if (value.getString().length() == 0 && field != null) {
return;
}
-
serializeString(generator, field, value.getString());
}
@@ -237,10 +236,6 @@ public class JsonSerializationHelper {
}
public static void serializeString(JsonGenerator generator, FieldBase field, String value) {
- if (value.length() == 0) {
- return;
- }
-
fieldNameIfNotNull(generator, field);
wrapIOException(() -> generator.writeString(value));
}
diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
index 50d02d70c9b..0ab00b4e2bc 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
@@ -213,7 +213,7 @@ public class JsonWriterTestCase {
@Test
public void arrayTest() throws IOException {
roundTripEquality("id:unittest:testarray::whee", "{ \"actualarray\": ["
- + " \"nalle\"," + " \"tralle\"]}");
+ + " \"nalle\"," + " \"\"," + " \"tralle\"]}");
}
@Test
@@ -342,6 +342,10 @@ public class JsonWriterTestCase {
ObjectMapper m = new ObjectMapper();
Map<?, ?> exp = m.readValue(expected, Map.class);
Map<?, ?> gen = m.readValue(generated, Map.class);
+ if (! exp.equals(gen)) {
+ System.err.println("expected: "+Utf8.toString(expected));
+ System.err.println("generated: "+Utf8.toString(generated));
+ }
assertEquals(exp, gen);
}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java
index 6983eb4fbd9..10a316f7751 100755
--- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java
+++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java
@@ -301,63 +301,50 @@ public class DocumentProtocol implements Protocol {
putRoutingPolicyFactory("SubsetService", new RoutingPolicyFactories.SubsetServicePolicyFactory());
// Prepare version specifications to use when adding routable factories.
- VersionSpecification version50 = new VersionSpecification(5, 0);
- VersionSpecification version51 = new VersionSpecification(5, 1);
VersionSpecification version52 = new VersionSpecification(5, 115);
VersionSpecification version6 = new VersionSpecification(6, 221);
- List<VersionSpecification> from50 = Arrays.asList(version50, version51, version52, version6);
- List<VersionSpecification> from51 = Arrays.asList(version51, version52, version6);
List<VersionSpecification> from52 = Arrays.asList(version52, version6);
List<VersionSpecification> from6 = Collections.singletonList(version6); // TODO decide minor version...
- // 5.0 serialization (keep alphabetized please)
- putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories50.CreateVisitorMessageFactory(), from50);
- putRoutableFactory(MESSAGE_DESTROYVISITOR, new RoutableFactories50.DestroyVisitorMessageFactory(), from50);
- putRoutableFactory(MESSAGE_DOCUMENTLIST, new RoutableFactories50.DocumentListMessageFactory(), from50);
- putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, new RoutableFactories50.DocumentSummaryMessageFactory(), from50);
- putRoutableFactory(MESSAGE_EMPTYBUCKETS, new RoutableFactories50.EmptyBucketsMessageFactory(), from50);
- putRoutableFactory(MESSAGE_GETBUCKETLIST, new RoutableFactories50.GetBucketListMessageFactory(), from50);
- putRoutableFactory(MESSAGE_GETBUCKETSTATE, new RoutableFactories50.GetBucketStateMessageFactory(), from50);
- putRoutableFactory(MESSAGE_GETDOCUMENT, new RoutableFactories50.GetDocumentMessageFactory(), from50);
- putRoutableFactory(MESSAGE_MAPVISITOR, new RoutableFactories50.MapVisitorMessageFactory(), from50);
- putRoutableFactory(MESSAGE_PUTDOCUMENT, new RoutableFactories50.PutDocumentMessageFactory(), from50);
- putRoutableFactory(MESSAGE_QUERYRESULT, new RoutableFactories50.QueryResultMessageFactory(), from50);
- putRoutableFactory(MESSAGE_REMOVEDOCUMENT, new RoutableFactories50.RemoveDocumentMessageFactory(), from50);
- putRoutableFactory(MESSAGE_REMOVELOCATION, new RoutableFactories50.RemoveLocationMessageFactory(), from50);
- putRoutableFactory(MESSAGE_SEARCHRESULT, new RoutableFactories50.SearchResultMessageFactory(), from50);
- putRoutableFactory(MESSAGE_STATBUCKET, new RoutableFactories50.StatBucketMessageFactory(), from50);
- putRoutableFactory(MESSAGE_UPDATEDOCUMENT, new RoutableFactories50.UpdateDocumentMessageFactory(), from50);
- putRoutableFactory(MESSAGE_VISITORINFO, new RoutableFactories50.VisitorInfoMessageFactory(), from50);
- putRoutableFactory(REPLY_CREATEVISITOR, new RoutableFactories50.CreateVisitorReplyFactory(), from50);
- putRoutableFactory(REPLY_DESTROYVISITOR, new RoutableFactories50.DestroyVisitorReplyFactory(), from50);
- putRoutableFactory(REPLY_DOCUMENTLIST, new RoutableFactories50.DocumentListReplyFactory(), from50);
- putRoutableFactory(REPLY_DOCUMENTSUMMARY, new RoutableFactories50.DocumentSummaryReplyFactory(), from50);
- putRoutableFactory(REPLY_EMPTYBUCKETS, new RoutableFactories50.EmptyBucketsReplyFactory(), from50);
- putRoutableFactory(REPLY_GETBUCKETLIST, new RoutableFactories50.GetBucketListReplyFactory(), from50);
- putRoutableFactory(REPLY_GETBUCKETSTATE, new RoutableFactories50.GetBucketStateReplyFactory(), from50);
- putRoutableFactory(REPLY_GETDOCUMENT, new RoutableFactories50.GetDocumentReplyFactory(), from50);
- putRoutableFactory(REPLY_MAPVISITOR, new RoutableFactories50.MapVisitorReplyFactory(), from50);
- putRoutableFactory(REPLY_PUTDOCUMENT, new RoutableFactories50.PutDocumentReplyFactory(), from50);
- putRoutableFactory(REPLY_QUERYRESULT, new RoutableFactories50.QueryResultReplyFactory(), from50);
- putRoutableFactory(REPLY_REMOVEDOCUMENT, new RoutableFactories50.RemoveDocumentReplyFactory(), from50);
- putRoutableFactory(REPLY_REMOVELOCATION, new RoutableFactories50.RemoveLocationReplyFactory(), from50);
- putRoutableFactory(REPLY_SEARCHRESULT, new RoutableFactories50.SearchResultReplyFactory(), from50);
- putRoutableFactory(REPLY_STATBUCKET, new RoutableFactories50.StatBucketReplyFactory(), from50);
- putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories50.UpdateDocumentReplyFactory(), from50);
- putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories50.UpdateDocumentReplyFactory(), from50);
- putRoutableFactory(REPLY_VISITORINFO, new RoutableFactories50.VisitorInfoReplyFactory(), from50);
- putRoutableFactory(REPLY_WRONGDISTRIBUTION, new RoutableFactories50.WrongDistributionReplyFactory(), from50);
-
- // 5.1 serialization
- putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories51.CreateVisitorMessageFactory(), from51);
- putRoutableFactory(MESSAGE_GETDOCUMENT, new RoutableFactories51.GetDocumentMessageFactory(), from51);
- putRoutableFactory(REPLY_DOCUMENTIGNORED, new RoutableFactories51.DocumentIgnoredReplyFactory(), from51);
-
- // 5.2 serialization
+ // 5.2 serialization (keep alphabetized please)
+ putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories52.CreateVisitorMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_DESTROYVISITOR, new RoutableFactories52.DestroyVisitorMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_DOCUMENTLIST, new RoutableFactories52.DocumentListMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, new RoutableFactories52.DocumentSummaryMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_EMPTYBUCKETS, new RoutableFactories52.EmptyBucketsMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_GETBUCKETLIST, new RoutableFactories52.GetBucketListMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_GETBUCKETSTATE, new RoutableFactories52.GetBucketStateMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_GETDOCUMENT, new RoutableFactories52.GetDocumentMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_MAPVISITOR, new RoutableFactories52.MapVisitorMessageFactory(), from52);
putRoutableFactory(MESSAGE_PUTDOCUMENT, new RoutableFactories52.PutDocumentMessageFactory(), from52);
- putRoutableFactory(MESSAGE_UPDATEDOCUMENT, new RoutableFactories52.UpdateDocumentMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_QUERYRESULT, new RoutableFactories52.QueryResultMessageFactory(), from52);
putRoutableFactory(MESSAGE_REMOVEDOCUMENT, new RoutableFactories52.RemoveDocumentMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_REMOVELOCATION, new RoutableFactories52.RemoveLocationMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_SEARCHRESULT, new RoutableFactories52.SearchResultMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_STATBUCKET, new RoutableFactories52.StatBucketMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_UPDATEDOCUMENT, new RoutableFactories52.UpdateDocumentMessageFactory(), from52);
+ putRoutableFactory(MESSAGE_VISITORINFO, new RoutableFactories52.VisitorInfoMessageFactory(), from52);
+ putRoutableFactory(REPLY_CREATEVISITOR, new RoutableFactories52.CreateVisitorReplyFactory(), from52);
+ putRoutableFactory(REPLY_DESTROYVISITOR, new RoutableFactories52.DestroyVisitorReplyFactory(), from52);
+ putRoutableFactory(REPLY_DOCUMENTIGNORED, new RoutableFactories52.DocumentIgnoredReplyFactory(), from52);
+ putRoutableFactory(REPLY_DOCUMENTLIST, new RoutableFactories52.DocumentListReplyFactory(), from52);
+ putRoutableFactory(REPLY_DOCUMENTSUMMARY, new RoutableFactories52.DocumentSummaryReplyFactory(), from52);
+ putRoutableFactory(REPLY_EMPTYBUCKETS, new RoutableFactories52.EmptyBucketsReplyFactory(), from52);
+ putRoutableFactory(REPLY_GETBUCKETLIST, new RoutableFactories52.GetBucketListReplyFactory(), from52);
+ putRoutableFactory(REPLY_GETBUCKETSTATE, new RoutableFactories52.GetBucketStateReplyFactory(), from52);
+ putRoutableFactory(REPLY_GETDOCUMENT, new RoutableFactories52.GetDocumentReplyFactory(), from52);
+ putRoutableFactory(REPLY_MAPVISITOR, new RoutableFactories52.MapVisitorReplyFactory(), from52);
+ putRoutableFactory(REPLY_PUTDOCUMENT, new RoutableFactories52.PutDocumentReplyFactory(), from52);
+ putRoutableFactory(REPLY_QUERYRESULT, new RoutableFactories52.QueryResultReplyFactory(), from52);
+ putRoutableFactory(REPLY_REMOVEDOCUMENT, new RoutableFactories52.RemoveDocumentReplyFactory(), from52);
+ putRoutableFactory(REPLY_REMOVELOCATION, new RoutableFactories52.RemoveLocationReplyFactory(), from52);
+ putRoutableFactory(REPLY_SEARCHRESULT, new RoutableFactories52.SearchResultReplyFactory(), from52);
+ putRoutableFactory(REPLY_STATBUCKET, new RoutableFactories52.StatBucketReplyFactory(), from52);
+ putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories52.UpdateDocumentReplyFactory(), from52);
+ putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories52.UpdateDocumentReplyFactory(), from52);
+ putRoutableFactory(REPLY_VISITORINFO, new RoutableFactories52.VisitorInfoReplyFactory(), from52);
+ putRoutableFactory(REPLY_WRONGDISTRIBUTION, new RoutableFactories52.WrongDistributionReplyFactory(), from52);
// 6.x serialization
putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories60.CreateVisitorMessageFactory(), from6);
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java
index 5a68e5bda2a..051ec44dd07 100755
--- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java
+++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java
@@ -13,6 +13,7 @@ import com.yahoo.messagebus.routing.RoutingContext;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* This policy implements the necessary logic to communicate with an external Vespa application and resolve its list of
@@ -30,6 +31,7 @@ public class ExternPolicy implements DocumentProtocolRoutingPolicy {
private int offset = 0;
private int generation = 0;
private final List<Hop> recipients = new ArrayList<>();
+ private final AtomicBoolean destroyed = new AtomicBoolean(false);
/**
* Constructs a new instance of this policy. The argument given is the connection spec to the slobrok to use for
@@ -110,13 +112,6 @@ public class ExternPolicy implements DocumentProtocolRoutingPolicy {
}
@Override
- @SuppressWarnings("deprecation") // finalize() is deprecated from Java 9
- public void finalize() throws Throwable {
- super.finalize();
- mirror.shutdown();
- orb.transport().shutdown().join();
- }
-
public void select(RoutingContext ctx) {
if (error != null) {
ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, error);
@@ -135,13 +130,19 @@ public class ExternPolicy implements DocumentProtocolRoutingPolicy {
}
}
+ @Override
public void merge(RoutingContext ctx) {
DocumentProtocol.merge(ctx);
}
+ @Override
public void destroy() {
+ if (destroyed.getAndSet(true)) throw new RuntimeException("Already destroyed");
+ mirror.shutdown();
+ orb.transport().shutdown().join();
}
+ @Override
public MetricSet getMetrics() {
return null;
}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories50.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories50.java
deleted file mode 100755
index c7c1121ae68..00000000000
--- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories50.java
+++ /dev/null
@@ -1,917 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.document.*;
-import com.yahoo.document.serialization.DocumentDeserializer;
-import com.yahoo.document.serialization.DocumentSerializer;
-import com.yahoo.document.serialization.DocumentSerializerFactory;
-import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
-import com.yahoo.log.LogLevel;
-import com.yahoo.messagebus.Routable;
-import com.yahoo.vdslib.DocumentList;
-import com.yahoo.vdslib.DocumentSummary;
-import com.yahoo.vdslib.SearchResult;
-import com.yahoo.vdslib.VisitorStatistics;
-import com.yahoo.vespa.objects.Deserializer;
-import com.yahoo.vespa.objects.Serializer;
-
-import java.util.Map;
-import java.util.logging.Logger;
-
-
-/**
- * This class encapsulates all the {@link RoutableFactory} classes needed to implement serialization for the document
- * protocol. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now.
- */
-public abstract class RoutableFactories50 {
-
- /**
- * Implements the shared factory logic required for {@link DocumentMessage} objects, and it offers a more convenient
- * interface for implementing {@link RoutableFactory}.
- *
- * @author Simon Thoresen Hult
- */
- public static abstract class DocumentMessageFactory extends AbstractRoutableFactory {
-
- /**
- * This method encodes the given message using the given serializer. You are guaranteed to only receive messages
- * of the type that this factory was registered for.
- * <p>
- * This method is NOT exception safe. Return false to
- * signal failure.
- *
- * @param msg The message to encode.
- * @param serializer The serializer to use for encoding.
- * @return True if the message was encoded.
- */
- protected abstract boolean doEncode(DocumentMessage msg, DocumentSerializer serializer);
-
- /**
- * This method decodes a message from the given deserializer. You are guaranteed to only receive byte buffers
- * generated by a previous call to {@link #doEncode(DocumentMessage, DocumentSerializer)}.
- * <p>
- * This method is NOT exception safe. Return null to signal failure.
- *
- * @param deserializer The deserializer to use for decoding.
- * @return The decoded message.
- */
- protected abstract DocumentMessage doDecode(DocumentDeserializer deserializer);
-
- public boolean encode(Routable obj, DocumentSerializer out) {
- if (!(obj instanceof DocumentMessage)) {
- throw new AssertionError(
- "Document message factory (" + getClass().getName() + ") registered for incompatible " +
- "routable type " + obj.getType() + "(" + obj.getClass().getName() + ").");
- }
- DocumentMessage msg = (DocumentMessage)obj;
- out.putByte(null, (byte)(msg.getPriority().getValue()));
- out.putInt(null, msg.getLoadType().getId());
- return doEncode(msg, out);
- }
-
- public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) {
- byte pri = in.getByte(null);
- int loadType = in.getInt(null);
- DocumentMessage msg = doDecode(in);
- if (msg != null) {
- msg.setPriority(DocumentProtocol.getPriority(pri));
- msg.setLoadType(loadTypes.getIdMap().get(loadType));
- }
- return msg;
- }
- }
-
- /**
- * Implements the shared factory logic required for {@link DocumentReply} objects, and it offers a more convenient
- * interface for implementing {@link RoutableFactory}.
- *
- * @author Simon Thoresen Hult
- */
- public static abstract class DocumentReplyFactory extends AbstractRoutableFactory {
-
- /**
- * This method encodes the given reply into the given byte buffer. You are guaranteed to only receive replies of
- * the type that this factory was registered for.
- * <p>
- * This method is NOT exception safe. Return false to signal
- * failure.
- *
- * @param reply The reply to encode.
- * @param buf The byte buffer to write to.
- * @return True if the message was encoded.
- */
- protected abstract boolean doEncode(DocumentReply reply, DocumentSerializer buf);
-
- /**
- * This method decodes a reply from the given byte buffer. You are guaranteed to only receive byte buffers
- * generated by a previous call to {@link #doEncode(DocumentReply, com.yahoo.document.serialization.DocumentSerializer)}.
- *
- * <p>
- * This method is NOT exception safe. Return null to signal failure.
- *
- * @param buf The byte buffer to read from.
- * @return The decoded reply.
- */
- protected abstract DocumentReply doDecode(DocumentDeserializer buf);
-
- public boolean encode(Routable obj, DocumentSerializer out) {
- if (!(obj instanceof DocumentReply)) {
- throw new AssertionError(
- "Document reply factory (" + getClass().getName() + ") registered for incompatible " +
- "routable type " + obj.getType() + "(" + obj.getClass().getName() + ").");
- }
- DocumentReply reply = (DocumentReply)obj;
- out.putByte(null, (byte)(reply.getPriority().getValue()));
- return doEncode(reply, out);
- }
-
- public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) {
- byte pri = in.getByte(null);
- DocumentReply reply = doDecode(in);
- if (reply != null) {
- reply.setPriority(DocumentProtocol.getPriority(pri));
- }
- return reply;
- }
- }
-
- public static class CreateVisitorMessageFactory extends DocumentMessageFactory {
-
- @Override
- @SuppressWarnings("deprecation")
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- CreateVisitorMessage msg = new CreateVisitorMessage();
- msg.setLibraryName(decodeString(buf));
- msg.setInstanceId(decodeString(buf));
- msg.setControlDestination(decodeString(buf));
- msg.setDataDestination(decodeString(buf));
- msg.setDocumentSelection(decodeString(buf));
- msg.setMaxPendingReplyCount(buf.getInt(null));
-
- int size = buf.getInt(null);
- for (int i = 0; i < size; i++) {
- long reversed = buf.getLong(null);
- long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) |
- ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) |
- ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) |
- ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l);
- msg.getBuckets().add(new BucketId(rawid));
- }
-
- msg.setFromTimestamp(buf.getLong(null));
- msg.setToTimestamp(buf.getLong(null));
- msg.setVisitRemoves(buf.getByte(null) == (byte)1);
- buf.getByte(null); // removed feature "visitHeadersOnly"
- msg.setVisitInconsistentBuckets(buf.getByte(null) == (byte)1);
-
- size = buf.getInt(null);
- for (int i = 0; i < size; i++) {
- String key = decodeString(buf);
- int sz = buf.getInt(null);
- msg.getParameters().put(key, buf.getBytes(null, sz));
- }
-
- msg.setVisitorOrdering(buf.getInt(null));
- msg.setMaxBucketsPerVisitor(buf.getInt(null));
- msg.setVisitorDispatcherVersion(50);
- return msg;
- }
-
- @Override
- @SuppressWarnings("deprecation")
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- CreateVisitorMessage msg = (CreateVisitorMessage)obj;
- encodeString(msg.getLibraryName(), buf);
- encodeString(msg.getInstanceId(), buf);
- encodeString(msg.getControlDestination(), buf);
- encodeString(msg.getDataDestination(), buf);
- encodeString(msg.getDocumentSelection(), buf);
- buf.putInt(null, msg.getMaxPendingReplyCount());
-
- buf.putInt(null, msg.getBuckets().size());
- for (BucketId id : msg.getBuckets()) {
- long rawid = id.getRawId();
- long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) |
- ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) |
- ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) |
- ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l);
- buf.putLong(null, reversed);
- }
-
- buf.putLong(null, msg.getFromTimestamp());
- buf.putLong(null, msg.getToTimestamp());
- buf.putByte(null, msg.getVisitRemoves() ? (byte)1 : (byte)0);
- buf.putByte(null, (byte)0); // removed feature "visitHeadersOnly"
- buf.putByte(null, msg.getVisitInconsistentBuckets() ? (byte)1 : (byte)0);
-
- buf.putInt(null, msg.getParameters().size());
- for (Map.Entry<String, byte[]> pairs : msg.getParameters().entrySet()) {
- encodeString(pairs.getKey(), buf);
- byte[] b = pairs.getValue();
- buf.putInt(null, b.length);
- buf.put(null, b);
- }
-
- buf.putInt(null, msg.getVisitorOrdering());
- buf.putInt(null, msg.getMaxBucketsPerVisitor());
- return true;
- }
- }
-
- public static class CreateVisitorReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR);
- reply.setLastBucket(new BucketId(buf.getLong(null)));
-
- VisitorStatistics vs = new VisitorStatistics();
- vs.setBucketsVisited(buf.getInt(null));
- vs.setDocumentsVisited(buf.getLong(null));
- vs.setBytesVisited(buf.getLong(null));
- vs.setDocumentsReturned(buf.getLong(null));
- vs.setBytesReturned(buf.getLong(null));
- vs.setSecondPassDocumentsReturned(buf.getLong(null));
- vs.setSecondPassBytesReturned(buf.getLong(null));
- reply.setVisitorStatistics(vs);
- return reply;
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- CreateVisitorReply reply = (CreateVisitorReply)obj;
- buf.putLong(null, reply.getLastBucket().getRawId());
- buf.putInt(null, reply.getVisitorStatistics().getBucketsVisited());
- buf.putLong(null, reply.getVisitorStatistics().getDocumentsVisited());
- buf.putLong(null, reply.getVisitorStatistics().getBytesVisited());
- buf.putLong(null, reply.getVisitorStatistics().getDocumentsReturned());
- buf.putLong(null, reply.getVisitorStatistics().getBytesReturned());
- buf.putLong(null, reply.getVisitorStatistics().getSecondPassDocumentsReturned());
- buf.putLong(null, reply.getVisitorStatistics().getSecondPassBytesReturned());
- return true;
- }
- }
-
- public static class DestroyVisitorMessageFactory extends DocumentMessageFactory {
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- DestroyVisitorMessage msg = new DestroyVisitorMessage();
- msg.setInstanceId(decodeString(buf));
- return msg;
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- DestroyVisitorMessage msg = (DestroyVisitorMessage)obj;
- encodeString(msg.getInstanceId(), buf);
- return true;
- }
- }
-
- public static class DestroyVisitorReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- return new VisitorReply(DocumentProtocol.REPLY_DESTROYVISITOR);
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- return true;
- }
- }
-
- public static class DocumentListMessageFactory extends DocumentMessageFactory {
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- DocumentListMessage msg = new DocumentListMessage();
- msg.setBucketId(new BucketId(buf.getLong(null)));
- int len = buf.getInt(null);
- for (int i = 0; i < len; i++) {
- msg.getDocuments().add(new DocumentListEntry(buf));
- }
- return msg;
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- DocumentListMessage msg = (DocumentListMessage)obj;
- buf.putLong(null, msg.getBucketId().getRawId());
- buf.putInt(null, msg.getDocuments().size());
-
- for (int i = 0; i < msg.getDocuments().size(); i++) {
- msg.getDocuments().get(i).serialize(buf);
- }
- return true;
- }
- }
-
- public static class DocumentListReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTLIST);
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- return true;
- }
- }
-
- public static class DocumentSummaryMessageFactory extends DocumentMessageFactory {
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- DocumentSummaryMessage msg = new DocumentSummaryMessage();
- msg.setDocumentSummary(new DocumentSummary(buf));
- return msg;
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- return false; // not supported
- }
- }
-
- public static class DocumentSummaryReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTSUMMARY);
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- return true;
- }
- }
-
- public static class EmptyBucketsMessageFactory extends DocumentMessageFactory {
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- EmptyBucketsMessage msg = new EmptyBucketsMessage();
- int size = buf.getInt(null);
- for (int i = 0; i < size; ++i) {
- msg.getBucketIds().add(new BucketId(buf.getLong(null)));
- }
- return msg;
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- EmptyBucketsMessage msg = (EmptyBucketsMessage)obj;
- buf.putInt(null, msg.getBucketIds().size());
- for (BucketId bid : msg.getBucketIds()) {
- buf.putLong(null, bid.getRawId());
- }
- return true;
- }
- }
-
- public static class EmptyBucketsReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- return new VisitorReply(DocumentProtocol.REPLY_EMPTYBUCKETS);
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- return true;
- }
- }
-
- public static class GetBucketListMessageFactory extends DocumentMessageFactory {
-
- protected String decodeBucketSpace(Deserializer deserializer) {
- return FixedBucketSpaces.defaultSpace();
- }
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- GetBucketListMessage msg = new GetBucketListMessage();
- msg.setBucketId(new BucketId(buf.getLong(null)));
- msg.setBucketSpace(decodeBucketSpace(buf));
- return msg;
- }
-
- protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) {
- return FixedBucketSpaces.defaultSpace().equals(bucketSpace);
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- GetBucketListMessage msg = (GetBucketListMessage)obj;
- buf.putLong(null, msg.getBucketId().getRawId());
- return encodeBucketSpace(msg.getBucketSpace(), buf);
- }
- }
-
- public static class GetBucketListReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- GetBucketListReply reply = new GetBucketListReply();
- int len = buf.getInt(null);
- for (int i = 0; i < len; i++) {
- GetBucketListReply.BucketInfo info = new GetBucketListReply.BucketInfo();
- info.bucket = new BucketId(buf.getLong(null));
- info.bucketInformation = decodeString(buf);
- reply.getBuckets().add(info);
- }
- return reply;
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- GetBucketListReply reply = (GetBucketListReply)obj;
- buf.putInt(null, reply.getBuckets().size());
- for (GetBucketListReply.BucketInfo info : reply.getBuckets()) {
- buf.putLong(null, info.bucket.getRawId());
- encodeString(info.bucketInformation, buf);
- }
- return true;
- }
- }
-
- public static class GetBucketStateMessageFactory extends DocumentMessageFactory {
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- GetBucketStateMessage msg = new GetBucketStateMessage();
- msg.setBucketId(new BucketId(buf.getLong(null)));
- return msg;
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- GetBucketStateMessage msg = (GetBucketStateMessage)obj;
- buf.putLong(null, msg.getBucketId().getRawId());
- return true;
- }
- }
-
- public static class GetBucketStateReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- GetBucketStateReply reply = new GetBucketStateReply();
- int size = buf.getInt(null);
- for (int i = 0; i < size; i++) {
- reply.getBucketState().add(new DocumentState(buf));
- }
- return reply;
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- GetBucketStateReply reply = (GetBucketStateReply)obj;
- buf.putInt(null, reply.getBucketState().size());
- for (DocumentState stat : reply.getBucketState()) {
- stat.serialize(buf);
- }
- return true;
- }
- }
-
- public static class GetDocumentMessageFactory extends DocumentMessageFactory {
-
- @Override
- @SuppressWarnings("deprecation")
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- GetDocumentMessage msg = new GetDocumentMessage();
- msg.setDocumentId(new DocumentId(buf));
- buf.getInt(null); // removed feature "flags"; ignore
- return msg;
- }
-
- @Override
- @SuppressWarnings("deprecation")
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- GetDocumentMessage msg = (GetDocumentMessage)obj;
- msg.getDocumentId().serialize(buf);
- buf.putInt(null, 0); // removed feature "flags"
- return true;
- }
- }
-
- public static class GetDocumentReplyFactory extends DocumentReplyFactory {
-
- private final LazyDecoder decoder = new LazyDecoder() {
-
- public void decode(Routable obj, DocumentDeserializer buf) {
- GetDocumentReply reply = (GetDocumentReply)obj;
-
- Document doc = null;
- byte flag = buf.getByte(null);
- if (flag != 0) {
- doc = Document.createDocument(buf);
- reply.setDocument(doc);
- }
- long lastModified = buf.getLong(null);
- reply.setLastModified(lastModified);
- if (doc != null) {
- doc.setLastModified(lastModified);
- }
- }
- };
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- GetDocumentReply reply = new GetDocumentReply(decoder, buf);
-
- return reply;
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- GetDocumentReply reply = (GetDocumentReply)obj;
- if (reply.getSerializedBuffer() != null) {
- buf.put(null, reply.getSerializedBuffer());
- } else {
- Document document = reply.getDocument();
- buf.putByte(null, (byte)(document == null ? 0 : 1));
- if (document != null) {
- document.serialize(buf);
- }
- buf.putLong(null, reply.getLastModified());
- }
- return true;
- }
- }
-
- public static class MapVisitorMessageFactory extends DocumentMessageFactory {
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- MapVisitorMessage msg = new MapVisitorMessage();
- int size = buf.getInt(null);
- for (int i = 0; i < size; i++) {
- String key = decodeString(buf);
- String value = decodeString(buf);
- msg.getData().put(key, value);
- }
- return msg;
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- MapVisitorMessage msg = (MapVisitorMessage)obj;
- buf.putInt(null, msg.getData().size());
- for (Map.Entry<String, String> pairs : msg.getData().entrySet()) {
- encodeString(pairs.getKey(), buf);
- encodeString(pairs.getValue(), buf);
- }
- return true;
- }
- }
-
- public static class MapVisitorReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- return new VisitorReply(DocumentProtocol.REPLY_MAPVISITOR);
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- return true;
- }
- }
-
- public static class PutDocumentMessageFactory extends DocumentMessageFactory {
- protected void decodeInto(PutDocumentMessage msg, DocumentDeserializer buf) {
- msg.setDocumentPut(new DocumentPut(Document.createDocument(buf)));
- msg.setTimestamp(buf.getLong(null));
- }
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buffer) {
- final LazyDecoder decoder = (obj, buf) -> {
- decodeInto((PutDocumentMessage) obj, buf);
- };
-
- return new PutDocumentMessage(decoder, buffer);
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- PutDocumentMessage msg = (PutDocumentMessage)obj;
- if (msg.getSerializedBuffer() != null) {
- buf.put(null, msg.getSerializedBuffer());
- } else {
- msg.getDocumentPut().getDocument().serialize(buf);
- buf.putLong(null, msg.getTimestamp());
- }
- return true;
- }
- }
-
- public static class PutDocumentReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- WriteDocumentReply rep = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT);
- rep.setHighestModificationTimestamp(buf.getLong(null));
- return rep;
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- WriteDocumentReply rep = (WriteDocumentReply)obj;
- buf.putLong(null, rep.getHighestModificationTimestamp());
- return true;
- }
- }
-
- public static class RemoveDocumentMessageFactory extends DocumentMessageFactory {
- protected void decodeInto(RemoveDocumentMessage msg, DocumentDeserializer buf) {
- msg.setDocumentId(new DocumentId(buf));
- }
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- RemoveDocumentMessage msg = new RemoveDocumentMessage();
- decodeInto(msg, buf);
- return msg;
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- RemoveDocumentMessage msg = (RemoveDocumentMessage)obj;
- msg.getDocumentId().serialize(buf);
- return true;
- }
- }
-
- public static class RemoveDocumentReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- RemoveDocumentReply reply = new RemoveDocumentReply();
- byte flag = buf.getByte(null);
- reply.setWasFound(flag != 0);
- reply.setHighestModificationTimestamp(buf.getLong(null));
- return reply;
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- RemoveDocumentReply reply = (RemoveDocumentReply)obj;
- buf.putByte(null, (byte)(reply.wasFound() ? 1 : 0));
- buf.putLong(null, reply.getHighestModificationTimestamp());
- return true;
- }
- }
-
- public static class RemoveLocationMessageFactory extends DocumentMessageFactory {
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- return new RemoveLocationMessage(decodeString(buf));
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- RemoveLocationMessage msg = (RemoveLocationMessage)obj;
- encodeString(msg.getDocumentSelection(), buf);
- return true;
- }
- }
-
- public static class RemoveLocationReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- return new DocumentReply(DocumentProtocol.REPLY_REMOVELOCATION);
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- return true;
- }
- }
-
- public static class SearchResultMessageFactory extends DocumentMessageFactory {
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- SearchResultMessage msg = new SearchResultMessage();
- msg.setSearchResult(new SearchResult(buf));
- return msg;
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- return false; // not supported
- }
- }
-
- public static class QueryResultMessageFactory extends DocumentMessageFactory {
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- QueryResultMessage msg = new QueryResultMessage();
- msg.setSearchResult(new SearchResult(buf));
- msg.setSummary(new DocumentSummary(buf));
- return msg;
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- return false; // not supported
- }
- }
-
- public static class SearchResultReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- return new VisitorReply(DocumentProtocol.REPLY_SEARCHRESULT);
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- return true;
- }
- }
-
- public static class QueryResultReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- return new VisitorReply(DocumentProtocol.REPLY_QUERYRESULT);
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- return true;
- }
- }
-
- public static class StatBucketMessageFactory extends DocumentMessageFactory {
-
- protected String decodeBucketSpace(Deserializer deserializer) {
- return FixedBucketSpaces.defaultSpace();
- }
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- StatBucketMessage msg = new StatBucketMessage();
- msg.setBucketId(new BucketId(buf.getLong(null)));
- msg.setDocumentSelection(decodeString(buf));
- msg.setBucketSpace(decodeBucketSpace(buf));
- return msg;
- }
-
- protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) {
- return FixedBucketSpaces.defaultSpace().equals(bucketSpace);
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- StatBucketMessage msg = (StatBucketMessage)obj;
- buf.putLong(null, msg.getBucketId().getRawId());
- encodeString(msg.getDocumentSelection(), buf);
- return encodeBucketSpace(msg.getBucketSpace(), buf);
- }
- }
-
- public static class StatBucketReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- StatBucketReply reply = new StatBucketReply();
- reply.setResults(decodeString(buf));
- return reply;
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- StatBucketReply reply = (StatBucketReply)obj;
- encodeString(reply.getResults(), buf);
- return true;
- }
- }
-
- public static class UpdateDocumentMessageFactory extends DocumentMessageFactory {
- protected void decodeInto(UpdateDocumentMessage msg, DocumentDeserializer buf) {
- msg.setDocumentUpdate(new DocumentUpdate(buf));
- msg.setOldTimestamp(buf.getLong(null));
- msg.setNewTimestamp(buf.getLong(null));
- }
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buffer) {
- final LazyDecoder decoder = (obj, buf) -> {
- decodeInto((UpdateDocumentMessage) obj, buf);
- };
-
- return new UpdateDocumentMessage(decoder, buffer);
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- UpdateDocumentMessage msg = (UpdateDocumentMessage)obj;
- if (msg.getSerializedBuffer() != null) {
- buf.put(null, msg.getSerializedBuffer());
- } else {
- msg.getDocumentUpdate().serialize(buf);
- buf.putLong(null, msg.getOldTimestamp());
- buf.putLong(null, msg.getNewTimestamp());
- }
- return true;
- }
- }
-
- public static class UpdateDocumentReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- UpdateDocumentReply rep = new UpdateDocumentReply();
- byte flag = buf.getByte(null);
- rep.setWasFound(flag != 0);
- rep.setHighestModificationTimestamp(buf.getLong(null));
- return rep;
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- UpdateDocumentReply rep = (UpdateDocumentReply)obj;
- buf.putByte(null, (byte)(rep.wasFound() ? 1 : 0));
- buf.putLong(null, rep.getHighestModificationTimestamp());
- return true;
- }
- }
-
- public static class VisitorInfoMessageFactory extends DocumentMessageFactory {
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- VisitorInfoMessage msg = new VisitorInfoMessage();
- int size = buf.getInt(null);
- for (int i = 0; i < size; i++) {
- long reversed = buf.getLong(null);
- long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) |
- ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) |
- ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) |
- ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l);
- msg.getFinishedBuckets().add(new BucketId(rawid));
- }
-
- msg.setErrorMessage(decodeString(buf));
- return msg;
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- VisitorInfoMessage msg = (VisitorInfoMessage)obj;
- buf.putInt(null, msg.getFinishedBuckets().size());
- for (BucketId id : msg.getFinishedBuckets()) {
- long rawid = id.getRawId();
- long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) |
- ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) |
- ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) |
- ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l);
- buf.putLong(null, reversed);
- }
- encodeString(msg.getErrorMessage(), buf);
- return true;
- }
- }
-
- public static class VisitorInfoReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- return new VisitorReply(DocumentProtocol.REPLY_VISITORINFO);
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- return true;
- }
- }
-
- public static class WrongDistributionReplyFactory extends DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- WrongDistributionReply reply = new WrongDistributionReply();
- reply.setSystemState(decodeString(buf));
- return reply;
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- WrongDistributionReply reply = (WrongDistributionReply)obj;
- encodeString(reply.getSystemState(), buf);
- return true;
- }
- }
-
-}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories51.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories51.java
deleted file mode 100755
index 0ef5835008a..00000000000
--- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories51.java
+++ /dev/null
@@ -1,137 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.document.BucketId;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.FixedBucketSpaces;
-import com.yahoo.document.serialization.DocumentDeserializer;
-import com.yahoo.document.serialization.DocumentSerializer;
-import com.yahoo.vespa.objects.Deserializer;
-
-import java.util.Map;
-
-/**
- * This class encapsulates all the {@link RoutableFactory} classes needed to implement serialization for the document
- * protocol. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now.
- *
- */
-public abstract class RoutableFactories51 extends RoutableFactories50 {
-
- public static class CreateVisitorMessageFactory extends DocumentMessageFactory {
-
- protected String decodeBucketSpace(Deserializer deserializer) {
- return FixedBucketSpaces.defaultSpace();
- }
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- CreateVisitorMessage msg = new CreateVisitorMessage();
- msg.setLibraryName(decodeString(buf));
- msg.setInstanceId(decodeString(buf));
- msg.setControlDestination(decodeString(buf));
- msg.setDataDestination(decodeString(buf));
- msg.setDocumentSelection(decodeString(buf));
- msg.setMaxPendingReplyCount(buf.getInt(null));
-
- int size = buf.getInt(null);
- for (int i = 0; i < size; i++) {
- long reversed = buf.getLong(null);
- long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) |
- ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) |
- ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) |
- ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l);
- msg.getBuckets().add(new BucketId(rawid));
- }
-
- msg.setFromTimestamp(buf.getLong(null));
- msg.setToTimestamp(buf.getLong(null));
- msg.setVisitRemoves(buf.getByte(null) == (byte)1);
- msg.setFieldSet(decodeString(buf));
- msg.setVisitInconsistentBuckets(buf.getByte(null) == (byte)1);
-
- size = buf.getInt(null);
- for (int i = 0; i < size; i++) {
- String key = decodeString(buf);
- int sz = buf.getInt(null);
- msg.getParameters().put(key, buf.getBytes(null, sz));
- }
-
- msg.setVisitorOrdering(buf.getInt(null));
- msg.setMaxBucketsPerVisitor(buf.getInt(null));
- msg.setVisitorDispatcherVersion(50);
- msg.setBucketSpace(decodeBucketSpace(buf));
- return msg;
- }
-
- protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) {
- return FixedBucketSpaces.defaultSpace().equals(bucketSpace);
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- CreateVisitorMessage msg = (CreateVisitorMessage)obj;
- encodeString(msg.getLibraryName(), buf);
- encodeString(msg.getInstanceId(), buf);
- encodeString(msg.getControlDestination(), buf);
- encodeString(msg.getDataDestination(), buf);
- encodeString(msg.getDocumentSelection(), buf);
- buf.putInt(null, msg.getMaxPendingReplyCount());
-
- buf.putInt(null, msg.getBuckets().size());
- for (BucketId id : msg.getBuckets()) {
- long rawid = id.getRawId();
- long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) |
- ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) |
- ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) |
- ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l);
- buf.putLong(null, reversed);
- }
-
- buf.putLong(null, msg.getFromTimestamp());
- buf.putLong(null, msg.getToTimestamp());
- buf.putByte(null, msg.getVisitRemoves() ? (byte)1 : (byte)0);
- encodeString(msg.getFieldSet(), buf);
- buf.putByte(null, msg.getVisitInconsistentBuckets() ? (byte)1 : (byte)0);
-
- buf.putInt(null, msg.getParameters().size());
- for (Map.Entry<String, byte[]> pairs : msg.getParameters().entrySet()) {
- encodeString(pairs.getKey(), buf);
- byte[] b = pairs.getValue();
- buf.putInt(null, b.length);
- buf.put(null, b);
- }
-
- buf.putInt(null, msg.getVisitorOrdering());
- buf.putInt(null, msg.getMaxBucketsPerVisitor());
- return encodeBucketSpace(msg.getBucketSpace(), buf);
- }
- }
-
- public static class GetDocumentMessageFactory extends DocumentMessageFactory {
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- return new GetDocumentMessage(new DocumentId(buf), decodeString(buf));
- }
-
- @Override
- protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- GetDocumentMessage msg = (GetDocumentMessage)obj;
- msg.getDocumentId().serialize(buf);
- encodeString(msg.getFieldSet(), buf);
- return true;
- }
- }
-
- public static class DocumentIgnoredReplyFactory extends DocumentReplyFactory {
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- return new DocumentIgnoredReply();
- }
-
- @Override
- protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
- return true;
- }
- }
-}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java
index 2e18c681c50..89a0aa22bb3 100644
--- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java
+++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java
@@ -1,82 +1,948 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.documentapi.messagebus.protocol;
-import com.google.common.annotations.Beta;
+import com.yahoo.document.BucketId;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.FixedBucketSpaces;
import com.yahoo.document.TestAndSetCondition;
import com.yahoo.document.serialization.DocumentDeserializer;
import com.yahoo.document.serialization.DocumentSerializer;
+import com.yahoo.document.serialization.DocumentSerializerFactory;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.log.LogLevel;
+import com.yahoo.messagebus.Routable;
+import com.yahoo.vdslib.DocumentList;
+import com.yahoo.vdslib.DocumentSummary;
+import com.yahoo.vdslib.SearchResult;
+import com.yahoo.vdslib.VisitorStatistics;
+import com.yahoo.vespa.objects.Deserializer;
+import com.yahoo.vespa.objects.Serializer;
+
+import java.util.Map;
+import java.util.logging.Logger;
import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.decodeString;
import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.encodeString;
+
/**
* @author Vegard Sjonfjell
+ * This class encapsulates all the {@link RoutableFactory} classes needed to implement serialization for the document
+ * protocol. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now.
*/
+public abstract class RoutableFactories52 {
+
+ /**
+ * Implements the shared factory logic required for {@link DocumentMessage} objects, and it offers a more convenient
+ * interface for implementing {@link RoutableFactory}.
+ *
+ * @author Simon Thoresen Hult
+ */
+ public static abstract class DocumentMessageFactory extends AbstractRoutableFactory {
+
+ /**
+ * This method encodes the given message using the given serializer. You are guaranteed to only receive messages
+ * of the type that this factory was registered for.
+ * <p>
+ * This method is NOT exception safe. Return false to
+ * signal failure.
+ *
+ * @param msg The message to encode.
+ * @param serializer The serializer to use for encoding.
+ * @return True if the message was encoded.
+ */
+ protected abstract boolean doEncode(DocumentMessage msg, DocumentSerializer serializer);
+
+ /**
+ * This method decodes a message from the given deserializer. You are guaranteed to only receive byte buffers
+ * generated by a previous call to {@link #doEncode(DocumentMessage, DocumentSerializer)}.
+ * <p>
+ * This method is NOT exception safe. Return null to signal failure.
+ *
+ * @param deserializer The deserializer to use for decoding.
+ * @return The decoded message.
+ */
+ protected abstract DocumentMessage doDecode(DocumentDeserializer deserializer);
+
+ public boolean encode(Routable obj, DocumentSerializer out) {
+ if (!(obj instanceof DocumentMessage)) {
+ throw new AssertionError(
+ "Document message factory (" + getClass().getName() + ") registered for incompatible " +
+ "routable type " + obj.getType() + "(" + obj.getClass().getName() + ").");
+ }
+ DocumentMessage msg = (DocumentMessage)obj;
+ out.putByte(null, (byte)(msg.getPriority().getValue()));
+ out.putInt(null, msg.getLoadType().getId());
+ return doEncode(msg, out);
+ }
+
+ public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) {
+ byte pri = in.getByte(null);
+ int loadType = in.getInt(null);
+ DocumentMessage msg = doDecode(in);
+ if (msg != null) {
+ msg.setPriority(DocumentProtocol.getPriority(pri));
+ msg.setLoadType(loadTypes.getIdMap().get(loadType));
+ }
+ return msg;
+ }
+ }
+
+ /**
+ * Implements the shared factory logic required for {@link DocumentReply} objects, and it offers a more convenient
+ * interface for implementing {@link RoutableFactory}.
+ *
+ * @author Simon Thoresen Hult
+ */
+ public static abstract class DocumentReplyFactory extends AbstractRoutableFactory {
+
+ /**
+ * This method encodes the given reply into the given byte buffer. You are guaranteed to only receive replies of
+ * the type that this factory was registered for.
+ * <p>
+ * This method is NOT exception safe. Return false to signal
+ * failure.
+ *
+ * @param reply The reply to encode.
+ * @param buf The byte buffer to write to.
+ * @return True if the message was encoded.
+ */
+ protected abstract boolean doEncode(DocumentReply reply, DocumentSerializer buf);
+
+ /**
+ * This method decodes a reply from the given byte buffer. You are guaranteed to only receive byte buffers
+ * generated by a previous call to {@link #doEncode(DocumentReply, com.yahoo.document.serialization.DocumentSerializer)}.
+ *
+ * <p>
+ * This method is NOT exception safe. Return null to signal failure.
+ *
+ * @param buf The byte buffer to read from.
+ * @return The decoded reply.
+ */
+ protected abstract DocumentReply doDecode(DocumentDeserializer buf);
+
+ public boolean encode(Routable obj, DocumentSerializer out) {
+ if (!(obj instanceof DocumentReply)) {
+ throw new AssertionError(
+ "Document reply factory (" + getClass().getName() + ") registered for incompatible " +
+ "routable type " + obj.getType() + "(" + obj.getClass().getName() + ").");
+ }
+ DocumentReply reply = (DocumentReply)obj;
+ out.putByte(null, (byte)(reply.getPriority().getValue()));
+ return doEncode(reply, out);
+ }
+
+ public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) {
+ byte pri = in.getByte(null);
+ DocumentReply reply = doDecode(in);
+ if (reply != null) {
+ reply.setPriority(DocumentProtocol.getPriority(pri));
+ }
+ return reply;
+ }
+ }
+
+ public static class CreateVisitorMessageFactory extends DocumentMessageFactory {
+
+ protected String decodeBucketSpace(Deserializer deserializer) {
+ return FixedBucketSpaces.defaultSpace();
+ }
-@Beta
-public abstract class RoutableFactories52 extends RoutableFactories51 {
- protected static class PutDocumentMessageFactory extends RoutableFactories51.PutDocumentMessageFactory {
@Override
- protected void decodeInto(PutDocumentMessage msg, DocumentDeserializer buf) {
- super.decodeInto(msg, buf);
- decodeTasCondition(msg, buf);
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ CreateVisitorMessage msg = new CreateVisitorMessage();
+ msg.setLibraryName(decodeString(buf));
+ msg.setInstanceId(decodeString(buf));
+ msg.setControlDestination(decodeString(buf));
+ msg.setDataDestination(decodeString(buf));
+ msg.setDocumentSelection(decodeString(buf));
+ msg.setMaxPendingReplyCount(buf.getInt(null));
+
+ int size = buf.getInt(null);
+ for (int i = 0; i < size; i++) {
+ long reversed = buf.getLong(null);
+ long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) |
+ ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) |
+ ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) |
+ ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l);
+ msg.getBuckets().add(new BucketId(rawid));
+ }
+
+ msg.setFromTimestamp(buf.getLong(null));
+ msg.setToTimestamp(buf.getLong(null));
+ msg.setVisitRemoves(buf.getByte(null) == (byte)1);
+ msg.setFieldSet(decodeString(buf));
+ msg.setVisitInconsistentBuckets(buf.getByte(null) == (byte)1);
+
+ size = buf.getInt(null);
+ for (int i = 0; i < size; i++) {
+ String key = decodeString(buf);
+ int sz = buf.getInt(null);
+ msg.getParameters().put(key, buf.getBytes(null, sz));
+ }
+
+ msg.setVisitorOrdering(buf.getInt(null));
+ msg.setMaxBucketsPerVisitor(buf.getInt(null));
+ msg.setVisitorDispatcherVersion(50);
+ msg.setBucketSpace(decodeBucketSpace(buf));
+ return msg;
+ }
+
+ protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) {
+ return FixedBucketSpaces.defaultSpace().equals(bucketSpace);
}
@Override
protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- if (!super.doEncode(obj, buf)) {
- return false;
+ CreateVisitorMessage msg = (CreateVisitorMessage)obj;
+ encodeString(msg.getLibraryName(), buf);
+ encodeString(msg.getInstanceId(), buf);
+ encodeString(msg.getControlDestination(), buf);
+ encodeString(msg.getDataDestination(), buf);
+ encodeString(msg.getDocumentSelection(), buf);
+ buf.putInt(null, msg.getMaxPendingReplyCount());
+
+ buf.putInt(null, msg.getBuckets().size());
+ for (BucketId id : msg.getBuckets()) {
+ long rawid = id.getRawId();
+ long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) |
+ ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) |
+ ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) |
+ ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l);
+ buf.putLong(null, reversed);
}
- // If the serialized buffer exists, the test and set condition has already been encoded
- if (((PutDocumentMessage)obj).getSerializedBuffer() == null) {
- encodeTasCondition(buf, (TestAndSetMessage) obj);
+ buf.putLong(null, msg.getFromTimestamp());
+ buf.putLong(null, msg.getToTimestamp());
+ buf.putByte(null, msg.getVisitRemoves() ? (byte)1 : (byte)0);
+ encodeString(msg.getFieldSet(), buf);
+ buf.putByte(null, msg.getVisitInconsistentBuckets() ? (byte)1 : (byte)0);
+
+ buf.putInt(null, msg.getParameters().size());
+ for (Map.Entry<String, byte[]> pairs : msg.getParameters().entrySet()) {
+ encodeString(pairs.getKey(), buf);
+ byte[] b = pairs.getValue();
+ buf.putInt(null, b.length);
+ buf.put(null, b);
+ }
+
+ buf.putInt(null, msg.getVisitorOrdering());
+ buf.putInt(null, msg.getMaxBucketsPerVisitor());
+ return encodeBucketSpace(msg.getBucketSpace(), buf);
+ }
+ }
+
+ public static class CreateVisitorReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR);
+ reply.setLastBucket(new BucketId(buf.getLong(null)));
+
+ VisitorStatistics vs = new VisitorStatistics();
+ vs.setBucketsVisited(buf.getInt(null));
+ vs.setDocumentsVisited(buf.getLong(null));
+ vs.setBytesVisited(buf.getLong(null));
+ vs.setDocumentsReturned(buf.getLong(null));
+ vs.setBytesReturned(buf.getLong(null));
+ vs.setSecondPassDocumentsReturned(buf.getLong(null));
+ vs.setSecondPassBytesReturned(buf.getLong(null));
+ reply.setVisitorStatistics(vs);
+ return reply;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ CreateVisitorReply reply = (CreateVisitorReply)obj;
+ buf.putLong(null, reply.getLastBucket().getRawId());
+ buf.putInt(null, reply.getVisitorStatistics().getBucketsVisited());
+ buf.putLong(null, reply.getVisitorStatistics().getDocumentsVisited());
+ buf.putLong(null, reply.getVisitorStatistics().getBytesVisited());
+ buf.putLong(null, reply.getVisitorStatistics().getDocumentsReturned());
+ buf.putLong(null, reply.getVisitorStatistics().getBytesReturned());
+ buf.putLong(null, reply.getVisitorStatistics().getSecondPassDocumentsReturned());
+ buf.putLong(null, reply.getVisitorStatistics().getSecondPassBytesReturned());
+ return true;
+ }
+ }
+
+ public static class DestroyVisitorMessageFactory extends DocumentMessageFactory {
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ DestroyVisitorMessage msg = new DestroyVisitorMessage();
+ msg.setInstanceId(decodeString(buf));
+ return msg;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ DestroyVisitorMessage msg = (DestroyVisitorMessage)obj;
+ encodeString(msg.getInstanceId(), buf);
+ return true;
+ }
+ }
+
+ public static class DestroyVisitorReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ return new VisitorReply(DocumentProtocol.REPLY_DESTROYVISITOR);
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ return true;
+ }
+ }
+
+ public static class DocumentIgnoredReplyFactory extends DocumentReplyFactory {
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ return new DocumentIgnoredReply();
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ return true;
+ }
+ }
+
+ public static class DocumentListMessageFactory extends DocumentMessageFactory {
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ DocumentListMessage msg = new DocumentListMessage();
+ msg.setBucketId(new BucketId(buf.getLong(null)));
+ int len = buf.getInt(null);
+ for (int i = 0; i < len; i++) {
+ msg.getDocuments().add(new DocumentListEntry(buf));
}
+ return msg;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ DocumentListMessage msg = (DocumentListMessage)obj;
+ buf.putLong(null, msg.getBucketId().getRawId());
+ buf.putInt(null, msg.getDocuments().size());
+ for (int i = 0; i < msg.getDocuments().size(); i++) {
+ msg.getDocuments().get(i).serialize(buf);
+ }
return true;
- }
+ }
}
- protected static class RemoveDocumentMessageFactory extends RoutableFactories51.RemoveDocumentMessageFactory {
+ public static class DocumentListReplyFactory extends DocumentReplyFactory {
+
@Override
- protected void decodeInto(RemoveDocumentMessage msg, DocumentDeserializer buf) {
- super.decodeInto(msg, buf);
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTLIST);
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ return true;
+ }
+ }
+
+ public static class DocumentSummaryMessageFactory extends DocumentMessageFactory {
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ DocumentSummaryMessage msg = new DocumentSummaryMessage();
+ msg.setDocumentSummary(new DocumentSummary(buf));
+ return msg;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ return false; // not supported
+ }
+ }
+
+ public static class DocumentSummaryReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTSUMMARY);
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ return true;
+ }
+ }
+
+ public static class EmptyBucketsMessageFactory extends DocumentMessageFactory {
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ EmptyBucketsMessage msg = new EmptyBucketsMessage();
+ int size = buf.getInt(null);
+ for (int i = 0; i < size; ++i) {
+ msg.getBucketIds().add(new BucketId(buf.getLong(null)));
+ }
+ return msg;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ EmptyBucketsMessage msg = (EmptyBucketsMessage)obj;
+ buf.putInt(null, msg.getBucketIds().size());
+ for (BucketId bid : msg.getBucketIds()) {
+ buf.putLong(null, bid.getRawId());
+ }
+ return true;
+ }
+ }
+
+ public static class EmptyBucketsReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ return new VisitorReply(DocumentProtocol.REPLY_EMPTYBUCKETS);
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ return true;
+ }
+ }
+
+ public static class GetBucketListMessageFactory extends DocumentMessageFactory {
+
+ protected String decodeBucketSpace(Deserializer deserializer) {
+ return FixedBucketSpaces.defaultSpace();
+ }
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ GetBucketListMessage msg = new GetBucketListMessage();
+ msg.setBucketId(new BucketId(buf.getLong(null)));
+ msg.setBucketSpace(decodeBucketSpace(buf));
+ return msg;
+ }
+
+ protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) {
+ return FixedBucketSpaces.defaultSpace().equals(bucketSpace);
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ GetBucketListMessage msg = (GetBucketListMessage)obj;
+ buf.putLong(null, msg.getBucketId().getRawId());
+ return encodeBucketSpace(msg.getBucketSpace(), buf);
+ }
+ }
+
+ public static class GetBucketListReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ GetBucketListReply reply = new GetBucketListReply();
+ int len = buf.getInt(null);
+ for (int i = 0; i < len; i++) {
+ GetBucketListReply.BucketInfo info = new GetBucketListReply.BucketInfo();
+ info.bucket = new BucketId(buf.getLong(null));
+ info.bucketInformation = decodeString(buf);
+ reply.getBuckets().add(info);
+ }
+ return reply;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ GetBucketListReply reply = (GetBucketListReply)obj;
+ buf.putInt(null, reply.getBuckets().size());
+ for (GetBucketListReply.BucketInfo info : reply.getBuckets()) {
+ buf.putLong(null, info.bucket.getRawId());
+ encodeString(info.bucketInformation, buf);
+ }
+ return true;
+ }
+ }
+
+ public static class GetBucketStateMessageFactory extends DocumentMessageFactory {
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ GetBucketStateMessage msg = new GetBucketStateMessage();
+ msg.setBucketId(new BucketId(buf.getLong(null)));
+ return msg;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ GetBucketStateMessage msg = (GetBucketStateMessage)obj;
+ buf.putLong(null, msg.getBucketId().getRawId());
+ return true;
+ }
+ }
+
+ public static class GetBucketStateReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ GetBucketStateReply reply = new GetBucketStateReply();
+ int size = buf.getInt(null);
+ for (int i = 0; i < size; i++) {
+ reply.getBucketState().add(new DocumentState(buf));
+ }
+ return reply;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ GetBucketStateReply reply = (GetBucketStateReply)obj;
+ buf.putInt(null, reply.getBucketState().size());
+ for (DocumentState stat : reply.getBucketState()) {
+ stat.serialize(buf);
+ }
+ return true;
+ }
+ }
+
+ public static class GetDocumentMessageFactory extends DocumentMessageFactory {
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ return new GetDocumentMessage(new DocumentId(buf), decodeString(buf));
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ GetDocumentMessage msg = (GetDocumentMessage)obj;
+ msg.getDocumentId().serialize(buf);
+ encodeString(msg.getFieldSet(), buf);
+ return true;
+ }
+ }
+
+ public static class GetDocumentReplyFactory extends DocumentReplyFactory {
+
+ private final LazyDecoder decoder = new LazyDecoder() {
+
+ public void decode(Routable obj, DocumentDeserializer buf) {
+ GetDocumentReply reply = (GetDocumentReply)obj;
+
+ Document doc = null;
+ byte flag = buf.getByte(null);
+ if (flag != 0) {
+ doc = Document.createDocument(buf);
+ reply.setDocument(doc);
+ }
+ long lastModified = buf.getLong(null);
+ reply.setLastModified(lastModified);
+ if (doc != null) {
+ doc.setLastModified(lastModified);
+ }
+ }
+ };
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ GetDocumentReply reply = new GetDocumentReply(decoder, buf);
+
+ return reply;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ GetDocumentReply reply = (GetDocumentReply)obj;
+ if (reply.getSerializedBuffer() != null) {
+ buf.put(null, reply.getSerializedBuffer());
+ } else {
+ Document document = reply.getDocument();
+ buf.putByte(null, (byte)(document == null ? 0 : 1));
+ if (document != null) {
+ document.serialize(buf);
+ }
+ buf.putLong(null, reply.getLastModified());
+ }
+ return true;
+ }
+ }
+
+ public static class MapVisitorMessageFactory extends DocumentMessageFactory {
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ MapVisitorMessage msg = new MapVisitorMessage();
+ int size = buf.getInt(null);
+ for (int i = 0; i < size; i++) {
+ String key = decodeString(buf);
+ String value = decodeString(buf);
+ msg.getData().put(key, value);
+ }
+ return msg;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ MapVisitorMessage msg = (MapVisitorMessage)obj;
+ buf.putInt(null, msg.getData().size());
+ for (Map.Entry<String, String> pairs : msg.getData().entrySet()) {
+ encodeString(pairs.getKey(), buf);
+ encodeString(pairs.getValue(), buf);
+ }
+ return true;
+ }
+ }
+
+ public static class MapVisitorReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ return new VisitorReply(DocumentProtocol.REPLY_MAPVISITOR);
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ return true;
+ }
+ }
+
+ public static class PutDocumentMessageFactory extends DocumentMessageFactory {
+ protected void decodeInto(PutDocumentMessage msg, DocumentDeserializer buf) {
+ msg.setDocumentPut(new DocumentPut(Document.createDocument(buf)));
+ msg.setTimestamp(buf.getLong(null));
decodeTasCondition(msg, buf);
}
@Override
+ protected DocumentMessage doDecode(DocumentDeserializer buffer) {
+ final LazyDecoder decoder = (obj, buf) -> {
+ decodeInto((PutDocumentMessage) obj, buf);
+ };
+
+ return new PutDocumentMessage(decoder, buffer);
+ }
+
+ @Override
protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- if (!super.doEncode(obj, buf)) {
- return false;
+ PutDocumentMessage msg = (PutDocumentMessage)obj;
+ if (msg.getSerializedBuffer() != null) {
+ buf.put(null, msg.getSerializedBuffer());
+ } else {
+ msg.getDocumentPut().getDocument().serialize(buf);
+ buf.putLong(null, msg.getTimestamp());
+ encodeTasCondition(buf, (TestAndSetMessage) obj);
}
+ return true;
+ }
+ }
+ public static class PutDocumentReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ WriteDocumentReply rep = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT);
+ rep.setHighestModificationTimestamp(buf.getLong(null));
+ return rep;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ WriteDocumentReply rep = (WriteDocumentReply)obj;
+ buf.putLong(null, rep.getHighestModificationTimestamp());
+ return true;
+ }
+ }
+
+ public static class RemoveDocumentMessageFactory extends DocumentMessageFactory {
+ protected void decodeInto(RemoveDocumentMessage msg, DocumentDeserializer buf) {
+ msg.setDocumentId(new DocumentId(buf));
+ decodeTasCondition(msg, buf);
+ }
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ RemoveDocumentMessage msg = new RemoveDocumentMessage();
+ decodeInto(msg, buf);
+ return msg;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ RemoveDocumentMessage msg = (RemoveDocumentMessage)obj;
+ msg.getDocumentId().serialize(buf);
encodeTasCondition(buf, (TestAndSetMessage) obj);
return true;
}
}
- protected static class UpdateDocumentMessageFactory extends RoutableFactories51.UpdateDocumentMessageFactory {
+ public static class RemoveDocumentReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ RemoveDocumentReply reply = new RemoveDocumentReply();
+ byte flag = buf.getByte(null);
+ reply.setWasFound(flag != 0);
+ reply.setHighestModificationTimestamp(buf.getLong(null));
+ return reply;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ RemoveDocumentReply reply = (RemoveDocumentReply)obj;
+ buf.putByte(null, (byte)(reply.wasFound() ? 1 : 0));
+ buf.putLong(null, reply.getHighestModificationTimestamp());
+ return true;
+ }
+ }
+
+ public static class RemoveLocationMessageFactory extends DocumentMessageFactory {
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ return new RemoveLocationMessage(decodeString(buf));
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ RemoveLocationMessage msg = (RemoveLocationMessage)obj;
+ encodeString(msg.getDocumentSelection(), buf);
+ return true;
+ }
+ }
+
+ public static class RemoveLocationReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ return new DocumentReply(DocumentProtocol.REPLY_REMOVELOCATION);
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ return true;
+ }
+ }
+
+ public static class SearchResultMessageFactory extends DocumentMessageFactory {
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ SearchResultMessage msg = new SearchResultMessage();
+ msg.setSearchResult(new SearchResult(buf));
+ return msg;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ return false; // not supported
+ }
+ }
+
+ public static class QueryResultMessageFactory extends DocumentMessageFactory {
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ QueryResultMessage msg = new QueryResultMessage();
+ msg.setSearchResult(new SearchResult(buf));
+ msg.setSummary(new DocumentSummary(buf));
+ return msg;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ return false; // not supported
+ }
+ }
+
+ public static class SearchResultReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ return new VisitorReply(DocumentProtocol.REPLY_SEARCHRESULT);
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ return true;
+ }
+ }
+
+ public static class QueryResultReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ return new VisitorReply(DocumentProtocol.REPLY_QUERYRESULT);
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ return true;
+ }
+ }
+
+ public static class StatBucketMessageFactory extends DocumentMessageFactory {
+
+ protected String decodeBucketSpace(Deserializer deserializer) {
+ return FixedBucketSpaces.defaultSpace();
+ }
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ StatBucketMessage msg = new StatBucketMessage();
+ msg.setBucketId(new BucketId(buf.getLong(null)));
+ msg.setDocumentSelection(decodeString(buf));
+ msg.setBucketSpace(decodeBucketSpace(buf));
+ return msg;
+ }
+
+ protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) {
+ return FixedBucketSpaces.defaultSpace().equals(bucketSpace);
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ StatBucketMessage msg = (StatBucketMessage)obj;
+ buf.putLong(null, msg.getBucketId().getRawId());
+ encodeString(msg.getDocumentSelection(), buf);
+ return encodeBucketSpace(msg.getBucketSpace(), buf);
+ }
+ }
+
+ public static class StatBucketReplyFactory extends DocumentReplyFactory {
+
@Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ StatBucketReply reply = new StatBucketReply();
+ reply.setResults(decodeString(buf));
+ return reply;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ StatBucketReply reply = (StatBucketReply)obj;
+ encodeString(reply.getResults(), buf);
+ return true;
+ }
+ }
+
+ public static class UpdateDocumentMessageFactory extends DocumentMessageFactory {
protected void decodeInto(UpdateDocumentMessage msg, DocumentDeserializer buf) {
- super.decodeInto(msg, buf);
+ msg.setDocumentUpdate(new DocumentUpdate(buf));
+ msg.setOldTimestamp(buf.getLong(null));
+ msg.setNewTimestamp(buf.getLong(null));
decodeTasCondition(msg, buf);
}
@Override
+ protected DocumentMessage doDecode(DocumentDeserializer buffer) {
+ final LazyDecoder decoder = (obj, buf) -> {
+ decodeInto((UpdateDocumentMessage) obj, buf);
+ };
+
+ return new UpdateDocumentMessage(decoder, buffer);
+ }
+
+ @Override
protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
- if (!super.doEncode(obj, buf)) {
- return false;
+ UpdateDocumentMessage msg = (UpdateDocumentMessage)obj;
+ if (msg.getSerializedBuffer() != null) {
+ buf.put(null, msg.getSerializedBuffer());
+ } else {
+ msg.getDocumentUpdate().serialize(buf);
+ buf.putLong(null, msg.getOldTimestamp());
+ buf.putLong(null, msg.getNewTimestamp());
+ encodeTasCondition(buf, (TestAndSetMessage) obj);
}
+ return true;
+ }
+ }
- // If the serialized buffer exists, the test and set condition has already been encoded
- if (((UpdateDocumentMessage)obj).getSerializedBuffer() == null) {
- encodeTasCondition(buf, (TestAndSetMessage) obj);
+ public static class UpdateDocumentReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ UpdateDocumentReply rep = new UpdateDocumentReply();
+ byte flag = buf.getByte(null);
+ rep.setWasFound(flag != 0);
+ rep.setHighestModificationTimestamp(buf.getLong(null));
+ return rep;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ UpdateDocumentReply rep = (UpdateDocumentReply)obj;
+ buf.putByte(null, (byte)(rep.wasFound() ? 1 : 0));
+ buf.putLong(null, rep.getHighestModificationTimestamp());
+ return true;
+ }
+ }
+
+ public static class VisitorInfoMessageFactory extends DocumentMessageFactory {
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ VisitorInfoMessage msg = new VisitorInfoMessage();
+ int size = buf.getInt(null);
+ for (int i = 0; i < size; i++) {
+ long reversed = buf.getLong(null);
+ long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) |
+ ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) |
+ ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) |
+ ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l);
+ msg.getFinishedBuckets().add(new BucketId(rawid));
}
+ msg.setErrorMessage(decodeString(buf));
+ return msg;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) {
+ VisitorInfoMessage msg = (VisitorInfoMessage)obj;
+ buf.putInt(null, msg.getFinishedBuckets().size());
+ for (BucketId id : msg.getFinishedBuckets()) {
+ long rawid = id.getRawId();
+ long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) |
+ ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) |
+ ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) |
+ ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l);
+ buf.putLong(null, reversed);
+ }
+ encodeString(msg.getErrorMessage(), buf);
return true;
}
}
+ public static class VisitorInfoReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ return new VisitorReply(DocumentProtocol.REPLY_VISITORINFO);
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ return true;
+ }
+ }
+
+ public static class WrongDistributionReplyFactory extends DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ WrongDistributionReply reply = new WrongDistributionReply();
+ reply.setSystemState(decodeString(buf));
+ return reply;
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) {
+ WrongDistributionReply reply = (WrongDistributionReply)obj;
+ encodeString(reply.getSystemState(), buf);
+ return true;
+ }
+ }
static void decodeTasCondition(TestAndSetMessage msg, DocumentDeserializer buf) {
msg.setCondition(new TestAndSetCondition(decodeString(buf)));
}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolTest.java
deleted file mode 100644
index 50778e00519..00000000000
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolTest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.component.Version;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.DocumentTypeManager;
-import com.yahoo.document.DocumentTypeManagerConfigurer;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author Simon Thoresen Hult
- */
-public class DocumentProtocolTest {
-
- private final DocumentTypeManager manager = new DocumentTypeManager();
-
- @Before
- public void setUp() {
- DocumentTypeManagerConfigurer.configure(manager, "file:./test/cfg/testdoc.cfg");
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void requireThat50SerializationPrecedes5xSerialization() {
- DocumentProtocol protocol = new DocumentProtocol(manager);
- GetDocumentMessage prev = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo");
- byte[] buf = protocol.encode(new Version(5, 0), prev);
-
- GetDocumentMessage next = (GetDocumentMessage)protocol.decode(new Version(5, 0), buf);
- assertEquals(GetDocumentMessage.DEFAULT_FIELD_SET, next.getFieldSet());
- }
-
-}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java
deleted file mode 100644
index 5eca8f49967..00000000000
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java
+++ /dev/null
@@ -1,904 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol.test;
-
-import com.yahoo.component.Version;
-import com.yahoo.document.*;
-import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
-import com.yahoo.document.idstring.IdString;
-import com.yahoo.document.select.OrderingSpecification;
-import com.yahoo.documentapi.messagebus.protocol.*;
-import com.yahoo.messagebus.Routable;
-import com.yahoo.text.Utf8;
-import com.yahoo.vdslib.DocumentList;
-import com.yahoo.vdslib.Entry;
-import com.yahoo.vdslib.SearchResult;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.*;
-
-/**
- * @author Simon Thoresen Hult
- */
-public class Messages50TestCase extends MessagesTestBase {
-
- @Override
- protected void registerTests(Map<Integer, RunnableTest> out) {
- // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
- // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
- out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage());
- out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage());
- out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage());
- out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage());
- out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage());
- out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage());
- out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage());
- out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage());
- out.put(DocumentProtocol.MESSAGE_MAPVISITOR, new testMapVisitorMessage());
- out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage());
- out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage());
- out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage());
- out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage());
- out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage());
- out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage());
- out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage());
- out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage());
- out.put(DocumentProtocol.REPLY_CREATEVISITOR, new testCreateVisitorReply());
- out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply());
- out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply());
- out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply());
- out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply());
- out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply());
- out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply());
- out.put(DocumentProtocol.REPLY_GETDOCUMENT, new testGetDocumentReply());
- out.put(DocumentProtocol.REPLY_MAPVISITOR, new testMapVisitorReply());
- out.put(DocumentProtocol.REPLY_PUTDOCUMENT, new testPutDocumentReply());
- out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply());
- out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply());
- out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply());
- out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply());
- out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply());
- out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply());
- out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply());
- out.put(DocumentProtocol.REPLY_WRONGDISTRIBUTION, new testWrongDistributionReply());
- }
-
- @Override
- protected Version version() {
- return new Version(5, 0);
- }
-
- @Override
- protected boolean shouldTestCoverage() {
- return false;
- }
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Tests
- //
- ////////////////////////////////////////////////////////////////////////////////
-
- private static int BASE_MESSAGE_LENGTH = 5;
-
- public class testRemoveLocationMessage implements RunnableTest {
-
- @Override
- public void run() {
- {
- RemoveLocationMessage msg = new RemoveLocationMessage("id.group == \"mygroup\"");
- assertEquals(BASE_MESSAGE_LENGTH + 29, serialize("RemoveLocationMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (RemoveLocationMessage)deserialize("RemoveLocationMessage", DocumentProtocol.MESSAGE_REMOVELOCATION, lang);
- assertEquals("id.group == \"mygroup\"", msg.getDocumentSelection());
- }
- }
- }
- }
-
- public class testGetBucketListMessage implements RunnableTest {
-
- @Override
- public void run() {
- GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123));
- msg.setLoadType(loadTypes.getNameMap().get("foo"));
- assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketListMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang);
- assertEquals(new BucketId(16, 123), msg.getBucketId());
- assertEquals("foo", msg.getLoadType().getName());
- }
- }
- }
-
-
- public class testStatBucketMessage implements RunnableTest {
-
- @Override
- public void run() {
- StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123");
- msg.setLoadType(null);
- assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("StatBucketMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang);
- assertEquals(new BucketId(16, 123), msg.getBucketId());
- assertEquals("id.user=123", msg.getDocumentSelection());
- assertEquals("default", msg.getLoadType().getName());
- }
- }
- }
-
- public class testGetBucketStateMessage implements RunnableTest {
-
- @Override
- public void run() {
- GetBucketStateMessage msg = new GetBucketStateMessage(new BucketId(16, 666));
- assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketStateMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (GetBucketStateMessage)deserialize("GetBucketStateMessage", DocumentProtocol.MESSAGE_GETBUCKETSTATE, lang);
- assertEquals(16, msg.getBucketId().getUsedBits());
- assertEquals(4611686018427388570l, msg.getBucketId().getId());
- }
- }
- }
-
- public class testCreateVisitorMessage implements RunnableTest {
-
- @Override
- @SuppressWarnings("deprecation")
- public void run() {
- CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london");
- msg.setDocumentSelection("true and false or true");
- msg.getParameters().put("myvar", Utf8.toBytes("somevalue"));
- msg.getParameters().put("anothervar", Utf8.toBytes("34"));
- msg.getBuckets().add(new BucketId(16, 1234));
- msg.setVisitRemoves(true);
- msg.setVisitorOrdering(OrderingSpecification.DESCENDING);
- msg.setMaxBucketsPerVisitor(2);
- assertEquals(BASE_MESSAGE_LENGTH + 168, serialize("CreateVisitorMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang);
- assertEquals("SomeLibrary", msg.getLibraryName());
- assertEquals("myvisitor", msg.getInstanceId());
- assertEquals("newyork", msg.getControlDestination());
- assertEquals("london", msg.getDataDestination());
- assertEquals("true and false or true", msg.getDocumentSelection());
- assertEquals(8, msg.getMaxPendingReplyCount());
- assertEquals(true, msg.getVisitRemoves());
- assertEquals(false, msg.getVisitInconsistentBuckets());
- assertEquals(1, msg.getBuckets().size());
- assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next());
- assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar")));
- assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar")));
- assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering());
- assertEquals(2, msg.getMaxBucketsPerVisitor());
- }
-
- msg.getBuckets().clear();
-
- assertEquals("CreateVisitorMessage(" +
- "No buckets, " +
- "selection 'true and false or true', " +
- "bucket space 'default', " +
- "library SomeLibrary, including removes, " +
- "get fields: [all]" +
- ")",
- msg.toString());
-
- msg.getBuckets().add(new BucketId(16, 1234));
-
- assertEquals("CreateVisitorMessage(" +
- "Bucket BucketId(0x40000000000004d2), " +
- "selection 'true and false or true', " +
- "bucket space 'default', " +
- "library SomeLibrary, including removes, " +
- "get fields: [all]" +
- ")",
- msg.toString());
-
- msg.getBuckets().add(new BucketId(16, 1235));
- msg.getBuckets().add(new BucketId(16, 1236));
- msg.getBuckets().add(new BucketId(16, 1237));
- msg.getBuckets().add(new BucketId(16, 1238));
- msg.setFromTimestamp(10001);
- msg.setToTimestamp(20002);
- msg.setVisitInconsistentBuckets(true);
- assertEquals("CreateVisitorMessage(" +
- "5 buckets: BucketId(0x40000000000004d2) BucketId(0x40000000000004d3) BucketId(0x40000000000004d4) ..., " +
- "time 10001-20002, " +
- "selection 'true and false or true', " +
- "bucket space 'default', " +
- "library SomeLibrary, including removes, " +
- "get fields: [all], " +
- "visit inconsistent buckets" +
- ")",
- msg.toString());
- }
- }
-
- public class testCreateVisitorReply implements RunnableTest {
-
- @Override
- public void run() {
- CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR);
- reply.setLastBucket(new BucketId(16, 123));
- reply.getVisitorStatistics().setBucketsVisited(3);
- reply.getVisitorStatistics().setDocumentsVisited(1000);
- reply.getVisitorStatistics().setBytesVisited(1024000);
- reply.getVisitorStatistics().setDocumentsReturned(123);
- reply.getVisitorStatistics().setBytesReturned(512000);
- reply.getVisitorStatistics().setSecondPassDocumentsReturned(456);
- reply.getVisitorStatistics().setSecondPassBytesReturned(789100);
-
- assertEquals(65, serialize("CreateVisitorReply", reply));
-
- for (Language lang : LANGUAGES) {
- reply = (CreateVisitorReply)deserialize("CreateVisitorReply", DocumentProtocol.REPLY_CREATEVISITOR, lang);
- assertNotNull(reply);
- assertEquals(new BucketId(16, 123), reply.getLastBucket());
- assertEquals(3, reply.getVisitorStatistics().getBucketsVisited());
- assertEquals(1000, reply.getVisitorStatistics().getDocumentsVisited());
- assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited());
- assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned());
- assertEquals(512000, reply.getVisitorStatistics().getBytesReturned());
- assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned());
- assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned());
- }
- }
- }
-
- public class testDestroyVisitorReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("DestroyVisitorReply", DocumentProtocol.REPLY_DESTROYVISITOR);
- }
- }
-
- public class testDocumentListReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("DocumentListReply", DocumentProtocol.REPLY_DOCUMENTLIST);
- }
- }
-
- public class testDocumentSummaryReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY);
- }
- }
-
- public class testEmptyBucketsReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("EmptyBucketsReply", DocumentProtocol.REPLY_EMPTYBUCKETS);
- }
- }
-
- public class testDestroyVisitorMessage implements RunnableTest {
-
- @Override
- public void run() {
- DestroyVisitorMessage msg = new DestroyVisitorMessage("myvisitor");
- assertEquals(BASE_MESSAGE_LENGTH + 17, serialize("DestroyVisitorMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (DestroyVisitorMessage)deserialize("DestroyVisitorMessage", DocumentProtocol.MESSAGE_DESTROYVISITOR, lang);
- assertEquals("myvisitor", msg.getInstanceId());
- }
- }
- }
-
- public class testDocumentListMessage implements RunnableTest {
-
- @Override
- public void run() {
- DocumentListMessage msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.CPP);
- assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString());
- assertEquals(1234, msg.getDocuments().get(0).getTimestamp());
- assertFalse(msg.getDocuments().get(0).isRemoveEntry());
-
- assertEquals(BASE_MESSAGE_LENGTH + 63, serialize("DocumentListMessage", msg));
- msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.JAVA);
- assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString());
- assertEquals(1234, msg.getDocuments().get(0).getTimestamp());
- assertFalse(msg.getDocuments().get(0).isRemoveEntry());
-
- }
- }
-
- public class testEmptyBucketsMessage implements RunnableTest {
-
- @Override
- public void run() {
- List<BucketId> bids = new ArrayList<>();
- for (int i = 0; i < 13; ++i) {
- bids.add(new BucketId(16, i));
- }
-
- EmptyBucketsMessage ebm = new EmptyBucketsMessage(bids);
- assertEquals(BASE_MESSAGE_LENGTH + 112, serialize("EmptyBucketsMessage", ebm));
- for (Language lang : LANGUAGES) {
- ebm = (EmptyBucketsMessage)deserialize("EmptyBucketsMessage", DocumentProtocol.MESSAGE_EMPTYBUCKETS, lang);
- for (int i = 0; i < 13; ++i) {
- assertEquals(new BucketId(16, i), ebm.getBucketIds().get(i));
- }
- }
- }
- }
-
- public class testDocumentSummaryMessage implements RunnableTest {
-
- @Override
- public void run() {
- try {
- FileInputStream stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-1.dat"));
- byte[] data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- Routable routable = decode(data);
- assertTrue(routable instanceof DocumentSummaryMessage);
-
- DocumentSummaryMessage msg = (DocumentSummaryMessage)routable;
- assertEquals(0, msg.getResult().getSummaryCount());
-
- stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-2.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof DocumentSummaryMessage);
-
- msg = (DocumentSummaryMessage)routable;
- assertEquals(2, msg.getResult().getSummaryCount());
- com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0);
- assertEquals("doc1", s.getDocId());
- byte[] b = s.getSummary();
- assertEquals(8, b.length);
- byte[] c = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(c[i], b[i]);
- }
-
- s = msg.getResult().getSummary(1);
- assertEquals("aoc17", s.getDocId());
- b = s.getSummary();
- assertEquals(9, b.length);
- byte[] d = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(d[i], b[i]);
- }
-
- stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-3.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof DocumentSummaryMessage);
-
- msg = (DocumentSummaryMessage)routable;
- assertEquals(2, msg.getResult().getSummaryCount());
-
- s = msg.getResult().getSummary(0);
- assertEquals("aoc17", s.getDocId());
- b = s.getSummary();
- assertEquals(9, b.length);
- byte[] e = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(e[i], b[i]);
- }
-
- s = msg.getResult().getSummary(1);
- assertEquals("doc1", s.getDocId());
- b = s.getSummary();
- assertEquals(8, b.length);
- byte[] f = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(f[i], b[i]);
- }
- } catch (IOException e) {
- fail(e.toString());
- }
- }
- }
-
-
- public class testGetDocumentMessage implements RunnableTest {
-
- @Override
- @SuppressWarnings("deprecation")
- public void run() {
- GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"));
- assertEquals(BASE_MESSAGE_LENGTH + 20, serialize("GetDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang);
- assertEquals("doc:scheme:", msg.getDocumentId().toString());
- }
- }
- }
-
-
- public class testRemoveDocumentMessage implements RunnableTest {
-
- @Override
- public void run() {
- RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
- assertEquals(BASE_MESSAGE_LENGTH + 16, serialize("RemoveDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang);
- assertEquals("doc:scheme:", msg.getDocumentId().toString());
- }
- }
- }
-
- public class testMapVisitorMessage implements RunnableTest {
-
- @Override
- public void run() {
- MapVisitorMessage msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.CPP);
- assertEquals("3", msg.getData().get("foo"));
- assertEquals("5", msg.getData().get("bar"));
-
- assertEquals(BASE_MESSAGE_LENGTH + 32, serialize("MapVisitorMessage", msg));
-
- msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.JAVA);
- assertEquals("3", msg.getData().get("foo"));
- assertEquals("5", msg.getData().get("bar"));
- }
- }
-
-
- public class testVisitorInfoMessage implements RunnableTest {
-
- @Override
- public void run() {
- VisitorInfoMessage msg = new VisitorInfoMessage();
- msg.getFinishedBuckets().add(new BucketId(16, 1));
- msg.getFinishedBuckets().add(new BucketId(16, 2));
- msg.getFinishedBuckets().add(new BucketId(16, 4));
- msg.setErrorMessage("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6");
- assertEquals(BASE_MESSAGE_LENGTH + 67, serialize("VisitorInfoMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (VisitorInfoMessage)deserialize("VisitorInfoMessage", DocumentProtocol.MESSAGE_VISITORINFO, lang);
- assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 1)));
- assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 2)));
- assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 4)));
- assertEquals("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6", msg.getErrorMessage());
- }
- }
- }
-
- public class testSearchResultMessage implements RunnableTest {
-
- @Override
- public void run() throws Exception {
- FileInputStream stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-1.dat"));
- byte[] data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- Routable routable = decode(data);
- assertTrue(routable instanceof SearchResultMessage);
-
- SearchResultMessage msg = (SearchResultMessage)routable;
- assertEquals(0, msg.getResult().getHitCount());
-
- stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-2.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof SearchResultMessage);
-
- msg = (SearchResultMessage)routable;
- assertEquals(2, msg.getResult().getHitCount());
- com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0);
- assertEquals(89.0, h.getRank(), 1E-6);
- assertEquals("doc1", h.getDocId());
- h = msg.getResult().getHit(1);
- assertEquals(109.0, h.getRank(), 1E-6);
- assertEquals("doc17", h.getDocId());
-
- stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-3.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof SearchResultMessage);
-
- msg = (SearchResultMessage)routable;
- assertEquals(2, msg.getResult().getHitCount());
- h = msg.getResult().getHit(0);
- assertEquals(109.0, h.getRank(), 1E-6);
- assertEquals("doc17", h.getDocId());
- h = msg.getResult().getHit(1);
- assertEquals(89.0, h.getRank(), 1E-6);
- assertEquals("doc1", h.getDocId());
-
- stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-4.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof SearchResultMessage);
-
- msg = (SearchResultMessage)routable;
- assertEquals(3, msg.getResult().getHitCount());
- h = msg.getResult().getHit(0);
- assertTrue(h instanceof SearchResult.HitWithSortBlob);
- assertEquals(89.0, h.getRank(), 1E-6);
- assertEquals("doc1", h.getDocId());
- byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
- assertEquals(9, b.length);
- byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(e[i], b[i]);
- }
- h = msg.getResult().getHit(1);
- assertTrue(h instanceof SearchResult.HitWithSortBlob);
- assertEquals(109.0, h.getRank(), 1E-6);
- assertEquals("doc17", h.getDocId());
- b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
- assertEquals(9, b.length);
- byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(d[i], b[i]);
- }
- h = msg.getResult().getHit(2);
- assertTrue(h instanceof SearchResult.HitWithSortBlob);
- assertEquals(90.0, h.getRank(), 1E-6);
- assertEquals("doc18", h.getDocId());
- b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
- assertEquals(9, b.length);
- byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(c[i], b[i]);
- }
- }
- }
-
- public class testPutDocumentMessage implements RunnableTest {
-
- @Override
- public void run() {
- PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")));
- msg.setTimestamp(666);
- assertEquals(BASE_MESSAGE_LENGTH + 41, serialize("PutDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);
- assertEquals("testdoc", msg.getDocumentPut().getDocument().getDataType().getName());
- assertEquals("doc:scheme:", msg.getDocumentPut().getDocument().getId().toString());
- assertEquals(666, msg.getTimestamp());
- }
- }
- }
-
- public class testPutDocumentReply implements RunnableTest {
-
- @Override
- public void run() {
- WriteDocumentReply reply = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT);
- reply.setHighestModificationTimestamp(30);
-
- assertEquals(13, serialize("PutDocumentReply", reply));
-
- for (Language lang : LANGUAGES) {
- WriteDocumentReply obj = (WriteDocumentReply)deserialize("PutDocumentReply", DocumentProtocol.REPLY_PUTDOCUMENT, lang);
- assertNotNull(obj);
- assertEquals(30, obj.getHighestModificationTimestamp());
- }
- }
- }
-
- public class testUpdateDocumentMessage implements RunnableTest {
-
- @Override
- public void run() {
- DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");
- DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:"));
- update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
- UpdateDocumentMessage msg = new UpdateDocumentMessage(update);
- msg.setNewTimestamp(777);
- msg.setOldTimestamp(666);
-
- assertEquals(BASE_MESSAGE_LENGTH + 89, serialize("UpdateDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (UpdateDocumentMessage)deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang);
- assertEquals(update, msg.getDocumentUpdate());
- assertEquals(777, msg.getNewTimestamp());
- assertEquals(666, msg.getOldTimestamp());
- }
- }
- }
-
- public class testUpdateDocumentReply implements RunnableTest {
-
- @Override
- public void run() {
- UpdateDocumentReply reply = new UpdateDocumentReply();
- reply.setHighestModificationTimestamp(30);
- reply.setWasFound(false);
-
- assertEquals(14, serialize("UpdateDocumentReply", reply));
-
- for (Language lang : LANGUAGES) {
- UpdateDocumentReply obj = (UpdateDocumentReply)deserialize("UpdateDocumentReply", DocumentProtocol.REPLY_UPDATEDOCUMENT, lang);
- assertNotNull(obj);
- assertEquals(30, reply.getHighestModificationTimestamp());
- assertEquals(false, obj.wasFound());
- }
- }
- }
-
- public class testVisitorInfoReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("VisitorInfoReply", DocumentProtocol.REPLY_VISITORINFO);
- }
- }
-
- public class testWrongDistributionReply implements RunnableTest {
-
- @Override
- public void run() {
- WrongDistributionReply reply = new WrongDistributionReply("distributor:3 storage:2");
- assertEquals(32, serialize("WrongDistributionReply", reply));
-
- for (Language lang : LANGUAGES) {
- reply = (WrongDistributionReply)deserialize("WrongDistributionReply", DocumentProtocol.REPLY_WRONGDISTRIBUTION, lang);
- assertEquals("distributor:3 storage:2", reply.getSystemState());
- }
- }
- }
-
- public class testRemoveDocumentReply implements RunnableTest {
-
- @Override
- public void run() {
- RemoveDocumentReply reply = new RemoveDocumentReply();
- reply.setHighestModificationTimestamp(30);
- reply.setWasFound(false);
-
- assertEquals(14, serialize("RemoveDocumentReply", reply));
-
- for (Language lang : LANGUAGES) {
- RemoveDocumentReply obj = (RemoveDocumentReply)deserialize("RemoveDocumentReply", DocumentProtocol.REPLY_REMOVEDOCUMENT, lang);
- assertNotNull(obj);
- assertEquals(30, obj.getHighestModificationTimestamp());
- assertEquals(false, obj.wasFound());
- }
- }
- }
-
- public class testRemoveLocationReply implements RunnableTest {
-
- @Override
- public void run() {
- testDocumentReply("RemoveLocationReply", DocumentProtocol.REPLY_REMOVELOCATION);
- }
- }
-
- public class testSearchResultReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT);
- }
- }
-
- public class testStatBucketReply implements RunnableTest {
-
- @Override
- public void run() {
- StatBucketReply msg = new StatBucketReply();
- msg.setResults("These are the votes of the Norwegian jury");
-
- assertEquals(50, serialize("StatBucketReply", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (StatBucketReply)deserialize("StatBucketReply", DocumentProtocol.REPLY_STATBUCKET, lang);
- assertEquals("These are the votes of the Norwegian jury", msg.getResults());
- }
- }
- }
-
- public class testQueryResultReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("QueryResultReply", DocumentProtocol.REPLY_QUERYRESULT);
- }
- }
-
- public class testQueryResultMessage implements RunnableTest {
-
- @Override
- public void run() throws Exception {
- FileInputStream stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-1.dat"));
- byte[] data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- Routable routable = decode(data);
- assertTrue(routable instanceof QueryResultMessage);
-
- QueryResultMessage msg = (QueryResultMessage)routable;
- assertEquals(0, msg.getResult().getHitCount());
-
- stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-2.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof QueryResultMessage);
-
- msg = (QueryResultMessage)routable;
- assertEquals(2, msg.getResult().getHitCount());
- com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0);
- assertEquals(89.0, h.getRank(), 1E-6);
- assertEquals("doc1", h.getDocId());
- h = msg.getResult().getHit(1);
- assertEquals(109.0, h.getRank(), 1E-6);
- assertEquals("doc17", h.getDocId());
-
- stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-3.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof QueryResultMessage);
-
- msg = (QueryResultMessage)routable;
- assertEquals(2, msg.getResult().getHitCount());
- h = msg.getResult().getHit(0);
- assertEquals(109.0, h.getRank(), 1E-6);
- assertEquals("doc17", h.getDocId());
- h = msg.getResult().getHit(1);
- assertEquals(89.0, h.getRank(), 1E-6);
- assertEquals("doc1", h.getDocId());
-
- stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-4.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof QueryResultMessage);
-
- msg = (QueryResultMessage)routable;
- assertEquals(3, msg.getResult().getHitCount());
- h = msg.getResult().getHit(0);
- assertTrue(h instanceof SearchResult.HitWithSortBlob);
- assertEquals(89.0, h.getRank(), 1E-6);
- assertEquals("doc1", h.getDocId());
- byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
- assertEquals(9, b.length);
- byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(e[i], b[i]);
- }
- h = msg.getResult().getHit(1);
- assertTrue(h instanceof SearchResult.HitWithSortBlob);
- assertEquals(109.0, h.getRank(), 1E-6);
- assertEquals("doc17", h.getDocId());
- b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
- assertEquals(9, b.length);
- byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(d[i], b[i]);
- }
- h = msg.getResult().getHit(2);
- assertTrue(h instanceof SearchResult.HitWithSortBlob);
- assertEquals(90.0, h.getRank(), 1E-6);
- assertEquals("doc18", h.getDocId());
- b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
- assertEquals(9, b.length);
- byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(c[i], b[i]);
- }
- }
- }
-
- public class testGetBucketListReply implements RunnableTest {
-
- public void run() {
- GetBucketListReply reply = new GetBucketListReply();
- reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo"));
- reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar"));
- reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink"));
-
- assertEquals(56, serialize("GetBucketListReply", reply));
-
- for (Language lang : LANGUAGES) {
- reply = (GetBucketListReply)deserialize("GetBucketListReply", DocumentProtocol.REPLY_GETBUCKETLIST, lang);
- assertEquals(reply.getBuckets().get(0), new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo"));
- assertEquals(reply.getBuckets().get(1), new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar"));
- assertEquals(reply.getBuckets().get(2), new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink"));
- }
- }
- }
-
- public class testGetBucketStateReply implements RunnableTest {
-
- public void run() {
- GlobalId foo = new GlobalId(IdString.createIdString("doc:scheme:foo"));
- GlobalId bar = new GlobalId(IdString.createIdString("doc:scheme:bar"));
-
- GetBucketStateReply reply = new GetBucketStateReply();
- List<DocumentState> state = new ArrayList<>(2);
- state.add(new DocumentState(foo, 777, false));
- state.add(new DocumentState(bar, 888, true));
- reply.setBucketState(state);
- assertEquals(53, serialize("GetBucketStateReply", reply));
-
- for (Language lang : LANGUAGES) {
- reply = (GetBucketStateReply)deserialize("GetBucketStateReply", DocumentProtocol.REPLY_GETBUCKETSTATE, lang);
- assertEquals(777, reply.getBucketState().get(0).getTimestamp());
- assertEquals(foo, reply.getBucketState().get(0).getGid());
- assertEquals(false, reply.getBucketState().get(0).isRemoveEntry());
- assertEquals(888, reply.getBucketState().get(1).getTimestamp());
- assertEquals(bar, reply.getBucketState().get(1).getGid());
- assertEquals(true, reply.getBucketState().get(1).isRemoveEntry());
- }
- }
- }
-
- public class testGetDocumentReply implements RunnableTest {
-
- public void run() {
- GetDocumentReply reply = new GetDocumentReply(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:"));
- assertEquals(43, serialize("GetDocumentReply", reply));
-
- for (Language lang : LANGUAGES) {
- reply = (GetDocumentReply)deserialize("GetDocumentReply", DocumentProtocol.REPLY_GETDOCUMENT, lang);
- assertEquals("testdoc", reply.getDocument().getDataType().getName());
- assertEquals("doc:scheme:", reply.getDocument().getId().toString());
- }
- }
- }
-
- public class testMapVisitorReply implements RunnableTest {
-
- public void run() {
- testVisitorReply("MapVisitorReply", DocumentProtocol.REPLY_MAPVISITOR);
- }
- }
-
- protected void testDocumentReply(String filename, int type) {
- DocumentReply reply = new DocumentReply(type);
- assertEquals(5, serialize(filename, reply));
-
- for (Language lang : LANGUAGES) {
- reply = (DocumentReply)deserialize(filename, type, lang);
- assertNotNull(reply);
- }
- }
-
- protected void testVisitorReply(String filename, int type) {
- VisitorReply reply = new VisitorReply(type);
- assertEquals(5, serialize(filename, reply));
-
- for (Language lang : LANGUAGES) {
- reply = (VisitorReply)deserialize(filename, type, lang);
- assertNotNull(reply);
- }
- }
-
-}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java
deleted file mode 100644
index 862244236f8..00000000000
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java
+++ /dev/null
@@ -1,121 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol.test;
-
-import com.yahoo.component.Version;
-import com.yahoo.document.BucketId;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.select.OrderingSpecification;
-import com.yahoo.documentapi.messagebus.protocol.CreateVisitorMessage;
-import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
-import com.yahoo.text.Utf8;
-
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author Simon Thoresen Hult
- */
-public class Messages51TestCase extends Messages50TestCase {
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Setup
- //
- ///////////////////////////////////////////////////////////////////////////////
-
- @Override
- protected void registerTests(Map<Integer, RunnableTest> out) {
- super.registerTests(out);
-
- // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
- // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
- out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage());
- out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage());
- out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply());
- }
-
- @Override
- protected Version version() {
- return new Version(5, 1);
- }
-
- @Override
- protected boolean shouldTestCoverage() {
- return true;
- }
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Tests
- //
- ////////////////////////////////////////////////////////////////////////////////
-
- protected static int BASE_MESSAGE_LENGTH = 5;
-
- public class testCreateVisitorMessage implements RunnableTest {
-
- @Override
- public void run() {
- CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london");
- msg.setDocumentSelection("true and false or true");
- msg.getParameters().put("myvar", Utf8.toBytes("somevalue"));
- msg.getParameters().put("anothervar", Utf8.toBytes("34"));
- msg.getBuckets().add(new BucketId(16, 1234));
- msg.setVisitRemoves(true);
- msg.setFieldSet("foo bar");
- msg.setVisitorOrdering(OrderingSpecification.DESCENDING);
- msg.setMaxBucketsPerVisitor(2);
- assertEquals(BASE_MESSAGE_LENGTH + 178, serialize("CreateVisitorMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang);
- assertEquals("SomeLibrary", msg.getLibraryName());
- assertEquals("myvisitor", msg.getInstanceId());
- assertEquals("newyork", msg.getControlDestination());
- assertEquals("london", msg.getDataDestination());
- assertEquals("true and false or true", msg.getDocumentSelection());
- assertEquals(8, msg.getMaxPendingReplyCount());
- assertEquals(true, msg.getVisitRemoves());
- assertEquals("foo bar", msg.getFieldSet());
- assertEquals(false, msg.getVisitInconsistentBuckets());
- assertEquals(1, msg.getBuckets().size());
- assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next());
- assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar")));
- assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar")));
- assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering());
- assertEquals(2, msg.getMaxBucketsPerVisitor());
- }
- }
- }
-
- public class testGetDocumentMessage implements RunnableTest {
-
- @Override
- public void run() {
- GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo bar");
- assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("GetDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang);
- assertEquals("doc:scheme:", msg.getDocumentId().toString());
- assertEquals("foo bar", msg.getFieldSet());
- }
- }
- }
-
- public class testDocumentIgnoredReply implements RunnableTest {
-
- @Override
- public void run() {
- DocumentIgnoredReply reply = new DocumentIgnoredReply();
- assertEquals(BASE_MESSAGE_LENGTH, serialize("DocumentIgnoredReply", reply));
-
- for (Language lang : LANGUAGES) {
- reply = (DocumentIgnoredReply)deserialize("DocumentIgnoredReply", DocumentProtocol.REPLY_DOCUMENTIGNORED, lang);
- }
- }
- }
-}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java
index 1bda3ea3610..448c8fcb520 100644
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java
@@ -2,19 +2,108 @@
package com.yahoo.documentapi.messagebus.protocol.test;
import com.yahoo.component.Version;
-import com.yahoo.document.*;
+import com.yahoo.document.BucketId;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.GlobalId;
+import com.yahoo.document.TestAndSetCondition;
import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
-import com.yahoo.documentapi.messagebus.protocol.*;
+import com.yahoo.document.idstring.IdString;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.documentapi.messagebus.protocol.CreateVisitorMessage;
+import com.yahoo.documentapi.messagebus.protocol.CreateVisitorReply;
+import com.yahoo.documentapi.messagebus.protocol.DestroyVisitorMessage;
+import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply;
+import com.yahoo.documentapi.messagebus.protocol.DocumentListMessage;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.documentapi.messagebus.protocol.DocumentReply;
+import com.yahoo.documentapi.messagebus.protocol.DocumentState;
+import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage;
+import com.yahoo.documentapi.messagebus.protocol.EmptyBucketsMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketListMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketStateMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetBucketStateReply;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply;
+import com.yahoo.documentapi.messagebus.protocol.MapVisitorMessage;
+import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage;
+import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentReply;
+import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage;
+import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage;
+import com.yahoo.documentapi.messagebus.protocol.StatBucketMessage;
+import com.yahoo.documentapi.messagebus.protocol.StatBucketReply;
+import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentReply;
+import com.yahoo.documentapi.messagebus.protocol.VisitorInfoMessage;
+import com.yahoo.documentapi.messagebus.protocol.VisitorReply;
+import com.yahoo.documentapi.messagebus.protocol.WriteDocumentReply;
+import com.yahoo.documentapi.messagebus.protocol.WrongDistributionReply;
+import com.yahoo.messagebus.Routable;
+import com.yahoo.text.Utf8;
+import com.yahoo.vdslib.SearchResult;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
/**
+ * @author Simon Thoresen Hult
* @author Vegard Sjonfjell
*/
-public class Messages52TestCase extends Messages51TestCase {
+public class Messages52TestCase extends MessagesTestBase {
+ @Override
+ protected void registerTests(Map<Integer, RunnableTest> out) {
+ // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
+ // version 5.2. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
+ out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage());
+ out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage());
+ out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage());
+ out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage());
+ out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage());
+ out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage());
+ out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage());
+ out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage());
+ out.put(DocumentProtocol.MESSAGE_MAPVISITOR, new testMapVisitorMessage());
+ out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage());
+ out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage());
+ out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage());
+ out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage());
+ out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage());
+ out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage());
+ out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage());
+ out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage());
+ out.put(DocumentProtocol.REPLY_CREATEVISITOR, new testCreateVisitorReply());
+ out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply());
+ out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply());
+ out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply());
+ out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply());
+ out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply());
+ out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply());
+ out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply());
+ out.put(DocumentProtocol.REPLY_GETDOCUMENT, new testGetDocumentReply());
+ out.put(DocumentProtocol.REPLY_MAPVISITOR, new testMapVisitorReply());
+ out.put(DocumentProtocol.REPLY_PUTDOCUMENT, new testPutDocumentReply());
+ out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply());
+ out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply());
+ out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply());
+ out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply());
+ out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply());
+ out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply());
+ out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply());
+ out.put(DocumentProtocol.REPLY_WRONGDISTRIBUTION, new testWrongDistributionReply());
+ }
@Override
protected Version version() {
@@ -26,21 +115,488 @@ public class Messages52TestCase extends Messages51TestCase {
return true;
}
- @Override
- protected void registerTests(Map<Integer, RunnableTest> out) {
- super.registerTests(out);
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Tests
+ //
+ ////////////////////////////////////////////////////////////////////////////////
- // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
- // version 5.2. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
+ protected static int BASE_MESSAGE_LENGTH = 5;
- out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage());
- out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage());
- out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage());
+ public class testRemoveLocationMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ {
+ RemoveLocationMessage msg = new RemoveLocationMessage("id.group == \"mygroup\"");
+ assertEquals(BASE_MESSAGE_LENGTH + 29, serialize("RemoveLocationMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (RemoveLocationMessage)deserialize("RemoveLocationMessage", DocumentProtocol.MESSAGE_REMOVELOCATION, lang);
+ assertEquals("id.group == \"mygroup\"", msg.getDocumentSelection());
+ }
+ }
+ }
+ }
+
+ public class testGetBucketListMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123));
+ msg.setLoadType(loadTypes.getNameMap().get("foo"));
+ assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketListMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang);
+ assertEquals(new BucketId(16, 123), msg.getBucketId());
+ assertEquals("foo", msg.getLoadType().getName());
+ }
+ }
+ }
+
+
+ public class testStatBucketMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123");
+ msg.setLoadType(null);
+ assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("StatBucketMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang);
+ assertEquals(new BucketId(16, 123), msg.getBucketId());
+ assertEquals("id.user=123", msg.getDocumentSelection());
+ assertEquals("default", msg.getLoadType().getName());
+ }
+ }
+ }
+
+ public class testGetBucketStateMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ GetBucketStateMessage msg = new GetBucketStateMessage(new BucketId(16, 666));
+ assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketStateMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (GetBucketStateMessage)deserialize("GetBucketStateMessage", DocumentProtocol.MESSAGE_GETBUCKETSTATE, lang);
+ assertEquals(16, msg.getBucketId().getUsedBits());
+ assertEquals(4611686018427388570l, msg.getBucketId().getId());
+ }
+ }
+ }
+
+ public class testCreateVisitorMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london");
+ msg.setDocumentSelection("true and false or true");
+ msg.getParameters().put("myvar", Utf8.toBytes("somevalue"));
+ msg.getParameters().put("anothervar", Utf8.toBytes("34"));
+ msg.getBuckets().add(new BucketId(16, 1234));
+ msg.setVisitRemoves(true);
+ msg.setFieldSet("foo bar");
+ msg.setVisitorOrdering(OrderingSpecification.DESCENDING);
+ msg.setMaxBucketsPerVisitor(2);
+ assertEquals(BASE_MESSAGE_LENGTH + 178, serialize("CreateVisitorMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang);
+ assertEquals("SomeLibrary", msg.getLibraryName());
+ assertEquals("myvisitor", msg.getInstanceId());
+ assertEquals("newyork", msg.getControlDestination());
+ assertEquals("london", msg.getDataDestination());
+ assertEquals("true and false or true", msg.getDocumentSelection());
+ assertEquals(8, msg.getMaxPendingReplyCount());
+ assertEquals(true, msg.getVisitRemoves());
+ assertEquals("foo bar", msg.getFieldSet());
+ assertEquals(false, msg.getVisitInconsistentBuckets());
+ assertEquals(1, msg.getBuckets().size());
+ assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next());
+ assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar")));
+ assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar")));
+ assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering());
+ assertEquals(2, msg.getMaxBucketsPerVisitor());
+ }
+
+ msg.getBuckets().clear();
+
+ assertEquals("CreateVisitorMessage(" +
+ "No buckets, " +
+ "selection 'true and false or true', " +
+ "bucket space 'default', " +
+ "library SomeLibrary, including removes, " +
+ "get fields: foo bar" +
+ ")",
+ msg.toString());
+
+ msg.getBuckets().add(new BucketId(16, 1234));
+
+ assertEquals("CreateVisitorMessage(" +
+ "Bucket BucketId(0x40000000000004d2), " +
+ "selection 'true and false or true', " +
+ "bucket space 'default', " +
+ "library SomeLibrary, including removes, " +
+ "get fields: foo bar" +
+ ")",
+ msg.toString());
+
+ msg.getBuckets().add(new BucketId(16, 1235));
+ msg.getBuckets().add(new BucketId(16, 1236));
+ msg.getBuckets().add(new BucketId(16, 1237));
+ msg.getBuckets().add(new BucketId(16, 1238));
+ msg.setFromTimestamp(10001);
+ msg.setToTimestamp(20002);
+ msg.setVisitInconsistentBuckets(true);
+ assertEquals("CreateVisitorMessage(" +
+ "5 buckets: BucketId(0x40000000000004d2) BucketId(0x40000000000004d3) BucketId(0x40000000000004d4) ..., " +
+ "time 10001-20002, " +
+ "selection 'true and false or true', " +
+ "bucket space 'default', " +
+ "library SomeLibrary, including removes, " +
+ "get fields: foo bar, " +
+ "visit inconsistent buckets" +
+ ")",
+ msg.toString());
+ }
+ }
+
+ public class testCreateVisitorReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR);
+ reply.setLastBucket(new BucketId(16, 123));
+ reply.getVisitorStatistics().setBucketsVisited(3);
+ reply.getVisitorStatistics().setDocumentsVisited(1000);
+ reply.getVisitorStatistics().setBytesVisited(1024000);
+ reply.getVisitorStatistics().setDocumentsReturned(123);
+ reply.getVisitorStatistics().setBytesReturned(512000);
+ reply.getVisitorStatistics().setSecondPassDocumentsReturned(456);
+ reply.getVisitorStatistics().setSecondPassBytesReturned(789100);
+
+ assertEquals(65, serialize("CreateVisitorReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (CreateVisitorReply)deserialize("CreateVisitorReply", DocumentProtocol.REPLY_CREATEVISITOR, lang);
+ assertNotNull(reply);
+ assertEquals(new BucketId(16, 123), reply.getLastBucket());
+ assertEquals(3, reply.getVisitorStatistics().getBucketsVisited());
+ assertEquals(1000, reply.getVisitorStatistics().getDocumentsVisited());
+ assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited());
+ assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned());
+ assertEquals(512000, reply.getVisitorStatistics().getBytesReturned());
+ assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned());
+ assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned());
+ }
+ }
+ }
+
+ public class testDestroyVisitorReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("DestroyVisitorReply", DocumentProtocol.REPLY_DESTROYVISITOR);
+ }
+ }
+
+ public class testDocumentIgnoredReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ DocumentIgnoredReply reply = new DocumentIgnoredReply();
+ assertEquals(BASE_MESSAGE_LENGTH, serialize("DocumentIgnoredReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (DocumentIgnoredReply)deserialize("DocumentIgnoredReply", DocumentProtocol.REPLY_DOCUMENTIGNORED, lang);
+ }
+ }
+ }
+
+ public class testDocumentListReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("DocumentListReply", DocumentProtocol.REPLY_DOCUMENTLIST);
+ }
+ }
+
+ public class testDocumentSummaryReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY);
+ }
+ }
+
+ public class testEmptyBucketsReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("EmptyBucketsReply", DocumentProtocol.REPLY_EMPTYBUCKETS);
+ }
+ }
+
+ public class testDestroyVisitorMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ DestroyVisitorMessage msg = new DestroyVisitorMessage("myvisitor");
+ assertEquals(BASE_MESSAGE_LENGTH + 17, serialize("DestroyVisitorMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (DestroyVisitorMessage)deserialize("DestroyVisitorMessage", DocumentProtocol.MESSAGE_DESTROYVISITOR, lang);
+ assertEquals("myvisitor", msg.getInstanceId());
+ }
+ }
+ }
+
+ public class testDocumentListMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ DocumentListMessage msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.CPP);
+ assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString());
+ assertEquals(1234, msg.getDocuments().get(0).getTimestamp());
+ assertFalse(msg.getDocuments().get(0).isRemoveEntry());
+
+ assertEquals(BASE_MESSAGE_LENGTH + 63, serialize("DocumentListMessage", msg));
+ msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.JAVA);
+ assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString());
+ assertEquals(1234, msg.getDocuments().get(0).getTimestamp());
+ assertFalse(msg.getDocuments().get(0).isRemoveEntry());
+
+ }
+ }
+
+ public class testEmptyBucketsMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ List<BucketId> bids = new ArrayList<>();
+ for (int i = 0; i < 13; ++i) {
+ bids.add(new BucketId(16, i));
+ }
+
+ EmptyBucketsMessage ebm = new EmptyBucketsMessage(bids);
+ assertEquals(BASE_MESSAGE_LENGTH + 112, serialize("EmptyBucketsMessage", ebm));
+ for (Language lang : LANGUAGES) {
+ ebm = (EmptyBucketsMessage)deserialize("EmptyBucketsMessage", DocumentProtocol.MESSAGE_EMPTYBUCKETS, lang);
+ for (int i = 0; i < 13; ++i) {
+ assertEquals(new BucketId(16, i), ebm.getBucketIds().get(i));
+ }
+ }
+ }
+ }
+
+ public class testDocumentSummaryMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ Routable routable = deserialize("DocumentSummaryMessage-1", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP);
+ assertTrue(routable instanceof DocumentSummaryMessage);
+
+ DocumentSummaryMessage msg = (DocumentSummaryMessage) routable;
+ assertEquals(0, msg.getResult().getSummaryCount());
+
+ routable = deserialize("DocumentSummaryMessage-2", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP);
+ assertTrue(routable instanceof DocumentSummaryMessage);
+
+ msg = (DocumentSummaryMessage) routable;
+ assertEquals(2, msg.getResult().getSummaryCount());
+ com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0);
+ assertEquals("doc1", s.getDocId());
+ byte[] b = s.getSummary();
+ assertEquals(8, b.length);
+ byte[] c = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '1'};
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(c[i], b[i]);
+ }
+
+ s = msg.getResult().getSummary(1);
+ assertEquals("aoc17", s.getDocId());
+ b = s.getSummary();
+ assertEquals(9, b.length);
+ byte[] d = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5'};
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(d[i], b[i]);
+ }
+ routable = deserialize("DocumentSummaryMessage-3", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP);
+ assertTrue(routable instanceof DocumentSummaryMessage);
+
+ msg = (DocumentSummaryMessage) routable;
+ assertEquals(2, msg.getResult().getSummaryCount());
+
+ s = msg.getResult().getSummary(0);
+ assertEquals("aoc17", s.getDocId());
+ b = s.getSummary();
+ assertEquals(9, b.length);
+ byte[] e = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5'};
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(e[i], b[i]);
+ }
+
+ s = msg.getResult().getSummary(1);
+ assertEquals("doc1", s.getDocId());
+ b = s.getSummary();
+ assertEquals(8, b.length);
+ byte[] f = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '1'};
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(f[i], b[i]);
+ }
+ }
+ }
+
+
+ public class testGetDocumentMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo bar");
+ assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("GetDocumentMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang);
+ assertEquals("doc:scheme:", msg.getDocumentId().toString());
+ assertEquals("foo bar", msg.getFieldSet());
+ }
+ }
+ }
+
+
+ public class testRemoveDocumentMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ final RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+ msg.setCondition(new TestAndSetCondition(CONDITION_STRING));
+
+ assertEquals(BASE_MESSAGE_LENGTH + 16 + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ final RemoveDocumentMessage deserializedMsg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang);
+ assertEquals(deserializedMsg.getDocumentId().toString(), msg.getDocumentId().toString());
+ }
+ }
+ }
+
+ public class testMapVisitorMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ MapVisitorMessage msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.CPP);
+ assertEquals("3", msg.getData().get("foo"));
+ assertEquals("5", msg.getData().get("bar"));
+
+ assertEquals(BASE_MESSAGE_LENGTH + 32, serialize("MapVisitorMessage", msg));
+
+ msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.JAVA);
+ assertEquals("3", msg.getData().get("foo"));
+ assertEquals("5", msg.getData().get("bar"));
+ }
+ }
+
+
+ public class testVisitorInfoMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ VisitorInfoMessage msg = new VisitorInfoMessage();
+ msg.getFinishedBuckets().add(new BucketId(16, 1));
+ msg.getFinishedBuckets().add(new BucketId(16, 2));
+ msg.getFinishedBuckets().add(new BucketId(16, 4));
+ msg.setErrorMessage("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6");
+ assertEquals(BASE_MESSAGE_LENGTH + 67, serialize("VisitorInfoMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (VisitorInfoMessage)deserialize("VisitorInfoMessage", DocumentProtocol.MESSAGE_VISITORINFO, lang);
+ assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 1)));
+ assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 2)));
+ assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 4)));
+ assertEquals("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6", msg.getErrorMessage());
+ }
+ }
+ }
+
+ public class testSearchResultMessage implements RunnableTest {
+
+ @Override
+ public void run() throws Exception {
+ Routable routable = deserialize("SearchResultMessage-1", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP);
+ assertTrue(routable instanceof SearchResultMessage);
+
+ SearchResultMessage msg = (SearchResultMessage)routable;
+ assertEquals(0, msg.getResult().getHitCount());
+
+ routable = deserialize("SearchResultMessage-2", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP);
+ assertTrue(routable instanceof SearchResultMessage);
+
+ msg = (SearchResultMessage)routable;
+ assertEquals(2, msg.getResult().getHitCount());
+ com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0);
+ assertEquals(89.0, h.getRank(), 1E-6);
+ assertEquals("doc1", h.getDocId());
+ h = msg.getResult().getHit(1);
+ assertEquals(109.0, h.getRank(), 1E-6);
+ assertEquals("doc17", h.getDocId());
+
+ routable = deserialize("SearchResultMessage-3", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP);
+ assertTrue(routable instanceof SearchResultMessage);
+
+ msg = (SearchResultMessage)routable;
+ assertEquals(2, msg.getResult().getHitCount());
+ h = msg.getResult().getHit(0);
+ assertEquals(109.0, h.getRank(), 1E-6);
+ assertEquals("doc17", h.getDocId());
+ h = msg.getResult().getHit(1);
+ assertEquals(89.0, h.getRank(), 1E-6);
+ assertEquals("doc1", h.getDocId());
+
+ routable = deserialize("SearchResultMessage-4", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP);
+ assertTrue(routable instanceof SearchResultMessage);
+
+ msg = (SearchResultMessage)routable;
+ assertEquals(3, msg.getResult().getHitCount());
+ h = msg.getResult().getHit(0);
+ assertTrue(h instanceof SearchResult.HitWithSortBlob);
+ assertEquals(89.0, h.getRank(), 1E-6);
+ assertEquals("doc1", h.getDocId());
+ byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+ assertEquals(9, b.length);
+ byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(e[i], b[i]);
+ }
+ h = msg.getResult().getHit(1);
+ assertTrue(h instanceof SearchResult.HitWithSortBlob);
+ assertEquals(109.0, h.getRank(), 1E-6);
+ assertEquals("doc17", h.getDocId());
+ b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+ assertEquals(9, b.length);
+ byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(d[i], b[i]);
+ }
+ h = msg.getResult().getHit(2);
+ assertTrue(h instanceof SearchResult.HitWithSortBlob);
+ assertEquals(90.0, h.getRank(), 1E-6);
+ assertEquals("doc18", h.getDocId());
+ b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+ assertEquals(9, b.length);
+ byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(c[i], b[i]);
+ }
+ }
}
private static String CONDITION_STRING = "There's just one condition";
public class testPutDocumentMessage implements RunnableTest {
+
@Override
public void run() {
PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")));
@@ -60,7 +616,25 @@ public class Messages52TestCase extends Messages51TestCase {
}
}
+ public class testPutDocumentReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ WriteDocumentReply reply = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT);
+ reply.setHighestModificationTimestamp(30);
+
+ assertEquals(13, serialize("PutDocumentReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ WriteDocumentReply obj = (WriteDocumentReply)deserialize("PutDocumentReply", DocumentProtocol.REPLY_PUTDOCUMENT, lang);
+ assertNotNull(obj);
+ assertEquals(30, obj.getHighestModificationTimestamp());
+ }
+ }
+ }
+
public class testUpdateDocumentMessage implements RunnableTest {
+
@Override
public void run() {
DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");
@@ -84,21 +658,263 @@ public class Messages52TestCase extends Messages51TestCase {
}
}
- public class testRemoveDocumentMessage implements RunnableTest {
+ public class testUpdateDocumentReply implements RunnableTest {
+
@Override
public void run() {
- final RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
- msg.setCondition(new TestAndSetCondition(CONDITION_STRING));
+ UpdateDocumentReply reply = new UpdateDocumentReply();
+ reply.setHighestModificationTimestamp(30);
+ reply.setWasFound(false);
- assertEquals(BASE_MESSAGE_LENGTH + 16 + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg));
+ assertEquals(14, serialize("UpdateDocumentReply", reply));
for (Language lang : LANGUAGES) {
- final RemoveDocumentMessage deserializedMsg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang);
- assertEquals(deserializedMsg.getDocumentId().toString(), msg.getDocumentId().toString());
+ UpdateDocumentReply obj = (UpdateDocumentReply)deserialize("UpdateDocumentReply", DocumentProtocol.REPLY_UPDATEDOCUMENT, lang);
+ assertNotNull(obj);
+ assertEquals(30, reply.getHighestModificationTimestamp());
+ assertEquals(false, obj.wasFound());
+ }
+ }
+ }
+
+ public class testVisitorInfoReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("VisitorInfoReply", DocumentProtocol.REPLY_VISITORINFO);
+ }
+ }
+
+ public class testWrongDistributionReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ WrongDistributionReply reply = new WrongDistributionReply("distributor:3 storage:2");
+ assertEquals(32, serialize("WrongDistributionReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (WrongDistributionReply)deserialize("WrongDistributionReply", DocumentProtocol.REPLY_WRONGDISTRIBUTION, lang);
+ assertEquals("distributor:3 storage:2", reply.getSystemState());
+ }
+ }
+ }
+
+ public class testRemoveDocumentReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ RemoveDocumentReply reply = new RemoveDocumentReply();
+ reply.setHighestModificationTimestamp(30);
+ reply.setWasFound(false);
+
+ assertEquals(14, serialize("RemoveDocumentReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ RemoveDocumentReply obj = (RemoveDocumentReply)deserialize("RemoveDocumentReply", DocumentProtocol.REPLY_REMOVEDOCUMENT, lang);
+ assertNotNull(obj);
+ assertEquals(30, obj.getHighestModificationTimestamp());
+ assertEquals(false, obj.wasFound());
+ }
+ }
+ }
+
+ public class testRemoveLocationReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testDocumentReply("RemoveLocationReply", DocumentProtocol.REPLY_REMOVELOCATION);
+ }
+ }
+
+ public class testSearchResultReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT);
+ }
+ }
+
+ public class testStatBucketReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ StatBucketReply msg = new StatBucketReply();
+ msg.setResults("These are the votes of the Norwegian jury");
+
+ assertEquals(50, serialize("StatBucketReply", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (StatBucketReply)deserialize("StatBucketReply", DocumentProtocol.REPLY_STATBUCKET, lang);
+ assertEquals("These are the votes of the Norwegian jury", msg.getResults());
+ }
+ }
+ }
+
+ public class testQueryResultReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("QueryResultReply", DocumentProtocol.REPLY_QUERYRESULT);
+ }
+ }
+
+ public class testQueryResultMessage implements RunnableTest {
+
+ @Override
+ public void run() throws Exception {
+ Routable routable = deserialize("QueryResultMessage-1", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP);
+ assertTrue(routable instanceof QueryResultMessage);
+
+ QueryResultMessage msg = (QueryResultMessage)routable;
+ assertEquals(0, msg.getResult().getHitCount());
+
+ routable = deserialize("QueryResultMessage-2", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP);
+ assertTrue(routable instanceof QueryResultMessage);
+
+ msg = (QueryResultMessage)routable;
+ assertEquals(2, msg.getResult().getHitCount());
+ com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0);
+ assertEquals(89.0, h.getRank(), 1E-6);
+ assertEquals("doc1", h.getDocId());
+ h = msg.getResult().getHit(1);
+ assertEquals(109.0, h.getRank(), 1E-6);
+ assertEquals("doc17", h.getDocId());
+
+ routable = deserialize("QueryResultMessage-3", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP);
+ assertTrue(routable instanceof QueryResultMessage);
+
+ msg = (QueryResultMessage)routable;
+ assertEquals(2, msg.getResult().getHitCount());
+ h = msg.getResult().getHit(0);
+ assertEquals(109.0, h.getRank(), 1E-6);
+ assertEquals("doc17", h.getDocId());
+ h = msg.getResult().getHit(1);
+ assertEquals(89.0, h.getRank(), 1E-6);
+ assertEquals("doc1", h.getDocId());
+
+ routable = deserialize("QueryResultMessage-4", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP);
+ assertTrue(routable instanceof QueryResultMessage);
+
+ msg = (QueryResultMessage)routable;
+ assertEquals(3, msg.getResult().getHitCount());
+ h = msg.getResult().getHit(0);
+ assertTrue(h instanceof SearchResult.HitWithSortBlob);
+ assertEquals(89.0, h.getRank(), 1E-6);
+ assertEquals("doc1", h.getDocId());
+ byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+ assertEquals(9, b.length);
+ byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(e[i], b[i]);
+ }
+ h = msg.getResult().getHit(1);
+ assertTrue(h instanceof SearchResult.HitWithSortBlob);
+ assertEquals(109.0, h.getRank(), 1E-6);
+ assertEquals("doc17", h.getDocId());
+ b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+ assertEquals(9, b.length);
+ byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(d[i], b[i]);
+ }
+ h = msg.getResult().getHit(2);
+ assertTrue(h instanceof SearchResult.HitWithSortBlob);
+ assertEquals(90.0, h.getRank(), 1E-6);
+ assertEquals("doc18", h.getDocId());
+ b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+ assertEquals(9, b.length);
+ byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(c[i], b[i]);
+ }
+ }
+ }
+
+ public class testGetBucketListReply implements RunnableTest {
+
+ public void run() {
+ GetBucketListReply reply = new GetBucketListReply();
+ reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo"));
+ reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar"));
+ reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink"));
+
+ assertEquals(56, serialize("GetBucketListReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (GetBucketListReply)deserialize("GetBucketListReply", DocumentProtocol.REPLY_GETBUCKETLIST, lang);
+ assertEquals(reply.getBuckets().get(0), new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo"));
+ assertEquals(reply.getBuckets().get(1), new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar"));
+ assertEquals(reply.getBuckets().get(2), new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink"));
+ }
+ }
+ }
+
+ public class testGetBucketStateReply implements RunnableTest {
+
+ public void run() {
+ GlobalId foo = new GlobalId(IdString.createIdString("doc:scheme:foo"));
+ GlobalId bar = new GlobalId(IdString.createIdString("doc:scheme:bar"));
+
+ GetBucketStateReply reply = new GetBucketStateReply();
+ List<DocumentState> state = new ArrayList<>(2);
+ state.add(new DocumentState(foo, 777, false));
+ state.add(new DocumentState(bar, 888, true));
+ reply.setBucketState(state);
+ assertEquals(53, serialize("GetBucketStateReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (GetBucketStateReply)deserialize("GetBucketStateReply", DocumentProtocol.REPLY_GETBUCKETSTATE, lang);
+ assertEquals(777, reply.getBucketState().get(0).getTimestamp());
+ assertEquals(foo, reply.getBucketState().get(0).getGid());
+ assertEquals(false, reply.getBucketState().get(0).isRemoveEntry());
+ assertEquals(888, reply.getBucketState().get(1).getTimestamp());
+ assertEquals(bar, reply.getBucketState().get(1).getGid());
+ assertEquals(true, reply.getBucketState().get(1).isRemoveEntry());
}
}
}
+ public class testGetDocumentReply implements RunnableTest {
+
+ public void run() {
+ GetDocumentReply reply = new GetDocumentReply(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:"));
+ assertEquals(43, serialize("GetDocumentReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (GetDocumentReply)deserialize("GetDocumentReply", DocumentProtocol.REPLY_GETDOCUMENT, lang);
+ assertEquals("testdoc", reply.getDocument().getDataType().getName());
+ assertEquals("doc:scheme:", reply.getDocument().getId().toString());
+ }
+ }
+ }
+
+ public class testMapVisitorReply implements RunnableTest {
+
+ public void run() {
+ testVisitorReply("MapVisitorReply", DocumentProtocol.REPLY_MAPVISITOR);
+ }
+ }
+
+ protected void testDocumentReply(String filename, int type) {
+ DocumentReply reply = new DocumentReply(type);
+ assertEquals(5, serialize(filename, reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (DocumentReply)deserialize(filename, type, lang);
+ assertNotNull(reply);
+ }
+ }
+
+ protected void testVisitorReply(String filename, int type) {
+ VisitorReply reply = new VisitorReply(type);
+ assertEquals(5, serialize(filename, reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (VisitorReply)deserialize(filename, type, lang);
+ assertNotNull(reply);
+ }
+ }
+
static int serializedLength(String str) {
return 4 + str.length();
}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java
index 89698e44705..59cbf5b65ae 100755
--- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java
@@ -8,7 +8,7 @@ import com.yahoo.document.serialization.DocumentSerializer;
import com.yahoo.documentapi.messagebus.protocol.DocumentMessage;
import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
import com.yahoo.documentapi.messagebus.protocol.DocumentReply;
-import com.yahoo.documentapi.messagebus.protocol.RoutableFactories50;
+import com.yahoo.documentapi.messagebus.protocol.RoutableFactories52;
import com.yahoo.jrt.ListenFailedException;
import com.yahoo.jrt.slobrok.server.Slobrok;
import com.yahoo.messagebus.DestinationSession;
@@ -130,7 +130,7 @@ public class RoutableFactoryTestCase {
assertFalse(reply.hasErrors());
}
- private static class MyMessageFactory extends RoutableFactories50.DocumentMessageFactory {
+ private static class MyMessageFactory extends RoutableFactories52.DocumentMessageFactory {
@Override
protected DocumentMessage doDecode(DocumentDeserializer buf) {
@@ -143,7 +143,7 @@ public class RoutableFactoryTestCase {
}
}
- private static class MyReplyFactory extends RoutableFactories50.DocumentReplyFactory {
+ private static class MyReplyFactory extends RoutableFactories52.DocumentReplyFactory {
@Override
protected DocumentReply doDecode(DocumentDeserializer buf) {
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java
index ed862b75828..6160c9deca6 100644
--- a/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java
+++ b/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java
@@ -33,7 +33,7 @@ public abstract class AbstractDocumentApiTestCase {
@Test
public void requireThatSyncSessionWorks() {
- SyncSession session = access().createSyncSession(new SyncParameters());
+ SyncSession session = access().createSyncSession(new SyncParameters.Builder().build());
DocumentType type = access().getDocumentTypeManager().getDocumentType("music");
Document doc1 = new Document(type, new DocumentId("doc:music:1"));
diff --git a/documentapi/src/tests/messagebus/messagebus_test.cpp b/documentapi/src/tests/messagebus/messagebus_test.cpp
index dff3f3f1bac..22a46951ecc 100644
--- a/documentapi/src/tests/messagebus/messagebus_test.cpp
+++ b/documentapi/src/tests/messagebus/messagebus_test.cpp
@@ -4,6 +4,7 @@
#include <vespa/document/datatype/documenttype.h>
#include <vespa/document/fieldvalue/document.h>
#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/update/documentupdate.h>
#include <vespa/documentapi/documentapi.h>
#include <vespa/documentapi/loadtypes/loadtypeset.h>
#include <vespa/vdslib/state/clusterstate.h>
@@ -69,10 +70,10 @@ void Test::testMessage() {
LoadTypeSet set;
DocumentProtocol protocol(set, _repo);
- Blob blob = protocol.encode(vespalib::Version(5,0), upd1);
+ Blob blob = protocol.encode(vespalib::Version(5,115), upd1);
EXPECT_TRUE(blob.size() > 0);
- Routable::UP dec1 = protocol.decode(vespalib::Version(5,0), blob);
+ Routable::UP dec1 = protocol.decode(vespalib::Version(5,115), blob);
EXPECT_TRUE(dec1.get() != NULL);
EXPECT_TRUE(dec1->isReply() == false);
EXPECT_TRUE(dec1->getType() == DocumentProtocol::MESSAGE_UPDATEDOCUMENT);
diff --git a/documentapi/src/tests/messages/CMakeLists.txt b/documentapi/src/tests/messages/CMakeLists.txt
index b762eaca657..19e1901003d 100644
--- a/documentapi/src/tests/messages/CMakeLists.txt
+++ b/documentapi/src/tests/messages/CMakeLists.txt
@@ -1,23 +1,4 @@
# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(documentapi_messages50_test_app TEST
- SOURCES
- testbase.cpp
- messages50test.cpp
- messages50app.cpp
- DEPENDS
- documentapi
-)
-vespa_add_test(NAME documentapi_messages50_test_app COMMAND documentapi_messages50_test_app)
-vespa_add_executable(documentapi_messages51_test_app TEST
- SOURCES
- testbase.cpp
- messages50test.cpp
- messages51test.cpp
- messages51app.cpp
- DEPENDS
- documentapi
-)
-vespa_add_test(NAME documentapi_messages51_test_app COMMAND documentapi_messages51_test_app)
vespa_add_executable(documentapi_messages52_test_app TEST
SOURCES
testbase.cpp
diff --git a/documentapi/src/tests/messages/messages50app.cpp b/documentapi/src/tests/messages/messages50app.cpp
deleted file mode 100644
index aa42302beba..00000000000
--- a/documentapi/src/tests/messages/messages50app.cpp
+++ /dev/null
@@ -1,5 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "messages50test.h"
-
-TEST_APPHOOK(Messages50Test);
diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp
index 6705277d5c3..cee0af5ad8e 100644
--- a/documentapi/src/tests/messages/messages50test.cpp
+++ b/documentapi/src/tests/messages/messages50test.cpp
@@ -5,6 +5,7 @@
#include <vespa/document/datatype/documenttype.h>
#include <vespa/document/select/parser.h>
#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/update/documentupdate.h>
#include <vespa/document/update/fieldpathupdates.h>
#include <vespa/documentapi/documentapi.h>
#include <vespa/document/bucket/fixed_bucket_spaces.h>
@@ -22,7 +23,6 @@ Messages50Test::Messages50Test()
{
// This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
// version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
- putTest(DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE, TEST_METHOD(Messages50Test::testBatchDocumentUpdateMessage));
putTest(DocumentProtocol::MESSAGE_CREATEVISITOR, TEST_METHOD(Messages50Test::testCreateVisitorMessage));
putTest(DocumentProtocol::MESSAGE_DESTROYVISITOR, TEST_METHOD(Messages50Test::testDestroyVisitorMessage));
putTest(DocumentProtocol::MESSAGE_DOCUMENTLIST, TEST_METHOD(Messages50Test::testDocumentListMessage));
@@ -41,7 +41,6 @@ Messages50Test::Messages50Test()
putTest(DocumentProtocol::MESSAGE_UPDATEDOCUMENT, TEST_METHOD(Messages50Test::testUpdateDocumentMessage));
putTest(DocumentProtocol::MESSAGE_VISITORINFO, TEST_METHOD(Messages50Test::testVisitorInfoMessage));
- putTest(DocumentProtocol::REPLY_BATCHDOCUMENTUPDATE, TEST_METHOD(Messages50Test::testBatchDocumentUpdateReply));
putTest(DocumentProtocol::REPLY_CREATEVISITOR, TEST_METHOD(Messages50Test::testCreateVisitorReply));
putTest(DocumentProtocol::REPLY_DESTROYVISITOR, TEST_METHOD(Messages50Test::testDestroyVisitorReply));
putTest(DocumentProtocol::REPLY_DOCUMENTLIST, TEST_METHOD(Messages50Test::testDocumentListReply));
@@ -259,38 +258,27 @@ Messages50Test::testDocumentSummaryMessage()
EXPECT_EQUAL(srm.hasSequenceId(), false);
EXPECT_EQUAL(srm.getSummaryCount(), size_t(0));
- mbus::Blob data = encode(srm);
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(12), serialize("DocumentSummaryMessage-1", srm));
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(12), data.size());
-
- writeFile(getPath("5-cpp-DocumentSummaryMessage-1.dat"), data);
- // print(data);
-
- mbus::Routable::UP routable = decode(data);
+ mbus::Routable::UP routable = deserialize("DocumentSummaryMessage-1", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_DOCUMENTSUMMARY);
DocumentSummaryMessage * dm = static_cast<DocumentSummaryMessage *>(routable.get());
EXPECT_EQUAL(dm->getSummaryCount(), size_t(0));
srm.addSummary("doc1", "summary1", 8);
srm.addSummary("aoc17", "summary45", 9);
- data = encode(srm);
- //print(data);
-
const void *summary(NULL);
const char *docId(NULL);
size_t sz(0);
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, data.size());
- writeFile(getPath("5-cpp-DocumentSummaryMessage-2.dat"), data);
- routable = decode(data);
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, serialize("DocumentSummaryMessage-2", srm));
+ routable = deserialize("DocumentSummaryMessage-2", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_DOCUMENTSUMMARY);
dm = static_cast<DocumentSummaryMessage *>(routable.get());
EXPECT_EQUAL(dm->getSummaryCount(), size_t(2));
dm->getSummary(0, docId, summary, sz);
@@ -304,14 +292,11 @@ Messages50Test::testDocumentSummaryMessage()
srm.sort();
- data = encode(srm);
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, data.size());
- writeFile(getPath("5-cpp-DocumentSummaryMessage-3.dat"), data);
- routable = decode(data);
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, serialize("DocumentSummaryMessage-3", srm));
+ routable = deserialize("DocumentSummaryMessage-3", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_DOCUMENTSUMMARY);
dm = static_cast<DocumentSummaryMessage *>(routable.get());
EXPECT_EQUAL(dm->getSummaryCount(), size_t(2));
dm->getSummary(0, docId, summary, sz);
@@ -524,38 +509,25 @@ Messages50Test::testSearchResultMessage()
EXPECT_EQUAL(srm.vdslib::SearchResult::getSerializedSize(), 20u);
EXPECT_EQUAL(srm.getSerializedSize(), 20u);
- mbus::Blob data = encode(srm);
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(24), serialize("SearchResultMessage-1", srm));
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(24), data.size());
-
- writeFile(getPath("5-cpp-SearchResultMessage-1.dat"), data);
- // print(data);
-
- mbus::Routable::UP routable = decode(data);
+ mbus::Routable::UP routable = deserialize("SearchResultMessage-1", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT);
SearchResultMessage * dm = static_cast<SearchResultMessage *>(routable.get());
EXPECT_EQUAL(dm->getSequenceId(), size_t(0));
EXPECT_EQUAL(dm->getHitCount(), size_t(0));
srm.addHit(0, "doc1", 89);
srm.addHit(1, "doc17", 109);
- //srm.setSequenceId(567);
-
- data = encode(srm);
- //EXPECT_EQUAL(srm.getSequenceId(), size_t(567));
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, data.size());
- writeFile(getPath("5-cpp-SearchResultMessage-2.dat"), data);
- routable = decode(data);
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, serialize("SearchResultMessage-2", srm));
+ routable = deserialize("SearchResultMessage-2", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT);
dm = static_cast<SearchResultMessage *>(routable.get());
-// EXPECT_EQUAL(dm->getSequenceId(), size_t(567));
EXPECT_EQUAL(dm->getHitCount(), size_t(2));
const char *docId;
SearchResultMessage::RankType rank;
@@ -568,16 +540,12 @@ Messages50Test::testSearchResultMessage()
srm.sort();
- data = encode(srm);
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, data.size());
- writeFile(getPath("5-cpp-SearchResultMessage-3.dat"), data);
- routable = decode(data);
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, serialize("SearchResultMessage-3", srm));
+ routable = deserialize("SearchResultMessage-3", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT);
dm = static_cast<SearchResultMessage *>(routable.get());
-// EXPECT_EQUAL(dm->getSequenceId(), size_t(567));
EXPECT_EQUAL(dm->getHitCount(), size_t(2));
dm->getHit(0, docId, rank);
EXPECT_EQUAL(rank, SearchResultMessage::RankType(109));
@@ -590,18 +558,13 @@ Messages50Test::testSearchResultMessage()
srm2.addHit(0, "doc1", 89, "sortdata2", 9);
srm2.addHit(1, "doc17", 109, "sortdata1", 9);
srm2.addHit(2, "doc18", 90, "sortdata3", 9);
- //srm2.setSequenceId(567);
- data = encode(srm2);
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, data.size());
- writeFile(getPath("5-cpp-SearchResultMessage-4.dat"), data);
- routable = decode(data);
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, serialize("SearchResultMessage-4", srm2));
+ routable = deserialize("SearchResultMessage-4", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT);
dm = static_cast<SearchResultMessage *>(routable.get());
- //EXPECT_EQUAL(dm->getSequenceId(), size_t(567));
EXPECT_EQUAL(dm->getHitCount(), size_t(3));
dm->getHit(0, docId, rank);
EXPECT_EQUAL(rank, SearchResultMessage::RankType(89));
@@ -635,16 +598,12 @@ Messages50Test::testSearchResultMessage()
EXPECT_EQUAL(rank, SearchResultMessage::RankType(90));
EXPECT_EQUAL(strcmp("doc18", docId), 0);
- data = encode(srm2);
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, data.size());
- writeFile(getPath("5-cpp-SearchResultMessage-5.dat"), data);
- routable = decode(data);
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, serialize("SearchResultMessage-5", srm2));
+ routable = deserialize("SearchResultMessage-5", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT);
dm = static_cast<SearchResultMessage *>(routable.get());
-// EXPECT_EQUAL(dm->getSequenceId(), size_t(567));
EXPECT_EQUAL(dm->getHitCount(), size_t(3));
dm->getHit(0, docId, rank);
dm->getSortBlob(0, buf, sz);
@@ -693,90 +652,6 @@ Messages50Test::testUpdateDocumentMessage()
}
bool
-Messages50Test::testBatchDocumentUpdateMessage()
-{
- const DocumentTypeRepo &repo = getTypeRepo();
- const document::DocumentType &docType = *repo.getDocumentType("testdoc");
-
- BatchDocumentUpdateMessage msg(1234);
-
- {
- document::DocumentUpdate::SP upd;
- upd.reset(new document::DocumentUpdate(repo, docType, document::DocumentId("userdoc:footype:1234:foo")));
- upd->addFieldPathUpdate(document::FieldPathUpdate::CP(
- new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0")));
- msg.addUpdate(upd);
- }
- {
- document::DocumentUpdate::SP upd;
- upd.reset(new document::DocumentUpdate(repo, docType, document::DocumentId("orderdoc(32,17):footype:1234:123456789:foo")));
- upd->addFieldPathUpdate(document::FieldPathUpdate::CP(
- new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0")));
- msg.addUpdate(upd);
- }
- try {
- document::DocumentUpdate::SP upd;
- upd.reset(new document::DocumentUpdate(repo, docType, document::DocumentId("userdoc:footype:5678:foo")));
- upd->addFieldPathUpdate(document::FieldPathUpdate::CP(
- new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0")));
- msg.addUpdate(upd);
- EXPECT_TRUE(false);
- } catch (...) {
- }
- try {
- document::DocumentUpdate::SP upd;
- upd.reset(new document::DocumentUpdate(repo, docType, document::DocumentId("groupdoc:footype:hable:foo")));
- upd->addFieldPathUpdate(document::FieldPathUpdate::CP(
- new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0")));
- msg.addUpdate(upd);
- EXPECT_TRUE(false);
- } catch (...) {
- }
-
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 202u, serialize("BatchDocumentUpdateMessage", msg));
- for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) {
- mbus::Routable::UP obj = deserialize("BatchDocumentUpdateMessage", DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE, lang);
- if (EXPECT_TRUE(obj.get() != NULL)) {
- BatchDocumentUpdateMessage &ref = static_cast<BatchDocumentUpdateMessage&>(*obj);
- EXPECT_EQUAL(2u, ref.getUpdates().size());
- }
- }
-
- return true;
-}
-
-bool
-Messages50Test::testBatchDocumentUpdateReply()
-{
- BatchDocumentUpdateReply reply;
- reply.setHighestModificationTimestamp(30);
- {
- std::vector<bool> notFound(3);
- notFound[0] = false;
- notFound[1] = true;
- notFound[2] = true;
- reply.getDocumentsNotFound() = notFound;
- }
-
- EXPECT_EQUAL(20u, serialize("BatchDocumentUpdateReply", reply));
-
- for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) {
- mbus::Routable::UP obj = deserialize("BatchDocumentUpdateReply", DocumentProtocol::REPLY_BATCHDOCUMENTUPDATE, lang);
- if (EXPECT_TRUE(obj.get() != NULL)) {
- BatchDocumentUpdateReply &ref = dynamic_cast<BatchDocumentUpdateReply&>(*obj);
- EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp());
- {
- const std::vector<bool>& notFound = ref.getDocumentsNotFound();
- EXPECT_TRUE(notFound[0] == false);
- EXPECT_TRUE(notFound[1] == true);
- EXPECT_TRUE(notFound[2] == true);
- }
- }
- }
- return true;
-}
-
-bool
Messages50Test::testQueryResultMessage()
{
QueryResultMessage srm;
@@ -787,18 +662,12 @@ Messages50Test::testQueryResultMessage()
EXPECT_EQUAL(sr.getSerializedSize(), 20u);
EXPECT_EQUAL(srm.getApproxSize(), 28u);
- mbus::Blob data = encode(srm);
-
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(32), data.size());
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(32), serialize("QueryResultMessage-1", srm));
- writeFile(getPath("5-cpp-QueryResultMessage-1.dat"), data);
- // print(data);
-
- mbus::Routable::UP routable = decode(data);
+ mbus::Routable::UP routable = deserialize("QueryResultMessage-1", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT);
QueryResultMessage * dm = static_cast<QueryResultMessage *>(routable.get());
vdslib::SearchResult * dr(&dm->getSearchResult());
EXPECT_EQUAL(dm->getSequenceId(), size_t(0));
@@ -807,15 +676,11 @@ Messages50Test::testQueryResultMessage()
sr.addHit(0, "doc1", 89);
sr.addHit(1, "doc17", 109);
- data = encode(srm);
-
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, data.size());
- writeFile(getPath("5-cpp-QueryResultMessage-2.dat"), data);
- routable = decode(data);
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, serialize("QueryResultMessage-2", srm));
+ routable = deserialize("QueryResultMessage-2", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT);
dm = static_cast<QueryResultMessage *>(routable.get());
dr = &dm->getSearchResult();
EXPECT_EQUAL(dr->getHitCount(), size_t(2));
@@ -830,14 +695,11 @@ Messages50Test::testQueryResultMessage()
sr.sort();
- data = encode(srm);
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, data.size());
- writeFile(getPath("5-cpp-QueryResultMessage-3.dat"), data);
- routable = decode(data);
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, serialize("QueryResultMessage-3", srm));
+ routable = deserialize("QueryResultMessage-3", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT);
dm = static_cast<QueryResultMessage *>(routable.get());
dr = &dm->getSearchResult();
EXPECT_EQUAL(dr->getHitCount(), size_t(2));
@@ -853,15 +715,12 @@ Messages50Test::testQueryResultMessage()
sr2.addHit(0, "doc1", 89, "sortdata2", 9);
sr2.addHit(1, "doc17", 109, "sortdata1", 9);
sr2.addHit(2, "doc18", 90, "sortdata3", 9);
- data = encode(srm2);
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, data.size());
- writeFile(getPath("5-cpp-QueryResultMessage-4.dat"), data);
- routable = decode(data);
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, serialize("QueryResultMessage-4", srm2));
+ routable = deserialize("QueryResultMessage-4", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT);
dm = static_cast<QueryResultMessage *>(routable.get());
dr = &dm->getSearchResult();
EXPECT_EQUAL(dr->getHitCount(), size_t(3));
@@ -897,14 +756,11 @@ Messages50Test::testQueryResultMessage()
EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(90));
EXPECT_EQUAL(strcmp("doc18", docId), 0);
- data = encode(srm2);
- EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, data.size());
- writeFile(getPath("5-cpp-QueryResultMessage-5.dat"), data);
- routable = decode(data);
+ EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, serialize("QueryResultMessage-5", srm2));
+ routable = deserialize("QueryResultMessage-5", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP);
if (!EXPECT_TRUE(routable.get() != NULL)) {
return false;
}
- EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT);
dm = static_cast<QueryResultMessage *>(routable.get());
dr = &dm->getSearchResult();
EXPECT_EQUAL(dr->getHitCount(), size_t(3));
diff --git a/documentapi/src/tests/messages/messages50test.h b/documentapi/src/tests/messages/messages50test.h
index c764f814a45..96c1be78c27 100644
--- a/documentapi/src/tests/messages/messages50test.h
+++ b/documentapi/src/tests/messages/messages50test.h
@@ -13,8 +13,6 @@ protected:
public:
Messages50Test();
- bool testBatchDocumentUpdateMessage();
- bool testBatchDocumentUpdateReply();
bool testCreateVisitorMessage();
bool testCreateVisitorReply();
bool testDestroyVisitorMessage();
diff --git a/documentapi/src/tests/messages/messages51app.cpp b/documentapi/src/tests/messages/messages51app.cpp
deleted file mode 100644
index 654fb932568..00000000000
--- a/documentapi/src/tests/messages/messages51app.cpp
+++ /dev/null
@@ -1,5 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "messages51test.h"
-
-TEST_APPHOOK(Messages51Test);
diff --git a/documentapi/src/tests/messages/messages52test.cpp b/documentapi/src/tests/messages/messages52test.cpp
index 8f5d7381500..1da48d4aa41 100644
--- a/documentapi/src/tests/messages/messages52test.cpp
+++ b/documentapi/src/tests/messages/messages52test.cpp
@@ -5,6 +5,7 @@
#include "messages52test.h"
#include <vespa/documentapi/documentapi.h>
#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/update/documentupdate.h>
#include <vespa/document/update/fieldpathupdates.h>
#include <vespa/document/datatype/documenttype.h>
diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp
index 3e804c30415..c01cdfde30c 100644
--- a/documentapi/src/tests/policies/policies_test.cpp
+++ b/documentapi/src/tests/policies/policies_test.cpp
@@ -22,6 +22,7 @@
#include <vespa/document/fieldvalue/longfieldvalue.h>
#include <vespa/document/datatype/documenttype.h>
#include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/document/update/documentupdate.h>
#include <vespa/vespalib/testkit/testapp.h>
#include <vespa/vespalib/util/stringfmt.h>
diff --git a/documentapi/src/vespa/documentapi/documentapi.h b/documentapi/src/vespa/documentapi/documentapi.h
index dff4125e624..0056dd2abdf 100644
--- a/documentapi/src/vespa/documentapi/documentapi.h
+++ b/documentapi/src/vespa/documentapi/documentapi.h
@@ -1,8 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once
-#include <vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h>
-#include <vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h>
#include <vespa/documentapi/messagebus/messages/getbucketstatemessage.h>
#include <vespa/documentapi/messagebus/messages/getbucketstatereply.h>
#include <vespa/documentapi/messagebus/messages/getdocumentmessage.h>
diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp
index d2661d0fe6c..b70a76690f9 100644
--- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp
@@ -1,7 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include "routablefactories50.h"
-#include "routablefactories51.h"
#include "routablefactories52.h"
#include "routablefactories60.h"
#include "routingpolicyfactories.h"
@@ -46,64 +44,49 @@ DocumentProtocol::DocumentProtocol(const LoadTypeSet& loadTypes,
putRoutingPolicyFactory("LoadBalancer", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::LoadBalancerPolicyFactory()));
// Prepare version specifications to use when adding routable factories.
- vespalib::VersionSpecification version50(5, 0);
- vespalib::VersionSpecification version51(5, 1);
vespalib::VersionSpecification version52(5, 115);
vespalib::VersionSpecification version6(6, 221);
- std::vector<vespalib::VersionSpecification> from50 = { version50, version51, version52, version6 };
- std::vector<vespalib::VersionSpecification> from51 = { version51, version52, version6 };
std::vector<vespalib::VersionSpecification> from52 = { version52, version6 };
std::vector<vespalib::VersionSpecification> from6 = { version6 };
- // Add 5.0 serialization
- putRoutableFactory(MESSAGE_BATCHDOCUMENTUPDATE, IRoutableFactory::SP(new RoutableFactories50::BatchDocumentUpdateMessageFactory(*_repo)), from50);
- putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories50::CreateVisitorMessageFactory(*_repo)), from50);
- putRoutableFactory(MESSAGE_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories50::DestroyVisitorMessageFactory()), from50);
- putRoutableFactory(MESSAGE_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories50::DocumentListMessageFactory(*_repo)), from50);
- putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories50::DocumentSummaryMessageFactory()), from50);
- putRoutableFactory(MESSAGE_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories50::EmptyBucketsMessageFactory()), from50);
- putRoutableFactory(MESSAGE_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories50::GetBucketListMessageFactory()), from50);
- putRoutableFactory(MESSAGE_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories50::GetBucketStateMessageFactory()), from50);
- putRoutableFactory(MESSAGE_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::GetDocumentMessageFactory()), from50);
- putRoutableFactory(MESSAGE_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories50::MapVisitorMessageFactory(*_repo)), from50);
- putRoutableFactory(MESSAGE_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::PutDocumentMessageFactory(*_repo)), from50);
- putRoutableFactory(MESSAGE_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories50::QueryResultMessageFactory()), from50);
- putRoutableFactory(MESSAGE_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::RemoveDocumentMessageFactory()), from50);
- putRoutableFactory(MESSAGE_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories50::RemoveLocationMessageFactory(*_repo)), from50);
- putRoutableFactory(MESSAGE_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories50::SearchResultMessageFactory()), from50);
- putRoutableFactory(MESSAGE_STATBUCKET, IRoutableFactory::SP(new RoutableFactories50::StatBucketMessageFactory()), from50);
- putRoutableFactory(MESSAGE_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::UpdateDocumentMessageFactory(*_repo)), from50);
- putRoutableFactory(MESSAGE_VISITORINFO, IRoutableFactory::SP(new RoutableFactories50::VisitorInfoMessageFactory()), from50);
- putRoutableFactory(REPLY_BATCHDOCUMENTUPDATE, IRoutableFactory::SP(new RoutableFactories50::BatchDocumentUpdateReplyFactory()), from50);
- putRoutableFactory(REPLY_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories50::CreateVisitorReplyFactory()), from50);
- putRoutableFactory(REPLY_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories50::DestroyVisitorReplyFactory()), from50);
- putRoutableFactory(REPLY_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories50::DocumentListReplyFactory()), from50);
- putRoutableFactory(REPLY_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories50::DocumentSummaryReplyFactory()), from50);
- putRoutableFactory(REPLY_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories50::EmptyBucketsReplyFactory()), from50);
- putRoutableFactory(REPLY_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories50::GetBucketListReplyFactory()), from50);
- putRoutableFactory(REPLY_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories50::GetBucketStateReplyFactory()), from50);
- putRoutableFactory(REPLY_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::GetDocumentReplyFactory(*_repo)), from50);
- putRoutableFactory(REPLY_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories50::MapVisitorReplyFactory()), from50);
- putRoutableFactory(REPLY_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::PutDocumentReplyFactory()), from50);
- putRoutableFactory(REPLY_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories50::QueryResultReplyFactory()), from50);
- putRoutableFactory(REPLY_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::RemoveDocumentReplyFactory()), from50);
- putRoutableFactory(REPLY_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories50::RemoveLocationReplyFactory()), from50);
- putRoutableFactory(REPLY_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories50::SearchResultReplyFactory()), from50);
- putRoutableFactory(REPLY_STATBUCKET, IRoutableFactory::SP(new RoutableFactories50::StatBucketReplyFactory()), from50);
- putRoutableFactory(REPLY_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::UpdateDocumentReplyFactory()), from50);
- putRoutableFactory(REPLY_VISITORINFO, IRoutableFactory::SP(new RoutableFactories50::VisitorInfoReplyFactory()), from50);
- putRoutableFactory(REPLY_WRONGDISTRIBUTION, IRoutableFactory::SP(new RoutableFactories50::WrongDistributionReplyFactory()), from50);
-
- // Add 5.1 serialization
- putRoutableFactory(MESSAGE_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories51::GetDocumentMessageFactory()), from51);
- putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories51::CreateVisitorMessageFactory(*_repo)), from51);
- putRoutableFactory(REPLY_DOCUMENTIGNORED, IRoutableFactory::SP(new RoutableFactories51::DocumentIgnoredReplyFactory()), from51);
-
// Add 5.2 serialization
+ putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories52::CreateVisitorMessageFactory(*_repo)), from52);
+ putRoutableFactory(MESSAGE_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories52::DestroyVisitorMessageFactory()), from52);
+ putRoutableFactory(MESSAGE_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories52::DocumentListMessageFactory(*_repo)), from52);
+ putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories52::DocumentSummaryMessageFactory()), from52);
+ putRoutableFactory(MESSAGE_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories52::EmptyBucketsMessageFactory()), from52);
+ putRoutableFactory(MESSAGE_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories52::GetBucketListMessageFactory()), from52);
+ putRoutableFactory(MESSAGE_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories52::GetBucketStateMessageFactory()), from52);
+ putRoutableFactory(MESSAGE_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::GetDocumentMessageFactory()), from52);
+ putRoutableFactory(MESSAGE_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories52::MapVisitorMessageFactory(*_repo)), from52);
putRoutableFactory(MESSAGE_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::PutDocumentMessageFactory(*_repo)), from52);
- putRoutableFactory(MESSAGE_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::UpdateDocumentMessageFactory(*_repo)), from52);
+ putRoutableFactory(MESSAGE_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories52::QueryResultMessageFactory()), from52);
putRoutableFactory(MESSAGE_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::RemoveDocumentMessageFactory()), from52);
+ putRoutableFactory(MESSAGE_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories52::RemoveLocationMessageFactory(*_repo)), from52);
+ putRoutableFactory(MESSAGE_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories52::SearchResultMessageFactory()), from52);
+ putRoutableFactory(MESSAGE_STATBUCKET, IRoutableFactory::SP(new RoutableFactories52::StatBucketMessageFactory()), from52);
+ putRoutableFactory(MESSAGE_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::UpdateDocumentMessageFactory(*_repo)), from52);
+ putRoutableFactory(MESSAGE_VISITORINFO, IRoutableFactory::SP(new RoutableFactories52::VisitorInfoMessageFactory()), from52);
+ putRoutableFactory(REPLY_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories52::CreateVisitorReplyFactory()), from52);
+ putRoutableFactory(REPLY_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories52::DestroyVisitorReplyFactory()), from52);
+ putRoutableFactory(REPLY_DOCUMENTIGNORED, IRoutableFactory::SP(new RoutableFactories52::DocumentIgnoredReplyFactory()), from52);
+ putRoutableFactory(REPLY_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories52::DocumentListReplyFactory()), from52);
+ putRoutableFactory(REPLY_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories52::DocumentSummaryReplyFactory()), from52);
+ putRoutableFactory(REPLY_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories52::EmptyBucketsReplyFactory()), from52);
+ putRoutableFactory(REPLY_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories52::GetBucketListReplyFactory()), from52);
+ putRoutableFactory(REPLY_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories52::GetBucketStateReplyFactory()), from52);
+ putRoutableFactory(REPLY_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::GetDocumentReplyFactory(*_repo)), from52);
+ putRoutableFactory(REPLY_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories52::MapVisitorReplyFactory()), from52);
+ putRoutableFactory(REPLY_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::PutDocumentReplyFactory()), from52);
+ putRoutableFactory(REPLY_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories52::QueryResultReplyFactory()), from52);
+ putRoutableFactory(REPLY_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::RemoveDocumentReplyFactory()), from52);
+ putRoutableFactory(REPLY_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories52::RemoveLocationReplyFactory()), from52);
+ putRoutableFactory(REPLY_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories52::SearchResultReplyFactory()), from52);
+ putRoutableFactory(REPLY_STATBUCKET, IRoutableFactory::SP(new RoutableFactories52::StatBucketReplyFactory()), from52);
+ putRoutableFactory(REPLY_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::UpdateDocumentReplyFactory()), from52);
+ putRoutableFactory(REPLY_VISITORINFO, IRoutableFactory::SP(new RoutableFactories52::VisitorInfoReplyFactory()), from52);
+ putRoutableFactory(REPLY_WRONGDISTRIBUTION, IRoutableFactory::SP(new RoutableFactories52::WrongDistributionReplyFactory()), from52);
// Add 6.x serialization (TODO finalize version)
putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories60::CreateVisitorMessageFactory(*_repo)), from6);
diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h
index 32e6dc1d95e..79f7d7c0ccc 100644
--- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h
+++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h
@@ -68,7 +68,6 @@ public:
MESSAGE_EMPTYBUCKETS = DOCUMENT_MESSAGE + 23,
MESSAGE_REMOVELOCATION = DOCUMENT_MESSAGE + 24,
MESSAGE_QUERYRESULT = DOCUMENT_MESSAGE + 25,
- MESSAGE_BATCHDOCUMENTUPDATE = DOCUMENT_MESSAGE + 26,
// MESSAGE_GARBAGECOLLECT = DOCUMENT_MESSAGE + 27,
DOCUMENT_REPLY = 200000,
@@ -92,7 +91,6 @@ public:
REPLY_EMPTYBUCKETS = DOCUMENT_REPLY + 23,
REPLY_REMOVELOCATION = DOCUMENT_REPLY + 24,
REPLY_QUERYRESULT = DOCUMENT_REPLY + 25,
- REPLY_BATCHDOCUMENTUPDATE = DOCUMENT_REPLY + 26,
// REPLY_GARBAGECOLLECT = DOCUMENT_REPLY + 27,
REPLY_WRONGDISTRIBUTION = DOCUMENT_REPLY + 1000,
REPLY_DOCUMENTIGNORED = DOCUMENT_REPLY + 1001
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt
index 3fa7e6086e9..7bd18a14719 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt
@@ -1,8 +1,6 @@
# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
vespa_add_library(documentapi_documentapimessages OBJECT
SOURCES
- batchdocumentupdatemessage.cpp
- batchdocumentupdatereply.cpp
documentignoredreply.cpp
documentmessage.cpp
documentreply.cpp
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp
deleted file mode 100644
index 2b38f9b24ae..00000000000
--- a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "batchdocumentupdatemessage.h"
-#include "batchdocumentupdatereply.h"
-#include <vespa/documentapi/messagebus/documentprotocol.h>
-#include <vespa/document/bucket/bucketidfactory.h>
-#include <vespa/vespalib/util/exceptions.h>
-
-namespace documentapi {
-
-BatchDocumentUpdateMessage::BatchDocumentUpdateMessage(uint64_t userId)
- : _userId(userId)
-{
- setBucketId(document::UserDocIdString(vespalib::make_string("userdoc:foo:%lu:bar", _userId)));
-}
-
-BatchDocumentUpdateMessage::BatchDocumentUpdateMessage(const string& group)
- : _userId(0),
- _group(group)
-{
- setBucketId(document::GroupDocIdString("groupdoc:foo:" + _group + ":bar"));
-}
-
-BatchDocumentUpdateMessage::~BatchDocumentUpdateMessage() {}
-
-void
-BatchDocumentUpdateMessage::setBucketId(const document::IdString& idString)
-{
- document::BucketIdFactory factory;
- _bucketId = factory.getBucketId(document::DocumentId(idString));
-}
-
-void
-BatchDocumentUpdateMessage::addUpdate(document::DocumentUpdate::SP update)
-{
- verifyUpdate(*update);
- _updates.push_back(update);
-}
-
-void
-BatchDocumentUpdateMessage::verifyUpdate(const document::DocumentUpdate& update) {
- const document::IdString& idString = update.getId().getScheme();
-
- if (_group.length()) {
- string group;
-
- if (idString.hasGroup()) {
- group = idString.getGroup();
- } else {
- throw vespalib::IllegalArgumentException("Batch update message can only contain groupdoc or orderdoc items");
- }
-
- if (group != _group) {
- throw vespalib::IllegalArgumentException(vespalib::make_string("Batch update message can not contain messages from group %s, only group %s", group.c_str(), _group.c_str()));
- }
- } else {
- uint64_t userId;
-
- if (idString.hasNumber()) {
- userId = idString.getNumber();
- } else {
- throw vespalib::IllegalArgumentException("Batch update message can only contain userdoc or orderdoc items");
- }
-
- if (userId != _userId) {
- throw vespalib::IllegalArgumentException(vespalib::make_string("Batch update message can not contain messages from user %llu, only user %llu", (long long unsigned)userId, (long long unsigned)_userId));
- }
- }
-}
-
-DocumentReply::UP
-BatchDocumentUpdateMessage::doCreateReply() const
-{
- return DocumentReply::UP(new BatchDocumentUpdateReply());
-}
-
-uint32_t
-BatchDocumentUpdateMessage::getType() const {
- return DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE;
-}
-
-}
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h
deleted file mode 100644
index 54599c2627b..00000000000
--- a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h
+++ /dev/null
@@ -1,81 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include "documentmessage.h"
-#include "writedocumentreply.h"
-#include <vespa/document/update/documentupdate.h>
-#include <vespa/document/bucket/bucketid.h>
-#include <vespa/document/base/idstring.h>
-
-namespace documentapi {
-
-/**
- Message to use to send multiple updates for documents
- belonging to the same user or group to Vespa. Using this
- message improves performance in VDS mainly.
-*/
-class BatchDocumentUpdateMessage : public DocumentMessage
-{
-public:
- typedef std::vector<document::DocumentUpdate::SP > UpdateList;
-
- /**
- Creates a batch update message that can contain only updates
- for documents belonging to the given user.
- */
- BatchDocumentUpdateMessage(uint64_t userId);
-
- /**
- Creates a batch update message that can contain only updates
- for documents belonging to the given group.
- */
- BatchDocumentUpdateMessage(const string& group);
- ~BatchDocumentUpdateMessage();
-
- /**
- @return Returns a list of the updates to be performed.
- */
- const UpdateList& getUpdates() const { return _updates; };
-
- /**
- Adds an update to be performed.
- */
- void addUpdate(document::DocumentUpdate::SP update);
-
- /**
- Returns the user id that this batch can contain.
- Only valid if this object was created with the first constructor.
- */
- uint64_t getUserId() const { return _userId; };
-
- /**
- Returns the grouo that this batch can contain.
- Only valid if this object was created with the second constructor.
- */
- const string& getGroup() const { return _group; }
-
- uint32_t getType() const override;
-
- /**
- Returns a bucket id suitable for routing this message.
- */
- const document::BucketId& getBucketId() const { return _bucketId; }
-
- string toString() const override { return "batchdocumentupdatemessage"; }
-
-protected:
- DocumentReply::UP doCreateReply() const override;
-
-private:
- uint64_t _userId;
- string _group;
-
- UpdateList _updates;
- document::BucketId _bucketId;
-
- void verifyUpdate(const document::DocumentUpdate& update);
- void setBucketId(const document::IdString& idString);
-};
-
-}
-
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.cpp
deleted file mode 100644
index 483740233c0..00000000000
--- a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "batchdocumentupdatereply.h"
-#include <vespa/documentapi/messagebus/documentprotocol.h>
-
-namespace documentapi {
-
-BatchDocumentUpdateReply::BatchDocumentUpdateReply()
- : WriteDocumentReply(DocumentProtocol::REPLY_BATCHDOCUMENTUPDATE)
-{ }
-
-BatchDocumentUpdateReply::~BatchDocumentUpdateReply() {}
-
-}
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h
deleted file mode 100644
index be1f5bc75d5..00000000000
--- a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include "writedocumentreply.h"
-
-namespace documentapi {
-
-class BatchDocumentUpdateReply : public WriteDocumentReply
-{
- /**
- * If all documents to update are found, this vector will be empty. If
- * one or more documents are not found, this vector will have the size of
- * the initial number of updates, with entries set to true where the
- * corresponding update was not found.
- */
- std::vector<bool> _documentsNotFound;
-public:
- typedef std::unique_ptr<BatchDocumentUpdateReply> UP;
- typedef std::shared_ptr<BatchDocumentUpdateReply> SP;
-
- BatchDocumentUpdateReply();
- ~BatchDocumentUpdateReply();
-
- const std::vector<bool>& getDocumentsNotFound() const { return _documentsNotFound; }
- std::vector<bool>& getDocumentsNotFound() { return _documentsNotFound; }
-
- string toString() const override { return "batchdocumentupdatereply"; }
-};
-
-}
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp
index 42120b8052a..62135150d98 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp
@@ -4,7 +4,6 @@
#include <vespa/document/bucket/bucketidfactory.h>
#include <vespa/document/select/parser.h>
#include <vespa/documentapi/messagebus/documentprotocol.h>
-#include <vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h>
#include <vespa/documentapi/messagebus/messages/putdocumentmessage.h>
#include <vespa/documentapi/messagebus/messages/updatedocumentmessage.h>
#include <vespa/documentapi/messagebus/messages/documentignoredreply.h>
@@ -138,17 +137,6 @@ DocumentRouteSelectorPolicy::select(mbus::RoutingContext &context, const vespali
return true;
}
}
-
- case DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE:
- {
- const BatchDocumentUpdateMessage& mom = static_cast<const BatchDocumentUpdateMessage&>(msg);
- for (uint32_t i = 0; i < mom.getUpdates().size(); i++) {
- if (it->second->contains(*mom.getUpdates()[i]) == Result::False) {
- return false;
- }
- }
- return true;
- }
default:
return true;
}
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp
index 723c6fc836a..472440d7863 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp
@@ -2,6 +2,7 @@
#include "storagepolicy.h"
#include <vespa/document/base/documentid.h>
+#include <vespa/document/update/documentupdate.h>
#include <vespa/messagebus/emptyreply.h>
#include <vespa/messagebus/routing/verbatimdirective.h>
#include <vespa/documentapi/documentapi.h>
@@ -150,10 +151,6 @@ StoragePolicy::doSelect(mbus::RoutingContext &context)
id = static_cast<const RemoveLocationMessage&>(msg).getBucketId();
break;
- case DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE:
- id = static_cast<const BatchDocumentUpdateMessage&>(msg).getBucketId();
- break;
-
default:
LOG(error, "Message type '%d' not supported.", msg.getType());
return;
diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp
index 26d85b57522..120ee1facd0 100644
--- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp
@@ -4,6 +4,7 @@
#include <vespa/document/bucket/bucketidfactory.h>
#include <vespa/document/bucket/fixed_bucket_spaces.h>
#include <vespa/document/select/parser.h>
+#include <vespa/document/update/documentupdate.h>
#include <vespa/documentapi/documentapi.h>
#include <vespa/documentapi/loadtypes/loadtypeset.h>
#include <vespa/vespalib/objects/nbostream.h>
@@ -67,69 +68,6 @@ RoutableFactories50::DocumentReplyFactory::decode(document::ByteBuffer &in, cons
////////////////////////////////////////////////////////////////////////////////
DocumentMessage::UP
-RoutableFactories50::BatchDocumentUpdateMessageFactory::doDecode(document::ByteBuffer &buf) const
-{
- uint64_t userId = (uint64_t)decodeLong(buf);
- string group = decodeString(buf);
-
- auto msg = (group.length())
- ? std::make_unique<BatchDocumentUpdateMessage>(group)
- : std::make_unique<BatchDocumentUpdateMessage>(userId);
-
- uint32_t len = decodeInt(buf);
- for (uint32_t i = 0; i < len; i++) {
- document::DocumentUpdate::SP upd = document::DocumentUpdate::createHEAD(_repo, buf);
- msg->addUpdate(upd);
- }
-
- return msg;
-}
-
-bool
-RoutableFactories50::BatchDocumentUpdateMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const
-{
- const BatchDocumentUpdateMessage &msg = static_cast<const BatchDocumentUpdateMessage&>(obj);
-
- buf.putLong(msg.getUserId());
- buf.putString(msg.getGroup());
- buf.putInt(msg.getUpdates().size());
-
- vespalib::nbostream stream;
- for (const auto & update : msg.getUpdates()) {
- update->serializeHEAD(stream);
- }
- buf.putBytes(stream.c_str(), stream.size());
-
- return true;
-}
-
-DocumentReply::UP
-RoutableFactories50::BatchDocumentUpdateReplyFactory::doDecode(document::ByteBuffer &buf) const
-{
- auto reply = std::make_unique<BatchDocumentUpdateReply>();
- reply->setHighestModificationTimestamp(decodeLong(buf));
- std::vector<bool>& notFound = reply->getDocumentsNotFound();
- notFound.resize(decodeInt(buf));
- for (std::size_t i = 0; i < notFound.size(); ++i) {
- notFound[i] = decodeBoolean(buf);
- }
- return reply;
-}
-
-bool
-RoutableFactories50::BatchDocumentUpdateReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const
-{
- const BatchDocumentUpdateReply& reply = static_cast<const BatchDocumentUpdateReply&>(obj);
- buf.putLong(reply.getHighestModificationTimestamp());
- const std::vector<bool>& notFoundV = reply.getDocumentsNotFound();
- buf.putInt(notFoundV.size());
- for (bool notFound : notFoundV) {
- buf.putBoolean(notFound);
- }
- return true;
-}
-
-DocumentMessage::UP
RoutableFactories50::CreateVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const
{
auto msg = std::make_unique<CreateVisitorMessage>();
diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.h
index 12d5c560786..f96b9641800 100644
--- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.h
+++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.h
@@ -157,19 +157,6 @@ public:
// Factories
//
////////////////////////////////////////////////////////////////////////////////
-
- class BatchDocumentUpdateMessageFactory : public DocumentMessageFactory {
- const document::DocumentTypeRepo &_repo;
- DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override;
- bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override;
- public:
- BatchDocumentUpdateMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {}
- };
- class BatchDocumentUpdateReplyFactory : public DocumentReplyFactory {
- protected:
- DocumentReply::UP doDecode(document::ByteBuffer &buf) const override;
- bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override;
- };
class CreateVisitorMessageFactory : public DocumentMessageFactory {
const document::DocumentTypeRepo &_repo;
protected:
diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h
index fc20324b418..b71c478c540 100644
--- a/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h
+++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h
@@ -10,7 +10,7 @@ namespace documentapi {
* This class encapsulates all the {@link RoutableFactory} classes needed to implement factories for the document
* routable. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now.
*/
-class RoutableFactories51 {
+class RoutableFactories51 : public RoutableFactories50 {
public:
RoutableFactories51() = delete;
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateMessage.dat
deleted file mode 100644
index 95e663088c3..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateReply.dat
deleted file mode 100644
index 216db17f80e..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-BatchMessage.dat
deleted file mode 100644
index c1c43e8081b..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-BatchMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-BatchReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-BatchReply.dat
deleted file mode 100644
index be3a9ba1913..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-BatchReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorMessage.dat
deleted file mode 100644
index a506fc760bd..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorReply.dat
deleted file mode 100644
index e13917227d1..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorMessage.dat
deleted file mode 100644
index f39b31217e6..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorReply.dat
deleted file mode 100644
index 1468f027b15..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-DocumentListMessage.dat
deleted file mode 100644
index 2d8d12d2704..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-DocumentListMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-DocumentListReply.dat
deleted file mode 100644
index c8a1cd888f0..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-DocumentListReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryReply.dat
deleted file mode 100644
index 16b1e4bc4ef..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsMessage.dat
deleted file mode 100644
index b9df278fd7a..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsReply.dat
deleted file mode 100644
index 05510dd8c1e..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListMessage.dat
deleted file mode 100644
index fa3de45ac5b..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListReply.dat
deleted file mode 100644
index 830994ed785..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateMessage.dat
deleted file mode 100644
index aa2d206ca3a..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateReply.dat
deleted file mode 100644
index 0f91e3759f8..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentMessage.dat
deleted file mode 100644
index 1444d617c22..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentReply.dat
deleted file mode 100644
index c1ad7920a2e..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorMessage.dat
deleted file mode 100644
index 1a8a837ea16..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorReply.dat
deleted file mode 100644
index 541cd718a66..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationMessage.dat
deleted file mode 100644
index 6efc54a6d09..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationReply.dat
deleted file mode 100644
index 8ad31a95bd5..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentMessage.dat
deleted file mode 100644
index 9e4a5c5f82f..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentReply.dat
deleted file mode 100644
index 480544045bb..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultReply.dat
deleted file mode 100644
index 003f35d63a7..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentMessage.dat
deleted file mode 100644
index 78b7972eb5c..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentReply.dat
deleted file mode 100644
index bf5db8761e2..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessage.dat
deleted file mode 100644
index 16850a6aff3..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageGroup.dat
deleted file mode 100644
index abd648184d7..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageGroup.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageUser.dat
deleted file mode 100644
index c2a63cb94c0..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageUser.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationReply.dat
deleted file mode 100644
index 752c4dba399..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultReply.dat
deleted file mode 100644
index cce9c6f8d14..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-StatBucketMessage.dat
deleted file mode 100644
index 1fc2b1cf3c1..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-StatBucketMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-StatBucketReply.dat
deleted file mode 100644
index 0b98e240018..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-StatBucketReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentMessage.dat
deleted file mode 100644
index f1bca25ceca..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentReply.dat
deleted file mode 100644
index c7151299366..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoMessage.dat
deleted file mode 100644
index 80b44e0c6fd..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoReply.dat
deleted file mode 100644
index 57a656c9b2d..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-WrongDistributionReply.dat
deleted file mode 100644
index 0dbe13225ae..00000000000
--- a/documentapi/test/crosslanguagefiles/5-cpp-WrongDistributionReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateMessage.dat
deleted file mode 100644
index 95e663088c3..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateReply.dat
deleted file mode 100644
index 216db17f80e..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5-java-BatchMessage.dat
deleted file mode 100644
index c1c43e8081b..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-BatchMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-BatchReply.dat b/documentapi/test/crosslanguagefiles/5-java-BatchReply.dat
deleted file mode 100644
index be3a9ba1913..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-BatchReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-java-CreateVisitorMessage.dat
deleted file mode 100644
index a506fc760bd..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-CreateVisitorMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-java-CreateVisitorReply.dat
deleted file mode 100644
index e13917227d1..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-CreateVisitorReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorMessage.dat
deleted file mode 100644
index f39b31217e6..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorReply.dat
deleted file mode 100644
index 1468f027b15..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5-java-DocumentListMessage.dat
deleted file mode 100644
index 2d8d12d2704..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-DocumentListMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5-java-DocumentListReply.dat
deleted file mode 100644
index c8a1cd888f0..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-DocumentListReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5-java-DocumentSummaryReply.dat
deleted file mode 100644
index 16b1e4bc4ef..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-DocumentSummaryReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsMessage.dat
deleted file mode 100644
index b9df278fd7a..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsReply.dat
deleted file mode 100644
index 05510dd8c1e..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5-java-GetBucketListMessage.dat
deleted file mode 100644
index fa3de45ac5b..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-GetBucketListMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5-java-GetBucketListReply.dat
deleted file mode 100644
index 830994ed785..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-GetBucketListReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5-java-GetBucketStateMessage.dat
deleted file mode 100644
index aa2d206ca3a..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-GetBucketStateMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5-java-GetBucketStateReply.dat
deleted file mode 100644
index 0f91e3759f8..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-GetBucketStateReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-java-GetDocumentMessage.dat
deleted file mode 100644
index 1444d617c22..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-GetDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-java-GetDocumentReply.dat
deleted file mode 100644
index c1ad7920a2e..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-GetDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-java-MapVisitorMessage.dat
deleted file mode 100644
index 1a8a837ea16..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-MapVisitorMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-java-MapVisitorReply.dat
deleted file mode 100644
index 541cd718a66..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-MapVisitorReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5-java-MultiOperationMessage.dat
deleted file mode 100644
index 6efc54a6d09..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-MultiOperationMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5-java-MultiOperationReply.dat
deleted file mode 100644
index 8ad31a95bd5..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-MultiOperationReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-java-PutDocumentMessage.dat
deleted file mode 100644
index 9e4a5c5f82f..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-PutDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-java-PutDocumentReply.dat
deleted file mode 100644
index 480544045bb..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-PutDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5-java-QueryResultReply.dat
deleted file mode 100644
index 003f35d63a7..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-QueryResultReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentMessage.dat
deleted file mode 100644
index 78b7972eb5c..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentReply.dat
deleted file mode 100644
index bf5db8761e2..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessage.dat
deleted file mode 100644
index 16850a6aff3..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageGroup.dat
deleted file mode 100644
index 9d7901a8b6e..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageGroup.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageUser.dat
deleted file mode 100644
index c2a63cb94c0..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageUser.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationReply.dat
deleted file mode 100644
index 752c4dba399..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5-java-SearchResultReply.dat
deleted file mode 100644
index cce9c6f8d14..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-SearchResultReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5-java-StatBucketMessage.dat
deleted file mode 100644
index 1fc2b1cf3c1..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-StatBucketMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5-java-StatBucketReply.dat
deleted file mode 100644
index 0b98e240018..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-StatBucketReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentMessage.dat
deleted file mode 100644
index f1bca25ceca..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentReply.dat
deleted file mode 100644
index c7151299366..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5-java-VisitorInfoMessage.dat
deleted file mode 100644
index 80b44e0c6fd..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-VisitorInfoMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5-java-VisitorInfoReply.dat
deleted file mode 100644
index 57a656c9b2d..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-VisitorInfoReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-java-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5-java-WrongDistributionReply.dat
deleted file mode 100644
index 0dbe13225ae..00000000000
--- a/documentapi/test/crosslanguagefiles/5-java-WrongDistributionReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateMessage.dat
deleted file mode 100644
index 95e663088c3..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateReply.dat
deleted file mode 100644
index 216db17f80e..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchMessage.dat
deleted file mode 100644
index c1c43e8081b..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchReply.dat
deleted file mode 100644
index be3a9ba1913..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorMessage.dat
deleted file mode 100644
index 4ca8648e702..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorReply.dat
deleted file mode 100644
index e13917227d1..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorMessage.dat
deleted file mode 100644
index f39b31217e6..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorReply.dat
deleted file mode 100644
index 1468f027b15..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentIgnoredReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentIgnoredReply.dat
deleted file mode 100644
index 15a7afe2a59..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentIgnoredReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListMessage.dat
deleted file mode 100644
index 2d8d12d2704..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListReply.dat
deleted file mode 100644
index c8a1cd888f0..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-1.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-1.dat
deleted file mode 100644
index 0107dd5f350..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-1.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-2.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-2.dat
deleted file mode 100644
index 57187093f28..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-2.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-3.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-3.dat
deleted file mode 100644
index 6a516d38d17..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-3.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryReply.dat
deleted file mode 100644
index 16b1e4bc4ef..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsMessage.dat
deleted file mode 100644
index b9df278fd7a..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsReply.dat
deleted file mode 100644
index 05510dd8c1e..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListMessage.dat
deleted file mode 100644
index fa3de45ac5b..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListReply.dat
deleted file mode 100644
index 830994ed785..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateMessage.dat
deleted file mode 100644
index aa2d206ca3a..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateReply.dat
deleted file mode 100644
index 0f91e3759f8..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentMessage.dat
deleted file mode 100644
index 3df64ed657f..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentReply.dat
deleted file mode 100644
index c1ad7920a2e..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorMessage.dat
deleted file mode 100644
index 1a8a837ea16..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorReply.dat
deleted file mode 100644
index 541cd718a66..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationMessage.dat
deleted file mode 100644
index 6efc54a6d09..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationReply.dat
deleted file mode 100644
index 8ad31a95bd5..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentMessage.dat
deleted file mode 100644
index 9e4a5c5f82f..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentReply.dat
deleted file mode 100644
index 480544045bb..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-1.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-1.dat
deleted file mode 100644
index dbf830c9365..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-1.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-2.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-2.dat
deleted file mode 100644
index 094143cf78d..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-2.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-3.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-3.dat
deleted file mode 100644
index 3341d74052b..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-3.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-4.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-4.dat
deleted file mode 100644
index 8aaaefff491..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-4.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-5.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-5.dat
deleted file mode 100644
index e66ed1f07d4..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-5.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultReply.dat
deleted file mode 100644
index 003f35d63a7..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentMessage.dat
deleted file mode 100644
index 78b7972eb5c..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentReply.dat
deleted file mode 100644
index bf5db8761e2..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessage.dat
deleted file mode 100644
index 16850a6aff3..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageGroup.dat
deleted file mode 100644
index abd648184d7..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageGroup.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageUser.dat
deleted file mode 100644
index c2a63cb94c0..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageUser.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationReply.dat
deleted file mode 100644
index 752c4dba399..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-1.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-1.dat
deleted file mode 100644
index 988f9fdab1f..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-1.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-2.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-2.dat
deleted file mode 100644
index ac277d09643..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-2.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-3.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-3.dat
deleted file mode 100644
index 03b49c8a0ac..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-3.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-4.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-4.dat
deleted file mode 100644
index d52e574ea44..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-4.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-5.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-5.dat
deleted file mode 100644
index e68654e9941..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-5.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultReply.dat
deleted file mode 100644
index cce9c6f8d14..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketMessage.dat
deleted file mode 100644
index 1fc2b1cf3c1..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketReply.dat
deleted file mode 100644
index 0b98e240018..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentMessage.dat
deleted file mode 100644
index f1bca25ceca..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentReply.dat
deleted file mode 100644
index c7151299366..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoMessage.dat
deleted file mode 100644
index 80b44e0c6fd..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoReply.dat
deleted file mode 100644
index 57a656c9b2d..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-WrongDistributionReply.dat
deleted file mode 100644
index 0dbe13225ae..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-cpp-WrongDistributionReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateMessage.dat
deleted file mode 100644
index 95e663088c3..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateReply.dat
deleted file mode 100644
index 216db17f80e..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-BatchMessage.dat
deleted file mode 100644
index c1c43e8081b..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-BatchMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-BatchReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-BatchReply.dat
deleted file mode 100644
index be3a9ba1913..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-BatchReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorMessage.dat
deleted file mode 100644
index 4ca8648e702..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorReply.dat
deleted file mode 100644
index e13917227d1..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorMessage.dat
deleted file mode 100644
index f39b31217e6..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorReply.dat
deleted file mode 100644
index 1468f027b15..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DocumentIgnoredReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-DocumentIgnoredReply.dat
deleted file mode 100644
index 15a7afe2a59..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-DocumentIgnoredReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-DocumentListMessage.dat
deleted file mode 100644
index 2d8d12d2704..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-DocumentListMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-DocumentListReply.dat
deleted file mode 100644
index c8a1cd888f0..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-DocumentListReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-DocumentSummaryReply.dat
deleted file mode 100644
index 16b1e4bc4ef..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-DocumentSummaryReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsMessage.dat
deleted file mode 100644
index b9df278fd7a..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsReply.dat
deleted file mode 100644
index 05510dd8c1e..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListMessage.dat
deleted file mode 100644
index fa3de45ac5b..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListReply.dat
deleted file mode 100644
index 830994ed785..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateMessage.dat
deleted file mode 100644
index aa2d206ca3a..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateReply.dat
deleted file mode 100644
index 0f91e3759f8..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentMessage.dat
deleted file mode 100644
index 3df64ed657f..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentReply.dat
deleted file mode 100644
index c1ad7920a2e..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorMessage.dat
deleted file mode 100644
index 1a8a837ea16..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorReply.dat
deleted file mode 100644
index 541cd718a66..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationMessage.dat
deleted file mode 100644
index 6efc54a6d09..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationReply.dat
deleted file mode 100644
index 8ad31a95bd5..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentMessage.dat
deleted file mode 100644
index 9e4a5c5f82f..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentReply.dat
deleted file mode 100644
index 480544045bb..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-QueryResultReply.dat
deleted file mode 100644
index 003f35d63a7..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-QueryResultReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentMessage.dat
deleted file mode 100644
index 78b7972eb5c..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentReply.dat
deleted file mode 100644
index bf5db8761e2..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessage.dat
deleted file mode 100644
index 16850a6aff3..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageGroup.dat
deleted file mode 100644
index 9d7901a8b6e..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageGroup.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageUser.dat
deleted file mode 100644
index c2a63cb94c0..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageUser.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationReply.dat
deleted file mode 100644
index 752c4dba399..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-SearchResultReply.dat
deleted file mode 100644
index cce9c6f8d14..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-SearchResultReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-StatBucketMessage.dat
deleted file mode 100644
index 1fc2b1cf3c1..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-StatBucketMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-StatBucketReply.dat
deleted file mode 100644
index 0b98e240018..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-StatBucketReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentMessage.dat
deleted file mode 100644
index f1bca25ceca..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentReply.dat
deleted file mode 100644
index c7151299366..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoMessage.dat
deleted file mode 100644
index 80b44e0c6fd..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoReply.dat
deleted file mode 100644
index 57a656c9b2d..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.1-java-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-WrongDistributionReply.dat
deleted file mode 100644
index 0dbe13225ae..00000000000
--- a/documentapi/test/crosslanguagefiles/5.1-java-WrongDistributionReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateMessage.dat
deleted file mode 100644
index 95e663088c3..00000000000
--- a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateReply.dat
deleted file mode 100644
index 216db17f80e..00000000000
--- a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateMessage.dat
deleted file mode 100644
index 95e663088c3..00000000000
--- a/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateReply.dat
deleted file mode 100644
index 216db17f80e..00000000000
--- a/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateMessage.dat
deleted file mode 100644
index 95e663088c3..00000000000
--- a/documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateReply.dat
deleted file mode 100644
index 216db17f80e..00000000000
--- a/documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateReply.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-1.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-1.dat
index 0107dd5f350..0107dd5f350 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-1.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-1.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-2.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-2.dat
index 57187093f28..57187093f28 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-2.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-2.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-3.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-3.dat
index 6a516d38d17..6a516d38d17 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-3.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-3.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-1.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-1.dat
index dbf830c9365..dbf830c9365 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-1.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-1.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-2.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-2.dat
index 094143cf78d..094143cf78d 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-2.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-2.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-3.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-3.dat
index 3341d74052b..3341d74052b 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-3.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-3.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-4.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-4.dat
index 8aaaefff491..8aaaefff491 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-4.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-4.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-5.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-5.dat
index e66ed1f07d4..e66ed1f07d4 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-5.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-5.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-1.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-1.dat
index 988f9fdab1f..988f9fdab1f 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-1.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-1.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-2.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-2.dat
index ac277d09643..ac277d09643 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-2.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-2.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-3.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-3.dat
index 03b49c8a0ac..03b49c8a0ac 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-3.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-3.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-4.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-4.dat
index d52e574ea44..d52e574ea44 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-4.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-4.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-5.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-5.dat
index e68654e9941..e68654e9941 100644
--- a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-5.dat
+++ b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-5.dat
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateMessage.dat
deleted file mode 100644
index 95e663088c3..00000000000
--- a/documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateMessage.dat
+++ /dev/null
Binary files differ
diff --git a/documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateReply.dat
deleted file mode 100644
index 216db17f80e..00000000000
--- a/documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateReply.dat
+++ /dev/null
Binary files differ
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java
index 252f6c5bd12..43ccd6d48a8 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java
@@ -48,23 +48,31 @@ public class SimpleAdapterFactory implements AdapterFactory {
DocumentId docId = upd.getId();
Document complete = new Document(docType, upd.getId());
for (FieldPathUpdate fieldUpd : upd) {
- if (FieldPathUpdateHelper.isComplete(fieldUpd)) {
- // A 'complete' field path update is basically a regular top-level field update
- // in wolf's clothing. Convert it to a regular field update to be friendlier
- // towards the search core backend.
- FieldPathUpdateHelper.applyUpdate(fieldUpd, complete);
- } else {
- ret.add(new IdentityFieldPathUpdateAdapter(fieldUpd, newDocumentAdapter(complete, true)));
+ try {
+ if (FieldPathUpdateHelper.isComplete(fieldUpd)) {
+ // A 'complete' field path update is basically a regular top-level field update
+ // in wolf's clothing. Convert it to a regular field update to be friendlier
+ // towards the search core backend.
+ FieldPathUpdateHelper.applyUpdate(fieldUpd, complete);
+ } else {
+ ret.add(new IdentityFieldPathUpdateAdapter(fieldUpd, newDocumentAdapter(complete, true)));
+ }
+ } catch (NullPointerException e) {
+ throw new IllegalArgumentException("Exception during handling of update '" + fieldUpd + "' to field '" + fieldUpd.getFieldPath() + "'", e);
}
}
for (FieldUpdate fieldUpd : upd.getFieldUpdates()) {
Field field = fieldUpd.getField();
for (ValueUpdate valueUpd : fieldUpd.getValueUpdates()) {
- if (FieldUpdateHelper.isComplete(field, valueUpd)) {
- FieldUpdateHelper.applyUpdate(field, valueUpd, complete);
- } else {
- Document partial = FieldUpdateHelper.newPartialDocument(docType, docId, field, valueUpd);
- ret.add(FieldUpdateAdapter.fromPartialUpdate(expressionSelector.selectExpression(docType, field.getName()),newDocumentAdapter(partial, true), valueUpd));
+ try {
+ if (FieldUpdateHelper.isComplete(field, valueUpd)) {
+ FieldUpdateHelper.applyUpdate(field, valueUpd, complete);
+ } else {
+ Document partial = FieldUpdateHelper.newPartialDocument(docType, docId, field, valueUpd);
+ ret.add(FieldUpdateAdapter.fromPartialUpdate(expressionSelector.selectExpression(docType, field.getName()), newDocumentAdapter(partial, true), valueUpd));
+ }
+ } catch (NullPointerException e) {
+ throw new IllegalArgumentException("Exception during handling of update '" + valueUpd + "' to field '" + field + "'", e);
}
}
}
diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java
index 5a3ccfed490..9321f8e290d 100644
--- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java
+++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java
@@ -8,6 +8,7 @@ import org.glassfish.jersey.client.proxy.WebResourceFactory;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
+import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.ClientRequestFilter;
import javax.ws.rs.client.WebTarget;
@@ -16,17 +17,17 @@ import javax.ws.rs.core.UriBuilder;
import java.util.Collections;
/**
- * @author bakksjo
+ * Factory for creating Jersey clients from a JAX-RS resource interface.
+ *
+ * @author Oyvind Bakksjo
*/
public class JerseyJaxRsClientFactory implements JaxRsClientFactory {
+
private static final int DEFAULT_CONNECT_TIMEOUT_MS = 30000;
private static final int DEFAULT_READ_TIMEOUT_MS = 30000;
- private final int connectTimeoutMs;
- private final int readTimeoutMs;
- private final SSLContext sslContext;
- private final String userAgent;
- private final HostnameVerifier hostnameVerifier;
+ // Client is a heavy-weight object with a finalizer so we create only one and re-use it
+ private final Client client;
public JerseyJaxRsClientFactory() {
this(DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_READ_TIMEOUT_MS);
@@ -36,33 +37,23 @@ public class JerseyJaxRsClientFactory implements JaxRsClientFactory {
this(DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_READ_TIMEOUT_MS, sslContext, hostnameVerifier, userAgent);
}
- public JerseyJaxRsClientFactory(final int connectTimeoutMs, final int readTimeoutMs) {
+ public JerseyJaxRsClientFactory(int connectTimeoutMs, int readTimeoutMs) {
this(connectTimeoutMs, readTimeoutMs, null, null, null);
}
public JerseyJaxRsClientFactory(int connectTimeoutMs, int readTimeoutMs, SSLContext sslContext,
HostnameVerifier hostnameVerifier, String userAgent) {
- this.connectTimeoutMs = connectTimeoutMs;
- this.readTimeoutMs = readTimeoutMs;
- this.sslContext = sslContext;
- this.hostnameVerifier = hostnameVerifier;
- this.userAgent = userAgent;
- }
-
- /**
- * Contains some workarounds for HTTP/JAX-RS/Jersey issues. See:
- * https://jersey.java.net/apidocs/latest/jersey/org/glassfish/jersey/client/ClientProperties.html#SUPPRESS_HTTP_COMPLIANCE_VALIDATION
- * https://jersey.java.net/apidocs/latest/jersey/org/glassfish/jersey/client/HttpUrlConnectorProvider.html#SET_METHOD_WORKAROUND
- */
- @Override
- public <T> T createClient(final Class<T> apiClass, final HostName hostName, final int port, final String pathPrefix, String scheme) {
- final UriBuilder uriBuilder = UriBuilder.fromPath(pathPrefix).host(hostName.s()).port(port).scheme(scheme);
+ /*
+ * Configure client with some workarounds for HTTP/JAX-RS/Jersey issues. See:
+ * https://jersey.java.net/apidocs/latest/jersey/org/glassfish/jersey/client/ClientProperties.html#SUPPRESS_HTTP_COMPLIANCE_VALIDATION
+ * https://jersey.java.net/apidocs/latest/jersey/org/glassfish/jersey/client/HttpUrlConnectorProvider.html#SET_METHOD_WORKAROUND
+ */
ClientBuilder builder = ClientBuilder.newBuilder()
- .property(ClientProperties.CONNECT_TIMEOUT, connectTimeoutMs)
- .property(ClientProperties.READ_TIMEOUT, readTimeoutMs)
- .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true) // Allow empty PUT. TODO: Fix API.
- .property(HttpUrlConnectorProvider.SET_METHOD_WORKAROUND, true) // Allow e.g. PATCH method.
- .property(ClientProperties.FOLLOW_REDIRECTS, true);
+ .property(ClientProperties.CONNECT_TIMEOUT, connectTimeoutMs)
+ .property(ClientProperties.READ_TIMEOUT, readTimeoutMs)
+ .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true) // Allow empty PUT. TODO: Fix API.
+ .property(HttpUrlConnectorProvider.SET_METHOD_WORKAROUND, true) // Allow e.g. PATCH method.
+ .property(ClientProperties.FOLLOW_REDIRECTS, true);
if (sslContext != null) {
builder.sslContext(sslContext);
}
@@ -70,11 +61,16 @@ public class JerseyJaxRsClientFactory implements JaxRsClientFactory {
builder.hostnameVerifier(hostnameVerifier);
}
if (userAgent != null) {
- builder.register((ClientRequestFilter) context ->
- context.getHeaders().put(HttpHeaders.USER_AGENT, Collections.singletonList(userAgent)));
+ builder.register((ClientRequestFilter) context -> context.getHeaders().put(HttpHeaders.USER_AGENT, Collections.singletonList(userAgent)));
}
- final WebTarget target = builder.build().target(uriBuilder);
- // TODO: Check if this fills up non-heap memory with loaded classes.
+ this.client = builder.build();
+ }
+
+ @Override
+ public <T> T createClient(Class<T> apiClass, HostName hostName, int port, String pathPrefix, String scheme) {
+ UriBuilder uriBuilder = UriBuilder.fromPath(pathPrefix).host(hostName.s()).port(port).scheme(scheme);
+ WebTarget target = client.target(uriBuilder);
return WebResourceFactory.newResource(apiClass, target);
}
+
}
diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml
index efa4e62affd..1a0c27c71a6 100644
--- a/jdisc_core/pom.xml
+++ b/jdisc_core/pom.xml
@@ -232,6 +232,7 @@
<configuration>
<mainClass>com.yahoo.jdisc.core.ExportPackages</mainClass>
<classpathScope>test</classpathScope>
+ <cleanupDaemonThreads>false</cleanupDaemonThreads>
<arguments>
<argument>${exportPackagesFile}</argument>
<argument>${project.build.directory}/dependency/commons-daemon.jar</argument>
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java
index 86ab016bded..e8e6373f2ed 100644
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java
+++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java
@@ -1,20 +1,31 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.jdisc.core;
-import org.apache.felix.framework.util.Util;
+import com.yahoo.io.IOUtils;
+import com.yahoo.yolean.Exceptions;
+import org.apache.felix.framework.Felix;
+import org.osgi.framework.BundleException;
import org.osgi.framework.Constants;
+import org.osgi.framework.launch.Framework;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
import java.util.jar.JarInputStream;
+import java.util.logging.Logger;
/**
* @author Simon Thoresen Hult
+ * @author gjoranv
*/
public class ExportPackages {
+ private static final Logger log = Logger.getLogger(ExportPackages.class.getName());
public static final String PROPERTIES_FILE = "/exportPackages.properties";
public static final String EXPORT_PACKAGES = "exportPackages";
@@ -75,7 +86,38 @@ public class ExportPackages {
}
public static String getSystemPackages() {
- return Util.getDefaultProperty(null, "org.osgi.framework.system.packages");
+ File cache;
+ try {
+ cache = Files.createTempDirectory("felix-cache").toAbsolutePath().toFile();
+ } catch (IOException e) {
+ throw new RuntimeException("Could not create temp bundle-cache.", e);
+ }
+ Framework framework = new Felix(felixCacheParams(cache.getAbsolutePath()));
+ try {
+ framework.init();
+ framework.start();
+ return framework.getHeaders().get(Constants.EXPORT_PACKAGE);
+ } catch (BundleException e) {
+ throw new RuntimeException("Failed retrieving exported system packages. ", e);
+ } finally {
+ try {
+ framework.stop();
+ framework.waitForStop(10000);
+ } catch (BundleException | InterruptedException e) {
+ log.warning("Failed to stop Felix framework:\n" + Exceptions.toMessageString(e));
+ }
+ if (! IOUtils.recursiveDeleteDir(cache)) {
+ log.warning("Failed to delete temp dir, must be deleted manually: " + cache.getAbsolutePath());
+ }
+ }
+ }
+
+ private static Map<String, Object> felixCacheParams(String cache) {
+ Map<String, Object> params = new HashMap<>();
+ params.put("felix.cache.profiledir", cache);
+ params.put("felix.cache.dir", cache);
+ params.put(Constants.FRAMEWORK_STORAGE, cache);
+ return params;
}
private static String getExportedPackages(String argument) throws IOException {
@@ -92,4 +134,5 @@ public class ExportPackages {
return jar.getManifest().getMainAttributes().getValue(Constants.EXPORT_PACKAGE);
}
}
+
}
diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java
index 55fca97841a..6836ebde496 100644
--- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java
+++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java
@@ -29,17 +29,19 @@ public class FelixParamsTestCase {
@Test
public void requireThatSystemPackagesAreNotReplaced() {
+ String systemPackages = ExportPackages.getSystemPackages();
+
FelixParams params = new FelixParams();
Map<String, String> config = params.toConfig();
assertNotNull(config);
String str = config.get(Constants.FRAMEWORK_SYSTEMPACKAGES);
assertNotNull(str);
- assertTrue(str.contains(ExportPackages.getSystemPackages()));
+ assertTrue(str.contains(systemPackages));
params.exportPackage("foo");
assertNotNull(config = params.toConfig());
assertNotNull(str = config.get(Constants.FRAMEWORK_SYSTEMPACKAGES));
- assertTrue(str.contains(ExportPackages.getSystemPackages()));
+ assertTrue(str.contains(systemPackages));
assertTrue(str.contains("foo"));
}
diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java
index 6388643148c..03e56d7c26d 100644
--- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java
+++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java
@@ -22,7 +22,7 @@ public class BundleActivatorIntegrationTest {
OsgiFramework osgi = driver.osgiFramework();
BundleInstaller installer = new BundleInstaller(driver.osgiFramework());
Bundle bundle = installer.installAndStart("my-bundle-activator.jar").get(0);
- Class<?> serviceClass = bundle.loadClass("com.yahoo.jdisc.bundle.MyService");
+ Class<?> serviceClass = bundle.loadClass("com.yahoo.jdisc.bundle.my_act.MyService");
assertNotNull(serviceClass);
BundleContext ctx = osgi.bundleContext();
ServiceReference<?> serviceRef = ctx.getServiceReference(serviceClass.getName());
@@ -37,7 +37,7 @@ public class BundleActivatorIntegrationTest {
public void requireThatApplicationBundleActivatorHasAccessToCurrentContainer() throws Exception {
TestDriver driver = TestDriver.newApplicationBundleInstance("app-g-act.jar", false);
OsgiFramework osgi = driver.osgiFramework();
- Class<?> serviceClass = osgi.bundles().get(1).loadClass("com.yahoo.jdisc.bundle.MyService");
+ Class<?> serviceClass = osgi.bundles().get(1).loadClass("com.yahoo.jdisc.bundle.g_act.MyService");
assertNotNull(serviceClass);
BundleContext ctx = osgi.bundleContext();
ServiceReference<?> serviceRef = ctx.getServiceReference(serviceClass.getName());
diff --git a/jdisc_core_test/test_bundles/app-g-act/pom.xml b/jdisc_core_test/test_bundles/app-g-act/pom.xml
index b91b95f9fdd..aaa740efade 100644
--- a/jdisc_core_test/test_bundles/app-g-act/pom.xml
+++ b/jdisc_core_test/test_bundles/app-g-act/pom.xml
@@ -23,13 +23,13 @@
<configuration>
<instructions>
<X-JDisc-Application>
- com.yahoo.jdisc.bundle.ApplicationG
+ com.yahoo.jdisc.bundle.g_act.ApplicationG
</X-JDisc-Application>
<Bundle-Activator>
- com.yahoo.jdisc.bundle.MyBundleActivator
+ com.yahoo.jdisc.bundle.g_act.MyBundleActivator
</Bundle-Activator>
<Export-Package>
- com.yahoo.jdisc.bundle
+ com.yahoo.jdisc.bundle.g_act
</Export-Package>
</instructions>
</configuration>
diff --git a/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/ApplicationG.java b/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/ApplicationG.java
index 9b4d7501ac0..02a9cd5a542 100644
--- a/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/ApplicationG.java
+++ b/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/ApplicationG.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.jdisc.bundle;
+package com.yahoo.jdisc.bundle.g_act;
import com.yahoo.jdisc.application.Application;
diff --git a/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/MyBundleActivator.java b/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/MyBundleActivator.java
index 51a95b8e3a9..c60f4939409 100644
--- a/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/MyBundleActivator.java
+++ b/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/MyBundleActivator.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.jdisc.bundle;
+package com.yahoo.jdisc.bundle.g_act;
import com.yahoo.jdisc.service.CurrentContainer;
import org.osgi.framework.BundleActivator;
diff --git a/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/MyService.java b/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/MyService.java
index a5cb9be9a47..518f7304100 100644
--- a/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/MyService.java
+++ b/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/MyService.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.jdisc.bundle;
+package com.yahoo.jdisc.bundle.g_act;
import com.yahoo.jdisc.service.AbstractServerProvider;
import com.yahoo.jdisc.service.CurrentContainer;
diff --git a/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml b/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml
index 531af08b915..8931c26a73e 100644
--- a/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml
+++ b/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml
@@ -23,10 +23,10 @@
<configuration>
<instructions>
<Bundle-Activator>
- com.yahoo.jdisc.bundle.MyBundleActivator
+ com.yahoo.jdisc.bundle.my_act.MyBundleActivator
</Bundle-Activator>
<Export-Package>
- com.yahoo.jdisc.bundle
+ com.yahoo.jdisc.bundle.my_act
</Export-Package>
</instructions>
</configuration>
diff --git a/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/MyBundleActivator.java b/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/my_act/MyBundleActivator.java
index b4af03bdf60..4212fc2fd09 100644
--- a/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/MyBundleActivator.java
+++ b/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/my_act/MyBundleActivator.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.jdisc.bundle;
+package com.yahoo.jdisc.bundle.my_act;
import com.yahoo.jdisc.service.CurrentContainer;
import org.osgi.framework.BundleActivator;
diff --git a/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/MyService.java b/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/my_act/MyService.java
index 8eb1543776d..b24ee137f59 100644
--- a/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/MyService.java
+++ b/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/my_act/MyService.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.jdisc.bundle;
+package com.yahoo.jdisc.bundle.my_act;
/**
* @author Simon Thoresen Hult
diff --git a/jrt/src/com/yahoo/jrt/CryptoEngine.java b/jrt/src/com/yahoo/jrt/CryptoEngine.java
index 2ef936ec7ed..c27aba73873 100644
--- a/jrt/src/com/yahoo/jrt/CryptoEngine.java
+++ b/jrt/src/com/yahoo/jrt/CryptoEngine.java
@@ -2,10 +2,10 @@
package com.yahoo.jrt;
-import com.yahoo.security.tls.TransportSecurityOptions;
+import com.yahoo.security.tls.TransportSecurityUtils;
+import com.yahoo.security.tls.TransportSecurityUtils.MixedMode;
import java.nio.channels.SocketChannel;
-import java.nio.file.Paths;
/**
@@ -16,12 +16,22 @@ import java.nio.file.Paths;
**/
public interface CryptoEngine {
public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer);
- static public CryptoEngine createDefault() { // TODO Move this logic to a dedicated factory class
- String tlsConfigParameter = System.getenv("VESPA_TLS_CONFIG_FILE");
- if (tlsConfigParameter != null && !tlsConfigParameter.isEmpty()) {
- return new TlsCryptoEngine(TransportSecurityOptions.fromJsonFile(Paths.get(tlsConfigParameter)));
- } else {
+ static public CryptoEngine createDefault() {
+ if (!TransportSecurityUtils.isTransportSecurityEnabled()) {
return new NullCryptoEngine();
}
+ TlsCryptoEngine tlsCryptoEngine = new TlsCryptoEngine(TransportSecurityUtils.getOptions().get());
+ if (!TransportSecurityUtils.isInsecureMixedModeEnabled()) {
+ return tlsCryptoEngine;
+ }
+ MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode().get();
+ switch (mixedMode) {
+ case PLAINTEXT_CLIENT_MIXED_SERVER:
+ return new MaybeTlsCryptoEngine(tlsCryptoEngine, false);
+ case TLS_CLIENT_MIXED_SERVER:
+ return new MaybeTlsCryptoEngine(tlsCryptoEngine, true);
+ default:
+ throw new IllegalArgumentException(mixedMode.toString());
+ }
}
}
diff --git a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java
index 8cb560246e8..6664934799c 100644
--- a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java
+++ b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java
@@ -7,28 +7,28 @@ import java.nio.channels.SocketChannel;
* A crypto engine that supports both tls encrypted connections and
* unencrypted connections. The use of tls for incoming connections is
* auto-detected using clever heuristics. The use of tls for outgoing
- * connections is controlled by the useTls flag given to the
+ * connections is controlled by the useTlsWhenClient flag given to the
* constructor.
**/
public class MaybeTlsCryptoEngine implements CryptoEngine {
private final TlsCryptoEngine tlsEngine;
- private final boolean useTls;
+ private final boolean useTlsWhenClient;
- public MaybeTlsCryptoEngine(TlsCryptoEngine tlsEngine, boolean useTls) {
+ public MaybeTlsCryptoEngine(TlsCryptoEngine tlsEngine, boolean useTlsWhenClient) {
this.tlsEngine = tlsEngine;
- this.useTls = useTls;
+ this.useTlsWhenClient = useTlsWhenClient;
}
@Override public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) {
if (isServer) {
return new MaybeTlsCryptoSocket(channel, tlsEngine);
- } else if (useTls) {
+ } else if (useTlsWhenClient) {
return tlsEngine.createCryptoSocket(channel, false);
} else {
return new NullCryptoSocket(channel);
}
}
- @Override public String toString() { return "MaybeTlsCryptoEngine(useTls:" + useTls + ")"; }
+ @Override public String toString() { return "MaybeTlsCryptoEngine(useTlsWhenClient:" + useTlsWhenClient + ")"; }
}
diff --git a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java
index 7cedbcda9a1..1cf3dfd1261 100644
--- a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java
+++ b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java
@@ -16,13 +16,13 @@ import java.nio.channels.SocketChannel;
**/
public class MaybeTlsCryptoSocket implements CryptoSocket {
- private static final int snoop_size = 9;
+ private static final int SNOOP_SIZE = 9;
private CryptoSocket socket;
// 'data' is the first 9 bytes received from the client
public static boolean looksLikeTlsToMe(byte[] data) {
- if (data.length != snoop_size) {
+ if (data.length != SNOOP_SIZE) {
return false; // wrong data size for tls detection
}
if (data[0] != 22) {
@@ -67,13 +67,13 @@ public class MaybeTlsCryptoSocket implements CryptoSocket {
@Override public HandshakeResult handshake() throws IOException {
if (factory != null) {
- channel().read(buffer.getWritable(snoop_size));
- if (buffer.bytes() < snoop_size) {
+ channel().read(buffer.getWritable(SNOOP_SIZE));
+ if (buffer.bytes() < SNOOP_SIZE) {
return HandshakeResult.NEED_READ;
}
- byte[] data = new byte[snoop_size];
+ byte[] data = new byte[SNOOP_SIZE];
ByteBuffer src = buffer.getReadable();
- for (int i = 0; i < snoop_size; i++) {
+ for (int i = 0; i < SNOOP_SIZE; i++) {
data[i] = src.get(i);
}
if (looksLikeTlsToMe(data)) {
diff --git a/jrt/tests/com/yahoo/jrt/TlsDetectionTest.java b/jrt/tests/com/yahoo/jrt/TlsDetectionTest.java
index 9bd37e25772..47a8a20deab 100644
--- a/jrt/tests/com/yahoo/jrt/TlsDetectionTest.java
+++ b/jrt/tests/com/yahoo/jrt/TlsDetectionTest.java
@@ -1,20 +1,18 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.jrt;
+import static org.junit.Assert.assertEquals;
+
public class TlsDetectionTest {
- static private String message(byte[] data, boolean actual) {
- String msg = "[";
+ static private String message(byte[] data) {
+ String msg = "isTls([";
String delimiter = "";
for (byte b: data) {
msg += delimiter + (b & 0xff);
delimiter = ", ";
}
- if (actual) {
- msg += "] wrongfully detected as tls";
- } else {
- msg += "] wrongfully rejected as not tls";
- }
+ msg += "])";
return msg;
}
@@ -23,10 +21,7 @@ public class TlsDetectionTest {
for (int i = 0; i < data.length; i++) {
data[i] = (byte) values[i];
}
- boolean actual = MaybeTlsCryptoSocket.looksLikeTlsToMe(data);
- if(actual != expect) {
- throw new AssertionError(message(data, actual));
- }
+ assertEquals(message(data), expect, MaybeTlsCryptoSocket.looksLikeTlsToMe(data));
}
@org.junit.Test public void testValidHandshake() {
diff --git a/metrics/src/vespa/metrics/countmetric.h b/metrics/src/vespa/metrics/countmetric.h
index dd4c9e6fc92..912fc52449b 100644
--- a/metrics/src/vespa/metrics/countmetric.h
+++ b/metrics/src/vespa/metrics/countmetric.h
@@ -78,11 +78,6 @@ public:
void set(T value);
void inc(T value = 1);
void dec(T value = 1);
- CountMetric & operator++() { inc(); return *this; }
- CountMetric & operator--() { dec(); return *this; }
-
- CountMetric operator++(int);
- CountMetric operator--(int);
CountMetric & operator+=(const CountMetric &);
CountMetric & operator-=(const CountMetric &);
diff --git a/metrics/src/vespa/metrics/countmetric.hpp b/metrics/src/vespa/metrics/countmetric.hpp
index 516f6c660b6..e150ce0d632 100644
--- a/metrics/src/vespa/metrics/countmetric.hpp
+++ b/metrics/src/vespa/metrics/countmetric.hpp
@@ -75,22 +75,6 @@ CountMetric<T, SumOnAdd>::operator-=(const CountMetric<T, SumOnAdd>& other)
}
template <typename T, bool SumOnAdd>
-CountMetric<T, SumOnAdd>
-CountMetric<T, SumOnAdd>:: operator++(int) {
- CountMetric tmp(*this);
- inc();
- return tmp;
-}
-
-template <typename T, bool SumOnAdd>
-CountMetric<T, SumOnAdd>
-CountMetric<T, SumOnAdd>::operator--(int) {
- CountMetric tmp(*this);
- inc();
- return tmp;
-}
-
-template <typename T, bool SumOnAdd>
void
CountMetric<T, SumOnAdd>::set(T value)
{
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerNetworking.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerNetworking.java
index 7678ad8169a..7f0ade2e500 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerNetworking.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerNetworking.java
@@ -28,7 +28,7 @@ public enum DockerNetworking {
}
public static DockerNetworking from(String cloud, NodeType nodeType, boolean hostAdmin) {
- if (cloud.equals("AWS")) {
+ if (cloud.equalsIgnoreCase("aws")) {
return DockerNetworking.NPT;
} else if (nodeType == NodeType.confighost || nodeType == NodeType.proxyhost) {
return DockerNetworking.HOST_NETWORK;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java
index 4a19e5fe215..6d1d51ead43 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java
@@ -30,18 +30,21 @@ public interface DockerOperations {
ProcessResult executeCommandInNetworkNamespace(ContainerName containerName, String... command);
- void resumeNode(ContainerName containerName);
-
- void restartVespaOnNode(ContainerName containerName);
- void stopServicesOnNode(ContainerName containerName);
+ /** Resume node. Resuming a node means that it is ready to take on traffic. */
+ void resumeNode(ContainerName containerName);
/**
- * Try to suspend node. Suspending a node means the node should be taken offline,
- * such that maintenance can be done of the node (upgrading, rebooting, etc),
- * and such that we will start serving again as soon as possible afterwards.
+ * Suspend node. Suspending a node means the node should be taken temporarly offline,
+ * such that maintenance of the node can be done (upgrading, rebooting, etc).
*/
- void trySuspendNode(ContainerName containerName);
+ void suspendNode(ContainerName containerName);
+
+ void restartVespa(ContainerName containerName);
+
+ void startServices(ContainerName containerName);
+
+ void stopServices(ContainerName containerName);
Optional<ContainerStats> getContainerStats(ContainerName containerName);
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
index 45c2e93c93e..7809f263ad9 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
@@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.dockerapi.Docker;
import com.yahoo.vespa.hosted.dockerapi.DockerImage;
import com.yahoo.vespa.hosted.dockerapi.DockerNetworkCreator;
import com.yahoo.vespa.hosted.dockerapi.ProcessResult;
-import com.yahoo.vespa.hosted.dockerapi.exception.ContainerNotFoundException;
import com.yahoo.vespa.hosted.node.admin.component.Environment;
import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData;
@@ -292,29 +291,26 @@ public class DockerOperationsImpl implements DockerOperations {
}
@Override
- public void restartVespaOnNode(ContainerName containerName) {
+ public void suspendNode(ContainerName containerName) {
+ executeCommandInContainer(containerName, nodeProgram, "suspend");
+ }
+
+ @Override
+ public void restartVespa(ContainerName containerName) {
executeCommandInContainer(containerName, nodeProgram, "restart-vespa");
}
@Override
- public void stopServicesOnNode(ContainerName containerName) {
- executeCommandInContainer(containerName, nodeProgram, "stop");
+ public void startServices(ContainerName containerName) {
+ executeCommandInContainer(containerName, nodeProgram, "start");
}
@Override
- public void trySuspendNode(ContainerName containerName) {
- try {
- executeCommandInContainer(containerName, nodeProgram, "suspend");
- } catch (ContainerNotFoundException e) {
- throw e;
- } catch (RuntimeException e) {
- PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName);
- // It's bad to continue as-if nothing happened, but on the other hand if we do not proceed to
- // remove container, we will not be able to upgrade to fix any problems in the suspend logic!
- logger.warning("Failed trying to suspend container " + containerName.asString(), e);
- }
+ public void stopServices(ContainerName containerName) {
+ executeCommandInContainer(containerName, nodeProgram, "stop");
}
+
@Override
public Optional<ContainerStats> getContainerStats(ContainerName containerName) {
return docker.getContainerStats(containerName);
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
index ba8a2e55587..12c1b9bcf11 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java
@@ -145,7 +145,10 @@ public class NodeAdminImpl implements NodeAdmin {
// Each container may spend 1-1:30 minutes stopping
nodeAgentsByHostname.values().parallelStream()
.filter(nodeAgent -> hostnames.contains(nodeAgent.getHostname()))
- .forEach(NodeAgent::stopServices);
+ .forEach(nodeAgent -> {
+ nodeAgent.suspend();
+ nodeAgent.stopServices();
+ });
}
public int getNumberOfNodeAgents() {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java
index 92c44969d5e..9b759b208eb 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java
@@ -18,9 +18,18 @@ public interface NodeAgent {
*/
boolean setFrozen(boolean frozen);
+ /**
+ * Stop services running on node. Depending on the state of the node, {@link #suspend()} might need to be
+ * called before calling this method.
+ */
void stopServices();
/**
+ * Suspend node. Take node offline (e.g. take node out of VIP, drain traffic, prepare for restart etc.)
+ */
+ void suspend();
+
+ /**
* Returns a map containing all relevant NodeAgent variables and their current values.
*/
Map<String, Object> debugInfo();
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index ad38306547d..36a2672b1a6 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -88,7 +88,8 @@ public class NodeAgentImpl implements NodeAgent {
private Consumer<String> serviceRestarter;
private Optional<Future<?>> currentFilebeatRestarter = Optional.empty();
- private boolean resumeScriptRun = false;
+ private boolean hasResumedNode = false;
+ private boolean hasStartedServices = false;
/**
* ABSENT means container is definitely absent - A container that was absent will not suddenly appear without
@@ -216,17 +217,26 @@ public class NodeAgentImpl implements NodeAgent {
*/
protected void verifyHealth(NodeSpec node) { }
- void runLocalResumeScriptIfNeeded(NodeSpec node) {
- if (! resumeScriptRun) {
- storageMaintainer.writeMetricsConfig(containerName, node);
- storageMaintainer.writeFilebeatConfig(containerName, node);
- stopFilebeatSchedulerIfNeeded();
- currentFilebeatRestarter = Optional.of(filebeatRestarter.scheduleWithFixedDelay(
- () -> serviceRestarter.accept("filebeat"), 1, 1, TimeUnit.DAYS));
+ void startServicesIfNeeded() {
+ if (!hasStartedServices) {
+ logger.info("Starting services");
+ dockerOperations.startServices(containerName);
+ hasStartedServices = true;
+ }
+ }
+
+ void resumeNodeIfNeeded(NodeSpec node) {
+ if (!hasResumedNode) {
+ if (!currentFilebeatRestarter.isPresent()) {
+ storageMaintainer.writeMetricsConfig(containerName, node);
+ storageMaintainer.writeFilebeatConfig(containerName, node);
+ currentFilebeatRestarter = Optional.of(filebeatRestarter.scheduleWithFixedDelay(
+ () -> serviceRestarter.accept("filebeat"), 1, 1, TimeUnit.DAYS));
+ }
logger.debug("Starting optional node program resume command");
dockerOperations.resumeNode(containerName);
- resumeScriptRun = true;
+ hasResumedNode = true;
}
}
@@ -260,7 +270,8 @@ public class NodeAgentImpl implements NodeAgent {
dockerOperations.startContainer(containerName);
lastCpuMetric = new CpuUsageReporter();
- resumeScriptRun = false;
+ hasStartedServices = true; // Automatically started with the container
+ hasResumedNode = false;
logger.info("Container successfully started, new containerState is " + containerState);
}
@@ -293,7 +304,7 @@ public class NodeAgentImpl implements NodeAgent {
logger.info("Restarting services");
// Since we are restarting the services we need to suspend the node.
orchestratorSuspendNode();
- dockerOperations.restartVespaOnNode(containerName);
+ dockerOperations.restartVespa(containerName);
}
}
@@ -302,13 +313,29 @@ public class NodeAgentImpl implements NodeAgent {
logger.info("Stopping services");
if (containerState == ABSENT) return;
try {
- dockerOperations.trySuspendNode(containerName);
- dockerOperations.stopServicesOnNode(containerName);
+ hasStartedServices = hasResumedNode = false;
+ dockerOperations.stopServices(containerName);
} catch (ContainerNotFoundException e) {
containerState = ABSENT;
}
}
+ @Override
+ public void suspend() {
+ logger.info("Suspending services on node");
+ if (containerState == ABSENT) return;
+ try {
+ hasResumedNode = false;
+ dockerOperations.suspendNode(containerName);
+ } catch (ContainerNotFoundException e) {
+ containerState = ABSENT;
+ } catch (RuntimeException e) {
+ // It's bad to continue as-if nothing happened, but on the other hand if we do not proceed to
+ // remove container, we will not be able to upgrade to fix any problems in the suspend logic!
+ logger.warning("Failed trying to suspend container " + containerName.asString(), e);
+ }
+ }
+
private Optional<String> shouldRemoveContainer(NodeSpec node, Container existingContainer) {
final Node.State nodeState = node.getState();
if (nodeState == Node.State.dirty || nodeState == Node.State.provisioned) {
@@ -344,6 +371,9 @@ public class NodeAgentImpl implements NodeAgent {
}
try {
+ if (node.getState() != Node.State.dirty) {
+ suspend();
+ }
stopServices();
} catch (Exception e) {
logger.info("Failed stopping services, ignoring", e);
@@ -484,7 +514,8 @@ public class NodeAgentImpl implements NodeAgent {
}
verifyHealth(node);
- runLocalResumeScriptIfNeeded(node);
+ startServicesIfNeeded();
+ resumeNodeIfNeeded(node);
athenzCredentialsMaintainer.converge();
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java
index 319207f9e95..a46defc991b 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java
@@ -194,9 +194,9 @@ public class RunInContainerTest {
// Allow stopping services in active nodes
doNothing().when(dockerOperationsMock)
- .trySuspendNode(eq(new ContainerName("host1")));
+ .suspendNode(eq(new ContainerName("host1")));
doNothing().when(dockerOperationsMock)
- .stopServicesOnNode(eq(new ContainerName("host1")));
+ .stopServices(eq(new ContainerName("host1")));
assertTrue(verifyWithRetries("suspend", false));
assertTrue(verifyWithRetries("suspend", true));
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java
index 56373dda2f8..9d5555e1f92 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java
@@ -49,13 +49,13 @@ public class AclMaintainerTest {
@Before
public void before() {
when(dockerOperations.getAllManagedContainers()).thenReturn(containerList);
- when(env.getCloud()).thenReturn("AWS");
+ when(env.getCloud()).thenReturn("aws");
when(env.getDockerNetworking()).thenReturn(DockerNetworking.NPT);
}
@Test
public void no_redirect_in_yahoo() {
- when(env.getCloud()).thenReturn("YAHOO");
+ when(env.getCloud()).thenReturn("yahoo");
when(env.getDockerNetworking()).thenReturn(DockerNetworking.MACVLAN);
Container container = addContainer("container1", "container1.host.com", Container.State.RUNNING);
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index e0031f9b9b3..635b50246f4 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -131,6 +131,7 @@ public class NodeAgentImplTest {
final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository);
// TODO: Verify this isn't run unless 1st time
+ inOrder.verify(dockerOperations, times(1)).startServices(eq(containerName));
inOrder.verify(dockerOperations, times(1)).resumeNode(eq(containerName));
inOrder.verify(orchestrator).resume(hostName);
}
@@ -159,6 +160,41 @@ public class NodeAgentImplTest {
verify(storageMaintainer, times(1)).removeOldFilesFromNode(eq(containerName));
}
+ @Test
+ public void startsAfterStoppingServices() {
+ final InOrder inOrder = inOrder(dockerOperations);
+ final NodeSpec node = nodeBuilder
+ .wantedDockerImage(dockerImage)
+ .currentDockerImage(dockerImage)
+ .state(Node.State.active)
+ .wantedVespaVersion(vespaVersion)
+ .vespaVersion(vespaVersion)
+ .build();
+
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
+ when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node));
+ when(storageMaintainer.getDiskUsageFor(eq(containerName))).thenReturn(Optional.of(187500000000L));
+
+ nodeAgent.converge();
+ inOrder.verify(dockerOperations, times(1)).startServices(eq(containerName));
+ inOrder.verify(dockerOperations, times(1)).resumeNode(eq(containerName));
+
+ nodeAgent.suspend();
+ nodeAgent.converge();
+ inOrder.verify(dockerOperations, never()).startServices(eq(containerName));
+ inOrder.verify(dockerOperations, times(1)).resumeNode(eq(containerName)); // Expect a resume, but no start services
+
+ // No new suspends/stops, so no need to resume/start
+ nodeAgent.converge();
+ inOrder.verify(dockerOperations, never()).startServices(eq(containerName));
+ inOrder.verify(dockerOperations, never()).resumeNode(eq(containerName));
+
+ nodeAgent.suspend();
+ nodeAgent.stopServices();
+ nodeAgent.converge();
+ inOrder.verify(dockerOperations, times(1)).startServices(eq(containerName));
+ inOrder.verify(dockerOperations, times(1)).resumeNode(eq(containerName));
+ }
@Test
public void absentContainerCausesStart() throws Exception {
@@ -184,6 +220,7 @@ public class NodeAgentImplTest {
nodeAgent.converge();
verify(dockerOperations, never()).removeContainer(any());
+ verify(dockerOperations, never()).startServices(any());
verify(orchestrator, never()).suspend(any(String.class));
final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository, aclMaintainer);
@@ -426,6 +463,8 @@ public class NodeAgentImplTest {
verify(dockerOperations, never()).createContainer(eq(containerName), any(), any());
verify(dockerOperations, never()).startContainer(eq(containerName));
+ verify(dockerOperations, never()).suspendNode(eq(containerName));
+ verify(dockerOperations, times(1)).stopServices(eq(containerName));
verify(orchestrator, never()).resume(any(String.class));
verify(orchestrator, never()).suspend(any(String.class));
// current Docker image and vespa version should be cleared
@@ -579,7 +618,7 @@ public class NodeAgentImplTest {
verify(dockerOperations, never()).removeContainer(any());
verify(dockerOperations, times(1)).createContainer(eq(containerName), eq(node), any());
verify(dockerOperations, times(1)).startContainer(eq(containerName));
- verify(nodeAgent, never()).runLocalResumeScriptIfNeeded(any());
+ verify(nodeAgent, never()).resumeNodeIfNeeded(any());
// The docker container was actually started and is running, but subsequent exec calls to set up
// networking failed
@@ -589,7 +628,7 @@ public class NodeAgentImplTest {
verify(dockerOperations, times(1)).removeContainer(any());
verify(dockerOperations, times(2)).createContainer(eq(containerName), eq(node), any());
verify(dockerOperations, times(2)).startContainer(eq(containerName));
- verify(nodeAgent, times(1)).runLocalResumeScriptIfNeeded(any());
+ verify(nodeAgent, times(1)).resumeNodeIfNeeded(any());
}
@Test
diff --git a/parent/pom.xml b/parent/pom.xml
index 96075974567..891eabd9c83 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -461,7 +461,7 @@
<artifactId>commons-pool</artifactId>
<version>1.5.6</version>
</dependency>
- <!-- Explicitly included to get Zookeeper version 3.4.12,
+ <!-- Explicitly included to get Zookeeper version 3.4.13,
can be excluded if you want the Zookeeper version
used by curator by default
-->
@@ -673,6 +673,11 @@
<artifactId>language-detector</artifactId>
<version>0.6</version>
</dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna</artifactId>
+ <version>${jna.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
@@ -680,6 +685,7 @@
<antlr.version>3.5.2</antlr.version>
<antlr4.version>4.5</antlr4.version>
<asm.version>6.2</asm.version>
+ <jna.version>4.5.2</jna.version>
<!-- Athenz dependencies. Make sure these dependencies matches those in Vespa's internal repositories -->
<athenz.version>1.7.43</athenz.version>
<commons-lang.version>2.6</commons-lang.version>
diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp
index ea2cc00c642..f4b80f7961c 100644
--- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp
+++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp
@@ -11,7 +11,6 @@
#include <vespa/storage/distributor/externaloperationhandler.h>
#include <vespa/storageapi/message/persistence.h>
#include <vespa/storage/distributor/operations/external/twophaseupdateoperation.h>
-#include <vespa/storageapi/message/batch.h>
#include <tests/distributor/distributortestutil.h>
#include <vespa/document/test/make_document_bucket.h>
#include <vespa/storage/distributor/distributor.h>
diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
index 369e820f987..61c55248370 100644
--- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
+++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp
@@ -12,12 +12,12 @@
#include <vespa/storage/persistence/filestorage/filestormanager.h>
#include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h>
#include <vespa/document/update/assignvalueupdate.h>
+#include <vespa/document/update/documentupdate.h>
#include <vespa/document/select/parser.h>
#include <vespa/vdslib/state/random.h>
#include <vespa/storageapi/message/bucketsplitting.h>
#include <vespa/persistence/dummyimpl/dummypersistence.h>
#include <vespa/persistence/spi/test.h>
-#include <vespa/storageapi/message/batch.h>
#include <vespa/config/common/exceptions.h>
#include <vespa/fastos/file.h>
diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp
index 40d561bd589..7f8b1b8f34a 100644
--- a/storage/src/tests/storageserver/documentapiconvertertest.cpp
+++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp
@@ -8,11 +8,11 @@
#include <vespa/document/repo/documenttyperepo.h>
#include <vespa/document/select/parser.h>
#include <vespa/document/test/make_document_bucket.h>
+#include <vespa/document/update/documentupdate.h>
#include <vespa/documentapi/documentapi.h>
#include <vespa/messagebus/emptyreply.h>
#include <vespa/storage/common/bucket_resolver.h>
#include <vespa/storage/storageserver/documentapiconverter.h>
-#include <vespa/storageapi/message/batch.h>
#include <vespa/storageapi/message/datagram.h>
#include <vespa/storageapi/message/persistence.h>
#include <vespa/storageapi/message/removelocation.h>
@@ -115,7 +115,6 @@ struct DocumentApiConverterTest : public CppUnit::TestFixture
void testCreateVisitorReplyLastBucket();
void testDestroyVisitor();
void testVisitorInfo();
- void testBatchDocumentUpdate();
void testStatBucket();
void testGetBucketList();
void testRemoveLocation();
@@ -133,7 +132,6 @@ struct DocumentApiConverterTest : public CppUnit::TestFixture
CPPUNIT_TEST(testCreateVisitorReplyLastBucket);
CPPUNIT_TEST(testDestroyVisitor);
CPPUNIT_TEST(testVisitorInfo);
- CPPUNIT_TEST(testBatchDocumentUpdate);
CPPUNIT_TEST(testStatBucket);
CPPUNIT_TEST(testGetBucketList);
CPPUNIT_TEST(testRemoveLocation);
@@ -321,58 +319,6 @@ DocumentApiConverterTest::testVisitorInfo()
}
void
-DocumentApiConverterTest::testBatchDocumentUpdate()
-{
- std::vector<document::DocumentUpdate::SP > updates;
-
- {
- document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test1"));
- auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, docId);
- updates.push_back(update);
- }
-
- {
- document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test2"));
- auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, docId);
- updates.push_back(update);
- }
-
- {
- document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test3"));
- auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, docId);
- updates.push_back(update);
- }
-
- auto msg = std::make_shared<documentapi::BatchDocumentUpdateMessage>(1234);
- for (std::size_t i = 0; i < updates.size(); ++i) {
- msg->addUpdate(updates[i]);
- }
-
- auto batchCmd = toStorageAPI<api::BatchDocumentUpdateCommand>(*msg);
- CPPUNIT_ASSERT_EQUAL(updates.size(), batchCmd->getUpdates().size());
- for (std::size_t i = 0; i < updates.size(); ++i) {
- CPPUNIT_ASSERT_EQUAL(*updates[i], *batchCmd->getUpdates()[i]);
- }
-
- api::BatchDocumentUpdateReply batchReply(*batchCmd);
- batchReply.getDocumentsNotFound().resize(3);
- batchReply.getDocumentsNotFound()[0] = true;
- batchReply.getDocumentsNotFound()[2] = true;
-
- std::unique_ptr<mbus::Reply> mbusReply = msg->createReply();
- documentapi::BatchDocumentUpdateReply* mbusBatchReply(
- dynamic_cast<documentapi::BatchDocumentUpdateReply*>(mbusReply.get()));
- CPPUNIT_ASSERT(mbusBatchReply != 0);
-
- _converter->transferReplyState(batchReply, *mbusReply);
-
- CPPUNIT_ASSERT_EQUAL(std::size_t(3), mbusBatchReply->getDocumentsNotFound().size());
- CPPUNIT_ASSERT(mbusBatchReply->getDocumentsNotFound()[0] == true);
- CPPUNIT_ASSERT(mbusBatchReply->getDocumentsNotFound()[1] == false);
- CPPUNIT_ASSERT(mbusBatchReply->getDocumentsNotFound()[2] == true);
-}
-
-void
DocumentApiConverterTest::testStatBucket()
{
documentapi::StatBucketMessage msg(BucketId(123), "");
diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp
index 0837b0d1463..d553b41de7c 100644
--- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp
+++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp
@@ -438,7 +438,7 @@ StorageBucketDBInitializer::registerBucket(const document::Bucket &bucket,
_system._nodeState, _system._nodeIndex, bucketId.stripUnused(),
lib::Distribution::IDEAL_DISK_EVEN_IF_DOWN));
if (disk != partition) {
- ++_metrics._wrongDisk;
+ _metrics._wrongDisk.inc();
}
_metrics._insertedCount.inc();
diff --git a/storage/src/vespa/storage/common/messagebucket.cpp b/storage/src/vespa/storage/common/messagebucket.cpp
index ecbad310a58..4a6f638262d 100644
--- a/storage/src/vespa/storage/common/messagebucket.cpp
+++ b/storage/src/vespa/storage/common/messagebucket.cpp
@@ -9,7 +9,6 @@
#include <vespa/storageapi/message/removelocation.h>
#include <vespa/storage/persistence/messages.h>
#include <vespa/storageapi/message/stat.h>
-#include <vespa/storageapi/message/batch.h>
#include <vespa/vespalib/util/exceptions.h>
@@ -31,8 +30,6 @@ getStorageMessageBucket(const api::StorageMessage& msg)
return static_cast<const api::RevertCommand&>(msg).getBucket();
case api::MessageType::STATBUCKET_ID:
return static_cast<const api::StatBucketCommand&>(msg).getBucket();
- case api::MessageType::BATCHPUTREMOVE_ID:
- return static_cast<const api::BatchPutRemoveCommand&>(msg).getBucket();
case api::MessageType::REMOVELOCATION_ID:
return static_cast<const api::RemoveLocationCommand&>(msg).getBucket();
case api::MessageType::CREATEBUCKET_ID:
diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp
index 2d490ea9923..1664dd0d9a1 100644
--- a/storage/src/vespa/storage/distributor/distributor.cpp
+++ b/storage/src/vespa/storage/distributor/distributor.cpp
@@ -616,8 +616,6 @@ bool is_client_request(const api::StorageMessage& msg) noexcept {
case api::MessageType::STATBUCKET_ID:
case api::MessageType::UPDATE_ID:
case api::MessageType::REMOVELOCATION_ID:
- case api::MessageType::BATCHPUTREMOVE_ID: // Deprecated
- case api::MessageType::BATCHDOCUMENTUPDATE_ID: // Deprecated
return true;
default:
return false;
diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp
index bb8c2b0608a..b22592af327 100644
--- a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp
+++ b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp
@@ -15,7 +15,6 @@
#include <vespa/document/util/stringutil.h>
#include <vespa/storageapi/message/persistence.h>
#include <vespa/storageapi/message/removelocation.h>
-#include <vespa/storageapi/message/batch.h>
#include <vespa/storageapi/message/stat.h>
#include "distributor_bucket_space_repo.h"
#include "distributor_bucket_space.h"
@@ -79,11 +78,11 @@ ExternalOperationHandler::checkTimestampMutationPreconditions(api::StorageComman
LOG(debug, "Distributor manager received %s, bucket %s with wrong distribution",
cmd.toString().c_str(), bucket.toString().c_str());
- persistenceMetrics.failures.wrongdistributor++;
+ persistenceMetrics.failures.wrongdistributor.inc();
return false;
}
if (!checkSafeTimeReached(cmd)) {
- persistenceMetrics.failures.safe_time_not_reached++;
+ persistenceMetrics.failures.safe_time_not_reached.inc();
return false;
}
return true;
@@ -97,7 +96,7 @@ ExternalOperationHandler::makeConcurrentMutationRejectionReply(api::StorageComma
auto err_msg = vespalib::make_string("A mutating operation for document '%s' is already in progress",
docId.toString().c_str());
LOG(debug, "Aborting incoming %s operation: %s", cmd.getType().toString().c_str(), err_msg.c_str());
- persistenceMetrics.failures.concurrent_mutations++;
+ persistenceMetrics.failures.concurrent_mutations.inc();
api::StorageReply::UP reply(cmd.makeReply());
reply->setResult(api::ReturnCode(api::ReturnCode::BUSY, err_msg));
return std::shared_ptr<api::StorageMessage>(reply.release());
@@ -190,7 +189,7 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, RemoveLocation)
if (!checkDistribution(*cmd, bucket)) {
LOG(debug, "Distributor manager received %s with wrong distribution", cmd->toString().c_str());
- getMetrics().removelocations[cmd->getLoadType()].failures.wrongdistributor++;
+ getMetrics().removelocations[cmd->getLoadType()].failures.wrongdistributor.inc();
return true;
}
@@ -206,7 +205,7 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Get)
LOG(debug, "Distributor manager received get for %s, bucket %s with wrong distribution",
cmd->getDocumentId().toString().c_str(), bucket.toString().c_str());
- getMetrics().gets[cmd->getLoadType()].failures.wrongdistributor++;
+ getMetrics().gets[cmd->getLoadType()].failures.wrongdistributor.inc();
return true;
}
diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp
index 5b1e585a6f8..04fcd4bd01c 100644
--- a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp
@@ -193,19 +193,19 @@ GetOperation::sendReply(DistributorMessageSender& sender)
repl->setResult(_returnCode);
if (_returnCode.success()) {
- ++_metric.ok;
+ _metric.ok.inc();
} else if (_returnCode.getResult() == api::ReturnCode::TIMEOUT) {
- ++_metric.failures.timeout;
+ _metric.failures.timeout.inc();
} else if (_returnCode.isBusy()) {
- ++_metric.failures.busy;
+ _metric.failures.busy.inc();
} else if (_returnCode.isNodeDownOrNetwork()) {
- ++_metric.failures.notconnected;
+ _metric.failures.notconnected.inc();
} else {
- ++_metric.failures.storagefailure;
+ _metric.failures.storagefailure.inc();
}
if (!_doc.get()) {
- ++_metric.failures.notfound;
+ _metric.failures.notfound.inc();
}
_metric.latency.addValue(_operationTimer.getElapsedTimeAsDouble());
diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp
index 0b0fc05763f..c652f787b2e 100644
--- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp
@@ -6,8 +6,8 @@
#include "updateoperation.h"
#include <vespa/storage/distributor/distributor_bucket_space.h>
#include <vespa/storageapi/message/persistence.h>
-#include <vespa/storageapi/message/batch.h>
#include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/fieldvalue/document.h>
#include <vespa/document/select/parser.h>
#include <vespa/vespalib/stllike/hash_map.hpp>
diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h
index e3fb6c93a3a..6efce913e70 100644
--- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h
+++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h
@@ -15,7 +15,6 @@ namespace storage {
namespace api {
class UpdateCommand;
-class BatchDocumentUpdateCommand;
class CreateBucketReply;
}
diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp
index 3475e3e12ba..784ea7253b6 100644
--- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp
@@ -308,9 +308,7 @@ constexpr std::array<uint32_t, 7> WRITE_FEED_MESSAGE_TYPES {{
api::MessageType::PUT_ID,
api::MessageType::REMOVE_ID,
api::MessageType::UPDATE_ID,
- api::MessageType::REMOVELOCATION_ID,
- api::MessageType::BATCHPUTREMOVE_ID,
- api::MessageType::BATCHDOCUMENTUPDATE_ID
+ api::MessageType::REMOVELOCATION_ID
}};
}
diff --git a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp
index 88b28941e65..d5f73a9e916 100644
--- a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp
+++ b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp
@@ -79,21 +79,21 @@ void
PersistenceOperationMetricSet::updateFromResult(const api::ReturnCode& result)
{
if (result.success()) {
- ++ok;
+ ok.inc();
} else if (result.getResult() == api::ReturnCode::WRONG_DISTRIBUTION) {
- ++failures.wrongdistributor;
+ failures.wrongdistributor.inc();
} else if (result.getResult() == api::ReturnCode::TIMEOUT) {
- ++failures.timeout;
+ failures.timeout.inc();
} else if (result.isBusy()) {
- ++failures.busy;
+ failures.busy.inc();
} else if (result.isBucketDisappearance()) {
// Bucket not found/deleted codes imply that replicas are transiently
// inconsistent in our DB or across replica nodes.
- ++failures.inconsistent_bucket;
+ failures.inconsistent_bucket.inc();
} else if (result.isNodeDownOrNetwork()) {
- ++failures.notconnected;
+ failures.notconnected.inc();
} else {
- ++failures.storagefailure;
+ failures.storagefailure.inc();
}
}
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
index f9571228ef9..3ccb35ef5a7 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
@@ -12,7 +12,6 @@
#include <vespa/storage/common/messagebucket.h>
#include <vespa/storage/persistence/messages.h>
#include <vespa/storageapi/message/stat.h>
-#include <vespa/storageapi/message/batch.h>
#include <vespa/vespalib/stllike/hash_map.hpp>
#include <vespa/vespalib/util/exceptions.h>
@@ -249,8 +248,6 @@ FileStorHandlerImpl::messageMayBeAborted(const api::StorageMessage& msg)
case api::MessageType::JOINBUCKETS_ID:
case api::MessageType::UPDATE_ID:
case api::MessageType::REMOVELOCATION_ID:
- case api::MessageType::BATCHPUTREMOVE_ID:
- case api::MessageType::BATCHDOCUMENTUPDATE_ID:
case api::MessageType::SETBUCKETSTATE_ID:
return true;
default:
@@ -587,7 +584,6 @@ FileStorHandlerImpl::remapMessage(api::StorageMessage& msg, const document::Buck
break;
}
case api::MessageType::STAT_ID:
- case api::MessageType::BATCHPUTREMOVE_ID:
case api::MessageType::REVERT_ID:
case api::MessageType::REMOVELOCATION_ID:
case api::MessageType::SETBUCKETSTATE_ID:
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
index bf0244255c1..e7323d07480 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
@@ -11,7 +11,6 @@
#include <vespa/storage/config/config-stor-server.h>
#include <vespa/storage/persistence/bucketownershipnotifier.h>
#include <vespa/storage/persistence/persistencethread.h>
-#include <vespa/storageapi/message/batch.h>
#include <vespa/storageapi/message/bucketsplitting.h>
#include <vespa/storageapi/message/state.h>
#include <vespa/vespalib/stllike/hash_map.hpp>
@@ -325,16 +324,6 @@ FileStorManager::onRevert(const shared_ptr<api::RevertCommand>& cmd)
}
bool
-FileStorManager::onBatchPutRemove(const std::shared_ptr<api::BatchPutRemoveCommand>& cmd)
-{
- StorBucketDatabase::WrappedEntry entry(mapOperationToBucketAndDisk(*cmd, 0));
- if (entry.exist()) {
- handlePersistenceMessage(cmd, entry->disk);
- }
- return true;
-}
-
-bool
FileStorManager::onRemoveLocation(const std::shared_ptr<api::RemoveLocationCommand>& cmd)
{
StorBucketDatabase::WrappedEntry entry(mapOperationToDisk(*cmd, cmd->getBucket()));
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
index 4bf2c1049cf..5c52e6c6a23 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
@@ -151,7 +151,6 @@ private:
bool onGet(const std::shared_ptr<api::GetCommand>&) override;
bool onRemove(const std::shared_ptr<api::RemoveCommand>&) override;
bool onRevert(const std::shared_ptr<api::RevertCommand>&) override;
- bool onBatchPutRemove(const std::shared_ptr<api::BatchPutRemoveCommand>&) override;
bool onStatBucket(const std::shared_ptr<api::StatBucketCommand>&) override;
// Bucket operations
diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp
index c3dc7cba80a..26cfb0e6566 100644
--- a/storage/src/vespa/storage/persistence/mergehandler.cpp
+++ b/storage/src/vespa/storage/persistence/mergehandler.cpp
@@ -1301,7 +1301,7 @@ void
MergeHandler::handleGetBucketDiffReply(api::GetBucketDiffReply& reply,
MessageSender& sender)
{
- ++_env._metrics.getBucketDiffReply;
+ _env._metrics.getBucketDiffReply.inc();
spi::Bucket bucket(reply.getBucket(), spi::PartitionId(_env._partition));
LOG(debug, "GetBucketDiffReply(%s)", bucket.toString().c_str());
@@ -1479,7 +1479,7 @@ void
MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,
MessageSender& sender)
{
- ++_env._metrics.applyBucketDiffReply;
+ _env._metrics.applyBucketDiffReply.inc();
spi::Bucket bucket(reply.getBucket(), spi::PartitionId(_env._partition));
std::vector<api::ApplyBucketDiffCommand::Entry>& diff(reply.getDiff());
LOG(debug, "%s", reply.toString().c_str());
diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp
index 0eeb4eeb84d..1b221ea0b7c 100644
--- a/storage/src/vespa/storage/persistence/persistencethread.cpp
+++ b/storage/src/vespa/storage/persistence/persistencethread.cpp
@@ -137,7 +137,7 @@ PersistenceThread::handleRemove(api::RemoveCommand& cmd)
tracker->setReply(std::make_shared<api::RemoveReply>(cmd, response.wasFound() ? cmd.getTimestamp() : 0));
}
if (!response.wasFound()) {
- ++_env._metrics.remove[cmd.getLoadType()].notFound;
+ _env._metrics.remove[cmd.getLoadType()].notFound.inc();
}
return tracker;
}
@@ -177,7 +177,7 @@ PersistenceThread::handleGet(api::GetCommand& cmd)
if (checkForError(result, *tracker)) {
if (!result.hasDocument()) {
- ++_env._metrics.get[cmd.getLoadType()].notFound;
+ _env._metrics.get[cmd.getLoadType()].notFound.inc();
}
tracker->setReply(std::make_shared<api::GetReply>(cmd, result.getDocumentPtr(), result.getTimestamp()));
}
@@ -202,7 +202,7 @@ PersistenceThread::handleRepairBucket(RepairBucketCommand& cmd)
reply->setAltered(!(after == before));
if (reply->bucketAltered()) {
notifyGuard.notifyAlways(cmd.getBucket(), after);
- ++_env._metrics.repairFixed;
+ _env._metrics.repairFixed.inc();
}
_env.updateBucketDatabase(cmd.getBucket(), after);
@@ -810,7 +810,7 @@ PersistenceThread::processMessage(api::StorageMessage& msg)
{
MBUS_TRACE(msg.getTrace(), 5, "PersistenceThread: Processing message in persistence layer");
- ++_env._metrics.operations;
+ _env._metrics.operations.inc();
if (msg.getType().isReply()) {
try{
LOG(debug, "Handling reply: %s", msg.toString().c_str());
@@ -837,7 +837,7 @@ PersistenceThread::processMessage(api::StorageMessage& msg)
&& tracker->getReply()->getResult().failed())
|| tracker->getResult().failed())
{
- ++_env._metrics.failedOperations;
+ _env._metrics.failedOperations.inc();
}
}
diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp
index 888dc93dd82..4bee5df4d04 100644
--- a/storage/src/vespa/storage/persistence/persistenceutil.cpp
+++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp
@@ -23,7 +23,7 @@ MessageTracker::MessageTracker(FileStorThreadMetrics::Op& metric,
_result(api::ReturnCode::OK),
_timer(clock)
{
- ++_metric.count;
+ _metric.count.inc();
}
MessageTracker::~MessageTracker()
@@ -53,7 +53,7 @@ MessageTracker::generateReply(api::StorageCommand& cmd)
}
if (!_reply->getResult().success()) {
- ++_metric.failed;
+ _metric.failed.inc();
LOGBP(debug, "Failed to handle command %s: %s",
cmd.toString().c_str(),
_result.toString().c_str());
diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp
index 00bc395b536..b6ac2a4b219 100644
--- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp
+++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp
@@ -3,9 +3,9 @@
#include "documentapiconverter.h"
#include "priorityconverter.h"
#include <vespa/document/bucket/bucketidfactory.h>
+#include <vespa/document/update/documentupdate.h>
#include <vespa/documentapi/documentapi.h>
#include <vespa/storage/common/bucket_resolver.h>
-#include <vespa/storageapi/message/batch.h>
#include <vespa/storageapi/message/datagram.h>
#include <vespa/storageapi/message/documentsummary.h>
#include <vespa/storageapi/message/persistence.h>
@@ -100,12 +100,6 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg)
toMsg = std::make_unique<api::DestroyVisitorCommand>(from.getInstanceId());
break;
}
- case DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE:
- {
- documentapi::BatchDocumentUpdateMessage& from(static_cast<documentapi::BatchDocumentUpdateMessage&>(fromMsg));
- toMsg = std::make_unique<api::BatchDocumentUpdateCommand>(from.getUpdates());
- break;
- }
case DocumentProtocol::MESSAGE_STATBUCKET:
{
documentapi::StatBucketMessage& from(static_cast<documentapi::StatBucketMessage&>(fromMsg));
@@ -377,10 +371,6 @@ DocumentApiConverter::transferReplyState(api::StorageReply& fromMsg, mbus::Reply
documentapi::CreateVisitorReply& to(static_cast<documentapi::CreateVisitorReply&>(toMsg));
to.setLastBucket(from.getLastBucket());
to.setVisitorStatistics(from.getVisitorStatistics());
- } else if (toMsg.getType() == DocumentProtocol::REPLY_BATCHDOCUMENTUPDATE) {
- api::BatchDocumentUpdateReply& from(static_cast<api::BatchDocumentUpdateReply&>(fromMsg));
- documentapi::BatchDocumentUpdateReply& to(static_cast<documentapi::BatchDocumentUpdateReply&>(toMsg));
- to.getDocumentsNotFound() = from.getDocumentsNotFound();
}
}
diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp
index 22af1a73633..a9d54c196c9 100644
--- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp
+++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp
@@ -471,33 +471,33 @@ MergeThrottler::updateOperationMetrics(
{
switch (result.getResult()) {
case api::ReturnCode::OK:
- ++metrics.ok;
+ metrics.ok.inc();
break;
case api::ReturnCode::NOT_READY:
- ++metrics.failures.notready;
+ metrics.failures.notready.inc();
break;
case api::ReturnCode::TIMEOUT:
- ++metrics.failures.timeout;
+ metrics.failures.timeout.inc();
break;
case api::ReturnCode::ABORTED:
- ++metrics.failures.aborted;
+ metrics.failures.aborted.inc();
break;
case api::ReturnCode::WRONG_DISTRIBUTION:
- ++metrics.failures.wrongdistribution;
+ metrics.failures.wrongdistribution.inc();
break;
case api::ReturnCode::EXISTS:
- ++metrics.failures.exists;
+ metrics.failures.exists.inc();
break;
case api::ReturnCode::REJECTED:
- ++metrics.failures.rejected;
+ metrics.failures.rejected.inc();
break;
default:
if (result.isBusy()) {
- ++metrics.failures.busy;
+ metrics.failures.busy.inc();
} else if (result.isBucketDisappearance()) {
- ++metrics.failures.bucketnotfound;
+ metrics.failures.bucketnotfound.inc();
} else {
- ++metrics.failures.other;
+ metrics.failures.other.inc();
}
}
}
diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
index cfc69bcde45..da7e8cb743e 100644
--- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
+++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
@@ -5,7 +5,6 @@
#include <vespa/storageapi/message/bucketsplitting.h>
#include <vespa/storageapi/message/internal.h>
#include <vespa/storageapi/message/removelocation.h>
-#include <vespa/storageapi/message/batch.h>
#include <vespa/storageapi/mbusprot/storageprotocol.h>
#include <vespa/storageapi/mbusprot/storagecommand.h>
#include <vespa/storageapi/mbusprot/storagereply.h>
@@ -89,7 +88,6 @@ struct StorageProtocolTest : public CppUnit::TestFixture {
void testSplitBucket51();
void testSplitBucketChain51();
void testJoinBuckets51();
- void testBatchPutRemove51();
void testCreateVisitor51();
void testDestroyVisitor51();
void testRemoveLocation51();
@@ -133,7 +131,6 @@ struct StorageProtocolTest : public CppUnit::TestFixture {
CPPUNIT_TEST(testCreateVisitor51);
CPPUNIT_TEST(testDestroyVisitor51);
CPPUNIT_TEST(testRemoveLocation51);
- CPPUNIT_TEST(testBatchPutRemove51);
CPPUNIT_TEST(testInternalMessage);
CPPUNIT_TEST(testSetBucketState51);
@@ -782,55 +779,6 @@ StorageProtocolTest::testApplyBucketDiff51()
recordSerialization50();
}
-void
-StorageProtocolTest::testBatchPutRemove51()
-{
- ScopedName test("testBatchPutRemove51");
-
- document::BucketId bucketId(20, 0xf1f1f1f1f1ull);
- document::Bucket bucket(makeDocumentBucket(bucketId));
- BatchPutRemoveCommand::SP cmd(new BatchPutRemoveCommand(bucket));
- cmd->addPut(_testDoc, 100);
- cmd->addHeaderUpdate(_testDoc, 101, 1234);
- cmd->addRemove(_testDoc->getId(), 102);
- cmd->forceMsgId(556677);
- BatchPutRemoveCommand::SP cmd2(copyCommand(cmd, _version5_1));
- CPPUNIT_ASSERT_EQUAL(bucketId, cmd2->getBucketId());
- CPPUNIT_ASSERT_EQUAL(3, (int)cmd2->getOperationCount());
- CPPUNIT_ASSERT_EQUAL(*_testDoc, *(dynamic_cast<const BatchPutRemoveCommand::PutOperation&>(cmd2->getOperation(0)).document));
- CPPUNIT_ASSERT_EQUAL((uint64_t)100, cmd2->getOperation(0).timestamp);
- {
- vespalib::nbostream header;
- _testDoc->serializeHeader(header);
- document::Document headerDoc(_docMan.getTypeRepo(), header);
- CPPUNIT_ASSERT_EQUAL(
- headerDoc,
- *(dynamic_cast<const BatchPutRemoveCommand::HeaderUpdateOperation&>(
- cmd2->getOperation(1)).document));
- }
- CPPUNIT_ASSERT_EQUAL((uint64_t)101, cmd2->getOperation(1).timestamp);
- CPPUNIT_ASSERT_EQUAL(1234, (int)dynamic_cast<const BatchPutRemoveCommand::HeaderUpdateOperation&>(cmd2->getOperation(1)).timestampToUpdate);
- CPPUNIT_ASSERT_EQUAL(_testDoc->getId(), dynamic_cast<const BatchPutRemoveCommand::RemoveOperation&>(cmd2->getOperation(2)).documentId);
- CPPUNIT_ASSERT_EQUAL((uint64_t)102, cmd2->getOperation(2).timestamp);
- CPPUNIT_ASSERT_EQUAL(uint64_t(556677), cmd2->getMsgId());
-
- BatchPutRemoveReply::SP reply(new BatchPutRemoveReply(*cmd2));
- reply->getDocumentsNotFound().push_back(document::DocumentId("userdoc:footype:1234:foo1"));
- reply->getDocumentsNotFound().push_back(document::DocumentId("userdoc:footype:1234:foo2"));
- reply->getDocumentsNotFound().push_back(document::DocumentId("userdoc:footype:1234:foo3"));
-
- BatchPutRemoveReply::SP reply2(copyReply(reply));
-
- CPPUNIT_ASSERT_EQUAL(3, (int)reply2->getDocumentsNotFound().size());
- CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:footype:1234:foo1"), reply2->getDocumentsNotFound()[0]);
- CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:footype:1234:foo2"), reply2->getDocumentsNotFound()[1]);
- CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:footype:1234:foo3"), reply2->getDocumentsNotFound()[2]);
-
- recordOutput(*cmd2);
- recordOutput(*reply2);
- recordSerialization50();
-}
-
namespace {
struct MyCommand : public api::InternalCommand {
MyCommand() : InternalCommand(101) {}
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp
index d08464da715..172cd6c8de5 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp
@@ -7,7 +7,6 @@
#include <vespa/storageapi/message/bucketsplitting.h>
#include <vespa/storageapi/message/visitor.h>
#include <vespa/storageapi/message/removelocation.h>
-#include <vespa/storageapi/message/batch.h>
#include <vespa/vespalib/util/exceptions.h>
@@ -132,12 +131,6 @@ ProtocolSerialization::encode(const api::StorageMessage& msg) const
case api::MessageType::REMOVELOCATION_REPLY_ID:
onEncode(buf, static_cast<const api::RemoveLocationReply&>(msg));
break;
- case api::MessageType::BATCHPUTREMOVE_ID:
- onEncode(buf, static_cast<const api::BatchPutRemoveCommand&>(msg));
- break;
- case api::MessageType::BATCHPUTREMOVE_REPLY_ID:
- onEncode(buf, static_cast<const api::BatchPutRemoveReply&>(msg));
- break;
case api::MessageType::SETBUCKETSTATE_ID:
onEncode(buf, static_cast<const api::SetBucketStateCommand&>(msg));
break;
@@ -205,8 +198,6 @@ ProtocolSerialization::decodeCommand(mbus::BlobRef data) const
cmd = onDecodeDestroyVisitorCommand(buf); break;
case api::MessageType::REMOVELOCATION_ID:
cmd = onDecodeRemoveLocationCommand(buf); break;
- case api::MessageType::BATCHPUTREMOVE_ID:
- cmd = onDecodeBatchPutRemoveCommand(buf); break;
case api::MessageType::SETBUCKETSTATE_ID:
cmd = onDecodeSetBucketStateCommand(buf); break;
default:
@@ -269,8 +260,6 @@ ProtocolSerialization::decodeReply(mbus::BlobRef data, const api::StorageCommand
reply = onDecodeDestroyVisitorReply(cmd, buf); break;
case api::MessageType::REMOVELOCATION_REPLY_ID:
reply = onDecodeRemoveLocationReply(cmd, buf); break;
- case api::MessageType::BATCHPUTREMOVE_REPLY_ID:
- reply = onDecodeBatchPutRemoveReply(cmd, buf); break;
case api::MessageType::SETBUCKETSTATE_REPLY_ID:
reply = onDecodeSetBucketStateReply(cmd, buf); break;
default:
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h
index e7d80a0614f..9c3ddb88bdf 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h
@@ -50,10 +50,6 @@ class SetBucketStateReply;
class CreateVisitorCommand;
class RemoveLocationCommand;
class RemoveLocationReply;
-class BatchPutRemoveCommand;
-class BatchPutRemoveReply;
-class BatchDocumentUpdateCommand;
-class BatchDocumentUpdateReply;
}
namespace storage::mbusprot {
@@ -125,8 +121,6 @@ protected:
virtual void onEncode(GBBuf&, const api::DestroyVisitorReply&) const = 0;
virtual void onEncode(GBBuf&, const api::RemoveLocationCommand&) const = 0;
virtual void onEncode(GBBuf&, const api::RemoveLocationReply&) const = 0;
- virtual void onEncode(GBBuf&, const api::BatchPutRemoveCommand&) const = 0;
- virtual void onEncode(GBBuf&, const api::BatchPutRemoveReply&) const = 0;
virtual SCmd::UP onDecodePutCommand(BBuf&) const = 0;
virtual SRep::UP onDecodePutReply(const SCmd&, BBuf&) const = 0;
@@ -166,8 +160,6 @@ protected:
virtual SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const = 0;
virtual SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const = 0;
virtual SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const = 0;
- virtual SCmd::UP onDecodeBatchPutRemoveCommand(BBuf&) const = 0;
- virtual SRep::UP onDecodeBatchPutRemoveReply(const SCmd&, BBuf&) const = 0;
virtual document::Bucket getBucket(document::ByteBuffer& buf) const = 0;
virtual void putBucket(const document::Bucket& bucket, vespalib::GrowableByteBuffer& buf) const = 0;
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp
index 2532f76f3a0..74a0c964d19 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp
@@ -7,7 +7,6 @@
#include "storagereply.h"
#include <vespa/storageapi/message/bucketsplitting.h>
-#include <vespa/storageapi/message/batch.h>
#include <vespa/storageapi/message/visitor.h>
#include <vespa/storageapi/message/removelocation.h>
#include <vespa/vespalib/util/exceptions.h>
@@ -25,104 +24,6 @@ ProtocolSerialization4_2::ProtocolSerialization4_2(
{
}
-void
-ProtocolSerialization4_2::onEncode(
- GBBuf& buf, const api::BatchPutRemoveCommand& msg) const
-{
- // Serialization format - allow different types of serialization depending on source.
- buf.putByte(0);
- putBucket(msg.getBucket(), buf);
- buf.putInt(msg.getOperationCount());
-
- for (uint32_t i = 0; i < msg.getOperationCount(); i++) {
- const api::BatchPutRemoveCommand::Operation& op = msg.getOperation(i);
- buf.putByte((uint8_t)op.type);
- buf.putLong(op.timestamp);
-
- switch (op.type) {
- case api::BatchPutRemoveCommand::Operation::REMOVE:
- buf.putString(static_cast<const api::BatchPutRemoveCommand::RemoveOperation&>(op).documentId.toString());
- break;
- case api::BatchPutRemoveCommand::Operation::HEADERUPDATE:
- {
- buf.putLong(static_cast<const api::BatchPutRemoveCommand::HeaderUpdateOperation&>(op).timestampToUpdate);
-
- vespalib::nbostream stream;
- static_cast<const api::BatchPutRemoveCommand::HeaderUpdateOperation&>(op).document->serializeHeader(stream);
- buf.putInt(stream.size());
- buf.putBytes(stream.peek(), stream.size());
- break;
- }
- case api::BatchPutRemoveCommand::Operation::PUT:
- SH::putDocument(static_cast<const api::BatchPutRemoveCommand::PutOperation&>(op).document.get(), buf);
- break;
- }
- }
- onEncodeBucketInfoCommand(buf, msg);
-}
-
-api::StorageCommand::UP
-ProtocolSerialization4_2::onDecodeBatchPutRemoveCommand(BBuf& buf) const
-{
- SH::getByte(buf);
- document::Bucket bucket = getBucket(buf);
- std::unique_ptr<api::BatchPutRemoveCommand> cmd(new api::BatchPutRemoveCommand(bucket));
- int length = SH::getInt(buf);
-
- for (int i = 0; i < length; i++) {
- int type = SH::getByte(buf);
- long timestamp = SH::getLong(buf);
-
- switch (type) {
- case api::BatchPutRemoveCommand::Operation::REMOVE:
- cmd->addRemove(document::DocumentId(SH::getString(buf)), timestamp);
- break;
- case api::BatchPutRemoveCommand::Operation::HEADERUPDATE:
- {
- long newTimestamp = SH::getLong(buf);
- cmd->addHeaderUpdate(document::Document::SP(
- SH::getDocument(buf, getTypeRepo())),
- timestamp, newTimestamp);
- break;
- }
- case api::BatchPutRemoveCommand::Operation::PUT:
- cmd->addPut(document::Document::SP(SH::getDocument(
- buf, getTypeRepo())), timestamp);
- break;
- }
- }
-
- onDecodeBucketInfoCommand(buf, *cmd);
-
- return api::StorageCommand::UP(cmd.release());
-}
-
-void ProtocolSerialization4_2::onEncode(
- GBBuf& buf, const api::BatchPutRemoveReply& msg) const
-{
- buf.putInt(msg.getDocumentsNotFound().size());
- for (uint32_t i = 0; i < msg.getDocumentsNotFound().size(); i++) {
- buf.putString(msg.getDocumentsNotFound()[i].toString());
- }
-
- onEncodeBucketInfoReply(buf, msg);
-}
-
-api::StorageReply::UP
-ProtocolSerialization4_2::onDecodeBatchPutRemoveReply(const SCmd& cmd,
- BBuf& buf) const
-{
- api::BatchPutRemoveReply::UP msg(new api::BatchPutRemoveReply(
- static_cast<const api::BatchPutRemoveCommand&>(cmd)));
- uint32_t count = SH::getInt(buf);
- for (uint32_t i = 0; i < count; i++) {
- msg->getDocumentsNotFound().push_back(document::DocumentId(SH::getString(buf)));
- }
-
- onDecodeBucketInfoReply(buf, *msg);
- return api::StorageReply::UP(msg.release());
-}
-
void ProtocolSerialization4_2::onEncode(
GBBuf& buf, const api::GetCommand& msg) const
{
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h
index 3a6f77e46a3..56aa3d4ed30 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h
@@ -28,8 +28,6 @@ protected:
void onEncode(GBBuf&, const api::RemoveLocationReply&) const override;
// Not supported on 4.2, but implemented here for simplicity.
- void onEncode(GBBuf&, const api::BatchPutRemoveCommand&) const override;
- void onEncode(GBBuf&, const api::BatchPutRemoveReply&) const override;
void onEncode(GBBuf&, const api::SetBucketStateCommand&) const override;
void onEncode(GBBuf&, const api::SetBucketStateReply&) const override;
@@ -59,10 +57,6 @@ protected:
SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const override;
SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const override;
- // Not supported on 4.2, but implemented here for simplicity.
- SCmd::UP onDecodeBatchPutRemoveCommand(BBuf&) const override;
- SRep::UP onDecodeBatchPutRemoveReply(const SCmd&, BBuf&) const override;
-
virtual void onDecodeBucketInfoCommand(BBuf&, api::BucketInfoCommand&) const;
virtual void onDecodeBucketInfoReply(BBuf&, api::BucketInfoReply&) const = 0;
virtual void onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const = 0;
diff --git a/storageapi/src/vespa/storageapi/message/CMakeLists.txt b/storageapi/src/vespa/storageapi/message/CMakeLists.txt
index dbbaad8eed1..cde9183482f 100644
--- a/storageapi/src/vespa/storageapi/message/CMakeLists.txt
+++ b/storageapi/src/vespa/storageapi/message/CMakeLists.txt
@@ -12,7 +12,6 @@ vespa_add_library(storageapi_message OBJECT
stat.cpp
removelocation.cpp
queryresult.cpp
- batch.cpp
internal.cpp
DEPENDS
)
diff --git a/storageapi/src/vespa/storageapi/message/batch.cpp b/storageapi/src/vespa/storageapi/message/batch.cpp
deleted file mode 100644
index de8ac849dee..00000000000
--- a/storageapi/src/vespa/storageapi/message/batch.cpp
+++ /dev/null
@@ -1,173 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-//
-#include "batch.h"
-#include <vespa/document/bucket/bucketidfactory.h>
-#include <ostream>
-
-using namespace storage::api;
-using document::BucketSpace;
-
-IMPLEMENT_COMMAND(BatchPutRemoveCommand, BatchPutRemoveReply)
-IMPLEMENT_REPLY(BatchPutRemoveReply)
-IMPLEMENT_COMMAND(BatchDocumentUpdateCommand, BatchDocumentUpdateReply)
-IMPLEMENT_REPLY(BatchDocumentUpdateReply)
-
-
-BatchPutRemoveCommand::Operation::Operation(uint64_t ts, Type tp)
- : timestamp(ts),
- type(tp)
-{
-}
-
-BatchPutRemoveCommand::PutOperation::PutOperation(document::Document::SP doc, uint64_t ts)
- : Operation(ts, PUT),
- document(doc)
-{
-}
-
-BatchPutRemoveCommand::HeaderUpdateOperation::HeaderUpdateOperation(document::Document::SP doc, uint64_t newTimestamp, uint64_t timestampToUpdate_)
- : Operation(newTimestamp, HEADERUPDATE),
- document(doc),
- timestampToUpdate(timestampToUpdate_)
-{
-}
-
-BatchPutRemoveCommand::RemoveOperation::RemoveOperation(const document::DocumentId& docId, uint64_t ts)
- : Operation(ts, REMOVE),
- documentId(docId)
-{
-}
-
-BatchPutRemoveCommand::BatchPutRemoveCommand(const document::Bucket &bucket)
- : BucketInfoCommand(MessageType::BATCHPUTREMOVE, bucket),
- _approxSize(0)
-{
-}
-
-void
-BatchPutRemoveCommand::addPut(document::Document::SP document, uint64_t ts)
-{
- _operations.push_back(std::unique_ptr<Operation>(new PutOperation(document, ts)));
- _approxSize += document->serialize()->getLength();
-}
-
-void
-BatchPutRemoveCommand::addHeaderUpdate(document::Document::SP document, uint64_t ts, uint64_t timestampToUpdate)
-{
- _operations.push_back(std::unique_ptr<Operation>(new HeaderUpdateOperation(document, ts, timestampToUpdate)));
- _approxSize += document->serialize()->getLength();
-}
-
-void
-BatchPutRemoveCommand::addRemove(const document::DocumentId& docId, uint64_t ts)
-{
- _operations.push_back(std::unique_ptr<Operation>(new RemoveOperation(docId, ts)));
- _approxSize += docId.toString().length();
-}
-
-void
-BatchPutRemoveCommand::addOperation(const Operation& op, bool cloneDocument)
-{
- switch (op.type) {
- case Operation::PUT:
- {
- document::Document::SP doc;
- if (!cloneDocument) {
- doc = static_cast<const PutOperation&>(op).document;
- } else {
- doc.reset(static_cast<const PutOperation&>(op).document->clone());
- }
- addPut(doc, op.timestamp);
- break;
- }
- case Operation::REMOVE:
- addRemove(static_cast<const RemoveOperation&>(op).documentId, op.timestamp);
- break;
- case Operation::HEADERUPDATE:
- {
- const HeaderUpdateOperation& hup = static_cast<const HeaderUpdateOperation&>(op);
- document::Document::SP doc;
- if (!cloneDocument) {
- doc = hup.document;
- } else {
- doc.reset(hup.document->clone());
- }
- addHeaderUpdate(doc, op.timestamp, hup.timestampToUpdate);
- break;
- }
- }
-}
-
-void
-BatchPutRemoveCommand::print(std::ostream& out, bool verbose,
- const std::string& indent) const {
- out << "BatchPutRemove(" << getBucketId() << ", " << _operations.size() << " operations)";
-
- if (verbose) {
- out << " : ";
- BucketInfoCommand::print(out, verbose, indent);
- }
-}
-
-BatchPutRemoveReply::BatchPutRemoveReply(const BatchPutRemoveCommand& cmd)
- : BucketInfoReply(cmd)
-{
-}
-
-void
-BatchPutRemoveReply::print(std::ostream& out, bool verbose,
- const std::string& indent) const {
- out << "BatchPutRemoveReply(";
- out << _documentsNotFound.size() << " documents not found)";
-
- if (verbose) {
- out << " {";
- for (std::vector<document::DocumentId>::const_iterator it =
- _documentsNotFound.begin();
- it != _documentsNotFound.end(); ++it)
- {
- out << "\n" << indent << " " << (*it);
- }
- out << "\n" << indent << "} : ";
- BucketInfoReply::print(out, verbose, indent);
- }
-}
-
-BatchDocumentUpdateCommand::BatchDocumentUpdateCommand(const UpdateList& updates)
- : StorageCommand(MessageType::BATCHDOCUMENTUPDATE),
- _updates(updates),
- _bucket(BucketSpace::placeHolder(), document::BucketId())
-{
- document::BucketIdFactory factory;
- _bucket = document::Bucket(BucketSpace::placeHolder(), factory.getBucketId(updates[0]->getId()));
-}
-
-void
-BatchDocumentUpdateCommand::print(std::ostream& out, bool verbose,
- const std::string& indent) const {
- out << "BatchDocumentUpdate(" << _updates.size() << " operations)";
-
- if (verbose) {
- out << " : ";
- StorageCommand::print(out, verbose, indent);
- }
-}
-
-BatchDocumentUpdateReply::BatchDocumentUpdateReply(const BatchDocumentUpdateCommand& cmd)
- : StorageReply(cmd),
- _documentsNotFound()
-{
-}
-
-void
-BatchDocumentUpdateReply::print(std::ostream& out, bool verbose,
- const std::string& indent) const {
- out << "BatchDocumentUpdateReply("
- << std::count(_documentsNotFound.begin(), _documentsNotFound.end(), true)
- << " not found)";
-
- if (verbose) {
- out << " : ";
- StorageReply::print(out, verbose, indent);
- }
-}
diff --git a/storageapi/src/vespa/storageapi/message/batch.h b/storageapi/src/vespa/storageapi/message/batch.h
deleted file mode 100644
index 56cca3cfe38..00000000000
--- a/storageapi/src/vespa/storageapi/message/batch.h
+++ /dev/null
@@ -1,186 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/document/fieldvalue/document.h>
-#include <vespa/document/update/documentupdate.h>
-#include <vespa/storageapi/messageapi/bucketinfocommand.h>
-#include <vespa/storageapi/messageapi/bucketinforeply.h>
-
-namespace storage::api {
-
-/**
- * @class BatchPutRemoveCommand
- * @ingroup message
- *
- * @brief Sends a batch of puts and removes
- */
-class BatchPutRemoveCommand : public BucketInfoCommand {
-public:
- class Operation {
- public:
- enum Type {
- REMOVE, // Removes a document
- HEADERUPDATE, // Updates the header of a document, if it already exists.
- PUT // Inserts a new document.
- };
-
- Operation(uint64_t ts, Type type);
- virtual ~Operation() {};
-
- uint64_t timestamp;
- Type type;
-
- virtual const document::DocumentId& getDocumentId() const = 0;
- };
-
- explicit BatchPutRemoveCommand(const document::Bucket &bucket);
-
- class PutOperation : public Operation {
- public:
- PutOperation(document::Document::SP document, uint64_t timestamp);
-
- document::Document::SP document;
-
- const document::DocumentId& getDocumentId() const override {
- return document->getId();
- }
- };
-
- class HeaderUpdateOperation : public Operation {
- public:
- HeaderUpdateOperation(document::Document::SP document, uint64_t newTimestamp, uint64_t timestampToUpdate);
-
- document::Document::SP document;
- uint64_t timestampToUpdate;
-
- const document::DocumentId& getDocumentId() const override {
- return document->getId();
- }
- };
-
- class RemoveOperation : public Operation {
- public:
- RemoveOperation(const document::DocumentId& docId, uint64_t timestamp);
-
- document::DocumentId documentId;
-
- const document::DocumentId& getDocumentId() const override {
- return documentId;
- }
- };
-
- /**
- Adds a PUT operation to be performed.
- */
- void addPut(document::Document::SP document, uint64_t timestamp);
-
- /**
- Adds a PUT operation to be performed.
- */
- void addHeaderUpdate(document::Document::SP document, uint64_t newTimestamp, uint64_t timestampToUpdate);
-
- /**
- Adds a REMOVE operation to be performed.
- */
- void addRemove(const document::DocumentId& docId, uint64_t timestamp);
-
- /**
- * Adds an operation to be performed. Optionally deep-clones the
- * operation's document.
- */
- void addOperation(const Operation& op, bool cloneDocument = false);
-
- /**
- Returns the number of operations in this batch.
- */
- uint32_t getOperationCount() const { return _operations.size(); }
-
- /**
- Returns the nth operation in this batch.
- */
- const Operation& getOperation(uint32_t index) const { return *_operations[index]; }
-
- /**
- Returns the nth operation in this batch.
- */
- Operation& getOperation(uint32_t index) { return *_operations[index]; }
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- DECLARE_STORAGECOMMAND(BatchPutRemoveCommand, onBatchPutRemove)
-
-private:
- std::vector<std::unique_ptr<Operation> > _operations;
- uint32_t _approxSize;
-};
-
-/**
- * @class BatchPutRemoveReply
- * @ingroup message
- *
- * @brief Confirm that a given docoperations have been received.
- */
-class BatchPutRemoveReply : public BucketInfoReply {
-private:
- std::vector<document::DocumentId> _documentsNotFound;
-
-public:
- explicit BatchPutRemoveReply(const BatchPutRemoveCommand&);
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- const std::vector<document::DocumentId>& getDocumentsNotFound() const { return _documentsNotFound; }
- std::vector<document::DocumentId>& getDocumentsNotFound() { return _documentsNotFound; }
-
- DECLARE_STORAGEREPLY(BatchPutRemoveReply, onBatchPutRemoveReply)
-};
-
-class BatchDocumentUpdateCommand : public StorageCommand
-{
-public:
- typedef std::vector<document::DocumentUpdate::SP > UpdateList;
-
- /**
- Creates a batch update message containing the given updates.
- */
- BatchDocumentUpdateCommand(const UpdateList& updates);
-
- /**
- @return Returns a list of the updates to be performed.
- */
- const UpdateList& getUpdates() const { return _updates; };
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- /**
- Returns a bucket suitable for routing this message.
- */
- document::Bucket getBucket() const override { return _bucket; }
- bool hasSingleBucketId() const override { return true; }
-
- DECLARE_STORAGECOMMAND(BatchDocumentUpdateCommand, onBatchDocumentUpdate)
-
-private:
- UpdateList _updates;
- document::Bucket _bucket;
-};
-
-/**
- * @class BatchDocumentUpdateReply
- * @ingroup message
- *
- * @brief Confirm that a given docoperations have been received.
- */
-class BatchDocumentUpdateReply : public StorageReply {
- // 1-1 mapping of found/not found state for documents
- std::vector<bool> _documentsNotFound;
-public:
- explicit BatchDocumentUpdateReply(const BatchDocumentUpdateCommand&);
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
- const std::vector<bool>& getDocumentsNotFound() const { return _documentsNotFound; }
- std::vector<bool>& getDocumentsNotFound() { return _documentsNotFound; }
-
- DECLARE_STORAGEREPLY(BatchDocumentUpdateReply, onBatchDocumentUpdateReply)
-};
-
-}
diff --git a/storageapi/src/vespa/storageapi/messageapi/messagehandler.h b/storageapi/src/vespa/storageapi/messageapi/messagehandler.h
index e7bd2b5bf27..a9c1dfb8f26 100644
--- a/storageapi/src/vespa/storageapi/messageapi/messagehandler.h
+++ b/storageapi/src/vespa/storageapi/messageapi/messagehandler.h
@@ -25,8 +25,6 @@ class PutCommand; // Add document
class UpdateCommand; // Update document
class RemoveCommand; // Remove document
class RevertCommand; // Revert put/remove operation
-class BatchPutRemoveCommand;
-class BatchDocumentUpdateCommand;
class CreateVisitorCommand; // Create a new visitor
class DestroyVisitorCommand; // Destroy a running visitor
@@ -64,8 +62,6 @@ class PutReply;
class UpdateReply;
class RemoveReply;
class RevertReply;
-class BatchPutRemoveReply;
-class BatchDocumentUpdateReply;
class CreateVisitorReply;
class DestroyVisitorReply;
@@ -143,18 +139,6 @@ public:
{ return false; }
virtual bool onRevertReply(const std::shared_ptr<api::RevertReply>&)
{ return false; }
- virtual bool onBatchPutRemove(
- const std::shared_ptr<api::BatchPutRemoveCommand>&)
- { return false; }
- virtual bool onBatchPutRemoveReply(
- const std::shared_ptr<api::BatchPutRemoveReply>&)
- { return false; }
- virtual bool onBatchDocumentUpdate(
- const std::shared_ptr<api::BatchDocumentUpdateCommand>&)
- { return false; }
- virtual bool onBatchDocumentUpdateReply(
- const std::shared_ptr<api::BatchDocumentUpdateReply>&)
- { return false; }
// Visiting
virtual bool onCreateVisitor(
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp
index 563a2aab7c1..bab475eea32 100644
--- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp
+++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp
@@ -105,10 +105,6 @@ const MessageType MessageType::REMOVELOCATION("Removelocation", REMOVELOCATION_I
const MessageType MessageType::REMOVELOCATION_REPLY("Removelocation Reply", REMOVELOCATION_REPLY_ID, &MessageType::REMOVELOCATION);
const MessageType MessageType::QUERYRESULT("QueryResult", QUERYRESULT_ID);
const MessageType MessageType::QUERYRESULT_REPLY("QueryResult reply", QUERYRESULT_REPLY_ID, &MessageType::QUERYRESULT);
-const MessageType MessageType::BATCHPUTREMOVE("BatchPutRemove", BATCHPUTREMOVE_ID);
-const MessageType MessageType::BATCHPUTREMOVE_REPLY("BatchPutRemove reply", BATCHPUTREMOVE_REPLY_ID, &MessageType::BATCHPUTREMOVE);
-const MessageType MessageType::BATCHDOCUMENTUPDATE("BatchDocumentUpdate", BATCHDOCUMENTUPDATE_ID);
-const MessageType MessageType::BATCHDOCUMENTUPDATE_REPLY("BatchDocumentUpdate reply", BATCHDOCUMENTUPDATE_REPLY_ID, &MessageType::BATCHDOCUMENTUPDATE);
const MessageType MessageType::SETBUCKETSTATE("SetBucketState", SETBUCKETSTATE_ID);
const MessageType MessageType::SETBUCKETSTATE_REPLY("SetBucketStateReply", SETBUCKETSTATE_REPLY_ID, &MessageType::SETBUCKETSTATE);
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
index c81ee51ae22..c9f6e737a47 100644
--- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
+++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
@@ -147,10 +147,6 @@ public:
REMOVELOCATION_REPLY_ID = 87,
QUERYRESULT_ID = 88,
QUERYRESULT_REPLY_ID = 89,
- BATCHPUTREMOVE_ID = 90,
- BATCHPUTREMOVE_REPLY_ID = 91,
- BATCHDOCUMENTUPDATE_ID = 92,
- BATCHDOCUMENTUPDATE_REPLY_ID = 93,
SETBUCKETSTATE_ID = 94,
SETBUCKETSTATE_REPLY_ID = 95,
MESSAGETYPE_MAX_ID
@@ -233,10 +229,6 @@ public:
static const MessageType REMOVELOCATION_REPLY;
static const MessageType QUERYRESULT;
static const MessageType QUERYRESULT_REPLY;
- static const MessageType BATCHPUTREMOVE;
- static const MessageType BATCHPUTREMOVE_REPLY;
- static const MessageType BATCHDOCUMENTUPDATE;
- static const MessageType BATCHDOCUMENTUPDATE_REPLY;
static const MessageType SETBUCKETSTATE;
static const MessageType SETBUCKETSTATE_REPLY;
diff --git a/vespabase/src/start-cbinaries.sh b/vespabase/src/start-cbinaries.sh
index e61a3d8c8b1..02eac2f42bc 100755
--- a/vespabase/src/start-cbinaries.sh
+++ b/vespabase/src/start-cbinaries.sh
@@ -92,7 +92,7 @@ check_bname_in_value () {
configure_valgrind () {
no_valgrind=true
- if which valgrind >/dev/null; then
+ if which valgrind >/dev/null 2>&1; then
if check_bname_in_value $VESPA_USE_VALGRIND; then
no_valgrind=false
valgrind_log=$VESPA_HOME/tmp/valgrind.$bname.log.$$
diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml
index 5b9c143a447..741b4b58cc9 100644
--- a/vespajlib/pom.xml
+++ b/vespajlib/pom.xml
@@ -31,6 +31,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
</dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna</artifactId>
+ </dependency>
<!-- provided scope -->
diff --git a/vespajlib/src/main/java/com/yahoo/io/NativeIO.java b/vespajlib/src/main/java/com/yahoo/io/NativeIO.java
new file mode 100644
index 00000000000..f08e01070d7
--- /dev/null
+++ b/vespajlib/src/main/java/com/yahoo/io/NativeIO.java
@@ -0,0 +1,104 @@
+package com.yahoo.io;
+
+import java.io.File;
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.SyncFailedException;
+import java.lang.reflect.Field;
+import java.util.logging.Logger;
+
+import com.sun.jna.LastErrorException;
+import com.sun.jna.Native;
+import com.sun.jna.Platform;
+
+/**
+ * Provides functionality only possible through native C library.
+ */
+public class NativeIO {
+ private final static Logger logger = Logger.getLogger(NativeIO.class.getName());
+ private static final int POSIX_FADV_DONTNEED = 4; // See /usr/include/linux/fadvise.h
+ private static boolean initialized = false;
+ private static Throwable initError = null;
+ static {
+ try {
+ if (Platform.isLinux()) {
+ Native.register(Platform.C_LIBRARY_NAME);
+ initialized = true;
+ }
+ } catch (Throwable throwable) {
+ initError = throwable;
+ }
+ }
+
+ private static final Field fieldFD = getField(FileDescriptor.class, "fd");
+
+
+ private static native int posix_fadvise(int fd, long offset, long len, int flag) throws LastErrorException;
+
+ public NativeIO() {
+ if (!initialized) {
+ logger.warning("native IO not possible due to " + getError().getMessage());
+ }
+ }
+
+ public boolean valid() { return initialized; }
+ public Throwable getError() {
+ if (initError != null) {
+ return initError;
+ } else {
+ return new RuntimeException("Platform is unsúpported. Only supported on linux.");
+ }
+ }
+
+ /**
+ * Will hint the OS that this is will not be accessed again and should hence be dropped from the buffer cache.
+ * @param fd The file descriptor to drop from buffer cache.
+ */
+ public void dropFileFromCache(FileDescriptor fd) {
+ try {
+ fd.sync();
+ } catch (SyncFailedException e) {
+ logger.warning("Sync failed while dropping cache: " + e.getMessage());
+ }
+ if (initialized) {
+ posix_fadvise(getNativeFD(fd), 0, 0, POSIX_FADV_DONTNEED);
+ }
+ }
+
+ /**
+ * Will hint the OS that this is will not be accessed again and should hence be dropped from the buffer cache.
+ * @param file File to drop from buffer cache
+ */
+ public void dropFileFromCache(File file) {
+ try {
+ dropFileFromCache(new FileInputStream(file).getFD());
+ } catch (FileNotFoundException e) {
+ logger.warning("No point in dropping a non-existing file from the buffer cache: " + e.getMessage());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ private static Field getField(Class<?> clazz, String fieldName) {
+ Field field;
+ try {
+ field = clazz.getDeclaredField(fieldName);
+ } catch (NoSuchFieldException e) {
+ throw new RuntimeException(e);
+ }
+ field.setAccessible(true);
+ return field;
+ }
+
+ private static int getNativeFD(FileDescriptor fd) {
+ try {
+ return fieldFD.getInt(fd);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
diff --git a/vespajlib/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java b/vespajlib/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java
new file mode 100644
index 00000000000..5595d33a9b5
--- /dev/null
+++ b/vespajlib/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java
@@ -0,0 +1,66 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.security.tls;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Optional;
+
+/**
+ * Utility class for retrieving {@link TransportSecurityOptions} from the system.
+ *
+ * @author bjorncs
+ */
+public class TransportSecurityUtils {
+
+ public static final String CONFIG_FILE_ENVIRONMENT_VARIABLE = "VESPA_TLS_CONFIG_FILE";
+ public static final String INSECURE_MIXED_MODE_ENVIRONMENT_VARIABLE = "VESPA_TLS_INSECURE_MIXED_MODE";
+
+ public enum MixedMode {
+ PLAINTEXT_CLIENT_MIXED_SERVER("plaintext_client_mixed_server"),
+ TLS_CLIENT_MIXED_SERVER("tls_client_mixed_server");
+
+ final String configValue;
+
+ MixedMode(String configValue) {
+ this.configValue = configValue;
+ }
+
+ static MixedMode fromConfigValue(String configValue) {
+ return Arrays.stream(values())
+ .filter(v -> v.configValue.equals(configValue))
+ .findFirst()
+ .orElseThrow(() -> new IllegalArgumentException("Unknown value: " + configValue));
+ }
+ }
+
+ private TransportSecurityUtils() {}
+
+ public static boolean isTransportSecurityEnabled() {
+ return getConfigFile().isPresent();
+ }
+
+ public static boolean isInsecureMixedModeEnabled() {
+ return getInsecureMixedMode().isPresent();
+ }
+
+ public static Optional<MixedMode> getInsecureMixedMode() {
+ if (!isTransportSecurityEnabled()) return Optional.empty();
+ return getEnvironmentVariable(INSECURE_MIXED_MODE_ENVIRONMENT_VARIABLE)
+ .map(MixedMode::fromConfigValue);
+ }
+
+ public static Optional<Path> getConfigFile() {
+ return getEnvironmentVariable(CONFIG_FILE_ENVIRONMENT_VARIABLE).map(Paths::get);
+ }
+
+ public static Optional<TransportSecurityOptions> getOptions() {
+ return getConfigFile()
+ .map(TransportSecurityOptions::fromJsonFile);
+ }
+
+ private static Optional<String> getEnvironmentVariable(String environmentVariable) {
+ return Optional.ofNullable(System.getenv(environmentVariable))
+ .filter(var -> !var.isEmpty());
+ }
+}
diff --git a/vespajlib/src/test/java/com/yahoo/io/NativeIOTestCase.java b/vespajlib/src/test/java/com/yahoo/io/NativeIOTestCase.java
new file mode 100644
index 00000000000..6018a195b6a
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/io/NativeIOTestCase.java
@@ -0,0 +1,35 @@
+package com.yahoo.io;
+
+import com.sun.jna.Platform;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+public class NativeIOTestCase {
+
+ @Test
+ public void requireThatDropFileFromCacheDoesNotThrow() throws IOException {
+ File testFile = new File("testfile");
+ FileOutputStream output = new FileOutputStream(testFile);
+ output.write('t');
+ output.flush();
+ output.close();
+ NativeIO nativeIO = new NativeIO();
+ if (Platform.isLinux()) {
+ assertTrue(nativeIO.valid());
+ } else {
+ assertFalse(nativeIO.valid());
+ assertEquals("Platform is unsúpported. Only supported on linux.", nativeIO.getError().getMessage());
+ }
+ nativeIO.dropFileFromCache(output.getFD());
+ nativeIO.dropFileFromCache(testFile);
+ testFile.delete();
+ nativeIO.dropFileFromCache(new File("file.that.does.not.exist"));
+ }
+}
diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt
index b32c875cb26..663d281179d 100644
--- a/vespalib/CMakeLists.txt
+++ b/vespalib/CMakeLists.txt
@@ -12,6 +12,7 @@ vespa_define_module(
src/apps/make_fixture_macros
src/apps/vespa-detect-hostname
src/apps/vespa-validate-hostname
+ src/apps/vespa-drop-file-from-cache
TESTS
src/tests/alignedmemory
@@ -36,6 +37,7 @@ vespa_define_module(
src/tests/data/smart_buffer
src/tests/delegatelist
src/tests/dotproduct
+ src/tests/drop-file-from-cache
src/tests/dual_merge_director
src/tests/eventbarrier
src/tests/exception_classes
@@ -59,6 +61,7 @@ vespa_define_module(
src/tests/net/socket_spec
src/tests/net/tls/direct_buffer_bio
src/tests/net/tls/openssl_impl
+ src/tests/net/tls/protocol_snooping
src/tests/net/tls/transport_options
src/tests/objects/nbostream
src/tests/optimized
diff --git a/vespalib/src/apps/vespa-drop-file-from-cache/.gitignore b/vespalib/src/apps/vespa-drop-file-from-cache/.gitignore
new file mode 100644
index 00000000000..24dc558b6bc
--- /dev/null
+++ b/vespalib/src/apps/vespa-drop-file-from-cache/.gitignore
@@ -0,0 +1 @@
+vespa-drop-file-from-cache
diff --git a/vespalib/src/apps/vespa-drop-file-from-cache/CMakeLists.txt b/vespalib/src/apps/vespa-drop-file-from-cache/CMakeLists.txt
new file mode 100644
index 00000000000..409fd5bb725
--- /dev/null
+++ b/vespalib/src/apps/vespa-drop-file-from-cache/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_vespa-drop-file-from-cache_app
+ SOURCES
+ drop_file_from_cache.cpp
+ OUTPUT_NAME vespa-drop-file-from-cache
+ INSTALL bin
+ DEPENDS
+ vespalib
+)
diff --git a/vespalib/src/apps/vespa-drop-file-from-cache/drop_file_from_cache.cpp b/vespalib/src/apps/vespa-drop-file-from-cache/drop_file_from_cache.cpp
new file mode 100644
index 00000000000..042681a90fe
--- /dev/null
+++ b/vespalib/src/apps/vespa-drop-file-from-cache/drop_file_from_cache.cpp
@@ -0,0 +1,32 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <cstdio>
+#include <cerrno>
+#include <cstring>
+#include <fcntl.h>
+#include <unistd.h>
+
+int main(int argc, char **argv) {
+ char errorBuf[200];
+ if (argc != 2) {
+ fprintf(stderr, "%s <filename>\n", argv[0]);
+ return 1;
+ }
+ const char *fileName = argv[1];
+ int fh = open(fileName, O_RDONLY);
+ if (fh == -1) {
+ const char *errorString = strerror_r(errno, errorBuf, sizeof(errorBuf));
+ fprintf(stderr, "Failed opening file %s: %s\n", fileName, errorString);
+ return 2;
+ }
+
+ int retval = 0;
+ int err = posix_fadvise(fh, 0, 0, POSIX_FADV_DONTNEED);
+ if (err != 0) {
+ const char *errorString = strerror_r(errno, errorBuf, sizeof(errorBuf));
+ fprintf(stderr, "posix_fadvise failed: %s\n", errorString);
+ retval = 3;
+ }
+ close(fh);
+ return retval;
+}
diff --git a/vespalib/src/tests/drop-file-from-cache/CMakeLists.txt b/vespalib/src/tests/drop-file-from-cache/CMakeLists.txt
new file mode 100644
index 00000000000..d6f4a51a4c5
--- /dev/null
+++ b/vespalib/src/tests/drop-file-from-cache/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_drop_file_from_cache_test_app TEST
+ SOURCES
+ drop_file_from_cache_test.cpp
+ DEPENDS
+ vespalib
+)
+vespa_add_test(NAME vespalib_drop_file_from_cache_test_app COMMAND vespalib_drop_file_from_cache_test_app)
diff --git a/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp b/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp
new file mode 100644
index 00000000000..63defa58c41
--- /dev/null
+++ b/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp
@@ -0,0 +1,25 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/util/slaveproc.h>
+
+using vespalib::SlaveProc;
+
+TEST("no arguments") {
+ SlaveProc drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache");
+ drop.wait();
+ EXPECT_EQUAL(1, drop.getExitCode());
+}
+
+TEST("file does not exist") {
+ SlaveProc drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache not_exist");
+ drop.wait();
+ EXPECT_EQUAL(2, drop.getExitCode());
+}
+
+TEST("All is well") {
+ SlaveProc drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache vespalib_drop_file_from_cache_test_app");
+ drop.wait();
+ EXPECT_EQUAL(0, drop.getExitCode());
+}
+
+TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/net/tls/protocol_snooping/CMakeLists.txt b/vespalib/src/tests/net/tls/protocol_snooping/CMakeLists.txt
new file mode 100644
index 00000000000..1489859fe48
--- /dev/null
+++ b/vespalib/src/tests/net/tls/protocol_snooping/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_net_tls_protocol_snooping_test_app TEST
+ SOURCES
+ protocol_snooping_test.cpp
+ DEPENDS
+ vespalib
+)
+vespa_add_test(NAME vespalib_net_tls_protocol_snooping_test_app COMMAND vespalib_net_tls_protocol_snooping_test_app)
+
diff --git a/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp b/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp
new file mode 100644
index 00000000000..2d203047835
--- /dev/null
+++ b/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp
@@ -0,0 +1,78 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/net/tls/protocol_snooping.h>
+
+using namespace vespalib;
+using namespace vespalib::net::tls::snooping;
+
+TEST("min_header_bytes_to_observe() is 8") {
+ EXPECT_EQUAL(8u, min_header_bytes_to_observe());
+}
+
+TlsSnoopingResult do_snoop(const unsigned char* buf) {
+ return snoop_client_hello_header(reinterpret_cast<const char*>(buf));
+}
+
+TEST("Well-formed TLSv1.0 packet returns ProbablyTls") {
+ const unsigned char buf[] = { 22, 3, 1, 10, 255, 1, 0, 10 };
+ EXPECT_EQUAL(TlsSnoopingResult::ProbablyTls, do_snoop(buf));
+}
+
+TEST("Well-formed TLSv1.2 packet returns ProbablyTls") {
+ const unsigned char buf[] = { 22, 3, 3, 10, 255, 1, 0, 10 };
+ EXPECT_EQUAL(TlsSnoopingResult::ProbablyTls, do_snoop(buf));
+}
+
+TEST("Mismatching handshake header byte 1 returns HandshakeMismatch") {
+ const unsigned char buf[] = { 23, 3, 1, 10, 255, 1, 0, 10 };
+ EXPECT_EQUAL(TlsSnoopingResult::HandshakeMismatch, do_snoop(buf));
+}
+
+TEST("Mismatching major version byte returns ProtocolVersionMismatch") {
+ const unsigned char buf[] = { 22, 2, 1, 10, 255, 1, 0, 10 };
+ EXPECT_EQUAL(TlsSnoopingResult::ProtocolVersionMismatch, do_snoop(buf));
+}
+
+TEST("Mismatching minor version byte returns ProtocolVersionMismatch") {
+ const unsigned char buf[] = { 22, 3, 0, 10, 255, 1, 0, 10 };
+ EXPECT_EQUAL(TlsSnoopingResult::ProtocolVersionMismatch, do_snoop(buf));
+}
+
+TEST("Oversized record returns RecordSizeRfcViolation") {
+ const unsigned char buf1[] = { 22, 3, 1, 255, 255, 1, 0, 10 }; // 64k
+ // ^^^^^^^^ big endian record length
+ EXPECT_EQUAL(TlsSnoopingResult::RecordSizeRfcViolation, do_snoop(buf1));
+
+ const unsigned char buf2[] = { 22, 3, 1, 72, 1, 1, 0, 10 }; // 18K+1
+ EXPECT_EQUAL(TlsSnoopingResult::RecordSizeRfcViolation, do_snoop(buf2));
+}
+
+TEST("Undersized record returns RecordSizeRfcViolation") {
+ const unsigned char buf1[] = { 22, 3, 1, 0, 3, 1, 0, 0 };
+ EXPECT_EQUAL(TlsSnoopingResult::RecordSizeRfcViolation, do_snoop(buf1));
+}
+
+TEST("Non-ClientHello handshake record returns RecordNotClientHello") {
+ const unsigned char buf[] = { 22, 3, 1, 10, 255, 2, 0, 10 };
+ // ^ 1 == ClientHello
+ EXPECT_EQUAL(TlsSnoopingResult::RecordNotClientHello, do_snoop(buf));
+}
+
+TEST("Oversized or fragmented ClientHello record returns ClientHelloRecordTooBig") {
+ const unsigned char buf[] = { 22, 3, 1, 10, 255, 1, 1, 10 };
+ // ^ MSB of 24-bit record length
+ EXPECT_EQUAL(TlsSnoopingResult::ClientHelloRecordTooBig, do_snoop(buf));
+}
+
+TEST("Expected ClientHello record size mismatch returns ExpectedRecordSizeMismatch") {
+ const unsigned char buf[] = { 22, 3, 1, 10, 2, 1, 0, 10 };
+ // ^^ bits [8,16) of record length, should be 9
+ EXPECT_EQUAL(TlsSnoopingResult::ExpectedRecordSizeMismatch, do_snoop(buf));
+}
+
+TEST("Valid ClientHello record size with LSB < 4 returns ProbablyTls") {
+ const unsigned char buf[] = { 22, 3, 1, 10, 3, 1, 0, 9 };
+ EXPECT_EQUAL(TlsSnoopingResult::ProbablyTls, do_snoop(buf));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
index 2d34a3e1c80..17abd82366d 100644
--- a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
@@ -4,6 +4,7 @@ vespa_add_library(vespalib_vespalib_net_tls OBJECT
crypto_codec.cpp
crypto_codec_adapter.cpp
crypto_exception.cpp
+ protocol_snooping.cpp
tls_context.cpp
tls_crypto_engine.cpp
transport_security_options.cpp
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
index 4c253fdf24c..53693dd7f40 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
@@ -11,10 +11,8 @@ namespace vespalib::net::tls { class TlsContext; }
namespace vespalib::net::tls::impl {
/*
- * Frame-level OpenSSL-backed TLSv1.2 crypto codec implementation.
- *
- * Currently has sub-optimal buffer management, and is mostly intended
- * as a starting point.
+ * Frame-level OpenSSL-backed TLSv1.2/TLSv1.3 (depending on OpenSSL version)
+ * crypto codec implementation.
*
* NOT thread safe per instance, but independent instances may be
* used by different threads safely.
diff --git a/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.cpp b/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.cpp
new file mode 100644
index 00000000000..6ccfc7f27ac
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.cpp
@@ -0,0 +1,127 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "protocol_snooping.h"
+#include <iostream>
+#include <cstdlib>
+#include <stdint.h>
+
+namespace vespalib::net::tls::snooping {
+
+namespace {
+
+// Precondition for all helper functions: buffer is at least `min_header_bytes_to_observe()` bytes long
+
+// From RFC 5246:
+// 0x16 - Handshake content type byte of TLSCiphertext record
+inline bool is_tls_handshake_packet(const char* buf) {
+ return (buf[0] == 0x16);
+}
+
+// First byte of 2-byte ProtocolVersion, always 3 on TLSv1.2 and v1.3
+// Next is the TLS minor version, either 1 or 3 depending on version (though the
+// RFCs say it _should_ be 1 for backwards compatibility reasons).
+// Yes, the TLS spec says that you should technically ignore the protocol version
+// field here, but we want all the signals we can get.
+inline bool is_expected_tls_protocol_version(const char* buf) {
+ return ((buf[1] == 0x03) && ((buf[2] == 0x01) || (buf[2] == 0x03)));
+}
+
+// Length is big endian u16 in bytes 3, 4
+inline uint16_t tls_record_length(const char* buf) {
+ return (uint16_t(static_cast<unsigned char>(buf[3]) << 8)
+ + static_cast<unsigned char>(buf[4]));
+}
+
+// First byte of Handshake record in byte 5, which shall be ClientHello (0x01)
+inline bool is_client_hello_handshake_record(const char* buf) {
+ return (buf[5] == 0x01);
+}
+
+// Last 2 bytes are the 2 first big-endian bytes of a 3-byte Handshake
+// record length field. No support for records that are large enough that
+// the MSB should ever be non-zero.
+inline bool client_hello_record_size_within_expected_bounds(const char* buf) {
+ return (buf[6] == 0x00);
+}
+
+// The byte after the MSB of the 24-bit handshake record size should be equal
+// to the most significant byte of the record length value, minus the Handshake
+// record header size.
+// Again, we make the assumption that ClientHello messages are not fragmented,
+// so their max size must be <= 16KiB. This also just happens to be a lower
+// number than the minimum FS4/FRT packet type byte at the same location.
+// Oooh yeah, leaky abstractions to the rescue!
+inline bool handshake_record_size_matches_length(const char* buf, uint16_t length) {
+ return (static_cast<unsigned char>(buf[7]) == ((length - 4) >> 8));
+}
+
+} // anon ns
+
+TlsSnoopingResult snoop_client_hello_header(const char* buf) noexcept {
+ if (!is_tls_handshake_packet(buf)) {
+ return TlsSnoopingResult::HandshakeMismatch;
+ }
+ if (!is_expected_tls_protocol_version(buf)) {
+ return TlsSnoopingResult::ProtocolVersionMismatch;
+ }
+ // Length of TLS record follows. Must be <= 16KiB + 2048 (16KiB + 256 on v1.3).
+ // We expect that the first record contains _only_ a ClientHello with no coalescing
+ // and no fragmentation. This is technically a violation of the TLS spec, but this
+ // particular detection logic is only intended to be used against other Vespa nodes
+ // where we control frame sizes and where such fragmentation should not take place.
+ // We also do not support TLSv1.3 0-RTT which may trigger early data.
+ uint16_t length = tls_record_length(buf);
+ if ((length < 4) || (length > (16384 + 2048))) {
+ return TlsSnoopingResult::RecordSizeRfcViolation;
+ }
+ if (!is_client_hello_handshake_record(buf)) {
+ return TlsSnoopingResult::RecordNotClientHello;
+ }
+ if (!client_hello_record_size_within_expected_bounds(buf)) {
+ return TlsSnoopingResult::ClientHelloRecordTooBig;
+ }
+ if (!handshake_record_size_matches_length(buf, length)) {
+ return TlsSnoopingResult::ExpectedRecordSizeMismatch;
+ }
+ // Hooray! It very probably most likely is a TLS connection! :D
+ return TlsSnoopingResult::ProbablyTls;
+}
+
+const char* to_string(TlsSnoopingResult result) noexcept {
+ switch (result) {
+ case TlsSnoopingResult::ProbablyTls: return "ProbablyTls";
+ case TlsSnoopingResult::HandshakeMismatch: return "HandshakeMismatch";
+ case TlsSnoopingResult::ProtocolVersionMismatch: return "ProtocolVersionMismatch";
+ case TlsSnoopingResult::RecordSizeRfcViolation: return "RecordSizeRfcViolation";
+ case TlsSnoopingResult::RecordNotClientHello: return "RecordNotClientHello";
+ case TlsSnoopingResult::ClientHelloRecordTooBig: return "ClientHelloRecordTooBig";
+ case TlsSnoopingResult::ExpectedRecordSizeMismatch: return "ExpectedRecordSizeMismatch";
+ }
+ abort();
+}
+
+std::ostream& operator<<(std::ostream& os, TlsSnoopingResult result) {
+ os << to_string(result);
+ return os;
+}
+
+const char* describe_result(TlsSnoopingResult result) noexcept {
+ switch (result) {
+ case TlsSnoopingResult::ProbablyTls:
+ return "client data matches TLS heuristics, very likely a TLS connection";
+ case TlsSnoopingResult::HandshakeMismatch:
+ return "not a TLS handshake packet";
+ case TlsSnoopingResult::ProtocolVersionMismatch:
+ return "ProtocolVersion mismatch";
+ case TlsSnoopingResult::RecordSizeRfcViolation:
+ return "ClientHello record size is greater than RFC allows";
+ case TlsSnoopingResult::RecordNotClientHello:
+ return "record is not ClientHello";
+ case TlsSnoopingResult::ClientHelloRecordTooBig:
+ return "ClientHello record is too big (fragmented?)";
+ case TlsSnoopingResult::ExpectedRecordSizeMismatch:
+ return "ClientHello vs Handshake header record size mismatch";
+ }
+ abort();
+}
+
+}
diff --git a/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.h b/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.h
new file mode 100644
index 00000000000..f53e136e597
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.h
@@ -0,0 +1,32 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <iosfwd>
+#include <stddef.h>
+
+namespace vespalib::net::tls::snooping {
+
+constexpr inline size_t min_header_bytes_to_observe() { return 8; }
+
+enum class TlsSnoopingResult {
+ ProbablyTls, // Very safe to assume TLSv1.x client
+ HandshakeMismatch, // Almost guaranteed to trigger for plaintext RPC
+ ProtocolVersionMismatch,
+ RecordSizeRfcViolation,
+ RecordNotClientHello,
+ ClientHelloRecordTooBig,
+ ExpectedRecordSizeMismatch
+};
+
+const char* to_string(TlsSnoopingResult) noexcept;
+std::ostream& operator<<(std::ostream& os, TlsSnoopingResult);
+
+// Precondition: buf is at least `min_header_bytes_to_observe()` bytes long. This is the minimum amount
+// of bytes always sent for a packet in our existing plaintext production protocols and
+// therefore the maximum we can expect to always be present.
+// Yes, this is a pragmatic and delightfully leaky abstraction.
+TlsSnoopingResult snoop_client_hello_header(const char* buf) noexcept;
+
+const char* describe_result(TlsSnoopingResult result) noexcept;
+
+}
diff --git a/vespalib/src/vespa/vespalib/util/slaveproc.h b/vespalib/src/vespa/vespalib/util/slaveproc.h
index 72b20cfb069..c08a13f0b1d 100644
--- a/vespalib/src/vespa/vespalib/util/slaveproc.h
+++ b/vespalib/src/vespa/vespalib/util/slaveproc.h
@@ -8,14 +8,9 @@
#include <queue>
#include "sync.h"
-namespace vespalib {
-
-namespace slaveproc {
-
-class Timer;
-
-} // namespace slaveproc
+namespace vespalib::slaveproc { class Timer; }
+namespace vespalib {
/**
* @brief Slave Process utility class for running external programs
*
@@ -56,12 +51,12 @@ private:
bool _failed;
int _exitCode;
- SlaveProc(const SlaveProc &); // no copy
- SlaveProc &operator=(const SlaveProc &); // no assignment
-
void checkProc();
public:
+ SlaveProc(const SlaveProc &) = delete;
+ SlaveProc &operator=(const SlaveProc &) = delete;
+
/**
* @brief Run a slave process
*