diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /documentapi |
Publish
Diffstat (limited to 'documentapi')
765 files changed, 41728 insertions, 0 deletions
diff --git a/documentapi/.gitignore b/documentapi/.gitignore new file mode 100644 index 00000000000..9690261769a --- /dev/null +++ b/documentapi/.gitignore @@ -0,0 +1,5 @@ +documentapi.iml +target +/pom.xml.build +Makefile +Testing diff --git a/documentapi/CMakeLists.txt b/documentapi/CMakeLists.txt new file mode 100644 index 00000000000..c6eaab01917 --- /dev/null +++ b/documentapi/CMakeLists.txt @@ -0,0 +1,38 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + DEPENDS + fastos + vespalog + config_cloudconfig + vespalib + fnet + document + slobrok + messagebus + metrics + configdefinitions + vdslib + + LIBS + src/vespa/binref + src/vespa/documentapi + src/vespa/documentapi/loadtypes + src/vespa/documentapi/messagebus + src/vespa/documentapi/messagebus/messages + src/vespa/documentapi/messagebus/policies + src/vespa/documentapi/messagebus/systemstate + + TEST_DEPENDS + messagebus_messagebus-test + + TESTS + src/tests/loadtypes + src/tests/messagebus + src/tests/messages + src/tests/policies + src/tests/policyfactory + src/tests/priority + src/tests/replymerger + src/tests/routablefactory + src/tests/systemstate +) diff --git a/documentapi/OWNERS b/documentapi/OWNERS new file mode 100644 index 00000000000..0e39145d8c3 --- /dev/null +++ b/documentapi/OWNERS @@ -0,0 +1 @@ +dybdahl diff --git a/documentapi/README b/documentapi/README new file mode 100644 index 00000000000..311f3695822 --- /dev/null +++ b/documentapi/README @@ -0,0 +1 @@ +This is the API needed to access the Vespa System. diff --git a/documentapi/pom.xml b/documentapi/pom.xml new file mode 100644 index 00000000000..7f103965893 --- /dev/null +++ b/documentapi/pom.xml @@ -0,0 +1,108 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>documentapi</artifactId> + <packaging>container-plugin</packaging> + <version>6-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>messagebus</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vdslib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>document</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configdefinitions</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:deprecation</arg> + <arg>-Xlint:unchecked</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>javacc-maven-plugin</artifactId> + <executions> + <execution> + <id>javacc</id> + <goals> + <goal>javacc</goal> + </goals> + <configuration> + <lookAhead>1</lookAhead> + <isStatic>false</isStatic> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/documentapi/src/.gitignore b/documentapi/src/.gitignore new file mode 100644 index 00000000000..afa6b2d43c5 --- /dev/null +++ b/documentapi/src/.gitignore @@ -0,0 +1,4 @@ +Makefile.ini +config_command.sh +documentapi.mak +project.dsw diff --git a/documentapi/src/Doxyfile b/documentapi/src/Doxyfile new file mode 100644 index 00000000000..1ffe98ec28a --- /dev/null +++ b/documentapi/src/Doxyfile @@ -0,0 +1,1213 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Doxyfile 1.4.1 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = documentapi + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = ../../doc + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, +# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, +# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, +# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, +# Swedish, and Ukrainian. + +OUTPUT_LANGUAGE = English + +# This tag can be used to specify the encoding used in the generated output. +# The encoding is not always determined by the language that is chosen, +# but also whether or not the output is meant for Windows or non-Windows users. +# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES +# forces the Windows encoding (this is the default for the Windows binary), +# whereas setting the tag to NO uses a Unix-style encoding (the default for +# all platforms other than Windows). + +USE_WINDOWS_ENCODING = NO + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = YES + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like the Qt-style comments (thus requiring an +# explicit @brief command for a brief description. + +JAVADOC_AUTOBRIEF = YES + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the DETAILS_AT_TOP tag is set to YES then Doxygen +# will output the detailed description near the top, like JavaDoc. +# If set to NO, the detailed description appears after the member +# documentation. + +DETAILS_AT_TOP = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources +# only. Doxygen will then generate output that is more tailored for Java. +# For instance, namespaces will be presented as packages, qualified scopes +# will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. + +SHOW_DIRECTORIES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from the +# version control system). Doxygen will invoke the program by executing (via +# popen()) the command <command> <input-file>, where <command> is the value of +# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file +# provided by doxygen. Whatever the progam writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = documentapi + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm + +FILE_PATTERNS = *.h *.cpp + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. + +EXCLUDE_PATTERNS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command <filter> <input-file>, where <filter> +# is the value of the INPUT_FILTER tag, and <input-file> is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES (the default) +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = YES + +# If the REFERENCES_RELATION tag is set to YES (the default) +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = YES + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = NO + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be +# generated containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, +# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are +# probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = NO + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = NO + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_PREDEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = IAM_DOXYGEN + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will +# generate a call dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width +# (in pixels) of the graphs generated by dot. If a graph becomes larger than +# this value, doxygen will try to truncate the graph, so that it fits within +# the specified constraint. Beware that most browsers cannot cope with very +# large images. + +MAX_DOT_GRAPH_WIDTH = 1024 + +# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height +# (in pixels) of the graphs generated by dot. If a graph becomes larger than +# this value, doxygen will try to truncate the graph, so that it fits within +# the specified constraint. Beware that most browsers cannot cope with very +# large images. + +MAX_DOT_GRAPH_HEIGHT = 1024 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that a graph may be further truncated if the graph's +# image dimensions are not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH +# and MAX_DOT_GRAPH_HEIGHT). If 0 is used for the depth value (the default), +# the graph is not depth-constrained. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, which results in a white background. +# Warning: Depending on the platform used, enabling this option may lead to +# badly anti-aliased labels on the edges of a graph (i.e. they become hard to +# read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to the search engine +#--------------------------------------------------------------------------- + +# The SEARCHENGINE tag specifies whether or not a search engine should be +# used. If set to NO the values of all tags below this one will be ignored. + +SEARCHENGINE = NO diff --git a/documentapi/src/main/docapi-with-dependencies.xml b/documentapi/src/main/docapi-with-dependencies.xml new file mode 100644 index 00000000000..f16f31e0398 --- /dev/null +++ b/documentapi/src/main/docapi-with-dependencies.xml @@ -0,0 +1,19 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<assembly> + <id>jar-with-dependencies</id> + <formats> + <format>jar</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <dependencySets> + <dependencySet> + <unpack>true</unpack> + <scope>runtime</scope> + </dependencySet> + </dependencySets> + <fileSets> + <fileSet> + <directory>${project.build.outputDirectory}</directory> + </fileSet> + </fileSets> +</assembly> diff --git a/documentapi/src/main/java/com/yahoo/documentapi/AckToken.java b/documentapi/src/main/java/com/yahoo/documentapi/AckToken.java new file mode 100644 index 00000000000..6bacad38786 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/AckToken.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * Token to use to acknowledge data for visiting. + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public class AckToken { + + public Object ackObject; + + /** + * Creates ack token from the supplied parameter. + * + * @param ackObject the object to use to ack data + */ + public AckToken(Object ackObject) { + this.ackObject = ackObject; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/AsyncParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/AsyncParameters.java new file mode 100644 index 00000000000..3e06d5f8d94 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/AsyncParameters.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * Parameters for creating an async session + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class AsyncParameters extends Parameters { + + private ResponseHandler responseHandler = null; + + public ResponseHandler getResponseHandler() { + return responseHandler; + } + + public AsyncParameters setResponseHandler(ResponseHandler responseHandler) { + this.responseHandler = responseHandler; + return this; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java new file mode 100644 index 00000000000..183e4ea63d3 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java @@ -0,0 +1,142 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; + +/** + * <p>A session for asynchronous access to a document repository. + * This class provides document repository writes and random access with high + * throughput.</p> + * + * <p>All operations which are <i>accepted</i> by an async session will cause one or more + * {@link Response responses} to be returned within the timeout limit. When an operation fails, + * the response will contain the argument which was submitted to the operation.</p> + * + * @author bratseth + */ +public interface AsyncSession extends Session { + + /** + * <p>Puts a document. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. + * The response returned later will either be a success, or contain the document submitted here. + * If it was not a success, this method has no further effects.</p> + * + * @param document the Document to put + * @return the synchronous result of this operation + */ + Result put(Document document); + + /** + * <p>Puts a document. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. + * The response returned later will either be a success, or contain the document submitted here. + * If it was not a success, this method has no further effects.</p> + * + * @param document the Document to put + * @return the synchronous result of this operation + */ + Result put(Document document, DocumentProtocol.Priority priority); + + /** + * <p>Gets a document. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. + * The response returned later will contain the requested document if it is a success. + * If it was not a success, this method has no further effects.</p> + * + * @param id the id of the document to get + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support retrieving + */ + Result get(DocumentId id); + + /** + * <p>Gets a document. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. + * The response returned later will contain the requested document if it is a success. + * If it was not a success, this method has no further effects.</p> + * + * @param id the id of the document to get + * @param priority The priority with which to perform this operation. + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support retrieving + */ + Result get(DocumentId id, boolean headersOnly, DocumentProtocol.Priority priority); + + /** + * <p>Removes a document if it is present. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link RemoveResponse} objects to appear within the timeout time of this session. + * The response returned later will either be a success, or contain the document id submitted here. + * If it was not a success, this method has no further effects.</p> + * + * @param id the id of the document to remove + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support removal + */ + Result remove(DocumentId id); + + /** + * <p>Removes a document if it is present. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link DocumentIdResponse} objects to apprear within the timeout time of this session. + * The response returned later will either be a success, or contain the document id submitted here. + * If it was not a success, this method has no further effects.</p> + * + * @param id the id of the document to remove + * @param priority The priority with which to perform this operation. + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support removal + */ + Result remove(DocumentId id, DocumentProtocol.Priority priority); + + /** + * <p>Updates a document. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link DocumentUpdateResponse} within the timeout time of this session. + * The returned response returned later will either be a success or contain the update submitted here. + * If it was not a success, this method has no further effects.</p> + * + * @param update the updates to perform + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support update + */ + Result update(DocumentUpdate update); + + /** + * <p>Updates a document. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link DocumentUpdateResponse} within the timeout time of this session. + * The returned response returned later will either be a success or contain the update submitted here. + * If it was not a success, this method has no further effects.</p> + * + * @param update the updates to perform + * @param priority The priority with which to perform this operation. + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support update + */ + Result update(DocumentUpdate update, DocumentProtocol.Priority priority); + + /** + * Returns the current send window size of the session. + * + * @return Returns the window size. + */ + double getCurrentWindowSize(); + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/BucketListVisitorResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/BucketListVisitorResponse.java new file mode 100644 index 00000000000..1ccb9c1c2fa --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/BucketListVisitorResponse.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.BucketId; +import com.yahoo.documentapi.messagebus.protocol.DocumentListEntry; + +import java.util.List; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class BucketListVisitorResponse extends VisitorResponse { + private BucketId bucketId; + private List<DocumentListEntry> documents; + + public BucketListVisitorResponse(BucketId bucketId, List<DocumentListEntry> documents, AckToken token) { + super(token); + this.bucketId = bucketId; + this.documents = documents; + } + + public BucketId getBucketId() { + return bucketId; + } + + public List<DocumentListEntry> getDocuments() { + return documents; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java new file mode 100644 index 00000000000..0d781e4ca95 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.config.subscription.ConfigSubscriber; + +/** + * <p>This is the starting point of the <b>document api</b>. This api provides + * access to documents in a document repository. The document api contains four + * separate access types: </p> + * + * <ul><li><b>Synchronous random access</b> - provided by {@link SyncSession}, + * allows simple access where throughput is not a concern.</li> + * <li><b>Asynchronous random access</b> - provided by {@link AsyncSession}, + * allows document repository writes and random access with high + * throughput.</li> + * <li><b>Visiting</b> - provided by {@link VisitorSession}, allows a set of + * documents to be accessed in an order decided by the document repository. This + * allows much higher read throughput than random access.</li> + * <li><b>Subscription</b> - provided by {@link SubscriptionSession}, allows + * changes to a defined set of documents in the repository to be + * visited.</li></ul> + * + * <p>This class is the factory for creating the four session types mentioned + * above.</p> + * + * <p>There may be multiple implementations of the document api classes. If + * default configuration is sufficient, use the {@link #createDefault} method to + * return a running document access. Note that there are running threads within + * an access object, so you must shut it down when done.</p> + * + * <p>An implementation of the Document Api may support just a subset of the + * access types defined in this interface. For example, some document + * repositories, like indexes, are <i>write only</i>. Others may support random + * access, but not visiting and subscription. Any method which is not supported + * by the underlying implementation will throw + * UnsupportedOperationException.</p> + * + * <p>Access to this class is thread-safe.</p> + * + * @author bratseth + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar Rosenvinge</a> + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class DocumentAccess { + + protected DocumentTypeManager documentMgr; + protected ConfigSubscriber documentTypeManagerConfig; + + /** + * <p>This is a convenience method to return a document access object with + * all default parameter values. The client that calls this method is also + * responsible for shutting the object down when done. If an error occurred + * while attempting to create such an object, this method will throw an + * exception.</p> + * + * @return A running document access object with all default configuration. + */ + public static DocumentAccess createDefault() { + return new com.yahoo.documentapi.messagebus.MessageBusDocumentAccess(); + } + + /** + * <p>Constructs a new document access object.</p> + * + * @param params The parameters to use for setup. + */ + protected DocumentAccess(DocumentAccessParams params) { + super(); + documentMgr = new DocumentTypeManager(); + documentTypeManagerConfig = DocumentTypeManagerConfigurer.configure(documentMgr, params.getDocumentManagerConfigId()); + } + + /** + * <p>Returns a session for synchronous document access. Use this for simple + * access.</p> + * + * @param parameters The parameters of this sync session. + * @return A session to use for synchronous document access. + * @throws UnsupportedOperationException If this access implementation does + * not support synchronous access. + * @throws RuntimeException If an error prevented the session + * from being created. + */ + public abstract SyncSession createSyncSession(SyncParameters parameters); + + /** + * <p>Returns a session for asynchronous document access. Use this if high + * operation throughput is required.</p> + * + * @param parameters The parameters of this async session. + * @return A session to use for asynchronous document access. + * @throws UnsupportedOperationException If this access implementation does + * not support asynchronous access. + * @throws RuntimeException If an error prevented the session + * from being created. + */ + public abstract AsyncSession createAsyncSession(AsyncParameters parameters); + + /** + * <p>Run a visitor with the given visitor parameters, and get the result + * back here.</p> + * + * @param parameters The parameters of this visitor session. + * @return A session used to track progress of the visitor and get the + * actual data returned. + * @throws UnsupportedOperationException If this access implementation does + * not support visiting. + * @throws RuntimeException If an error prevented the session + * from being created. + * @throws ParseException If the document selection string + * could not be parsed. + */ + public abstract VisitorSession createVisitorSession(VisitorParameters parameters) throws ParseException; + + /** + * <p>Creates a destination session for receiving data from visiting. The + * visitor must be started and progress tracked through a visitor + * session.</p> + * + * @param parameters The parameters of this visitor destination session. + * @return A session used to get the actual data returned. + * @throws UnsupportedOperationException If this access implementation does + * not support visiting. + */ + public abstract VisitorDestinationSession createVisitorDestinationSession(VisitorDestinationParameters parameters); + + /** + * <p>Creates a subscription and returns a session for getting data from + * it. Use this to get document operations being done by other parties.</p> + * + * @param parameters The parameters of this subscription session. + * @return A session to use for document subscription. + * @throws UnsupportedOperationException If this access implementation does + * not support subscription. + * @throws RuntimeException If an error prevented the session + * from being created. + */ + public abstract SubscriptionSession createSubscription(SubscriptionParameters parameters); + + /** + * <p>Returns a session for document subscription. Use this to get document + * operations being done by other parties.</p> + * + * @param parameters The parameters of this subscription session. + * @return A session to use for document subscription. + * @throws UnsupportedOperationException If this access implementation does + * not support subscription. + * @throws RuntimeException If an error prevented the session + * from being created. + */ + public abstract SubscriptionSession openSubscription(SubscriptionParameters parameters); + + /** + * <p>Shuts down the underlying sessions used by this DocumentAccess; + * subsequent use of this DocumentAccess will throw unspecified exceptions, + * depending on implementation.</p> + */ + public abstract void shutdown(); + + /** + * <p>Returns the {@link DocumentTypeManager} used by this + * DocumentAccess.</p> + * + * @return The document type manager. + */ + public DocumentTypeManager getDocumentTypeManager() { + return documentMgr; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessException.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessException.java new file mode 100644 index 00000000000..4c9c3b0e817 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessException.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import java.util.Set; +import java.util.HashSet; + +/** + * General exception thrown from various methods in the Vespa Document API. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentAccessException extends RuntimeException { + + private Set<Integer> errorCodes = new HashSet<>(); + + public Set<Integer> getErrorCodes() { + return errorCodes; + } + + public DocumentAccessException() { + super(); + } + + public DocumentAccessException(String message) { + super(message); + } + + public DocumentAccessException(String message, Set<Integer> errorCodes) { + super(message); + this.errorCodes = errorCodes; + } + + public DocumentAccessException(String message, Throwable cause) { + super(message, cause); + } + + public DocumentAccessException(Throwable cause) { + super(cause); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessParams.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessParams.java new file mode 100755 index 00000000000..57cfdbd32e1 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessParams.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi;
+
+/**
+ * Superclass of the classes which contains the parameters for creating or opening a document access.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DocumentAccessParams {
+
+ // The id to resolve to document manager config.
+ private String documentManagerConfigId = "client";
+
+ /**
+ * Returns the config id that the document manager should subscribe to.
+ *
+ * @return The config id.
+ */
+ public String getDocumentManagerConfigId() {
+ return documentManagerConfigId;
+ }
+
+ /**
+ * Sets the config id that the document manager should subscribe to.
+ *
+ * @param configId The config id.
+ * @return This, to allow chaining.
+ */
+ public DocumentAccessParams setDocumentManagerConfigId(String configId) {
+ documentManagerConfigId = configId;
+ return this;
+ }
+}
\ No newline at end of file diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java new file mode 100644 index 00000000000..4d79f0973cb --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.DocumentId; + +/** + * The asynchronous response to a document remove operation. + * This is a <i>value object</i>. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentIdResponse extends Response { + + /** The document id of this response, if any */ + private DocumentId documentId = null; + + /** Creates a successful response */ + public DocumentIdResponse(long requestId) { + super(requestId); + } + + /** + * Creates a successful response containing a document id + * + * @param documentId the DocumentId to encapsulate in the Response + */ + public DocumentIdResponse(long requestId, DocumentId documentId) { + super(requestId); + this.documentId = documentId; + } + + /** + * Creates a response containing a textual message + * + * @param textMessage the message to encapsulate in the Response + * @param success true if the response represents a successful call + */ + public DocumentIdResponse(long requestId, String textMessage, boolean success) { + super(requestId, textMessage, success); + } + + /** + * Creates a response containing a textual message and/or a document id + * + * @param documentId the DocumentId to encapsulate in the Response + * @param textMessage the message to encapsulate in the Response + * @param success true if the response represents a successful call + */ + public DocumentIdResponse(long requestId, DocumentId documentId, String textMessage, boolean success) { + super(requestId, textMessage, success); + this.documentId = documentId; + } + + + /** + * Returns the document id of this response, or null if there is none + * + * @return the DocumentId, or null + */ + public DocumentId getDocumentId() { return documentId; } + + public int hashCode() { + return super.hashCode() + (documentId == null ? 0 : documentId.hashCode()); + } + + public boolean equals(Object o) { + if (!(o instanceof DocumentIdResponse)) { + return false; + } + + DocumentIdResponse docResp = (DocumentIdResponse) o; + + return super.equals(docResp) && ((documentId == null && docResp.documentId == null) || + (documentId != null && docResp.documentId != null && documentId.equals(docResp.documentId))); + } + + public String toString() { + return "DocumentId" + super.toString() + (documentId == null ? "" : " " + documentId); + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentListVisitorResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentListVisitorResponse.java new file mode 100644 index 00000000000..945538244c4 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentListVisitorResponse.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.vdslib.DocumentList; + +/** + * Visitor response containing a document list. All visitor responses have ack + * tokens that must be acked. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class DocumentListVisitorResponse extends VisitorResponse { + private DocumentList documents; + + /** + * Creates visitor response containing a document list and an ack token. + * + * @param docs the document list + * @param ack the ack token + */ + public DocumentListVisitorResponse(DocumentList docs, AckToken ack) { + super(ack); + documents = docs; + } + + /** @return the document list */ + public DocumentList getDocumentList() { return documents; } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java new file mode 100644 index 00000000000..5a32b19c342 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.Document; +import com.yahoo.component.Version; + +/** + * The asynchronous response to a document put or get operation. + * This is a <i>value object</i>. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentResponse extends Response { + + /** The document of this response, if any */ + private Document document = null; + + /** Creates a successful response */ + public DocumentResponse(long requestId) { + super(requestId); + } + + /** + * Creates a successful response containing a document + * + * @param document the Document to encapsulate in the Response + */ + public DocumentResponse(long requestId, Document document) { + super(requestId); + this.document = document; + } + + /** + * Creates a response containing a textual message + * + * @param textMessage the message to encapsulate in the Response + * @param success true if the response represents a successful call + */ + public DocumentResponse(long requestId, String textMessage, boolean success) { + super(requestId, textMessage, success); + } + + /** + * Creates a response containing a textual message and/or a document + * + * @param document the Document to encapsulate in the Response + * @param textMessage the message to encapsulate in the Response + * @param success true if the response represents a successful call + */ + public DocumentResponse(long requestId, Document document, String textMessage, boolean success) { + super(requestId, textMessage, success); + this.document = document; + } + + + /** + * Returns the document of this response, or null if there is none + * + * @return the Document, or null + */ + public Document getDocument() { return document; } + + public int hashCode() { + return super.hashCode() + (document == null ? 0 : document.hashCode()); + } + + public boolean equals(Object o) { + if (!(o instanceof DocumentResponse)) { + return false; + } + + DocumentResponse docResp = (DocumentResponse) o; + + return super.equals(docResp) && ((document == null && docResp.document == null) || + (document != null && docResp.document != null && document.equals(docResp.document))); + } + + public String toString() { + return "Document" + super.toString() + (document == null ? "" : " " + document); + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java new file mode 100644 index 00000000000..44044a41bc3 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.DocumentUpdate; + +/** + * The asynchronous response to a document update operation. + * This is a <i>value object</i>. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentUpdateResponse extends Response { + + /** The document update of this response, if any */ + private DocumentUpdate documentUpdate = null; + + /** Creates a successful response */ + public DocumentUpdateResponse(long requestId) { + super(requestId); + } + + /** + * Creates a successful response containing a document update + * + * @param documentUpdate the DocumentUpdate to encapsulate in the Response + */ + public DocumentUpdateResponse(long requestId, DocumentUpdate documentUpdate) { + super(requestId); + this.documentUpdate = documentUpdate; + } + + /** + * Creates a response containing a textual message + * + * @param textMessage the message to encapsulate in the Response + * @param success true if the response represents a successful call + */ + public DocumentUpdateResponse(long requestId, String textMessage, boolean success) { + super(requestId, textMessage, success); + } + + /** + * Creates a response containing a textual message and/or a document update + * + * @param documentUpdate the DocumentUpdate to encapsulate in the Response + * @param textMessage the message to encapsulate in the Response + * @param success true if the response represents a successful call + */ + public DocumentUpdateResponse(long requestId, DocumentUpdate documentUpdate, String textMessage, boolean success) { + super(requestId, textMessage, success); + this.documentUpdate = documentUpdate; + } + + + /** + * Returns the document update of this response or null if there is none + * + * @return the DocumentUpdate, or null + */ + public DocumentUpdate getDocumentUpdate() { return documentUpdate; } + + public int hashCode() { + return super.hashCode() + (documentUpdate == null ? 0 : documentUpdate.hashCode()); + } + + public boolean equals(Object o) { + if (!(o instanceof DocumentUpdateResponse)) { + return false; + } + + DocumentUpdateResponse docResp = (DocumentUpdateResponse) o; + + return super.equals(docResp) && ((documentUpdate == null && docResp.documentUpdate == null) || ( + documentUpdate != null && docResp.documentUpdate != null && + documentUpdate.equals(docResp.documentUpdate))); + } + + public String toString() { + return "Update" + super.toString() + (documentUpdate == null ? "" : " " + documentUpdate); + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentVisitor.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentVisitor.java new file mode 100644 index 00000000000..fd0e9725866 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentVisitor.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * Visitor that simply returns documents found in storage. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class DocumentVisitor extends VisitorParameters { + + /** + * Create a document visitor. + * + * @param documentSelection The document selection criteria. + */ + public DocumentVisitor(String documentSelection) { + super(documentSelection); + } + + // Inherited docs from VisitorParameters + public String getVisitorLibrary() { return "DumpVisitor"; } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DumpVisitorDataHandler.java b/documentapi/src/main/java/com/yahoo/documentapi/DumpVisitorDataHandler.java new file mode 100644 index 00000000000..8d09feec707 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/DumpVisitorDataHandler.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.messagebus.Message; + +/** + * <p>Implementation of VisitorDataHandler which invokes onDocument() for each + * received document and onRemove() for each document id that was returned as + * part of a remove entry. The latter only applies if the visitor was run with + * visitRemoves enabled.</p> + * + * <p>NOTE: onDocument and onRemove may be called in a re-entrant manner, as + * these run on top of a thread pool. Any mutation of shared state must be + * appropriately synchronized.</p> + */ +public abstract class DumpVisitorDataHandler extends VisitorDataHandler { + + public DumpVisitorDataHandler() { + } + + @Override + public void onMessage(Message m, AckToken token) { + if (m instanceof PutDocumentMessage) { + PutDocumentMessage pm = (PutDocumentMessage)m; + + onDocument(pm.getDocumentPut().getDocument(), pm.getTimestamp()); + } else if (m instanceof RemoveDocumentMessage) { + RemoveDocumentMessage rm = (RemoveDocumentMessage)m; + onRemove(rm.getDocumentId()); + } else { + throw new UnsupportedOperationException("Received unsupported message " + m.toString() + " to dump visitor data handler. This handler only accepts Put and Remove"); + } + ack(token); + } + + /** + * Called when a document is received. + * + * May be called from multiple threads concurrently. + * + * @param doc The document found + * @param timeStamp The time when the document was stored. + */ + public abstract void onDocument(Document doc, long timeStamp); + + /** + * Called when a remove is received. + * + * May be called from multiple threads concurrently. + * + * @param id The document id that was removed. + */ + public abstract void onRemove(DocumentId id); + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/EmptyBucketsVisitorResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/EmptyBucketsVisitorResponse.java new file mode 100644 index 00000000000..68431f8ecaa --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/EmptyBucketsVisitorResponse.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.BucketId; + +import java.util.List; + +/** + * Response containing list of empty buckets. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class EmptyBucketsVisitorResponse extends VisitorResponse { + private List<BucketId> bucketIds; + /** + * Creates visitor response containing an ack token. + * + * @param bucketIds the empty buckets + * @param token the ack token + */ + public EmptyBucketsVisitorResponse(List<BucketId> bucketIds, AckToken token) { + super(token); + this.bucketIds = bucketIds; + } + + public List<BucketId> getBucketIds() { return bucketIds; } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Parameters.java b/documentapi/src/main/java/com/yahoo/documentapi/Parameters.java new file mode 100644 index 00000000000..fdf57cecfc8 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/Parameters.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * Superclass of the classes which contains the parameters for creating or opening a session. This is currently empty, + * but keeping this parameter hierarchy in place means that we can later add parameters with default values that all + * clients will be able to use with no code changes. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class Parameters { + // empty +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java new file mode 100644 index 00000000000..ce200c89751 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java @@ -0,0 +1,816 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import java.util.Collections; +import java.util.Map; +import java.util.TreeMap; +import java.util.logging.Logger; + +import com.yahoo.document.*; +import com.yahoo.document.serialization.*; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.objects.Serializer; + +/** + * Token to use to keep track of progress for visiting. Can be used to resume + * visiting if visiting has been aborted for any reason. + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + * @author <a href="mailto:vekterli@yahoo-inc.com">Tor Brede Vekterli</a> + */ +public class ProgressToken { + + private static final Logger log = Logger.getLogger(ProgressToken.class.getName()); + /** + * Any bucket kept track of by a <code>ProgressToken</code> instance may + * be in one of two states: pending or active. <em>Pending</em> means that + * a bucket may be returned by a VisitorIterator, i.e. it is ready for + * visiting, while <em>active</em> means that a bucket is currently being + * visited and may thus not be returned from an iterator. + * + * Getting a pending bucket via the iterator sets its state to active and + * updating an active bucket sets its state back to pending again. + */ + public enum BucketState { + BUCKET_PENDING, + BUCKET_ACTIVE + } + + public static final BucketId NULL_BUCKET = new BucketId(); + public static final BucketId FINISHED_BUCKET = new BucketId(Integer.MAX_VALUE); + + /** + * When a bucket has its state kept by the progress token, we need to + * discern between active buckets (i.e. those that have been returned by + * {@link com.yahoo.documentapi.VisitorIterator#getNext()} but have not + * yet been update()'d) and pending buckets (i.e. those that have been + * update()'d and may be returned by getNext() at some point) + */ + public static class BucketEntry { + private BucketId progress; + private BucketState state; + + private BucketEntry(BucketId progress, BucketState state) { + this.progress = progress; + this.state = state; + } + + public BucketId getProgress() { + return progress; + } + + public void setProgress(BucketId progress) { + this.progress = progress; + } + + public BucketState getState() { + return state; + } + + public void setState(BucketState state) { + this.state = state; + } + } + + /** + * For consistent bucket key ordering, we need to ensure that reverse bucket + * IDs that have their MSB set actually are compared as being greater than + * those that don't. This is yet another issue caused by Java's lack of + * unsigned integers. + */ + public static class BucketKeyWrapper implements Comparable<BucketKeyWrapper> + { + private long key; + + public BucketKeyWrapper(long key) { + this.key = key; + } + + public int compareTo(BucketKeyWrapper other) { + if ((key & 0x8000000000000000L) != (other.key & 0x8000000000000000L)) { + // MSBs differ + return ((key >>> 63) > (other.key >>> 63)) ? 1 : -1; + } + // Mask off MSBs since we've already checked them, and with MSB != 1 + // we know the ordering will be consistent + if ((key & 0x7FFFFFFFFFFFFFFFL) < (other.key & 0x7FFFFFFFFFFFFFFFL)) { + return -1; + } else if ((key & 0x7FFFFFFFFFFFFFFFL) > (other.key & 0x7FFFFFFFFFFFFFFFL)) { + return 1; + } + return 0; + } + + public long getKey() { + return key; + } + + public BucketId toBucketId() { + return new BucketId(keyToBucketId(key)); + } + + @Override + public String toString() { + return Long.toHexString(key); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || !(o instanceof BucketKeyWrapper)) return false; + return key == ((BucketKeyWrapper)o).key; + } + + @Override + public int hashCode() { + return (int) (key ^ (key >>> 32)); + } + } + + /** + * By default, a ProgressToken's distribution bit count is set to the VDS + * standard value of 16, but it may be changed via the iterator using it + * or by a bucket source when importing an existing progress + */ + private int distributionBits = 16; + + private TreeMap<BucketKeyWrapper, BucketEntry> buckets = new TreeMap<BucketKeyWrapper, BucketEntry>(); + private long activeBucketCount = 0; + private long pendingBucketCount = 0; + private long finishedBucketCount = 0; + private long totalBucketCount = 0; + private TreeMap<BucketId, BucketId> failedBuckets = new TreeMap<BucketId, BucketId>(); + private String firstErrorMsg; + + /** + * The bucket cursor (i.e. current position in the bucket space) is used + * by the range source + */ + private long bucketCursor = 0; + + /** + * Set by the VisitorIterator during a distribution bit change when + * the token contains active/pending buckets with different used-bits + */ + private boolean inconsistentState = false; + + /** + * Creates a progress token. + */ + public ProgressToken() { + } + + public ProgressToken(int distributionBits) { + this.distributionBits = distributionBits; + } + + public ProgressToken(String serialized) { + String[] lines = serialized.split("\\n"); + if (lines.length < 5) { + throw new IllegalArgumentException("Progress file is malformed or a deprecated version"); + } + + // 1st token is simple header text. Just check that it starts with + // a known value. To be 5.0 backwards compatible, we do not check + // the rest of the line. + final String header = lines[0]; + if (!header.startsWith("VDS bucket progress file")) { + throw new IllegalArgumentException("File does not appear to be a " + + "valid VDS progress file; expected first line to start with " + + "'VDS bucket progress file'"); + } + // 2nd token contains the distribution bit count the progress file was + // saved with + distributionBits = Integer.parseInt(lines[1]); + bucketCursor = Long.parseLong(lines[2]); + finishedBucketCount = Long.parseLong(lines[3]); + totalBucketCount = Long.parseLong(lines[4]); + + if (totalBucketCount == finishedBucketCount) { + return; // We're done here + } + + // The rest of the tokens are super:sub bucket progress pairs + for (int i = 5; i < lines.length; ++i) { + String[] buckets = lines[i].split(":"); + if (buckets.length != 2) { + throw new IllegalArgumentException("Bucket progress file contained malformed line"); + } + // Due to Java's fantastically broken handling of unsigned integer + // conversion, the following workaround (i.e. hack) is used for now + // (it was also used in the past for presumably the same reason). + BucketId superId = new BucketId("BucketId(0x" + buckets[0] + ")"); + BucketId subId; + if ("0".equals(buckets[1])) { + subId = new BucketId(); + } else { + subId = new BucketId("BucketId(0x" + buckets[1] + ")"); + } + addBucket(superId, subId, BucketState.BUCKET_PENDING); + } + } + + public ProgressToken(byte[] serialized) { + DocumentDeserializer in = DocumentDeserializerFactory.create42(null, GrowableByteBuffer.wrap(serialized)); + distributionBits = in.getInt(null); + bucketCursor = in.getLong(null); + finishedBucketCount = in.getLong(null); + totalBucketCount = in.getLong(null); + + int progressCount = in.getInt(null); + for (int i = 0; i < progressCount; ++i) { + long key = in.getLong(null); + long value = in.getLong(null); + addBucket(new BucketId(key), new BucketId(value), BucketState.BUCKET_PENDING); + } + } + + public byte[] serialize() { + DocumentSerializer out = DocumentSerializerFactory.create42(new GrowableByteBuffer()); + out.putInt(null, distributionBits); + out.putLong(null, bucketCursor); + out.putLong(null, finishedBucketCount); + out.putLong(null, totalBucketCount); + + out.putInt(null, buckets.size()); + + // Append individual bucket progress + for (Map.Entry<BucketKeyWrapper, ProgressToken.BucketEntry> entry : buckets.entrySet()) { + out.putLong(null, keyToBucketId(entry.getKey().getKey())); + out.putLong(null, entry.getValue().getProgress().getRawId()); + } + + byte[] ret = new byte[out.getBuf().position()]; + out.getBuf().rewind(); + out.getBuf().get(ret); + return ret; + } + + public void addFailedBucket(BucketId superbucket, BucketId progress, String errorMsg) { + BucketId existing = failedBuckets.put(superbucket, progress); + if (existing != null) { + throw new IllegalStateException( + "Attempting to add a superbucket to failed buckets that has already been added: " + + superbucket + ":" + progress); + } + if (firstErrorMsg == null) { + firstErrorMsg = errorMsg; + } + } + + /** + * Get all failed buckets and their progress. Not thread safe. + * @return Unmodifiable map of all failed buckets + */ + public Map<BucketId, BucketId> getFailedBuckets() { + return Collections.unmodifiableMap(failedBuckets); + } + + /** + * Updates internal progress state for <code>bucket</code>, indicating it's currently + * at <code>progress</code>. Assumes that given a completely finished bucket, this + * function will not be called again to further update its progress after + * the finished-update. + * + * @see VisitorIterator#update(com.yahoo.document.BucketId, com.yahoo.document.BucketId) + * + * @param superbucket A valid superbucket ID that exists in <code>buckets</code> + * @param progress The sub-bucket progress that has been reached in the + * superbucket + */ + protected void updateProgress(BucketId superbucket, BucketId progress) { + // There exists a valid case in which the progress bucket may actually contains + // its superbucket from the POV of the storage code, so it has to be handled + // appropriately. + if (!progress.equals(NULL_BUCKET) + && !progress.equals(FINISHED_BUCKET) + && !superbucket.contains(progress) + && !progress.contains(superbucket)) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "updateProgress called with non-contained bucket " + + "pair " + superbucket + ":" + progress + ", but allowing anyway"); + } + } + + BucketKeyWrapper superKey = bucketToKeyWrapper(superbucket); + BucketEntry entry = buckets.get(superKey); + if (entry == null) { + throw new IllegalArgumentException( + "updateProgress with unknown superbucket " + + superbucket + ":" + progress); + } + + // If progress == Integer.MAX_VALUE, we're done. Otherwise, we're not + if (!progress.equals(FINISHED_BUCKET)) { + if (entry.getState() != BucketState.BUCKET_ACTIVE) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "updateProgress called with sub-bucket that was " + + "not marked as active " + superbucket + ":" + progress); + } + } else { + assert(activeBucketCount > 0); + --activeBucketCount; + ++pendingBucketCount; + } + // Mark bucket as pending instead of active, allowing it to be + // reused by the iterator + entry.setState(BucketState.BUCKET_PENDING); + entry.setProgress(progress); + } + else { + // Superbucket is finished, alongside its sub-bucket tree + ++finishedBucketCount; + if (entry.getState() == BucketState.BUCKET_PENDING) { + assert(pendingBucketCount > 0); + --pendingBucketCount; + } else { + assert(activeBucketCount > 0); + --activeBucketCount; + } + buckets.remove(superKey); + } + } + + /** + * <em>For use internally by DocumentAPI code only</em>. Using this method by + * itself will invariably lead to undefined ProgressToken state unless + * care is taken. Leave it to the VisitorIterator. + * + * @param superbucket Superbucket that will be progress-tracked + * @param progress Bucket progress thus far + * @param state Initial bucket state. Only pending buckets may be returned + */ + protected void addBucket(BucketId superbucket, BucketId progress, BucketState state) { + if (progress.equals(FINISHED_BUCKET)) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Trying to add already finished superbucket " + + superbucket + "; ignoring it"); + } + return; + } + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, "Adding bucket pair " + superbucket + + ":" + progress + " with state " + state); + } + + BucketEntry entry = new BucketEntry(progress, state); + BucketEntry existing = buckets.put(bucketToKeyWrapper(superbucket), entry); + if (existing != null) { + throw new IllegalStateException( + "Attempting to add a superbucket that has already been added: " + + superbucket + ":" + progress); + } + if (state == BucketState.BUCKET_PENDING) { + ++pendingBucketCount; + } else { + ++activeBucketCount; + } + } + + /** + * Directly generate a bucket Id key for the <code>n</code>th bucket in + * reverse sorted order. + * + * @param n a number in the range [0, 2**<code>distributionBits</code>) + * @param distributionBits Distribution bit count for the generated key + * @return A value where, if you had generated 2**<code>distributionBits</code> + * {@link BucketId}s with incremental numerical IDs and then sorted + * them on their reverse bit-order keys, the returned key would be equal + * to the nth element in the resulting sorted sequence. + */ + public static long makeNthBucketKey(long n, int distributionBits) { + return (n << (64 - distributionBits)) | distributionBits; + } + + public int getDistributionBitCount() { + return distributionBits; + } + + /** + * Set the internal number of distribution bits, which wil be used for writing + * the progress file and calculating correct percent-wise sub-bucket completion. + * + * Note that simply invoking this method on the progress token does not actually + * change any of its bucket structures/counts! <i>This is the bucket source's + * responsibility</i>, since only it knows how such a change will affect the + * progress semantics. + * + * @param distributionBits new distribution bit value + */ + protected void setDistributionBitCount(int distributionBits) { + this.distributionBits = distributionBits; + } + + public long getActiveBucketCount() { + return activeBucketCount; + } + + public long getBucketCursor() { + return bucketCursor; + } + + protected void setBucketCursor(long bucketCursor) { + this.bucketCursor = bucketCursor; + } + + public long getFinishedBucketCount() { + return finishedBucketCount; + } + + /** + * <em>For use by bucket sources and unit tests only!</em> + * + * @param finishedBucketCount Number of buckets the token has finished + */ + protected void setFinishedBucketCount(long finishedBucketCount) { + this.finishedBucketCount = finishedBucketCount; + } + + public long getTotalBucketCount() { + return totalBucketCount; + } + + /** + * <em>For use by bucket sources and unit tests only!</em> + * + * @param totalBucketCount Total number of buckets that the progress token spans + */ + protected void setTotalBucketCount(long totalBucketCount) { + this.totalBucketCount = totalBucketCount; + } + + public long getPendingBucketCount() { + return pendingBucketCount; + } + + public boolean hasPending() { + return pendingBucketCount > 0; + } + + public boolean hasActive() { + return activeBucketCount > 0; + } + + public boolean isFinished() { + return finishedBucketCount == totalBucketCount; + } + + public boolean isEmpty() { + return buckets.isEmpty(); + } + + public String getFirstErrorMsg() { + return firstErrorMsg; + } + + public boolean containsFailedBuckets() { + return !failedBuckets.isEmpty(); + } + + public boolean isInconsistentState() { + return inconsistentState; + } + + public void setInconsistentState(boolean inconsistentState) { + this.inconsistentState = inconsistentState; + } + + /** + * Get internal progress token bucket state map. <em>For internal use only!</em> + * @return Map of superbuckets → sub buckets + */ + protected TreeMap<BucketKeyWrapper, BucketEntry> getBuckets() { + return buckets; + } + + protected void setActiveBucketCount(long activeBucketCount) { + this.activeBucketCount = activeBucketCount; + } + + protected void setPendingBucketCount(long pendingBucketCount) { + this.pendingBucketCount = pendingBucketCount; + } + + /** + * The format of the bucket progress output is as follows: + * <pre> + * VDS bucket progress file (n% completed)\n + * distribution bit count\n + * current bucket cursor\n + * number of finished buckets\n + * total number of buckets\n + * hex-of-superbucket:hex-of-progress\n + * ... repeat above line for each pending bucket ... + * </pre> + * + * Note that unlike earlier versions of ProgressToken, the bucket IDs are + * not prefixed with '0x'. + */ + public synchronized String toString() { + StringBuilder sb = new StringBuilder(); + // Append header + sb.append("VDS bucket progress file ("); + sb.append(percentFinished()); + sb.append("% completed)\n"); + sb.append(distributionBits); + sb.append('\n'); + sb.append(bucketCursor); + sb.append('\n'); + long doneBucketCount = Math.max(0l, finishedBucketCount - failedBuckets.size()); + sb.append(doneBucketCount); + sb.append('\n'); + sb.append(totalBucketCount); + sb.append('\n'); + // Append individual bucket progress + for (Map.Entry<BucketKeyWrapper, ProgressToken.BucketEntry> entry : buckets.entrySet()) { + sb.append(Long.toHexString(keyToBucketId(entry.getKey().getKey()))); + sb.append(':'); + sb.append(Long.toHexString(entry.getValue().getProgress().getRawId())); + sb.append('\n'); + } + for (Map.Entry<BucketId, BucketId> entry : failedBuckets.entrySet()) { + sb.append(Long.toHexString(entry.getKey().getRawId())); + sb.append(':'); + sb.append(Long.toHexString(entry.getValue().getRawId())); + sb.append('\n'); + } + return sb.toString(); + } + + /** + * Calculate an estimate on how far we've managed to iterate over both the + * superbuckets and the sub-buckets. + * + * Runs in <em>O(n+m)</em> time, where <em>n</em> is the number of active buckets + * and <em>m</em> is the number of pending buckets. Both these values should + * be fairly small in practice, however. + * + * Method is synchronized, as legacy code treats this as an atomic read. + * + * @return A value in the range [0, 100] estimating the progress. + */ + public synchronized double percentFinished() { + long superTotal = totalBucketCount; + long superFinished = finishedBucketCount; + + if (superTotal == 0 || superTotal == superFinished) return 100; + + double superDelta = 100.0 / superTotal; + double cumulativeSubProgress = 0; + + // Calculate cumulative for all non-finished buckets. 0 means the + // bucket has yet to see any progress + // There are numerical precision issues here, but this hardly requires + // aerospace engineering result-accuracy + for (Map.Entry<BucketKeyWrapper, ProgressToken.BucketEntry> entry : buckets.entrySet()) { + BucketId superbucket = new BucketId(keyToBucketId(entry.getKey().getKey())); + BucketId progress = entry.getValue().getProgress(); + // Prevent calculation of bucket progress on inconsistent buckets + if (progress.getId() != 0 && superbucket.contains(progress)) { + cumulativeSubProgress += superDelta * progressFraction(superbucket, progress); + } + } + + return (((double)superFinished / (double)superTotal) * 100.0) + + cumulativeSubProgress; + } + + /* + * Based on the following C++ code from document/bucket/bucketid.cpp: + * + * BucketId::Type + * BucketId::bucketIdToKey(Type id) + * { + * Type retVal = reverse(id); + * + * Type usedCountLSB = id >> maxNumBits(); + * retVal >>= CountBits; + * retVal <<= CountBits; + * retVal |= usedCountLSB; + * + * return retVal; + * } + * + * static uint32_t maxNumBits() { return (8 * sizeof(Type) - CountBits);} + */ + // TODO: this should probably be moved to BucketId at some point? + public static long bucketToKey(long id) { + long retVal = Long.reverse(id); + long usedCountLSB = id >>> (64 - BucketId.COUNT_BITS); + retVal >>>= BucketId.COUNT_BITS; + retVal <<= BucketId.COUNT_BITS; + retVal |= usedCountLSB; + + return retVal; + } + + private static BucketKeyWrapper bucketToKeyWrapper(BucketId bucket) { + return new BucketKeyWrapper(bucketToKey(bucket.getId())); + } + /* + * BucketId::Type + * BucketId::keyToBucketId(Type key) + * { + * Type retVal = reverse(key); + * + * Type usedCountMSB = key << maxNumBits(); + * retVal <<= CountBits; + * retVal >>= CountBits; + * retVal |= usedCountMSB; + * + * return retVal; + * } + */ + public static long keyToBucketId(long key) { + long retVal = Long.reverse(key); + long usedCountMSB = key << (64 - BucketId.COUNT_BITS); + retVal <<= BucketId.COUNT_BITS; + retVal >>>= BucketId.COUNT_BITS; + retVal |= usedCountMSB; + + return retVal; + } + + /** + * @param superbucket The superbucket of which <code>progress</code> is + * a sub-bucket + * @param progress The sub-bucket for which a fractional progress should + * be calculated + * @return a value in [0, 1] specifying how far the (sub-bucket) has + * reached in its superbucket. This is calculated by looking at the + * bucket's split factor. + */ + public synchronized double progressFraction(BucketId superbucket, BucketId progress) { + long revBits = bucketToKey(progress.getId()); + int superUsed = superbucket.getUsedBits(); + int progressUsed = progress.getUsedBits(); + + if (progressUsed == 0 || progressUsed < superUsed) { + return 0; + } + + int splitCount = progressUsed - superUsed; + + if (splitCount == 0) return 1; // Superbucket or inconsistent used-bits + + // Extract reversed split-bits + revBits <<= superUsed; + revBits >>>= 64 - splitCount; + + return (double)(revBits + 1) / (double)(1L << splitCount); + } + + /** + * Checks whether or not a given bucket is certain to be finished. Only + * looks at the super-bucket part of the given bucket ID, so it's possible + * that the bucket has in fact finished on a sub-bucket progress level. + * This does not affect the correctness of the result, however. + * + * During a distribution bit change, the token's buckets may be inconsistent. + * In this scenario, false is always returned since we can't tell for + * sure if the bucket is still active until the buckets have been made + * consistent. + * + * <strong>FIXME: verify correctness with regards to orderdoc et al. + * Don't make this method public until this has been done!</strong> + * + * @param bucket Bucket to check whether or not is finished. + * @return <code>true</code> if <code>bucket</code>'s super-bucket is + * finished, <code>false</code> otherwise. + */ + protected synchronized boolean isBucketFinished(BucketId bucket) { + if (inconsistentState) { + return false; + } + // Token only knows of super-buckets, not sub buckets + BucketId superbucket = new BucketId(distributionBits, bucket.getId()); + // Bucket is done if the current cursor location implies a visitor for + // the associated superbucket has already been sent off at some point + // and there is no pending visitor for the superbucket. The cursor is + // used to directly generate bucket keys, so we can compare against it + // directly. + // Example: given db=3 and cursor=2, the buckets 000 and 100 will have + // been returned by the iterator. By reversing the id and "right- + // aligning" it, we get the cursor location that would be required to + // generate it. + // We also return false if we're inconsistent, since the active/pending + // check is done on exact key values, requiring a uniform distribution + // bit value. + long reverseId = Long.reverse(superbucket.getId()) + >>> (64 - distributionBits); // No count bits + + if (reverseId >= bucketCursor) { + return false; + } + // Bucket has been generated, and it must have been finished if it's + // not listed as active/pending since we always remove finished buckets + BucketEntry entry = buckets.get(bucketToKeyWrapper(superbucket)); + if (entry == null) { + return true; + } + // If key of bucket progress > key of bucket id, we've finished it + long bucketKey = bucketToKey(bucket.getId()); + long progressKey = bucketToKey(entry.getProgress().getId()); + // TODO: verify correctness for all bucket orderings! + return progressKey > bucketKey; + } + + /** + * + * @param bucket BucketId to be split into two buckets. Bucket's used-bits + * do not need to match the ProgressToken's current distribution bit count, + * as it is assumed the client knows what it's doing and will bring the + * token into a consistent state eventually. + */ + protected void splitPendingBucket(BucketId bucket) { + BucketKeyWrapper bucketKey = bucketToKeyWrapper(bucket); + BucketEntry entry = buckets.get(bucketKey); + if (entry == null) { + throw new IllegalArgumentException( + "Attempting to split unknown bucket: " + bucket); + } + if (entry.getState() != BucketState.BUCKET_PENDING) { + throw new IllegalArgumentException( + "Attempting to split non-pending bucket: " + bucket); + } + + int splitDistBits = bucket.getUsedBits() + 1; + // Original bucket is replaced by two split children + BucketId splitLeft = new BucketId(splitDistBits, bucket.getId()); + // Right split sibling becomes logically at location original_bucket*2 in the + // bucket space due to the key ordering and setting the MSB of the split + BucketId splitRight = new BucketId(splitDistBits, bucket.getId() + | (1L << bucket.getUsedBits())); + + addBucket(splitLeft, entry.getProgress(), BucketState.BUCKET_PENDING); + addBucket(splitRight, entry.getProgress(), BucketState.BUCKET_PENDING); + + // Remove old bucket + buckets.remove(bucketKey); + --pendingBucketCount; + } + + protected void mergePendingBucket(BucketId bucket) { + BucketKeyWrapper bucketKey = bucketToKeyWrapper(bucket); + BucketEntry entry = buckets.get(bucketKey); + if (entry == null) { + throw new IllegalArgumentException( + "Attempting to join unknown bucket: " + bucket); + } + if (entry.getState() != BucketState.BUCKET_PENDING) { + throw new IllegalArgumentException( + "Attempting to join non-pending bucket: " + bucket); + } + + int usedBits = bucket.getUsedBits(); + // If MSB is 0, we should look for the bucket's right sibling. If not, + // we know that there's no left sibling, as it should otherwise have been + // merged already by the caller, due to it being ordered before the + // right sibling in the pending mapping + if ((bucket.getId() & (1L << (usedBits - 1))) == 0) { + BucketId rightCheck = new BucketId(usedBits, bucket.getId() | (1L << (usedBits - 1))); + BucketEntry rightSibling = buckets.get(bucketToKeyWrapper(rightCheck)); + // Must not merge if sibling isn't pending + if (rightSibling != null) { + assert(rightSibling.getState() == BucketState.BUCKET_PENDING); + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, "Merging " + bucket + " with rhs " + rightCheck); + } + // If right sibling has progress, it will unfortunately have to + // be discarded + if (rightSibling.getProgress().getUsedBits() != 0 + && log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Bucket progress for " + rightCheck + + " will be lost due to merging; potential for duplicates in result-set"); + } + buckets.remove(bucketToKeyWrapper(rightCheck)); + --pendingBucketCount; + } + } else { + BucketId leftSanityCheck = new BucketId(usedBits, bucket.getId() & ~(1L << (usedBits - 1))); + BucketEntry leftSibling = buckets.get(bucketToKeyWrapper(leftSanityCheck)); + assert(leftSibling == null) : "bucket merge sanity checking failed"; + } + + BucketId newMerged = new BucketId(usedBits - 1, bucket.getId()); + addBucket(newMerged, entry.getProgress(), BucketState.BUCKET_PENDING); + // Remove original bucket, leaving only the merged bucket + buckets.remove(bucketKey); + --pendingBucketCount; + assert(pendingBucketCount > 0); + } + + protected void setAllBucketsToState(BucketState state) { + for (Map.Entry<BucketKeyWrapper, ProgressToken.BucketEntry> entry + : buckets.entrySet()) { + entry.getValue().setState(state); + } + } + + protected void clearAllBuckets() { + buckets.clear(); + pendingBucketCount = 0; + activeBucketCount = 0; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java new file mode 100644 index 00000000000..f712240e7f9 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/RemoveResponse.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * This response is provided for successful document remove operations. Use the + * wasFound() method to check whether or not the document was actually found. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class RemoveResponse extends Response { + + private final boolean wasFound; + + public RemoveResponse(long requestId, boolean wasFound) { + super(requestId); + this.wasFound = wasFound; + } + + public boolean wasFound() { + return wasFound; + } + + @Override + public int hashCode() { + return super.hashCode() + Boolean.valueOf(wasFound).hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof RemoveResponse)) { + return false; + } + if (!super.equals(obj)) { + return false; + } + RemoveResponse rhs = (RemoveResponse)obj; + if (wasFound != rhs.wasFound) { + return false; + } + return true; + } + + @Override + public String toString() { + return "Remove" + super.toString() + " " + wasFound; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Response.java b/documentapi/src/main/java/com/yahoo/documentapi/Response.java new file mode 100644 index 00000000000..5a079ec8580 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/Response.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * <p>An asynchronous response from the document api. + * Subclasses of this provide additional response information for particular operations.</p> + * + * <p>This is a <i>value object</i>.</p> + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class Response { + + private long requestId; + private String textMessage = null; + private boolean success = true; + + /** Creates a successful response containing no information */ + public Response(long requestId) { + this(requestId, null, true); + } + + /** + * Creates a successful response containing a textual message + * + * @param textMessage the message to encapsulate in the Response + */ + public Response(long requestId, String textMessage) { + this(requestId, textMessage, true); + } + + /** + * Creates a response containing a textual message + * + * @param textMessage the message to encapsulate in the Response + * @param success true if the response represents a successful call + */ + public Response(long requestId, String textMessage, boolean success) { + this.requestId = requestId; + this.textMessage = textMessage; + this.success = success; + } + + /** + * Returns the text message of this response or null if there is none + * + * @return the message, or null + */ + public String getTextMessage() { return textMessage; } + + /** + * Returns whether this response encodes a success or a failure + * + * @return true if success + */ + public boolean isSuccess() { return success; } + + public long getRequestId() { return requestId; } + + public int hashCode() { + return (new Long(requestId).hashCode()) + (textMessage == null ? 0 : textMessage.hashCode()) + + (success ? 1 : 0); + } + + public boolean equals(Object o) { + if (!(o instanceof Response)) { + return false; + } + Response other = (Response) o; + + return requestId == other.requestId && success == other.success && ( + textMessage == null && other.textMessage == null || + textMessage != null && other.textMessage != null && textMessage.equals(other.textMessage)); + } + + public String toString() { + return "Response " + requestId + (textMessage == null ? "" : textMessage) + + (success ? " SUCCESSFUL" : " UNSUCCESSFUL"); + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/ResponseHandler.java b/documentapi/src/main/java/com/yahoo/documentapi/ResponseHandler.java new file mode 100644 index 00000000000..05d6973e4b0 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/ResponseHandler.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface ResponseHandler { + + /** + * This method is called once for each document api operation invoked on a {@link AsyncSession}. There is no + * guarantee as to which thread calls this, so any implementation must be thread-safe. + * + * @param response The response to process. + */ + public void handleResponse(Response response); +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Result.java b/documentapi/src/main/java/com/yahoo/documentapi/Result.java new file mode 100644 index 00000000000..e5982d297f7 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/Result.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * The <i>synchronous</i> result of submitting an asynchronous operation. + * A result is either a success or not. If it is not a success, it will contain an explanation of why. + * Document repositories may return subclasses which contain more information. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class Result { + + /** Null if this is a success, set to the error occurring if this is a failure */ + private Error error = null; + + /** The id of this operation */ + private long requestId; + + private ResultType type = ResultType.SUCCESS; + + /** + * Creates a successful result + * + * @param requestId the ID of the request + */ + public Result(long requestId) { + this.requestId = requestId; + } + + /** + * Creates a unsuccessful result + * + * @param type the type of failure + * @param error the error to encapsulate in this Result + * @see com.yahoo.documentapi.Result.ResultType + */ + public Result(ResultType type, Error error) { + this.type = type; + this.error = error; + } + + /** + * Returns whether this operation is a success. + * If it is a success, the operation is accepted and one or more responses are guaranteed + * to arrive within this sessions timeout limit. + * If this is not a success, this operation has no further consequences. + * + * @return true if success + */ + public boolean isSuccess() { return type == ResultType.SUCCESS; } + + /** + * Returns the error causes by this. If this was not a success, this method always returns an error + * If this was a success, this method returns null. + * + * @return the Error, or null + */ + public Error getError() { return error; } + + /** + * Returns the id of this operation. The asynchronous response to this operation + * will contain the same id to allow clients who desire to, to match operations to responses. + * + * @return the if of this operation + */ + public long getRequestId() { return requestId; } + + /** + * Returns the type of result. + * + * @return the type of result, typically if this is an error or a success, and what kind of error + * @see com.yahoo.documentapi.Result.ResultType + */ + public ResultType getType() { return type; } + + /** The types that a Result can have. */ + public enum ResultType { + /** The request was successful, no error information is attached. */ + SUCCESS, + /** The request failed, but may be successful if retried at a later time. */ + TRANSIENT_ERROR, + /** The request failed, and retrying is pointless. */ + FATAL_ERROR + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Session.java b/documentapi/src/main/java/com/yahoo/documentapi/Session.java new file mode 100644 index 00000000000..d0fef420f1d --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/Session.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * Superclass of all document api sessions. A session provides a handle through + * which an application can work with a document repository. There are various + * session subclasses for various types of interaction with the repository. + * <p> + * Each session can be used by multiple client application threads, i.e they are + * multithread safe. + * + * @author bratseth + */ +public interface Session { + + /** + * Returns the next response of this session. This method returns immediately. + * + * @return the next response, or null if no response is ready at this time + */ + public Response getNext(); + + /** + * Returns the next response of this session. This will block until a response is ready + * or until the given timeout is reached + * + * @param timeoutMilliseconds the max time to wait for a response. + * @return the next response, or null if no response becomes ready before the timeout expires + * @throws InterruptedException if this thread is interrupted while waiting + */ + public Response getNext(int timeoutMilliseconds) throws InterruptedException; + + /** + * Destroys this session and frees up any resources it has held. Making further calls on a destroyed + * session causes a runtime exception. + */ + public void destroy(); + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SimpleVisitorDocumentQueue.java b/documentapi/src/main/java/com/yahoo/documentapi/SimpleVisitorDocumentQueue.java new file mode 100644 index 00000000000..3930bd1b7ec --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/SimpleVisitorDocumentQueue.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.vdslib.DocumentList; + +import java.util.LinkedList; +import java.util.List; + +/** + * A simple document queue that queues up all results and automatically acks + * them. + * <p> + * Retrieving the list is not thread safe, so wait until visitor is done. This + * is a simple class merely meant for testing. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class SimpleVisitorDocumentQueue extends DumpVisitorDataHandler { + private final List<Document> documents = new LinkedList<Document>(); + + // Inherit doc from VisitorDataHandler + public void reset() { + super.reset(); + documents.clear(); + } + + @Override + public void onDocument(Document doc, long timestamp) { + documents.add(doc); + } + + public void onRemove(DocumentId docId) {} + + /** @return a list of all documents retrieved so far */ + public List<Document> getDocuments() { + return documents; + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SubscriptionParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/SubscriptionParameters.java new file mode 100644 index 00000000000..abd24099ff9 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/SubscriptionParameters.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * Parameters for creating or opening a visitor session + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class SubscriptionParameters extends Parameters { +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SubscriptionSession.java b/documentapi/src/main/java/com/yahoo/documentapi/SubscriptionSession.java new file mode 100644 index 00000000000..588a5e6f118 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/SubscriptionSession.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * This class provides document <i>subscription</i> - accessing document changes to a + * document repository. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public interface SubscriptionSession extends Session { + + /** + * Closes this subscription session without closing the subscription + * registered on the document repository. + * The same subscription can be accessed later by another subscription session. + */ + public void close(); + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java new file mode 100755 index 00000000000..24b68613208 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi;
+
+/**
+ * Parameters for creating a synchronous session
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SyncParameters extends Parameters {
+ // empty
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java new file mode 100755 index 00000000000..f864898fb5b --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.TestAndSetCondition;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+
+/**
+ * <p>A session for synchronous access to a document repository. This class
+ * provides simple document access where throughput is not a concern.</p>
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface SyncSession extends Session {
+
+ /**
+ * <p>Puts a document. When this method returns, the document is safely
+ * received. This enables setting condition compared to using Document.</p>
+ *
+ * @param documentPut The DocumentPut operation
+ */
+ void put(DocumentPut documentPut);
+
+ /**
+ * <p>Puts a document. When this method returns, the document is safely
+ * received.</p>
+ *
+ * @param documentPut The DocumentPut operation
+ * @param priority The priority with which to perform this operation.
+ */
+ void put(DocumentPut documentPut, DocumentProtocol.Priority priority);
+
+ /**
+ * <p>Gets a document.</p>
+ *
+ * @param id The id of the document to get.
+ * @return The known document having this id, or null if there is no
+ * document having this id.
+ * @throws UnsupportedOperationException Thrown if this access does not
+ * support retrieving.
+ */
+ Document get(DocumentId id);
+
+ /**
+ * <p>Gets a document.</p>
+ *
+ * @param id The id of the document to get.
+ * @param fieldSet A comma-separated list of fields to retrieve
+ * @param priority The priority with which to perform this operation.
+ * @return The known document having this id, or null if there is no
+ * document having this id.
+ * @throws UnsupportedOperationException Thrown if this access does not
+ * support retrieving.
+ */
+ Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority);
+
+ /**
+ * <p>Removes a document if it is present and condition is fulfilled.</p>
+ * @param documentRemove document to delete
+ * @return true If the document with this id was removed, false otherwise.
+ */
+ boolean remove(DocumentRemove documentRemove);
+
+ /**
+ * <p>Removes a document if it is present.</p>
+ *
+ * @param documentRemove Document remove operation
+ * @param priority The priority with which to perform this operation.
+ * @return true If the document with this id was removed, false otherwise.
+ * @throws UnsupportedOperationException Thrown if this access does not
+ * support removal.
+ */
+ boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority priority);
+
+ /**
+ * <p>Updates a document.</p>
+ *
+ * @param update The updates to perform.
+ * @return True, if the document was found and updated.
+ * @throws UnsupportedOperationException Thrown if this access does not
+ * support update.
+ */
+ boolean update(DocumentUpdate update);
+
+ /**
+ * <p>Updates a document.</p>
+ *
+ * @param update The updates to perform.
+ * @param priority The priority with which to perform this operation.
+ * @return True, if the document was found and updated.
+ * @throws UnsupportedOperationException Thrown if this access does not
+ * support update.
+ */
+ boolean update(DocumentUpdate update, DocumentProtocol.Priority priority);
+
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/ThroughputLimitQueue.java b/documentapi/src/main/java/com/yahoo/documentapi/ThroughputLimitQueue.java new file mode 100644 index 00000000000..a24dbc07bfa --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/ThroughputLimitQueue.java @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.concurrent.SystemTimer; +import com.yahoo.concurrent.Timer; + +import java.util.Collection; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * Queue that limits it's size based on the throughput. Allows the queue to keep a certain number of + * seconds of processing in its queue. + */ +public class ThroughputLimitQueue<M> extends LinkedBlockingQueue<M> { + private static Logger log = Logger.getLogger(ThroughputLimitQueue.class.getName()); + + double averageWaitTime = 0; + long maxWaitTime = 0; + long startTime; + int capacity = 2; + Timer timer; + + /** + * Creates a new queue. + * + * @param queueSizeInMs The maximum time we wish to have objects waiting in the queue. + */ + public ThroughputLimitQueue(long queueSizeInMs) { + this(SystemTimer.INSTANCE, queueSizeInMs); + } + + /** + * Creates a new queue. Used for unit testing. + * + * @param t Used to measure time spent in the queue. Subclass for unit testing, or use SystemTimer for regular use. + * @param queueSizeInMs The maximum time we wish to have objects waiting in the queue. + */ + public ThroughputLimitQueue(Timer t, long queueSizeInMs) { + maxWaitTime = queueSizeInMs; + timer = t; + } + + // Doc inherited from BlockingQueue + public boolean add(M m) { + if (!offer(m)) { + throw new IllegalStateException("Queue full"); + } + return true; + } + + // Doc inherited from BlockingQueue + public boolean offer(M m) { + return remainingCapacity() > 0 && super.offer(m); + } + + /** + * Calculates the average waiting time and readjusts the queue capacity. + * + * @param m The last message that was read from queue, if any. + * @return Returns m. + */ + private M calculateAverage(M m) { + if (m == null) { + startTime = 0; + return null; + } + + if (startTime != 0) { + long waitTime = timer.milliTime() - startTime; + + if (averageWaitTime == 0) { + averageWaitTime = waitTime; + } else { + averageWaitTime = 0.99 * averageWaitTime + 0.01 * waitTime; + } + + int newCapacity = Math.max(2, (int)Math.round(maxWaitTime / averageWaitTime)); + if (newCapacity != capacity) { + log.fine("Capacity of throughput queue changed from " + capacity + " to " + newCapacity); + capacity = newCapacity; + } + } + + if (!isEmpty()) { + startTime = timer.milliTime(); + } else { + startTime = 0; + } + + return m; + } + + // Doc inherited from BlockingQueue + public M poll() { + return calculateAverage(super.poll()); + } + + // Doc inherited from BlockingQueue + public void put(M m) throws InterruptedException { + offer(m, Long.MAX_VALUE, TimeUnit.SECONDS); + } + + // Doc inherited from BlockingQueue + public boolean offer(M m, long l, TimeUnit timeUnit) throws InterruptedException { + long timeWaited = 0; + while (timeWaited < timeUnit.toMillis(l)) { + if (offer(m)) { + return true; + } + + Thread.sleep(10); + timeWaited += 10; + } + + return false; + } + + // Doc inherited from BlockingQueue + public M take() throws InterruptedException { + return poll(Long.MAX_VALUE, TimeUnit.SECONDS); + } + + // Doc inherited from BlockingQueue + public M poll(long l, TimeUnit timeUnit) throws InterruptedException { + long timeWaited = 0; + while (timeWaited < timeUnit.toMillis(l)) { + M elem = poll(); + if (elem != null) { + return elem; + } + + Thread.sleep(10); + timeWaited += 10; + } + + return null; + } + + /** + * @return Returns the maximum capacity of the queue + */ + public int capacity() { + return capacity; + } + + // Doc inherited from BlockingQueue + public int remainingCapacity() { + int sz = capacity - size(); + return (sz > 0) ? sz : 0; + } + + // Doc inherited from BlockingQueue + public boolean addAll(Collection<? extends M> ms) { + for (M m : ms) { + if (!offer(m)) { + return false; + } + } + + return true; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java new file mode 100644 index 00000000000..ed96234ba64 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/UpdateResponse.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * This response is provided for successful document update operations. Use the + * wasFound() method to check whether or not the document was actually found. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UpdateResponse extends Response { + + private final boolean wasFound; + + public UpdateResponse(long requestId, boolean wasFound) { + super(requestId); + this.wasFound = wasFound; + } + + public boolean wasFound() { + return wasFound; + } + + @Override + public int hashCode() { + return super.hashCode() + Boolean.valueOf(wasFound).hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof UpdateResponse)) { + return false; + } + if (!super.equals(obj)) { + return false; + } + UpdateResponse rhs = (UpdateResponse)obj; + if (wasFound != rhs.wasFound) { + return false; + } + return true; + } + + @Override + public String toString() { + return "Update" + super.toString() + " " + wasFound; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorControlHandler.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorControlHandler.java new file mode 100644 index 00000000000..b46308e0daf --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorControlHandler.java @@ -0,0 +1,160 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.vdslib.VisitorStatistics; + +/** + * A class for controlling a visitor supplied through visitor parameters when + * creating the visitor session. The class defines callbacks for reporting + * progress and that the visitor is done. If you want to reimplement the + * default behavior of those callbacks, you can write your own subclass. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class VisitorControlHandler { + /** Possible completion codes for visiting. */ + public enum CompletionCode { + /** Visited all specified data successfully. */ + SUCCESS, + /** Aborted by user. */ + ABORTED, + /** Failure */ + FAILURE, + /** Create visitor reply did not return within the specified timeframe. */ + TIMEOUT + }; + + /** + * The result of the visitor, containing a completion code and an optional + * error message. + */ + public class Result { + public CompletionCode code; + public String message; + + public String toString() { + switch(code) { + case SUCCESS: + return "OK: " + message; + case ABORTED: + return "ABORTED: " + message; + case FAILURE: + return "FAILURE: " + message; + case TIMEOUT: + return "TIMEOUT: " + message; + } + + return "Unknown error"; + } + }; + + private VisitorControlSession session; + private ProgressToken currentProgress; + private boolean completed = false; + private Result result; + private VisitorStatistics currentStatistics; + + /** + * Called before the visitor starts. Override this method if you need + * to reset local data. Remember to call the superclass' method as well. + */ + public void reset() { + synchronized (this) { + session = null; + currentProgress = null; + completed = false; + result = null; + } + } + + /** + * Callback called when progress has changed. + * + * @param token the most recent progress token for this visitor + */ + public void onProgress(ProgressToken token) { + currentProgress = token; + } + + /** + * Callback for visitor error messages. + * + * @param message the error message + */ + public void onVisitorError(String message) { + } + + /** + * Callback for visitor statistics updates. + * + * @param vs The current statistics for this visitor. + */ + public void onVisitorStatistics(VisitorStatistics vs) { + currentStatistics = vs; + } + + /** + * Callback called when the visitor is done. + * + * @param code the completion code + * @param message an optional error message + */ + public void onDone(CompletionCode code, String message) { + synchronized (this) { + completed = true; + result = new Result(); + result.code = code; + result.message = message; + notifyAll(); + } + } + + /** @param session the visitor session used for this visitor */ + public void setSession(VisitorControlSession session) { + this.session = session; + } + + /** @return Retrieves the last progress token gotten for this visitor. If visitor has not been started, returns null.*/ + public ProgressToken getProgress() { return currentProgress; } + + public VisitorStatistics getVisitorStatistics() { return currentStatistics; } + + /** @return True if the visiting is done (either by error or success). */ + public boolean isDone() { + synchronized (this) { + return completed; + } + } + + /** + * Waits until visiting is done, or the given timeout (in ms) expires. + * Will wait forever if timeout is 0. + * + * @param timeoutMs The maximum amount of milliseconds to wait. + * @return True if visiting is done (either by error or success). + * @throws InterruptedException If an interrupt signal was received while waiting. + */ + public boolean waitUntilDone(long timeoutMs) throws InterruptedException { + synchronized (this) { + if (completed) return true; + if (timeoutMs == 0) { + while (!completed) { + wait(); + } + } else { + wait(timeoutMs); + } + return completed; + } + } + + /** + * Abort this visitor + */ + public void abort() { session.abort(); } + + /** + @return The result of the visiting, if done. If not done, returns null. + */ + public Result getResult() { return result; }; +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorControlSession.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorControlSession.java new file mode 100644 index 00000000000..4296407d633 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorControlSession.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * Superclass for document <i>visiting</i> functionality - accessing + * documents in an order decided by the document repository. This allows much + * higher read throughput than random access. + * <p> + * The class supplies an interface for functions that are common for different + * kinds of visitor sessions, such as acking visitor data and aborting the + * session. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public interface VisitorControlSession { + /** + * Acknowledges a response previously retrieved by the <code>getNext</code> + * method. + * + * @param token The ack token. You must get this from the visitor response + * returned by the <code>getNext</code> method. + */ + public void ack(AckToken token); + + /** + * Aborts the session. + */ + public void abort(); + + /** + * Returns the next response of this session. This method returns immediately. + * + * @return the next response, or null if no response is ready at this time + */ + public VisitorResponse getNext(); + + /** + * Returns the next response of this session. This will block until a response is ready + * or until the given timeout is reached + * + * @param timeoutMilliseconds the max time to wait for a response. If the number is 0, this will block + * without any timeout limit + * @return the next response, or null if no response becomes ready before the timeout expires + * @throws InterruptedException if this thread is interrupted while waiting + */ + public VisitorResponse getNext(int timeoutMilliseconds) throws InterruptedException; + + /** + * Destroys this session and frees up any resources it has held. + */ + public void destroy(); + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataHandler.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataHandler.java new file mode 100644 index 00000000000..4cee27a9fda --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataHandler.java @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.messagebus.Message; +import com.yahoo.vdslib.DocumentList; +import com.yahoo.vdslib.Entry; +import com.yahoo.vdslib.SearchResult; +import com.yahoo.vdslib.DocumentSummary; +import com.yahoo.document.BucketId; +import java.util.List; + +/** + * A data handler is a class that handles responses from a visitor. + * Different clients might want different interfaces. + * Some might want a callback interface, some might want a polling interface. + * Some want good control of acking, while others just want something simple. + * <p> + * Use a data handler that fits your needs to be able to use visiting easily. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public abstract class VisitorDataHandler { + protected VisitorControlSession session; + + /** Creates a new visitor data handler. */ + public VisitorDataHandler() { + } + + /** + * Called before the visitor starts. Override this method if you need + * to reset local data. Remember to call the superclass' method as well. + */ + public void reset() { + session = null; + } + + /** + * Sets which session this visitor data handler belongs to. This is done by + * the session itself and should not be called manually. The session is + * needed for ack to work. + * + * @param session the session currently using this data handler + */ + public void setSession(VisitorControlSession session) { + this.session = session; + } + + /** + * Returns the next response of this session. This method returns + * immediately. + * + * @return the next response, or null if no response is ready at this time + * @throws UnsupportedOperationException if data handler does not support + * the operation + */ + public VisitorResponse getNext() { + throw new UnsupportedOperationException("This datahandler doesn't support polling"); + } + + /** + * Returns the next response of this session. This will block until a + * response is ready or the given timeout is reached. + * + * @param timeoutMilliseconds the max time to wait for a response. If the + * number is 0, this will block without any + * timeout limit + * @return the next response, or null if no response becomes ready before + * the timeout expires + * @throws InterruptedException if this thread is interrupted while waiting + * @throws UnsupportedOperationException if data handler does not support + * the operation + */ + public VisitorResponse getNext(int timeoutMilliseconds) throws InterruptedException { + throw new UnsupportedOperationException("This datahandler doesn't support polling"); + } + + /** + * Called when visiting is done, to notify clients waiting on getNext(). + */ + public void onDone() {} + + /** + * Called when a data message is received. + * + * May be called concurrently from multiple threads. Any internal state + * mutations must be done in a thread-safe manner. + * + * @param m The message received + * @param token A token to reply with when finished processing the message. + */ + public abstract void onMessage(Message m, AckToken token); + + /** + * Function used to ack data. You need to ack data periodically, as storage + * will halt visiting when it has too much client requests pending. + * + * @param token The token to ack. Gotten from an earlier callback. + */ + public void ack(AckToken token) { + session.ack(token); + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataQueue.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataQueue.java new file mode 100644 index 00000000000..5e65ee534ba --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataQueue.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.BucketId; +import com.yahoo.documentapi.messagebus.protocol.DocumentListEntry; +import com.yahoo.messagebus.Message; +import com.yahoo.vdslib.DocumentList; + +import java.util.LinkedList; +import java.util.List; + + +/** + * A visitor data handler that queues up documents in visitor responses and + * implements the <code>getNext</code> methods, thus implementing the polling + * API defined in VisitorDataHandler. + * <p> + * Visitor responses containing document lists should be polled for with the + * <code>getNext</code> methods and need to be acked when processed for + * visiting not to halt. The class is thread safe. + * + * @author <a href="mailto:humbe@yahoo-inc.com">HÃ¥kon Humberset</a> + */ +public class VisitorDataQueue extends VisitorDataHandler { + + final LinkedList<VisitorResponse> pendingResponses = new LinkedList<VisitorResponse>(); + + /** Creates a new visitor data queue. */ + public VisitorDataQueue() { + } + + // Inherit doc from VisitorDataHandler + public void reset() { + super.reset(); + synchronized (pendingResponses) { + pendingResponses.clear(); + } + } + + public void onMessage(Message m, AckToken token) { + } + + // Inherit doc from VisitorDataHandler + public void onDocuments(DocumentList docs, AckToken token) { + synchronized (pendingResponses) { + pendingResponses.add(new DocumentListVisitorResponse(docs, token)); + pendingResponses.notifyAll(); + } + } + + // Inherit doc from VisitorDataHandler + public VisitorResponse getNext() { + synchronized (pendingResponses) { + return (pendingResponses.isEmpty() + ? null : pendingResponses.removeFirst()); + } + } + + // Inherit doc from VisitorDataHandler + public VisitorResponse getNext(int timeoutMilliseconds) throws InterruptedException { + synchronized (pendingResponses) { + if (pendingResponses.isEmpty()) { + if (timeoutMilliseconds == 0) { + while (pendingResponses.isEmpty()) { + pendingResponses.wait(); + } + } else { + pendingResponses.wait(timeoutMilliseconds); + } + } + return (pendingResponses.isEmpty() + ? null : pendingResponses.removeFirst()); + } + } + + @Override + public void onDone() { + synchronized (pendingResponses) { + pendingResponses.notifyAll(); + } + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorDestinationParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorDestinationParameters.java new file mode 100644 index 00000000000..a27e1bb405c --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorDestinationParameters.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * Parameters for creating or opening a visitor destination session. + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public class VisitorDestinationParameters extends Parameters { + private String sessionName; + private VisitorDataHandler dataHandler; + + /** + * Creates visitor destination parameters from the supplied parameters. + * + * @param sessionName The name of the destination session. + * @param handler The data handler. + */ + public VisitorDestinationParameters(String sessionName, VisitorDataHandler handler) { + this.sessionName = sessionName; + dataHandler = handler; + } + + /** @return the name of the destination session */ + public String getSessionName() { return sessionName; }; + + /** @return the data handler */ + public VisitorDataHandler getDataHandler() { return dataHandler; }; +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorDestinationSession.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorDestinationSession.java new file mode 100644 index 00000000000..bb2b3975292 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorDestinationSession.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * A visitor destination session for receiving data from a visitor. + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public interface VisitorDestinationSession extends VisitorControlSession { +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java new file mode 100755 index 00000000000..cde434df141 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java @@ -0,0 +1,797 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSelector;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.log.LogLevel;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.logging.Logger;
+
+/**
+ * <p>Enables transparent iteration of super/sub-buckets</p>
+ *
+ * <p>Thread safety: safe for threads to hold their own iterators (no shared state),
+ * as long as they also hold the ProgressToken object associated with it. No two
+ * VisitorIterator instances may share the same progress token instance at the
+ * same time.
+ * Concurrent access to a single VisitorIterator instance is not safe and must
+ * be handled atomically by the caller.</p>
+ *
+ * @author <a href="mailto:vekterli@yahoo-inc.com">Tor Brede Vekterli</a>
+ */
+public class VisitorIterator {
+ private ProgressToken progressToken;
+ private BucketSource bucketSource;
+ private int distributionBitCount;
+
+ private static final Logger log = Logger.getLogger(VisitorIterator.class.getName());
+
+ public static class BucketProgress {
+ private BucketId superbucket;
+ private BucketId progress;
+
+ public BucketProgress(BucketId superbucket, BucketId progress) {
+ this.superbucket = superbucket;
+ this.progress = progress;
+ }
+
+ public BucketId getProgress() {
+ return progress;
+ }
+
+ public BucketId getSuperbucket() {
+ return superbucket;
+ }
+ }
+
+ /**
+ * Provides an abstract interface to <code>VisitorIterator</code> for
+ * how pending buckets are acquired, decoupling this from the iteration
+ * itself.
+ *
+ * <em>Important</em>: it is the responsibility of the {@link BucketSource} implementation
+ * to ensure that progress information is honored for (partially) finished buckets.
+ * From the point of view of the iterator itself, it should not have to deal with
+ * filtering away already finished buckets, as this is a detail best left to
+ * bucket sources.
+ */
+ protected static interface BucketSource {
+ public boolean hasNext();
+ public boolean shouldYield();
+ public boolean visitsAllBuckets();
+ public BucketProgress getNext();
+ public long getTotalBucketCount();
+ public int getDistributionBitCount();
+ public void setDistributionBitCount(int distributionBitCount,
+ ProgressToken progress);
+ public void update(BucketId superbucket, BucketId progress,
+ ProgressToken token);
+ }
+
+ /**
+ * Provides a bucket source that encompasses the entire range available
+ * through a given value of distribution bits
+ */
+ protected static class DistributionRangeBucketSource implements BucketSource {
+ private boolean flushActive = false;
+ private int distributionBitCount;
+ // Wouldn't need this if this were a non-static class, but do it for
+ // the sake of keeping things identical in Java and C++
+ private ProgressToken progressToken;
+
+ public DistributionRangeBucketSource(int distributionBitCount,
+ ProgressToken progress) {
+ progressToken = progress;
+
+ // New progress token (could also be empty, in which this is a
+ // no-op anyway)
+ if (progressToken.getTotalBucketCount() == 0) {
+ assert(progressToken.isEmpty()) : "inconsistent progress state";
+ progressToken.setTotalBucketCount(1L << distributionBitCount);
+ progressToken.setDistributionBitCount(distributionBitCount);
+ progressToken.setBucketCursor(0);
+ progressToken.setFinishedBucketCount(0);
+ this.distributionBitCount = distributionBitCount;
+ }
+ else {
+ this.distributionBitCount = progressToken.getDistributionBitCount();
+ // Quick consistency check to ensure the user isn't trying to eg.
+ // pass a progress token for an explicit document selection
+ if (progressToken.getTotalBucketCount() != (1L << progressToken.getDistributionBitCount())) {
+ throw new IllegalArgumentException("Total bucket count in existing progress is not "
+ + "consistent with that of the current document selection");
+ }
+ }
+
+ if (!progress.isFinished()) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Importing unfinished progress token with " +
+ "bits: " + progressToken.getDistributionBitCount() +
+ ", active: " + progressToken.getActiveBucketCount() +
+ ", pending: " + progressToken.getPendingBucketCount() +
+ ", cursor: " + progressToken.getBucketCursor() +
+ ", finished: " + progressToken.getFinishedBucketCount() +
+ ", total: " + progressToken.getTotalBucketCount());
+ }
+ if (!progress.isEmpty()) {
+ // Lower all active to pending
+ if (progressToken.getActiveBucketCount() > 0) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Progress token had active buckets upon range " +
+ "construction. Setting these as pending");
+ }
+ progressToken.setAllBucketsToState(ProgressToken.BucketState.BUCKET_PENDING);
+ }
+ // Fixup for any buckets that were active when progress was written
+ // but are now pending and with wrong dist bits (used-bits). Buckets
+ // split here may very well be split/merged again if we set a new dist
+ // bit count, but that is the desired process
+ correctInconsistentPending(progressToken.getDistributionBitCount());
+ // Fixup for bucket cursor in case of bucket space downscaling
+ correctTruncatedBucketCursor();
+
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Partial bucket space progress; continuing "+
+ "from position " + progressToken.getBucketCursor());
+ }
+ }
+ progressToken.setFinishedBucketCount(progressToken.getBucketCursor() -
+ progressToken.getPendingBucketCount());
+ } else {
+ assert(progressToken.getBucketCursor() == progressToken.getTotalBucketCount());
+ }
+ // Should be all fixed up and good to go
+ progressToken.setInconsistentState(false);
+ }
+
+ protected boolean isLosslessResetPossible() {
+ // #pending must be equal to cursor, i.e. all buckets ever fetched
+ // must be located in the set of pending
+ if (progressToken.getPendingBucketCount() != progressToken.getBucketCursor()) {
+ return false;
+ }
+ // Check if all pending buckets have a progress of 0
+ for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry
+ : progressToken.getBuckets().entrySet()) {
+ if (entry.getValue().getState() != ProgressToken.BucketState.BUCKET_PENDING) {
+ return false;
+ }
+ if (entry.getValue().getProgress().getId() != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Ensure that a given <code>ProgressToken</code> instance only has
+ * buckets pending that have a used-bits count of that of the
+ * <code>targetDistCits</code>. This is done by splitting or merging
+ * all inconsistent buckets until the desired state is reached.
+ *
+ * Time complexity is approx <i>O(4bn)</i> where <i>b</i> is the maximum
+ * delta of bits to change anywhere in the set of pending and <i>n</i>
+ * is the number of pending. This includes the time spent making shallow
+ * map copies.
+ *
+ * @param targetDistBits The desired distribution bit count of the buckets
+ */
+ private void correctInconsistentPending(int targetDistBits) {
+ boolean maybeInconsistent = true;
+ long bucketsSplit = 0, bucketsMerged = 0;
+ long pendingBefore = progressToken.getPendingBucketCount();
+ ProgressToken p = progressToken;
+
+ // Optimization: before doing any splitting/merging at all, we check
+ // to see if we can't simply just reset the entire internal state
+ // with the new distribution bit count. This ensures that if we go
+ // from eg. 1 bit to 20 bits, we won't have to perform a grueling
+ // half a million splits to cover the same bucket space as that 1
+ // single-bit bucket once did
+ if (isLosslessResetPossible()) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "At start of bucket space and all " +
+ "buckets have no progress; doing a lossless reset " +
+ "instead of splitting/merging");
+ }
+ assert(p.getActiveBucketCount() == 0);
+ p.clearAllBuckets();
+ p.setBucketCursor(0);
+ return;
+ }
+
+ while (maybeInconsistent) {
+ BucketId lastMergedBucket = null;
+ maybeInconsistent = false;
+ // Make a shallow working copy of the bucket map. BucketKeyWrapper
+ // keys are considered immutable, and should thus not be at risk
+ // for being changed during the inner loop
+ // Do separate passes for splitting and merging just to make
+ // absolutely sure that the two ops won't step on each others'
+ // toes. This isn't wildly efficient, but the data sets in question
+ // are presumed to be low in size and this is presumed to be a very
+ // infrequent operation
+ TreeMap<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> buckets
+ = new TreeMap<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry>(p.getBuckets());
+ for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry
+ : buckets.entrySet()) {
+ assert(entry.getValue().getState() == ProgressToken.BucketState.BUCKET_PENDING);
+ BucketId pending = new BucketId(ProgressToken.keyToBucketId(entry.getKey().getKey()));
+ if (pending.getUsedBits() < targetDistBits) {
+ if (pending.getUsedBits() + 1 < targetDistBits) {
+ maybeInconsistent = true; // Do another pass
+ }
+ p.splitPendingBucket(pending);
+ ++bucketsSplit;
+ }
+ }
+
+ // Make new map copy with potentially split buckets
+ buckets = new TreeMap<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry>(p.getBuckets());
+ for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry
+ : buckets.entrySet()) {
+ assert(entry.getValue().getState() == ProgressToken.BucketState.BUCKET_PENDING);
+ BucketId pending = new BucketId(ProgressToken.keyToBucketId(entry.getKey().getKey()));
+ if (pending.getUsedBits() > targetDistBits) {
+ // If this is the right sibling of an already merged left sibling,
+ // it's already been merged away, so we should skip it
+ if (lastMergedBucket != null) {
+ BucketId rightCheck = new BucketId(lastMergedBucket.getUsedBits(),
+ lastMergedBucket.getId() | (1L << (lastMergedBucket.getUsedBits() - 1)));
+ if (pending.equals(rightCheck)) {
+ if (log.isLoggable(LogLevel.SPAM)) {
+ log.log(LogLevel.SPAM, "Skipped " + pending +
+ ", as it was right sibling of " + lastMergedBucket);
+ }
+ continue;
+ }
+ }
+ if (pending.getUsedBits() - 1 > targetDistBits) {
+ maybeInconsistent = true; // Do another pass
+ }
+ p.mergePendingBucket(pending);
+ ++bucketsMerged;
+
+ lastMergedBucket = pending;
+ }
+ }
+ }
+ if ((bucketsSplit > 0 || bucketsMerged > 0) && log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Existing progress' pending buckets had inconsistent " +
+ "distribution bits; performed " + bucketsSplit + " split ops and " +
+ bucketsMerged + " merge ops. Pending: " + pendingBefore + " -> " +
+ p.getPendingBucketCount());
+ }
+ }
+
+ private void correctTruncatedBucketCursor() {
+ // We've truncated the bucket cursor, but in doing so we might
+ // have moved back beyond where there are pending buckets. Consider
+ // having a cursor value of 3 at 31 bits and then moving to 11 bits.
+ // With 1 pending we'll normally reach a cursor of 0, even though it
+ // should be 1
+ for (ProgressToken.BucketKeyWrapper bucketKey
+ : progressToken.getBuckets().keySet()) {
+ BucketId bid = bucketKey.toBucketId();
+ long idx = bucketKey.getKey() >>> (64 - bid.getUsedBits());
+ if (bid.getUsedBits() == distributionBitCount
+ && idx >= progressToken.getBucketCursor()) {
+ progressToken.setBucketCursor(idx + 1);
+ }
+ }
+ if (log.isLoggable(LogLevel.SPAM)) {
+ log.log(LogLevel.SPAM, "New range bucket cursor is " +
+ progressToken.getBucketCursor());
+ }
+ }
+
+ public boolean hasNext() {
+ return progressToken.getBucketCursor() < (1L << distributionBitCount);
+ }
+
+ public boolean shouldYield() {
+ // If we need to flush all active buckets, stall the iteration until
+ // this has been done
+ return flushActive;
+ }
+
+ public boolean visitsAllBuckets() {
+ return true;
+ }
+
+ public long getTotalBucketCount() {
+ return 1L << distributionBitCount;
+ }
+
+ public BucketProgress getNext() {
+ assert(hasNext()) : "getNext() called with hasNext() == false";
+ long currentPosition = progressToken.getBucketCursor();
+ long key = ProgressToken.makeNthBucketKey(currentPosition, distributionBitCount);
+ ++currentPosition;
+ progressToken.setBucketCursor(currentPosition);
+ return new BucketProgress(
+ new BucketId(ProgressToken.keyToBucketId(key)),
+ new BucketId());
+ }
+
+ public int getDistributionBitCount() {
+ return distributionBitCount;
+ }
+
+ public void setDistributionBitCount(int distributionBitCount,
+ ProgressToken progress)
+ {
+ this.distributionBitCount = distributionBitCount;
+
+ // There might be a case where we're waiting for active buckets
+ // already when a new distribution bit change comes in. If so,
+ // don't do anything at all yet with the set of pending
+ if (progressToken.getActiveBucketCount() > 0) {
+ flushActive = true;
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Holding off new/pending buckets and consistency " +
+ "correction until all " + progress.getActiveBucketCount() +
+ " active buckets have been updated");
+ }
+ progressToken.setInconsistentState(true);
+ } else {
+ // Only perform the actual distribution bit bucket ops if we've
+ // got no pending buckets
+ int delta = distributionBitCount - progressToken.getDistributionBitCount();
+
+ // Must do this before setting the bucket cursor to allow
+ // reset-checking to be performed
+ correctInconsistentPending(distributionBitCount);
+ if (delta > 0) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Increasing distribution bits for full bucket " +
+ "space range source from " + progressToken.getDistributionBitCount() + " to " +
+ distributionBitCount);
+ }
+ progressToken.setFinishedBucketCount(progressToken.getFinishedBucketCount() << delta);
+ // By n-doubling the position, the bucket key ordering ensures
+ // we go from eg. 3:0x02 to 4:0x02 to 5:02 etc.
+ progressToken.setBucketCursor(progressToken.getBucketCursor() << delta);
+ } else if (delta < 0) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Decreasing distribution bits for full bucket " +
+ "space range source from " + progressToken.getDistributionBitCount() +
+ " to " + distributionBitCount + " bits");
+ }
+ // Scale down bucket space and cursor
+ progressToken.setBucketCursor(progressToken.getBucketCursor() >>> -delta);
+ progressToken.setFinishedBucketCount(progressToken.getFinishedBucketCount() >>> -delta);
+ }
+
+ progressToken.setTotalBucketCount(1L << distributionBitCount);
+ progressToken.setDistributionBitCount(distributionBitCount);
+
+ correctTruncatedBucketCursor();
+ progressToken.setInconsistentState(false);
+ }
+ }
+
+ public void update(BucketId superbucket, BucketId progress,
+ ProgressToken token) {
+ progressToken.updateProgress(superbucket, progress);
+
+ if (superbucket.getUsedBits() != distributionBitCount) {
+ if (!progress.equals(ProgressToken.FINISHED_BUCKET)) {
+ // We should now always flush active buckets before doing a
+ // consistency fix. This simplifies things greatly
+ assert(flushActive);
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Received non-finished bucket " +
+ superbucket + " with wrong distribution bit count (" +
+ superbucket.getUsedBits() + "). Waiting to correct " +
+ "until all active are done");
+ }
+ } else {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Received finished bucket " +
+ superbucket + " with wrong distribution bit count (" +
+ superbucket.getUsedBits() + "). Waiting to correct " +
+ "until all active are done");
+ }
+ }
+ }
+
+ if (progressToken.getActiveBucketCount() == 0) {
+ if (flushActive) {
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "All active buckets flushed, " +
+ "correcting progress token and continuing normal operation");
+ }
+ // Trigger the actual bucket state change this time
+ setDistributionBitCount(distributionBitCount, progressToken);
+ assert(progressToken.getDistributionBitCount() == distributionBitCount);
+ }
+ flushActive = false;
+ // Update #finished since we might have had inconsistent active
+ // buckets that have prevent us from getting a correct value. At
+ // this point, however, all pending buckets should presumably be
+ // at the same, correct dist bit count, so we can safely compute
+ // a new count
+ // TODO: ensure this is consistent
+ if (progressToken.getPendingBucketCount() <= progressToken.getBucketCursor()) {
+ progressToken.setFinishedBucketCount(progressToken.getBucketCursor() -
+ progressToken.getPendingBucketCount());
+ }
+ }
+ }
+ }
+
+ /**
+ * Provides an explicit set of bucket IDs to iterate over. Will immediately
+ * set these as pending in the {@link ProgressToken}, as it is presumed this set is
+ * rather small. Changing the distribution bit count for this source is
+ * effectively a no-op, as explicit bucket IDs should not be implicitly
+ * changed.
+ */
+ protected static class ExplicitBucketSource implements BucketSource {
+ private int distributionBitCount;
+ private long totalBucketCount = 0;
+
+ public ExplicitBucketSource(Set<BucketId> superbuckets,
+ int distributionBitCount,
+ ProgressToken progress) {
+ this.distributionBitCount = progress.getDistributionBitCount();
+ this.totalBucketCount = superbuckets.size();
+
+ // New progress token?
+ if (progress.getTotalBucketCount() == 0) {
+ progress.setTotalBucketCount(this.totalBucketCount);
+ progress.setDistributionBitCount(distributionBitCount);
+ this.distributionBitCount = distributionBitCount;
+ }
+ else {
+ // Quick consistency check to ensure the user isn't trying to eg.
+ // pass a progress token for another document selection
+ if (progress.getTotalBucketCount() != totalBucketCount
+ || (progress.getFinishedBucketCount() + progress.getPendingBucketCount()
+ + progress.getActiveBucketCount() != totalBucketCount)) {
+ throw new IllegalArgumentException("Total bucket count in existing progress is not " +
+ "consistent with that of the current document selection");
+ }
+ if (progress.getBucketCursor() != 0) {
+ // Trying to use a range source progress file
+ throw new IllegalArgumentException("Cannot use given progress file with the "+
+ "current document selection");
+ }
+ this.distributionBitCount = progress.getDistributionBitCount();
+ }
+
+ if (progress.isFinished() || !progress.isEmpty()) return;
+
+ for (BucketId id : superbuckets) {
+ // Add all superbuckets with zero sub-bucket progress and pending
+ progress.addBucket(id, new BucketId(), ProgressToken.BucketState.BUCKET_PENDING);
+ }
+ }
+
+ public boolean hasNext() {
+ return false;
+ }
+
+ public boolean shouldYield() {
+ return false;
+ }
+
+ public boolean visitsAllBuckets() {
+ return false;
+ }
+
+ public long getTotalBucketCount() {
+ return totalBucketCount;
+ }
+
+ // All explicit buckets should have been placed in the progress
+ // token during construction, so this method should never be called
+ public BucketProgress getNext() {
+ throw new IllegalStateException("getNext() called on ExplicitBucketSource");
+ }
+
+ public int getDistributionBitCount() {
+ return distributionBitCount;
+ }
+
+ public void setDistributionBitCount(int distributionBitCount,
+ ProgressToken progress)
+ {
+ // Setting distribution bits for explicit bucket source is essentially
+ // a no-op, since its buckets already are fixed at 32 used bits.
+ progress.setDistributionBitCount(distributionBitCount);
+ this.distributionBitCount = distributionBitCount;
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Set distribution bit count to "
+ + distributionBitCount + " for explicit bucket source (no-op)");
+ }
+ }
+
+ public void update(BucketId superbucket, BucketId progress,
+ ProgressToken token) {
+ // Simply delegate to ProgressToken, as it maintains all progress state
+ token.updateProgress(superbucket, progress);
+ }
+ }
+
+ /**
+ * @param bucketSource An instance of {@link BucketSource}, providing the working set for
+ * the iterator
+ * @param progressToken A {@link ProgressToken} instance, allowing the progress of
+ * finished or partially finished buckets to be tracked
+ *
+ * @see BucketSource
+ * @see ProgressToken
+ */
+ private VisitorIterator(ProgressToken progressToken,
+ BucketSource bucketSource) {
+ assert(progressToken.getDistributionBitCount() == bucketSource.getDistributionBitCount())
+ : "inconsistent distribution bit counts";
+ this.distributionBitCount = progressToken.getDistributionBitCount();
+ this.progressToken = progressToken;
+ this.bucketSource = bucketSource;
+ }
+
+
+ /**
+ * @return The pair [superbucket, progress] that specifies the next iterable
+ * bucket. When a superbucket is initially returned, the pair is equal to
+ * that of [superbucket, 0], as there has been no progress into its sub-buckets
+ * yet (if they exist).
+ *
+ * Precondition: <code>hasNext() == true</code>
+ */
+ public BucketProgress getNext() {
+ assert(progressToken.getDistributionBitCount() == bucketSource.getDistributionBitCount())
+ : "inconsistent distribution bit counts for progress and source";
+ assert(hasNext());
+ // We prioritize returning buckets in the pending map over those
+ // that may be in the bucket source, since we want to avoid growing
+ // the map too much
+ if (progressToken.hasPending()) {
+ // Find first pending bucket in token
+ TreeMap<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> buckets = progressToken.getBuckets();
+ ProgressToken.BucketEntry pending = null;
+ BucketId superbucket = null;
+ for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry : buckets.entrySet()) {
+ if (entry.getValue().getState() == ProgressToken.BucketState.BUCKET_PENDING) {
+ pending = entry.getValue();
+ superbucket = new BucketId(ProgressToken.keyToBucketId(entry.getKey().getKey()));
+ break;
+ }
+ }
+ assert(pending != null) : "getNext() called with inconsistent state";
+
+ // Set bucket to active, since it's not awaiting an update
+ pending.setState(ProgressToken.BucketState.BUCKET_ACTIVE);
+
+ progressToken.setActiveBucketCount(progressToken.getActiveBucketCount() + 1);
+ progressToken.setPendingBucketCount(progressToken.getPendingBucketCount() - 1);
+
+ return new BucketProgress(superbucket, pending.getProgress());
+ } else {
+ BucketProgress ret = bucketSource.getNext();
+ progressToken.addBucket(ret.getSuperbucket(), ret.getProgress(),
+ ProgressToken.BucketState.BUCKET_ACTIVE);
+ return ret;
+ }
+ }
+
+ /**
+ * <p>Check whether or not it is valid to call {@link #getNext()} with the current
+ * iterator state.</p>
+ *
+ * <p>There exists a case wherein <code>hasNext</code> may return false before {@link #update} is
+ * called, but true afterwards. This happens when the set of pending buckets is
+ * empty, the bucket source is empty <em>but</em> the set of active buckets is
+ * not. A future progress update on any of the buckets in the active set may
+ * or may not make that bucket available to the pending set again.
+ * This must be handled explicitly by the caller by checking {@link #isDone()}
+ * and ensuring that {@link #update} is called before retrying <code>hasNext</code>.</p>
+ *
+ * <p>This method will also return false if the number of distribution bits have
+ * changed and there are active buckets needing to be flushed before the
+ * iterator will allow new buckets to be handed out.</p>
+ *
+ * @return Whether or not it is valid to call {@link #getNext()} with the current
+ * iterator state.
+ */
+ public boolean hasNext() {
+ return (progressToken.hasPending() || bucketSource.hasNext()) && !bucketSource.shouldYield();
+ }
+
+ /**
+ * Check if the iterator is actually done
+ *
+ * @see #hasNext()
+ *
+ * @return <code>true</code> <em>iff</em> the bucket source is empty and
+ * there are no pending or active buckets in the progress token.
+ */
+ public boolean isDone() {
+ return !(hasNext() || progressToken.hasActive());
+ }
+
+ /**
+ * <p>Tell the iterator that we've finished processing up to <i>and
+ * including</i> <code>progress</code>. <code>progress</code> may be a sub-bucket <i>or</i>
+ * the invalid 0-bucket (in case the caller fails to process the bucket and
+ * must return it to the set of pending) <em>or</em> the special case <code>BucketId(Integer.MAX_VALUE)</code>,
+ * the latter indicating to the iterator that traversal is complete for
+ * <code>superbucket</code>'s tree. The null bucket should only be used if no
+ * non-null updates have yet been given for the superbucket.</p>
+ *
+ * <p>It is a requirement that each superbucket returned by {@link #getNext()} must
+ * eventually result in 1-n update operations, where the last update operation
+ * has the special progress==super case.</p>
+ *
+ * <p>If the document selection used to create the iterator is unknown and there
+ * were active buckets at the time of a distribution bit state change, such
+ * a bucket passed to <code>update()</code> will be in an inconsistent state
+ * with regards to the number of bits it uses. For unfinished buckets, this
+ * is handled by splitting or merging it until it's consistent, depending on
+ * whether or not it had a lower or higher distribution bit count than that of
+ * the current system state. For finished buckets of a lower dist bit count,
+ * the amount of finished buckets in the ProgressToken is adjusted upwards
+ * to compensate for the fact that a bucket using fewer distribution bits
+ * actually covers more of the bucket space than the ones that are currently
+ * in use. For finished buckets of a higher dist bit count, the number of
+ * finished buckets is <em>not</em> increased at that point in time, since
+ * such a bucket doesn't actually cover an entire bucket with the current state.</p>
+ *
+ * <p>All this is done automatically and transparently to the caller once all
+ * active buckets have been updated.</p>
+ *
+ * @param superbucket A valid bucket ID that has been retrieved earlier through
+ * {@link #getNext()}
+ * @param progress A bucket logically contained within <code>super</code>. Subsequent
+ * updates for the same superbucket must have <code>progress</code> be in an increasing
+ * order, where order is defined as the in-order traversal of the bucket split
+ * tree. May also be the null bucket if the superbucket has not seen any "proper"
+ * progress updates yet or the special case Integer.MAX_VALUE. Note that inconsistent
+ * splitting might actually see <code>progress</code> as containing <code>super</code>
+ * rather than vice versa, so this is explicitly allowed to pass by the code.
+ */
+ public void update(BucketId superbucket, BucketId progress) {
+ // Delegate to bucket source, as it knows how to deal with buckets
+ // that are in an inconsistent state wrt distribution bit count
+ bucketSource.update(superbucket, progress, progressToken);
+ }
+
+ /**
+ * @return The total number of iterable buckets that remain to be processed
+ *
+ * Note: currently includes all non-finished (i.e. active and pending
+ * buckets) as well
+ */
+ public long getRemainingBucketCount() {
+ return progressToken.getTotalBucketCount() - progressToken.getFinishedBucketCount();
+ }
+
+ /**
+ * @return Internal bucket source instance. Do <i>NOT</i> modify!
+ */
+ protected BucketSource getBucketSource() {
+ return bucketSource;
+ }
+
+ public ProgressToken getProgressToken() {
+ return progressToken;
+ }
+
+ public int getDistributionBitCount() {
+ return distributionBitCount;
+ }
+
+ /**
+ * <p>Set the distribution bit count for the iterator and the buckets it
+ * currently maintains and will return in the future.</p>
+ *
+ * <p>For document selections that result in a explicit set of buckets, this
+ * is essentially a no-op, so in such a case, disregard the rest of this text.</p>
+ *
+ * <p>Changing the number of distribution bits for an unknown document
+ * selection will effectively scale the bucket space that will be visited;
+ * each bit increase or decrease doubling or halving its size, respectively.
+ * When increasing, any pending buckets will be split to ensure the total
+ * bucket space covered remains the same. Correspondingly, when decreasing,
+ * any pending buckets will be merged appropriately.</p>
+ *
+ * <p>If there are buckets active at the time of the change, the actual
+ * bucket splitting/merging operations are kept on hold until all active
+ * buckets have been updated, at which point they will be automatically
+ * performed. The iterator will force such an update by not giving out
+ * any new or pending buckets until that happens.</p>
+ *
+ * <p><em>Note:</em> when decreasing the number of distribution bits,
+ * there is a chance of losing superbucket progress in a bucket that
+ * is merged with another bucket, leading to potential duplicate
+ * results.</p>
+ *
+ * @param distBits New system state distribution bit count
+ */
+ public void setDistributionBitCount(int distBits) {
+ if (distributionBitCount != distBits) {
+ bucketSource.setDistributionBitCount(distBits, progressToken);
+ distributionBitCount = distBits;
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "Set visitor iterator distribution bit count to "
+ + distBits);
+ }
+ }
+ }
+
+ public boolean visitsAllBuckets() {
+ return bucketSource.visitsAllBuckets();
+ }
+
+ /**
+ * Create a new <code>VisitorIterator</code> instance based on the given document
+ * selection string.
+ *
+ * @param documentSelection Document selection string used to create the
+ * <code>VisitorIterator</code> instance. Depending on the characteristics of the
+ * selection, the iterator may iterate over only a small subset of the buckets or
+ * every bucket in the system. Both cases will be handled efficiently.
+ * @param idFactory {@link BucketId} factory specifying the number of distribution bits
+ * to use et al.
+ * @param progress A unique {@link ProgressToken} instance which is used for maintaining the state
+ * of the iterator. Can <em>not</em> be shared with other iterator instances at the same time.
+ * If <code>progress</code> contains work done in an earlier iteration run, the iterator will pick
+ * up from where it left off
+ * @return A new <code>VisitorIterator</code> instance
+ * @throws ParseException if <code>documentSelection</code> fails to properly parse
+ */
+ public static VisitorIterator createFromDocumentSelection(
+ String documentSelection,
+ BucketIdFactory idFactory,
+ int distributionBitCount,
+ ProgressToken progress) throws ParseException {
+ BucketSelector bucketSel = new BucketSelector(idFactory);
+ Set<BucketId> rawBuckets = bucketSel.getBucketList(documentSelection);
+ BucketSource src;
+
+ // Depending on whether the expression yielded an unknown number of
+ // buckets, we create either an explicit bucket source or a distribution
+ // bit-based range source
+ if (rawBuckets == null) {
+ // Range source
+ src = new DistributionRangeBucketSource(distributionBitCount, progress);
+ } else {
+ // Explicit source
+ src = new ExplicitBucketSource(rawBuckets, distributionBitCount, progress);
+ }
+
+ return new VisitorIterator(progress, src);
+ }
+
+ /**
+ * Create a new <code>VisitorIterator</code> instance based on the given
+ * set of buckets. This is supported for internal use only, and is required
+ * by Synchronization. Use {@link #createFromDocumentSelection} instead for
+ * all normal purposes.
+ *
+ * @param bucketsToVisit The set of buckets that will be visited
+ * @param distributionBitCount Number of distribution bits to use
+ * @param progress A unique ProgressToken instance which is used for maintaining the state
+ * of the iterator. Can <em>not</em> be shared with other iterator instances at the same time.
+ * If <code>progress</code> contains work done in an earlier iteration run, the iterator will pick
+ * up from where it left off
+ * @return A new <code>VisitorIterator</code> instance
+ */
+ public static VisitorIterator createFromExplicitBucketSet(
+ Set<BucketId> bucketsToVisit,
+ int distributionBitCount,
+ ProgressToken progress) {
+ // For obvious reasons, always create an explicit source here
+ BucketSource src = new ExplicitBucketSource(bucketsToVisit,
+ distributionBitCount, progress);
+ return new VisitorIterator(progress, src);
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java new file mode 100644 index 00000000000..b81025c7286 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java @@ -0,0 +1,369 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.BucketId; +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.messagebus.ThrottlePolicy; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.text.Utf8; + +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * Parameters for creating or opening a visitor session + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class VisitorParameters extends Parameters { + + private String documentSelection; + private String visitorLibrary = "DumpVisitor"; + private int maxPending = 32; + private long timeoutMs = -1; + private long fromTimestamp = 0; + private long toTimestamp = 0; + boolean visitRemoves = false; + private String fieldSet = "[all]"; + boolean visitInconsistentBuckets = false; + private ProgressToken resumeToken = null; + private String resumeFileName = ""; + private String remoteDataHandler = null; + private VisitorDataHandler localDataHandler; + private VisitorControlHandler controlHandler; + private Map<String, byte []> libraryParameters = new TreeMap<String, byte []>(); + private Route visitRoute = null; + private float weight = 1; + private long maxFirstPassHits = -1; + private long maxTotalHits = -1; + private int visitorOrdering = 0; + private int maxBucketsPerVisitor = 1; + private boolean dynamicallyIncreaseMaxBucketsPerVisitor = false; + private float dynamicMaxBucketsIncreaseFactor = 2; + private LoadType loadType = LoadType.DEFAULT; + private DocumentProtocol.Priority priority = null; + private int traceLevel = 0; + private ThrottlePolicy throttlePolicy = null; + private boolean skipBucketsOnFatalErrors = false; + + // Advanced parameter, only for internal use. + Set<BucketId> bucketsToVisit = null; + + /** + * Creates visitor parameters from a document selection expression, using + * defaults for other parameters. + * + * @param documentSelection document selection expression + */ + public VisitorParameters(String documentSelection) { + this.documentSelection = documentSelection; + } + + /** + * Copy constructor. + * + * @param params object to copy + */ + public VisitorParameters(VisitorParameters params) { + setDocumentSelection(params.getDocumentSelection()); + setVisitorLibrary(params.getVisitorLibrary()); + setMaxPending(params.getMaxPending()); + setTimeoutMs(params.getTimeoutMs()); + setFromTimestamp(params.getFromTimestamp()); + setToTimestamp(params.getToTimestamp()); + visitRemoves(params.visitRemoves()); + fieldSet(params.fieldSet()); + visitInconsistentBuckets(params.visitInconsistentBuckets()); + setLibraryParameters(params.getLibraryParameters()); + setRoute(params.getRoute()); + setResumeFileName(params.getResumeFileName()); + setResumeToken(params.getResumeToken()); + if (params.getRemoteDataHandler() != null) { + setRemoteDataHandler(params.getRemoteDataHandler()); + } else { + setLocalDataHandler(params.getLocalDataHandler()); + } + setControlHandler(params.getControlHandler()); + setMaxFirstPassHits(params.getMaxFirstPassHits()); + setMaxTotalHits(params.getMaxTotalHits()); + setVisitorOrdering(params.getVisitorOrdering()); + setMaxBucketsPerVisitor(params.getMaxBucketsPerVisitor()); + setLoadType(params.getLoadType()); + setPriority(params.getPriority()); + setDynamicallyIncreaseMaxBucketsPerVisitor( + params.getDynamicallyIncreaseMaxBucketsPerVisitor()); + setDynamicMaxBucketsIncreaseFactor( + params.getDynamicMaxBucketsIncreaseFactor()); + setTraceLevel(params.getTraceLevel()); + skipBucketsOnFatalErrors(params.skipBucketsOnFatalErrors()); + } + + // Get functions + + // TODO: s/@return/Returns/ - this javadoc will not contain text in the method overview + + /** @return The selection string used for visiting. */ + public String getDocumentSelection() { return documentSelection; } + + /** @return What visitor library to use for the visiting. The library in question must be installed on each storage node in the target cluster. */ + public String getVisitorLibrary() { return visitorLibrary; } + + /** @return The maximum number of messages each storage visitor will have pending before waiting for acks from client. */ + public int getMaxPending() { return maxPending; } + + /** @return The timeout for the visitor in milliseconds. */ + public long getTimeoutMs() { return timeoutMs; } + + /** @return The minimum timestamp (in microsecs) of documents the visitor will visit. */ + public long getFromTimestamp() { return fromTimestamp; } + + /** @return The maximum timestamp (in microsecs) of documents the visitor will visit. */ + public long getToTimestamp() { return toTimestamp; } + + /** @return If this method returns true, the visitor will visit remove entries as well as documents (you can see what documents have been deleted). */ + public boolean visitRemoves() { return visitRemoves; } + + public boolean getVisitRemoves() { return visitRemoves; } + + public boolean getVisitHeadersOnly() { return "[header]".equals(fieldSet); } + + /** @return The field set to use. */ + public String fieldSet() { return fieldSet; } + + public String getFieldSet() { return fieldSet; } + + /** @return If this method returns true, the visitor will visit inconsistent buckets. */ + public boolean visitInconsistentBuckets() { return visitInconsistentBuckets; } + + public boolean getVisitInconsistentBuckets() { return visitInconsistentBuckets; } + + /** @return Returns a map of string → string of arguments that are passed to the visitor library. */ + public Map<String, byte []> getLibraryParameters() { return libraryParameters; } + + /** @return The progress token, which can be used to resume visitor. */ + public ProgressToken getResumeToken() { return resumeToken; } + + /** @return The filename for reading/storing progress token. */ + public String getResumeFileName() { return resumeFileName; } + + /** @return Address to the remote data handler. */ + public String getRemoteDataHandler() { return remoteDataHandler; } + + /** @return The local data handler. */ + public VisitorDataHandler getLocalDataHandler() { return localDataHandler; } + + /** @return The control handler. */ + public VisitorControlHandler getControlHandler() { return controlHandler; } + + /** @return Whether or not max buckets per visitor value should be dynamically + * increased when using orderdoc and visitors do not return at least half + * the desired amount of documents + */ + public boolean getDynamicallyIncreaseMaxBucketsPerVisitor() { + return dynamicallyIncreaseMaxBucketsPerVisitor; + } + + /** @return Factor with which max buckets are dynamically increased each time */ + public float getDynamicMaxBucketsIncreaseFactor() { + return dynamicMaxBucketsIncreaseFactor; + } + + public DocumentProtocol.Priority getPriority() { + if (priority != null) { + return priority; + } else if (loadType != null) { + return loadType.getPriority(); + } else { + return DocumentProtocol.Priority.NORMAL_3; + } + } + + // Set functions + + /** Set the document selection expression */ + public void setDocumentSelection(String selection) { documentSelection = selection; } + + /** Set which visitor library is used for visiting in storage. DumpVisitor is most common implementation. */ + public void setVisitorLibrary(String library) { visitorLibrary = library; } + + /** Set maximum pending messages one storage visitor will have pending to this client before stalling, waiting for acks. */ + public void setMaxPending(int maxPending) { this.maxPending = maxPending; } + + /** Set the timeout for the visitor in milliseconds. */ + public void setTimeoutMs(long timeoutMs) { this.timeoutMs = timeoutMs; } + + /** Set from timestamp in microseconds. Documents put/updated before this timestamp will not be visited. */ + public void setFromTimestamp(long timestamp) { fromTimestamp = timestamp; } + + /** Set to timestamp in microseconds. Documents put/updated after this timestamp will not be visited. */ + public void setToTimestamp(long timestamp) { toTimestamp = timestamp; } + + /** Set whether to visit remove entries. That is, entries saying that some document has been removed. */ + public void visitRemoves(boolean visitRemoves) { this.visitRemoves = visitRemoves; } + + public void setVisitRemoves(boolean visitRemoves) { this.visitRemoves = visitRemoves; } + + public void setVisitHeadersOnly(boolean headersOnly) { this.fieldSet = headersOnly ? "[header]" : "[all]"; } + + /** Set field set to use. */ + public void fieldSet(String fieldSet) { this.fieldSet = fieldSet; } + public void setFieldSet(String fieldSet) { this.fieldSet = fieldSet; } + + /** Set whether to visit inconsistent buckets. */ + public void visitInconsistentBuckets(boolean visitInconsistentBuckets) { this.visitInconsistentBuckets = visitInconsistentBuckets; } + + public void setVisitInconsistentBuckets(boolean visitInconsistentBuckets) { this.visitInconsistentBuckets = visitInconsistentBuckets; } + + /** Set a visitor library specific parameter. */ + public void setLibraryParameter(String param, String value) { + libraryParameters.put(param, Utf8.toBytes(value)); + } + + /** Set a visitor library specific parameter. */ + public void setLibraryParameter(String param, byte [] value) { libraryParameters.put(param, value); } + + /** Set all visitor library specific parameters. */ + public void setLibraryParameters(Map<String, byte []> params) { libraryParameters = params; } + + /** Set progress token, which can be used to resume visitor. */ + public void setResumeToken(ProgressToken token) { resumeToken = token; } + + /** + * Set filename for reading/storing progress token. If the file exists and + * contains progress data, visitor should resume visiting from this point. + */ + public void setResumeFileName(String fileName) { resumeFileName = fileName; } + + /** Set address for the remote data handler. */ + public void setRemoteDataHandler(String remoteDataHandler) { this.remoteDataHandler = remoteDataHandler; localDataHandler = null; } + + /** Set local data handler. */ + public void setLocalDataHandler(VisitorDataHandler localDataHandler) { this.localDataHandler = localDataHandler; remoteDataHandler = null; } + + /** Set control handler. */ + public void setControlHandler(VisitorControlHandler controlHandler) { this.controlHandler = controlHandler; } + + /** Set the name of the storage cluster route to visit. Default is "storage/cluster.storage". */ + public void setRoute(String route) { setRoute(Route.parse(route)); } + + /** Set the route to visit. */ + public void setRoute(Route route) { visitRoute = route; } + + /** @return Returns the name of the storage cluster to visit. */ + // TODO: Document: Where is the default - does this ever return null, or does it return "storage" if input is null? + public Route getRoute() { return visitRoute; } + + /** Set the maximum number of documents to visit (max documents returned by the visitor) */ + public void setMaxFirstPassHits(long max) { maxFirstPassHits = max; } + + /** @return Returns the maximum number of documents to visit (max documents returned by the visitor) */ + public long getMaxFirstPassHits() { return maxFirstPassHits; } + + /** Set the maximum number of documents to visit (max documents returned by the visitor) */ + public void setMaxTotalHits(long max) { maxTotalHits = max; } + + /** @return Returns the maximum number of documents to visit (max documents returned by the visitor) */ + public long getMaxTotalHits() { return maxTotalHits; } + + public Set<BucketId> getBucketsToVisit() { return bucketsToVisit; } + + public void setBucketsToVisit(Set<BucketId> buckets) { bucketsToVisit = buckets; } + + public int getVisitorOrdering() { return visitorOrdering; } + + public void setVisitorOrdering(int order) { visitorOrdering = order; } + + public int getMaxBucketsPerVisitor() { return maxBucketsPerVisitor; } + + public void setMaxBucketsPerVisitor(int max) { maxBucketsPerVisitor = max; } + + public void setTraceLevel(int traceLevel) { this.traceLevel = traceLevel; } + + public int getTraceLevel() { return traceLevel; } + + public void setPriority(DocumentProtocol.Priority priority) { + this.priority = priority; + } + + public ThrottlePolicy getThrottlePolicy() { + return throttlePolicy; + } + + public void setThrottlePolicy(ThrottlePolicy policy) { + throttlePolicy = policy; + } + + public void setLoadType(LoadType loadType) { + this.loadType = loadType; + } + + public LoadType getLoadType() { + return loadType; + } + + public boolean skipBucketsOnFatalErrors() { return skipBucketsOnFatalErrors; } + + public void skipBucketsOnFatalErrors(boolean skipBucketsOnFatalErrors) { this.skipBucketsOnFatalErrors = skipBucketsOnFatalErrors; } + + /** + * Set whether or not max buckets per visitor value should be dynamically + * increased when using orderdoc and visitors do not return at least half + * the desired amount of documents + * + * @param dynamicallyIncreaseMaxBucketsPerVisitor whether or not to increase + */ + public void setDynamicallyIncreaseMaxBucketsPerVisitor(boolean dynamicallyIncreaseMaxBucketsPerVisitor) { + this.dynamicallyIncreaseMaxBucketsPerVisitor = dynamicallyIncreaseMaxBucketsPerVisitor; + } + + /** + * Set factor with which max buckets are dynamically increased each time + * @param dynamicMaxBucketsIncreaseFactor increase factor (must be 1 or more) + */ + public void setDynamicMaxBucketsIncreaseFactor(float dynamicMaxBucketsIncreaseFactor) { + this.dynamicMaxBucketsIncreaseFactor = dynamicMaxBucketsIncreaseFactor; + } + + // Inherit docs from Object + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("VisitorParameters(\n") + .append(" Document selection: ").append(documentSelection).append('\n') + .append(" Visitor library: ").append(visitorLibrary).append('\n') + .append(" Max pending: ").append(maxPending).append('\n') + .append(" Timeout (ms): ").append(timeoutMs).append('\n') + .append(" Time period: ").append(fromTimestamp).append(" - ").append(toTimestamp).append('\n'); + if (visitRemoves) { + sb.append(" Visiting remove entries\n"); + } + if (visitInconsistentBuckets) { + sb.append(" Visiting inconsistent buckets\n"); + } + if (libraryParameters.size() > 0) { + sb.append(" Visitor library parameters:\n"); + for (Map.Entry<String, byte[]> e : libraryParameters.entrySet()) { + sb.append(" ").append(e.getKey()).append(" : "); + sb.append(Utf8.toString(e.getValue())).append('\n'); + } + } + sb.append(" Field set: ").append(fieldSet).append('\n'); + sb.append(" Route: ").append(visitRoute).append('\n'); + sb.append(" Weight: ").append(weight).append('\n'); + sb.append(" Max firstpass hits: ").append(maxFirstPassHits).append('\n'); + sb.append(" Max total hits: ").append(maxTotalHits).append('\n'); + sb.append(" Visitor ordering: ").append(visitorOrdering).append('\n'); + sb.append(" Max buckets: ").append(maxBucketsPerVisitor).append('\n'); + sb.append(" Priority: ").append(getPriority().toString()).append('\n'); + if (dynamicallyIncreaseMaxBucketsPerVisitor) { + sb.append(" Dynamically increasing max buckets per visitor\n"); + sb.append(" Increase factor: ") + .append(dynamicMaxBucketsIncreaseFactor) + .append('\n'); + } + sb.append(')'); + + return sb.toString(); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorResponse.java new file mode 100644 index 00000000000..87a44caceb5 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorResponse.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +/** + * Common class for all visitor responses. All visitor responses have ack + * tokens that must be acked. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class VisitorResponse { + private AckToken token; + + /** + * Creates visitor response containing an ack token. + * + * @param token the ack token + */ + public VisitorResponse(AckToken token) { + this.token = token; + } + + /** @return The ack token. */ + public AckToken getAckToken() { return token; } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorSession.java new file mode 100644 index 00000000000..55fd2790fe5 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorSession.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.messagebus.Trace; + +/** + * A session for tracking progress for and potentially receiving data from a + * visitor. + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public interface VisitorSession extends VisitorControlSession { + /** + * Checks if visiting is done. + * + * @return True if visiting is done (either by error or success). + */ + public boolean isDone(); + + /** + * Retrieves the last progress token gotten for this visitor. + * + * @return The progress token. + */ + public ProgressToken getProgress(); + + /** + * Returns the tracing information so far about the visitor. + * + * @return Returns the trace. + */ + public Trace getTrace(); + + /** + * Waits until visiting is done, or the given timeout (in ms) expires. + * Will wait forever if timeout is 0. + * + * @param timeoutMs The maximum amount of milliseconds to wait. + * @return True if visiting is done (either by error or success). + * @throws InterruptedException If an interrupt signal was received while waiting. + */ + public boolean waitUntilDone(long timeoutMs) throws InterruptedException; +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java new file mode 100644 index 00000000000..423c1d9f913 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java @@ -0,0 +1,137 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.local; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; + +import java.util.LinkedList; +import java.util.List; +import java.util.Random; + +/** + * @author bratseth + */ +public class LocalAsyncSession implements AsyncSession { + + private final List<Response> responses = new LinkedList<>(); + private final ResponseHandler handler; + private final LocalDocumentAccess access; + private final SyncSession syncSession; + private long requestId = 0; + private Random random = new Random(); + + private synchronized long getNextRequestId() { + requestId++; + return requestId; + } + + public LocalAsyncSession(AsyncParameters params, LocalDocumentAccess access) { + this.access = access; + this.handler = params.getResponseHandler(); + random.setSeed(System.currentTimeMillis()); + syncSession = access.createSyncSession(new SyncParameters()); + } + + @Override + public double getCurrentWindowSize() { + return 1000; + } + + @Override + public Result put(Document document) { + return put(document, DocumentProtocol.Priority.NORMAL_3); + } + + @Override + public Result put(Document document, DocumentProtocol.Priority pri) { + long req = getNextRequestId(); + try { + syncSession.put(new DocumentPut(document), pri); + addResponse(new DocumentResponse(req)); + } catch (Exception e) { + addResponse(new DocumentResponse(req, document, e.getMessage(), false)); + } + return new Result(req); + } + + @Override + public Result get(DocumentId id) { + return get(id, false, DocumentProtocol.Priority.NORMAL_3); + } + + @Override + public Result get(DocumentId id, boolean headersOnly, DocumentProtocol.Priority pri) { + long req = getNextRequestId(); + try { + addResponse(new DocumentResponse(req, syncSession.get(id))); + } catch (Exception e) { + addResponse(new DocumentResponse(req, e.getMessage(), false)); + } + return new Result(req); + } + + @Override + public Result remove(DocumentId id) { + return remove(id, DocumentProtocol.Priority.NORMAL_3); + } + + @Override + public Result remove(DocumentId id, DocumentProtocol.Priority pri) { + long req = getNextRequestId(); + if (syncSession.remove(new DocumentRemove(id), pri)) { + addResponse(new RemoveResponse(req, true)); + } else { + addResponse(new DocumentIdResponse(req, id, "Document not found.", false)); + } + return new Result(req); + } + + @Override + public Result update(DocumentUpdate update) { + return update(update, DocumentProtocol.Priority.NORMAL_3); + } + + @Override + public Result update(DocumentUpdate update, DocumentProtocol.Priority pri) { + long req = getNextRequestId(); + if (syncSession.update(update, pri)) { + addResponse(new UpdateResponse(req, true)); + } else { + addResponse(new DocumentUpdateResponse(req, update, "Document not found.", false)); + } + return new Result(req); + } + + @Override + public Response getNext() { + if (responses.isEmpty()) { + return null; + } + int index = random.nextInt(responses.size()); + return responses.remove(index); + } + + @Override + public Response getNext(int timeout) { + return getNext(); + } + + @Override + public void destroy() { + // empty + } + + private void addResponse(Response response) { + if (handler != null) { + handler.handleResponse(response); + } else { + responses.add(response); + } + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java new file mode 100644 index 00000000000..5ac77abd3ae --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.local; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.documentapi.*; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The main class of the local implementation of the document api + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class LocalDocumentAccess extends DocumentAccess { + + Map<DocumentId, Document> documents = new LinkedHashMap<DocumentId, Document>(); + + public LocalDocumentAccess(DocumentAccessParams params) { + super(params); + } + + @Override + public void shutdown() { + if (documentTypeManagerConfig != null) { + documentTypeManagerConfig.close(); + } + } + + @Override + public SyncSession createSyncSession(SyncParameters parameters) { + return new LocalSyncSession(this); + } + + @Override + public AsyncSession createAsyncSession(AsyncParameters parameters) { + return new LocalAsyncSession(parameters, this); + } + + @Override + public VisitorSession createVisitorSession(VisitorParameters parameters) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public VisitorDestinationSession createVisitorDestinationSession(VisitorDestinationParameters parameters) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public SubscriptionSession createSubscription(SubscriptionParameters parameters) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public SubscriptionSession openSubscription(SubscriptionParameters parameters) { + throw new UnsupportedOperationException("Not supported yet"); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java new file mode 100755 index 00000000000..966caa46969 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.local;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.TestAndSetCondition;
+import com.yahoo.documentapi.Response;
+import com.yahoo.documentapi.Result;
+import com.yahoo.documentapi.SyncSession;
+import com.yahoo.documentapi.messagebus.protocol.*;
+
+/**
+ * @author bratseth
+ */
+public class LocalSyncSession implements SyncSession {
+
+ private LocalDocumentAccess access;
+
+ public LocalSyncSession(LocalDocumentAccess access) {
+ this.access = access;
+ }
+
+ @Override
+ public void put(DocumentPut documentPut) {
+ if (documentPut.getCondition().isPresent()) {
+ throw new UnsupportedOperationException("test-and-set is not supported.");
+ }
+
+ access.documents.put(documentPut.getId(), documentPut.getDocument());
+ }
+
+ @Override
+ public void put(DocumentPut documentPut, DocumentProtocol.Priority priority) {
+ access.documents.put(documentPut.getId(), documentPut.getDocument());
+ }
+
+ @Override
+ public Document get(DocumentId id) {
+ return access.documents.get(id);
+ }
+
+ @Override
+ public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri) {
+ // FIXME: More than half the get() methods are deprecated, but they all
+ // call exactly the same method, including this one, throwing away most
+ // of the parameters
+ return access.documents.get(id);
+ }
+
+ @Override
+ public boolean remove(DocumentRemove documentRemove) {
+ if (documentRemove.getCondition().isPresent()) {
+ throw new UnsupportedOperationException("test-and-set is not supported.");
+ }
+ access.documents.remove(documentRemove.getId());
+ return true;
+ }
+
+ @Override
+ public boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority priority) {
+ return remove(documentRemove);
+ }
+
+ @Override
+ public boolean update(DocumentUpdate update) {
+ Document document = access.documents.get(update.getId());
+ if (document == null) {
+ return false;
+ }
+ update.applyTo(document);
+ return true;
+ }
+
+ @Override
+ public boolean update(DocumentUpdate update, DocumentProtocol.Priority pri) {
+ Document document = access.documents.get(update.getId());
+ if (document == null) {
+ return false;
+ }
+ update.applyTo(document);
+ return true;
+ }
+
+ @Override
+ public Response getNext() {
+ throw new UnsupportedOperationException("Queue not supported.");
+ }
+
+ @Override
+ public Response getNext(int timeout) {
+ throw new UnsupportedOperationException("Queue not supported.");
+ }
+
+ @Override
+ public void destroy() {
+ access = null;
+ }
+
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/package-info.java b/documentapi/src/main/java/com/yahoo/documentapi/local/package-info.java new file mode 100644 index 00000000000..edaf8ac3a94 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.documentapi.local; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java new file mode 100644 index 00000000000..7b895845f3d --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -0,0 +1,300 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.Result; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.*; + +import java.lang.Error; +import java.util.Set; +import java.util.HashSet; +import java.util.Queue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Logger; + +/** + * An access session which wraps a messagebus source session sending document messages. + * The sessions are multithread safe. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar Rosenvinge</a> + */ +public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { + + private static final Logger log = Logger.getLogger(MessageBusAsyncSession.class.getName()); + private final AtomicLong requestId = new AtomicLong(0); + private final BlockingQueue<Response> responses = new LinkedBlockingQueue<Response>(); + private final ThrottlePolicy throttlePolicy; + private final SourceSession session; + private String route; + private int traceLevel; + + /** + * Creates a new async session running on message bus logic. + * + * @param asyncParams Common asyncsession parameters, not used. + * @param bus The message bus on which to run. + * @param mbusParams Parameters concerning message bus configuration. + */ + MessageBusAsyncSession(AsyncParameters asyncParams, MessageBus bus, MessageBusParams mbusParams) { + this(asyncParams, bus, mbusParams, null); + } + + /** + * Creates a new async session running on message bus logic with a specified reply handler. + * + * @param asyncParams Common asyncsession parameters, not used. + * @param bus The message bus on which to run. + * @param mbusParams Parameters concerning message bus configuration. + * @param handler The external reply handler. + */ + MessageBusAsyncSession(AsyncParameters asyncParams, MessageBus bus, MessageBusParams mbusParams, + ReplyHandler handler) { + route = mbusParams.getRoute(); + traceLevel = mbusParams.getTraceLevel(); + throttlePolicy = mbusParams.getSourceSessionParams().getThrottlePolicy(); + if (handler == null) { + handler = new MyReplyHandler(asyncParams.getResponseHandler(), responses); + } + session = bus.createSourceSession(handler, mbusParams.getSourceSessionParams()); + } + + @Override + public Result put(Document document) { + return put(document, DocumentProtocol.Priority.NORMAL_3); + } + + @Override + public Result put(Document document, DocumentProtocol.Priority pri) { + PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(document)); + msg.setPriority(pri); + return send(msg); + } + + @Override + public Result get(DocumentId id) { + return get(id, false, DocumentProtocol.Priority.NORMAL_1); + } + + @Override + public Result get(DocumentId id, boolean headersOnly, DocumentProtocol.Priority pri) { + GetDocumentMessage msg = new GetDocumentMessage(id, headersOnly ? "[header]" : "[all]"); + msg.setPriority(pri); + return send(msg); + } + + @Override + public Result remove(DocumentId id) { + return remove(id, DocumentProtocol.Priority.NORMAL_2); + } + + @Override + public Result remove(DocumentId id, DocumentProtocol.Priority pri) { + RemoveDocumentMessage msg = new RemoveDocumentMessage(id); + msg.setPriority(pri); + return send(msg); + } + + @Override + public Result update(DocumentUpdate update) { + return update(update, DocumentProtocol.Priority.NORMAL_2); + } + + @Override + public Result update(DocumentUpdate update, DocumentProtocol.Priority pri) { + UpdateDocumentMessage msg = new UpdateDocumentMessage(update); + msg.setPriority(pri); + return send(msg); + } + + /** + * A convenience method for assigning the internal trace level and route string to a message before sending it + * through the internal mbus session object. + * + * @param msg The message to send. + * @return The document api result object. + */ + public Result send(Message msg) { + try { + long reqId = requestId.incrementAndGet(); + msg.setContext(reqId); + msg.getTrace().setLevel(traceLevel); + if (route != null) { + return toResult(reqId, session.send(msg, route, true)); + } else { + return toResult(reqId, session.send(msg)); + } + } catch (Exception e) { + return new Result(Result.ResultType.FATAL_ERROR, new Error(e.getMessage(), e)); + } + } + + @Override + public Response getNext() { + return responses.poll(); + } + + @Override + public Response getNext(int timeoutMilliseconds) throws InterruptedException { + return responses.poll(timeoutMilliseconds, TimeUnit.MILLISECONDS); + } + + @Override + public void destroy() { + session.destroy(); + } + + @Override + public String getRoute() { + return route; + } + + @Override + public void setRoute(String route) { + this.route = route; + } + + @Override + public int getTraceLevel() { + return traceLevel; + } + + @Override + public void setTraceLevel(int traceLevel) { + this.traceLevel = traceLevel; + } + + @Override + public double getCurrentWindowSize() { + if (throttlePolicy instanceof StaticThrottlePolicy) { + return ((StaticThrottlePolicy)throttlePolicy).getMaxPendingCount(); + } + return 0; + } + + /** + * Returns a concatenated error string from the errors contained in a reply. + * + * @param reply The reply whose errors to concatenate. + * @return The error string. + */ + static String getErrorMessage(Reply reply) { + if (!reply.hasErrors()) { + return null; + } + StringBuilder errors = new StringBuilder(); + for (int i = 0; i < reply.getNumErrors(); ++i) { + errors.append(reply.getError(i)).append(" "); + } + return errors.toString(); + } + + static Set<Integer> + getErrorCodes(Reply reply) { + Set<Integer> errorCodes = new HashSet<>(); + for (int i = 0; i < reply.getNumErrors(); ++i) { + errorCodes.add(reply.getError(i).getCode()); + } + return errorCodes; + } + + private static Result toResult(long reqId, com.yahoo.messagebus.Result mbusResult) { + if (mbusResult.isAccepted()) { + return new Result(reqId); + } else if (mbusResult.getError().getCode() == ErrorCode.SEND_QUEUE_FULL) { + return new Result(Result.ResultType.TRANSIENT_ERROR, + new Error(mbusResult.getError().getMessage() + " (" + mbusResult.getError().getCode() + ")")); + } else { + return new Result(Result.ResultType.FATAL_ERROR, + new Error(mbusResult.getError().getMessage() + " (" + mbusResult.getError().getCode() + ")")); + } + } + + private static Response toResponse(Reply reply) { + long reqId = (Long)reply.getContext(); + return reply.hasErrors() ? toError(reply, reqId) : toSuccess(reply, reqId); + } + + private static Response toError(Reply reply, long reqId) { + Message msg = reply.getMessage(); + String err = getErrorMessage(reply); + switch (msg.getType()) { + case DocumentProtocol.MESSAGE_PUTDOCUMENT: + return new DocumentResponse(reqId, ((PutDocumentMessage)msg).getDocumentPut().getDocument(), err, false); + case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: + return new DocumentUpdateResponse(reqId, ((UpdateDocumentMessage)msg).getDocumentUpdate(), err, false); + case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: + return new DocumentIdResponse(reqId, ((RemoveDocumentMessage)msg).getDocumentId(), err, false); + case DocumentProtocol.MESSAGE_GETDOCUMENT: + return new DocumentIdResponse(reqId, ((GetDocumentMessage)msg).getDocumentId(), err, false); + default: + return new Response(reqId, err, false); + } + } + + @SuppressWarnings("deprecation") + private static Response toSuccess(Reply reply, long reqId) { + switch (reply.getType()) { + case DocumentProtocol.REPLY_GETDOCUMENT: + GetDocumentReply docReply = ((GetDocumentReply) reply); + Document getDoc = docReply.getDocument(); + if (getDoc != null) { + getDoc.setLastModified(docReply.getLastModified()); + } + return new DocumentResponse(reqId, getDoc); + case DocumentProtocol.REPLY_REMOVEDOCUMENT: + return new RemoveResponse(reqId, ((RemoveDocumentReply)reply).wasFound()); + case DocumentProtocol.REPLY_UPDATEDOCUMENT: + return new UpdateResponse(reqId, ((UpdateDocumentReply)reply).wasFound()); + case DocumentProtocol.REPLY_PUTDOCUMENT: + break; + default: + return new Response(reqId); + } + Message msg = reply.getMessage(); + switch (msg.getType()) { + case DocumentProtocol.MESSAGE_PUTDOCUMENT: + return new DocumentResponse(reqId, ((PutDocumentMessage)msg).getDocumentPut().getDocument()); + case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: + return new DocumentIdResponse(reqId, ((RemoveDocumentMessage)msg).getDocumentId()); + case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: + return new DocumentUpdateResponse(reqId, ((UpdateDocumentMessage)msg).getDocumentUpdate()); + default: + return new Response(reqId); + } + } + + private static class MyReplyHandler implements ReplyHandler { + + final ResponseHandler handler; + final Queue<Response> queue; + + MyReplyHandler(ResponseHandler handler, Queue<Response> queue) { + this.handler = handler; + this.queue = queue; + } + + @Override + public void handleReply(Reply reply) { + if (reply.getTrace().getLevel() > 0) { + log.log(LogLevel.INFO, reply.getTrace().toString()); + } + Response response = toResponse(reply); + if (handler != null) { + handler.handleResponse(response); + } else { + queue.add(response); + } + } + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java new file mode 100644 index 00000000000..818bc204784 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -0,0 +1,134 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus; + +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.RPCMessageBus; +import com.yahoo.messagebus.network.Network; +import com.yahoo.messagebus.routing.RoutingTable; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +/** + * This class implements the {@link DocumentAccess} interface using message bus for communication. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar Rosenvinge</a> + * @author bratseth + */ +public class MessageBusDocumentAccess extends DocumentAccess { + + private final RPCMessageBus bus; + private final MessageBusParams params; + // TODO: make pool size configurable? ScheduledExecutorService is not dynamic + private final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool( + Runtime.getRuntime().availableProcessors(), ThreadFactoryFactory.getDaemonThreadFactory("mbus.access.scheduler")); + + /** + * Creates a new document access using default values for all parameters. + */ + public MessageBusDocumentAccess() { + this(new MessageBusParams()); + } + + /** + * Creates a new document access using the supplied parameters. + * + * @param params All parameters for construction. + */ + public MessageBusDocumentAccess(MessageBusParams params) { + super(params); + this.params = params; + try { + com.yahoo.messagebus.MessageBusParams mbusParams = new com.yahoo.messagebus.MessageBusParams(params.getMessageBusParams()); + mbusParams.addProtocol(new DocumentProtocol(getDocumentTypeManager(), params.getProtocolConfigId(), params.getLoadTypes())); + bus = new RPCMessageBus(mbusParams, + params.getRPCNetworkParams(), + params.getRoutingConfigId()); + } + catch (Exception e) { + throw new DocumentAccessException(e); + } + } + + @Override + public void shutdown() { + bus.destroy(); + if (documentTypeManagerConfig != null) { + documentTypeManagerConfig.close(); + } + scheduledExecutorService.shutdownNow(); + } + + @Override + public MessageBusSyncSession createSyncSession(SyncParameters parameters) { + return new MessageBusSyncSession(parameters, bus.getMessageBus(), this.params); + } + + @Override + public MessageBusAsyncSession createAsyncSession(AsyncParameters parameters) { + return new MessageBusAsyncSession(parameters, bus.getMessageBus(), this.params); + } + + @Override + public MessageBusVisitorSession createVisitorSession(VisitorParameters params) throws ParseException, IllegalArgumentException { + MessageBusVisitorSession.AsyncTaskExecutor executor = new MessageBusVisitorSession.ThreadAsyncTaskExecutor(scheduledExecutorService); + MessageBusVisitorSession.MessageBusSenderFactory senderFactory = new MessageBusVisitorSession.MessageBusSenderFactory(bus.getMessageBus()); + MessageBusVisitorSession.MessageBusReceiverFactory receiverFactory = new MessageBusVisitorSession.MessageBusReceiverFactory(bus.getMessageBus()); + RoutingTable table = bus.getMessageBus().getRoutingTable(DocumentProtocol.NAME); + + MessageBusVisitorSession session = new MessageBusVisitorSession(params, executor, senderFactory, receiverFactory, table); + session.start(); + return session; + } + + @Override + public MessageBusVisitorDestinationSession createVisitorDestinationSession(VisitorDestinationParameters params) { + return new MessageBusVisitorDestinationSession(params, bus.getMessageBus()); + } + + @Override + public SubscriptionSession createSubscription(SubscriptionParameters parameters) { + throw new UnsupportedOperationException("Subscriptions not supported."); + } + + @Override + public SubscriptionSession openSubscription(SubscriptionParameters parameters) { + throw new UnsupportedOperationException("Subscriptions not supported."); + } + + /** + * Returns the internal message bus object so that clients can use it directly. + * + * @return The internal message bus. + */ + public MessageBus getMessageBus() { + return bus.getMessageBus(); + } + + /** + * Returns the network layer of the internal message bus object so that clients can use it directly. This may seem + * abit arbitrary, but the fact is that the RPCNetwork actually implements the IMirror API as well as exposing the + * SystemState object. + * + * @return The network layer. + */ + public Network getNetwork() { + return bus.getRPCNetwork(); + } + + /** + * Returns the parameter object that controls the underlying message bus. Changes to these parameters do not affect + * previously created sessions. + * + * @return The parameter object. + */ + public MessageBusParams getParams() { + return params; + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java new file mode 100755 index 00000000000..ecf6927f20c --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java @@ -0,0 +1,193 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus; + +import com.yahoo.documentapi.DocumentAccessParams; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.network.rpc.RPCNetworkParams; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class MessageBusParams extends DocumentAccessParams { + + private String routingConfigId = null; + private String protocolConfigId = null; + private String route = "route:default"; + private int traceLevel = 0; + private RPCNetworkParams rpcNetworkParams = new RPCNetworkParams(); + private com.yahoo.messagebus.MessageBusParams mbusParams = new com.yahoo.messagebus.MessageBusParams(); + private SourceSessionParams sourceSessionParams = new SourceSessionParams(); + private LoadTypeSet loadTypes; + + public MessageBusParams() { + this(new LoadTypeSet()); + } + + public MessageBusParams(LoadTypeSet loadTypes) { + this.loadTypes = loadTypes; + } + + /** + * + * @return Returns the set of load types accepted by this Vespa installation + */ + public LoadTypeSet getLoadTypes() { + return loadTypes; + } + + /** + * Returns the id to resolve to routing config. + * + * @return The config id. + */ + public String getRoutingConfigId() { + return routingConfigId; + } + + /** + * Sets the id to resolve to routing config. This has a proper default value that holds for Vespa applications, and + * can therefore be left unset. + * + * @param configId The config id. + * @return This object for chaining. + */ + public MessageBusParams setRoutingConfigId(String configId) { + routingConfigId = configId; + return this; + } + + /** + * Returns the id to resolve to protocol config. + * + * @return The config id. + */ + public String getProtocolConfigId() { + return protocolConfigId; + } + + /** + * Sets the id to resolve to protocol config. This has a proper default value that holds for Vespa applications, + * and can therefore be left usnet. + * + * @param configId The config id. + * @return This, to allow chaining. + */ + public MessageBusParams setProtocolConfigId(String configId) { + protocolConfigId = configId; + return this; + } + + /** + * Sets the name of the route to send appropriate requests to. This is a convenience method for prefixing a route + * with "route:", and using {@link #setRoute} instead. + * + * @param routeName The route name. + * @return This object for chaining. + */ + public MessageBusParams setRouteName(String routeName) { + return setRoute("route:" + routeName); + } + + /** + * Sets the route string to send all requests to. This string will be parsed as a route string, so setting a route + * name directly will not necessarily have the intended consequences. Use "route:<routename>" syntax for route + * names, or the convenience method {@link #setRouteName} for this. + * + * @param route The route string. + * @return This object for chaining. + */ + public MessageBusParams setRoute(String route) { + this.route = route; + return this; + } + + /** + * Returns the route string that all requests will be sent to. + * + * @return The route string. + */ + public String getRoute() { + return route; + } + + /** + * Returns the trace level to use when sending. + * + * @return The trace level. + */ + public int getTraceLevel() { + return traceLevel; + } + + /** + * Sets the trace level to use when sending. + * + * @param traceLevel The trace level. + * @return This object for chaining. + */ + public MessageBusParams setTraceLevel(int traceLevel) { + this.traceLevel = traceLevel; + return this; + } + + /** + * Returns the params object used to instantiate the rpc network layer for message bus. + * + * @return The params object. + */ + public RPCNetworkParams getRPCNetworkParams() { + return rpcNetworkParams; + } + + /** + * Sets the params object used to instantiate the rpc network layer for message bus. + * + * @param params The params object. + * @return This object for chaining. + */ + public MessageBusParams setRPCNetworkParams(RPCNetworkParams params) { + rpcNetworkParams = new RPCNetworkParams(params); + return this; + } + + /** + * Returns the params object used to instantiate the message bus. + * + * @return The params object. + */ + public com.yahoo.messagebus.MessageBusParams getMessageBusParams() { + return mbusParams; + } + + /** + * Sets the params object used to instantiate the message bus. + * + * @param params The params object. + * @return This object for chaining. + */ + public MessageBusParams setMessageBusParams(com.yahoo.messagebus.MessageBusParams params) { + mbusParams = new com.yahoo.messagebus.MessageBusParams(params); + return this; + } + + /** + * Returns a reference to the extended source session params object. + * + * @return The params object. + */ + public SourceSessionParams getSourceSessionParams() { + return sourceSessionParams; + } + + /** + * Sets the extended source session params. + * + * @param params The params object. + * @return This object for chaining. + */ + public MessageBusParams setSourceSessionParams(SourceSessionParams params) { + sourceSessionParams = new SourceSessionParams(params); + return this; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSession.java new file mode 100755 index 00000000000..7f5544a93ce --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSession.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus;
+
+/**
+ * This class defines a common interface for message bus sessions.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface MessageBusSession {
+
+ /**
+ * Returns the route to send all messages to when sending through this session.
+ *
+ * @return The route string.
+ */
+ public String getRoute();
+
+ /**
+ * Sets the route to send all messages to when sending through this session.
+ *
+ * @param route The route string.
+ */
+ public void setRoute(String route);
+
+ /**
+ * Returns the trace level used when sending messages through this session.
+ *
+ * @return The trace level.
+ */
+ public int getTraceLevel();
+
+ /**
+ * Sets the trace level used when sending messages through this session.
+ *
+ * @param traceLevel The trace level to set.
+ */
+ public void setTraceLevel(int traceLevel);
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java new file mode 100755 index 00000000000..5be94564556 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java @@ -0,0 +1,225 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.documentapi.messagebus;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.documentapi.AsyncParameters;
+import com.yahoo.documentapi.DocumentAccessException;
+import com.yahoo.documentapi.Response;
+import com.yahoo.documentapi.Result;
+import com.yahoo.documentapi.SyncParameters;
+import com.yahoo.documentapi.SyncSession;
+import com.yahoo.documentapi.messagebus.protocol.*;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.MessageBus;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.ReplyHandler;
+
+/**
+ * An implementation of the SyncSession interface running over message bus.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class MessageBusSyncSession implements MessageBusSession, SyncSession, ReplyHandler {
+
+ private MessageBusAsyncSession session;
+
+ /**
+ * Creates a new sync session running on message bus logic.
+ *
+ * @param syncParams Common syncsession parameters, not used.
+ * @param bus The message bus on which to run.
+ * @param mbusParams Parameters concerning message bus configuration.
+ */
+ MessageBusSyncSession(SyncParameters syncParams, MessageBus bus, MessageBusParams mbusParams) {
+ session = new MessageBusAsyncSession(new AsyncParameters(), bus, mbusParams, this);
+ }
+
+ @Override
+ public void handleReply(Reply reply) {
+ if (reply.getContext() instanceof RequestMonitor) {
+ ((RequestMonitor)reply.getContext()).replied(reply);
+ } else {
+ ReplyHandler handler = reply.getCallStack().pop(reply);
+ handler.handleReply(reply); // not there yet
+ }
+ }
+
+ @Override
+ public Response getNext() {
+ throw new UnsupportedOperationException("Queue not supported.");
+ }
+
+ @Override
+ public Response getNext(int timeout) {
+ throw new UnsupportedOperationException("Queue not supported.");
+ }
+
+ @Override
+ public void destroy() {
+ session.destroy();
+ }
+
+ /**
+ * Perform a synchronous sending of a message. This method block until the message is successfuly sent and a
+ * corresponding reply has been received.
+ *
+ * @param msg The message to send.
+ * @return The reply received.
+ */
+ public Reply syncSend(Message msg) {
+ try {
+ RequestMonitor monitor = new RequestMonitor();
+ msg.setContext(monitor);
+ msg.pushHandler(this); // store monitor
+ Result result = null;
+ while (result == null || result.getType() == Result.ResultType.TRANSIENT_ERROR) {
+ result = session.send(msg);
+ if (result != null && result.isSuccess()) {
+ break;
+ }
+ Thread.sleep(100);
+ }
+ if (!result.isSuccess()) {
+ throw new DocumentAccessException(result.getError().toString());
+ }
+ return monitor.waitForReply();
+ } catch (InterruptedException e) {
+ throw new DocumentAccessException(e);
+ }
+ }
+
+ @Override
+ public void put(DocumentPut documentPut) {
+ put(documentPut, DocumentProtocol.Priority.NORMAL_3);
+ }
+
+ @Override
+ public void put(DocumentPut documentPut, DocumentProtocol.Priority priority) {
+ PutDocumentMessage msg = new PutDocumentMessage(documentPut);
+ msg.setPriority(priority);
+ syncSendPutDocumentMessage(msg);
+ }
+
+ @Override
+ public Document get(DocumentId id) {
+ return get(id, "[all]", DocumentProtocol.Priority.NORMAL_1);
+ }
+
+ @Override
+ public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri) {
+ GetDocumentMessage msg = new GetDocumentMessage(id, fieldSet);
+ msg.setPriority(pri);
+
+ Reply reply = syncSend(msg);
+ if (reply.hasErrors()) {
+ throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply));
+ }
+ if (reply.getType() != DocumentProtocol.REPLY_GETDOCUMENT) {
+ throw new DocumentAccessException("Received unknown response: " + reply);
+ }
+ GetDocumentReply docReply = ((GetDocumentReply)reply);
+ Document doc = docReply.getDocument();
+ if (doc != null) {
+ doc.setLastModified(docReply.getLastModified());
+ }
+ return doc;
+ }
+
+ @Override
+ public boolean remove(DocumentRemove documentRemove) {
+ RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId());
+ msg.setCondition(documentRemove.getCondition());
+ return remove(msg);
+ }
+
+ @Override
+ public boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority pri) {
+ RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId());
+ msg.setPriority(pri);
+ msg.setCondition(documentRemove.getCondition());
+ return remove(msg);
+ }
+
+ private boolean remove(RemoveDocumentMessage msg) {
+ Reply reply = syncSend(msg);
+ if (reply.hasErrors()) {
+ throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply));
+ }
+ if (reply.getType() != DocumentProtocol.REPLY_REMOVEDOCUMENT) {
+ throw new DocumentAccessException("Received unknown response: " + reply);
+ }
+ return ((RemoveDocumentReply)reply).wasFound();
+ }
+
+ @Override
+ public boolean update(DocumentUpdate update) {
+ return update(update, DocumentProtocol.Priority.NORMAL_2);
+ }
+
+ @Override
+ public boolean update(DocumentUpdate update, DocumentProtocol.Priority pri) {
+ UpdateDocumentMessage msg = new UpdateDocumentMessage(update);
+ msg.setPriority(pri);
+ Reply reply = syncSend(msg);
+ if (reply.hasErrors()) {
+ throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply),
+ MessageBusAsyncSession.getErrorCodes(reply));
+ }
+ if (reply.getType() != DocumentProtocol.REPLY_UPDATEDOCUMENT) {
+ throw new DocumentAccessException("Received unknown response: " + reply);
+ }
+ return ((UpdateDocumentReply)reply).wasFound();
+ }
+
+ @Override
+ public String getRoute() {
+ return session.getRoute();
+ }
+
+ @Override
+ public void setRoute(String route) {
+ session.setRoute(route);
+ }
+
+ @Override
+ public int getTraceLevel() {
+ return session.getTraceLevel();
+ }
+
+ @Override
+ public void setTraceLevel(int traceLevel) {
+ session.setTraceLevel(traceLevel);
+ }
+
+ /**
+ * This class implements a monitor for waiting for a reply to arrive.
+ */
+ static class RequestMonitor {
+ private Reply reply = null;
+
+ synchronized Reply waitForReply() throws InterruptedException {
+ while (reply == null) {
+ wait();
+ }
+ return reply;
+ }
+
+ synchronized void replied(Reply reply) {
+ this.reply = reply;
+ notify();
+ }
+ }
+
+ private void syncSendPutDocumentMessage(PutDocumentMessage putDocumentMessage) {
+ Reply reply = syncSend(putDocumentMessage);
+ if (reply.hasErrors()) {
+ throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply),
+ MessageBusAsyncSession.getErrorCodes(reply));
+ }
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java new file mode 100644 index 00000000000..d12ee0a35df --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus; + +import com.yahoo.documentapi.AckToken; +import com.yahoo.documentapi.VisitorDestinationParameters; +import com.yahoo.documentapi.VisitorDestinationSession; +import com.yahoo.documentapi.VisitorResponse; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.*; + +import java.util.logging.Logger; + +/** + * A visitor destination session for receiving data from a visitor using a + * messagebus destination session. The default behaviour of the visitor session + * is to control visiting and receive the data. As an alternative, you may set + * up one or more visitor destination sessions and tell the visitor to send + * data to the remote destination(s). This is convenient if you want to receive + * data decoupled from controlling the visitor, but also to avoid a single data + * destination becoming a bottleneck. + * <p> + * Create the visitor destination session by calling the + * <code>MessageBusDocumentAccess.createVisitorDestinationSession</code> + * method. The visitor must be started by calling the + * <code>MessageBusDocumentAccess.createVisitorSession</code> method and + * progress tracked through the resulting visitor session. + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public class MessageBusVisitorDestinationSession implements VisitorDestinationSession,MessageHandler +{ + private static final Logger log = Logger.getLogger(MessageBusVisitorDestinationSession.class.getName()); + + private DestinationSession session; + private VisitorDestinationParameters params; + + /** + * Creates a message bus visitor destination session. + * + * @param params the parameters for the visitor destination session + * @param bus the message bus to use + */ + public MessageBusVisitorDestinationSession(VisitorDestinationParameters params, MessageBus bus) { + this.params = params; + session = bus.createDestinationSession(params.getSessionName(), true, this); + params.getDataHandler().setSession(this); + } + + public void handleMessage(Message message) { + Reply reply = ((DocumentMessage)message).createReply(); + message.swapState(reply); + + params.getDataHandler().onMessage(message, new AckToken(reply)); + } + + public void ack(AckToken token) { + try { + log.log(LogLevel.DEBUG, "Sending ack " + token.ackObject); + session.reply((Reply) token.ackObject); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public void destroy() { + session.destroy(); + session = null; + } + + public void abort() { + destroy(); + } + + public VisitorResponse getNext() { + return params.getDataHandler().getNext(); + } + + public VisitorResponse getNext(int timeoutMilliseconds) throws InterruptedException { + return params.getDataHandler().getNext(timeoutMilliseconds); + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java new file mode 100755 index 00000000000..a784ccd61e4 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -0,0 +1,1071 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus; + +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.Result; +import com.yahoo.messagebus.routing.RoutingTable; +import com.yahoo.vdslib.VisitorStatistics; +import com.yahoo.vdslib.state.ClusterState; + +import java.util.Arrays; +import java.util.concurrent.*; +import java.util.logging.Logger; + +/** + * <p> + * A visitor session for tracking progress for and potentially receiving data from + * a visitor using a MessageBus source and destination session. The source session + * is used to initiate visiting by sending create visitor messages to storage and + * the destination session is used for receiving progress. If the visitor is not + * set up to send data to a remote destination, data will also be received through + * the destination session. + * </p> + * <p> + * Create the visitor session by calling the + * <code>DocumentAccess.createVisitorSession</code> method. + * </p> + */ +public class MessageBusVisitorSession implements VisitorSession { + /** + * Abstract away notion of source session into a generic Sender + * interface to allow easy mocking. + */ + public static interface Sender { + public Result send(Message msg); + public int getPendingCount(); + public void destroy(); + } + + public static interface SenderFactory { + public Sender createSender(ReplyHandler replyHandler, VisitorParameters visitorParameters); + } + + /** + * Abstract away notion of destination session into a generic Receiver + * interface to allow easy mocking. + * The implementation must be thread safe since reply() can be invoked + * from an arbitrary thread. + */ + public static interface Receiver { + public void reply(Reply reply); + public void destroy(); + /** + * Get connection spec that can be used by other clients to send + * messages to this Receiver. + * @return connection spec + */ + public String getConnectionSpec(); + } + + public static interface ReceiverFactory { + public Receiver createReceiver(MessageHandler messageHandler, String sessionName); + } + + public static interface AsyncTaskExecutor { + public void submitTask(Runnable event); + public void scheduleTask(Runnable event, long delay, TimeUnit unit); + } + + public static class VisitingProgress { + private final VisitorIterator iterator; + private final ProgressToken token; + + public VisitingProgress(VisitorIterator iterator, ProgressToken token) { + this.iterator = iterator; + this.token = token; + } + + public VisitorIterator getIterator() { + return iterator; + } + + public ProgressToken getToken() { + return token; + } + } + + public enum State { + NOT_STARTED(false), + WORKING(false), + COMPLETED(false), + ABORTED(true), + FAILED(true), + TIMED_OUT(true); + + private final boolean failure; + private State(boolean failure) { + this.failure = failure; + } + + public boolean isFailure() { + return failure; + } + } + + public class StateDescription { + private final State state; + private final String description; + + public StateDescription(State state, String description) { + this.state = state; + this.description = description; + } + + public StateDescription(State state) { + this.state = state; + this.description = ""; + } + + public State getState() { + return state; + } + + public String getDescription() { + return description; + } + + VisitorControlHandler.CompletionCode toCompletionCode() { + switch (state) { + case COMPLETED: return VisitorControlHandler.CompletionCode.SUCCESS; + case ABORTED: return VisitorControlHandler.CompletionCode.ABORTED; + case FAILED: return VisitorControlHandler.CompletionCode.FAILURE; + case TIMED_OUT: return VisitorControlHandler.CompletionCode.TIMEOUT; + default: + throw new IllegalStateException("Current state did not have a valid value: " + state); + } + } + + public boolean failed() { + return state.isFailure(); + } + + public String toString() { + return state + ": " + description; + } + } + + /** + * Message bus implementations of interfaces + */ + + public static class MessageBusSender implements Sender { + private final SourceSession sourceSession; + + public MessageBusSender(SourceSession sourceSession) { + this.sourceSession = sourceSession; + } + + @Override + public Result send(Message msg) { + return sourceSession.send(msg); + } + + @Override + public int getPendingCount() { + return sourceSession.getPendingCount(); + } + + @Override + public void destroy() { + sourceSession.destroy(); + } + } + + public static class MessageBusSenderFactory implements SenderFactory { + private final MessageBus messageBus; + + public MessageBusSenderFactory(MessageBus messageBus) { + this.messageBus = messageBus; + } + + private SourceSessionParams createSourceSessionParams(VisitorParameters visitorParameters) { + SourceSessionParams sourceParams = new SourceSessionParams(); + + if (visitorParameters.getThrottlePolicy() != null) { + sourceParams.setThrottlePolicy(visitorParameters.getThrottlePolicy()); + } else { + sourceParams.setThrottlePolicy(new DynamicThrottlePolicy()); + } + + return sourceParams; + } + + @Override + public Sender createSender(ReplyHandler replyHandler, VisitorParameters visitorParameters) { + messageBus.setMaxPendingCount(0); + SourceSessionParams sessionParams = createSourceSessionParams(visitorParameters); + return new MessageBusSender(messageBus.createSourceSession(replyHandler, sessionParams)); + } + } + + public static class MessageBusReceiver implements Receiver { + private final DestinationSession destinationSession; + + public MessageBusReceiver(DestinationSession destinationSession) { + this.destinationSession = destinationSession; + } + + @Override + public void reply(Reply reply) { + destinationSession.reply(reply); + } + + @Override + public void destroy() { + destinationSession.destroy(); + } + + @Override + public String getConnectionSpec() { + return destinationSession.getConnectionSpec(); + } + } + + public static class MessageBusReceiverFactory implements ReceiverFactory { + private final MessageBus messageBus; + + public MessageBusReceiverFactory(MessageBus messageBus) { + this.messageBus = messageBus; + } + + private DestinationSessionParams createDestinationParams(MessageHandler messageHandler, String visitorName) { + DestinationSessionParams destparams = new DestinationSessionParams(); + destparams.setName(visitorName); + destparams.setBroadcastName(false); + destparams.setMessageHandler(messageHandler); + return destparams; + } + + @Override + public Receiver createReceiver(MessageHandler messageHandler, String sessionName) { + DestinationSessionParams destinationParams = createDestinationParams(messageHandler, sessionName); + return new MessageBusReceiver(messageBus.createDestinationSession(destinationParams)); + } + } + + public static class ThreadAsyncTaskExecutor implements AsyncTaskExecutor { + private final ScheduledExecutorService executor; + + public ThreadAsyncTaskExecutor(ScheduledExecutorService executor) { + this.executor = executor; + } + + @Override + public void submitTask(Runnable task) { + executor.submit(task); + } + + @Override + public void scheduleTask(Runnable task, long delay, TimeUnit unit) { + executor.schedule(task, delay, unit); + } + } + + private static final Logger log = Logger.getLogger(MessageBusVisitorSession.class.getName()); + + private static long sessionCounter = 0; + private static synchronized long getNextSessionId() { + return ++sessionCounter; + } + private static String createSessionName() { + StringBuilder sb = new StringBuilder(); + sb.append("visitor-").append(getNextSessionId()).append('-').append(System.currentTimeMillis()); + return sb.toString(); + } + + private final VisitorParameters params; + private final Sender sender; + private final Receiver receiver; + private final AsyncTaskExecutor taskExecutor; + private final VisitingProgress progress; + private final VisitorStatistics statistics; + private final String sessionName = createSessionName(); + private final String dataDestination; + private StateDescription state; + private long visitorCounter = 0; + private boolean scheduledSendCreateVisitors = false; + private boolean done = false; + private boolean destroying = false; // For testing and sanity checking + private final Object completionMonitor = new Object(); + private Trace trace; + /** + * We keep our own track of pending messages since the sender's pending + * count cannot be relied on in an async task execution context. This + * because it is decremented before the message is actually processed. + */ + private int pendingMessageCount = 0; + + public MessageBusVisitorSession(VisitorParameters visitorParameters, + AsyncTaskExecutor taskExecutor, + SenderFactory senderFactory, + ReceiverFactory receiverFactory, + RoutingTable routingTable) + throws ParseException + { + this.params = visitorParameters; // TODO(vekterli): make copy? legacy impl does not copy + initializeRoute(routingTable); + this.sender = senderFactory.createSender(createReplyHandler(), this.params); + this.receiver = receiverFactory.createReceiver(createMessageHandler(), sessionName); + this.taskExecutor = taskExecutor; + this.progress = createVisitingProgress(params); + this.statistics = new VisitorStatistics(); + this.state = new StateDescription(State.NOT_STARTED); + initializeHandlers(); + trace = new Trace(visitorParameters.getTraceLevel()); + dataDestination = (params.getLocalDataHandler() == null + ? params.getRemoteDataHandler() + : receiver.getConnectionSpec()); + + validateSessionParameters(); + + // If we're already done, no need to do anything at all! + if (progress.getIterator().isDone()) { + markSessionCompleted(); + } + } + + private void validateSessionParameters() { + if (dataDestination == null) { + throw new IllegalStateException("No data destination specified"); + } + } + + public void start() { + synchronized (progress.getToken()) { + if (progress.getIterator().isDone()) { + log.log(LogLevel.DEBUG, sessionName + ": progress token indicates " + + "session is done before it could even start; no-op"); + return; + } + transitionTo(new StateDescription(State.WORKING)); + taskExecutor.submitTask(new SendCreateVisitorsTask()); + } + } + + /** + * Attempt to transition to a new state. Depending on the current state, + * some transitions may be disallowed, such as transitioning from ABORTED + * to COMPLETED, since failures take precedence. Transitioning multiple + * times to the same state is a no-op in order to conserve the textual + * description given by the first transition to said state (which most + * likely is the most useful one for the end-user). + * + * @param newState State to attempt to transition to. + * @return State which is current after the transition. If transition was + * successful, will be equal to newState. + */ + private StateDescription transitionTo(StateDescription newState) { + log.log(LogLevel.DEBUG, sessionName + ": attempting transition to state " + newState); + if (newState.getState() == State.WORKING) { + assert(state.getState() == State.NOT_STARTED); + state = newState; + } else if (newState.getState() == State.COMPLETED) { + if (state.getState() != State.ABORTED && state.getState() != State.FAILED) { + state = newState; + } // else: don't override aborted state + } else if (newState.getState() == State.ABORTED) { + state = newState; + } else if (newState.getState() == State.FAILED) { + if (state.getState() != State.FAILED) { + state = newState; + } // else: don't override failed state + } else { + assert(false); + } + log.log(LogLevel.DEBUG, "Session '" + sessionName + "' is now in state " + state); + return state; + } + + private ReplyHandler createReplyHandler() { + return new ReplyHandler() { + @Override + public void handleReply(Reply reply) { + // Generally, handleReply will run in the context of the + // underlying transport layer's processing thread(s), so we + // schedule our own reply handling task to avoid blocking it. + try { + taskExecutor.submitTask(new HandleReplyTask(reply)); + } catch (RejectedExecutionException e) { + // We cannot reliably handle reply tasks failing to be submitted, since + // the reply task performs all our internal state handling logic. As such, + // we just immediately go into a failure destruction mode as soon as this + // happens, in which we do not wait for any active messages to be replied + // to. + log.log(LogLevel.WARNING, "Visitor session '" + sessionName + + "': failed to submit reply task to executor service! " + + "Session cannot reliably continue; terminating it early.", e); + + synchronized (progress.getToken()) { + transitionTo(new StateDescription(State.FAILED, "Failed to submit reply task to executor service: " + e.getMessage())); + if (!done) { + markSessionCompleted(); + } + } + } + } + }; + } + + private MessageHandler createMessageHandler() { + return new MessageHandler() { + @Override + public void handleMessage(Message message) { + try { + taskExecutor.submitTask(new HandleMessageTask(message)); + } catch (RejectedExecutionException e) { + Reply reply = ((DocumentMessage)message).createReply(); + message.swapState(reply); + reply.addError(new Error( + DocumentProtocol.ERROR_ABORTED, + "Visitor session has been aborted")); + receiver.reply(reply); + } + } + }; + } + + private void initializeRoute(RoutingTable routingTable) { + // If no cluster route has been set by user arguments, attempt to retrieve it from mbus config. + if (params.getRoute() == null || !params.getRoute().hasHops()) { + params.setRoute(getClusterRoute(routingTable)); + log.log(LogLevel.DEBUG, "No route specified; resolved implicit " + + "storage cluster: " + params.getRoute().toString()); + } + } + + private String getClusterRoute(RoutingTable routingTable) throws IllegalArgumentException{ + String route = null; + for (RoutingTable.RouteIterator it = routingTable.getRouteIterator(); + it.isValid(); it.next()) + { + String str = it.getName(); + if (str.startsWith("storage/cluster.")) { + if (route != null) { + throw new IllegalArgumentException( + "There are multiple storage clusters in your application, " + + "please specify which one to visit."); + } + route = str; + } + } + if (route == null) { + throw new IllegalArgumentException("No storage cluster found in your application."); + } + return route; + } + + /** + * Called from the constructor to ensure control and data handlers + * are set and initialized. + */ + private void initializeHandlers() { + if (this.params.getLocalDataHandler() != null) { + this.params.getLocalDataHandler().reset(); + this.params.getLocalDataHandler().setSession(this); + } else if (this.params.getRemoteDataHandler() == null) { + this.params.setLocalDataHandler(new VisitorDataQueue()); + this.params.getLocalDataHandler().setSession(this); + } + + if (params.getControlHandler() != null) { + params.getControlHandler().reset(); + } else { + params.setControlHandler(new VisitorControlHandler()); + } + params.getControlHandler().setSession(this); + } + + private VisitingProgress createVisitingProgress(VisitorParameters params) + throws ParseException + { + ProgressToken progressToken; + if (params.getResumeToken() != null) { + progressToken = params.getResumeToken(); + } else { + progressToken = new ProgressToken(); + } + VisitorIterator visitorIterator; + + if (params.getBucketsToVisit() == null + || params.getBucketsToVisit().isEmpty()) + { + // Use 1 distribution bit as a starting point. This will almost certainly + // trigger a ERROR_WRONG_DISTRIBUTION reply immediately, meaning that we'll + // get a fresh system state from the start. Since no buckets should ever + // return with a OK result in such a case, we recognize this as a special + // case in the iterator and simply reset its entire internal state using + // the new db count rather than doing any splitting. + BucketIdFactory bucketIdFactory = new BucketIdFactory(); + visitorIterator = VisitorIterator.createFromDocumentSelection( + params.getDocumentSelection(), + bucketIdFactory, + 1, + progressToken); + } else { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "parameters specify explicit bucket set " + + "to visit; using it rather than document selection (" + + params.getBucketsToVisit().size() + " buckets given)"); + } + // Allow override of target buckets iff an explicit set of buckets + // to visit is given by the visitor parameters. This was primarily + // used for the defunct synchronization functionality, but since it's + // so easy to support, don't deprecate it just yet. + visitorIterator = VisitorIterator.createFromExplicitBucketSet( + params.getBucketsToVisit(), + 1, + progressToken); + } + return new VisitingProgress(visitorIterator, progressToken); + } + + private class SendCreateVisitorsTask implements Runnable { + // All private methods in this task must be protected by a lock around + // the progress token! + + private String getNextVisitorId() { + StringBuilder sb = new StringBuilder(); + ++visitorCounter; + sb.append(sessionName).append('-').append(visitorCounter); + return sb.toString(); + } + + private CreateVisitorMessage createMessage(VisitorIterator.BucketProgress bucket) { + CreateVisitorMessage msg = new CreateVisitorMessage( + params.getVisitorLibrary(), + getNextVisitorId(), + receiver.getConnectionSpec(), + dataDestination); + + msg.getTrace().setLevel(params.getTraceLevel()); + msg.setTimeRemaining(params.getTimeoutMs() != -1 + ? params.getTimeoutMs() + : 5 * 60 * 1000); + msg.setBuckets(Arrays.asList(bucket.getSuperbucket(), bucket.getProgress())); + msg.setDocumentSelection(params.getDocumentSelection()); + msg.setFromTimestamp(params.getFromTimestamp()); + msg.setToTimestamp(params.getToTimestamp()); + msg.setMaxPendingReplyCount(params.getMaxPending()); + msg.setFieldSet(params.fieldSet()); + msg.setVisitInconsistentBuckets(params.visitInconsistentBuckets()); + msg.setVisitRemoves(params.visitRemoves()); + msg.setParameters(params.getLibraryParameters()); + msg.setRoute(params.getRoute()); + msg.setVisitorOrdering(params.getVisitorOrdering()); + msg.setMaxBucketsPerVisitor(params.getMaxBucketsPerVisitor()); + msg.setLoadType(params.getLoadType()); + msg.setPriority(params.getPriority()); + + msg.setRetryEnabled(false); + + return msg; + } + + public void run() { + // Must sync around token as legacy API exposes it to handlers + // and they expect to be able to sync around it. + synchronized (progress.getToken()) { + try { + scheduledSendCreateVisitors = false; + while (progress.getIterator().hasNext()) { + VisitorIterator.BucketProgress bucket = progress.getIterator().getNext(); + Result result = sender.send(createMessage(bucket)); + if (result.isAccepted()) { + log.log(LogLevel.DEBUG, sessionName + ": sent CreateVisitor for bucket " + + bucket.getSuperbucket() + " with progress " + bucket.getProgress()); + ++pendingMessageCount; + } else { + // Must reinsert bucket without progress into iterator since + // we failed to send visitor. + progress.getIterator().update(bucket.getSuperbucket(), bucket.getProgress()); + break; + } + } + } catch (Exception e) { + String msg = "Got exception of type " + e.getClass().getName() + + " with message '" + e.getMessage() + + "' while attempting to send visitors"; + log.log(LogLevel.WARNING, msg); + transitionTo(new StateDescription(State.FAILED, msg)); + // It's likely that the exception caused a failure to send a + // visitor message, meaning we won't get a reply task in the + // future from which we can execute logic to complete the + // session. Thusly, we have to do this here and now. + continueVisiting(); + } catch (Throwable t) { + // We can't reliably handle this; take a nosedive + com.yahoo.protect.Process.logAndDie("Caught unhandled error when trying to send visitors", t); + } + } + } + } + + private void continueVisiting() { + if (visitingCompleted()) { + markSessionCompleted(); + } else { + scheduleSendCreateVisitorsIfApplicable(); + } + } + + private void markSessionCompleted() { + // 'done' is only ever written when token mutex is held, so safe to check + // outside of completionMonitor lock. + assert(!done) : "Session was marked as completed more than once"; + log.log(LogLevel.DEBUG, "Visitor session '" + sessionName + "' has completed"); + if (params.getLocalDataHandler() != null) { + params.getLocalDataHandler().onDone(); + } + // If skipFatalErrors is set and a fatal error did occur, fail + // the session now with the first encountered error message. + if (progress.getToken().containsFailedBuckets()) { + transitionTo(new StateDescription(State.FAILED, progress.getToken().getFirstErrorMsg())); + } + // NOTE: transitioning to COMPLETED will not override a failure + // state, so it's safe to always do this. + transitionTo(new StateDescription(State.COMPLETED)); + params.getControlHandler().onDone(state.toCompletionCode(), state.getDescription()); + synchronized (completionMonitor) { + done = true; + completionMonitor.notifyAll(); + } + } + + private class HandleReplyTask implements Runnable { + private Reply reply; + public HandleReplyTask(Reply reply) { + this.reply = reply; + } + + @Override + public void run() { + synchronized (progress.getToken()) { + try { + assert(pendingMessageCount > 0); + --pendingMessageCount; + if (reply.hasErrors()) { + handleErrorReply(reply); + } else if (reply instanceof CreateVisitorReply) { + handleCreateVisitorReply((CreateVisitorReply)reply); + } else { + String msg = "Received reply we do not know how to handle: " + + reply.getClass().getName(); + log.log(LogLevel.ERROR, msg); + transitionTo(new StateDescription(State.FAILED, msg)); + } + } catch (Exception e) { + String msg = "Got exception of type " + e.getClass().getName() + + " with message '" + e.getMessage() + + "' while processing reply in visitor session"; + e.printStackTrace(); + transitionTo(new StateDescription(State.FAILED, msg)); + } catch (Throwable t) { + // We can't reliably handle this; take a nosedive + com.yahoo.protect.Process.logAndDie("Caught unhandled error when running reply task", t); + } finally { + continueVisiting(); + } + } + } + } + + private class HandleMessageTask implements Runnable { + private Message message; + + private HandleMessageTask(Message message) { + this.message = message; + } + + @Override + public void run() { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Visitor session " + sessionName + ": Received message " + message); + } + try { + if (message instanceof VisitorInfoMessage) { + handleVisitorInfoMessage((VisitorInfoMessage)message); // always replies + } else { + handleDocumentMessage((DocumentMessage)message); // always replies on error + } + } catch (Throwable t) { + com.yahoo.protect.Process.logAndDie("Caught unhandled error when processing message", t); + } + } + } + + private void handleMessageProcessingException(Reply reply, Exception e, String what) { + final String errorDesc = formatProcessingException(e, what); + final String fullMsg = formatIdentifyingVisitorErrorString(errorDesc); + log.log(LogLevel.ERROR, fullMsg, e); + int errorCode; + synchronized (progress.getToken()) { + if (!params.skipBucketsOnFatalErrors()) { + errorCode = ErrorCode.APP_FATAL_ERROR; + transitionTo(new StateDescription(State.FAILED, errorDesc)); + } else { + errorCode = DocumentProtocol.ERROR_UNPARSEABLE; + } + } + reply.addError(new Error(errorCode, errorDesc)); + } + + private String formatProcessingException(Exception e, String whileProcessing) { + return String.format( + "Got exception of type %s with message '%s' while processing %s", + e.getClass().getName(), + e.getMessage(), + whileProcessing); + } + + private String formatIdentifyingVisitorErrorString(String details) { + return String.format( + "Visitor %s (selection '%s'): %s", + sessionName, + params.getDocumentSelection(), + details); + } + + /** + * NOTE: not called from within lock, function must take lock itself + */ + private void handleVisitorInfoMessage(VisitorInfoMessage msg) { + + Reply reply = msg.createReply(); + msg.swapState(reply); + + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Visitor session " + sessionName + + ": Received VisitorInfo with " + + msg.getFinishedBuckets().size() + " finished buckets"); + } + + try { + if (msg.getErrorMessage().length() > 0) { + params.getControlHandler().onVisitorError(msg.getErrorMessage()); + } + synchronized (progress.getToken()) { + // NOTE: control handlers shall sync on token themselves if + // they want to access it, but recursive locking is OK and by + // always locking we make screwing it up harder. + if (!isDone()) { + params.getControlHandler().onProgress(progress.getToken()); + } else { + reply.addError(new Error(ErrorCode.APP_FATAL_ERROR, "Visitor has been shut down")); + } + } + } catch (Exception e) { + handleMessageProcessingException(reply, e, "VisitorInfoMessage"); + } finally { + receiver.reply(reply); + } + } + + private void handleDocumentMessage(DocumentMessage msg) { + Reply reply = msg.createReply(); + msg.swapState(reply); + + if (params.getLocalDataHandler() == null) { + log.log(LogLevel.ERROR, sessionName + ": Got visitor data back to client with no local data destination."); + reply.addError(new Error(ErrorCode.APP_FATAL_ERROR, "Visitor data with no local data destination")); + receiver.reply(reply); + return; + } + try { + params.getLocalDataHandler().onMessage(msg, new AckToken(reply)); + } catch (Exception e) { + handleMessageProcessingException(reply, e, "DocumentMessage"); + // Immediately reply since we cannot count on AckToken being registered + receiver.reply(reply); + } + } + + private boolean isFatalError(Reply reply) { + Error error = reply.getError(0); + switch (error.getCode()) { + case ErrorCode.TIMEOUT: + case DocumentProtocol.ERROR_BUCKET_NOT_FOUND: + case DocumentProtocol.ERROR_WRONG_DISTRIBUTION: + return false; + } + return error.isFatal(); + } + + /** + * Return whether a (transient) error shall be exempt from visitor + * error reporting. This to prevent spamming handlers and output with + * errors for things that are happening naturally in the system. + * @return true if the error should be reported + */ + private boolean shouldReportError(Reply reply) { + Error error = reply.getError(0); + switch (error.getCode()) { + case DocumentProtocol.ERROR_BUCKET_NOT_FOUND: + case DocumentProtocol.ERROR_BUCKET_DELETED: + return false; + } + return true; + } + + private static String getErrorMessage(Error r) { + return DocumentProtocol.getErrorName(r.getCode()) + ": " + r.getMessage(); + } + + private static boolean isErrorOfType(Reply reply, int errorCode) { + return reply.getError(0).getCode() == errorCode; + } + + private void reportVisitorError(String message) { + params.getControlHandler().onVisitorError(message); + } + + private void handleErrorReply(Reply reply) { + CreateVisitorMessage msg = (CreateVisitorMessage)reply.getMessage(); + // Must reset bucket progress back to what it was before sending. + BucketId bucket = msg.getBuckets().get(0); + BucketId subProgress = msg.getBuckets().get(1); + progress.getIterator().update(bucket, subProgress); + + String message = getErrorMessage(reply.getError(0)); + log.log(LogLevel.DEBUG, sessionName + ": received error reply for bucket " + + bucket + " with message '" + message + "'"); + + if (isFatalError(reply)) { + if (params.skipBucketsOnFatalErrors()) { + progress.getToken().addFailedBucket(bucket, subProgress, message); + progress.getIterator().update(bucket, ProgressToken.FINISHED_BUCKET); + } else { + reportVisitorError(message); + transitionTo(new StateDescription(State.FAILED, message)); + return; // no additional visitors will be scheduled post-failure + } + } + if (isErrorOfType(reply, DocumentProtocol.ERROR_WRONG_DISTRIBUTION)) { + handleWrongDistributionReply((WrongDistributionReply)reply); + } else { + if (shouldReportError(reply)) { + reportVisitorError(message); + } + // Wait 100ms before new visitor task is executed. Will prevent + // visitors from being scheduled from caller. + scheduleSendCreateVisitorsIfApplicable(100, TimeUnit.MILLISECONDS); + } + } + + private boolean enoughHitsReceived() { + if (params.getMaxFirstPassHits() != -1 + && statistics.getDocumentsReturned() >= params.getMaxFirstPassHits()) + { + return true; + } + if (params.getMaxTotalHits() != -1 + && ((statistics.getDocumentsReturned() + + statistics.getSecondPassDocumentsReturned()) + >= params.getMaxTotalHits())) + { + return true; + } + return false; + } + + /** + * A session is considered completed if one or more of the following holds true: + * - All buckets have been visited (i.e. no active or pending visitors). + * - Visiting has failed fatally (or has been aborted) AND there are no + * active visitors remaining. 'Active' here means that we're waiting + * for a reply. + * - We have received sufficient number of documents (set via visitor + * parameters) from the buckets already visited AND there are no + * active visitors remaining. + * @return true if visiting has completed, false otherwise + */ + private boolean visitingCompleted() { + return (pendingMessageCount == 0) + && (progress.getIterator().isDone() + || state.failed() + || enoughHitsReceived()); + } + + /** + * Schedule a new SendCreateVisitors task iff there are still buckets to + * visit, the visiting has not failed fatally and we haven't already + * scheduled such a task. + */ + private void scheduleSendCreateVisitorsIfApplicable(long delay, TimeUnit unit) { + if (scheduledSendCreateVisitors + || !progress.getIterator().hasNext() + || state.failed() + || enoughHitsReceived()) + { + return; + } + taskExecutor.scheduleTask(new SendCreateVisitorsTask(), delay, unit); + scheduledSendCreateVisitors = true; + } + + private void scheduleSendCreateVisitorsIfApplicable() { + scheduleSendCreateVisitorsIfApplicable(0, TimeUnit.MILLISECONDS); + } + + private void handleCreateVisitorReply(CreateVisitorReply reply) { + CreateVisitorMessage msg = (CreateVisitorMessage)reply.getMessage(); + + BucketId superbucket = msg.getBuckets().get(0); + BucketId subBucketProgress = reply.getLastBucket(); + + log.log(LogLevel.DEBUG, sessionName + ": received CreateVisitorReply for bucket " + + superbucket + " with progress " + subBucketProgress); + + progress.getIterator().update(superbucket, subBucketProgress); + params.getControlHandler().onProgress(progress.getToken()); + statistics.add(reply.getVisitorStatistics()); + params.getControlHandler().onVisitorStatistics(statistics); + trace.getRoot().addChild(reply.getTrace().getRoot()); + + if (params.getDynamicallyIncreaseMaxBucketsPerVisitor() + && (reply.getVisitorStatistics().getDocumentsReturned() + < params.getMaxFirstPassHits() / 2.0)) + { + // Attempt to increase parallelism to reduce latency of visiting + // Ensure new count is within [1, 128] + int newMaxBuckets = Math.max(Math.min((int)(params.getMaxBucketsPerVisitor() + * params.getDynamicMaxBucketsIncreaseFactor()), 128), 1); + params.setMaxBucketsPerVisitor(newMaxBuckets); + log.log(LogLevel.DEBUG, sessionName + ": increasing max buckets per visitor to " + + params.getMaxBucketsPerVisitor()); + } + } + + private void handleWrongDistributionReply(WrongDistributionReply reply) { + try { + // Classnames clash with documentapi classes, so be explicit + ClusterState newState = new ClusterState(reply.getSystemState()); + int stateBits = newState.getDistributionBitCount(); + if (stateBits != progress.getIterator().getDistributionBitCount()) { + log.log(LogLevel.DEBUG, "System state changed; now at " + + stateBits + " distribution bits"); + // Update the internal state of the visitor iterator. If we're increasing + // the number of distribution bits, this may lead to splitting of pending + // buckets. If we're decreasing, it may lead to merging of pending buckets + // and potential loss of sub-bucket progress. In either way, the iterator + // will not let any new buckets out before all active buckets have been + // updated. + progress.getIterator().setDistributionBitCount(stateBits); + } + } catch (Exception e) { + log.log(LogLevel.ERROR, "Failed to parse new system state string: " + + reply.getSystemState()); + transitionTo(new StateDescription(State.FAILED, "Failed to parse cluster state '" + + reply.getSystemState() + "'")); + } + } + + public String getSessionName() { + return sessionName; + } + + @Override + public boolean isDone() { + synchronized (progress.getToken()) { + return done; + } + } + + @Override + public ProgressToken getProgress() { + return progress.getToken(); + } + + @Override + public Trace getTrace() { + return trace; + } + + @Override + public boolean waitUntilDone(long timeoutMs) throws InterruptedException { + return params.getControlHandler().waitUntilDone(timeoutMs); + } + + @Override + public void ack(AckToken token) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Visitor session " + sessionName + + ": Sending ack " + token.ackObject); + } + // No locking here; replying should be thread safe in itself + receiver.reply((Reply)token.ackObject); + } + + @Override + public void abort() { + synchronized (progress.getToken()) { + transitionTo(new StateDescription(State.ABORTED, "Visitor aborted by user")); + } + } + + @Override + public VisitorResponse getNext() { + if (params.getLocalDataHandler() == null) { + throw new IllegalStateException("Data has been routed to external source for this visitor"); + } + return params.getLocalDataHandler().getNext(); + } + + @Override + public VisitorResponse getNext(int timeoutMilliseconds) throws InterruptedException { + if (params.getLocalDataHandler() == null) { + throw new IllegalStateException("Data has been routed to external source for this visitor"); + } + return params.getLocalDataHandler().getNext(timeoutMilliseconds); + } + + /** + * For unit test purposes only, not to be used by any external parties. + * @return true if destroy() has been--or is being--invoked. + */ + public boolean isDestroying() { + synchronized (completionMonitor) { + return destroying; + } + } + + @Override + public void destroy() { + log.log(LogLevel.DEBUG, sessionName + ": synchronous destroy() called"); + try { + synchronized (progress.getToken()) { + synchronized (completionMonitor) { + // If we are destroying the session before it has completed (e.g. because + // waitUntilDone timed out or an interactive visiting was interrupted) + // set us to aborted state so that we'll seize + if (!done) { + transitionTo(new StateDescription(State.ABORTED, "Session explicitly destroyed before completion")); + } + } + } + synchronized (completionMonitor) { + assert(!destroying) : "Attempted to destroy VisitorSession more than once"; + destroying = true; + while (!done) { + completionMonitor.wait(); + } + } + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + try { + sender.destroy(); + receiver.destroy(); + } catch (Exception e) { + log.log(LogLevel.ERROR, "Caught exception destroying communication interfaces", e); + } + log.log(LogLevel.DEBUG, sessionName + ": synchronous destroy() done"); + } + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/ScheduledEventQueue.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/ScheduledEventQueue.java new file mode 100755 index 00000000000..f15d27e7cd8 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/ScheduledEventQueue.java @@ -0,0 +1,189 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus; + +import com.yahoo.concurrent.SystemTimer; +import com.yahoo.concurrent.Timer; + +import java.util.*; +import java.util.concurrent.RejectedExecutionException; + +/** + * Simple, lightweight event scheduler that does not maintain any executor + * threads of its own, but rather makes it the responsibility of the caller + * to run the events as the queue hands them over. + * + * Fully thread safe for multiple readers and writers. + */ +public class ScheduledEventQueue { + private final Set<Entry> tasks = new TreeSet<Entry>(); + private long sequenceCounter = 0; + private Timer timer; + private volatile boolean waiting = false; + private volatile boolean shutdown = false; + + private static class Entry implements Comparable<Entry> { + private Runnable task; + private long timestamp; + private long sequenceId; + + public Entry(Runnable task, long timestamp, long sequenceId) { + this.task = task; + this.timestamp = timestamp; + this.sequenceId = sequenceId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Entry entry = (Entry) o; + + if (sequenceId != entry.sequenceId) return false; + if (timestamp != entry.timestamp) return false; + if (!task.equals(entry.task)) return false; + + return true; + } + + public int compareTo(Entry o) { + if (timestamp < o.timestamp) return -1; + if (timestamp > o.timestamp) return 1; + if (sequenceId < o.sequenceId) return -1; + if (sequenceId > o.sequenceId) return 1; + return 0; + } + + public Runnable getTask() { + return task; + } + + public long getTimestamp() { + return timestamp; + } + + public long getSequenceId() { + return sequenceId; + } + } + + public ScheduledEventQueue() { + this.timer = SystemTimer.INSTANCE; + } + + public ScheduledEventQueue(Timer timer) { + this.timer = timer; + } + + public void pushTask(Runnable task) { + synchronized (tasks) { + if (shutdown) { + throw new RejectedExecutionException("Tasks can't be scheduled since queue has been shut down."); + } + + tasks.add(new Entry(task, 0, sequenceCounter++)); + tasks.notifyAll(); + } + } + + public void pushTask(Runnable task, long milliSecondsToWait) { + synchronized (tasks) { + if (shutdown) { + throw new RejectedExecutionException("Tasks can't be scheduled since queue has been shut down."); + } + + tasks.add(new Entry(task, timer.milliTime() + milliSecondsToWait, sequenceCounter++)); + tasks.notifyAll(); + } + } + + public boolean isWaiting() { + synchronized (tasks) { + return waiting; + } + } + + /** + * Waits until the queue has a task that is ready for scheduling, removes that + * task from the queue and returns it. + * + * @return The next task. + */ + public Runnable getNextTask() { + try { + while (true) { + synchronized (tasks) { + Iterator<Entry> iter = tasks.iterator(); + if (!iter.hasNext()) { + if (shutdown) { + return null; + } + // Set flag for unit tests to coordinate with. + waiting = true; + tasks.wait(); + waiting = false; + continue; + } + Entry retEntry = iter.next(); + long timeNow = timer.milliTime(); + if (retEntry.getTimestamp() > timeNow) { + waiting = true; + tasks.wait(retEntry.getTimestamp() - timeNow); + waiting = false; + continue; + } + iter.remove(); + return retEntry.getTask(); + } + } + } catch (InterruptedException e) { + return null; + } + } + + /** + * If there is a task ready for scheduling, remove it from the queue and return it. + * + * @return The next task. + */ + public Runnable popTask() { + synchronized (tasks) { + Iterator<Entry> iter = tasks.iterator(); + if (!iter.hasNext()) { + return null; + } + Entry retEntry = iter.next(); + if (retEntry.getTimestamp() > timer.milliTime()) { + return null; + } + iter.remove(); + return retEntry.getTask(); + } + } + + /** For unit testing only */ + public void wakeTasks() { + synchronized (tasks) { + tasks.notifyAll(); + } + } + + public void shutdown() { + synchronized (tasks) { + shutdown = true; + tasks.notifyAll(); + } + } + + public boolean isShutdown() { + synchronized (tasks) { + return shutdown; + } + } + + public long size() { + synchronized (tasks) { + return tasks.size(); + } + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java new file mode 100644 index 00000000000..a8edfe16bb5 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.loadtypes; + +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; + +/** + * @author thomasg + */ +public class LoadType { + private int id; + private String name; + private DocumentProtocol.Priority priority; + + public static LoadType DEFAULT = new LoadType(0, "default", DocumentProtocol.Priority.NORMAL_3); + + public LoadType(int id, String name, DocumentProtocol.Priority priority) { + this.id = id; + this.name = name; + this.priority = priority; + } + + public boolean equals(Object other) { + if (!(other instanceof LoadType)) { + return false; + } + + LoadType o = (LoadType)other; + + return name.equals(o.getName()) && id == o.getId() && priority == o.getPriority(); + } + + public String getName() { return name; } + + public String toString() { return name + " (id " + id + ")"; } + + public DocumentProtocol.Priority getPriority() { return priority; } + + public int getId() { return id; } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java new file mode 100644 index 00000000000..cb453559ab1 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.loadtypes; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +/** + * This class keeps track of all configured load types. + * + * For production use, you should only use the String constructor, + * and supply a valid config id. Only the load types configured will + * be propagated throughout the system, so there is no point in using other + * load types. + * + * For testing, you may want to use the empty constructor and add + * load types yourself with addType(). + */ +public class LoadTypeSet { + class DualMap { + Map<String, LoadType> nameMap = new TreeMap<String, LoadType>(); + Map<Integer, LoadType> idMap = new HashMap<Integer, LoadType>(); + + void put(LoadType l) { + if (nameMap.containsKey(l.getName()) || idMap.containsKey(l.getId())) { + throw new IllegalArgumentException( + "ID or name conflict when adding " + l); + } + + nameMap.put(l.getName(), l); + idMap.put(l.getId(), l); + } + } + + DualMap map; + + public LoadTypeSet() { + map = new DualMap(); + map.put(LoadType.DEFAULT); + } + + public LoadTypeSet(String configId) { + configure(new ConfigGetter<>(LoadTypeConfig.class).getConfig(configId)); + } + + public Map<String, LoadType> getNameMap() { + return map.nameMap; + } + + public Map<Integer, LoadType> getIdMap() { + return map.idMap; + } + + /** + * Used by config to generate priorities for a name, and add them to the load type set. + */ + public void addType(String name, String priority) { + try { + MessageDigest algorithm = MessageDigest.getInstance("MD5"); + algorithm.reset(); + algorithm.update(name.getBytes()); + byte messageDigest[] = algorithm.digest(); + + int id = 0; + for (int i = 0; i < 4; i++) { + int temp = ((int)messageDigest[i] & 0xff); + id <<= 8; + id |= temp; + } + + map.put(new LoadType(id, name, DocumentProtocol.Priority.valueOf(priority != null ? priority : "NORMAL_3"))); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + public void addLoadType(int id, String name, DocumentProtocol.Priority priority) { + map.put(new LoadType(id, name, priority)); + } + + public void configure(LoadTypeConfig config) { + DualMap newMap = new DualMap(); + + // Default should always be available. + newMap.put(LoadType.DEFAULT); + + for (LoadTypeConfig.Type t : config.type()) { + newMap.put(new LoadType(t.id(), t.name(), DocumentProtocol.Priority.valueOf(t.priority()))); + } + + map = newMap; + } +} + diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/package-info.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/package-info.java new file mode 100644 index 00000000000..34608cfc8db --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.documentapi.messagebus.loadtypes; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/package-info.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/package-info.java new file mode 100644 index 00000000000..70ed2af38ce --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.documentapi.messagebus; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ANDPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ANDPolicy.java new file mode 100755 index 00000000000..04818f80672 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ANDPolicy.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.messagebus.metrics.MetricSet;
+import com.yahoo.messagebus.routing.Hop;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.messagebus.routing.RoutingContext;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An AND policy is a routing policy that can be used to write simple routes that split a message between multiple other
+ * destinations. It can either be configured in a routing config, which will then produce a policy that always selects
+ * all configured recipients, or it can be configured using the policy parameter (i.e. a string following the name of
+ * the policy). Note that configured recipients take precedence over recipients configured in the parameter string.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ANDPolicy implements DocumentProtocolRoutingPolicy {
+
+ // A list of hops that are to always be selected when select() is invoked.
+ private final List<Hop> hops = new ArrayList<Hop>();
+
+ /**
+ * Constructs a new AND policy that requires all recipients to be ok for it to merge their replies to an ok reply.
+ * I.e. all errors in all child replies are copied into the merged reply.
+ *
+ * @param param A string of recipients to select unless recipients have been configured.
+ */
+ public ANDPolicy(String param) {
+ if (param == null || param.isEmpty()) {
+ return;
+ }
+ Route route = Route.parse(param);
+ for (int i = 0; i < route.getNumHops(); ++i) {
+ hops.add(route.getHop(i));
+ }
+ }
+
+ // Inherit doc from RoutingPolicy.
+ public void select(RoutingContext context) {
+ if (hops.isEmpty()) {
+ context.addChildren(context.getAllRecipients());
+ } else {
+ for (Hop hop : hops) {
+ Route route = new Route(context.getRoute());
+ route.setHop(0, hop);
+ context.addChild(route);
+ }
+ }
+ context.setSelectOnRetry(false);
+ context.addConsumableError(DocumentProtocol.ERROR_MESSAGE_IGNORED);
+ }
+
+ // Inherit doc from RoutingPolicy.
+ public void merge(RoutingContext context) {
+ DocumentProtocol.merge(context);
+ }
+
+ public void destroy() {
+ }
+
+ public MetricSet getMetrics() {
+ return null;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AbstractRoutableFactory.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AbstractRoutableFactory.java new file mode 100644 index 00000000000..bd192dea745 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AbstractRoutableFactory.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.text.Utf8; +import com.yahoo.vespa.objects.Deserializer; +import com.yahoo.vespa.objects.Serializer; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public abstract class AbstractRoutableFactory implements RoutableFactory { + + /** + * Reads a string from the given buffer that was previously written by {@link #encodeString(String, + * com.yahoo.vespa.objects.Serializer)}. + * + * @param in The byte buffer to read from. + * @return The decoded string. + */ + public static String decodeString(Deserializer in) { + int length = in.getInt(null); + if (length == 0) { + return ""; + } + return Utf8.toString(in.getBytes(null, length)); + } + + /** + * Writes the given string to the given byte buffer in such a way that it can be decoded using {@link + * #decodeString(com.yahoo.vespa.objects.Deserializer)}. + * + * @param str The string to encode. + * @param out The byte buffer to write to. + */ + public static void encodeString(String str, Serializer out) { + if (str == null || str.isEmpty()) { + out.putInt(null, 0); + } else { + byte[] nameBytes = Utf8.toBytes(str); + out.putInt(null, nameBytes.length); + out.put(null, nameBytes); + } + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AsyncInitializationPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AsyncInitializationPolicy.java new file mode 100644 index 00000000000..7e4e1d3a5ca --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AsyncInitializationPolicy.java @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.RoutingContext; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ScheduledThreadPoolExecutor; + +/** + * Abstract class for routing policies that need asynchronous initialization. + * This is recommended if the routing policy needs configuration, or synchronization with + * other sources. If this policy is not used in those cases, the messagebus thread might hang + * waiting for the other sources, causing messages to other routes to be blocked. + */ +public abstract class AsyncInitializationPolicy implements DocumentProtocolRoutingPolicy, Runnable { + + protected enum InitState { + NOT_STARTED, + RUNNING, + DONE + }; + + InitState initState; + ScheduledThreadPoolExecutor executor; + Exception initException; + boolean syncInit = true; + + public static Map<String, String> parse(String param) { + Map<String, String> map = new TreeMap<String, String>(); + + if (param != null) { + String[] p = param.split(";"); + for (String s : p) { + String[] keyValue = s.split("="); + + if (keyValue.length == 1) { + map.put(keyValue[0], "true"); + } else if (keyValue.length == 2) { + map.put(keyValue[0], keyValue[1]); + } + } + } + + return map; + } + + public AsyncInitializationPolicy(Map<String, String> params) { + initState = InitState.NOT_STARTED; + } + + public void needAsynchronousInitialization() { + syncInit = false; + } + + public abstract void init(); + + public abstract void doSelect(RoutingContext routingContext); + + private synchronized void checkStartInit() { + if (initState == InitState.NOT_STARTED) { + if (syncInit) { + init(); + initState = InitState.DONE; + } else { + executor = new ScheduledThreadPoolExecutor(1); + executor.execute(this); + initState = InitState.RUNNING; + } + } + } + + @Override + public void select(RoutingContext routingContext) { + synchronized (this) { + if (initException != null) { + Reply reply = new EmptyReply(); + reply.addError(new com.yahoo.messagebus.Error(ErrorCode.POLICY_ERROR, initException.getMessage())); + routingContext.setReply(reply); + return; + } + + checkStartInit(); + + if (initState == InitState.RUNNING) { + Reply reply = new EmptyReply(); + reply.addError(new com.yahoo.messagebus.Error(ErrorCode.SESSION_BUSY, "Policy is waiting to be initialized.")); + routingContext.setReply(reply); + return; + } + } + + doSelect(routingContext); + } + + public void run() { + try { + init(); + } catch (Exception e) { + initException = e; + } + + synchronized (this) { + initState = InitState.DONE; + this.notifyAll(); + } + } + + @Override + public void destroy() { + if (executor != null) { + executor.shutdownNow(); + } + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/BatchDocumentUpdateMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/BatchDocumentUpdateMessage.java new file mode 100644 index 00000000000..cfc6dc6ef82 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/BatchDocumentUpdateMessage.java @@ -0,0 +1,184 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.idstring.GroupDocIdString; +import com.yahoo.document.idstring.IdString; +import com.yahoo.document.idstring.UserDocIdString; +import com.yahoo.document.serialization.DocumentDeserializer; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class BatchDocumentUpdateMessage extends DocumentMessage { + + private DocumentDeserializer buffer = null; + private List<DocumentUpdate> updates = new ArrayList<DocumentUpdate>(); + private LazyDecoder decoder = null; + private String group = null; + private Long userId = null; + private BucketId bucketId = null; + + public String getGroup() { + return group; + } + + public Long getUserId() { + return userId; + } + + /** + * Constructs a new message for deserialization. + */ + BatchDocumentUpdateMessage() { + // empty + } + + /** + * Constructs a new message from a byte buffer. + * @param decoder The decoder to use for deserialization. + * @param buffer A byte buffer that contains a serialized message. + */ + public BatchDocumentUpdateMessage(long userId, LazyDecoder decoder, DocumentDeserializer buffer) { + this.userId = userId; + this.decoder = decoder; + this.buffer = buffer; + setBucketId(new UserDocIdString("foo", userId, "bar")); + } + + /** + * Constructs a new message from a byte buffer. + * @param decoder The decoder to use for deserialization. + * @param buffer A byte buffer that contains a serialized message. + */ + public BatchDocumentUpdateMessage(String group, LazyDecoder decoder, DocumentDeserializer buffer) { + this.group = group; + this.decoder = decoder; + this.buffer = buffer; + setBucketId(new GroupDocIdString("foo", group, "bar")); + } + + /** + * Constructs a batch document update message, to contain updates for documents for the given user. + */ + public BatchDocumentUpdateMessage(long userId) { + super(); + this.userId = userId; + setBucketId(new UserDocIdString("foo", userId, "bar")); + } + + /** + * Constructs a batch document update message, to contain updates for documents for the given user. + */ + public BatchDocumentUpdateMessage(String group) { + super(); + this.group = group; + setBucketId(new GroupDocIdString("foo", group, "bar")); + } + + void setBucketId(IdString id) { + BucketIdFactory factory = new BucketIdFactory(); + bucketId = factory.getBucketId(new DocumentId(id)); + } + + BucketId getBucketId() { + return bucketId; + } + + /** + * This method will make sure that any serialized content is deserialized into proper message content on first + * entry. Any subsequent entry into this function will do nothing. + */ + private void deserialize() { + if (decoder != null && buffer != null) { + decoder.decode(this, buffer); + decoder = null; + buffer = null; + } + } + + /** + * Returns the list of document updates to perform. + * + * @return The updates. + */ + public List<DocumentUpdate> getUpdates() { + deserialize(); + return updates; + } + + /** + * Adds a document update to perform. + * + * @param upd The document update to set. + */ + public void addUpdate(DocumentUpdate upd) { + buffer = null; + decoder = null; + + verifyUpdate(upd); + updates.add(upd); + } + + /** + * Returns the raw serialized buffer. This buffer is stored as the message is received from accross the network, and + * deserialized from as soon as a member is requested. This method will return null if the buffer has been decoded. + * + * @return The buffer containing the serialized data for this message, or null. + */ + ByteBuffer getSerializedBuffer() { + return buffer != null ? buffer.getBuf().getByteBuffer() : null; + } + + @Override + public DocumentReply createReply() { + return new BatchDocumentUpdateReply(); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE; + } + + void verifyUpdate(DocumentUpdate update) { + if (update == null) { + throw new IllegalArgumentException("Document update can not be null."); + } + + IdString idString = update.getId().getScheme(); + + if (group != null) { + String idGroup; + + if (idString.hasGroup()) { + idGroup = idString.getGroup(); + } else { + throw new IllegalArgumentException("Batch update message can only contain groupdoc or orderdoc items"); + } + + if (!group.equals(idGroup)) { + throw new IllegalArgumentException("Batch update message can not contain messages from group " + idGroup + " only group " + group); + } + } else { + long foundUserId = 0; + + if (idString.hasNumber()) { + foundUserId = idString.getNumber(); + } else { + throw new IllegalArgumentException("Batch update message can only contain userdoc or orderdoc items"); + } + + if (userId != foundUserId) { + throw new IllegalArgumentException("Batch update message can not contain messages from user " + foundUserId + " only user " + userId); + } + } + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/BatchDocumentUpdateReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/BatchDocumentUpdateReply.java new file mode 100755 index 00000000000..48eb41fdb5c --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/BatchDocumentUpdateReply.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import java.util.ArrayList;
+
+public class BatchDocumentUpdateReply extends WriteDocumentReply {
+
+ private ArrayList<Boolean> documentsNotFound = new ArrayList<Boolean>();
+
+ /**
+ * Constructs a new reply with no content.
+ */
+ public BatchDocumentUpdateReply() {
+ super(DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE);
+ }
+
+ /**
+ * If all documents to update are found, this vector will be empty. If
+ * one or more documents are not found, this vector will have the size of
+ * the initial number of updates, with entries set to true where the
+ * corresponding update was not found.
+ *
+ * @return Vector containing indices of not found documents, or empty
+ * if all documents were found
+ */
+ public ArrayList<Boolean> getDocumentsNotFound() {
+ return documentsNotFound;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java new file mode 100644 index 00000000000..4dd224bcc45 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import java.util.Map; + +/** + * Policy to talk to content clusters. + */ +public class ContentPolicy extends StoragePolicy { + + public static class ContentParameters extends Parameters { + + public ContentParameters(Map<String, String> parameters) { + super(parameters); + } + + @Override + public String getDistributionConfigId() { + if (distributionConfigId != null) { + return distributionConfigId; + } + return clusterName; + } + + @Override + public SlobrokHostPatternGenerator createPatternGenerator() { + return new SlobrokHostPatternGenerator(getClusterName()) { + public String getDistributorHostPattern(Integer distributor) { + return "storage/cluster." + getClusterName() + "/distributor/" + (distributor == null ? "*" : distributor) + "/default"; + } + }; + } + } + + public ContentPolicy(Map<String, String> params) { + super(new ContentParameters(params), params); + } + + public ContentPolicy(String parameters) { + this(parse(parameters)); + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java new file mode 100644 index 00000000000..9e6c5cb793b --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java @@ -0,0 +1,217 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; + +import java.util.*; + +public class CreateVisitorMessage extends DocumentMessage { + + private String libName = "DumpVisitor"; + private String instanceId = ""; + private String controlDestination = ""; + private String dataDestination = ""; + private String docSelection = ""; + private int maxPendingReplyCount = 8; + private List<BucketId> buckets = new ArrayList<>(); + private long fromTime = 0; + private long toTime = 0; + private boolean visitRemoves = false; + private String fieldSet = "[all]"; + private boolean visitInconsistentBuckets = false; + private Map<String, byte[]> params = new TreeMap<>(); + private int version = 42; + private int ordering = 0; + private int maxBucketsPerVisitor = 1; + + CreateVisitorMessage() { + // must be deserialized into + } + + public CreateVisitorMessage(String libraryName, String instanceId, String controlDestination, + String dataDestination) { + libName = libraryName; + this.instanceId = instanceId; + this.controlDestination = controlDestination; + this.dataDestination = dataDestination; + } + + public String getLibraryName() { + return libName; + } + + public void setLibraryName(String libraryName) { + libName = libraryName; + } + + public String getInstanceId() { + return instanceId; + } + + public void setInstanceId(String instanceId) { + this.instanceId = instanceId; + } + + public String getControlDestination() { + return controlDestination; + } + + public void setControlDestination(String controlDestination) { + this.controlDestination = controlDestination; + } + + public String getDataDestination() { + return dataDestination; + } + + public void setDataDestination(String dataDestination) { + this.dataDestination = dataDestination; + } + + public String getDocumentSelection() { + return docSelection; + } + + public void setDocumentSelection(String documentSelection) { + docSelection = documentSelection; + } + + public int getMaxPendingReplyCount() { + return maxPendingReplyCount; + } + + public void setMaxPendingReplyCount(int count) { + maxPendingReplyCount = count; + } + + public Map<String, byte[]> getParameters() { + return params; + } + + public void setParameters(Map<String, byte[]> parameters) { + params = parameters; + } + + public List<BucketId> getBuckets() { + return buckets; + } + + public void setBuckets(List<BucketId> buckets) { + this.buckets = buckets; + } + + public boolean getVisitRemoves() { + return visitRemoves; + } + + public void setVisitRemoves(boolean visitRemoves) { + this.visitRemoves = visitRemoves; + } + + public String getFieldSet() { + return fieldSet; + } + + public void setFieldSet(String fieldSet) { + this.fieldSet = fieldSet; + } + + public boolean getVisitInconsistentBuckets() { + return visitInconsistentBuckets; + } + + public void setVisitInconsistentBuckets(boolean visitInconsistentBuckets) { + this.visitInconsistentBuckets = visitInconsistentBuckets; + } + + public void setFromTimestamp(long from) { + fromTime = from; + } + + public void setToTimestamp(long to) { + toTime = to; + } + + public long getFromTimestamp() { + return fromTime; + } + + public long getToTimestamp() { + return toTime; + } + + public void setVisitorDispatcherVersion(int version) { + this.version = version; + } + + public int getVisitorDispatcherVersion() { + return version; + } + + public void setVisitorOrdering(int ordering) { + this.ordering = ordering; + } + + public int getVisitorOrdering() { + return ordering; + } + + public void setMaxBucketsPerVisitor(int max) { + this.maxBucketsPerVisitor = max; + } + + public int getMaxBucketsPerVisitor() { + return maxBucketsPerVisitor; + } + + @Override + public DocumentReply createReply() { + return new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); + } + + @Override + public int getApproxSize() { + return buckets.size() * 8; + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_CREATEVISITOR; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder().append("CreateVisitorMessage("); + if (buckets.size() == 0) { + sb.append("No buckets"); + } else if (buckets.size() == 1) { + sb.append("Bucket ").append(buckets.iterator().next().toString()); + } else if (buckets.size() < 65536) { + sb.append(buckets.size()).append(" buckets:"); + Iterator<BucketId> it = buckets.iterator(); + for (int i = 0; it.hasNext() && i < 3; ++i) { + sb.append(' ').append(it.next().toString()); + } + if (it.hasNext()) { + sb.append(" ..."); + } + } else { + sb.append("All buckets"); + } + if (fromTime != 0 || toTime != 0) { + sb.append(", time ").append(fromTime).append('-').append(toTime); + } + sb.append(", selection '").append(docSelection).append('\''); + if (!libName.equals("DumpVisitor")) { + sb.append(", library ").append(libName); + } + if (visitRemoves) { + sb.append(", including removes"); + } + sb.append(", get fields: " + fieldSet); + if (visitInconsistentBuckets) { + sb.append(", visit inconsistent buckets"); + } + return sb.append(')').toString(); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorReply.java new file mode 100644 index 00000000000..f9350b79cc4 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorReply.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; +import com.yahoo.vdslib.VisitorStatistics; + +public class CreateVisitorReply extends DocumentReply { + + private BucketId lastBucket; + private VisitorStatistics statistics = new VisitorStatistics(); + + public CreateVisitorReply(int type) { + super(type); + lastBucket = new BucketId(Integer.MAX_VALUE); + } + + public void setLastBucket(BucketId lastBucket) { + this.lastBucket = lastBucket; + } + + public BucketId getLastBucket() { + return lastBucket; + } + + public void setVisitorStatistics(VisitorStatistics statistics) { + this.statistics = statistics; + } + + public VisitorStatistics getVisitorStatistics() { + return statistics; + } + + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DestroyVisitorMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DestroyVisitorMessage.java new file mode 100644 index 00000000000..ab2125600f7 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DestroyVisitorMessage.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +public class DestroyVisitorMessage extends DocumentMessage { + + private String instanceId; + + DestroyVisitorMessage() { + // must be deserialized into + } + + public DestroyVisitorMessage(String instanceId) { + this.instanceId = instanceId; + } + + public DestroyVisitorMessage(DestroyVisitorMessage cmd) { + instanceId = cmd.instanceId; + } + + public String getInstanceId() { + return instanceId; + } + + public void setInstanceId(String instanceId) { + this.instanceId = instanceId; + } + + @Override + public DocumentReply createReply() { + return new DocumentReply(DocumentProtocol.REPLY_DESTROYVISITOR); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_DESTROYVISITOR; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentAcceptedReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentAcceptedReply.java new file mode 100644 index 00000000000..f2bbcd281b6 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentAcceptedReply.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +/** + * Common base class for replies that indicate that a document was routed + * to some recipient. Does not imply that the reply contains no errors! + */ +public abstract class DocumentAcceptedReply extends DocumentReply { + protected DocumentAcceptedReply(int type) { + super(type); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentIgnoredReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentIgnoredReply.java new file mode 100644 index 00000000000..3993c95f31f --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentIgnoredReply.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +public class DocumentIgnoredReply extends DocumentReply { + public DocumentIgnoredReply() { + super(DocumentProtocol.REPLY_DOCUMENTIGNORED); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListEntry.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListEntry.java new file mode 100755 index 00000000000..8de0cfd204c --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListEntry.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.serialization.DocumentDeserializer;
+import com.yahoo.vespa.objects.Serializer;
+
+public class DocumentListEntry {
+
+ private Document doc;
+ private long timestamp;
+ private boolean removeEntry;
+
+ public DocumentListEntry(Document doc, long timestamp, boolean removeEntry) {
+ this.doc = doc;
+ this.timestamp = timestamp;
+ this.removeEntry = removeEntry;
+ }
+
+ public void serialize(Serializer buf) {
+ buf.putLong(null, timestamp);
+ doc.serialize(buf);
+ buf.putByte(null, (byte)(removeEntry ? 1 : 0));
+ }
+
+ public static int getApproxSize() {
+ return 60; // optimzation. approximation is sufficient
+ }
+
+ public DocumentListEntry(DocumentDeserializer buf) {
+ timestamp = buf.getLong(null);
+ doc = new Document(buf);
+ removeEntry = buf.getByte(null) > 0;
+ }
+
+ public Document getDocument() {
+ return doc;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public boolean isRemoveEntry() {
+ return removeEntry;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListMessage.java new file mode 100755 index 00000000000..448c2820ec3 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListMessage.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.document.BucketId;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DocumentListMessage extends VisitorMessage {
+
+ private BucketId bucket = new BucketId(16, 0);
+ private final List<DocumentListEntry> entries = new ArrayList<DocumentListEntry>();
+
+ public DocumentListMessage() {
+ // empty
+ }
+
+ public DocumentListMessage(DocumentListMessage cmd) {
+ bucket = cmd.bucket;
+ entries.addAll(cmd.entries);
+ }
+
+ public BucketId getBucketId() {
+ return bucket;
+ }
+
+ public void setBucketId(BucketId id) {
+ bucket = id;
+ }
+
+ public List<DocumentListEntry> getDocuments() {
+ return entries;
+ }
+
+ @Override
+ public DocumentReply createReply() {
+ return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTLIST);
+ }
+
+ @Override
+ public int getType() {
+ return DocumentProtocol.MESSAGE_DOCUMENTLIST;
+ }
+
+ @Override
+ public int getApproxSize() {
+ return DocumentListEntry.getApproxSize() * entries.size();
+ }
+
+ @Override
+ public String toString() {
+ return "DocumentListMessage(" + entries.toString() + ")";
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java new file mode 100755 index 00000000000..c4839c87f69 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.documentapi.messagebus.loadtypes.LoadType;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.Routable;
+import com.yahoo.text.Utf8String;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class DocumentMessage extends Message {
+
+ private DocumentProtocol.Priority priority = DocumentProtocol.Priority.NORMAL_3;
+ private LoadType loadType = LoadType.DEFAULT;
+
+ /**
+ * Constructs a new message with no content.
+ */
+ public DocumentMessage() {
+ // empty
+ }
+
+ /**
+ * Creates and returns a reply to this message.
+ *
+ * @return The created reply.
+ */
+ public abstract DocumentReply createReply();
+
+ @Override
+ public void swapState(Routable rhs) {
+ super.swapState(rhs);
+ if (rhs instanceof DocumentMessage) {
+ DocumentMessage msg = (DocumentMessage)rhs;
+
+ DocumentProtocol.Priority pri = this.priority;
+ this.priority = msg.priority;
+ msg.priority = pri;
+
+ LoadType lt = this.loadType;
+ this.loadType = msg.loadType;
+ msg.loadType = lt;
+ }
+ }
+
+ /**
+ * Returns the priority tag for this message. This is an optional tag added for VDS that is not interpreted by the
+ * document protocol.
+ *
+ * @return The priority.
+ */
+ public DocumentProtocol.Priority getPriority() { return priority; }
+
+ /**
+ * Sets the priority tag for this message.
+ *
+ * @param priority The priority to set.
+ */
+ public void setPriority(DocumentProtocol.Priority priority) {
+ this.priority = priority;
+ }
+
+ public LoadType getLoadType() {
+ return loadType;
+ }
+
+ public void setLoadType(LoadType loadType) {
+ if (loadType != null) {
+ this.loadType = loadType;
+ } else {
+ this.loadType = LoadType.DEFAULT;
+ }
+ }
+
+ @Override
+ public int getApproxSize() {
+ return 4 + 1; // type + priority
+ }
+
+ @Override
+ public Utf8String getProtocol() {
+ return DocumentProtocol.NAME;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java new file mode 100755 index 00000000000..e4988d3c8b8 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -0,0 +1,578 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.google.common.annotations.Beta; +import com.yahoo.collections.Tuple2; +import com.yahoo.component.Version; +import com.yahoo.component.VersionSpecification; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.metrics.DocumentProtocolMetricSet; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.RoutingContext; +import com.yahoo.messagebus.routing.RoutingNodeIterator; +import com.yahoo.messagebus.routing.RoutingPolicy; +import com.yahoo.text.Utf8String; + +import java.util.*; +import java.util.logging.Logger; + +/** + * Implements the message bus protocol that is used by all components of Vespa. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class DocumentProtocol implements Protocol { + + private static final Logger log = Logger.getLogger(DocumentProtocol.class.getName()); + private final DocumentProtocolMetricSet metrics = new DocumentProtocolMetricSet(); + private final RoutingPolicyRepository routingPolicyRepository = new RoutingPolicyRepository(metrics); + private final RoutableRepository routableRepository; + private final DocumentTypeManager docMan; + + /** + * The name of this protocol. + */ + public static final Utf8String NAME = new Utf8String("document"); + + /** + * All message types that are implemented by this protocol. + */ + public static final int DOCUMENT_MESSAGE = 100000; + public static final int MESSAGE_GETDOCUMENT = DOCUMENT_MESSAGE + 3; + public static final int MESSAGE_PUTDOCUMENT = DOCUMENT_MESSAGE + 4; + public static final int MESSAGE_REMOVEDOCUMENT = DOCUMENT_MESSAGE + 5; + public static final int MESSAGE_UPDATEDOCUMENT = DOCUMENT_MESSAGE + 6; + public static final int MESSAGE_CREATEVISITOR = DOCUMENT_MESSAGE + 7; + public static final int MESSAGE_DESTROYVISITOR = DOCUMENT_MESSAGE + 8; + public static final int MESSAGE_VISITORINFO = DOCUMENT_MESSAGE + 9; + public static final int MESSAGE_SEARCHRESULT = DOCUMENT_MESSAGE + 11; + public static final int MESSAGE_DOCUMENTSUMMARY = DOCUMENT_MESSAGE + 14; + public static final int MESSAGE_MAPVISITOR = DOCUMENT_MESSAGE + 15; + public static final int MESSAGE_GETBUCKETSTATE = DOCUMENT_MESSAGE + 18; + public static final int MESSAGE_STATBUCKET = DOCUMENT_MESSAGE + 19; + public static final int MESSAGE_GETBUCKETLIST = DOCUMENT_MESSAGE + 20; + public static final int MESSAGE_DOCUMENTLIST = DOCUMENT_MESSAGE + 21; + public static final int MESSAGE_EMPTYBUCKETS = DOCUMENT_MESSAGE + 23; + public static final int MESSAGE_REMOVELOCATION = DOCUMENT_MESSAGE + 24; + public static final int MESSAGE_QUERYRESULT = DOCUMENT_MESSAGE + 25; + public static final int MESSAGE_BATCHDOCUMENTUPDATE = DOCUMENT_MESSAGE + 26; + + /** + * All reply types that are implemented by this protocol. + */ + public static final int DOCUMENT_REPLY = 200000; + public static final int REPLY_GETDOCUMENT = DOCUMENT_REPLY + 3; + public static final int REPLY_PUTDOCUMENT = DOCUMENT_REPLY + 4; + public static final int REPLY_REMOVEDOCUMENT = DOCUMENT_REPLY + 5; + public static final int REPLY_UPDATEDOCUMENT = DOCUMENT_REPLY + 6; + public static final int REPLY_CREATEVISITOR = DOCUMENT_REPLY + 7; + public static final int REPLY_DESTROYVISITOR = DOCUMENT_REPLY + 8; + public static final int REPLY_VISITORINFO = DOCUMENT_REPLY + 9; + public static final int REPLY_SEARCHRESULT = DOCUMENT_REPLY + 11; + public static final int REPLY_DOCUMENTSUMMARY = DOCUMENT_REPLY + 14; + public static final int REPLY_MAPVISITOR = DOCUMENT_REPLY + 15; + public static final int REPLY_GETBUCKETSTATE = DOCUMENT_REPLY + 18; + public static final int REPLY_STATBUCKET = DOCUMENT_REPLY + 19; + public static final int REPLY_GETBUCKETLIST = DOCUMENT_REPLY + 20; + public static final int REPLY_DOCUMENTLIST = DOCUMENT_REPLY + 21; + public static final int REPLY_EMPTYBUCKETS = DOCUMENT_REPLY + 23; + public static final int REPLY_REMOVELOCATION = DOCUMENT_REPLY + 24; + public static final int REPLY_QUERYRESULT = DOCUMENT_REPLY + 25; + public static final int REPLY_BATCHDOCUMENTUPDATE = DOCUMENT_REPLY + 26; + public static final int REPLY_WRONGDISTRIBUTION = DOCUMENT_REPLY + 1000; + public static final int REPLY_DOCUMENTIGNORED = DOCUMENT_REPLY + 1001; + + /** + * Important note on adding new error codes to the Document protocol: + * + * Changes to this protocol must be reflected in both the Java and C++ versions + * of the code. Furthermore, ErrorCodesTest must be updated across both languages + * to include the new error code. Otherwise, cross-language correctness may no + * longer be guaranteed. + */ + + /** + * Used by policies to indicate an inappropriate message. + */ + public static final int ERROR_MESSAGE_IGNORED = ErrorCode.APP_FATAL_ERROR + 1; + /** + * Used for error policy when policy creation failed. + */ + public static final int ERROR_POLICY_FAILURE = ErrorCode.APP_FATAL_ERROR + 2; + /** + * Document in operation cannot be found. (VDS Get and Remove) + */ + public static final int ERROR_DOCUMENT_NOT_FOUND = ErrorCode.APP_FATAL_ERROR + 1001; + /** + * Operation cannot be performed because token already exist. (Create bucket, create visitor) + */ + public static final int ERROR_DOCUMENT_EXISTS = ErrorCode.APP_FATAL_ERROR + 1002; + /** + * Node have not implemented support for the given operation. + */ + public static final int ERROR_NOT_IMPLEMENTED = ErrorCode.APP_FATAL_ERROR + 1004; + /** + * Parameters given in request is illegal. + */ + public static final int ERROR_ILLEGAL_PARAMETERS = ErrorCode.APP_FATAL_ERROR + 1005; + /** + * Unknown request received. (New client requesting from old server) + */ + public static final int ERROR_UNKNOWN_COMMAND = ErrorCode.APP_FATAL_ERROR + 1007; + /** + * Request cannot be decoded. + */ + public static final int ERROR_UNPARSEABLE = ErrorCode.APP_FATAL_ERROR + 1008; + /** + * Not enough free space on disk to perform operation. + */ + public static final int ERROR_NO_SPACE = ErrorCode.APP_FATAL_ERROR + 1009; + /** + * Request was not handled correctly. + */ + public static final int ERROR_IGNORED = ErrorCode.APP_FATAL_ERROR + 1010; + /** + * We failed in some way we didn't expect to fail. + */ + public static final int ERROR_INTERNAL_FAILURE = ErrorCode.APP_FATAL_ERROR + 1011; + /** + * Node refuse to perform operation. (Illegally formed message?) + */ + public static final int ERROR_REJECTED = ErrorCode.APP_FATAL_ERROR + 1012; + /** + * Test and set condition (selection) failed. + */ + @Beta + public static final int ERROR_TEST_AND_SET_CONDITION_FAILED = ErrorCode.APP_FATAL_ERROR + 1013; + + /** + * Failed to process the given request. (Used by docproc) + */ + public static final int ERROR_PROCESSING_FAILURE = ErrorCode.APP_FATAL_ERROR + 2001; + /** Unique timestamp specified for new operation is already in use. */ + public static final int ERROR_TIMESTAMP_EXIST = ErrorCode.APP_FATAL_ERROR + 2002; + /** + * Node not ready to perform operation. (Initializing VDS nodes) + */ + public static final int ERROR_NODE_NOT_READY = ErrorCode.APP_TRANSIENT_ERROR + 1001; + /** + * Wrong node to talk to in current state. (VDS system state disagreement) + */ + public static final int ERROR_WRONG_DISTRIBUTION = ErrorCode.APP_TRANSIENT_ERROR + 1002; + /** + * Operation cut short and aborted. (Destroy visitor, node stopping) + */ + public static final int ERROR_ABORTED = ErrorCode.APP_TRANSIENT_ERROR + 1004; + /** + * Node too busy to process request (Typically full queues) + */ + public static final int ERROR_BUSY = ErrorCode.APP_TRANSIENT_ERROR + 1005; + /** + * Lost connection with the node we requested something from. + */ + public static final int ERROR_NOT_CONNECTED = ErrorCode.APP_TRANSIENT_ERROR + 1006; + /** + * We failed accessing the disk, which we think is a disk hardware problem. + */ + public static final int ERROR_DISK_FAILURE = ErrorCode.APP_TRANSIENT_ERROR + 1007; + /** + * We failed during an IO operation, we dont think is a specific disk hardware problem. + */ + public static final int ERROR_IO_FAILURE = ErrorCode.APP_TRANSIENT_ERROR + 1008; + /** + * Bucket given in operation not found due to bucket database + * inconsistencies between storage and distributor nodes. + */ + public static final int ERROR_BUCKET_NOT_FOUND = ErrorCode.APP_TRANSIENT_ERROR + 1009; + /** + * Bucket recently removed, such that operation cannot be performed. + * Differs from BUCKET_NOT_FOUND in that there is no db inconsistency. + */ + public static final int ERROR_BUCKET_DELETED = ErrorCode.APP_TRANSIENT_ERROR + 1012; + /** + * Storage node received a timestamp that is stale. Likely clock skew. + */ + public static final int ERROR_STALE_TIMESTAMP = ErrorCode.APP_TRANSIENT_ERROR + 1013; + + /** + * The given node have gotten a critical error and have suspended itself. + */ + public static final int ERROR_SUSPENDED = ErrorCode.APP_TRANSIENT_ERROR + 2001; + + /** + * <p>Define the different priorities allowed for document api messages. Most user traffic should be fit into the + * NORMAL categories. Traffic in the HIGH end will be usually be prioritized over important maintenance operations. + * Traffic in the LOW end will be prioritized after these operations.</p> + */ + public static enum Priority { + HIGHEST(0), + VERY_HIGH(1), + HIGH_1(2), + HIGH_2(3), + HIGH_3(4), + NORMAL_1(5), + NORMAL_2(6), + NORMAL_3(7), + NORMAL_4(8), + NORMAL_5(9), + NORMAL_6(10), + LOW_1(11), + LOW_2(12), + LOW_3(13), + VERY_LOW(14), + LOWEST(15); + + private final int val; + + private Priority(int val) { + this.val = val; + } + + public int getValue() { + return val; + } + } + + /** + * Get a priority enum instance by its value. + * + * @param val The value of the priority to return. + * @return The priority enum instance. + * @throws IllegalArgumentException If priority value is unknown. + */ + public static Priority getPriority(int val) { + for (Priority pri : Priority.values()) { + if (val == pri.val) { + return pri; + } + } + throw new IllegalArgumentException("Unknown priority: " + val); + } + + /** + * Get priority enum instance by its name. + * + * @param name Name of priority. + * @return Priority enum instance, given that <code>name</code> is valid. + * @throws IllegalArgumentException If priority name is unknown. + */ + public static Priority getPriorityByName(String name) { + return Priority.valueOf(name); + } + + public DocumentProtocol(DocumentTypeManager docMan) { + this(docMan, null, new LoadTypeSet()); + } + + public DocumentProtocol(DocumentTypeManager docMan, String configId) { + this(docMan, configId, new LoadTypeSet()); + } + + public DocumentProtocol(DocumentTypeManager docMan, String configId, LoadTypeSet set) { + // Prepare config string for routing policy factories. + String cfg = (configId == null ? "client" : configId); + if (docMan != null) { + this.docMan = docMan; + } else { + this.docMan = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(this.docMan, cfg); + } + routableRepository = new RoutableRepository(set); + + // When adding factories to this list, please KEEP THEM ORDERED alphabetically like they are now. + putRoutingPolicyFactory("AND", new RoutingPolicyFactories.AndPolicyFactory()); + putRoutingPolicyFactory("Content", new RoutingPolicyFactories.ContentPolicyFactory()); + putRoutingPolicyFactory("DocumentRouteSelector", new RoutingPolicyFactories.DocumentRouteSelectorPolicyFactory(cfg)); + putRoutingPolicyFactory("Extern", new RoutingPolicyFactories.ExternPolicyFactory()); + putRoutingPolicyFactory("LocalService", new RoutingPolicyFactories.LocalServicePolicyFactory()); + putRoutingPolicyFactory("MessageType", new RoutingPolicyFactories.MessageTypePolicyFactory(cfg)); + putRoutingPolicyFactory("RoundRobin", new RoutingPolicyFactories.RoundRobinPolicyFactory()); + putRoutingPolicyFactory("LoadBalancer", new RoutingPolicyFactories.LoadBalancerPolicyFactory()); + putRoutingPolicyFactory("SearchColumn", new RoutingPolicyFactories.SearchColumnPolicyFactory()); + putRoutingPolicyFactory("SearchRow", new RoutingPolicyFactories.SearchRowPolicyFactory()); + putRoutingPolicyFactory("Storage", new RoutingPolicyFactories.StoragePolicyFactory()); + putRoutingPolicyFactory("SubsetService", new RoutingPolicyFactories.SubsetServicePolicyFactory()); + + // Prepare version specifications to use when adding routable factories. + VersionSpecification version50 = new VersionSpecification(5, 0); + VersionSpecification version51 = new VersionSpecification(5, 1); + VersionSpecification version52 = new VersionSpecification(5, 115); + + List<VersionSpecification> from50 = Arrays.asList(version50, version51, version52); + List<VersionSpecification> from51 = Arrays.asList(version51, version52); + List<VersionSpecification> from52 = Arrays.asList(version52); + + // 5.0 serialization (keep alphabetized please) + putRoutableFactory(MESSAGE_BATCHDOCUMENTUPDATE, new RoutableFactories50.BatchDocumentUpdateMessageFactory(), from50); + putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories50.CreateVisitorMessageFactory(), from50); + putRoutableFactory(MESSAGE_DESTROYVISITOR, new RoutableFactories50.DestroyVisitorMessageFactory(), from50); + putRoutableFactory(MESSAGE_DOCUMENTLIST, new RoutableFactories50.DocumentListMessageFactory(), from50); + putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, new RoutableFactories50.DocumentSummaryMessageFactory(), from50); + putRoutableFactory(MESSAGE_EMPTYBUCKETS, new RoutableFactories50.EmptyBucketsMessageFactory(), from50); + putRoutableFactory(MESSAGE_GETBUCKETLIST, new RoutableFactories50.GetBucketListMessageFactory(), from50); + putRoutableFactory(MESSAGE_GETBUCKETSTATE, new RoutableFactories50.GetBucketStateMessageFactory(), from50); + putRoutableFactory(MESSAGE_GETDOCUMENT, new RoutableFactories50.GetDocumentMessageFactory(), from50); + putRoutableFactory(MESSAGE_MAPVISITOR, new RoutableFactories50.MapVisitorMessageFactory(), from50); + putRoutableFactory(MESSAGE_PUTDOCUMENT, new RoutableFactories50.PutDocumentMessageFactory(), from50); + putRoutableFactory(MESSAGE_QUERYRESULT, new RoutableFactories50.QueryResultMessageFactory(), from50); + putRoutableFactory(MESSAGE_REMOVEDOCUMENT, new RoutableFactories50.RemoveDocumentMessageFactory(), from50); + putRoutableFactory(MESSAGE_REMOVELOCATION, new RoutableFactories50.RemoveLocationMessageFactory(), from50); + putRoutableFactory(MESSAGE_SEARCHRESULT, new RoutableFactories50.SearchResultMessageFactory(), from50); + putRoutableFactory(MESSAGE_STATBUCKET, new RoutableFactories50.StatBucketMessageFactory(), from50); + putRoutableFactory(MESSAGE_UPDATEDOCUMENT, new RoutableFactories50.UpdateDocumentMessageFactory(), from50); + putRoutableFactory(MESSAGE_VISITORINFO, new RoutableFactories50.VisitorInfoMessageFactory(), from50); + putRoutableFactory(REPLY_BATCHDOCUMENTUPDATE, new RoutableFactories50.BatchDocumentUpdateReplyFactory(), from50); + putRoutableFactory(REPLY_CREATEVISITOR, new RoutableFactories50.CreateVisitorReplyFactory(), from50); + putRoutableFactory(REPLY_DESTROYVISITOR, new RoutableFactories50.DestroyVisitorReplyFactory(), from50); + putRoutableFactory(REPLY_DOCUMENTLIST, new RoutableFactories50.DocumentListReplyFactory(), from50); + putRoutableFactory(REPLY_DOCUMENTSUMMARY, new RoutableFactories50.DocumentSummaryReplyFactory(), from50); + putRoutableFactory(REPLY_EMPTYBUCKETS, new RoutableFactories50.EmptyBucketsReplyFactory(), from50); + putRoutableFactory(REPLY_GETBUCKETLIST, new RoutableFactories50.GetBucketListReplyFactory(), from50); + putRoutableFactory(REPLY_GETBUCKETSTATE, new RoutableFactories50.GetBucketStateReplyFactory(), from50); + putRoutableFactory(REPLY_GETDOCUMENT, new RoutableFactories50.GetDocumentReplyFactory(), from50); + putRoutableFactory(REPLY_MAPVISITOR, new RoutableFactories50.MapVisitorReplyFactory(), from50); + putRoutableFactory(REPLY_PUTDOCUMENT, new RoutableFactories50.PutDocumentReplyFactory(), from50); + putRoutableFactory(REPLY_QUERYRESULT, new RoutableFactories50.QueryResultReplyFactory(), from50); + putRoutableFactory(REPLY_REMOVEDOCUMENT, new RoutableFactories50.RemoveDocumentReplyFactory(), from50); + putRoutableFactory(REPLY_REMOVELOCATION, new RoutableFactories50.RemoveLocationReplyFactory(), from50); + putRoutableFactory(REPLY_SEARCHRESULT, new RoutableFactories50.SearchResultReplyFactory(), from50); + putRoutableFactory(REPLY_STATBUCKET, new RoutableFactories50.StatBucketReplyFactory(), from50); + putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories50.UpdateDocumentReplyFactory(), from50); + putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories50.UpdateDocumentReplyFactory(), from50); + putRoutableFactory(REPLY_VISITORINFO, new RoutableFactories50.VisitorInfoReplyFactory(), from50); + putRoutableFactory(REPLY_WRONGDISTRIBUTION, new RoutableFactories50.WrongDistributionReplyFactory(), from50); + + // 5.1 serialization + putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories51.CreateVisitorMessageFactory(), from51); + putRoutableFactory(MESSAGE_GETDOCUMENT, new RoutableFactories51.GetDocumentMessageFactory(), from51); + putRoutableFactory(REPLY_DOCUMENTIGNORED, new RoutableFactories51.DocumentIgnoredReplyFactory(), from51); + + // 5.2 serialization + putRoutableFactory(MESSAGE_PUTDOCUMENT, new RoutableFactories52.PutDocumentMessageFactory(), from52); + putRoutableFactory(MESSAGE_UPDATEDOCUMENT, new RoutableFactories52.UpdateDocumentMessageFactory(), from52); + putRoutableFactory(MESSAGE_REMOVEDOCUMENT, new RoutableFactories52.RemoveDocumentMessageFactory(), from52); + } + + /** + * Adds a new routable factory to this protocol. This method is thread-safe, and may be invoked on a protocol object + * that is already in use by a message bus instance. Notice that the name you supply for a factory is the + * case-sensitive name that will be referenced by routes. + * + * @param name The name of the factory to add. + * @param factory The factory to add. + * @return This, to allow chaining. + */ + public DocumentProtocol putRoutingPolicyFactory(String name, RoutingPolicyFactory factory) { + routingPolicyRepository.putFactory(name, factory); + return this; + } + + /** + * Adds a new routable factory to this protocol. This method is thread-safe, and may be invoked on a protocol object + * that is already in use by a message bus instance. Notice that you must explicitly register a factory for each + * supported version. You can always bypass this by passing a default version specification object to this function, + * because that object will match any version. + * + * @param type The routable type to assign a factory to. + * @param factory The factory to add. + * @param version The version for which this factory can be used. + * @return This, to allow chaining. + */ + public DocumentProtocol putRoutableFactory(int type, RoutableFactory factory, VersionSpecification version) { + routableRepository.putFactory(version, type, factory); + return this; + } + + /** + * Convenience method to call {@link #putRoutableFactory(int, RoutableFactory, com.yahoo.component.VersionSpecification)} + * for multiple version specifications. + * + * @param type The routable type to assign a factory to. + * @param factory The factory to add. + * @param versions The versions for which this factory can be used. + * @return This, to allow chaining. + */ + public DocumentProtocol putRoutableFactory(int type, RoutableFactory factory, List<VersionSpecification> versions) { + for (VersionSpecification version : versions) { + putRoutableFactory(type, factory, version); + } + return this; + } + + /** + * Returns a string representation of the given error code. + * + * @param code The code whose string symbol to return. + * @return The error string. + */ + public static String getErrorName(int code) { + switch (code) { + case ERROR_MESSAGE_IGNORED: + return "MESSAGE_IGNORED"; + case ERROR_DOCUMENT_NOT_FOUND: + return "DOCUMENT_NOT_FOUND"; + case ERROR_DOCUMENT_EXISTS: + return "DOCUMENT_EXISTS"; + case ERROR_BUCKET_NOT_FOUND: + return "BUCKET_NOT_FOUND"; + case ERROR_BUCKET_DELETED: + return "BUCKET_DELETED"; + case ERROR_NOT_IMPLEMENTED: + return "NOT_IMPLEMENTED"; + case ERROR_ILLEGAL_PARAMETERS: + return "ILLEGAL_PARAMETERS"; + case ERROR_IGNORED: + return "IGNORED"; + case ERROR_UNKNOWN_COMMAND: + return "UNKNOWN_COMMAND"; + case ERROR_UNPARSEABLE: + return "UNPARSEABLE"; + case ERROR_NO_SPACE: + return "NO_SPACE"; + case ERROR_INTERNAL_FAILURE: + return "INTERNAL_FAILURE"; + case ERROR_PROCESSING_FAILURE: + return "PROCESSING_FAILURE"; + case ERROR_TIMESTAMP_EXIST: + return "TIMESTAMP_EXIST"; + case ERROR_NODE_NOT_READY: + return "NODE_NOT_READY"; + case ERROR_WRONG_DISTRIBUTION: + return "WRONG_DISTRIBUTION"; + case ERROR_REJECTED: + return "REJECTED"; + case ERROR_ABORTED: + return "ABORTED"; + case ERROR_BUSY: + return "BUSY"; + case ERROR_NOT_CONNECTED: + return "NOT_CONNECTED"; + case ERROR_DISK_FAILURE: + return "DISK_FAILURE"; + case ERROR_IO_FAILURE: + return "IO_FAILURE"; + case ERROR_SUSPENDED: + return "SUSPENDED"; + case ERROR_TEST_AND_SET_CONDITION_FAILED: + return "TEST_AND_SET_CONDITION_FAILED"; + default: + return ErrorCode.getName(code); + } + } + + /** + * This is a convenient entry to the {@link #merge(RoutingContext,Set)} method by way of a routing context object. + * The replies of all child contexts are merged and stored in the context. + * + * @param ctx The context whose children to merge. + */ + public static void merge(RoutingContext ctx) { + merge(ctx, new HashSet<Integer>(0)); + } + + /** + * This method implements the common way to merge document replies for whatever routing policy. In case of an error + * in any of the replies, it will prepare an EmptyReply() and add all errors to it. If there are no errors, this + * method will use the first reply in the list and transfer whatever feed answers might exist in the replies to it. + * + * @param ctx The context whose children to merge. + * @param mask The indexes of the children to skip. + */ + public static void merge(RoutingContext ctx, Set<Integer> mask) { + List<Reply> replies = new LinkedList<>(); + for (RoutingNodeIterator it = ctx.getChildIterator(); + it.isValid(); it.next()) { + Reply ref = it.getReplyRef(); + replies.add(ref); + } + Tuple2<Integer, Reply> tuple = merge(replies, mask); + if (tuple.first != null) { + ctx.getChildIterator().skip(tuple.first).removeReply(); + } + ctx.setReply(tuple.second); + } + + private static Tuple2<Integer, Reply> merge(List<Reply> replies, Set<Integer> mask) { + ReplyMerger rm = new ReplyMerger(); + for (int i = 0; i < replies.size(); ++i) { + if (mask.contains(i)) { + continue; + } + rm.merge(i, replies.get(i)); + } + return rm.mergedReply(); + } + + /** + * This method implements the common way to merge document replies for whatever routing policy. In case of an error + * in any of the replies, it will prepare an EmptyReply() and add all errors to it. If there are no errors, this + * method will use the first reply in the list and transfer whatever feed answers might exist in the replies to it. + * + * + * @param replies The replies to merge. + * @return The merged Reply. + */ + public static Reply merge(List<Reply> replies) { + return merge(replies, new HashSet<Integer>(0)).second; + } + + /** + * Returns true if the given reply has at least one error, and all errors are of the given type. + * + * @param reply The reply to check for error. + * @param errCode The error code to check for. + * @return Whether or not the reply has only the given error code. + */ + public static boolean hasOnlyErrorsOfType(Reply reply, int errCode) { + if (!reply.hasErrors()) { + return false; + } + for (int i = 0; i < reply.getNumErrors(); ++i) { + if (reply.getError(i).getCode() != errCode) { + return false; + } + } + return true; + } + + public String getName() { + return NAME.toString(); + } + + public RoutingPolicy createPolicy(String name, String param) { + return routingPolicyRepository.createPolicy(name, param); + } + + public byte[] encode(Version version, Routable routable) { + return routableRepository.encode(version, routable); + } + + public Routable decode(Version version, byte[] data) { + try { + return routableRepository.decode(docMan, version, data); + } catch (RuntimeException e) { + e.printStackTrace(); + log.warning(e.getMessage()); + return null; + } + } + + /** + * Returns a list of routable types that support the given version. + * + * @param version The version to return types for. + * @return The list of supported types. + */ + public List<Integer> getRoutableTypes(Version version) { + return routableRepository.getRoutableTypes(version); + } + + public MetricSet getMetrics() { + return metrics; + } + + final public DocumentTypeManager getDocumentTypeManager() { return docMan; } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolRoutingPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolRoutingPolicy.java new file mode 100644 index 00000000000..0f4bc33a944 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolRoutingPolicy.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.RoutingPolicy; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * @author thomasg + */ +public interface DocumentProtocolRoutingPolicy extends RoutingPolicy { + public MetricSet getMetrics(); +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentReply.java new file mode 100755 index 00000000000..126d85c5703 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentReply.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.messagebus.Reply;
+import com.yahoo.text.Utf8String;
+
+/**
+ * This class implements a generic document protocol reply that can be reused by document messages that require no
+ * special reply implementation while still allowing applications to distinguish between types.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DocumentReply extends Reply {
+
+ private final int type;
+ private DocumentProtocol.Priority priority = DocumentProtocol.Priority.NORMAL_3;
+
+ /**
+ * Constructs a new reply of given type.
+ *
+ * @param type The type code to assign to this.
+ */
+ public DocumentReply(int type) {
+ this.type = type;
+ }
+
+ /**
+ * Returns the priority tag for this message.
+ * @return The priority.
+ */
+ public DocumentProtocol.Priority getPriority() {
+ return priority;
+ }
+
+ /**
+ * Sets the priority tag for this message.
+ *
+ * @param priority The priority to set.
+ */
+ public void setPriority(DocumentProtocol.Priority priority) {
+ this.priority = priority;
+ }
+
+ @Override
+ public Utf8String getProtocol() {
+ return DocumentProtocol.NAME;
+ }
+
+ @Override
+ public final int getType() {
+ return type;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java new file mode 100755 index 00000000000..276732a494a --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java @@ -0,0 +1,179 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.select.DocumentSelector; +import com.yahoo.document.select.Result; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RoutingContext; + +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Logger; + +/** + * This policy is responsible for selecting among the given recipient routes according to the configured document + * selection properties. To facilitate this the "routing" plugin in the vespa model builds a mapping from the route + * names to a document selector and a feed name of every search cluster. This can very well be extended to include + * storage at a later time. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class DocumentRouteSelectorPolicy + implements DocumentProtocolRoutingPolicy, ConfigSubscriber.SingleSubscriber<DocumentrouteselectorpolicyConfig> { + + private static Logger log = Logger.getLogger(DocumentRouteSelectorPolicy.class.getName()); + private Map<String, DocumentSelector> config; + private String error = "Not configured."; + private ConfigSubscriber subscriber; + + /** + * This policy is constructed with a configuration identifier that can be subscribed to for the document selector + * config. If the string is either null or empty it will default to the proper one. + * + * @param configId The configuration identifier to subscribe to. + */ + public DocumentRouteSelectorPolicy(String configId) { + subscriber = new ConfigSubscriber(); + subscriber.subscribe(this, DocumentrouteselectorpolicyConfig.class, configId); + } + + /** + * This is a safety mechanism to allow the constructor to fail and signal that it can not be used. + * + * @return The error string, or null if no error. + */ + public synchronized String getError() { + return error; + } + + /** + * This method is called when configuration arrives from the config server. The received config object is traversed + * and a local map is constructed and swapped with the current {@link #config} map. + * + * @param cfg The configuration object given by subscription. + */ + @Override + public void configure(DocumentrouteselectorpolicyConfig cfg) { + String error = null; + Map<String, DocumentSelector> config = new HashMap<>(); + for (int i = 0; i < cfg.route().size(); i++) { + DocumentrouteselectorpolicyConfig.Route route = cfg.route(i); + if (route.selector().isEmpty()) { + continue; + } + DocumentSelector selector; + try { + selector = new DocumentSelector(route.selector()); + log.log(LogLevel.CONFIG, "Selector for route '" + route.name() + "' is '" + selector + "'."); + } catch (com.yahoo.document.select.parser.ParseException e) { + error = "Error parsing selector '" + route.selector() + "' for route '" + route.name() + "; " + + e.getMessage(); + break; + } + config.put(route.name(), selector); + } + synchronized (this) { + this.config = config; + this.error = error; + } + } + + @Override + public void select(RoutingContext context) { + // Require that recipients have been configured. + if (context.getNumRecipients() == 0) { + context.setError(DocumentProtocol.ERROR_POLICY_FAILURE, + "No recipients configured."); + return; + } + + // Invoke private select method for each candidate recipient. + synchronized (this) { + if (error != null) { + context.setError(DocumentProtocol.ERROR_POLICY_FAILURE, error); + return; + } + for (int i = 0; i < context.getNumRecipients(); ++i) { + Route recipient = context.getRecipient(i); + String routeName = recipient.toString(); + if (select(context, routeName)) { + Route route = context.getMessageBus().getRoutingTable(DocumentProtocol.NAME).getRoute(routeName); + context.addChild(route != null ? route : recipient); + } + } + } + context.setSelectOnRetry(false); + + // Notify that no children were selected, this is to differentiate this from the NO_RECIPIENTS_FOR_ROUTE error + // that message bus will generate if there are no recipients and no reply. + if (context.getNumChildren() == 0) { + context.setReply(new DocumentIgnoredReply()); + } + } + + /** + * This method runs the selector associated with the given location on the content of the message. If the selector + * validates the location, this method returns true. + * + * @param context The routing context that contains the necessary data. + * @param routeName The candidate route whose selector to run. + * @return Whether or not to send to the given recipient. + */ + private boolean select(RoutingContext context, String routeName) { + if (config == null) { + return true; + } + DocumentSelector selector = config.get(routeName); + if (selector == null) { + return true; + } + + // Select based on message content. + Message msg = context.getMessage(); + switch (msg.getType()) { + + case DocumentProtocol.MESSAGE_PUTDOCUMENT: + return selector.accepts(((PutDocumentMessage)msg).getDocumentPut()) == Result.TRUE; + + case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: + return selector.accepts(((UpdateDocumentMessage)msg).getDocumentUpdate()) != Result.FALSE; + + + case DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE: + BatchDocumentUpdateMessage bdu = (BatchDocumentUpdateMessage)msg; + for (int i = 0; i < bdu.getUpdates().size(); i++) { + if (selector.accepts(bdu.getUpdates().get(i)) == Result.FALSE) { + return false; + } + } + return true; + + default: + return true; + } + } + + @Override + public void merge(RoutingContext context) { + DocumentProtocol.merge(context); + } + + @Override + public void destroy() { + if (subscriber != null) { + subscriber.close(); + } + } + + @Override + public MetricSet getMetrics() { + return null; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentState.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentState.java new file mode 100644 index 00000000000..3735296eb4f --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentState.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.DocumentId; +import com.yahoo.document.GlobalId; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.objects.Deserializer; +import com.yahoo.vespa.objects.Serializer; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> +*/ +public class DocumentState implements Comparable { + private DocumentId docId; + private GlobalId gid; + private long timestamp; + private boolean removeEntry; + + public DocumentState(DocumentId docId, long timestamp, boolean removeEntry) { + this.docId = docId; + this.gid = new GlobalId(docId.getGlobalId()); + this.timestamp = timestamp; + this.removeEntry = removeEntry; + } + + public DocumentState(GlobalId gid, long timestamp, boolean removeEntry) { + this.gid = gid; + this.timestamp = timestamp; + this.removeEntry = removeEntry; + } + + public DocumentState(Deserializer buf) { + byte hasDocId = buf.getByte(null); + if (hasDocId == (byte) 1) { + docId = new DocumentId(buf); + } + gid = new GlobalId(buf); + timestamp = buf.getLong(null); + removeEntry = buf.getByte(null)>0; + } + + public DocumentId getDocId() { + return docId; + } + + public GlobalId getGid() { + return gid; + } + + public long getTimestamp() { + return timestamp; + } + + public boolean isRemoveEntry() { + return removeEntry; + } + + public void serialize(Serializer buf) { + if (docId != null) { + buf.putByte(null, (byte) 1); + docId.serialize(buf); + } else { + buf.putByte(null, (byte) 0); + } + gid.serialize(buf); + buf.putLong(null, timestamp); + buf.putByte(null, (byte)(removeEntry ? 1 : 0)); + } + + public int getSerializedSize() { + int size = 0; + if (docId != null) { + size += Utf8.byteCount(docId.toString()) + 1; + } + size += GlobalId.LENGTH; + size += 8; + size += 1; + return size; + } + + public int compareTo(Object o) { + DocumentState state = (DocumentState) o; + int comp = gid.compareTo(state.gid); + if (comp == 0) { + if (docId != null) { + if (state.docId != null) { + return docId.toString().compareTo(state.docId.toString()); + } else { + return 1; + } + } else if (state.docId != null){ + return -1; + } + } + return comp; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DocumentState)) return false; + + DocumentState that = (DocumentState) o; + + if (removeEntry != that.removeEntry) return false; + if (timestamp != that.timestamp) return false; + if (docId != null ? !docId.equals(that.docId) : that.docId != null) return false; + return gid.equals(that.gid); + } + + @Override + public int hashCode() { + int result; + result = (docId != null ? docId.hashCode() : 0); + result = 31 * result + gid.hashCode(); + result = 31 * result + (int) (timestamp ^ (timestamp >>> 32)); + result = 31 * result + (removeEntry ? 1 : 0); + return result; + } + + @Override + public String toString() { + return "DocumentState{" + + "docId=" + docId + + ", gid=" + gid + + ", timestamp=" + timestamp + + ", removeEntry=" + removeEntry + + '}'; + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentSummaryMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentSummaryMessage.java new file mode 100644 index 00000000000..f063d7885e4 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentSummaryMessage.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.vdslib.DocumentSummary; + +public class DocumentSummaryMessage extends VisitorMessage { + + private DocumentSummary documentSummary = null; + + public void setDocumentSummary(DocumentSummary summary) { + documentSummary = summary; + } + + public DocumentSummary getResult() { + return documentSummary; + } + + @Override + public DocumentReply createReply() { + return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTSUMMARY); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_DOCUMENTSUMMARY; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/EmptyBucketsMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/EmptyBucketsMessage.java new file mode 100644 index 00000000000..43514a156bc --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/EmptyBucketsMessage.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author banino + */ +public class EmptyBucketsMessage extends VisitorMessage { + + private final List<BucketId> bids = new ArrayList<BucketId>(); + + EmptyBucketsMessage() { + // must be deserialized into + } + + public EmptyBucketsMessage(List<BucketId> bids) { + this.bids.addAll(bids); + } + + public List<BucketId> getBucketIds() { + return bids; + } + + public void setBucketIds(List<BucketId> bids) { + this.bids.clear(); + this.bids.addAll(bids); + } + + @Override + public DocumentReply createReply() { + return new VisitorReply(DocumentProtocol.REPLY_EMPTYBUCKETS); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_EMPTYBUCKETS; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ErrorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ErrorPolicy.java new file mode 100755 index 00000000000..0b7310ad2fb --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ErrorPolicy.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.messagebus.EmptyReply;
+import com.yahoo.messagebus.metrics.MetricSet;
+import com.yahoo.messagebus.routing.RoutingContext;
+
+/**
+ * This policy assigns an error supplied at constructor time to the routing context when {@link #select(RoutingContext)}
+ * is invoked. This is useful for returning error states to the client instead of those auto-generated by mbus when a
+ * routing policy can not be created.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ErrorPolicy implements DocumentProtocolRoutingPolicy {
+
+ private final String msg;
+
+ /**
+ * Creates a new policy that will assign an {@link EmptyReply} with the given error to all routing contexts that
+ * invoke {@link #select(RoutingContext)}.
+ *
+ * @param msg The message of the error to assign.
+ */
+ public ErrorPolicy(String msg) {
+ this.msg = msg;
+ }
+
+ public void select(RoutingContext ctx) {
+ ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, msg);
+ }
+
+ public void merge(RoutingContext ctx) {
+ throw new AssertionError("Routing should not pass terminated selection.");
+ }
+
+ public void destroy() {
+ }
+
+
+ public MetricSet getMetrics() {
+ return null;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java new file mode 100755 index 00000000000..a843102f466 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java @@ -0,0 +1,147 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.jrt.Transport;
+import com.yahoo.jrt.slobrok.api.Mirror;
+import com.yahoo.jrt.slobrok.api.SlobrokList;
+import com.yahoo.messagebus.ErrorCode;
+import com.yahoo.messagebus.metrics.MetricSet;
+import com.yahoo.messagebus.routing.Hop;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.messagebus.routing.RoutingContext;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * This policy implements the necessary logic to communicate with an external Vespa application and resolve its list of
+ * recipients using that other application's slobrok servers.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ExternPolicy implements DocumentProtocolRoutingPolicy {
+
+ private Supervisor orb = null;
+ private Mirror mirror = null;
+ private String pattern = null;
+ private String session = null;
+ private final String error;
+ private int offset = 0;
+ private int generation = 0;
+ private final List<Hop> recipients = new ArrayList<>();
+
+ /**
+ * Constructs a new instance of this policy. The argument given is the connection spec to the slobrok to use for
+ * resolving recipients, as well as the pattern to use when querying. This constructor does _not_ wait for the
+ * mirror to become ready.
+ *
+ * @param arg The slobrok connection spec.
+ */
+ public ExternPolicy(String arg) {
+ if (arg == null || arg.length() == 0) {
+ error = "Expected parameter, got empty string.";
+ return;
+ }
+ String[] args = arg.split(";", 2);
+ if (args.length != 2 || args[0].length() == 0 || args[1].length() == 0) {
+ error = "Expected parameter on the form '<spec>;<pattern>', got '" + arg + "'.";
+ return;
+ }
+ int pos = args[1].lastIndexOf('/');
+ if (pos < 0) {
+ error = "Expected pattern on the form '<service>/<session>', got '" + args[1] + "'.";
+ return;
+ }
+ SlobrokList slobroks = new SlobrokList();
+ slobroks.setup(args[0].split(","));
+ pattern = args[1];
+ session = pattern.substring(pos);
+ orb = new Supervisor(new Transport());
+ mirror = new Mirror(orb, slobroks);
+ error = null;
+ }
+
+ /**
+ * This is a safety mechanism to allow the constructor to fail and signal that it can not be used.
+ *
+ * @return The error string, or null if no error.
+ */
+ public String getError() {
+ return error;
+ }
+
+ /**
+ * Returns the slobrok mirror used by this policy to resolve external recipients.
+ *
+ * @return The external mirror.
+ */
+ public Mirror getMirror() {
+ return mirror;
+ }
+
+ /**
+ * Returns the appropriate recipient hop. This method provides synchronized access to the internal mirror.
+ *
+ * @return The recipient hop to use.
+ */
+ private synchronized Hop getRecipient() {
+ update();
+ if (recipients.isEmpty()) {
+ return null;
+ }
+ int offset = ++this.offset & Integer.MAX_VALUE; // mask signed bit because of modulo
+ return new Hop(recipients.get(offset % recipients.size()));
+ }
+
+ /**
+ * Updates the list of matching recipients by querying the extern slobrok.
+ */
+ private void update() {
+ int upd = mirror.updates();
+ if (generation != upd) {
+ generation = upd;
+ recipients.clear();
+ Mirror.Entry[] arr = mirror.lookup(pattern);
+ for (Mirror.Entry entry : arr) {
+ recipients.add(Hop.parse(entry.getSpec() + session));
+ }
+ }
+ }
+
+ @Override
+ public void finalize() throws Throwable {
+ super.finalize();
+ mirror.shutdown();
+ orb.transport().shutdown().join();
+ }
+
+ public void select(RoutingContext ctx) {
+ if (error != null) {
+ ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, error);
+ } else if (mirror.ready()) {
+ Hop hop = getRecipient();
+ if (hop != null) {
+ Route route = new Route(ctx.getRoute());
+ route.setHop(0, hop);
+ ctx.addChild(route);
+ } else {
+ ctx.setError(ErrorCode.NO_ADDRESS_FOR_SERVICE,
+ "Could not resolve any recipients from '" + pattern + "'.");
+ }
+ } else {
+ ctx.setError(ErrorCode.APP_TRANSIENT_ERROR, "Extern slobrok not ready.");
+ }
+ }
+
+ public void merge(RoutingContext ctx) {
+ DocumentProtocol.merge(ctx);
+ }
+
+ public void destroy() {
+ }
+
+ public MetricSet getMetrics() {
+ return null;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternalSlobrokPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternalSlobrokPolicy.java new file mode 100644 index 00000000000..d60c8cb7b33 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternalSlobrokPolicy.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.jrt.slobrok.api.IMirror; +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.jrt.slobrok.api.SlobrokList; +import com.yahoo.messagebus.routing.RoutingContext; +import com.yahoo.cloud.config.SlobroksConfig; + +import java.util.Map; + +/** + * Abstract class for policies that allow you to specify which slobrok to use for the + * routing. + */ +public abstract class ExternalSlobrokPolicy extends AsyncInitializationPolicy implements ConfigSubscriber.SingleSubscriber<SlobroksConfig> { + String error; + Supervisor orb = null; + Mirror mirror = null; + SlobrokList slobroks = null; + boolean firstTry = true; + private ConfigSubscriber subscriber; + String[] configSources = null; + String slobrokConfigId = "admin/slobrok.0"; + + + public ExternalSlobrokPolicy(Map<String, String> param) { + super(param); + + String conf = param.get("config"); + if (conf != null) { + configSources = conf.split(","); + } + + String slbrk = param.get("slobroks"); + if (slbrk != null) { + slobroks = new SlobrokList(); + slobroks.setup(slbrk.split(",")); + } + + if (slobroks != null || configSources != null) { + needAsynchronousInitialization(); + } + } + + @Override + public void init() { + if (slobroks != null) { + orb = new Supervisor(new Transport()); + mirror = new Mirror(orb, slobroks); + } + + if (configSources != null) { + if (mirror == null) { + orb = new Supervisor(new Transport()); + subscriber = subscribe(slobrokConfigId, new ConfigSourceSet(configSources)); + } + } + } + + private ConfigSubscriber subscribe(String configId, final ConfigSourceSet configSourceSet) { + ConfigSubscriber subscriber = new ConfigSubscriber(configSourceSet); + subscriber.subscribe(this, SlobroksConfig.class, configId); + return subscriber; + } + + public IMirror getMirror() { + return mirror; + } + + public Mirror.Entry[] lookup(RoutingContext context, String pattern) { + IMirror mirror1 = (mirror != null ? mirror : context.getMirror()); + + Mirror.Entry[] arr = mirror1.lookup(pattern); + + if ((arr.length == 0) && firstTry) { + synchronized(this) { + try { + int count = 0; + while (arr.length == 0 && count < 100) { + Thread.sleep(50); + arr = mirror1.lookup(pattern); + count++; + } + } catch (InterruptedException e) { + } + + } + } + + firstTry = false; + return arr; + } + + @Override + public synchronized void configure(SlobroksConfig config) { + String[] slist = new String[config.slobrok().size()]; + + for(int i = 0; i < config.slobrok().size(); i++) { + slist[i] = config.slobrok(i).connectionspec(); + } + if (slobroks == null) { + slobroks = new SlobrokList(); + } + slobroks.setup(slist); + if (mirror == null) { + mirror = new Mirror(orb, slobroks); + } + + } + + @Override + public void destroy() { + if (subscriber!=null) subscriber.close(); + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListMessage.java new file mode 100755 index 00000000000..d38aa2e94f2 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListMessage.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.document.BucketId;
+
+public class GetBucketListMessage extends DocumentMessage {
+
+ private BucketId bucketId;
+
+ GetBucketListMessage() {
+ // must be deserialized into
+ }
+
+ public GetBucketListMessage(BucketId bucketId) {
+ this.bucketId = bucketId;
+ }
+
+ public BucketId getBucketId() {
+ return bucketId;
+ }
+
+ void setBucketId(BucketId id) {
+ bucketId = id;
+ }
+
+ @Override
+ public DocumentReply createReply() {
+ return new StatBucketReply();
+ }
+
+ @Override
+ public boolean hasSequenceId() {
+ return true;
+ }
+
+ @Override
+ public long getSequenceId() {
+ return bucketId.getRawId();
+ }
+
+ @Override
+ public int getApproxSize() {
+ return super.getApproxSize() + 8;
+ }
+
+ @Override
+ public int getType() {
+ return DocumentProtocol.MESSAGE_GETBUCKETLIST;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListReply.java new file mode 100755 index 00000000000..07013507d91 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListReply.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.document.BucketId;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class GetBucketListReply extends DocumentReply {
+
+ public static class BucketInfo {
+ BucketId bucket;
+ String bucketInformation;
+
+ BucketInfo() {
+ // must be deserialized into
+ }
+
+ public BucketInfo(BucketId bucket, String bucketInformation) {
+ this.bucket = bucket;
+ this.bucketInformation = bucketInformation;
+ }
+
+ public BucketId getBucketId() {
+ return bucket;
+ }
+
+ public String getBucketInformation() {
+ return bucketInformation;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof BucketInfo)) {
+ return false;
+ }
+ BucketInfo rhs = (BucketInfo)obj;
+ if (bucket == null) {
+ if (rhs.bucket != null) {
+ return false;
+ }
+ } else if (!bucket.equals(rhs.bucket)) {
+ return false;
+ }
+ if (bucketInformation == null) {
+ if (rhs.bucketInformation != null) {
+ return false;
+ }
+ } else if (!bucketInformation.equals(rhs.bucketInformation)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("BucketInfo(%s: %s)", bucket, bucketInformation);
+ }
+ }
+
+ private final List<BucketInfo> buckets = new ArrayList<BucketInfo>();
+
+ public GetBucketListReply() {
+ super(DocumentProtocol.REPLY_GETBUCKETLIST);
+ }
+
+ public List<BucketInfo> getBuckets() {
+ return buckets;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketStateMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketStateMessage.java new file mode 100755 index 00000000000..a6b23647f6c --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketStateMessage.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; + +/** + * This message is a request to return the state of a given bucket. The corresponding reply is {@link + * GetBucketStateReply}. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GetBucketStateMessage extends DocumentMessage { + + private BucketId bucket = null; + + /** + * Constructs a new message for deserialization. + */ + GetBucketStateMessage() { + // empty + } + + /** + * Constructs a new reply with initial content. + * + * @param bucket The bucket whose state to reply with. + */ + public GetBucketStateMessage(BucketId bucket) { + this.bucket = bucket; + } + + /** + * Returns the bucket whose state this contains. + * + * @return The bucket id. + */ + public BucketId getBucketId() { + return bucket; + } + + /** + * Sets the bucket whose state this contains. + * + * @param bucket The bucket id to set. + */ + public void setBucketId(BucketId bucket) { + this.bucket = bucket; + } + + @Override + public DocumentReply createReply() { + return new GetBucketStateReply(); + } + + @Override + public long getSequenceId() { + return bucket.getRawId(); + } + + @Override + public int getApproxSize() { + return super.getApproxSize() + 8; + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_GETBUCKETSTATE; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketStateReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketStateReply.java new file mode 100755 index 00000000000..f25543412f6 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketStateReply.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import java.util.ArrayList; +import java.util.List; + +/** + * This is a reply to a {@link GetBucketStateMessage}. It contains the state of the bucket id requested by the message. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GetBucketStateReply extends DocumentReply { + + private List<DocumentState> state; + + /** + * Constructs a new reply with no content. + */ + public GetBucketStateReply() { + super(DocumentProtocol.REPLY_GETBUCKETSTATE); + state = new ArrayList<DocumentState>(); + } + + /** + * Constructs a new reply with initial content. + * + * @param state The state to set. + */ + public GetBucketStateReply(List<DocumentState> state) { + super(DocumentProtocol.REPLY_GETBUCKETSTATE); + this.state = state; + } + + /** + * Sets the bucket state of this. + * + * @param state The state to set. + */ + public void setBucketState(List<DocumentState> state) { + this.state = state; + } + + /** + * Returns the bucket state contained in this. + * + * @return The state object. + */ + public List<DocumentState> getBucketState() { + return state; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java new file mode 100755 index 00000000000..cf66704d21f --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.document.DocumentId;
+
+import java.util.Arrays;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class GetDocumentMessage extends DocumentMessage {
+
+ final static String DEFAULT_FIELD_SET = "[all]";
+ private DocumentId documentId = null;
+ private String fieldSet = DEFAULT_FIELD_SET;
+
+ /**
+ * Constructs a new message for deserialization.
+ */
+ GetDocumentMessage() {
+ // empty
+ }
+
+ /**
+ * Constructs a new document get message.
+ *
+ * @param documentId The identifier of the document to get.
+ */
+ public GetDocumentMessage(DocumentId documentId) {
+ setDocumentId(documentId);
+ }
+
+ /**
+ * Constructs a new document get message.
+ *
+ * @param documentId The identifier of the document to get.
+ * @param fieldSet Which fields to retrieve from the document
+ */
+ public GetDocumentMessage(DocumentId documentId, String fieldSet) {
+ setDocumentId(documentId);
+ this.fieldSet = fieldSet;
+ }
+
+ /**
+ * Returns the identifier of the document to retrieve.
+ *
+ * @return The document id.
+ */
+ public DocumentId getDocumentId() {
+ return documentId;
+ }
+
+ /**
+ * Sets the identifier of the document to retrieve.
+ *
+ * @param documentId The document id to set.
+ */
+ public void setDocumentId(DocumentId documentId) {
+ if (documentId == null) {
+ throw new IllegalArgumentException("Document id can not be null.");
+ }
+ this.documentId = documentId;
+ }
+
+ public String getFieldSet() {
+ return fieldSet;
+ }
+
+ @Override
+ public DocumentReply createReply() {
+ return new GetDocumentReply();
+ }
+
+ @Override
+ public int getApproxSize() {
+ return super.getApproxSize() + 4 + documentId.toString().length();
+ }
+
+ @Override
+ public boolean hasSequenceId() {
+ return true;
+ }
+
+ @Override
+ public long getSequenceId() {
+ return Arrays.hashCode(documentId.getGlobalId());
+ }
+
+ @Override
+ public int getType() {
+ return DocumentProtocol.MESSAGE_GETDOCUMENT;
+ }
+
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentReply.java new file mode 100755 index 00000000000..f5f687bb18b --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentReply.java @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.serialization.DocumentDeserializer;
+
+import java.nio.ByteBuffer;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class GetDocumentReply extends DocumentAcceptedReply {
+
+ private DocumentDeserializer buffer = null;
+ private Document document = null;
+ private long lastModified = 0;
+ private LazyDecoder decoder = null;
+
+ /**
+ * Constructs a new reply for deserialization.
+ */
+ GetDocumentReply() {
+ super(DocumentProtocol.REPLY_GETDOCUMENT);
+ }
+
+ /**
+ * Constructs a new reply to lazily deserialize from a byte buffer.
+ * @param decoder The decoder to use for deserialization.
+ * @param buf A byte buffer that contains a serialized reply.
+ */
+ GetDocumentReply(LazyDecoder decoder, DocumentDeserializer buf) {
+ super(DocumentProtocol.REPLY_GETDOCUMENT);
+ this.decoder = decoder;
+ buffer = buf;
+ }
+
+ /**
+ * Constructs a new document get reply.
+ *
+ * @param doc The document requested.
+ */
+ public GetDocumentReply(Document doc) {
+ super(DocumentProtocol.REPLY_GETDOCUMENT);
+ document = doc;
+ }
+
+ /**
+ * This method will make sure that any serialized content is deserialized into proper message content on first
+ * entry. Any subsequent entry into this function will do nothing.
+ */
+ private void deserialize() {
+ if (decoder != null && buffer != null) {
+ decoder.decode(this, buffer);
+ decoder = null;
+ buffer = null;
+ }
+ }
+
+ /**
+ * Returns the document retrieved.
+ *
+ * @return The document.
+ */
+ public Document getDocument() {
+ deserialize();
+ return document;
+ }
+
+ /**
+ * Sets the document of this reply.
+ *
+ * @param doc The document to set.
+ */
+ public void setDocument(Document doc) {
+ buffer = null;
+ decoder = null;
+ document = doc;
+ lastModified = document != null && document.getLastModified() != null ? document.getLastModified() : 0;
+ }
+
+ /**
+ * Returns the date the document was last modified.
+ *
+ * @return The date.
+ */
+ public long getLastModified() {
+ deserialize();
+ return lastModified;
+ }
+
+ /**
+ * Set the date the document was last modified.
+ *
+ * @param modified The date.
+ */
+ void setLastModified(long modified) {
+ lastModified = modified;
+ }
+
+ /**
+ * Returns the internal buffer to deserialize from, may be null.
+ *
+ * @return The buffer.
+ */
+ public ByteBuffer getSerializedBuffer() {
+ return buffer != null ? buffer.getBuf().getByteBuffer() : null;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LazyDecoder.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LazyDecoder.java new file mode 100644 index 00000000000..2ac7f716850 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LazyDecoder.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.serialization.DocumentDeserializer; +import com.yahoo.messagebus.Routable; + +public interface LazyDecoder { + + public void decode(Routable obj, DocumentDeserializer buf); + +}
\ No newline at end of file diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java new file mode 100644 index 00000000000..79725e25e08 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java @@ -0,0 +1,151 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.messagebus.metrics.CountMetric; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.metrics.ValueMetric; + +import java.util.ArrayList; +import java.util.List; + +/** + * Load balances over a set of nodes based on statistics gathered from those nodes. + * + * @author thomasg + */ +public class LoadBalancer { + + public static class NodeMetrics extends MetricSet { + public CountMetric sent = new CountMetric("sent", this); + public CountMetric busy = new CountMetric("busy", this); + public ValueMetric<Double> weight = new ValueMetric<Double>("weight", 1.0, this); + + public NodeMetrics(String name, MetricSet owner) { + super(name); + owner.addMetric(this); + } + } + + public static class Metrics extends MetricSet { + MetricSet targets = new MetricSet("nodes"); + + public Metrics(String name) { + super(name); + addMetric(targets); + } + } + + public static class Node { + public Node(Mirror.Entry e, NodeMetrics m) { entry = e; metrics = m; } + + public Mirror.Entry entry; + public NodeMetrics metrics; + } + + /** Statistics on each node we are load balancing over. Populated lazily. */ + private List<NodeMetrics> nodeWeights = new ArrayList<NodeMetrics>(); + + private Metrics metrics; + private String cluster; + private double position = 0.0; + + public LoadBalancer(String cluster, String session, Metrics metrics) { + this.metrics = metrics; + this.cluster = cluster; + } + + public List<NodeMetrics> getNodeWeights() { + return nodeWeights; + } + + /** Returns the index from a node name string */ + public int getIndex(String nodeName) { + try { + String s = nodeName.substring(cluster.length() + 1); + s = s.substring(0, s.indexOf("/")); + s = s.substring(s.lastIndexOf(".") + 1); + return Integer.parseInt(s); + } catch (IndexOutOfBoundsException | NumberFormatException e) { + String err = "Expected recipient on the form '" + cluster + "/x/[y.]number/z', got '" + nodeName + "'."; + throw new IllegalArgumentException(err, e); + } + } + + /** + * The load balancing operation: Returns a node choice from the given choices, + * based on previously gathered statistics on the nodes, and a running "position" + * which is increased by 1 on each call to this. + * + * @param choices the node choices, represented as Slobrok entries + * @return the chosen node, or null only if the given choices were zero + */ + public Node getRecipient(Mirror.Entry[] choices) { + if (choices.length == 0) return null; + + double weightSum = 0.0; + Node selectedNode = null; + for (Mirror.Entry entry : choices) { + NodeMetrics nodeMetrics = getNodeMetrics(entry); + + weightSum += nodeMetrics.weight.get(); + + if (weightSum > position) { + selectedNode = new Node(entry, nodeMetrics); + break; + } + } + if (selectedNode == null) { // Position>sum of all weights: Wrap around (but keep the remainder for some reason) + position -= weightSum; + selectedNode = new Node(choices[0], getNodeMetrics(choices[0])); + } + position += 1.0; + selectedNode.metrics.sent.inc(1); + return selectedNode; + } + + /** + * Returns the node metrics at a given index. + * If there is no entry at the given index it is created by this call. + */ + private NodeMetrics getNodeMetrics(Mirror.Entry entry) { + int index = getIndex(entry.getName()); + // expand node array as needed + while (nodeWeights.size() < (index + 1)) + nodeWeights.add(null); + + NodeMetrics nodeMetrics = nodeWeights.get(index); + if (nodeMetrics == null) { // initialize statistics for this node + nodeMetrics = new NodeMetrics("node_" + index, metrics.targets); + nodeWeights.set(index, nodeMetrics); + } + return nodeMetrics; + } + + /** Scale weights such that ratios are preserved */ + private void increaseWeights() { + for (NodeMetrics n : nodeWeights) { + if (n == null) continue; + double want = n.weight.get() * 1.01010101010101010101; + if (want >= 1.0) { + n.weight.set(want); + } else { + n.weight.set(1.0); + } + } + } + + public void received(Node node, boolean busy) { + if (busy) { + double wantWeight = node.metrics.weight.get() - 0.01; + if (wantWeight < 1.0) { + increaseWeights(); + node.metrics.weight.set(1.0); + } else { + node.metrics.weight.set(wantWeight); + } + node.metrics.busy.inc(1); + } + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java new file mode 100644 index 00000000000..e6123dc3dc2 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.jrt.slobrok.api.IMirror; +import com.yahoo.jrt.slobrok.api.SlobrokList; +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.Hop; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RoutingContext; +import com.yahoo.messagebus.routing.RoutingNodeIterator; +import com.yahoo.cloud.config.SlobroksConfig; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Logger; + +/** + * Routing policy to load balance between nodes in a randomly distributed cluster, such as a docproc cluster. + * + * pattern=<pattern> (mandatory, determines the pattern of nodes to send to)<br> + * slobroks=<comma-separated connectionspecs> (optional, list of slobroks to use to find the pattern)<br> + * config=<comma-separated list of config servers> (optional, list of config servers to use to find slobrok config) + * + * If both slobroks and config is specified, the list from slobroks is used. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Haakon Humberset</a> + */ +public class LoadBalancerPolicy extends ExternalSlobrokPolicy { + String cluster = null; + String session = null; + private String pattern = null; + private AtomicLong count = new AtomicLong(0); + volatile Mirror.Entry [] lastLookup; + + LoadBalancer.Metrics metrics; + LoadBalancer loadBalancer; + + public LoadBalancerPolicy(String param) { + this(param, parse(param)); + } + + public LoadBalancerPolicy(String param, Map<String, String> params) { + super(params); + + cluster = params.get("cluster"); + session = params.get("session"); + + if (cluster == null) { + error = "Required parameter pattern not set"; + return; + } + + if (session == null) { + error = "Required parameter session not set"; + return; + } + + metrics = new LoadBalancer.Metrics(param); + metrics.setXmlTagName("loadbalancer"); + pattern = cluster + "/*/" + session; + loadBalancer = new LoadBalancer(cluster, session, metrics); + } + + @Override + public void doSelect(RoutingContext context) { + LoadBalancer.Node node = getRecipient(context); + + if (node != null) { + context.setContext(node); + Route route = new Route(context.getRoute()); + route.setHop(0, Hop.parse(node.entry.getSpec() + "/" + session)); + context.addChild(route); + } else { + context.setError(ErrorCode.NO_ADDRESS_FOR_SERVICE, + "Could not resolve any nodes to send to in pattern " + pattern); + } + } + + /** + Finds the TCP address of the target. + + @return Returns a hop representing the TCP address of the target, or null if none could be found. + */ + LoadBalancer.Node getRecipient(RoutingContext context) { + long c = count.getAndIncrement(); + if ((c%1024 == 0) || (lastLookup == null) || (lastLookup.length == 0)) { + lastLookup = lookup(context, pattern); + } + return loadBalancer.getRecipient(lastLookup); + } + + public void merge(RoutingContext context) { + RoutingNodeIterator it = context.getChildIterator(); + Reply reply = it.removeReply(); + LoadBalancer.Node target = (LoadBalancer.Node)context.getContext(); + + boolean busy = false; + for (int i = 0; i < reply.getNumErrors(); i++) { + if (reply.getError(i).getCode() == ErrorCode.SESSION_BUSY) { + busy = true; + } + } + loadBalancer.received(target, busy); + + context.setReply(reply); + } + + public MetricSet getMetrics() { + return metrics; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java new file mode 100755 index 00000000000..74ca65df547 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.jrt.slobrok.api.Mirror;
+import com.yahoo.messagebus.metrics.MetricSet;
+import com.yahoo.messagebus.routing.*;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This policy implements the logic to prefer local services that matches a slobrok pattern.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LocalServicePolicy implements DocumentProtocolRoutingPolicy {
+
+ private final String localAddress;
+ private Map<String, CacheEntry> cache = new HashMap<String, CacheEntry>();
+
+ /**
+ * Constructs a policy that will choose local services that match the slobrok pattern in which this policy occured.
+ * If no local service can be found, this policy simply returns the asterisk to allow the network to choose any.
+ *
+ * @param param The address to use for this, if empty this will resolve to hostname.
+ */
+ public LocalServicePolicy(String param) {
+ localAddress = (param != null && param.length() > 0) ? param : null;
+ }
+
+ // Inherit doc from RoutingPolicy.
+ public void select(RoutingContext ctx) {
+ Route route = new Route(ctx.getRoute());
+ route.setHop(0, getRecipient(ctx));
+ ctx.addChild(route);
+ }
+
+ // Inherit doc from RoutingPolicy.
+ public void merge(RoutingContext ctx) {
+ DocumentProtocol.merge(ctx);
+ }
+
+ /**
+ * Returns the appropriate recipient hop for the given routing context. This method provides synchronized access to
+ * the internal cache.
+ *
+ * @param ctx The routing context.
+ * @return The recipient hop to use.
+ */
+ private synchronized Hop getRecipient(RoutingContext ctx) {
+ CacheEntry entry = update(ctx);
+ if (entry.recipients.isEmpty()) {
+ Hop hop = new Hop(ctx.getRoute().getHop(0));
+ hop.setDirective(ctx.getDirectiveIndex(), new VerbatimDirective("*"));
+ return hop;
+ }
+ if (++entry.offset >= entry.recipients.size()) {
+ entry.offset = 0;
+ }
+ return new Hop(entry.recipients.get(entry.offset));
+ }
+
+ /**
+ * Updates and returns the cache entry for the given routing context. This method assumes that synchronization is
+ * handled outside of it.
+ *
+ * @param ctx The routing context.
+ * @return The updated cache entry.
+ */
+ private CacheEntry update(RoutingContext ctx) {
+ String key = getCacheKey(ctx);
+ CacheEntry entry = cache.get(key);
+ if (entry == null) {
+ entry = new CacheEntry();
+ cache.put(key, entry);
+ }
+ int upd = ctx.getMirror().updates();
+ if (entry.generation != upd) {
+ entry.generation = upd;
+ entry.recipients.clear();
+
+ Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix());
+ String self = localAddress != null ? localAddress : toAddress(ctx.getMessageBus().getConnectionSpec());
+ for (Mirror.Entry item : arr) {
+ if (self.equals(toAddress(item.getSpec()))) {
+ entry.recipients.add(Hop.parse(item.getName()));
+ }
+ }
+ }
+ return entry;
+ }
+
+ /**
+ * Returns a cache key for this instance of the policy. Because behaviour is based on the hop in which the policy
+ * occurs, the cache key is the hop string itself.
+ *
+ * @param ctx The routing context.
+ * @return The cache key.
+ */
+ private String getCacheKey(RoutingContext ctx) {
+ return ctx.getRoute().getHop(0).toString();
+ }
+
+ /**
+ * Defines the necessary cache data.
+ */
+ private class CacheEntry {
+ private final List<Hop> recipients = new ArrayList<Hop>();
+ private int generation = 0;
+ private int offset = 0;
+ }
+
+ /**
+ * Searches the given connection spec for a hostname or IP address. If an address is not found, this method returns
+ * null.
+ *
+ * @param connection The connection spec to search.
+ * @return The address, may be null.
+ */
+ private static String toAddress(String connection) {
+ if (connection.startsWith("tcp/")) {
+ int pos = connection.indexOf(':');
+ if (pos > 4) {
+ return connection.substring(4, pos);
+ }
+ }
+ return null;
+ }
+
+ public void destroy() {
+ }
+
+ public MetricSet getMetrics() {
+ return null;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MapVisitorMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MapVisitorMessage.java new file mode 100644 index 00000000000..97604cd0d27 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MapVisitorMessage.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import java.util.Map; +import java.util.TreeMap; + +public class MapVisitorMessage extends VisitorMessage { + + private final Map<String, String> data = new TreeMap<String, String>(); + + MapVisitorMessage() { + // must be deserialized into + } + + public MapVisitorMessage(MapVisitorMessage cmd) { + data.putAll(cmd.data); + } + + public Map<String, String> getData() { + return data; + } + + @Override + public DocumentReply createReply() { + return new VisitorReply(DocumentProtocol.REPLY_MAPVISITOR); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_MAPVISITOR; + } + + @Override + public int getApproxSize() { + int length = super.getApproxSize() + 4; + for (Map.Entry<String, String> pairs : data.entrySet()) { + length += 8; + length += (pairs.getKey()).length() + pairs.getValue().length(); + } + return length; + } + + @Override + public String toString() { + return "MapVisitorMessage(" + data.toString() + ")"; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java new file mode 100644 index 00000000000..c72994a9fc7 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RoutingContext; +import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class MessageTypePolicy implements DocumentProtocolRoutingPolicy, ConfigSubscriber.SingleSubscriber<MessagetyperouteselectorpolicyConfig> { + + private final AtomicReference<Map<Integer, Route>> configRef = new AtomicReference<Map<Integer, Route>>(); + private ConfigSubscriber subscriber; + private volatile Route defaultRoute; + + public MessageTypePolicy(String configId) { + subscriber = new ConfigSubscriber(); + subscriber.subscribe(this, MessagetyperouteselectorpolicyConfig.class, configId); + } + + @Override + public void select(RoutingContext context) { + int messageType = context.getMessage().getType(); + Route route = configRef.get().get(messageType); + if (route == null) { + route = defaultRoute; + } + context.addChild(route); + } + + @Override + public void merge(RoutingContext context) { + DocumentProtocol.merge(context); + } + + @Override + public void destroy() { + if (subscriber!=null) subscriber.close(); + } + + @Override + public MetricSet getMetrics() { + return null; + } + + @Override + public void configure(MessagetyperouteselectorpolicyConfig cfg) { + Map<Integer, Route> h = new HashMap<Integer, Route>(); + for (MessagetyperouteselectorpolicyConfig.Route selector : cfg.route()) { + h.put(selector.messagetype(), Route.parse(selector.name())); + } + configRef.set(h); + defaultRoute = Route.parse(cfg.defaultroute()); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java new file mode 100755 index 00000000000..9ff49e08365 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java @@ -0,0 +1,156 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.TestAndSetCondition; +import com.yahoo.document.serialization.DocumentDeserializer; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class PutDocumentMessage extends TestAndSetMessage { + + private DocumentDeserializer buffer = null; + private DocumentPut put = null; + private long time = 0; + private LazyDecoder decoder = null; + + /** + * Constructs a new message for deserialization. + */ + PutDocumentMessage() { + // empty + } + + /** + * Constructs a new message from a byte buffer. + * @param decoder The decoder to use for deserialization. + * @param buffer A byte buffer that contains a serialized message. + */ + public PutDocumentMessage(LazyDecoder decoder, DocumentDeserializer buffer) { + this.decoder = decoder; + this.buffer = buffer; + } + + /** + * Constructs a new document put message. + * + * @param put Document put operation + */ + public PutDocumentMessage(DocumentPut put) { + this.put = put; + } + + /** + * Creates an empty PutDocumentMessage + */ + public static PutDocumentMessage createEmpty() { + return new PutDocumentMessage(null); + } + + /** + * This method will make sure that any serialized content is deserialized into proper message content on first + * entry. Any subsequent entry into this function will do nothing. + */ + private void deserialize() { + if (decoder != null && buffer != null) { + decoder.decode(this, buffer); + decoder = null; + buffer = null; + } + } + + /** + * Returns the document put operation + */ + public DocumentPut getDocumentPut() { + deserialize(); + return put; + } + + /** + * Sets the document to put. + * + * @param put Put document operation + */ + public void setDocumentPut(DocumentPut put) { + buffer = null; + decoder = null; + this.put = put; + } + + /** + * Returns the timestamp of the document to put. + * + * @return The document timestamp. + */ + public long getTimestamp() { + deserialize(); + return time; + } + + /** + * Sets the timestamp of the document to put. + * + * @param time The timestamp to set. + */ + public void setTimestamp(long time) { + buffer = null; + decoder = null; + this.time = time; + } + + /** + * Returns the raw serialized buffer. This buffer is stored as the message is received from accross the network, and + * deserialized from as soon as a member is requested. This method will return null if the buffer has been decoded. + * + * @return The buffer containing the serialized data for this message, or null. + */ + ByteBuffer getSerializedBuffer() { + return buffer != null ? buffer.getBuf().getByteBuffer() : null; // TODO: very dirty. Must make interface. + } + + @Override + public DocumentReply createReply() { + return new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT); + } + + @Override + public int getApproxSize() { + if (buffer != null) { + return buffer.getBuf().remaining(); + } + return put.getDocument().getApproxSize(); + } + + @Override + public boolean hasSequenceId() { + return true; + } + + @Override + public long getSequenceId() { + deserialize(); + return Arrays.hashCode(put.getId().getGlobalId()); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_PUTDOCUMENT; + } + + @Override + public TestAndSetCondition getCondition() { + deserialize(); + return put.getCondition(); + } + + @Override + public void setCondition(TestAndSetCondition condition) { + put.setCondition(condition); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/QueryResultMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/QueryResultMessage.java new file mode 100644 index 00000000000..d2e7f4013e6 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/QueryResultMessage.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.vdslib.SearchResult; +import com.yahoo.vdslib.DocumentSummary; + +/** + */ +public class QueryResultMessage extends VisitorMessage { + + private SearchResult searchResult = null; + private DocumentSummary summary = null; + + public SearchResult getResult() { + return searchResult; + } + + public DocumentSummary getSummary() { + return summary; + } + + public void setSearchResult(SearchResult result) { + searchResult = result; + } + + public void setSummary(DocumentSummary summary) { + this.summary = summary; + } + + @Override + public DocumentReply createReply() { + return new VisitorReply(DocumentProtocol.REPLY_QUERYRESULT); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_QUERYRESULT; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentMessage.java new file mode 100755 index 00000000000..f6fcb5965ad --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentMessage.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.TestAndSetCondition;
+
+import java.util.Arrays;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class RemoveDocumentMessage extends TestAndSetMessage {
+ private DocumentRemove remove = null;
+
+ /**
+ * Constructs a new message for deserialization.
+ */
+ RemoveDocumentMessage() {
+ // empty
+ }
+
+ /**
+ * Constructs a new document remove message.
+ *
+ * @param documentId The identifier of the document to remove.
+ */
+ public RemoveDocumentMessage(DocumentId documentId) {
+ remove = new DocumentRemove(documentId);
+ }
+
+ /**
+ * Constructs a new document remove message.
+ *
+ * @param remove The DocumentRemove operation to perform
+ */
+ public RemoveDocumentMessage(DocumentRemove remove) {
+ this.remove = remove;
+ }
+
+ /**
+ * Returns the identifier of the document to remove.
+ *
+ * @return The document id.
+ */
+ public DocumentId getDocumentId() {
+ return remove.getId();
+ }
+
+ /**
+ * Sets the identifier of the document to remove.
+ *
+ * @param documentId The document id to set.
+ */
+ public void setDocumentId(DocumentId documentId) {
+ if (documentId == null) {
+ throw new IllegalArgumentException("Document id can not be null.");
+ }
+
+ remove = new DocumentRemove(documentId);
+ }
+
+ @Override
+ public DocumentReply createReply() {
+ return new RemoveDocumentReply();
+ }
+
+ @Override
+ public int getApproxSize() {
+ return super.getApproxSize() + 4 + remove.getId().toString().length();
+ }
+
+ @Override
+ public boolean hasSequenceId() {
+ return true;
+ }
+
+ @Override
+ public long getSequenceId() {
+ return Arrays.hashCode(remove.getId().getGlobalId());
+ }
+
+ @Override
+ public int getType() {
+ return DocumentProtocol.MESSAGE_REMOVEDOCUMENT;
+ }
+
+ @Override
+ public void setCondition(TestAndSetCondition condition) {
+ remove.setCondition(condition);
+ }
+
+ @Override
+ public TestAndSetCondition getCondition() {
+ return remove.getCondition();
+ }
+
+ public DocumentRemove getDocumentRemove() {
+ return remove;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentReply.java new file mode 100755 index 00000000000..c259aaa5731 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentReply.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class RemoveDocumentReply extends WriteDocumentReply {
+
+ private boolean found = true;
+
+ /**
+ * Constructs a new reply with no content.
+ */
+ public RemoveDocumentReply() {
+ super(DocumentProtocol.REPLY_REMOVEDOCUMENT);
+ }
+
+ /**
+ * Returns whether or not the document was found and removed.
+ *
+ * @return True if document was found.
+ */
+ public boolean wasFound() {
+ return found;
+ }
+
+ /**
+ * Set whether or not the document was found and removed.
+ *
+ * @param found True if the document was found.
+ */
+ public void setWasFound(boolean found) {
+ this.found = found;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveLocationMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveLocationMessage.java new file mode 100755 index 00000000000..4a7287b59d8 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveLocationMessage.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.*; +import com.yahoo.document.select.BucketSelector; +import java.util.Set; + +/** + * Message (VDS only) to remove an entire location for users using userdoc or groupdoc schemes. + * We use a document selection so the user can specify a subset of those documents to be deleted + * if they wish. + */ +public class RemoveLocationMessage extends DocumentMessage { + String documentSelection; + BucketId bucketId; + + public RemoveLocationMessage(String documentSelection) { + try { + this.documentSelection = documentSelection; + BucketSelector bucketSel = new BucketSelector(new BucketIdFactory()); + Set<BucketId> rawBuckets = bucketSel.getBucketList(documentSelection); + if (rawBuckets == null || rawBuckets.size() != 1) { + throw new IllegalArgumentException("Document selection for remove location must map to a single location (user or group)"); + } else { + // There can only be one. + for (BucketId id : rawBuckets) { + bucketId = id; + } + } + } catch (com.yahoo.document.select.parser.ParseException p) { + throw new IllegalArgumentException(p); + } + } + + public String getDocumentSelection() { + return documentSelection; + } + + @Override + public DocumentReply createReply() { + return new DocumentReply(DocumentProtocol.REPLY_REMOVELOCATION); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_REMOVELOCATION; + } + + public BucketId getBucketId() { + return bucketId; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ReplyMerger.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ReplyMerger.java new file mode 100644 index 00000000000..2dad8312fc9 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ReplyMerger.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.collections.Tuple2;
+import com.yahoo.messagebus.EmptyReply;
+import com.yahoo.messagebus.Reply;
+
+/**
+ * Encapsulated logic for merging replies from 1-n related DocumentProtocol messages.
+ * For internal use only. Not multithread safe.
+ */
+final class ReplyMerger {
+
+ private Reply successReply = null;
+ private int successIndex = -1;
+ private Reply error = null;
+ private Reply ignore = null;
+
+ public void merge(int i, Reply r) {
+ if (r.hasErrors()) {
+ mergeAllReplyErrors(r);
+ } else {
+ updateStateWithSuccessfulReply(i, r);
+ }
+ }
+
+ private boolean resourceWasFound(Reply r) {
+ if (r instanceof RemoveDocumentReply) {
+ return ((RemoveDocumentReply) r).wasFound();
+ }
+ if (r instanceof UpdateDocumentReply) {
+ return ((UpdateDocumentReply) r).wasFound();
+ }
+ if (r instanceof GetDocumentReply) {
+ return ((GetDocumentReply) r).getLastModified() > 0;
+ }
+ return false;
+ }
+
+ private boolean replyIsBetterThanCurrent(Reply r) {
+ return resourceWasFound(r) && !resourceWasFound(successReply);
+ }
+
+ private void updateStateWithSuccessfulReply(int i, Reply r) {
+ if (successReply == null || replyIsBetterThanCurrent(r)) {
+ setCurrentBestReply(i, r);
+ }
+ }
+
+ private void setCurrentBestReply(int i, Reply r) {
+ successReply = r;
+ successIndex = i;
+ }
+
+ private void mergeAllReplyErrors(Reply r) {
+ if (handleReplyWithOnlyIgnoredErrors(r)) {
+ return;
+ }
+ if (error == null) {
+ error = new EmptyReply();
+ }
+ for (int j = 0; j < r.getNumErrors(); ++j) {
+ error.addError(r.getError(j));
+ }
+ }
+
+ private boolean handleReplyWithOnlyIgnoredErrors(Reply r) {
+ if (DocumentProtocol.hasOnlyErrorsOfType(r, DocumentProtocol.ERROR_MESSAGE_IGNORED)) {
+ if (ignore == null) {
+ ignore = new EmptyReply();
+ }
+ ignore.addError(r.getError(0));
+ return true;
+ }
+ return false;
+ }
+
+ private boolean shouldReturnErrorReply() {
+ return (error != null || (ignore != null && successReply == null));
+ }
+
+ private Tuple2<Integer, Reply> createMergedErrorReplyResult() {
+ if (error != null) {
+ return new Tuple2<>(null, error);
+ }
+ if (ignore != null && successReply == null) {
+ return new Tuple2<>(null, ignore);
+ }
+ throw new IllegalStateException("createMergedErrorReplyResult called without error");
+ }
+
+ private boolean successfullyMergedAtLeastOneReply() {
+ return successReply != null;
+ }
+
+ private Tuple2<Integer, Reply> createEmptyReplyResult() {
+ return new Tuple2<>(null, (Reply)new EmptyReply());
+ }
+
+ public Tuple2<Integer, Reply> mergedReply() {
+ if (shouldReturnErrorReply()) {
+ return createMergedErrorReplyResult();
+ } else if (!successfullyMergedAtLeastOneReply()) {
+ return createEmptyReplyResult();
+ }
+ return new Tuple2<>(successIndex, successReply);
+ }
+
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java new file mode 100755 index 00000000000..f0e49146851 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java @@ -0,0 +1,125 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.jrt.slobrok.api.Mirror;
+import com.yahoo.messagebus.EmptyReply;
+import com.yahoo.messagebus.Error;
+import com.yahoo.messagebus.ErrorCode;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.metrics.MetricSet;
+import com.yahoo.messagebus.routing.Hop;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.messagebus.routing.RoutingContext;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This policy implements round-robin selection of the configured recipients that are currently registered in slobrok.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class RoundRobinPolicy implements DocumentProtocolRoutingPolicy {
+
+ private final Map<String, CacheEntry> cache = new HashMap<String, CacheEntry>();
+
+ // Inherit doc from RoutingPolicy.
+ public void select(RoutingContext ctx) {
+ Hop hop = getRecipient(ctx);
+ if (hop != null) {
+ Route route = new Route(ctx.getRoute());
+ route.setHop(0, hop);
+ ctx.addChild(route);
+ } else {
+ Reply reply = new EmptyReply();
+ reply.addError(new Error(ErrorCode.NO_ADDRESS_FOR_SERVICE,
+ "None of the configured recipients are currently available."));
+ ctx.setReply(reply);
+ }
+ }
+
+ // Inherit doc from RoutingPolicy.
+ public void merge(RoutingContext ctx) {
+ DocumentProtocol.merge(ctx);
+ }
+
+ /**
+ * Returns the appropriate recipient hop for the given routing context. This method provides synchronized access to
+ * the internal cache.
+ *
+ * @param ctx The routing context.
+ * @return The recipient hop to use.
+ */
+ private synchronized Hop getRecipient(RoutingContext ctx) {
+ CacheEntry entry = update(ctx);
+ if (entry.recipients.isEmpty()) {
+ return null;
+ }
+ if (++entry.offset >= entry.recipients.size()) {
+ entry.offset = 0;
+ }
+ return new Hop(entry.recipients.get(entry.offset));
+ }
+
+ /**
+ * Updates and returns the cache entry for the given routing context. This method assumes that synchronization is
+ * handled outside of it.
+ *
+ * @param ctx The routing context.
+ * @return The updated cache entry.
+ */
+ private CacheEntry update(RoutingContext ctx) {
+ String key = getCacheKey(ctx);
+ CacheEntry entry = cache.get(key);
+ if (entry == null) {
+ entry = new CacheEntry();
+ cache.put(key, entry);
+ }
+
+ int upd = ctx.getMirror().updates();
+ if (entry.generation != upd) {
+ entry.generation = upd;
+ entry.recipients.clear();
+ for (int i = 0; i < ctx.getNumRecipients(); ++i) {
+ Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getRecipient(i).getHop(0).toString());
+ for (Mirror.Entry item : arr) {
+ entry.recipients.add(Hop.parse(item.getName()));
+ }
+ }
+ }
+ return entry;
+ }
+
+ /**
+ * Returns a cache key for this instance of the policy. Because behaviour is based on the recipient list of this
+ * policy, the cache key is the concatenated string of recipient routes.
+ *
+ * @param ctx The routing context.
+ * @return The cache key.
+ */
+ private String getCacheKey(RoutingContext ctx) {
+ StringBuilder ret = new StringBuilder();
+ for (int i = 0; i < ctx.getNumRecipients(); ++i) {
+ ret.append(ctx.getRecipient(i).getHop(0).toString()).append(" ");
+ }
+ return ret.toString();
+ }
+
+ /**
+ * Defines the necessary cache data.
+ */
+ private class CacheEntry {
+ private final List<Hop> recipients = new ArrayList<Hop>();
+ private int generation = 0;
+ private int offset = 0;
+ }
+
+ public void destroy() {
+ }
+
+ public MetricSet getMetrics() {
+ return null;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories50.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories50.java new file mode 100755 index 00000000000..9ba2aee9227 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories50.java @@ -0,0 +1,984 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.serialization.DocumentDeserializer; +import com.yahoo.document.serialization.DocumentSerializer; +import com.yahoo.document.serialization.DocumentSerializerFactory; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.Routable; +import com.yahoo.vdslib.DocumentList; +import com.yahoo.vdslib.DocumentSummary; +import com.yahoo.vdslib.SearchResult; +import com.yahoo.vdslib.VisitorStatistics; +import com.yahoo.vespa.objects.Deserializer; +import com.yahoo.vespa.objects.Serializer; + +import java.util.Map; +import java.util.logging.Logger; + + +/** + * This class encapsulates all the {@link RoutableFactory} classes needed to implement serialization for the document + * protocol. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. + */ +public abstract class RoutableFactories50 { + + /** + * Implements the shared factory logic required for {@link DocumentMessage} objects, and it offers a more convenient + * interface for implementing {@link RoutableFactory}. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ + public static abstract class DocumentMessageFactory extends AbstractRoutableFactory { + + /** + * This method encodes the given message using the given serializer. You are guaranteed to only receive messages + * of the type that this factory was registered for. + * <p> + * This method is NOT exception safe. Return false to + * signal failure. + * + * @param msg The message to encode. + * @param serializer The serializer to use for encoding. + * @return True if the message was encoded. + */ + protected abstract boolean doEncode(DocumentMessage msg, DocumentSerializer serializer); + + /** + * This method decodes a message from the given deserializer. You are guaranteed to only receive byte buffers + * generated by a previous call to {@link #doEncode(DocumentMessage, DocumentSerializer)}. + * <p> + * This method is NOT exception safe. Return null to signal failure. + * + * @param deserializer The deserializer to use for decoding. + * @return The decoded message. + */ + protected abstract DocumentMessage doDecode(DocumentDeserializer deserializer); + + public boolean encode(Routable obj, DocumentSerializer out) { + if (!(obj instanceof DocumentMessage)) { + throw new AssertionError( + "Document message factory (" + getClass().getName() + ") registered for incompatible " + + "routable type " + obj.getType() + "(" + obj.getClass().getName() + ")."); + } + DocumentMessage msg = (DocumentMessage)obj; + out.putByte(null, (byte)(msg.getPriority().getValue())); + out.putInt(null, msg.getLoadType().getId()); + return doEncode(msg, out); + } + + public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { + byte pri = in.getByte(null); + int loadType = in.getInt(null); + DocumentMessage msg = doDecode(in); + if (msg != null) { + msg.setPriority(DocumentProtocol.getPriority(pri)); + msg.setLoadType(loadTypes.getIdMap().get(loadType)); + } + return msg; + } + } + + /** + * Implements the shared factory logic required for {@link DocumentReply} objects, and it offers a more convenient + * interface for implementing {@link RoutableFactory}. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ + public static abstract class DocumentReplyFactory extends AbstractRoutableFactory { + + /** + * This method encodes the given reply into the given byte buffer. You are guaranteed to only receive replies of + * the type that this factory was registered for. + * <p> + * This method is NOT exception safe. Return false to signal + * failure. + * + * @param reply The reply to encode. + * @param buf The byte buffer to write to. + * @return True if the message was encoded. + */ + protected abstract boolean doEncode(DocumentReply reply, DocumentSerializer buf); + + /** + * This method decodes a reply from the given byte buffer. You are guaranteed to only receive byte buffers + * generated by a previous call to {@link #doEncode(DocumentReply, com.yahoo.document.serialization.DocumentSerializer)}. + * + * <p> + * This method is NOT exception safe. Return null to signal failure. + * + * @param buf The byte buffer to read from. + * @return The decoded reply. + */ + protected abstract DocumentReply doDecode(DocumentDeserializer buf); + + public boolean encode(Routable obj, DocumentSerializer out) { + if (!(obj instanceof DocumentReply)) { + throw new AssertionError( + "Document reply factory (" + getClass().getName() + ") registered for incompatible " + + "routable type " + obj.getType() + "(" + obj.getClass().getName() + ")."); + } + DocumentReply reply = (DocumentReply)obj; + out.putByte(null, (byte)(reply.getPriority().getValue())); + return doEncode(reply, out); + } + + public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { + byte pri = in.getByte(null); + DocumentReply reply = doDecode(in); + if (reply != null) { + reply.setPriority(DocumentProtocol.getPriority(pri)); + } + return reply; + } + } + + public static class CreateVisitorMessageFactory extends DocumentMessageFactory { + + @Override + @SuppressWarnings("deprecation") + protected DocumentMessage doDecode(DocumentDeserializer buf) { + CreateVisitorMessage msg = new CreateVisitorMessage(); + msg.setLibraryName(decodeString(buf)); + msg.setInstanceId(decodeString(buf)); + msg.setControlDestination(decodeString(buf)); + msg.setDataDestination(decodeString(buf)); + msg.setDocumentSelection(decodeString(buf)); + msg.setMaxPendingReplyCount(buf.getInt(null)); + + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + long reversed = buf.getLong(null); + long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) | + ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) | + ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | + ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); + msg.getBuckets().add(new BucketId(rawid)); + } + + msg.setFromTimestamp(buf.getLong(null)); + msg.setToTimestamp(buf.getLong(null)); + msg.setVisitRemoves(buf.getByte(null) == (byte)1); + buf.getByte(null); // removed feature "visitHeadersOnly" + msg.setVisitInconsistentBuckets(buf.getByte(null) == (byte)1); + + size = buf.getInt(null); + for (int i = 0; i < size; i++) { + String key = decodeString(buf); + int sz = buf.getInt(null); + msg.getParameters().put(key, buf.getBytes(null, sz)); + } + + msg.setVisitorOrdering(buf.getInt(null)); + msg.setMaxBucketsPerVisitor(buf.getInt(null)); + msg.setVisitorDispatcherVersion(50); + return msg; + } + + @Override + @SuppressWarnings("deprecation") + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + CreateVisitorMessage msg = (CreateVisitorMessage)obj; + encodeString(msg.getLibraryName(), buf); + encodeString(msg.getInstanceId(), buf); + encodeString(msg.getControlDestination(), buf); + encodeString(msg.getDataDestination(), buf); + encodeString(msg.getDocumentSelection(), buf); + buf.putInt(null, msg.getMaxPendingReplyCount()); + + buf.putInt(null, msg.getBuckets().size()); + for (BucketId id : msg.getBuckets()) { + long rawid = id.getRawId(); + long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | + ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | + ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | + ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); + buf.putLong(null, reversed); + } + + buf.putLong(null, msg.getFromTimestamp()); + buf.putLong(null, msg.getToTimestamp()); + buf.putByte(null, msg.getVisitRemoves() ? (byte)1 : (byte)0); + buf.putByte(null, (byte)0); // removed feature "visitHeadersOnly" + buf.putByte(null, msg.getVisitInconsistentBuckets() ? (byte)1 : (byte)0); + + buf.putInt(null, msg.getParameters().size()); + for (Map.Entry<String, byte[]> pairs : msg.getParameters().entrySet()) { + encodeString(pairs.getKey(), buf); + byte[] b = pairs.getValue(); + buf.putInt(null, b.length); + buf.put(null, b); + } + + buf.putInt(null, msg.getVisitorOrdering()); + buf.putInt(null, msg.getMaxBucketsPerVisitor()); + return true; + } + } + + public static class CreateVisitorReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); + reply.setLastBucket(new BucketId(buf.getLong(null))); + + VisitorStatistics vs = new VisitorStatistics(); + vs.setBucketsVisited(buf.getInt(null)); + vs.setDocumentsVisited(buf.getLong(null)); + vs.setBytesVisited(buf.getLong(null)); + vs.setDocumentsReturned(buf.getLong(null)); + vs.setBytesReturned(buf.getLong(null)); + vs.setSecondPassDocumentsReturned(buf.getLong(null)); + vs.setSecondPassBytesReturned(buf.getLong(null)); + reply.setVisitorStatistics(vs); + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + CreateVisitorReply reply = (CreateVisitorReply)obj; + buf.putLong(null, reply.getLastBucket().getRawId()); + buf.putInt(null, reply.getVisitorStatistics().getBucketsVisited()); + buf.putLong(null, reply.getVisitorStatistics().getDocumentsVisited()); + buf.putLong(null, reply.getVisitorStatistics().getBytesVisited()); + buf.putLong(null, reply.getVisitorStatistics().getDocumentsReturned()); + buf.putLong(null, reply.getVisitorStatistics().getBytesReturned()); + buf.putLong(null, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); + buf.putLong(null, reply.getVisitorStatistics().getSecondPassBytesReturned()); + return true; + } + } + + public static class DestroyVisitorMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + DestroyVisitorMessage msg = new DestroyVisitorMessage(); + msg.setInstanceId(decodeString(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + DestroyVisitorMessage msg = (DestroyVisitorMessage)obj; + encodeString(msg.getInstanceId(), buf); + return true; + } + } + + public static class DestroyVisitorReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_DESTROYVISITOR); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class DocumentListMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + DocumentListMessage msg = new DocumentListMessage(); + msg.setBucketId(new BucketId(buf.getLong(null))); + int len = buf.getInt(null); + for (int i = 0; i < len; i++) { + msg.getDocuments().add(new DocumentListEntry(buf)); + } + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + DocumentListMessage msg = (DocumentListMessage)obj; + buf.putLong(null, msg.getBucketId().getRawId()); + buf.putInt(null, msg.getDocuments().size()); + + for (int i = 0; i < msg.getDocuments().size(); i++) { + msg.getDocuments().get(i).serialize(buf); + } + return true; + } + } + + public static class DocumentListReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTLIST); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class DocumentSummaryMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + DocumentSummaryMessage msg = new DocumentSummaryMessage(); + msg.setDocumentSummary(new DocumentSummary(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + return false; // not supported + } + } + + public static class DocumentSummaryReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTSUMMARY); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class EmptyBucketsMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + EmptyBucketsMessage msg = new EmptyBucketsMessage(); + int size = buf.getInt(null); + for (int i = 0; i < size; ++i) { + msg.getBucketIds().add(new BucketId(buf.getLong(null))); + } + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + EmptyBucketsMessage msg = (EmptyBucketsMessage)obj; + buf.putInt(null, msg.getBucketIds().size()); + for (BucketId bid : msg.getBucketIds()) { + buf.putLong(null, bid.getRawId()); + } + return true; + } + } + + public static class EmptyBucketsReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_EMPTYBUCKETS); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class GetBucketListMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + GetBucketListMessage msg = new GetBucketListMessage(); + msg.setBucketId(new BucketId(buf.getLong(null))); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + GetBucketListMessage msg = (GetBucketListMessage)obj; + buf.putLong(null, msg.getBucketId().getRawId()); + return true; + } + } + + public static class GetBucketListReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + GetBucketListReply reply = new GetBucketListReply(); + int len = buf.getInt(null); + for (int i = 0; i < len; i++) { + GetBucketListReply.BucketInfo info = new GetBucketListReply.BucketInfo(); + info.bucket = new BucketId(buf.getLong(null)); + info.bucketInformation = decodeString(buf); + reply.getBuckets().add(info); + } + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + GetBucketListReply reply = (GetBucketListReply)obj; + buf.putInt(null, reply.getBuckets().size()); + for (GetBucketListReply.BucketInfo info : reply.getBuckets()) { + buf.putLong(null, info.bucket.getRawId()); + encodeString(info.bucketInformation, buf); + } + return true; + } + } + + public static class GetBucketStateMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + GetBucketStateMessage msg = new GetBucketStateMessage(); + msg.setBucketId(new BucketId(buf.getLong(null))); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + GetBucketStateMessage msg = (GetBucketStateMessage)obj; + buf.putLong(null, msg.getBucketId().getRawId()); + return true; + } + } + + public static class GetBucketStateReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + GetBucketStateReply reply = new GetBucketStateReply(); + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + reply.getBucketState().add(new DocumentState(buf)); + } + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + GetBucketStateReply reply = (GetBucketStateReply)obj; + buf.putInt(null, reply.getBucketState().size()); + for (DocumentState stat : reply.getBucketState()) { + stat.serialize(buf); + } + return true; + } + } + + public static class GetDocumentMessageFactory extends DocumentMessageFactory { + + @Override + @SuppressWarnings("deprecation") + protected DocumentMessage doDecode(DocumentDeserializer buf) { + GetDocumentMessage msg = new GetDocumentMessage(); + msg.setDocumentId(new DocumentId(buf)); + buf.getInt(null); // removed feature "flags"; ignore + return msg; + } + + @Override + @SuppressWarnings("deprecation") + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + GetDocumentMessage msg = (GetDocumentMessage)obj; + msg.getDocumentId().serialize(buf); + buf.putInt(null, 0); // removed feature "flags" + return true; + } + } + + public static class GetDocumentReplyFactory extends DocumentReplyFactory { + + private final LazyDecoder decoder = new LazyDecoder() { + + public void decode(Routable obj, DocumentDeserializer buf) { + GetDocumentReply reply = (GetDocumentReply)obj; + + Document doc = null; + byte flag = buf.getByte(null); + if (flag != 0) { + doc = Document.createDocument(buf); + reply.setDocument(doc); + } + long lastModified = buf.getLong(null); + reply.setLastModified(lastModified); + if (doc != null) { + doc.setLastModified(lastModified); + } + } + }; + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + GetDocumentReply reply = new GetDocumentReply(decoder, buf); + + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + GetDocumentReply reply = (GetDocumentReply)obj; + if (reply.getSerializedBuffer() != null) { + buf.put(null, reply.getSerializedBuffer()); + } else { + Document document = reply.getDocument(); + buf.putByte(null, (byte)(document == null ? 0 : 1)); + if (document != null) { + document.serialize(buf); + } + buf.putLong(null, reply.getLastModified()); + } + return true; + } + } + + public static class MapVisitorMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + MapVisitorMessage msg = new MapVisitorMessage(); + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + String key = decodeString(buf); + String value = decodeString(buf); + msg.getData().put(key, value); + } + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + MapVisitorMessage msg = (MapVisitorMessage)obj; + buf.putInt(null, msg.getData().size()); + for (Map.Entry<String, String> pairs : msg.getData().entrySet()) { + encodeString(pairs.getKey(), buf); + encodeString(pairs.getValue(), buf); + } + return true; + } + } + + public static class MapVisitorReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_MAPVISITOR); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class BatchDocumentUpdateMessageFactory extends DocumentMessageFactory { + + private final LazyDecoder decoder = new LazyDecoder() { + + public void decode(Routable obj, DocumentDeserializer buf) { + BatchDocumentUpdateMessage msg = (BatchDocumentUpdateMessage)obj; + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + msg.addUpdate(new DocumentUpdate(buf)); + } + } + }; + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + long userId = buf.getLong(null); + String group = decodeString(buf); + + if (group.length() > 0) { + return new BatchDocumentUpdateMessage(group, decoder, buf); + } else { + return new BatchDocumentUpdateMessage(userId, decoder, buf); + } + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + BatchDocumentUpdateMessage msg = (BatchDocumentUpdateMessage)obj; + + if (msg.getSerializedBuffer() != null) { + buf.put(null, msg.getSerializedBuffer()); + } else { + if (msg.getUserId() != null) { + buf.putLong(null, msg.getUserId()); + } else { + buf.putLong(null, 0); + } + + if (msg.getGroup() != null) { + encodeString(msg.getGroup(), buf); + } else { + encodeString("", buf); + } + + buf.putInt(null, msg.getUpdates().size()); + for (int i = 0; i < msg.getUpdates().size(); i++) { + msg.getUpdates().get(i).serialize(buf); + } + } + + return true; + } + } + + public static class BatchDocumentUpdateReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + BatchDocumentUpdateReply rep = new BatchDocumentUpdateReply(); + rep.setHighestModificationTimestamp(buf.getLong(null)); + int size = buf.getInt(null); + rep.getDocumentsNotFound().ensureCapacity(size); + for (int i = 0; i < size; ++i) { + rep.getDocumentsNotFound().add(buf.getByte(null) == 1); + } + return rep; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + BatchDocumentUpdateReply rep = (BatchDocumentUpdateReply)obj; + buf.putLong(null, rep.getHighestModificationTimestamp()); + buf.putInt(null, rep.getDocumentsNotFound().size()); + for (int i = 0; i < rep.getDocumentsNotFound().size(); ++i) { + buf.putByte(null, (byte)(rep.getDocumentsNotFound().get(i) ? 1 : 0)); + } + return true; + } + } + + public static class PutDocumentMessageFactory extends DocumentMessageFactory { + protected void decodeInto(PutDocumentMessage msg, DocumentDeserializer buf) { + msg.setDocumentPut(new DocumentPut(Document.createDocument(buf))); + msg.setTimestamp(buf.getLong(null)); + } + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buffer) { + final LazyDecoder decoder = (obj, buf) -> { + decodeInto((PutDocumentMessage) obj, buf); + }; + + return new PutDocumentMessage(decoder, buffer); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + PutDocumentMessage msg = (PutDocumentMessage)obj; + if (msg.getSerializedBuffer() != null) { + buf.put(null, msg.getSerializedBuffer()); + } else { + msg.getDocumentPut().getDocument().serialize(buf); + buf.putLong(null, msg.getTimestamp()); + } + return true; + } + } + + public static class PutDocumentReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + WriteDocumentReply rep = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT); + rep.setHighestModificationTimestamp(buf.getLong(null)); + return rep; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + WriteDocumentReply rep = (WriteDocumentReply)obj; + buf.putLong(null, rep.getHighestModificationTimestamp()); + return true; + } + } + + public static class RemoveDocumentMessageFactory extends DocumentMessageFactory { + protected void decodeInto(RemoveDocumentMessage msg, DocumentDeserializer buf) { + msg.setDocumentId(new DocumentId(buf)); + } + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + RemoveDocumentMessage msg = new RemoveDocumentMessage(); + decodeInto(msg, buf); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + RemoveDocumentMessage msg = (RemoveDocumentMessage)obj; + msg.getDocumentId().serialize(buf); + return true; + } + } + + public static class RemoveDocumentReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + RemoveDocumentReply reply = new RemoveDocumentReply(); + byte flag = buf.getByte(null); + reply.setWasFound(flag != 0); + reply.setHighestModificationTimestamp(buf.getLong(null)); + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + RemoveDocumentReply reply = (RemoveDocumentReply)obj; + buf.putByte(null, (byte)(reply.wasFound() ? 1 : 0)); + buf.putLong(null, reply.getHighestModificationTimestamp()); + return true; + } + } + + public static class RemoveLocationMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + return new RemoveLocationMessage(decodeString(buf)); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + RemoveLocationMessage msg = (RemoveLocationMessage)obj; + encodeString(msg.getDocumentSelection(), buf); + return true; + } + } + + public static class RemoveLocationReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new DocumentReply(DocumentProtocol.REPLY_REMOVELOCATION); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class SearchResultMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + SearchResultMessage msg = new SearchResultMessage(); + msg.setSearchResult(new SearchResult(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + return false; // not supported + } + } + + public static class QueryResultMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + QueryResultMessage msg = new QueryResultMessage(); + msg.setSearchResult(new SearchResult(buf)); + msg.setSummary(new DocumentSummary(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + return false; // not supported + } + } + + public static class SearchResultReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_SEARCHRESULT); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class QueryResultReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_QUERYRESULT); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class StatBucketMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + StatBucketMessage msg = new StatBucketMessage(); + msg.setBucketId(new BucketId(buf.getLong(null))); + msg.setDocumentSelection(decodeString(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + StatBucketMessage msg = (StatBucketMessage)obj; + buf.putLong(null, msg.getBucketId().getRawId()); + encodeString(msg.getDocumentSelection(), buf); + return true; + } + } + + public static class StatBucketReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + StatBucketReply reply = new StatBucketReply(); + reply.setResults(decodeString(buf)); + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + StatBucketReply reply = (StatBucketReply)obj; + encodeString(reply.getResults(), buf); + return true; + } + } + + public static class UpdateDocumentMessageFactory extends DocumentMessageFactory { + protected void decodeInto(UpdateDocumentMessage msg, DocumentDeserializer buf) { + msg.setDocumentUpdate(new DocumentUpdate(buf)); + msg.setOldTimestamp(buf.getLong(null)); + msg.setNewTimestamp(buf.getLong(null)); + } + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buffer) { + final LazyDecoder decoder = (obj, buf) -> { + decodeInto((UpdateDocumentMessage) obj, buf); + }; + + return new UpdateDocumentMessage(decoder, buffer); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + UpdateDocumentMessage msg = (UpdateDocumentMessage)obj; + if (msg.getSerializedBuffer() != null) { + buf.put(null, msg.getSerializedBuffer()); + } else { + msg.getDocumentUpdate().serialize(buf); + buf.putLong(null, msg.getOldTimestamp()); + buf.putLong(null, msg.getNewTimestamp()); + } + return true; + } + } + + public static class UpdateDocumentReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + UpdateDocumentReply rep = new UpdateDocumentReply(); + byte flag = buf.getByte(null); + rep.setWasFound(flag != 0); + rep.setHighestModificationTimestamp(buf.getLong(null)); + return rep; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + UpdateDocumentReply rep = (UpdateDocumentReply)obj; + buf.putByte(null, (byte)(rep.wasFound() ? 1 : 0)); + buf.putLong(null, rep.getHighestModificationTimestamp()); + return true; + } + } + + public static class VisitorInfoMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + VisitorInfoMessage msg = new VisitorInfoMessage(); + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + long reversed = buf.getLong(null); + long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) | + ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) | + ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | + ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); + msg.getFinishedBuckets().add(new BucketId(rawid)); + } + + msg.setErrorMessage(decodeString(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + VisitorInfoMessage msg = (VisitorInfoMessage)obj; + buf.putInt(null, msg.getFinishedBuckets().size()); + for (BucketId id : msg.getFinishedBuckets()) { + long rawid = id.getRawId(); + long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | + ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | + ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | + ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); + buf.putLong(null, reversed); + } + encodeString(msg.getErrorMessage(), buf); + return true; + } + } + + public static class VisitorInfoReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_VISITORINFO); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class WrongDistributionReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + WrongDistributionReply reply = new WrongDistributionReply(); + reply.setSystemState(decodeString(buf)); + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + WrongDistributionReply reply = (WrongDistributionReply)obj; + encodeString(reply.getSystemState(), buf); + return true; + } + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories51.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories51.java new file mode 100755 index 00000000000..0005dc8d296 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories51.java @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; +import com.yahoo.document.DocumentId; +import com.yahoo.document.serialization.DocumentDeserializer; +import com.yahoo.document.serialization.DocumentSerializer; + +import java.util.Map; + +/** + * This class encapsulates all the {@link RoutableFactory} classes needed to implement serialization for the document + * protocol. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. + * + */ +public abstract class RoutableFactories51 extends RoutableFactories50 { + + public static class CreateVisitorMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + CreateVisitorMessage msg = new CreateVisitorMessage(); + msg.setLibraryName(decodeString(buf)); + msg.setInstanceId(decodeString(buf)); + msg.setControlDestination(decodeString(buf)); + msg.setDataDestination(decodeString(buf)); + msg.setDocumentSelection(decodeString(buf)); + msg.setMaxPendingReplyCount(buf.getInt(null)); + + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + long reversed = buf.getLong(null); + long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) | + ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) | + ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | + ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); + msg.getBuckets().add(new BucketId(rawid)); + } + + msg.setFromTimestamp(buf.getLong(null)); + msg.setToTimestamp(buf.getLong(null)); + msg.setVisitRemoves(buf.getByte(null) == (byte)1); + msg.setFieldSet(decodeString(buf)); + msg.setVisitInconsistentBuckets(buf.getByte(null) == (byte)1); + + size = buf.getInt(null); + for (int i = 0; i < size; i++) { + String key = decodeString(buf); + int sz = buf.getInt(null); + msg.getParameters().put(key, buf.getBytes(null, sz)); + } + + msg.setVisitorOrdering(buf.getInt(null)); + msg.setMaxBucketsPerVisitor(buf.getInt(null)); + msg.setVisitorDispatcherVersion(50); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + CreateVisitorMessage msg = (CreateVisitorMessage)obj; + encodeString(msg.getLibraryName(), buf); + encodeString(msg.getInstanceId(), buf); + encodeString(msg.getControlDestination(), buf); + encodeString(msg.getDataDestination(), buf); + encodeString(msg.getDocumentSelection(), buf); + buf.putInt(null, msg.getMaxPendingReplyCount()); + + buf.putInt(null, msg.getBuckets().size()); + for (BucketId id : msg.getBuckets()) { + long rawid = id.getRawId(); + long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | + ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | + ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | + ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); + buf.putLong(null, reversed); + } + + buf.putLong(null, msg.getFromTimestamp()); + buf.putLong(null, msg.getToTimestamp()); + buf.putByte(null, msg.getVisitRemoves() ? (byte)1 : (byte)0); + encodeString(msg.getFieldSet(), buf); + buf.putByte(null, msg.getVisitInconsistentBuckets() ? (byte)1 : (byte)0); + + buf.putInt(null, msg.getParameters().size()); + for (Map.Entry<String, byte[]> pairs : msg.getParameters().entrySet()) { + encodeString(pairs.getKey(), buf); + byte[] b = pairs.getValue(); + buf.putInt(null, b.length); + buf.put(null, b); + } + + buf.putInt(null, msg.getVisitorOrdering()); + buf.putInt(null, msg.getMaxBucketsPerVisitor()); + return true; + } + } + + public static class GetDocumentMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + return new GetDocumentMessage(new DocumentId(buf), decodeString(buf)); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + GetDocumentMessage msg = (GetDocumentMessage)obj; + msg.getDocumentId().serialize(buf); + encodeString(msg.getFieldSet(), buf); + return true; + } + } + + public static class DocumentIgnoredReplyFactory extends DocumentReplyFactory { + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new DocumentIgnoredReply(); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java new file mode 100644 index 00000000000..035309f373f --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.google.common.annotations.Beta; +import com.yahoo.document.TestAndSetCondition; +import com.yahoo.document.serialization.DocumentDeserializer; +import com.yahoo.document.serialization.DocumentSerializer; + +import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.decodeString; +import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.encodeString; + +/** + * @author Vegard Sjonfjell + */ + +@Beta +public abstract class RoutableFactories52 extends RoutableFactories51 { + protected static class PutDocumentMessageFactory extends RoutableFactories51.PutDocumentMessageFactory { + @Override + protected void decodeInto(PutDocumentMessage msg, DocumentDeserializer buf) { + super.decodeInto(msg, buf); + decodeTasCondition(msg, buf); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + if (!super.doEncode(obj, buf)) { + return false; + } + + // If the serialized buffer exists, the test and set condition has already been encoded + if (((PutDocumentMessage)obj).getSerializedBuffer() == null) { + encodeTasCondition(buf, (TestAndSetMessage) obj); + } + + return true; + } + } + + protected static class RemoveDocumentMessageFactory extends RoutableFactories51.RemoveDocumentMessageFactory { + @Override + protected void decodeInto(RemoveDocumentMessage msg, DocumentDeserializer buf) { + super.decodeInto(msg, buf); + decodeTasCondition(msg, buf); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + if (!super.doEncode(obj, buf)) { + return false; + } + + encodeTasCondition(buf, (TestAndSetMessage) obj); + return true; + } + } + + protected static class UpdateDocumentMessageFactory extends RoutableFactories51.UpdateDocumentMessageFactory { + @Override + protected void decodeInto(UpdateDocumentMessage msg, DocumentDeserializer buf) { + super.decodeInto(msg, buf); + decodeTasCondition(msg, buf); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + if (!super.doEncode(obj, buf)) { + return false; + } + + // If the serialized buffer exists, the test and set condition has already been encoded + if (((UpdateDocumentMessage)obj).getSerializedBuffer() == null) { + encodeTasCondition(buf, (TestAndSetMessage) obj); + } + + return true; + } + } + + static void decodeTasCondition(TestAndSetMessage msg, DocumentDeserializer buf) { + msg.setCondition(new TestAndSetCondition(decodeString(buf))); + } + + static void encodeTasCondition(DocumentSerializer buf, TestAndSetMessage msg) { + encodeString(msg.getCondition().getSelection(), buf); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java new file mode 100755 index 00000000000..d5c30101baf --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.serialization.DocumentDeserializer; +import com.yahoo.document.serialization.DocumentSerializer; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.messagebus.Routable; + +/** + * <p>This interface defines the necessary methods of a routable factory that can be plugged into a {@link + * DocumentProtocol} using the {@link DocumentProtocol#putRoutableFactory(int, RoutableFactory, + * com.yahoo.component.VersionSpecification)} method. </p> + * + * <p>Notice that no routable type is passed to the + * {@link #decode(DocumentDeserializer, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)} method, so + * you may NOT share a factory across multiple routable types. To share serialization logic between factory use a common + * superclass or composition with a common serialization utility.</p> + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface RoutableFactory { + + /** + * <p>This method encodes the content of the given routable into a byte buffer that can later be decoded using the + * {@link #decode(DocumentDeserializer, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)} method.</p> <p>Return false to signal failure.</p> + * <p>This method is NOT exception safe.</p> + * + * @param obj The routable to encode. + * @param out The buffer to write into. + * @return True if the routable could be encoded. + */ + boolean encode(Routable obj, DocumentSerializer out); + + /** + * <p>This method decodes the given byte bufer to a routable.</p> <p>Return false to signal failure.</p> <p>This + * method is NOT exception safe.</p> + * + * @param in The buffer to read from. + * @param loadTypes The LoadTypeSet to inject into the Routable. + * @return The decoded routable. + */ + Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes); + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java new file mode 100755 index 00000000000..6f044a1951f --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java @@ -0,0 +1,237 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.component.Version;
+import com.yahoo.component.VersionSpecification;
+import com.yahoo.concurrent.CopyOnWriteHashMap;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.serialization.*;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.log.LogLevel;
+import com.yahoo.messagebus.Routable;
+
+import java.util.*;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * This class encapsulates the logic required to map routable type and version to a corresponding {@link
+ * RoutableFactory}. It is owned and accessed through a {@link DocumentProtocol} instance. This class uses a factory
+ * cache to reduce the latency of matching version specifications to actual versions when resolving factories.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+final class RoutableRepository {
+
+ private static final Logger log = Logger.getLogger(RoutableRepository.class.getName());
+ private final CopyOnWriteHashMap<Integer, VersionMap> factoryTypes = new CopyOnWriteHashMap<>();
+ private final CopyOnWriteHashMap<CacheKey, RoutableFactory> cache = new CopyOnWriteHashMap<>();
+ private LoadTypeSet loadTypes;
+
+ public RoutableRepository(LoadTypeSet set) {
+ loadTypes = set;
+ }
+
+ /**
+ * Decodes a {@link Routable} from the given byte array. This uses the content of the byte array to dispatch the
+ * decode request to the appropriate {@link RoutableFactory} that was previously registered.
+ *
+ * If a routable can not be decoded, this method returns null.
+ *
+ * @param version The version of the encoded routable.
+ * @param data The byte array containing the encoded routable.
+ * @return The decoded routable.
+ */
+ Routable decode(DocumentTypeManager docMan, Version version, byte[] data) {
+ if (data == null || data.length == 0) {
+ log.log(LogLevel.ERROR, "Received empty byte array for deserialization.");
+ return null;
+ }
+ DocumentDeserializer in;
+
+ if (version.getMajor() >= 5) {
+ in = DocumentDeserializerFactory.createHead(docMan, GrowableByteBuffer.wrap(data));
+ } else {
+ in = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(data));
+ }
+
+ int type = in.getInt(null);
+ RoutableFactory factory = getFactory(version, type);
+ if (factory == null) {
+ log.log(LogLevel.ERROR, "No routable factory found for routable type " + type +
+ " (version " + version + ").");
+ return null;
+ }
+ Routable ret = factory.decode(in, loadTypes);
+ if (ret == null) {
+ log.log(LogLevel.ERROR, "Routable factory " + factory.getClass().getName() + " failed to deserialize " +
+ "routable of type " + type + " (version " + version + ").");
+ log.log(LogLevel.ERROR, Arrays.toString(data));
+ return null;
+ }
+ return ret;
+ }
+
+ /**
+ * Encodes a {@link Routable} into a byte array. This dispatches the encode request to the appropriate {@link
+ * RoutableFactory} that was previously registered.
+ *
+ * If a routable can not be encoded, this method returns an empty byte array.
+ *
+ * @param version The version to encode the routable as.
+ * @param obj The routable to encode.
+ * @return The byte array containing the encoded routable.
+ */
+ byte[] encode(Version version, Routable obj) {
+ int type = obj.getType();
+ RoutableFactory factory = getFactory(version, type);
+ if (factory == null) {
+ log.log(LogLevel.ERROR, "No routable factory found for routable type " + type +
+ " (version " + version + ").");
+ return new byte[0];
+ }
+ DocumentSerializer out;
+
+ if (version.getMajor() >= 5) {
+ out = DocumentSerializerFactory.createHead(new GrowableByteBuffer(8192));
+ } else {
+ out = DocumentSerializerFactory.create42(new GrowableByteBuffer(8192));
+ }
+
+ out.putInt(null, type);
+ if (!factory.encode(obj, out)) {
+ log.log(LogLevel.ERROR, "Routable factory " + factory.getClass().getName() + " failed to serialize " +
+ "routable of type " + type + " (version " + version + ").");
+ return new byte[0];
+ }
+ byte[] ret = new byte[out.getBuf().position()];
+ out.getBuf().rewind();
+ out.getBuf().get(ret);
+ return ret;
+ }
+
+ /**
+ * Registers a routable factory for a given version and routable type.
+ *
+ * @param version The version specification that the given factory supports.
+ * @param type The routable type that the given factory supports.
+ * @param factory The routable factory to register.
+ */
+ void putFactory(VersionSpecification version, int type, RoutableFactory factory) {
+ VersionMap versionMap = factoryTypes.get(type);
+ if (versionMap == null) {
+ versionMap = new VersionMap();
+
+ factoryTypes.put(type, versionMap);
+ }
+ if (versionMap.putFactory(version, factory)) {
+ cache.clear();
+ }
+ }
+
+ /**
+ * Returns the routable factory for a given version and routable type.
+ *
+ * @param version The version that the factory must support.
+ * @param type The routable type that the factory must support.
+ * @return The routable factory matching the criteria, or null.
+ */
+ RoutableFactory getFactory(Version version, int type) {
+ CacheKey cacheKey = new CacheKey(version, type);
+ RoutableFactory factory = cache.get(cacheKey);
+ if (factory != null) {
+ return factory;
+ }
+ VersionMap versionMap = factoryTypes.get(type);
+ if (versionMap == null) {
+ return null;
+ }
+ factory = versionMap.getFactory(version);
+ if (factory == null) {
+ return null;
+ }
+ cache.put(cacheKey, factory);
+ return factory;
+ }
+
+ /**
+ * Returns a list of routable types that support the given version.
+ *
+ * @param version The version to return types for.
+ * @return The list of supported types.
+ */
+ List<Integer> getRoutableTypes(Version version) {
+ List<Integer> ret = new ArrayList<>();
+ for (Map.Entry<Integer, VersionMap> entry : factoryTypes.entrySet()) {
+ if (entry.getValue().getFactory(version) != null) {
+ ret.add(entry.getKey());
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Internal helper class that implements a map from {@link VersionSpecification} to {@link RoutableFactory}.
+ */
+ private static class VersionMap {
+
+ final Map<VersionSpecification, RoutableFactory> factoryVersions = new HashMap<>();
+
+ boolean putFactory(VersionSpecification version, RoutableFactory factory) {
+ return factoryVersions.put(version, factory) == null;
+ }
+
+ RoutableFactory getFactory(Version version) {
+ VersionSpecification versionSpec = version.toSpecification();
+
+ // Retrieve the factory with the highest version lower than or equal to actual version
+ return factoryVersions.entrySet().stream()
+ // Drop factories that have a higher version than actual version
+ .filter(entry -> entry.getKey().compareTo(versionSpec) <= 0)
+
+ // Get the factory with the highest version
+ .max((entry1, entry2) -> entry1.getKey().compareTo(entry2.getKey()))
+ .map(Map.Entry::getValue)
+
+ // Return factory or null if no suitable factory found
+ .orElse(null);
+ }
+ }
+
+ /**
+ * Internal helper class that implements a cache key for mapping a {@link Version} and routable type to a {@link
+ * RoutableFactory}.
+ */
+ private static class CacheKey {
+
+ final Version version;
+ final int type;
+
+ public CacheKey(Version version, int type) {
+ this.version = version;
+ this.type = type;
+ }
+
+ @Override
+ public int hashCode() {
+ return version.hashCode() + type;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof CacheKey)) {
+ return false;
+ }
+ CacheKey rhs = (CacheKey)obj;
+ if (!version.equals(rhs.version)) {
+ return false;
+ }
+ if (type != rhs.type) {
+ return false;
+ }
+ return true;
+ }
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java new file mode 100755 index 00000000000..05e39503308 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java @@ -0,0 +1,148 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class RoutingPolicyFactories {
+
+ static class AndPolicyFactory implements RoutingPolicyFactory {
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ return new ANDPolicy(param);
+ }
+
+
+ public void destroy() {
+ }
+ }
+
+ static class StoragePolicyFactory implements RoutingPolicyFactory {
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ return new StoragePolicy(param);
+ }
+
+ public void destroy() {
+ }
+ }
+
+ static class ContentPolicyFactory implements RoutingPolicyFactory {
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ return new ContentPolicy(param);
+ }
+
+ public void destroy() {
+ }
+ }
+
+ static class MessageTypePolicyFactory implements RoutingPolicyFactory {
+ private final String configId;
+
+ public MessageTypePolicyFactory(String configId) {
+ this.configId = configId;
+ }
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ return new MessageTypePolicy((param == null || param.isEmpty()) ? configId : param);
+ }
+
+ public void destroy() {
+ }
+ }
+
+ static class DocumentRouteSelectorPolicyFactory implements RoutingPolicyFactory {
+
+ private final String configId;
+
+ public DocumentRouteSelectorPolicyFactory(String configId) {
+ this.configId = configId;
+ }
+
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ DocumentRouteSelectorPolicy ret = new DocumentRouteSelectorPolicy((param == null || param.isEmpty()) ?
+ configId : param);
+ String error = ret.getError();
+ if (error != null) {
+ return new ErrorPolicy(error);
+ }
+ return ret;
+ }
+
+
+ public void destroy() {
+ }
+ }
+
+ static class ExternPolicyFactory implements RoutingPolicyFactory {
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ ExternPolicy ret = new ExternPolicy(param);
+ String error = ret.getError();
+ if (error != null) {
+ return new ErrorPolicy(error);
+ }
+ return ret;
+ }
+
+
+ public void destroy() {
+ }
+ }
+
+ static class LocalServicePolicyFactory implements RoutingPolicyFactory {
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ return new LocalServicePolicy(param);
+ }
+
+
+ public void destroy() {
+ }
+ }
+
+ static class RoundRobinPolicyFactory implements RoutingPolicyFactory {
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ return new RoundRobinPolicy();
+ }
+
+
+ public void destroy() {
+ }
+ }
+
+ static class LoadBalancerPolicyFactory implements RoutingPolicyFactory {
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ return new LoadBalancerPolicy(param);
+ }
+
+
+ public void destroy() {
+ }
+ }
+
+ static class SearchColumnPolicyFactory implements RoutingPolicyFactory {
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ return new SearchColumnPolicy(param);
+ }
+
+
+ public void destroy() {
+ }
+ }
+
+ static class SearchRowPolicyFactory implements RoutingPolicyFactory {
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ return new SearchRowPolicy(param);
+ }
+
+
+ public void destroy() {
+ }
+ }
+
+ static class SubsetServicePolicyFactory implements RoutingPolicyFactory {
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ return new SubsetServicePolicy(param);
+ }
+
+
+ public void destroy() {
+ }
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactory.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactory.java new file mode 100755 index 00000000000..93e1aa7fbb5 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactory.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.messagebus.routing.RoutingPolicy; + +/** + * This interface defines the necessary methods of a routing policy factory that can be plugged into a {@link + * DocumentProtocol} using the {@link DocumentProtocol#putRoutingPolicyFactory(String, RoutingPolicyFactory)} method. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface RoutingPolicyFactory { + + /** + * This method creates and returns a routing policy that corresponds to the implementing class, using the given + * parameter string. There is only ever one instance of a routing policy for a given name and parameter combination, + * and because of this the policies must be state-less beyond what can be derived from the parameter string. Because + * there is only a single thread running route resolution within message bus, it is not necessary to make policies + * thread-safe. For more information see {@link RoutingPolicy}. + * + * Do NOT throw exceptions out of this method because that will cause the running thread to die, just return null to + * signal failure instead. + * + * @param param The parameter to use when creating the policy. + * @return The created routing policy. + */ + public DocumentProtocolRoutingPolicy createPolicy(String param); + + /** + * Destroys this factory and frees up any resources it has held. Making further calls on a destroyed + * factory causes a runtime exception. + */ + public void destroy(); +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java new file mode 100755 index 00000000000..3bfa85ac4d5 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.documentapi.metrics.DocumentProtocolMetricSet;
+import com.yahoo.messagebus.routing.RoutingPolicy;
+import com.yahoo.log.LogLevel;
+
+import java.util.Map;
+import java.util.logging.Logger;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+class RoutingPolicyRepository {
+
+ private static final Logger log = Logger.getLogger(RoutingPolicyRepository.class.getName());
+ private final Map<String, RoutingPolicyFactory> factories = new ConcurrentHashMap<String, RoutingPolicyFactory>();
+ private final DocumentProtocolMetricSet metrics;
+
+ RoutingPolicyRepository(DocumentProtocolMetricSet metrics) {
+ this.metrics = metrics;
+ }
+
+ /**
+ * Registers a routing policy factory for a given name.
+ *
+ * @param name The name of the factory to register.
+ * @param factory The factory to register.
+ */
+ void putFactory(String name, RoutingPolicyFactory factory) {
+ factories.put(name, factory);
+ }
+
+ /**
+ * Returns the routing policy factory for a given name.
+ *
+ * @param name The name of the factory to return.
+ * @return The routing policy factory matching the criteria, or null.
+ */
+ RoutingPolicyFactory getFactory(String name) {
+ return factories.get(name);
+ }
+
+ /**
+ * Creates and returns a routing policy using the named factory and the given parameter.
+ *
+ * @param name The name of the factory to use.
+ * @param param The parameter to pass to the factory.
+ * @return The created policy.
+ */
+ RoutingPolicy createPolicy(String name, String param) {
+ RoutingPolicyFactory factory = getFactory(name);
+ if (factory == null) {
+ log.log(LogLevel.ERROR, "No routing policy factory found for name '" + name + "'.");
+ return null;
+ }
+ DocumentProtocolRoutingPolicy ret;
+ try {
+ ret = factory.createPolicy(param);
+ } catch (Exception e) {
+ ret = new ErrorPolicy(e.getMessage());
+ }
+
+ if (ret.getMetrics() != null) {
+ metrics.routingPolicyMetrics.addMetric(ret.getMetrics());
+ }
+
+ if (ret == null) {
+ log.log(LogLevel.ERROR, "Routing policy factory " + factory.getClass().getName() + " failed to create a " +
+ "routing policy for parameter '" + name + "'.");
+ return null;
+ }
+ return ret;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchColumnPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchColumnPolicy.java new file mode 100644 index 00000000000..3a329e6e6cf --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchColumnPolicy.java @@ -0,0 +1,183 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.DocumentId; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.EmptyReply; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RoutingContext; +import com.yahoo.messagebus.routing.RoutingNodeIterator; +import com.yahoo.vdslib.BucketDistribution; + +import java.util.*; +import java.util.logging.Logger; + +/** + * <p>This policy implements the logic to select recipients for a single search column. It has 2 different modes of + * operation;</p> + * + * <ol> + * <li>If the "maxbadparts" parameter is 0, select recipient based on document id hash and use + * shared merge logic. Do not allow any out-of-service replies.</li> + * <li>Else do best-effort validation of system + * state. This means; + * <ol> + * <li>if the message is sending to all recipients (typicall start- and + * end-of-feed), allow at most "maxbadparts" out-of-service replies,</li> + * <li>else always allow out-of-service reply by masking it with an empty + * reply.</li> + * </ol> + * </li> + * </ol> + * <p>For systems that allow bad parts, one will not know whether or not feeding + * was a success until the RTX attempts to set the new index live, because it is + * only the RTX that is now able to verify that the service level requirements + * are met. Feeding will still break if a message that was supposed to be sent + * to all recipients receives more than "maxbadparts" out-of-service replies, + * according to (2.a) above.</p> + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SearchColumnPolicy implements DocumentProtocolRoutingPolicy { + + private static Logger log = Logger.getLogger(SearchColumnPolicy.class.getName()); + private BucketIdFactory factory = new BucketIdFactory(); + private Map<Integer, BucketDistribution> distributions = new HashMap<Integer, BucketDistribution>(); + private int maxOOS = 0; // The maximum OUT_OF_SERVICE replies to hide. + + public static final int DEFAULT_NUM_BUCKET_BITS = 16; + + /** + * Constructs a new policy object for the given parameter string. The string can be null or empty, which is a + * request to not allow any bad columns. + * + * @param param The maximum number of allowed bad columns. + */ + public SearchColumnPolicy(String param) { + if (param != null && param.length() > 0) { + try { + maxOOS = Integer.parseInt(param); + } catch (NumberFormatException e) { + log.log(LogLevel.WARNING, "Parameter '" + param + "' could not be parsed as an integer.", e); + } + if (maxOOS < 0) { + log.log(LogLevel.WARNING, "Ignoring a request to set the maximum number of OOS replies to " + maxOOS + + " because it makes no sense. This routing policy will not allow any recipient" + + " to be out of service."); + } + } + } + + @Override + public void select(RoutingContext context) { + List<Route> recipients = context.getMatchedRecipients(); + if (recipients == null || recipients.size() == 0) { + return; + } + DocumentId id = null; + BucketId bucketId = null; + Message msg = context.getMessage(); + switch (msg.getType()) { + + case DocumentProtocol.MESSAGE_PUTDOCUMENT: + id = ((PutDocumentMessage)msg).getDocumentPut().getDocument().getId(); + break; + + case DocumentProtocol.MESSAGE_GETDOCUMENT: + id = ((GetDocumentMessage)msg).getDocumentId(); + break; + + case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: + id = ((RemoveDocumentMessage)msg).getDocumentId(); + break; + + case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: + id = ((UpdateDocumentMessage)msg).getDocumentUpdate().getId(); + break; + + case DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE: + bucketId = ((BatchDocumentUpdateMessage)msg).getBucketId(); + break; + + case DocumentProtocol.MESSAGE_GETBUCKETSTATE: + bucketId = ((GetBucketStateMessage)msg).getBucketId(); + break; + + default: + throw new UnsupportedOperationException("Message type '" + msg.getType() + "' not supported."); + } + if (bucketId == null && id != null) { + bucketId = factory.getBucketId(id); + } + int recipient = getRecipient(bucketId, recipients.size()); + context.addChild(recipients.get(recipient)); + context.setSelectOnRetry(true); + if (maxOOS > 0) { + context.addConsumableError(ErrorCode.SERVICE_OOS); + } + } + + @Override + public void merge(RoutingContext context) { + if (maxOOS > 0) { + if (context.getNumChildren() > 1) { + Set<Integer> oosReplies = new HashSet<Integer>(); + int idx = 0; + for (RoutingNodeIterator it = context.getChildIterator(); + it.isValid(); it.next()) + { + Reply ref = it.getReplyRef(); + if (ref.hasErrors() && DocumentProtocol.hasOnlyErrorsOfType(ref, ErrorCode.SERVICE_OOS)) { + oosReplies.add(idx); + } + ++idx; + } + if (oosReplies.size() <= maxOOS) { + DocumentProtocol.merge(context, oosReplies); + return; // may the rtx be with you + } + } else { + Reply ref = context.getChildIterator().getReplyRef(); + if (ref.hasErrors() && DocumentProtocol.hasOnlyErrorsOfType(ref, ErrorCode.SERVICE_OOS)) { + context.setReply(new EmptyReply()); + return; // god help us all + } + } + } + DocumentProtocol.merge(context); + } + + /** + * Returns the recipient index for the given bucket id. This updates the shared internal distribution map, so it + * needs to be synchronized. + * + * @param bucketId The bucket whose recipient to return. + * @param numRecipients The number of recipients being distributed to. + * @return The recipient to use. + */ + private synchronized int getRecipient(BucketId bucketId, int numRecipients) { + BucketDistribution distribution = distributions.get(numRecipients); + if (distribution == null) { + distribution = new BucketDistribution(1, DEFAULT_NUM_BUCKET_BITS); + distribution.setNumColumns(numRecipients); + distributions.put(numRecipients, distribution); + } + return distribution.getColumn(bucketId); + } + + @Override + public void destroy() { + // empty + } + + @Override + public MetricSet getMetrics() { + return null; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchResultMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchResultMessage.java new file mode 100644 index 00000000000..9dd4085ffb3 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchResultMessage.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.vdslib.SearchResult; + +public class SearchResultMessage extends VisitorMessage { + + private SearchResult searchResult = null; + + public SearchResult getResult() { + return searchResult; + } + + public void setSearchResult(SearchResult result) { + searchResult = result; + } + + @Override + public DocumentReply createReply() { + return new VisitorReply(DocumentProtocol.REPLY_SEARCHRESULT); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_SEARCHRESULT; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchRowPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchRowPolicy.java new file mode 100755 index 00000000000..d36d3ee1e4c --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchRowPolicy.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.messagebus.ErrorCode;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.metrics.MetricSet;
+import com.yahoo.messagebus.routing.RoutingContext;
+import com.yahoo.messagebus.routing.RoutingNodeIterator;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Logger;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SearchRowPolicy implements DocumentProtocolRoutingPolicy {
+
+ private static Logger log = Logger.getLogger(SearchRowPolicy.class.getName());
+ private int minOk = 0; // Hide OUT_OF_SERVICE as long as this number of replies are something else.
+
+ /**
+ * Creates a search row policy that wraps the underlying search group policy in case the parameter is something
+ * other than an empty string.
+ *
+ * @param param The number of minimum non-OOS replies that this policy requires.
+ */
+ public SearchRowPolicy(String param) {
+ if (param != null && param.length() > 0) {
+ try {
+ minOk = Integer.parseInt(param);
+ }
+ catch (NumberFormatException e) {
+ log.log(LogLevel.WARNING, "Parameter '" + param + "' could not be parsed as an integer.", e);
+ }
+ if (minOk <= 0) {
+ log.log(LogLevel.WARNING, "Ignoring a request to set the minimum number of OK replies to " + minOk + " " +
+ "because it makes no sense. This routing policy will not allow any recipient " +
+ "to be out of service.");
+ }
+ }
+ }
+
+ @Override
+ public void select(RoutingContext context) {
+ context.addChildren(context.getMatchedRecipients());
+ context.setSelectOnRetry(false);
+ if (minOk > 0) {
+ context.addConsumableError(ErrorCode.SERVICE_OOS);
+ }
+ }
+
+ @Override
+ public void merge(RoutingContext context) {
+ if (minOk > 0) {
+ Set<Integer> oosReplies = new HashSet<Integer>();
+ int idx = 0;
+ for (RoutingNodeIterator it = context.getChildIterator();
+ it.isValid(); it.next())
+ {
+ Reply ref = it.getReplyRef();
+ if (ref.hasErrors() && DocumentProtocol.hasOnlyErrorsOfType(ref, ErrorCode.SERVICE_OOS)) {
+ oosReplies.add(idx);
+ }
+ ++idx;
+ }
+ if (context.getNumChildren() - oosReplies.size() >= minOk) {
+ DocumentProtocol.merge(context, oosReplies);
+ return;
+ }
+ }
+ DocumentProtocol.merge(context);
+ }
+
+ @Override
+ public void destroy() {
+ // empty
+ }
+
+ @Override
+ public MetricSet getMetrics() {
+ return null;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketMessage.java new file mode 100755 index 00000000000..3854637ba5f --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketMessage.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.document.BucketId;
+
+public class StatBucketMessage extends DocumentMessage {
+
+ private BucketId bucketId;
+ private String documentSelection;
+
+ StatBucketMessage() {
+ // need to deserialize into
+ }
+
+ public StatBucketMessage(BucketId bucket, String documentSelection) {
+ this.bucketId = bucket;
+ this.documentSelection = documentSelection;
+ }
+
+ public BucketId getBucketId() {
+ return bucketId;
+ }
+
+ void setBucketId(BucketId bucket) {
+ bucketId = bucket;
+ }
+
+ public String getDocumentSelection() {
+ return documentSelection;
+ }
+
+ void setDocumentSelection(String documentSelection) {
+ this.documentSelection = documentSelection;
+ }
+
+ @Override
+ public DocumentReply createReply() {
+ return new StatBucketReply();
+ }
+
+ @Override
+ public int getApproxSize() {
+ return super.getApproxSize() + 8 + documentSelection.length();
+ }
+
+ @Override
+ public boolean hasSequenceId() {
+ return true;
+ }
+
+ @Override
+ public long getSequenceId() {
+ return bucketId.getRawId();
+ }
+
+ @Override
+ public int getType() {
+ return DocumentProtocol.MESSAGE_STATBUCKET;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketReply.java new file mode 100755 index 00000000000..43c369106d1 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketReply.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+public class StatBucketReply extends DocumentReply {
+
+ private String results = "";
+
+ public StatBucketReply() {
+ super(DocumentProtocol.REPLY_STATBUCKET);
+ }
+
+ public String getResults() {
+ return results;
+ }
+
+ public void setResults(String result) {
+ results = result;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java new file mode 100644 index 00000000000..ae554b8b0c3 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java @@ -0,0 +1,469 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.jrt.slobrok.api.IMirror; +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.*; +import com.yahoo.vdslib.distribution.Distribution; +import com.yahoo.vdslib.state.ClusterState; +import com.yahoo.vdslib.state.Node; +import com.yahoo.vdslib.state.NodeType; +import com.yahoo.vdslib.state.State; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.logging.Logger; + +/** + * Routing policy to determine which distributor in a storage cluster to send data to. + * Using different key=value parameters separated by semicolon (";"), the user can control which cluster to send to. + * + * cluster=[clusterName] (Mandatory, determines the cluster name) + * config=[config] (Optional, a comma separated list of config servers to use. Used to talk to clusters not defined in this vespa application) + * slobrokconfigid=[id] (Optional, use given config id for slobrok instead of default) + * clusterconfigid=[id] (Optional, use given config id for distribution instead of default) + * + * @author <a href="mailto:humbe@yahoo-inc.com">Haakon Humberset</a> + */ +public class StoragePolicy extends ExternalSlobrokPolicy { + + private static final Logger log = Logger.getLogger(StoragePolicy.class.getName()); + public static final String owningBucketStates = "uim"; + public static final String upStates = "ui"; + + /** This class merely generates slobrok a host pattern for a given distributor. */ + public static class SlobrokHostPatternGenerator { + private final String clusterName; + public SlobrokHostPatternGenerator(String clusterName) { this.clusterName = clusterName; } + + /** + * Find host pattern of the hosts that are valid targets for this request. + * @param distributor Set to -1 if any distributor is valid target. + */ + public String getDistributorHostPattern(Integer distributor) { + return "storage/cluster." + clusterName + "/distributor/" + (distributor == null ? "*" : distributor) + "/default"; + } + } + + /** Helper class to match a host pattern with node to use. */ + public abstract static class HostFetcher { + private int requiredUpPercentageToSendToKnownGoodNodes = 60; + private List<Integer> validRandomTargets = new ArrayList<>(); + private int totalTargets = 1; + protected final Random randomizer = new Random(12345); // Use same randomizer each time to make unit testing easy. + + public void setRequiredUpPercentageToSendToKnownGoodNodes(int percent) { this.requiredUpPercentageToSendToKnownGoodNodes = percent; } + + public void updateValidTargets(ClusterState state) { + List<Integer> validRandomTargets = new ArrayList<>(); + for (int i=0; i<state.getNodeCount(NodeType.DISTRIBUTOR); ++i) { + if (state.getNodeState(new Node(NodeType.DISTRIBUTOR, i)).getState().oneOf(upStates)) validRandomTargets.add(i); + } + this.validRandomTargets = validRandomTargets; + this.totalTargets = state.getNodeCount(NodeType.DISTRIBUTOR); + } + public abstract String getTargetSpec(Integer distributor, RoutingContext context); + public String getRandomTargetSpec(RoutingContext context) { + // Try to use list of random targets, if at least X % of the nodes are up + while (100 * validRandomTargets.size() / totalTargets >= requiredUpPercentageToSendToKnownGoodNodes) { + int randIndex = randomizer.nextInt(validRandomTargets.size()); + String targetSpec = getTargetSpec(validRandomTargets.get(randIndex), context); + if (targetSpec != null) { + context.trace(3, "Sending to random node seen up in cluster state"); + return targetSpec; + } + validRandomTargets.remove(randIndex); + } + context.trace(3, "Too few nodes seen up in state. Sending totally random."); + return getTargetSpec(null, context); + } + public void close() {} + } + + /** Host fetcher using a slobrok mirror to find the hosts. */ + public static class SlobrokHostFetcher extends HostFetcher { + private final SlobrokHostPatternGenerator patternGenerator; + ExternalSlobrokPolicy policy; + + public SlobrokHostFetcher(SlobrokHostPatternGenerator patternGenerator, ExternalSlobrokPolicy policy) { + this.patternGenerator = patternGenerator; + this.policy = policy; + } + + private Mirror.Entry[] getEntries(String hostPattern, RoutingContext context) { + return policy.lookup(context, hostPattern); + } + + private String convertSlobrokNameToSessionName(String slobrokName) { return slobrokName + "/default"; } + + public IMirror getMirror(RoutingContext context) { return context.getMirror(); } + + public String getTargetSpec(Integer distributor, RoutingContext context) { + Mirror.Entry[] arr = getEntries(patternGenerator.getDistributorHostPattern(distributor), context); + if (arr.length == 0) return null; + if (distributor != null) { + if (arr.length == 1) { + return convertSlobrokNameToSessionName(arr[0].getSpec()); + } else { + log.log(LogLevel.WARNING, "Got " + arr.length + " matches for a distributor."); + } + } else { + return convertSlobrokNameToSessionName(arr[randomizer.nextInt(arr.length)].getSpec()); + } + return null; + } + } + + /** Class parsing the semicolon separated parameter string and exposes the appropriate value to the policy. */ + public static class Parameters { + protected String clusterName = null; + protected String distributionConfigId = null; + protected SlobrokHostPatternGenerator slobrokHostPatternGenerator = null; + + public Parameters(Map<String, String> params) { + clusterName = params.get("cluster"); + distributionConfigId = params.get("clusterconfigid"); + slobrokHostPatternGenerator = createPatternGenerator(); + if (clusterName == null) throw new IllegalArgumentException("Required parameter cluster with clustername not set"); + } + + public String getDistributionConfigId() { + return (distributionConfigId == null ? "storage/cluster." + clusterName : distributionConfigId); + } + public String getClusterName() { + return clusterName; + } + public SlobrokHostPatternGenerator createPatternGenerator() { + return new SlobrokHostPatternGenerator(getClusterName()); + } + public HostFetcher createHostFetcher(ExternalSlobrokPolicy policy) { + return new SlobrokHostFetcher(slobrokHostPatternGenerator, policy); + } + public Distribution createDistribution(ExternalSlobrokPolicy policy) { + return (policy.configSources != null ? + new Distribution(getDistributionConfigId(), new ConfigSourceSet(policy.configSources)) + : new Distribution(getDistributionConfigId())); + } + + /** + * When we have gotten this amount of failures from a node (Any kind of failures). We try to send to a random other node, just to see if the + * failure was related to node being bad. (Hard to detect from failure) + */ + public int getAttemptRandomOnFailuresLimit() { return 5; } + + /** + * If we receive more than this number of wrong distribution replies with old cluster states, we throw the current cached state and takes the + * old one. This guards us against version resets. + */ + public int maxOldClusterStatesSeenBeforeThrowingCachedState() { return 20; } + + /** + * When getting new cluster states we update good nodes. If we have more than this percentage of up nodes, we send to up nodes instead of totally random. + * (To avoid hitting trashing bad nodes still in slobrok) + */ + public int getRequiredUpPercentageToSendToKnownGoodNodes() { return 60; } + } + + /** Helper class to get the bucket identifier of a message. */ + public static class BucketIdCalculator { + private static final BucketIdFactory factory = new BucketIdFactory(); + + @SuppressWarnings("deprecation") + private BucketId getBucketId(Message msg) { + switch (msg.getType()) { + case DocumentProtocol.MESSAGE_PUTDOCUMENT: return factory.getBucketId(((PutDocumentMessage)msg).getDocumentPut().getDocument().getId()); + case DocumentProtocol.MESSAGE_GETDOCUMENT: return factory.getBucketId(((GetDocumentMessage)msg).getDocumentId()); + case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: return factory.getBucketId(((RemoveDocumentMessage)msg).getDocumentId()); + case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: return factory.getBucketId(((UpdateDocumentMessage)msg).getDocumentUpdate().getId()); + case DocumentProtocol.MESSAGE_GETBUCKETLIST: return ((GetBucketListMessage)msg).getBucketId(); + case DocumentProtocol.MESSAGE_STATBUCKET: return ((StatBucketMessage)msg).getBucketId(); + case DocumentProtocol.MESSAGE_CREATEVISITOR: return ((CreateVisitorMessage)msg).getBuckets().get(0); + case DocumentProtocol.MESSAGE_REMOVELOCATION: return ((RemoveLocationMessage)msg).getBucketId(); + case DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE: return ((BatchDocumentUpdateMessage)msg).getBucketId(); + default: + log.log(LogLevel.ERROR, "Message type '" + msg.getType() + "' not supported."); + return null; + } + } + + public BucketId handleBucketIdCalculation(RoutingContext context) { + BucketId id = getBucketId(context.getMessage()); + if (id == null || id.getRawId() == 0) { + Reply reply = new EmptyReply(); + reply.addError(new Error(ErrorCode.APP_FATAL_ERROR, "No bucket id available in message.")); + context.setReply(reply); + } + return id; + } + } + + /** Class handling the logic of picking a distributor */ + public static class DistributorSelectionLogic { + /** Class that tracks a failure of a given type per node. */ + public static class InstabilityChecker { + private List<Integer> nodeFailures = new ArrayList<>(); + private int failureLimit; + + public InstabilityChecker(int failureLimit) { this.failureLimit = failureLimit; } + + public boolean tooManyFailures(int nodeIndex) { + if (nodeFailures.size() > nodeIndex && nodeFailures.get(nodeIndex) > failureLimit) { + nodeFailures.set(nodeIndex, 0); + return true; + } else { + return false; + } + } + + public void addFailure(Integer calculatedDistributor) { + while (nodeFailures.size() <= calculatedDistributor) nodeFailures.add(0); + nodeFailures.set(calculatedDistributor, nodeFailures.get(calculatedDistributor) + 1); + } + } + /** Message context class. Contains data we want to inspect about a request at reply time. */ + private static class MessageContext { + Integer calculatedDistributor; + ClusterState usedState; + + public MessageContext(ClusterState usedState) { this.usedState = usedState; } + + public String toString() { + return "Context(Distributor " + calculatedDistributor + + ", state version " + usedState.getVersion() + ")"; + } + } + + private final HostFetcher hostFetcher; + private final Distribution distribution; + private final InstabilityChecker persistentFailureChecker; + private ClusterState cachedClusterState = null; + private int oldClusterVersionGottenCount = 0; + private final int maxOldClusterVersionBeforeSendingRandom; // Reset cluster version protection + + public DistributorSelectionLogic(Parameters params, ExternalSlobrokPolicy policy) { + this.hostFetcher = params.createHostFetcher(policy); + this.hostFetcher.setRequiredUpPercentageToSendToKnownGoodNodes(params.getRequiredUpPercentageToSendToKnownGoodNodes()); + this.distribution = params.createDistribution(policy); + persistentFailureChecker = new InstabilityChecker(params.getAttemptRandomOnFailuresLimit()); + maxOldClusterVersionBeforeSendingRandom = params.maxOldClusterStatesSeenBeforeThrowingCachedState(); + } + + public void destroy() { + hostFetcher.close(); + distribution.close(); + } + + public String getTargetSpec(RoutingContext context, BucketId bucketId) { + String sendRandomReason = null; + MessageContext messageContext = new MessageContext(cachedClusterState); + context.setContext(messageContext); + if (cachedClusterState != null) { // If we have a cached cluster state (regular case), we use that to calculate correct node. + try{ + Integer target = distribution.getIdealDistributorNode(cachedClusterState, bucketId, owningBucketStates); + // If we have had too many failures towards existing node, reset failure count and send to random + if (persistentFailureChecker.tooManyFailures(target)) { + sendRandomReason = "Too many failures detected versus distributor " + target + ". Sending to random instead of using cached state."; + target = null; + } + // If we have found a target, and the target exist in slobrok, send to it. + if (target != null) { + messageContext.calculatedDistributor = target; + String targetSpec = hostFetcher.getTargetSpec(target, context); + if (targetSpec != null) { + if (context.shouldTrace(1)) { + context.trace(1, "Using distributor " + messageContext.calculatedDistributor + " for " + + bucketId + " as our state version is " + cachedClusterState.getVersion()); + } + messageContext.usedState = cachedClusterState; + return targetSpec; + } else { + sendRandomReason = "Want to use distributor " + messageContext.calculatedDistributor + " but it is not in slobrok. Sending to random."; + log.log(LogLevel.DEBUG, "Target distributor is not in slobrok"); + } + } + } catch (Distribution.TooFewBucketBitsInUseException e) { + Reply reply = new WrongDistributionReply(cachedClusterState.toString(true)); + reply.addError(new Error(DocumentProtocol.ERROR_WRONG_DISTRIBUTION, + "Too few distribution bits used for given cluster state")); + context.setReply(reply); + return null; + } catch (Distribution.NoDistributorsAvailableException e) { + log.log(LogLevel.DEBUG, "No distributors available; clearing cluster state"); + cachedClusterState = null; + sendRandomReason = "No distributors available. Sending to random distributor."; + } + } else { + sendRandomReason = "No cluster state cached. Sending to random distributor."; + } + if (context.shouldTrace(1)) { + context.trace(1, sendRandomReason != null ? sendRandomReason : "Sending to random distributor for unknown reason"); + } + return hostFetcher.getRandomTargetSpec(context); + } + + public void handleWrongDistribution(WrongDistributionReply reply, RoutingContext routingContext) { + MessageContext context = (MessageContext) routingContext.getContext(); + ClusterState newState; + try { + newState = new ClusterState(reply.getSystemState()); + } catch (Exception e) { + reply.getTrace().trace(1, "Error when parsing system state string " + reply.getSystemState()); + return; + } + if (cachedClusterState != null && cachedClusterState.getVersion() > newState.getVersion()) { + if (++oldClusterVersionGottenCount >= maxOldClusterVersionBeforeSendingRandom) { + oldClusterVersionGottenCount = 0; + cachedClusterState = null; + } + } + if (context.usedState != null && newState.getVersion() <= context.usedState.getVersion()) { + reply.setRetryDelay(-1); + } else { + reply.setRetryDelay(0); + } + if (context.calculatedDistributor == null) { + if (cachedClusterState == null) { + if (reply.getTrace().shouldTrace(1)) { + reply.getTrace().trace(1, "Message sent to * with no previous state, received version " + newState.getVersion()); + } + } else if (newState.getVersion() == cachedClusterState.getVersion()) { + if (reply.getTrace().shouldTrace(1)) { + reply.getTrace().trace(1, "Message sent to * found that cluster state version " + newState.getVersion() + " was correct."); + } + } else if (newState.getVersion() > cachedClusterState.getVersion()) { + if (reply.getTrace().shouldTrace(1)) { + reply.getTrace().trace(1, "Message sent to * updated cluster state to version " + newState.getVersion()); + } + } else { + if (reply.getTrace().shouldTrace(1)) { + reply.getTrace().trace(1, "Message sent to * retrieved older cluster state version " + newState.getVersion()); + } + } + } else { + if (context.usedState == null) { + String msg = "Used state must be set as distributor is calculated. Bug."; + reply.getTrace().trace(1, msg); + log.log(LogLevel.ERROR, msg); + } else if (newState.getVersion() == context.usedState.getVersion()) { + String msg = "Message sent to distributor " + context.calculatedDistributor + + " retrieved cluster state version " + newState.getVersion() + + " which was the state we used to calculate distributor as target last time."; + reply.getTrace().trace(1, msg); + log.log(LogLevel.WARNING, msg); + } else if (newState.getVersion() > context.usedState.getVersion()) { + if (reply.getTrace().shouldTrace(1)) { + reply.getTrace().trace(1, "Message sent to distributor " + context.calculatedDistributor + + " updated cluster state from version " + context.usedState.getVersion() + + " to " + newState.getVersion()); + } + } else { + if (reply.getTrace().shouldTrace(1)) { + reply.getTrace().trace(1, "Message sent to distributor " + context.calculatedDistributor + + " returned older cluster state version " + newState.getVersion()); + } + } + } + if (cachedClusterState == null || newState.getVersion() >= cachedClusterState.getVersion()) { + cachedClusterState = newState; + if (newState.getClusterState().equals(State.UP)) { + hostFetcher.updateValidTargets(newState); + } + } else if (newState.getVersion() + 2000000000 < cachedClusterState.getVersion()) { + cachedClusterState = null; + } else if (context.calculatedDistributor != null) { + persistentFailureChecker.addFailure(context.calculatedDistributor); + } + } + public void handleErrorReply(Reply reply, Object untypedContext) { + MessageContext messageContext = (MessageContext) untypedContext; + if (messageContext.calculatedDistributor != null) { + persistentFailureChecker.addFailure(messageContext.calculatedDistributor); + if (reply.getTrace().shouldTrace(1)) { + reply.getTrace().trace(1, "Failed with " + messageContext.toString()); + } + } + } + } + + private final BucketIdCalculator bucketIdCalculator = new BucketIdCalculator(); + private DistributorSelectionLogic distributorSelectionLogic = null; + private Parameters parameters; + + /** Constructor used in production. */ + public StoragePolicy(String param) { + this(parse(param)); + } + + public StoragePolicy(Map<String, String> params) { + this(new Parameters(params), params); + } + + /** Constructor specifying a bit more in detail, so we can override what needs to be overridden in tests */ + public StoragePolicy(Parameters p, Map<String, String> params) { + super(params); + parameters = p; + } + + @Override + public void init() { + super.init(); + this.distributorSelectionLogic = new DistributorSelectionLogic(parameters, this); + } + + @Override + public void doSelect(RoutingContext context) { + if (context.shouldTrace(1)) { + context.trace(1, "Selecting route"); + } + + BucketId bucketId = bucketIdCalculator.handleBucketIdCalculation(context); + if (context.hasReply()) return; + + String targetSpec = distributorSelectionLogic.getTargetSpec(context, bucketId); + if (context.hasReply()) return; + if (targetSpec != null) { + Route route = new Route(context.getRoute()); + route.setHop(0, new Hop().addDirective(new VerbatimDirective(targetSpec))); + context.addChild(route); + } else { + context.setError(ErrorCode.NO_ADDRESS_FOR_SERVICE, + "Could not resolve any distributors to send to in cluster " + parameters.clusterName); + } + } + + @Override + public void merge(RoutingContext context) { + RoutingNodeIterator it = context.getChildIterator(); + Reply reply = it.removeReply(); + + if (reply instanceof WrongDistributionReply) { + distributorSelectionLogic.handleWrongDistribution((WrongDistributionReply) reply, context); + } else if (reply.hasErrors()) { + distributorSelectionLogic.handleErrorReply(reply, context.getContext()); + } else if (reply instanceof WriteDocumentReply) { + if (context.shouldTrace(9)) { + context.trace(9, "Modification timestamp: " + ((WriteDocumentReply)reply).getHighestModificationTimestamp()); + } + } + context.setReply(reply); + } + + @Override + public void destroy() { + distributorSelectionLogic.destroy(); + } + + @Override + public MetricSet getMetrics() { + return null; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java new file mode 100755 index 00000000000..dc06fe7042d --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java @@ -0,0 +1,145 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.jrt.slobrok.api.Mirror;
+import com.yahoo.log.LogLevel;
+import com.yahoo.messagebus.metrics.MetricSet;
+import com.yahoo.messagebus.routing.*;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+
+/**
+ * This policy implements the logic to select a subset of services that matches a slobrok pattern.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SubsetServicePolicy implements DocumentProtocolRoutingPolicy {
+
+ private static Logger log = Logger.getLogger(SubsetServicePolicy.class.getName());
+ private final int subsetSize;
+ private final Map<String, CacheEntry> cache = new HashMap<String, CacheEntry>();
+
+ /**
+ * Creates an instance of a subset service policy. The parameter string is parsed as an integer number that is the
+ * number of services to include in the set to choose from.
+ *
+ * @param param The number of services to include in the set.
+ */
+ public SubsetServicePolicy(String param) {
+ int subsetSize = 5;
+ if (param != null && param.length() > 0) {
+ try {
+ subsetSize = Integer.parseInt(param);
+ }
+ catch (NumberFormatException e) {
+ log.log(LogLevel.WARNING, "Parameter '" + param + "' could not be parsed as an integer.", e);
+ }
+ if (subsetSize <= 0) {
+ log.warning("Ignoring a request to set the subset size to " + subsetSize + " because it makes no " +
+ "sense. This routing policy will choose any one matching service.");
+ }
+ } else {
+ log.warning("No parameter given to SubsetService policy, using default value " + subsetSize + ".");
+ }
+ this.subsetSize = subsetSize;
+ }
+
+ // Inherit doc from RoutingPolicy.
+ public void select(RoutingContext ctx) {
+ Route route = new Route(ctx.getRoute());
+ route.setHop(0, getRecipient(ctx));
+ ctx.addChild(route);
+ }
+
+ // Inherit doc from RoutingPolicy.
+ public void merge(RoutingContext ctx) {
+ DocumentProtocol.merge(ctx);
+ }
+
+ /**
+ * Returns the appropriate recipient hop for the given routing context. This method provides synchronized access to
+ * the internal cache.
+ *
+ * @param ctx The routing context.
+ * @return The recipient hop to use.
+ */
+ private Hop getRecipient(RoutingContext ctx) {
+ Hop hop = null;
+ if (subsetSize > 0) {
+ synchronized (this) {
+ CacheEntry entry = update(ctx);
+ if (!entry.recipients.isEmpty()) {
+ if (++entry.offset >= entry.recipients.size()) {
+ entry.offset = 0;
+ }
+ hop = new Hop(entry.recipients.get(entry.offset));
+ }
+ }
+ }
+ if (hop == null) {
+ hop = new Hop(ctx.getRoute().getHop(0));
+ hop.setDirective(ctx.getDirectiveIndex(), new VerbatimDirective("*"));
+ }
+ return hop;
+ }
+
+ /**
+ * Updates and returns the cache entry for the given routing context. This method assumes that synchronization is
+ * handled outside of it.
+ *
+ * @param ctx The routing context.
+ * @return The updated cache entry.
+ */
+ private CacheEntry update(RoutingContext ctx) {
+ String key = getCacheKey(ctx);
+ CacheEntry entry = cache.get(key);
+ if (entry == null) {
+ entry = new CacheEntry();
+ cache.put(key, entry);
+ }
+
+ int upd = ctx.getMirror().updates();
+ if (entry.generation != upd) {
+ entry.generation = upd;
+ entry.recipients.clear();
+
+ Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix());
+ int pos = ctx.getMessageBus().getConnectionSpec().hashCode();
+ for (int i = 0; i < subsetSize && i < arr.length; ++i) {
+ entry.recipients.add(Hop.parse(arr[((pos + i) & Integer.MAX_VALUE) % arr.length].getName()));
+ }
+ }
+ return entry;
+ }
+
+ /**
+ * Returns a cache key for this instance of the policy. Because behaviour is based on the hop in which the policy
+ * occurs, the cache key is the hop string itself.
+ *
+ * @param ctx The routing context.
+ * @return The cache key.
+ */
+ private String getCacheKey(RoutingContext ctx) {
+ return ctx.getRoute().getHop(0).toString();
+ }
+
+ /**
+ * Defines the necessary cache data.
+ */
+ private class CacheEntry {
+ private final List<Hop> recipients = new ArrayList<Hop>();
+ private int generation = 0;
+ private int offset = 0;
+ }
+
+ public void destroy() {
+ }
+
+ public MetricSet getMetrics() {
+ return null;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/TestAndSetMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/TestAndSetMessage.java new file mode 100644 index 00000000000..1ffd956f0ba --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/TestAndSetMessage.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.google.common.annotations.Beta; +import com.yahoo.document.TestAndSetCondition; + +/** + * This class represents messages having an optional "test and set" condition + * + * @author Vegard Sjonfjell + */ +@Beta +public abstract class TestAndSetMessage extends DocumentMessage { + public abstract void setCondition(TestAndSetCondition condition); + public abstract TestAndSetCondition getCondition(); +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentMessage.java new file mode 100755 index 00000000000..8a5a733d026 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentMessage.java @@ -0,0 +1,175 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.TestAndSetCondition; +import com.yahoo.document.serialization.DocumentDeserializer; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UpdateDocumentMessage extends TestAndSetMessage { + + private DocumentDeserializer buffer = null; + private DocumentUpdate update = null; + private long oldTime = 0; + private long newTime = 0; + private LazyDecoder decoder = null; + + /** + * Constructs a new message for deserialization. + */ + UpdateDocumentMessage() { + // empty + } + + /** + * Constructs a new message from a byte buffer. + * @param decoder The decoder to use for deserialization. + * @param buffer A byte buffer that contains a serialized message. + */ + public UpdateDocumentMessage(LazyDecoder decoder, DocumentDeserializer buffer) { + this.decoder = decoder; + this.buffer = buffer; + } + + /** + * Constructs a new document update message. + * + * @param upd The document update to perform. + */ + public UpdateDocumentMessage(DocumentUpdate upd) { + super(); + update = upd; + } + + /** + * Creates an empty UpdateDocumentMessage + */ + public static UpdateDocumentMessage createEmpty() { + return new UpdateDocumentMessage(null); + } + + /** + * This method will make sure that any serialized content is deserialized into proper message content on first + * entry. Any subsequent entry into this function will do nothing. + */ + private void deserialize() { + if (decoder != null && buffer != null) { + decoder.decode(this, buffer); + decoder = null; + buffer = null; + } + } + + /** + * Returns the document update to perform. + * + * @return The update. + */ + public DocumentUpdate getDocumentUpdate() { + deserialize(); + return update; + } + + /** + * Sets the document update to perform. + * + * @param upd The document update to set. + */ + public void setDocumentUpdate(DocumentUpdate upd) { + if (upd == null) { + throw new IllegalArgumentException("Document update can not be null."); + } + buffer = null; + decoder = null; + update = upd; + } + + /** + * Returns the timestamp required for this update to be applied. + * + * @return The document timestamp. + */ + public long getOldTimestamp() { + deserialize(); + return oldTime; + } + + /** + * Sets the timestamp required for this update to be applied. + * + * @param time The timestamp to set. + */ + public void setOldTimestamp(long time) { + buffer = null; + decoder = null; + oldTime = time; + } + + /** + * Returns the timestamp to assign to the updated document. + * + * @return The document timestamp. + */ + public long getNewTimestamp() { + deserialize(); + return newTime; + } + + /** + * Sets the timestamp to assign to the updated document. + * + * @param time The timestamp to set. + */ + public void setNewTimestamp(long time) { + buffer = null; + decoder = null; + newTime = time; + } + + /** + * Returns the raw serialized buffer. This buffer is stored as the message is received from accross the network, and + * deserialized from as soon as a member is requested. This method will return null if the buffer has been decoded. + * + * @return The buffer containing the serialized data for this message, or null. + */ + ByteBuffer getSerializedBuffer() { + return buffer != null ? buffer.getBuf().getByteBuffer() : null; + } + + @Override + public DocumentReply createReply() { + return new UpdateDocumentReply(); + } + + @Override + public boolean hasSequenceId() { + return true; + } + + @Override + public long getSequenceId() { + deserialize(); + return Arrays.hashCode(update.getId().getGlobalId()); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_UPDATEDOCUMENT; + } + + @Override + public TestAndSetCondition getCondition() { + deserialize(); + return update.getCondition(); + } + + @Override + public void setCondition(TestAndSetCondition condition) { + this.update.setCondition(condition); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentReply.java new file mode 100755 index 00000000000..5f091101554 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentReply.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class UpdateDocumentReply extends WriteDocumentReply {
+
+ private boolean found = true;
+
+ /**
+ * Constructs a new reply with no content.
+ */
+ public UpdateDocumentReply() {
+ super(DocumentProtocol.REPLY_UPDATEDOCUMENT);
+ }
+
+ /**
+ * Returns whether or not the document was found and updated.
+ *
+ * @return true if document was found
+ */
+ public boolean wasFound() {
+ return found;
+ }
+
+ /**
+ * Sets whether or not the document was found and updated.
+ *
+ * @param found True if the document was found
+ */
+ public void setWasFound(boolean found) {
+ this.found = found;
+ }
+}
\ No newline at end of file diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/VisitorInfoMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/VisitorInfoMessage.java new file mode 100644 index 00000000000..10e56bb5ef8 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/VisitorInfoMessage.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; + +import java.util.Iterator; +import java.util.Set; +import java.util.TreeSet; + +public class VisitorInfoMessage extends VisitorMessage { + + private Set<BucketId> finishedBuckets = new TreeSet<BucketId>(); + private String errorMessage = ""; + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String s) { + errorMessage = s; + } + + public Set<BucketId> getFinishedBuckets() { + return finishedBuckets; + } + + public void setFinishedBuckets(Set<BucketId> finishedBuckets) { + this.finishedBuckets = finishedBuckets; + } + + @Override + public DocumentReply createReply() { + return new VisitorReply(DocumentProtocol.REPLY_VISITORINFO); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_VISITORINFO; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder().append("VisitorInfoMessage("); + if (finishedBuckets.size() == 0) { + sb.append("No buckets"); + } else if (finishedBuckets.size() == 1) { + sb.append("Bucket ").append(finishedBuckets.iterator().next().toString()); + } else if (finishedBuckets.size() < 65536) { + sb.append(finishedBuckets.size()).append(" buckets:"); + Iterator<BucketId> it = finishedBuckets.iterator(); + for (int i = 0; it.hasNext() && i < 3; ++i) { + sb.append(' ').append(it.next().toString()); + } + if (it.hasNext()) { + sb.append(" ..."); + } + } else { + sb.append("All buckets"); + } + sb.append(", error message '").append(errorMessage).append('\''); + return sb.append(')').toString(); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/VisitorMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/VisitorMessage.java new file mode 100644 index 00000000000..e775ea756a7 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/VisitorMessage.java @@ -0,0 +1,6 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +public abstract class VisitorMessage extends DocumentMessage { + // empty +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/VisitorReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/VisitorReply.java new file mode 100644 index 00000000000..bc62bb578b7 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/VisitorReply.java @@ -0,0 +1,9 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +public class VisitorReply extends WriteDocumentReply { + + public VisitorReply(int type) { + super(type); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/WriteDocumentReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/WriteDocumentReply.java new file mode 100755 index 00000000000..0db02c10b31 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/WriteDocumentReply.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +/** + * This reply class is used by operations that perform writes to VDS/search, + * that is: Put, Remove, Update. + */ +public class WriteDocumentReply extends DocumentAcceptedReply { + + private long highestModificationTimestamp = 0; + + public WriteDocumentReply(int type) { + super(type); + } + + /** + * Returns a unique VDS timestamp so that visiting up to and including that timestamp + * will return a state including this operation but not any operations sent to the same distributor + * after it. For PUT/UPDATE/REMOVE operations this timestamp will be the timestamp of the operation. + * + * @return Returns the modification timestamp. + */ + public long getHighestModificationTimestamp() { + return highestModificationTimestamp; + } + + /** + * Sets the modification timestamp. + */ + public void setHighestModificationTimestamp(long timestamp) { + this.highestModificationTimestamp = timestamp; + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/WrongDistributionReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/WrongDistributionReply.java new file mode 100644 index 00000000000..75e2525289d --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/WrongDistributionReply.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class WrongDistributionReply extends DocumentReply { + + private String systemState; + + public WrongDistributionReply() { + super(DocumentProtocol.REPLY_WRONGDISTRIBUTION); + } + + public WrongDistributionReply(String systemState) { + super(DocumentProtocol.REPLY_WRONGDISTRIBUTION); + this.systemState = systemState; + } + + public String getSystemState() { + return systemState; + } + + public void setSystemState(String state) { + systemState = state; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/package-info.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/package-info.java new file mode 100644 index 00000000000..f3b73bfa8a4 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Argument.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Argument.java new file mode 100755 index 00000000000..3a434eab101 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Argument.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.systemstate.rule;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Argument {
+
+ private final String name;
+ private final String value;
+
+ /**
+ * Constructs a new argument.
+ *
+ * @param name The name of this argument.
+ * @param value The value of this argument.
+ */
+ public Argument(String name, String value) {
+ this.name = name;
+ this.value = value;
+ }
+
+ /**
+ * Returns the name of this argument.
+ *
+ * @return The name.
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Returns the value of this argument.
+ *
+ * @return The value.
+ */
+ public String getValue() {
+ return value;
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Location.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Location.java new file mode 100755 index 00000000000..870a39c0122 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Location.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.systemstate.rule;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Arrays;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Location {
+
+ private List<String> items = new ArrayList<String>();
+
+ /**
+ * Constructs a new location with no items.
+ */
+ public Location() {
+ // empty
+ }
+
+ /**
+ * Constructs a new location based on a location string.
+ *
+ * @param loc The location string to parse.
+ */
+ public Location(String loc) {
+ items.addAll(Arrays.asList(loc.split("/")));
+ normalize();
+ }
+
+ /**
+ * Constructs a new location based on a list of items.
+ *
+ * @param items The components that make up this location.
+ */
+ public Location(List<String> items) {
+ this.items.addAll(items);
+ normalize();
+ }
+
+ /**
+ * Constructs a new location as a copy of another.
+ *
+ * @param loc The location to copy.
+ */
+ public Location(Location loc) {
+ items.addAll(loc.items);
+ }
+
+ /**
+ * Constructs a new location based on a working directory and a list of items.
+ *
+ * @param pwd The path of the working directory.
+ * @param items The components that make up this location.
+ */
+ public Location(Location pwd, List<String> items) {
+ this.items.addAll(pwd.getItems());
+ this.items.addAll(items);
+ normalize();
+ }
+
+ /**
+ * Returns a location object that represents the "next" step along this location path. This means removing the first
+ * elements of this location's items and returning a new location for this sublist.
+ *
+ * @return The next location along this path.
+ */
+ public Location getNext() {
+ List<String> next = new ArrayList<String>(items);
+ next.remove(0);
+ return new Location(next);
+ }
+
+ /**
+ * Returns the components of this location.
+ *
+ * @return The component array.
+ */
+ public List<String> getItems() {
+ return items;
+ }
+
+ /**
+ * Normalizes the items list of this location so that all PREV or THIS locations are replaced by their actual
+ * meaning. This carries some overhead since it is not done in place.
+ *
+ * @return This, to allow chaining.
+ */
+ private Location normalize() {
+ List<String> norm = new ArrayList<String>();
+ for (String item : items) {
+ if (item.equals(NodeState.NODE_PARENT)) {
+ if (norm.size() == 0) {
+ // ignore
+ }
+ else {
+ norm.remove(norm.size() - 1);
+ }
+ }
+ else if (!item.equals(NodeState.NODE_CURRENT)) {
+ norm.add(item);
+ }
+ }
+ items = norm;
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer ret = new StringBuffer();
+ for (int i = 0; i < items.size(); ++i) {
+ ret.append(items.get(i));
+ if (i < items.size() - 1) {
+ ret.append("/");
+ }
+ }
+ return ret.toString();
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java new file mode 100755 index 00000000000..f5920f32119 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java @@ -0,0 +1,310 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.systemstate.rule;
+
+import com.yahoo.log.LogLevel;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class NodeState {
+
+ /** A location string that expresses the use of the PARENT node. */
+ public static final String NODE_PARENT = "..";
+
+ /** A location string that expresses the use of THIS node. */
+ public static final String NODE_CURRENT = ".";
+
+ private static Logger log = Logger.getLogger(NodeState.class.getName());
+ private final Map<String, NodeState> children = new LinkedHashMap<String, NodeState>();
+ private final Map<String, String> state = new LinkedHashMap<String, String>();
+ private NodeState parent = null;
+ private String id = null;
+
+ /**
+ * Creates a node state that no internal content.
+ */
+ public NodeState() {
+ // empty
+ }
+
+ /**
+ * Creates a node state based on a list of argument objects. These arguments are iterated and added to this node's
+ * internal state map.
+ *
+ * @param args The arguments to use as state.
+ */
+ public NodeState(List<Argument> args) {
+ for (Argument arg : args) {
+ setState(arg.getName(), arg.getValue());
+ }
+ }
+
+ /**
+ * Adds a child to this node at the given location. The key can be a location string, in which case the necessary
+ * intermediate node states are created.
+ *
+ * @param key The location at which to add the child.
+ * @param child The child node to add.
+ * @return This, to allow chaining.
+ */
+ public NodeState addChild(String key, NodeState child) {
+ getChild(key, true).copy(child);
+ return this;
+ }
+
+ /**
+ * Returns the child at the given location relative to this.
+ *
+ * @param key The location of the child to return.
+ * @return The child object, null if not found.
+ */
+ public NodeState getChild(String key) {
+ return getChild(key, false);
+ }
+
+ /**
+ * Returns the child at the given location relative to this. This method can be forced to return a child node even
+ * if it does not exist, by adding all intermediate nodes and the target node itself.
+ *
+ * @param key The location of the child to return.
+ * @param force Whether or not to force a return value by creating missing nodes.
+ * @return The child object, null if not found.
+ */
+ public NodeState getChild(String key, boolean force) {
+ if (key == null || key.length() == 0) {
+ return this;
+ }
+ String arr[] = key.split("/", 2);
+ while (arr.length == 2 && arr[0].equals(NODE_CURRENT)) {
+ arr = arr[1].split("/", 2);
+ }
+ if (arr[0].equals(NODE_CURRENT)) {
+ return this;
+ }
+ if (arr[0].equals(NODE_PARENT)) {
+ if (parent == null) {
+ log.log(LogLevel.ERROR, "Location string '" + key + "' requests a parent above the top-most node, " +
+ "returning self to avoid crash.");
+ }
+ return parent.getChild(arr[1], force);
+ }
+ if (!children.containsKey(arr[0])) {
+ if (!force) {
+ return null;
+ }
+ children.put(arr[0], new NodeState());
+ children.get(arr[0]).setParent(this, arr[0]);
+ }
+ if (arr.length == 2) {
+ return children.get(arr[0]).getChild(arr[1], force);
+ }
+ return children.get(arr[0]);
+ }
+
+ /**
+ * Returns the map of child nodes for iteration.
+ *
+ * @return The internal child map.
+ */
+ public Map<String, NodeState> getChildren() {
+ return children;
+ }
+
+ /**
+ * Removes the named child node from this node, and attempts to compact the system state from this node upwards by
+ * removing empty nodes.
+ *
+ * @param key The child to remove.
+ * @return The result of invoking {@link #compact} after the remove.
+ */
+ public NodeState removeChild(String key) {
+ if (key == null || key.length() == 0) {
+ return this;
+ }
+ int pos = key.lastIndexOf("/");
+ if (pos > -1) {
+ NodeState parent = getChild(key.substring(0, pos), false);
+ if (parent != null) {
+ return parent.removeChild(key.substring(pos + 1));
+ }
+ }
+ else {
+ children.remove(key);
+ }
+ return compact();
+ }
+
+ /**
+ * Retrieves some arbitrary state information for a given key. The key can be a location string, in which case the
+ * necessary intermediate nodes are traversed. If the key is not found, this method returns null.
+ *
+ * @param key The name of the state information to return.
+ * @return The value of the state key.
+ */
+ public String getState(String key) {
+ if (key == null || key.length() == 0) {
+ return null;
+ }
+ int pos = key.lastIndexOf("/");
+ if (pos > -1) {
+ NodeState parent = getChild(key.substring(0, pos), false);
+ return parent != null ? parent.getState(key.substring(pos + 1)) : null;
+ }
+ return state.get(key);
+ }
+
+ /**
+ * Sets some arbitrary state data in this node. The key can be a location string, in which case the necessary
+ * intermediate nodes are traversed and even created if missing.
+ *
+ * @param key The key to set.
+ * @param value The value to assign to the key.
+ * @return This, to allow chaining.
+ */
+ public NodeState setState(String key, String value) {
+ if (key == null || key.length() == 0) {
+ return this;
+ }
+ int pos = key.lastIndexOf("/");
+ if (pos > -1) {
+ getChild(key.substring(0, pos), true).setState(key.substring(pos + 1), value);
+ }
+ else {
+ if (value == null || value.length() == 0) {
+ return removeState(key);
+ }
+ else {
+ state.put(key, value);
+ }
+ }
+ return this;
+ }
+
+ /**
+ * Removes the named (key, value) state pair from this node, and attempts to compact the system state from this node
+ * upwards by removing empty nodes.
+ *
+ * @param key The state variable to clear.
+ * @return The result of invoking {@link #compact} after the remove.
+ */
+ public NodeState removeState(String key) {
+ if (key == null || key.length() == 0) {
+ return this;
+ }
+ int pos = key.lastIndexOf("/");
+ if (pos > -1) {
+ NodeState parent = getChild(key.substring(0, pos), false);
+ if (parent != null) {
+ return parent.removeState(key.substring(pos + 1));
+ }
+ }
+ else {
+ state.remove(key);
+ }
+ return compact();
+ }
+
+ /**
+ * Compacts the system state tree from this node upwards. This will delete itself if it has a parent, but no
+ * internal state and no children.
+ *
+ * @return This or the first non-null ancestor, to allow chaining.
+ */
+ private NodeState compact() {
+ if (state.isEmpty() && children.isEmpty()) {
+ if (parent != null) {
+ return parent.removeChild(id);
+ }
+ }
+ return this;
+ }
+
+ /**
+ * Copies the state content of another node state object into this.
+ *
+ * @param node The node state to copy into this.
+ * @return This, to allow chaining.
+ */
+ public NodeState copy(NodeState node) {
+ for (String key : node.state.keySet()) {
+ state.put(key, node.state.get(key));
+ }
+ for (String key : node.children.keySet()) {
+ getChild(key, true).copy(node.children.get(key));
+ }
+ return this;
+ }
+
+ /**
+ * Clears both the internal state and child list, then compacts the tree from this node upwards.
+ *
+ * @return The result of invoking {@link #compact} after the remove.
+ */
+ public NodeState clear() {
+ state.clear();
+ children.clear();
+ return compact();
+ }
+
+ /**
+ * Sets the parent of this node.
+ *
+ * @param parent The parent node.
+ * @param id The identifier of this node as seen in the parent.
+ * @return This, to allow chaining.
+ */
+ public NodeState setParent(NodeState parent, String id) {
+ this.parent = parent;
+ this.id = id;
+ return this;
+ }
+
+ /**
+ * Returns a string representation of this node state.
+ *
+ * @param prefix The prefix to use for this string.
+ * @return A string representation of this.
+ * @throws UnsupportedEncodingException Thrown if the host system does not support UTF-8 encoding.
+ */
+ private String toString(String prefix) throws UnsupportedEncodingException {
+ StringBuffer buf = new StringBuffer();
+ if (!state.isEmpty()) {
+ buf.append(prefix.length() == 0 ? "." : prefix).append("?");
+ String[] arr = state.keySet().toArray(new String[state.keySet().size()]);
+ for (int i = 0; i < arr.length; ++i) {
+ buf.append(arr[i]).append("=").append(URLEncoder.encode(state.get(arr[i]), "UTF-8"));
+ if (i < arr.length - 1) {
+ buf.append("&");
+ }
+ }
+ buf.append(" ");
+ }
+ if (prefix.length() > 0) {
+ prefix += "/";
+ }
+ String[] keys = children.keySet().toArray(new String[children.keySet().size()]);
+ Arrays.sort(keys);
+ for (String loc : keys) {
+ buf.append(children.get(loc).toString(prefix + URLEncoder.encode(loc, "UTF-8")));
+ }
+ return buf.toString();
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return toString("").trim();
+ }
+ catch (UnsupportedEncodingException e) {
+ return e.toString();
+ }
+ }
+}
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/metrics/DocumentProtocolMetricSet.java b/documentapi/src/main/java/com/yahoo/documentapi/metrics/DocumentProtocolMetricSet.java new file mode 100644 index 00000000000..57090687867 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/metrics/DocumentProtocolMetricSet.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.metrics; + +import com.yahoo.messagebus.metrics.MetricSet; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author thomasg + */ +public class DocumentProtocolMetricSet extends MetricSet { + public MetricSet routingPolicyMetrics = new MetricSet("routingpolicies"); + + public DocumentProtocolMetricSet() { + super("document"); + addMetric(routingPolicyMetrics); + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/package-info.java b/documentapi/src/main/java/com/yahoo/documentapi/package-info.java new file mode 100644 index 00000000000..1fa2e8c8375 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.documentapi; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/documentapi/src/main/java/com/yahoo/documentapiclient/.gitignore b/documentapi/src/main/java/com/yahoo/documentapiclient/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapiclient/.gitignore diff --git a/documentapi/src/main/javacc/StateParser.jj b/documentapi/src/main/javacc/StateParser.jj new file mode 100755 index 00000000000..e2acfb446a9 --- /dev/null +++ b/documentapi/src/main/javacc/StateParser.jj @@ -0,0 +1,105 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * A system state parser. + * When this file is changed, do "ant compileparser" to rebuild the parser. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @version $Id: StateParser.jj,v 1.7 2007-11-15 13:24:45 simon Exp $ + */ +options { + CACHE_TOKENS = true; + STATIC = false; + DEBUG_PARSER = false; + IGNORE_CASE = true; + + // Flip for higher performance + ERROR_REPORTING = true; +} + +PARSER_BEGIN(StateParser) + +package com.yahoo.documentapi.messagebus.systemstate.parser; + +import com.yahoo.documentapi.messagebus.systemstate.rule.*; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.List; +import java.util.ArrayList; + +public class StateParser { + +} + +PARSER_END(StateParser) + +TOKEN : +{ + <WHITESPACE: " " | "\t" | "\r" | "\n" | "\f"> | + <SLASH: "/"> | + <DOTDOT: ".."> | + <DOT: "."> | + <ARG: "?"> | + <AND: "&"> | + <EQ: "="> | + <STRING: (<SPACE> | <CODE> | <ALPHANUM>)+> | + <#SPACE: "+"> | + <#CODE: "%" ["0"-"9","A"-"F","a"-"f"] ["0"-"9","A"-"F","a"-"f"]> | + <#ALPHANUM: ["0"-"9","A"-"Z","a"-"z","-",".","_","~"]> +} + +NodeState systemState() throws UnsupportedEncodingException : +{ + NodeState node = new NodeState(); + Location loc, pwd = new Location(); + List<Argument> arg; +} +{ + ( ( <WHITESPACE> )* + ( loc = location(pwd) { arg = null; } + [ <ARG> arg = argumentList() ] ) { if (arg == null) { pwd = new Location(loc); } + else { node.addChild(loc.toString(), new NodeState(arg)); } } )+ + { return node; } +} + +Location location(Location pwd) throws UnsupportedEncodingException : +{ + String item; + List<String> list = new ArrayList<String>(); +} +{ + ( ( <SLASH> { pwd.getItems().clear(); } )? + item = locationItem() { list.add(item); } + ( LOOKAHEAD(2) <SLASH> item = locationItem() { list.add(item); } )* + ( LOOKAHEAD(2) <SLASH> )? ) + { Location ret = new Location(pwd, list); return ret; } +} + +String locationItem() throws UnsupportedEncodingException : +{ + String ret; +} +{ + ( <DOTDOT> { return NodeState.NODE_PARENT; } | + <DOT> { return NodeState.NODE_CURRENT; } | + <STRING> { return URLDecoder.decode(token.image, "UTF-8"); } ) +} + +List<Argument> argumentList () throws UnsupportedEncodingException : +{ + Argument item; + List<Argument> list = new ArrayList<Argument>(); +} +{ + ( item = argument() { list.add(item); } ( <AND> item = argument() { list.add(item); } )* ) + { return list; } +} + +Argument argument() throws UnsupportedEncodingException : +{ + String key, val; +} +{ + ( <STRING> { key = token.image; } <EQ> + <STRING> { val = token.image; } ) + { return new Argument(URLDecoder.decode(key, "UTF-8"), URLDecoder.decode(val, "UTF-8")); } +} diff --git a/documentapi/src/main/resources/configdefinitions/documentrouteselectorpolicy.def b/documentapi/src/main/resources/configdefinitions/documentrouteselectorpolicy.def new file mode 100644 index 00000000000..81606d6f580 --- /dev/null +++ b/documentapi/src/main/resources/configdefinitions/documentrouteselectorpolicy.def @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=2 +namespace=documentapi.messagebus.protocol + +# The name of the route. +route[].name string + +# The document selector for this route. +route[].selector string + +# The feeds that this route accepts. +route[].feed string default="" diff --git a/documentapi/src/test/cfg/documentmanager.cfg b/documentapi/src/test/cfg/documentmanager.cfg new file mode 100644 index 00000000000..eec3a6a06a0 --- /dev/null +++ b/documentapi/src/test/cfg/documentmanager.cfg @@ -0,0 +1,30 @@ +datatype[3] +datatype[0].id -1910204744 +datatype[0].arraytype[0] +datatype[0].weightedsettype[0] +datatype[0].structtype[1] +datatype[0].structtype[0].name music.header +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].field[0] +datatype[0].documenttype[0] +datatype[1].id 993120973 +datatype[1].arraytype[0] +datatype[1].weightedsettype[0] +datatype[1].structtype[1] +datatype[1].structtype[0].name music.body +datatype[1].structtype[0].version 0 +datatype[1].structtype[0].field[1] +datatype[1].structtype[0].field[0].name artist +datatype[1].structtype[0].field[0].id[0] +datatype[1].structtype[0].field[0].datatype 2 +datatype[1].documenttype[0] +datatype[2].id 1412693671 +datatype[2].arraytype[0] +datatype[2].weightedsettype[0] +datatype[2].structtype[0] +datatype[2].documenttype[1] +datatype[2].documenttype[0].name music +datatype[2].documenttype[0].version 0 +datatype[2].documenttype[0].inherits[0] +datatype[2].documenttype[0].headerstruct -1910204744 +datatype[2].documenttype[0].bodystruct 993120973 diff --git a/documentapi/src/test/cfg/messagebus.cfg b/documentapi/src/test/cfg/messagebus.cfg new file mode 100644 index 00000000000..a4fd46136ac --- /dev/null +++ b/documentapi/src/test/cfg/messagebus.cfg @@ -0,0 +1,11 @@ +routingtable[1] +routingtable[0].protocol document +routingtable[0].hop[1] +routingtable[0].hop[0].name Search +routingtable[0].hop[0].selector test/destination/session +routingtable[0].hop[0].recipient[1] +routingtable[0].hop[0].recipient[0] test/destination/session +routingtable[0].route[1] +routingtable[0].route[0].name Route +routingtable[0].route[0].hop[1] +routingtable[0].route[0].hop[0] Search diff --git a/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java new file mode 100755 index 00000000000..99094b09a43 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java @@ -0,0 +1,1540 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi;
+
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.documentapi.ProgressToken;
+import com.yahoo.documentapi.VisitorIterator;
+import junit.framework.TestCase;
+import com.yahoo.document.BucketId;
+import com.yahoo.document.BucketIdFactory;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.Vector;
+
+/**
+ * Tests for VisitorIterator and ProgressToken (kept in one test case because their
+ * interactions are so tightly coupled)
+ * @author <a href="mailto:vekterli@yahoo-inc.com">Tor Brede Vekterli</a>
+ */
+public class VisitorIteratorTestCase extends TestCase {
+
+ public void testIterationSingleBucketUpdate() throws ParseException {
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken progress = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.user = 1234", idFactory, 1, progress);
+
+ assertFalse(progress.hasActive());
+ assertEquals(progress.getPendingBucketCount(), 1);
+ assertEquals(progress.getFinishedBucketCount(), 0);
+ assertEquals(progress.getTotalBucketCount(), 1);
+ assertFalse(iter.isDone());
+ assertTrue(iter.hasNext());
+ assertEquals(iter.getRemainingBucketCount(), 1);
+ VisitorIterator.BucketProgress b1 = iter.getNext();
+ // Upon first getNext of a superbucket, progress == 0
+ assertEquals(b1.getSuperbucket(), new BucketId(32, 1234));
+ assertEquals(b1.getProgress(), new BucketId());
+ assertFalse(iter.hasNext());
+ assertFalse(iter.isDone());
+ assertEquals(iter.getRemainingBucketCount(), 1);
+ // Should only be one active bucket; the one we just got
+ assertEquals(progress.getActiveBucketCount(), 1);
+ // No pending yet
+ assertFalse(progress.hasPending());
+ // Update the bucket with a sub-bucket, moving it from active to pending
+ BucketId sub = new BucketId(b1.getSuperbucket().getUsedBits() + 1, b1.getSuperbucket().getId());
+ iter.update(b1.getSuperbucket(), sub);
+ assertFalse(progress.hasActive());
+ assertEquals(progress.getPendingBucketCount(), 1);
+ assertTrue(iter.hasNext());
+ assertFalse(iter.isDone());
+ assertEquals(iter.getRemainingBucketCount(), 1);
+ // Get the pending bucket
+ VisitorIterator.BucketProgress b2 = iter.getNext();
+ assertEquals(b2.getSuperbucket(), new BucketId(32, 1234));
+ assertEquals(b2.getProgress(), new BucketId(33, 1234));
+ assertFalse(iter.hasNext());
+ assertEquals(progress.getActiveBucketCount(), 1);
+ assertFalse(progress.hasPending());
+ // Now update with progress==super, signalling that the bucket is done
+ iter.update(b1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ assertFalse(progress.hasActive());
+ assertFalse(progress.hasPending());
+ assertFalse(iter.hasNext());
+ assertTrue(iter.isDone());
+ assertTrue(progress.isFinished());
+ assertEquals(progress.getFinishedBucketCount(), 1);
+ assertEquals(iter.getRemainingBucketCount(), 0);
+ }
+
+ public void testProgressSerializationRange() throws ParseException {
+ int distBits = 4;
+
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken progress = new ProgressToken();
+
+ // docsel will be unknown --> entire bucket range will be covered
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, distBits, progress);
+
+ assertEquals(progress.getDistributionBitCount(), distBits);
+ assertTrue(iter.getBucketSource() instanceof VisitorIterator.DistributionRangeBucketSource);
+
+ assertEquals(progress.getFinishedBucketCount(), 0);
+ assertEquals(progress.getTotalBucketCount(), 1 << distBits);
+
+ // First, get+update half of the buckets, marking them as done
+ long bucketCount = 0;
+ long bucketStop = 1 << (distBits - 1);
+
+ while (iter.hasNext() && bucketCount != bucketStop) {
+ VisitorIterator.BucketProgress ids = iter.getNext();
+ iter.update(ids.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ ++bucketCount;
+ }
+ assertEquals(bucketCount, bucketStop);
+ // Should be no buckets in limbo at this point
+ assertFalse(progress.hasActive());
+ assertFalse(progress.hasPending());
+ assertFalse(iter.isDone());
+ assertTrue(iter.hasNext());
+ assertEquals(progress.getFinishedBucketCount(), bucketCount);
+ assertFalse(progress.isFinished());
+
+ StringBuilder desired = new StringBuilder();
+ desired.append("VDS bucket progress file (50.0% completed)\n");
+ desired.append(distBits);
+ desired.append('\n');
+ desired.append(bucketCount); // Finished == cursor for this
+ desired.append('\n');
+ desired.append(bucketCount);
+ desired.append('\n');
+ desired.append(1 << distBits);
+ desired.append('\n');
+
+ assertEquals(desired.toString(), progress.toString());
+
+ // Test import, in which case distribution bits are 1
+ BucketIdFactory idFactory2 = new BucketIdFactory();
+
+ // De-serialization with no pending buckets
+ {
+ ProgressToken progDs = new ProgressToken(progress.toString());
+
+ assertEquals(progDs.getDistributionBitCount(), distBits);
+ assertEquals(progDs.getTotalBucketCount(), 1 << distBits);
+ assertEquals(progDs.getFinishedBucketCount(), bucketCount);
+
+ VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory2, 1, progDs);
+
+ assertFalse(progDs.hasPending());
+ assertFalse(progDs.hasActive());
+ assertTrue(iterDs.hasNext());
+ assertFalse(iterDs.isDone());
+ assertEquals(distBits, iterDs.getDistributionBitCount());
+ assertEquals(distBits, progDs.getDistributionBitCount());
+
+ // Iterator must start up on next bucket in range
+ VisitorIterator.BucketProgress idDs = iterDs.getNext();
+ long resumeKey = ProgressToken.makeNthBucketKey(bucketCount, distBits);
+ assertEquals(idDs.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(resumeKey)));
+ assertEquals(idDs.getProgress(), new BucketId());
+ }
+
+ // Now fetch a subset of the remaining buckets without finishing them,
+ // keeping some in the active set and some in pending
+ int pendingTotal = 1 << (distBits - 3);
+ int activeTotal = 1 << (distBits - 3);
+ Vector<VisitorIterator.BucketProgress> buckets = new Vector<VisitorIterator.BucketProgress>();
+
+ // Pre-fetch, since otherwise we'd reuse pending buckets
+ for (int i = 0; i < pendingTotal + activeTotal; ++i) {
+ buckets.add(iter.getNext());
+ }
+
+ for (int i = 0; i < pendingTotal + activeTotal; ++i) {
+ VisitorIterator.BucketProgress idTemp = buckets.get(i);
+ if (i < activeTotal) {
+ // Make them 50% done
+ iter.update(idTemp.getSuperbucket(),
+ new BucketId(distBits + 2, idTemp.getSuperbucket().getId() | (2 << distBits)));
+ }
+ // else: leave hanging as active
+ }
+
+ assertEquals(progress.getActiveBucketCount(), activeTotal);
+ assertEquals(progress.getPendingBucketCount(), pendingTotal);
+
+ // we can't reuse the existing string builder, since the bucket cursor
+ // has changed
+ desired = new StringBuilder();
+ desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");
+ desired.append(distBits);
+ desired.append('\n');
+ desired.append(bucketCount + pendingTotal + activeTotal);
+ desired.append('\n');
+ desired.append(bucketCount);
+ desired.append('\n');
+ desired.append(1 << distBits);
+ desired.append('\n');
+
+ assertEquals(progress.getBuckets().entrySet().size(), pendingTotal + activeTotal);
+
+ for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry
+ : progress.getBuckets().entrySet()) {
+ desired.append(Long.toHexString(ProgressToken.keyToBucketId(entry.getKey().getKey())));
+ desired.append(':');
+ desired.append(Long.toHexString(entry.getValue().getProgress().getRawId()));
+ desired.append('\n');
+ }
+
+ assertEquals(progress.toString(), desired.toString());
+
+ {
+ // Deserialization with pending buckets
+ ProgressToken progDs = new ProgressToken(progress.toString());
+
+ assertEquals(progDs.getDistributionBitCount(), distBits);
+ assertEquals(progDs.getTotalBucketCount(), 1 << distBits);
+ assertEquals(progDs.getFinishedBucketCount(), bucketCount);
+
+ VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory2, 1, progDs);
+
+ // All started but nonfinished buckets get placed in pending upon
+ // deserialization
+ assertEquals(progDs.getPendingBucketCount(), pendingTotal + activeTotal);
+ assertEquals(distBits, progDs.getDistributionBitCount());
+ assertEquals(distBits, iterDs.getDistributionBitCount());
+ assertFalse(progDs.hasActive());
+ assertTrue(iterDs.hasNext());
+ assertFalse(iterDs.isDone());
+ assertEquals(progDs.getBucketCursor(), bucketCount + pendingTotal + activeTotal);
+ }
+
+ // Finish all the active buckets
+ for (int i = activeTotal; i < activeTotal + pendingTotal; ++i) {
+ iter.update(buckets.get(i).getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ ++bucketCount;
+ }
+
+ assertEquals(progress.getActiveBucketCount(), 0);
+ boolean consistentNext = true;
+ // Get all pending/remaining sourced and finish them all
+ while (!iter.isDone()) {
+ if (!iter.hasNext()) {
+ consistentNext = false;
+ break;
+ }
+ VisitorIterator.BucketProgress bp = iter.getNext();
+ iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ ++bucketCount;
+ }
+
+ assertTrue(consistentNext);
+ assertFalse(iter.hasNext());
+ assertTrue(progress.isFinished());
+ // Cumulative number of finished buckets must match 2^distbits
+ assertEquals(bucketCount, 1 << distBits);
+ StringBuilder finished = new StringBuilder();
+ finished.append("VDS bucket progress file (100.0% completed)\n");
+ finished.append(distBits);
+ finished.append('\n');
+ finished.append(1 << distBits); // Cursor
+ finished.append('\n');
+ finished.append(1 << distBits); // Finished
+ finished.append('\n');
+ finished.append(1 << distBits); // Total
+ finished.append('\n');
+
+ assertEquals(progress.toString(), finished.toString());
+ }
+
+ public void testProgressSerializationExplicit() throws ParseException {
+ int distBits = 16;
+
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken progress = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, progress);
+
+ assertEquals(progress.getDistributionBitCount(), distBits);
+ assertTrue(iter.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);
+
+ assertEquals(progress.getFinishedBucketCount(), 0);
+ assertEquals(progress.getTotalBucketCount(), 3);
+ assertEquals(progress.getPendingBucketCount(), 3);
+
+ VisitorIterator.BucketProgress bp1 = iter.getNext();
+ VisitorIterator.BucketProgress bp2 = iter.getNext();
+ assertEquals(progress.getPendingBucketCount(), 1);
+ assertEquals(progress.getActiveBucketCount(), 2);
+ // Buckets are ordered by their reverse bucket id key
+ assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));
+ assertEquals(bp1.getProgress(), new BucketId());
+ // Put bucket 1234 back into pending
+ iter.update(bp1.getSuperbucket(), new BucketId(36, 1234));
+ assertEquals(progress.getPendingBucketCount(), 2);
+
+ assertEquals(bp2.getSuperbucket(), new BucketId(32, 8009));
+ assertEquals(bp2.getProgress(), new BucketId());
+
+ {
+ StringBuilder desired = new StringBuilder();
+ desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");
+ desired.append(distBits);
+ desired.append('\n');
+ desired.append(0);
+ desired.append('\n');
+ desired.append(0);
+ desired.append('\n');
+ desired.append(3);
+ desired.append('\n');
+ // Pending/active buckets are written in an increasing (key, not
+ // bucket-id!) order
+ desired.append(Long.toHexString(new BucketId(32, 1234).getRawId()));
+ desired.append(':');
+ desired.append(Long.toHexString(new BucketId(36, 1234).getRawId()));
+ desired.append('\n');
+ desired.append(Long.toHexString(new BucketId(32, 8009).getRawId()));
+ desired.append(":0\n");
+ desired.append(Long.toHexString(new BucketId(32, 6789).getRawId()));
+ desired.append(":0\n");
+
+ assertEquals(desired.toString(), progress.toString());
+
+ ProgressToken prog2 = new ProgressToken(progress.toString());
+ assertEquals(prog2.getDistributionBitCount(), distBits);
+ assertEquals(prog2.getTotalBucketCount(), 3);
+ assertEquals(prog2.getFinishedBucketCount(), 0);
+
+ VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(
+ "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, prog2);
+
+ assertEquals(prog2.getPendingBucketCount(), 3);
+ assertFalse(prog2.hasActive());
+ assertTrue(iter2.hasNext());
+ assertFalse(iter2.isDone());
+
+ assertTrue(iter2.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);
+ assertFalse(iter2.getBucketSource().hasNext());
+
+ VisitorIterator.BucketProgress bp = iter2.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(32, 1234));
+ assertEquals(bp.getProgress(), new BucketId(36, 1234));
+ assertEquals(prog2.getPendingBucketCount(), 2);
+
+ assertTrue(iter2.hasNext());
+ assertFalse(iter2.isDone());
+ bp = iter2.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(32, 8009));
+ assertEquals(bp.getProgress(), new BucketId());
+ assertEquals(prog2.getPendingBucketCount(), 1);
+
+ assertTrue(iter2.hasNext());
+ assertFalse(iter2.isDone());
+ bp = iter2.getNext();
+ assertEquals(prog2.getPendingBucketCount(), 0);
+ assertEquals(bp.getSuperbucket(), new BucketId(32, 6789));
+ assertEquals(bp.getProgress(), new BucketId());
+ assertFalse(iter2.hasNext());
+ assertFalse(iter2.isDone()); // Active buckets
+ assertEquals(prog2.getActiveBucketCount(), 3);
+ }
+
+ // Finish off all active buckets
+ assertTrue(iter.hasNext());
+ assertFalse(iter.isDone());
+ bp1 = iter.getNext();
+ assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));
+ assertEquals(bp1.getProgress(), new BucketId(36, 1234));
+
+ iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+ assertTrue(iter.hasNext());
+ assertFalse(iter.isDone());
+ bp1 = iter.getNext();
+ assertEquals(bp1.getSuperbucket(), new BucketId(32, 6789));
+ assertEquals(bp1.getProgress(), new BucketId());
+
+ // Just to make sure Java serializes the long properly
+ assertEquals(
+ progress.toString(),
+ "VDS bucket progress file (" + progress.percentFinished() + "% completed)\n" +
+ "16\n" +
+ "0\n" +
+ "1\n" +
+ "3\n" +
+ "8000000000001f49:0\n" +
+ "8000000000001a85:0\n");
+
+ iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+ // At this point, we've got one active but no pending, so hasNext == false,
+ // but isDone is also == false
+ assertFalse(iter.hasNext());
+ assertFalse(iter.isDone());
+ assertEquals(progress.getPendingBucketCount(), 0);
+ assertEquals(progress.getActiveBucketCount(), 1);
+
+ iter.update(bp2.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ assertFalse(iter.hasNext());
+ assertTrue(iter.isDone());
+ assertTrue(progress.isFinished());
+ assertEquals(progress.getActiveBucketCount(), 0);
+
+ {
+ StringBuilder finished = new StringBuilder();
+ finished.append("VDS bucket progress file (100.0% completed)\n");
+ finished.append(distBits);
+ finished.append('\n');
+ finished.append(0); // Cursor (not used by explicit)
+ finished.append('\n');
+ finished.append(3); // Finished
+ finished.append('\n');
+ finished.append(3); // Total
+ finished.append('\n');
+
+ assertEquals(finished.toString(), progress.toString());
+ }
+ }
+
+ /**
+ * Test that doing update() on a bucket several times in a row (without re-fetching
+ * from getNext first) works
+ * @throws ParseException
+ */
+ public void testActiveUpdate() throws ParseException {
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken progress = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group = \"yahoo.com\"", idFactory, 16, progress);
+
+ VisitorIterator.BucketProgress bp = iter.getNext();
+
+ assertEquals(progress.getPendingBucketCount(), 0);
+ assertEquals(progress.getActiveBucketCount(), 1);
+
+ BucketId superbucket = bp.getSuperbucket();
+ int usedBits = superbucket.getUsedBits();
+
+ iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (2L << usedBits)));
+ assertEquals(progress.getPendingBucketCount(), 1);
+ assertEquals(progress.getActiveBucketCount(), 0);
+ iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
+ assertEquals(progress.getPendingBucketCount(), 1);
+ assertEquals(progress.getActiveBucketCount(), 0);
+
+ bp = iter.getNext();
+ assertEquals(bp.getSuperbucket(), superbucket);
+ assertEquals(bp.getProgress(), new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
+ assertEquals(progress.getPendingBucketCount(), 0);
+ assertEquals(progress.getActiveBucketCount(), 1);
+ }
+
+ /**
+ * Test that ensures doing update(superbucket, 0) simply puts the bucket back in
+ * pending
+ * @throws ParseException
+ */
+ public void testNullAndSuperUpdate() throws ParseException {
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken progress = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group = \"yahoo.com\"", idFactory, 16, progress);
+
+ assertEquals(progress.getPendingBucketCount(), 1);
+
+ VisitorIterator.BucketProgress bp = iter.getNext();
+ assertEquals(bp.getProgress(), new BucketId());
+ BucketId superbucket = bp.getSuperbucket();
+ BucketId sub = bp.getProgress();
+
+ assertFalse(iter.hasNext());
+ assertFalse(iter.isDone());
+ assertEquals(progress.getPendingBucketCount(), 0);
+ assertEquals(progress.getActiveBucketCount(), 1);
+
+ // 0-bucket
+ iter.update(superbucket, ProgressToken.NULL_BUCKET);
+ assertTrue(iter.hasNext());
+ assertFalse(iter.isDone());
+ assertEquals(progress.getPendingBucketCount(), 1);
+ assertEquals(progress.getActiveBucketCount(), 0);
+
+ VisitorIterator.BucketProgress bp2 = iter.getNext();
+ assertEquals(bp2.getSuperbucket(), superbucket);
+ assertEquals(bp2.getProgress(), ProgressToken.NULL_BUCKET);
+ assertEquals(progress.getPendingBucketCount(), 0);
+ assertEquals(progress.getActiveBucketCount(), 1);
+
+ // progress == super
+ iter.update(superbucket, superbucket);
+ assertTrue(iter.hasNext());
+ assertFalse(iter.isDone());
+ assertEquals(progress.getPendingBucketCount(), 1);
+ assertEquals(progress.getActiveBucketCount(), 0);
+
+ bp2 = iter.getNext();
+ assertEquals(bp2.getSuperbucket(), superbucket);
+ assertEquals(bp2.getProgress(), superbucket);
+ assertEquals(progress.getPendingBucketCount(), 0);
+ assertEquals(progress.getActiveBucketCount(), 1);
+ }
+
+ public void testDeserializedFinishedProgress() {
+ StringBuilder finished = new StringBuilder();
+ finished.append("VDS bucket progress file\n"); // legacy; no completion percentage
+ finished.append(17);
+ finished.append('\n');
+ finished.append(1L << 17); // Cursor
+ finished.append('\n');
+ finished.append(1L << 17); // Finished
+ finished.append('\n');
+ finished.append(1L << 17); // Total
+ finished.append('\n');
+
+ ProgressToken token = new ProgressToken(finished.toString());
+ assertEquals(token.getDistributionBitCount(), 17);
+ assertEquals(token.getTotalBucketCount(), 1L << 17);
+ assertEquals(token.getFinishedBucketCount(), 1L << 17);
+ assertEquals(token.getBucketCursor(), 1L << 17);
+ assertTrue(token.isFinished());
+
+ ProgressToken token2 = new ProgressToken(token.serialize());
+ assertEquals(17, token2.getDistributionBitCount());
+ assertEquals(1L << 17, token2.getTotalBucketCount());
+ assertEquals(1L << 17, token2.getFinishedBucketCount());
+ assertEquals(1L << 17, token2.getBucketCursor());
+ assertTrue(token2.isFinished());
+ }
+
+ public void testBucketProgressFraction() {
+ double epsilon = 0.00001;
+ // No progress
+ BucketId b_0 = new BucketId();
+ // No split; only superbucket (100%)
+ BucketId b_100_0 = new BucketId(16, 1234);
+ // 1 split (1/2)
+ BucketId b_50_1 = new BucketId(17, 1234);
+ BucketId b_100_1 = new BucketId(17, 1234 | (1 << 16));
+ // 2 splits (1/4)
+ BucketId b_25_2 = new BucketId(18, 1234);
+ BucketId b_50_2 = new BucketId(18, 1234 | (2 << 16));
+ BucketId b_75_2 = new BucketId(18, 1234 | (1 << 16));
+ BucketId b_100_2 = new BucketId(18, 1234 | (3 << 16));
+
+ ProgressToken p = new ProgressToken(16);
+
+ BucketId sb = new BucketId(16, 1234);
+
+ assertEquals(p.progressFraction(new BucketId(32, 1234), b_0), 0.0, epsilon);
+
+ assertEquals(p.progressFraction(sb, b_100_0), 1.0, epsilon);
+
+ assertEquals(p.progressFraction(sb, b_50_1), 0.5, epsilon);
+ assertEquals(p.progressFraction(sb, b_100_1), 1.0, epsilon);
+
+ assertEquals(p.progressFraction(sb, b_25_2), 0.25, epsilon);
+ assertEquals(p.progressFraction(sb, b_50_2), 0.5, epsilon);
+ assertEquals(p.progressFraction(sb, b_75_2), 0.75, epsilon);
+ assertEquals(p.progressFraction(sb, b_100_2), 1.0, epsilon);
+
+ assertEquals(p.progressFraction(new BucketId(0x8000000000000000L),
+ new BucketId(0xb0000fff00000000L)), 1.0, epsilon);
+ }
+
+ public void testProgressEstimation() throws ParseException {
+ int distBits = 4;
+
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken progress = new ProgressToken();
+
+ // Create a range of [0, 16) superbuckets
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, distBits, progress);
+
+ assertEquals(progress.getDistributionBitCount(), 4);
+
+ double epsilon = 0.00001;
+ assertEquals(progress.percentFinished(), 0, epsilon);
+ VisitorIterator.BucketProgress bp = iter.getNext();
+ // Finish first superbucket (6.25% total)
+ iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ assertEquals(progress.percentFinished(), 6.25, epsilon);
+ assertEquals(progress.getFinishedBucketCount(), 1);
+
+ bp = iter.getNext();
+ VisitorIterator.BucketProgress bp3 = iter.getNext();
+ VisitorIterator.BucketProgress bp4 = iter.getNext();
+
+ // Finish second (12.5% total)
+ iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ assertEquals(progress.percentFinished(), 12.5, epsilon);
+ assertEquals(progress.getFinishedBucketCount(), 2);
+
+ // Finish third bucket 75% through (17.1875% total)
+ iter.update(bp3.getSuperbucket(), new BucketId(distBits + 2, bp3.getSuperbucket().getId() | (1 << distBits)));
+ assertEquals(progress.percentFinished(), 17.1875, epsilon);
+ assertEquals(progress.getFinishedBucketCount(), 2);
+
+ // Finish fourth bucket 25% through (18.75% total)
+ iter.update(bp4.getSuperbucket(), new BucketId(distBits + 2, bp4.getSuperbucket().getId()));
+ assertEquals(progress.percentFinished(), 18.75, epsilon);
+ assertEquals(progress.getFinishedBucketCount(), 2);
+ // Finish all buckets
+ iter.update(bp4.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ iter.update(bp3.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ assertEquals(progress.percentFinished(), 25, epsilon);
+ assertEquals(progress.getFinishedBucketCount(), 4);
+
+ while (iter.hasNext()) {
+ bp = iter.getNext();
+ iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ }
+
+ assertEquals(progress.getFinishedBucketCount(), 16);
+ assertEquals(progress.percentFinished(), 100, epsilon);
+ }
+
+ public void testBucketKeyWrapperOrdering() {
+ ProgressToken.BucketKeyWrapper bk1 = new ProgressToken.BucketKeyWrapper(0x0000000000000001L);
+ ProgressToken.BucketKeyWrapper bk2 = new ProgressToken.BucketKeyWrapper(0x7FFFFFFFFFFFFFFFL);
+ ProgressToken.BucketKeyWrapper bk3 = new ProgressToken.BucketKeyWrapper(0x8000000000000000L);
+ ProgressToken.BucketKeyWrapper bk4 = new ProgressToken.BucketKeyWrapper(0xFFFFFFFFFFFFFFFFL);
+ assertTrue(bk1.compareTo(bk2) < 0);
+ assertTrue(bk2.compareTo(bk3) < 0);
+ assertTrue(bk3.compareTo(bk4) < 0);
+ assertTrue(bk2.compareTo(bk1) > 0);
+ assertTrue(bk3.compareTo(bk2) > 0);
+ assertTrue(bk4.compareTo(bk3) > 0);
+ ProgressToken.BucketKeyWrapper bk5 = new ProgressToken.BucketKeyWrapper(0x7FFFFFFFFFFFFFFFL);
+ ProgressToken.BucketKeyWrapper bk6 = new ProgressToken.BucketKeyWrapper(0x8000000000000000L);
+ assertTrue(bk5.compareTo(bk2) == 0);
+ assertTrue(bk6.compareTo(bk3) == 0);
+ }
+
+ private void doTestBucketKeyGeneration(int db) {
+ // Can't use longs since they won't sort properly when MSB is set
+ ProgressToken.BucketKeyWrapper[] keys = new ProgressToken.BucketKeyWrapper[1 << db];
+
+ // Generate entire bucket space for db
+ for (int i = 0; i < (1 << db); ++i) {
+ keys[i] = new ProgressToken.BucketKeyWrapper(
+ ProgressToken.bucketToKey(new BucketId(db, i).getId()));
+ }
+ Arrays.sort(keys);
+
+ boolean consistentKeys = true;
+ // Verify that makeNthBucketKey yields the same result as the equivalent
+ // ordered value in the array of keys
+ for (int i = 0; i < (1 << db); ++i) {
+ long genKey = ProgressToken.makeNthBucketKey(i, db);
+ long knownKey = keys[i].getKey();
+ if (genKey != knownKey) {
+ consistentKeys = false;
+ break;
+ }
+ }
+ assertTrue(consistentKeys);
+ }
+
+ public void testBucketKeyGeneration() {
+ // Due to the number of objects needed to be allocated, only test for a
+ // small set of distribution bits
+ for (int i = 1; i < 14; ++i) {
+ doTestBucketKeyGeneration(i);
+ }
+ }
+
+ public void testSingleBucketSplits() throws ParseException {
+ int db = 2;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ // Create a range of [0, 4) superbuckets
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, db, p);
+ VisitorIterator.BucketProgress bp = iter.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
+ // Put back as pending
+ iter.update(bp.getSuperbucket(), new BucketId());
+ assertEquals(p.getPendingBucketCount(), 1);
+ p.splitPendingBucket(new BucketId(db, 0));
+ assertEquals(p.getPendingBucketCount(), 2);
+ bp = iter.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 0)); // left split
+ assertEquals(bp.getProgress(), new BucketId(0));
+ bp = iter.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 4)); // right split
+ assertEquals(bp.getProgress(), new BucketId(0));
+
+ bp = iter.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(db, 2));
+ // Put back as pending, with a progress of 10010. This implies splitting
+ // the bucket should set both splits with a progress to 10010
+ iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x12));
+ assertEquals(p.getPendingBucketCount(), 1);
+ p.splitPendingBucket(new BucketId(db, 2));
+ assertEquals(p.getPendingBucketCount(), 2);
+ bp = iter.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 2)); // left split
+ assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));
+ bp = iter.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 6)); // right split
+ assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));
+
+ bp = iter.getNext();
+ // Put back as pending with a progress of 10101. This implies splitting the
+ // bucket should _discard_ left and set right's progress to 10101.
+ // Update: no it shouldn't, we now split with equal progress without
+ // discarding
+ assertEquals(bp.getSuperbucket(), new BucketId(db, 1));
+ iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x15));
+ assertEquals(p.getPendingBucketCount(), 1);
+ p.splitPendingBucket(new BucketId(db, 1));
+ assertEquals(p.getPendingBucketCount(), 2);
+ bp = iter.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 1));
+ assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));
+ bp = iter.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 5)); // right split
+ assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));
+ }
+
+ /**
+ * Test increasing the distribution bits for a full bucket space range
+ * source with no finished, active or pending buckets
+ * @throws ParseException upon docsel parse failure (shouldn't happen)
+ */
+ public void testRangeDistributionBitIncrease1NoPending() throws ParseException {
+ int db = 2;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ // Test for empty progress token. no splitting involved
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, db, p);
+
+ assertEquals(p.getTotalBucketCount(), 4);
+ iter.setDistributionBitCount(db + 1);
+ assertEquals(p.getTotalBucketCount(), 8);
+ assertEquals(p.getDistributionBitCount(), db + 1);
+ assertEquals(iter.getDistributionBitCount(), db + 1);
+ assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
+
+ int[] desired = new int[] { 0, 4, 2, 6, 1, 5, 3, 7 };
+ for (int i = 0; i < 8; ++i) {
+ VisitorIterator.BucketProgress bp = iter.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(db + 1, desired[i]));
+ }
+ }
+
+ public void testRangeDistributionBitIncrease1AllBucketStates() throws ParseException {
+ int db = 3;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, db, p);
+
+ // For this test, have 1 finished bucket, 3 pending and 0 active (we
+ // want to have the splitting to be triggered immediately)
+ VisitorIterator.BucketProgress bp = iter.getNext();
+ iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
+ bpp[0] = iter.getNext();
+ bpp[1] = iter.getNext();
+ bpp[2] = iter.getNext();
+ iter.update(bpp[0].getSuperbucket(), new BucketId());
+ iter.update(bpp[1].getSuperbucket(), new BucketId());
+ iter.update(bpp[2].getSuperbucket(), new BucketId());
+
+ assertEquals(p.getFinishedBucketCount(), 1);
+ assertEquals(p.getPendingBucketCount(), 3);
+ assertEquals(p.getActiveBucketCount(), 0);
+
+ iter.setDistributionBitCount(db + 1);
+
+ assertEquals(p.getTotalBucketCount(), 16);
+ assertEquals(p.getFinishedBucketCount(), 2);
+ assertEquals(p.getPendingBucketCount(), 6);
+ assertEquals(p.getActiveBucketCount(), 0);
+ assertEquals(p.getDistributionBitCount(), db + 1);
+ assertEquals(iter.getDistributionBitCount(), db + 1);
+ assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
+
+ // Bucket 3:0x4 -> 4:0x4 & 4:0xC
+ assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x04));
+ assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0C));
+ // Bucket 3:0x2 -> 4:0x2 & 4:0xA
+ assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x02));
+ assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0A));
+ // Bucket 3:0x6 -> 4:0x6 & 4:0xE
+ assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x06));
+ assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0E));
+
+ assertEquals(p.getPendingBucketCount(), 0);
+ // Bucket source should now begin returning from bucket 4:0x1
+ assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x01));
+ assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x09));
+ assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x05));
+ assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0D));
+ assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x03));
+ // Assume correct from here on
+ }
+
+ public void testRangeDistributionIncreaseMultipleBits() throws ParseException {
+ int db = 16;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, db, p);
+
+ // For this test, have 3 finished bucket, 2 pending and 1 active
+ for (int i = 0; i < 3; ++i) {
+ iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ }
+
+ VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];
+ bpp[0] = iter.getNext();
+ bpp[1] = iter.getNext();
+ VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active
+ iter.update(bpp[0].getSuperbucket(), new BucketId());
+ iter.update(bpp[1].getSuperbucket(), new BucketId());
+
+ iter.setDistributionBitCount(20);
+ // ProgressToken doesn't change yet, since it had active buckets
+ assertEquals(p.getDistributionBitCount(), 16);
+ assertEquals(iter.getDistributionBitCount(), 20);
+ assertEquals(iter.getBucketSource().getDistributionBitCount(), 20);
+
+ assertFalse(iter.hasNext());
+ assertFalse(iter.isDone());
+ assertTrue(iter.getBucketSource().shouldYield());
+ assertEquals(p.getPendingBucketCount(), 2);
+ assertEquals(p.getActiveBucketCount(), 1);
+
+ // Finish active, triggering the consistency fixes
+ iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+ assertEquals(p.getDistributionBitCount(), 20);
+ assertEquals(p.getPendingBucketCount(), 32);
+ assertEquals(p.getActiveBucketCount(), 0);
+ // Each bucket with db:16 becomes equal to 16 buckets with db:20, so
+ // the bucket space position must be 16 * 6 = 96
+ assertEquals(p.getBucketCursor(), 96);
+ // Each finished bucket also covers less ground, so count is upped
+ // accordingly
+ assertEquals(p.getFinishedBucketCount(), 16 * 4);
+
+ // Remove pending that came from the split
+ // Bucket space that should be covered by the 32 buckets is [48, 80)
+ // when using 20 distribution bits
+ for (int i = 0; i < 32; ++i) {
+ long testKey = ProgressToken.makeNthBucketKey(i + 48, 20);
+ VisitorIterator.BucketProgress bp = iter.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(testKey)));
+ iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ }
+ assertEquals(p.getPendingBucketCount(), 0);
+ assertEquals(p.getFinishedBucketCount(), 16 * 6);
+
+ // Bucket source should now begin returning from bucket 20:0x6000
+ assertEquals(iter.getNext().getSuperbucket(), new BucketId(20, 0x6000));
+ }
+
+ public void testSingleBucketMerge() throws ParseException {
+ int db = 2;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ // Create a range of [0, 4) superbuckets
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, db, p);
+
+ VisitorIterator.BucketProgress bp = iter.getNext();
+ // Put back as pending and split it
+ iter.update(bp.getSuperbucket(), new BucketId());
+ p.splitPendingBucket(new BucketId(db, 0));
+ assertEquals(p.getPendingBucketCount(), 2);
+ // Merge both back into one node. Merge from left sibling with right present
+ p.mergePendingBucket(new BucketId(db + 1, 0));
+ assertEquals(p.getPendingBucketCount(), 1);
+ bp = iter.getNext();
+ assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
+ }
+
+ public void testRangeDistributionBitDecrease1() throws ParseException {
+ int db = 16;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, db, p);
+
+ VisitorIterator.DistributionRangeBucketSource src
+ = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();
+
+ assertTrue(src.isLosslessResetPossible());
+
+ // For this test, have 3 finished buckets, 6 pending and 1 active
+ // This gives a sibling "distribution" of FF FP PP PP PA. When all
+ // active buckets have been updated, 3 merges should be triggered
+ for (int i = 0; i < 3; ++i) {
+ iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ }
+
+ assertFalse(src.isLosslessResetPossible());
+
+ VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[6];
+ for (int i = 0; i < 6; ++i) {
+ bpp[i] = iter.getNext();
+ }
+ VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active
+ for (int i = 0; i < 6; ++i) {
+ iter.update(bpp[i].getSuperbucket(), new BucketId());
+ }
+
+ assertEquals(p.getBucketCursor(), 10);
+
+ iter.setDistributionBitCount(db - 1);
+ assertEquals(iter.getDistributionBitCount(), db - 1);
+ assertEquals(p.getDistributionBitCount(), db);
+ assertEquals(iter.getBucketSource().getDistributionBitCount(), db - 1);
+ // The iterator is waiting patiently for all active buckets to be updated,
+ // at which point it will performed the merging and actually updating the
+ // progress token's distribution bit count
+ assertTrue(iter.getBucketSource().shouldYield());
+ assertFalse(iter.hasNext());
+ assertFalse(iter.isDone());
+ assertEquals(p.getActiveBucketCount(), 1);
+ iter.update(bpa.getSuperbucket(), new BucketId());
+
+ assertEquals(p.getDistributionBitCount(), db - 1);
+ assertEquals(p.getActiveBucketCount(), 0);
+ assertEquals(p.getPendingBucketCount(), 4); // 3 merges, P PP PP PP -> P P P P
+
+ assertEquals(p.getFinishedBucketCount(), 1);
+ assertEquals(p.getBucketCursor(), 5);
+ }
+
+ // Test that splitting and merging from and to the same db count gives
+ // back the initial state
+ public void testRangeDistributionBitIncreaseDecrease() throws ParseException {
+ int db = 16;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, db, p);
+
+ VisitorIterator.DistributionRangeBucketSource src
+ = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();
+
+ assertTrue(src.isLosslessResetPossible());
+
+ // "Sabotage" resetting by having at least 1 finished
+ iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+ VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];
+ for (int i = 0; i < 4; ++i) {
+ bpp[i] = iter.getNext();
+ }
+ for (int i = 0; i < 4; ++i) {
+ iter.update(bpp[i].getSuperbucket(), new BucketId());
+ }
+
+ assertFalse(src.isLosslessResetPossible());
+
+ iter.setDistributionBitCount(20);
+ assertEquals(p.getDistributionBitCount(), 20);
+ assertEquals(p.getPendingBucketCount(), 4 << 4);
+ assertFalse(iter.getBucketSource().shouldYield());
+ assertEquals(p.getBucketCursor(), 5 << 4);
+
+ iter.setDistributionBitCount(16);
+
+ assertEquals(p.getDistributionBitCount(), 16);
+ assertEquals(p.getPendingBucketCount(), 4);
+ assertFalse(iter.getBucketSource().shouldYield());
+ assertEquals(p.getBucketCursor(), 5);
+ }
+
+ // Test that intermittent changes in distribution are handled properly, e.g.
+ // changing from 11 -> 9 with X active and then before all those are flushed,
+ // the distribution goes up to 12
+ public void testRangeDistributionBitChangeWithoutDone() throws ParseException {
+ int db = 11;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, db, p);
+
+ VisitorIterator.DistributionRangeBucketSource src
+ = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();
+
+ VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];
+ for (int i = 0; i < 4; ++i) {
+ bpp[i] = iter.getNext();
+ }
+ for (int i = 0; i < 2; ++i) {
+ iter.update(bpp[i].getSuperbucket(), new BucketId());
+ }
+
+ assertFalse(src.isLosslessResetPossible());
+
+ // Now 2 pending, 2 active
+
+ iter.setDistributionBitCount(9);
+ assertEquals(p.getDistributionBitCount(), 11);
+ assertEquals(p.getActiveBucketCount(), 2);
+ assertEquals(p.getPendingBucketCount(), 2);
+ assertTrue(iter.getBucketSource().shouldYield());
+ // Update as pending, still with old count since there's 1 more active
+ // with bpp[2]. Have progress so that lossless reset isn't possible
+ iter.update(bpp[3].getSuperbucket(), new BucketId(15, bpp[3].getSuperbucket().getId()));
+
+ iter.setDistributionBitCount(12);
+ assertEquals(p.getActiveBucketCount(), 1);
+ assertEquals(p.getPendingBucketCount(), 3);
+ assertTrue(iter.getBucketSource().shouldYield());
+
+ // Serialize before token is updated to 12 bits
+ String serialized = p.toString();
+
+ iter.update(bpp[2].getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+ assertEquals(p.getActiveBucketCount(), 0);
+ // All active buckets are at db=11, so they should be split once each
+ assertEquals(p.getPendingBucketCount(), 3 * 2);
+ assertFalse(iter.getBucketSource().shouldYield());
+ assertEquals(p.getFinishedBucketCount(), 2);
+
+ // Ensure we get a consistent progress token imported
+ ProgressToken p2 = new ProgressToken(serialized);
+ assertEquals(p2.getDistributionBitCount(), 11); // Not yet updated
+
+ BucketIdFactory idFactory2 = new BucketIdFactory();
+ VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory2, 1, p2);
+
+ // Not yet updated, since we don't trust the initial BucketIdFactory
+ assertEquals(iter2.getDistributionBitCount(), 11);
+ assertEquals(p2.getDistributionBitCount(), 11);
+ iter2.setDistributionBitCount(12);
+ // Now it has been updated
+ assertEquals(p2.getDistributionBitCount(), 12);
+ assertEquals(p2.getPendingBucketCount(), 8);
+ assertEquals(p2.getBucketCursor(), 8);
+ assertEquals(p2.getFinishedBucketCount(), 0);
+ }
+
+ // Test a drop from 31->11 bits upon iteration start
+ public void testRangeDistributionBitInitialDrop() throws ParseException {
+ int db = 31;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, db, p);
+
+ VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[3];
+ bp[0] = iter.getNext();
+ bp[1] = iter.getNext();
+ bp[2] = iter.getNext();
+ iter.update(bp[2].getSuperbucket(), new BucketId());
+ iter.update(bp[1].getSuperbucket(), new BucketId());
+ assertEquals(p.getActiveBucketCount(), 1);
+
+ iter.setDistributionBitCount(11);
+
+ assertFalse(iter.hasNext());
+ assertFalse(iter.isDone());
+ assertEquals(p.getActiveBucketCount(), 1);
+
+ // Updating the active bucket allows the merging to take place
+ iter.update(new BucketId(31, 0), new BucketId());
+
+ assertTrue(iter.hasNext());
+ assertFalse(iter.isDone());
+
+ // All pending buckets should have been merged down to just 1 now
+ // Update: now rather gets reset
+ assertEquals(p.getPendingBucketCount(), 0);
+ assertEquals(p.getActiveBucketCount(), 0);
+ assertEquals(p.getFinishedBucketCount(), 0);
+ assertEquals(p.getBucketCursor(), 0);
+
+ bp[0] = iter.getNext();
+ assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
+ }
+
+ // Similar to testRangeDistributionBitInitialDrop, but going from 1 to 11
+ // This tests that doing so may be done in an optimized way rather than
+ // attempting to split enough buckets to cover the entire bucket space!
+ public void testRangeDistributionLosslessReset() throws ParseException {
+ int db = 1;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, db, p);
+
+ VisitorIterator.DistributionRangeBucketSource src
+ = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();
+
+ VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[2];
+ bp[0] = iter.getNext();
+ bp[1] = iter.getNext();
+
+ String serialized = p.toString();
+
+ assertFalse(src.isLosslessResetPossible());
+
+ iter.update(bp[1].getSuperbucket(), new BucketId());
+ assertEquals(p.getActiveBucketCount(), 1);
+
+ iter.setDistributionBitCount(11);
+
+ assertFalse(src.isLosslessResetPossible());
+ assertEquals(p.getDistributionBitCount(), 1); // Still at 1
+
+ assertFalse(iter.hasNext());
+ assertFalse(iter.isDone());
+ assertEquals(p.getActiveBucketCount(), 1);
+
+ // Updating the active bucket allows the reset to take place
+ iter.update(new BucketId(1, 0), new BucketId());
+
+ assertTrue(iter.hasNext());
+ assertFalse(iter.isDone());
+
+ // Should not be any buckets pending/active and the cursor should be
+ // back at 0
+ assertEquals(p.getPendingBucketCount(), 0);
+ assertEquals(p.getActiveBucketCount(), 0);
+ assertEquals(p.getFinishedBucketCount(), 0);
+ assertEquals(p.getBucketCursor(), 0);
+ assertEquals(p.getDistributionBitCount(), 11);
+
+ bp[0] = iter.getNext();
+ assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
+
+ // Ensure resetting also works when you're importing existing
+ // progress
+ p = new ProgressToken(serialized);
+ idFactory = new BucketIdFactory();
+ iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, 1, p);
+
+ iter.setDistributionBitCount(11);
+
+ assertEquals(p.getPendingBucketCount(), 0);
+ assertEquals(p.getActiveBucketCount(), 0);
+ assertEquals(p.getFinishedBucketCount(), 0);
+ assertEquals(p.getBucketCursor(), 0);
+ assertEquals(p.getDistributionBitCount(), 11);
+
+ bp[0] = iter.getNext();
+ assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
+ }
+
+ public void testExplicitDistributionBitIncrease() throws ParseException {
+ int distBits = 12;
+
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p);
+
+ assertEquals(iter.getDistributionBitCount(), distBits);
+ assertEquals(p.getDistributionBitCount(), distBits);
+ assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits);
+
+ iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ iter.setDistributionBitCount(16);
+
+ assertEquals(iter.getDistributionBitCount(), 16);
+ assertEquals(p.getDistributionBitCount(), 16);
+ assertEquals(iter.getBucketSource().getDistributionBitCount(), 16);
+ // Changing dist bits for explicit source should change nothing
+ assertEquals(p.getPendingBucketCount(), 2);
+ assertEquals(p.getFinishedBucketCount(), 1);
+ assertEquals(p.getTotalBucketCount(), 3);
+ }
+
+ public void testExplicitDistributionBitDecrease() throws ParseException {
+ int distBits = 20;
+
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p);
+
+ assertEquals(iter.getDistributionBitCount(), distBits);
+ assertEquals(p.getDistributionBitCount(), distBits);
+ assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits);
+
+ iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ iter.setDistributionBitCount(16);
+
+ assertEquals(iter.getDistributionBitCount(), 16);
+ assertEquals(p.getDistributionBitCount(), 16);
+ assertEquals(iter.getBucketSource().getDistributionBitCount(), 16);
+ // Changing dist bits for explicit source should change nothing
+ assertEquals(p.getPendingBucketCount(), 2);
+ assertEquals(p.getFinishedBucketCount(), 1);
+ assertEquals(p.getTotalBucketCount(), 3);
+ }
+
+ public void testExplicitDistributionImportNoTruncation() throws ParseException {
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, 20, p);
+ assertEquals(20, iter.getDistributionBitCount());
+ assertEquals(20, p.getDistributionBitCount());
+ assertEquals(20, iter.getBucketSource().getDistributionBitCount());
+
+ iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+ // Make sure no truncation is done on import
+ String serialized = p.toString();
+ ProgressToken p2 = new ProgressToken(serialized);
+ BucketIdFactory idFactory2 = new BucketIdFactory();
+ VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(
+ "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory2, 1, p2);
+ assertEquals(20, iter2.getDistributionBitCount());
+ assertEquals(20, p2.getDistributionBitCount());
+ assertEquals(20, iter2.getBucketSource().getDistributionBitCount());
+ assertEquals(2, p2.getPendingBucketCount());
+ assertEquals(1, p2.getFinishedBucketCount());
+ assertEquals(3, p2.getTotalBucketCount());
+ }
+
+ public void testImportProgressWithOutdatedDistribution() throws ParseException {
+ String input = "VDS bucket progress file\n" +
+ "10\n" +
+ "503\n" +
+ "500\n" +
+ "1024\n" +
+ "28000000000000be:0\n" +
+ "28000000000002be:0\n" +
+ "28000000000001be:0\n";
+
+ int db = 12;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken(input);
+ assertEquals(10, p.getDistributionBitCount());
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, 1, p);
+
+ iter.setDistributionBitCount(12);
+ assertEquals(iter.getDistributionBitCount(), 12);
+ assertEquals(p.getDistributionBitCount(), 12);
+ assertEquals(iter.getBucketSource().getDistributionBitCount(), 12);
+
+ assertEquals(p.getTotalBucketCount(), 1 << 12);
+ assertEquals(p.getFinishedBucketCount(), 500 << 2);
+ assertEquals(p.getPendingBucketCount(), 3 << 2);
+ assertEquals(p.getActiveBucketCount(), 0);
+ assertEquals(p.getBucketCursor(), 503 << 2);
+ assertTrue(iter.hasNext());
+
+ ProgressToken p2 = new ProgressToken(p.serialize());
+ assertEquals(p2.getDistributionBitCount(), 12);
+ assertEquals(p2.getTotalBucketCount(), 1 << 12);
+ assertEquals(p2.getFinishedBucketCount(), 500 << 2);
+ assertEquals(p2.getPendingBucketCount(), 3 << 2);
+ assertEquals(p2.getActiveBucketCount(), 0);
+ assertEquals(p2.getBucketCursor(), 503 << 2);
+ }
+
+ public void testImportInconsistentProgressIncrease() throws ParseException {
+ // Bucket progress "file" that upon time of changing from 4 to 7
+ // distribution bits and writing the progress had an active bucket
+ String input = "VDS bucket progress file\n" +
+ "7\n" +
+ "32\n" +
+ "24\n" +
+ "128\n" +
+ "100000000000000c:0\n";
+ // Now we're at 8 distribution bits
+ int db = 8;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken(input);
+ assertEquals(7, p.getDistributionBitCount());
+ assertEquals(p.getTotalBucketCount(), 1 << 7);
+ assertEquals(p.getFinishedBucketCount(), 24);
+ // Not yet corrected
+ assertEquals(p.getPendingBucketCount(), 1);
+ assertEquals(p.getActiveBucketCount(), 0);
+ assertEquals(32, p.getBucketCursor());
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, 1, p);
+
+ // Now the range handler should have corrected the progress
+ // (but not messed with the distribution bit count)
+ assertEquals(7, p.getDistributionBitCount());
+ assertEquals(p.getPendingBucketCount(), 1 << 3);
+ assertEquals(p.getActiveBucketCount(), 0);
+ assertEquals(24 + (1 << 3), p.getBucketCursor());
+
+ iter.setDistributionBitCount(8);
+
+ assertEquals(iter.getDistributionBitCount(), 8);
+ assertEquals(p.getDistributionBitCount(), 8);
+ assertEquals(iter.getBucketSource().getDistributionBitCount(), 8);
+
+ assertEquals(p.getTotalBucketCount(), 1 << 8);
+ assertEquals(p.getFinishedBucketCount(), 24 << 1);
+ assertEquals(p.getPendingBucketCount(), 1 << 4); // Split 4 -> 7 bits, then 7 -> 8 bits
+ assertEquals(p.getActiveBucketCount(), 0);
+ assertEquals(p.getBucketCursor(), 24*2 + (1 << 4));
+ assertTrue(iter.hasNext());
+ }
+
+ public void testImportInconsistentProgressDecrease() throws ParseException {
+ // Bucket progress "file" that upon time of changing from 4 to 7
+ // distribution bits and writing the progress had an active bucket
+ String input = "VDS bucket progress file\n" +
+ "7\n" +
+ "32\n" +
+ "24\n" +
+ "128\n" +
+ "100000000000000c:0\n";
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken(input);
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, 1, p);
+
+ assertEquals(iter.getDistributionBitCount(), 7);
+ // Now we're at 6 distribution bits
+ iter.setDistributionBitCount(6);
+
+ assertEquals(iter.getDistributionBitCount(), 6);
+ assertEquals(p.getDistributionBitCount(), 6);
+ assertEquals(iter.getBucketSource().getDistributionBitCount(), 6);
+
+ assertEquals(p.getTotalBucketCount(), 1 << 6);
+ assertEquals(p.getFinishedBucketCount(), 24 >> 1);
+ assertEquals(p.getPendingBucketCount(), 1 << 2); // Split 4 -> 7 bits, merge 7 -> 6 bits
+ assertEquals(p.getActiveBucketCount(), 0);
+ assertEquals(p.getBucketCursor(), 24/2 + (1 << 2));
+ assertTrue(iter.hasNext());
+ }
+
+ public void testEntireBucketSpaceCovered() throws ParseException {
+ int db = 4;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, db, p);
+
+ VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
+
+ for (int i = 0; i < 3; ++i) {
+ bpp[i] = iter.getNext();
+ }
+ for (int i = 0; i < 3; ++i) {
+ // Must use non-zero progress or all pending will be optimized
+ // away by the reset-logic
+ iter.update(bpp[i].getSuperbucket(),
+ new BucketId(db + 1, bpp[i].getSuperbucket().getId()));
+ }
+
+ Set<BucketId> buckets = new TreeSet<BucketId>();
+ db = 7;
+ for (int i = 0; i < (1 << db); ++i) {
+ buckets.add(new BucketId(db, i));
+ }
+
+ iter.setDistributionBitCount(db);
+ assertEquals(p.getFinishedBucketCount(), 0);
+ assertEquals(p.getPendingBucketCount(), 3 << 3);
+
+ // Ensure all buckets are visited once and only once
+ while (iter.hasNext()) {
+ VisitorIterator.BucketProgress bp = iter.getNext();
+ assertTrue(buckets.contains(bp.getSuperbucket()));
+ buckets.remove(bp.getSuperbucket());
+ }
+
+ assertTrue(buckets.isEmpty());
+ }
+
+ public void testExceptionOnWrongDocumentSelection() throws ParseException {
+ BucketIdFactory idFactory = new BucketIdFactory();
+ // Since we don't store the actual original document selection in the
+ // progress files, we can't really tell whether or not a "wrong" document
+ // selection has been given, so we just do a best effort by checking
+ // that the number of total buckets match up and that the bucket cursor
+ // isn't set for explicit sources
+
+ // Try to pass a known document selection to an unknown docsel iterator
+ boolean caughtIt = false;
+ try {
+ ProgressToken p = new ProgressToken("VDS bucket progress file\n16\n3\n1\n3\n"
+ + "8000000000001f49:0\n8000000000001a85:0\n");
+
+ VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 16, p);
+ }
+ catch (IllegalArgumentException e) {
+ caughtIt = true;
+ }
+ assertTrue(caughtIt);
+
+ // Now try it the other way around
+ caughtIt = false;
+ try {
+ ProgressToken p = new ProgressToken("VDS bucket progress file\n" +
+ "10\n" +
+ "503\n" +
+ "500\n" +
+ "1024\n" +
+ "28000000000000be:0\n" +
+ "28000000000002be:0\n" +
+ "28000000000001be:0\n");
+
+ VisitorIterator.createFromDocumentSelection("id.group=\"yahoo.com\" or id.user=555", idFactory, 16, p);
+ }
+ catch (IllegalArgumentException e) {
+ caughtIt = true;
+ }
+ assertTrue(caughtIt);
+ }
+
+ public void testIsBucketFinished() throws ParseException {
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, 4, p);
+
+ assertFalse(p.isBucketFinished(new BucketId(32, 0)));
+ // Finish superbucket 0x0000
+ iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ assertTrue(p.isBucketFinished(new BucketId(32, 0)));
+ // Cursor is 1, but bucket 0x1000 not yet returned
+ assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));
+ VisitorIterator.BucketProgress bp = iter.getNext();
+ // Cursor 2, 0x1000 returned but is contained in state, so not finished
+ assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));
+ iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ assertTrue(p.isBucketFinished(new BucketId(32, 1 << 3)));
+ // Only superbucket part is used
+ assertTrue(p.isBucketFinished(new BucketId(32, 0x12345670))); // ...0000
+ assertTrue(p.isBucketFinished(new BucketId(32, 0x12345678))); // ...1000
+ assertFalse(p.isBucketFinished(new BucketId(32, 0x12345671))); // ...0001
+ assertFalse(p.isBucketFinished(new BucketId(32, 0x12345679))); // ...1001
+ }
+
+ // Test that altering distribution bit count sets ProgressToken as
+ // inconsistent when there are active buckets
+ public void testInconsistentState() throws ParseException {
+ int db = 16;
+ BucketIdFactory idFactory = new BucketIdFactory();
+ ProgressToken p = new ProgressToken();
+
+ VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
+ "id.group != \"yahoo.com\"", idFactory, db, p);
+
+ // For this test, have 3 finished bucket, 2 pending and 1 active
+ for (int i = 0; i < 3; ++i) {
+ iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+ }
+
+ VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];
+ bpp[0] = iter.getNext();
+ bpp[1] = iter.getNext();
+ VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active
+ iter.update(bpp[0].getSuperbucket(), new BucketId());
+ iter.update(bpp[1].getSuperbucket(), new BucketId());
+
+ assertFalse(p.isInconsistentState());
+ iter.setDistributionBitCount(20);
+ assertTrue(p.isInconsistentState());
+
+ // Finish active, triggering the consistency fixes
+ iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
+
+ assertFalse(p.isInconsistentState());
+ }
+
+ public void testMalformedProgressFile() {
+ boolean caughtIt = false;
+ try {
+ new ProgressToken("VDS bucket progress file\n" +
+ "10\n" +
+ "503\n" +
+ "500\n" +
+ "1024\n" +
+ "28000000000000be:0\n" +
+ "28000000000002be:");
+ } catch (IllegalArgumentException e) {
+ caughtIt = true;
+ }
+ assertTrue(caughtIt);
+ }
+
+ public void testFailOnTooFewLinesInFile() {
+ boolean caughtIt = false;
+ try {
+ new ProgressToken("VDS bucket progress file\n" +
+ "10\n" +
+ "503\n");
+ } catch (IllegalArgumentException e) {
+ caughtIt = true;
+ }
+ assertTrue(caughtIt);
+ }
+
+ public void testUnknownFirstHeaderLine() {
+ boolean caughtIt = false;
+ try {
+ new ProgressToken("Smurf Time 3000\n" +
+ "10\n" +
+ "503\n" +
+ "500\n" +
+ "1024\n" +
+ "28000000000000be:0\n" +
+ "28000000000002be:0");
+ } catch (IllegalArgumentException e) {
+ caughtIt = true;
+ }
+ assertTrue(caughtIt);
+ }
+
+ public void testBinaryProgressSerialization() {
+ String input = "VDS bucket progress file (48.828125% completed)\n" +
+ "10\n" +
+ "503\n" +
+ "500\n" +
+ "1024\n" +
+ "28000000000000be:0\n" +
+ "28000000000002be:0\n" +
+ "28000000000001be:0\n";
+ ProgressToken p = new ProgressToken(input);
+ byte[] buf = p.serialize();
+ ProgressToken p2 = new ProgressToken(buf);
+ assertEquals(input, p2.toString());
+ }
+ }
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java new file mode 100644 index 00000000000..57753520a64 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import org.junit.Test; +import static org.junit.Assert.*; + +public class VisitorParametersTestCase { + private LoadType loadType = new LoadType(3, "samnmax", DocumentProtocol.Priority.HIGH_3); + + private VisitorParameters createVisitorParameters() { + VisitorParameters params = new VisitorParameters(""); + params.setDocumentSelection("id.user==5678"); + params.setFromTimestamp(9001); + params.setToTimestamp(10001); + params.setVisitorLibrary("CoolVisitor"); + params.setLibraryParameter("groovy", "dudes"); + params.setLibraryParameter("ninja", "turtles"); + params.setMaxBucketsPerVisitor(55); + params.setPriority(DocumentProtocol.Priority.HIGHEST); + params.setRoute("extraterrestrial/highway"); + params.setTimeoutMs(1337); + params.setMaxPending(111); + params.setFieldSet("[header]"); + params.setVisitorOrdering(123); + params.setLoadType(loadType); + params.setVisitRemoves(true); + params.setVisitInconsistentBuckets(true); + params.setTraceLevel(9); + params.setResumeFileName("foo.txt"); + params.setResumeToken(new ProgressToken()); + params.setRemoteDataHandler("mars_rover"); + params.setControlHandler(new VisitorControlHandler()); + params.setMaxFirstPassHits(555); + params.setMaxTotalHits(777); + params.setDynamicallyIncreaseMaxBucketsPerVisitor(true); + params.setDynamicMaxBucketsIncreaseFactor(2.5f); + params.skipBucketsOnFatalErrors(true); + + return params; + } + + @Test + public void testCopyConstructor() { + VisitorParameters params = createVisitorParameters(); + + VisitorParameters copy = new VisitorParameters(params); + + assertEquals("id.user==5678", copy.getDocumentSelection()); + assertEquals(9001, copy.getFromTimestamp()); + assertEquals(10001, copy.getToTimestamp()); + assertEquals("CoolVisitor", copy.getVisitorLibrary()); + assertEquals(2, copy.getLibraryParameters().size()); + assertEquals("dudes", new String(copy.getLibraryParameters().get("groovy"))); + assertEquals("turtles", new String(copy.getLibraryParameters().get("ninja"))); + assertEquals(55, copy.getMaxBucketsPerVisitor()); + assertEquals(DocumentProtocol.Priority.HIGHEST, copy.getPriority()); + assertEquals("extraterrestrial/highway", copy.getRoute().toString()); + assertEquals(1337, copy.getTimeoutMs()); + assertEquals(111, copy.getMaxPending()); + assertEquals("[header]", copy.getFieldSet()); + assertEquals(123, copy.getVisitorOrdering()); + assertEquals(loadType, copy.getLoadType()); + assertEquals(true, copy.getVisitRemoves()); + assertEquals(true, copy.getVisitInconsistentBuckets()); + assertEquals(9, copy.getTraceLevel()); + assertEquals("foo.txt", copy.getResumeFileName()); + assertEquals(params.getResumeToken(), copy.getResumeToken()); // instance compare + assertEquals("mars_rover", copy.getRemoteDataHandler()); + assertEquals(params.getControlHandler(), copy.getControlHandler()); + assertEquals(555, copy.getMaxFirstPassHits()); + assertEquals(777, copy.getMaxTotalHits()); + assertEquals(true, copy.getDynamicallyIncreaseMaxBucketsPerVisitor()); + assertEquals(2.5f, copy.getDynamicMaxBucketsIncreaseFactor(), 0.0001); + assertEquals(true, copy.skipBucketsOnFatalErrors()); + + // Test local data handler copy + VisitorParameters params2 = new VisitorParameters(""); + params2.setLocalDataHandler(new SimpleVisitorDocumentQueue()); + VisitorParameters copy2 = new VisitorParameters(params2); + assertEquals(params2.getLocalDataHandler(), copy2.getLocalDataHandler()); // instance compare + } + + @Test + public void testToString() { + VisitorParameters params = createVisitorParameters(); + + assertEquals( + "VisitorParameters(\n" + + " Document selection: id.user==5678\n" + + " Visitor library: CoolVisitor\n" + + " Max pending: 111\n" + + " Timeout (ms): 1337\n" + + " Time period: 9001 - 10001\n" + + " Visiting remove entries\n" + + " Visiting inconsistent buckets\n" + + " Visitor library parameters:\n" + + " groovy : dudes\n" + + " ninja : turtles\n" + + " Field set: [header]\n" + + " Route: extraterrestrial/highway\n" + + " Weight: 1.0\n" + + " Max firstpass hits: 555\n" + + " Max total hits: 777\n" + + " Visitor ordering: 123\n" + + " Max buckets: 55\n" + + " Priority: HIGHEST\n" + + " Dynamically increasing max buckets per visitor\n" + + " Increase factor: 2.5\n" + + ")", + params.toString()); + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/local/test/LocalDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/local/test/LocalDocumentApiTestCase.java new file mode 100644 index 00000000000..b00615eb8e5 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/local/test/LocalDocumentApiTestCase.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.local.test; + +import com.yahoo.document.*; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.local.LocalDocumentAccess; +import com.yahoo.documentapi.test.AbstractDocumentApiTestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Runs the superclass tests on this implementation + * + * @author bratseth + */ +public class LocalDocumentApiTestCase extends AbstractDocumentApiTestCase { + + protected DocumentAccess access; + + @Override + protected DocumentAccess access() { + return access; + } + + @Before + public void setUp() { + DocumentAccessParams params = new DocumentAccessParams(); + params.setDocumentManagerConfigId("file:src/test/cfg/documentmanager.cfg"); + access = new LocalDocumentAccess(params); + } + + @After + public void shutdownAccess() { + access.shutdown(); + } + + @Test + public void testNoExceptionFromAsync() { + AsyncSession session = access.createAsyncSession(new AsyncParameters()); + + DocumentType type = access.getDocumentTypeManager().getDocumentType("music"); + DocumentUpdate docUp = new DocumentUpdate(type, new DocumentId("doc:music:2")); + + Result result = session.update(docUp); + assertTrue(result.isSuccess()); + Response response = session.getNext(); + assertEquals(result.getRequestId(), response.getRequestId()); + assertFalse(response.isSuccess()); + session.destroy(); + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java new file mode 100644 index 00000000000..248f041b4a6 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.loadtypes.test; + +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import junit.framework.TestCase; + +/** + * @author thomasg + */ +public class LoadTypesTestCase extends TestCase { + public void testIdGeneration() { + LoadTypeSet set = new LoadTypeSet(); + set.addType("vespagrim", "VERY_HIGH"); + set.addType("slow", "VERY_LOW"); + set.addType("test", null); + + assertEquals("vespagrim", set.getNameMap().get("vespagrim").getName()); + assertEquals("slow", set.getNameMap().get("slow").getName()); + assertEquals("test", set.getNameMap().get("test").getName()); + assertEquals("default", set.getNameMap().get("default").getName()); + + assertEquals(0xc21803d4, set.getNameMap().get("vespagrim").getId()); + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolTest.java new file mode 100644 index 00000000000..035babf2af5 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolTest.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.component.Version; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class DocumentProtocolTest { + + private final DocumentTypeManager manager = new DocumentTypeManager(); + + @Before + public void setUp() { + DocumentTypeManagerConfigurer.configure(manager, "file:./test/cfg/testdoc.cfg"); + } + + @SuppressWarnings("deprecation") + @Test + public void requireThat50SerializationPrecedes5xSerialization() { + DocumentProtocol protocol = new DocumentProtocol(manager); + GetDocumentMessage prev = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo"); + byte[] buf = protocol.encode(new Version(5, 0), prev); + + GetDocumentMessage next = (GetDocumentMessage)protocol.decode(new Version(5, 0), buf); + assertEquals(GetDocumentMessage.DEFAULT_FIELD_SET, next.getFieldSet()); + } + +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/ReplyMergerTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/ReplyMergerTestCase.java new file mode 100644 index 00000000000..9f4eace387c --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/ReplyMergerTestCase.java @@ -0,0 +1,228 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol;
+
+import com.yahoo.collections.Tuple2;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.Error;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import static org.hamcrest.CoreMatchers.*;
+
+@SuppressWarnings("deprecation")
+public class ReplyMergerTestCase {
+
+ private ReplyMerger merger;
+
+ @Before
+ public void setUp() {
+ merger = new ReplyMerger();
+ }
+
+ @Test
+ public void mergingGenericRepliesWithNoErrorsPicksFirstReply() {
+ Reply r1 = new EmptyReply();
+ Reply r2 = new EmptyReply();
+ Reply r3 = new EmptyReply();
+ merger.merge(0, r1);
+ merger.merge(1, r2);
+ merger.merge(2, r3);
+ Tuple2<Integer, Reply> ret = merger.mergedReply();
+
+ assertThat(ret.first, is(0));
+ assertThat(ret.second, sameInstance(r1));
+ }
+
+ @Test
+ public void mergingSingleReplyWithOneErrorReturnsEmptyReplyWithError() {
+ Reply r1 = new EmptyReply();
+ Error error = new Error(1234, "oh no!");
+ r1.addError(error);
+ merger.merge(0, r1);
+ Tuple2<Integer, Reply> ret = merger.mergedReply();
+
+ assertThat(ret.first, nullValue());
+ assertThat(ret.second, not(sameInstance(r1)));
+ assertThatErrorsMatch(new Error[] { error }, ret);
+ }
+
+ @Test
+ public void mergingSingleReplyWithMultipleErrorsReturnsEmptyReplyWithAllErrors() {
+ Reply r1 = new EmptyReply();
+ Error errors[] = new Error[] {
+ new Error(1234, "oh no!"), new Error(4567, "oh dear!"),
+ };
+ r1.addError(errors[0]);
+ r1.addError(errors[1]);
+ merger.merge(0, r1);
+ Tuple2<Integer, Reply> ret = merger.mergedReply();
+
+ assertThat(ret.first, nullValue());
+ assertThat(ret.second, not(sameInstance(r1)));
+ assertThatErrorsMatch(errors, ret);
+ }
+
+ @Test
+ public void mergingMultipleRepliesWithMultipleErrorsReturnsEmptyReplyWithAllErrors() {
+ Reply r1 = new EmptyReply();
+ Reply r2 = new EmptyReply();
+ Error errors[] = new Error[] {
+ new Error(1234, "oh no!"), new Error(4567, "oh dear!"), new Error(678, "omg!"),
+ };
+ r1.addError(errors[0]);
+ r1.addError(errors[1]);
+ r2.addError(errors[2]);
+ merger.merge(0, r1);
+ merger.merge(1, r2);
+ Tuple2<Integer, Reply> ret = merger.mergedReply();
+
+ assertThat(ret.first, nullValue());
+ assertThat(ret.second, not(sameInstance(r1)));
+ assertThat(ret.second, not(sameInstance(r2)));
+ assertThatErrorsMatch(errors, ret);
+ }
+
+ @Test
+ public void returnIgnoredReplyWhenAllRepliesHaveOnlyIgnoredErrors() {
+ Reply r1 = new EmptyReply();
+ Reply r2 = new EmptyReply();
+ Error errors[] = new Error[] {
+ new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"),
+ new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh dear!"),
+ new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "omg!"),
+ };
+ r1.addError(errors[0]);
+ r1.addError(errors[1]);
+ r2.addError(errors[2]);
+
+ merger.merge(0, r1);
+ merger.merge(1, r2);
+ Tuple2<Integer, Reply> ret = merger.mergedReply();
+ assertThat(ret.first, nullValue());
+ assertThat(ret.second, not(sameInstance(r1)));
+ assertThat(ret.second, not(sameInstance(r2)));
+ // Only first ignore error from each reply
+ assertThatErrorsMatch(new Error[]{ errors[0], errors[2] }, ret);
+ }
+
+ @Test
+ public void successfulReplyTakesPrecedenceOverIgnoredReplyWhenNoErrors() {
+ Reply r1 = new EmptyReply();
+ Reply r2 = new EmptyReply();
+ Error errors[] = new Error[] {
+ new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"),
+ };
+ r1.addError(errors[0]);
+ merger.merge(0, r1);
+ merger.merge(1, r2);
+ Tuple2<Integer, Reply> ret = merger.mergedReply();
+ assertThat(ret.first, is(1));
+ assertThat(ret.second, sameInstance(r2));
+ // Only first ignore error from each reply
+ assertThatErrorsMatch(new Error[]{ }, ret);
+ }
+
+ @Test
+ public void nonIgnoredErrorTakesPrecedence() {
+ Reply r1 = new EmptyReply();
+ Reply r2 = new EmptyReply();
+ Error errors[] = new Error[] {
+ new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"),
+ new Error(DocumentProtocol.ERROR_ABORTED, "kablammo!"),
+ new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "omg!"),
+ };
+ r1.addError(errors[0]);
+ r1.addError(errors[1]);
+ r2.addError(errors[2]);
+
+ merger.merge(0, r1);
+ merger.merge(1, r2);
+ Tuple2<Integer, Reply> ret = merger.mergedReply();
+ assertThat(ret.first, nullValue());
+ assertThat(ret.second, not(sameInstance(r1)));
+ assertThat(ret.second, not(sameInstance(r2)));
+ // All errors from replies with errors are included, not those that
+ // are fully ignored.
+ assertThatErrorsMatch(new Error[]{ errors[0], errors[1] }, ret);
+ }
+
+ @Test
+ public void returnRemoveDocumentReplyWhereDocWasFound() {
+ RemoveDocumentReply r1 = new RemoveDocumentReply();
+ RemoveDocumentReply r2 = new RemoveDocumentReply();
+ RemoveDocumentReply r3 = new RemoveDocumentReply();
+ r1.setWasFound(false);
+ r2.setWasFound(true);
+ r3.setWasFound(false);
+
+ merger.merge(0, r1);
+ merger.merge(1, r2);
+ merger.merge(2, r3);
+ Tuple2<Integer, Reply> ret = merger.mergedReply();
+ assertThat(ret.first, is(1));
+ assertThat(ret.second, sameInstance((Reply) r2));
+ }
+
+ @Test
+ public void returnFirstRemoveDocumentReplyIfNoDocsWereFound() {
+ RemoveDocumentReply r1 = new RemoveDocumentReply();
+ RemoveDocumentReply r2 = new RemoveDocumentReply();
+ r1.setWasFound(false);
+ r2.setWasFound(false);
+
+ merger.merge(0, r1);
+ merger.merge(1, r2);
+ Tuple2<Integer, Reply> ret = merger.mergedReply();
+ assertThat(ret.first, is(0));
+ assertThat(ret.second, sameInstance((Reply)r1));
+ }
+
+ @Test
+ public void returnUpdateDocumentReplyWhereDocWasFound() {
+ UpdateDocumentReply r1 = new UpdateDocumentReply();
+ UpdateDocumentReply r2 = new UpdateDocumentReply();
+ UpdateDocumentReply r3 = new UpdateDocumentReply();
+ r1.setWasFound(false);
+ r2.setWasFound(true); // return first reply
+ r3.setWasFound(true);
+
+ merger.merge(0, r1);
+ merger.merge(1, r2);
+ merger.merge(2, r3);
+ Tuple2<Integer, Reply> ret = merger.mergedReply();
+ assertThat(ret.first, is(1));
+ assertThat(ret.second, sameInstance((Reply)r2));
+ }
+
+ @Test
+ public void returnGetDocumentReplyWhereDocWasFound() {
+ GetDocumentReply r1 = new GetDocumentReply(null);
+ GetDocumentReply r2 = new GetDocumentReply(null);
+ GetDocumentReply r3 = new GetDocumentReply(null);
+ r2.setLastModified(12345L);
+
+ merger.merge(0, r1);
+ merger.merge(1, r2);
+ merger.merge(2, r3);
+ Tuple2<Integer, Reply> ret = merger.mergedReply();
+ assertThat(ret.first, is(1));
+ assertThat(ret.second, sameInstance((Reply)r2));
+ }
+
+ @Test
+ public void mergingZeroRepliesReturnsDefaultEmptyReply() {
+ Tuple2<Integer, Reply> ret = merger.mergedReply();
+ assertThat(ret.first, nullValue());
+ assertThat(ret.second, instanceOf(EmptyReply.class));
+ assertThatErrorsMatch(new Error[]{}, ret);
+ }
+
+ private void assertThatErrorsMatch(Error[] errors, Tuple2<Integer, Reply> ret) {
+ assertThat(ret.second.getNumErrors(), is(errors.length));
+ for (int i = 0; i < ret.second.getNumErrors(); ++i) {
+ assertThat(ret.second.getError(i).getCode(), is(errors[i].getCode()));
+ assertThat(ret.second.getError(i).getMessage(), is(errors[i].getMessage()));
+ }
+ }
+
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/ErrorCodesTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/ErrorCodesTest.java new file mode 100644 index 00000000000..16a6347d2ad --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/ErrorCodesTest.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test; + +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import org.junit.Test; + +import java.nio.charset.Charset; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +/** + * Test for ensuring that the definitions of error codes in Java match their + * C++ implementation counterparts. Any new DocumentProtocol error codes must + * also be added to this test. + * + * @author vekterli + */ +public class ErrorCodesTest { + private class NamedErrorCodes { + private final TreeMap<String, Integer> nameAndCode = new TreeMap<>(); + + public void put(String name, int code) { + nameAndCode.put(name, code); + } + + public String toSortedKeyValueString() { + return nameAndCode.entrySet().stream() + .map((kv) -> kv.getKey() + " " + kv.getValue()) + .collect(Collectors.joining("\n")); + } + } + + /** + * Always meant to be run against HEAD. + */ + @Test + public void errorCodesMatchCppDefinitions() throws Exception { + final NamedErrorCodes codes = new NamedErrorCodes(); + enumerateAllDocumentProtocolErrorCodes(codes); + + final String javaGoldenFile = TestFileUtil.getPath("HEAD-java-golden-error-codes.txt"); + final String javaGoldenData = codes.toSortedKeyValueString(); + TestFileUtil.writeToFile(javaGoldenFile, javaGoldenData); + + final String cppGoldenFile = TestFileUtil.getPath("HEAD-cpp-golden-error-codes.txt"); + final String cppGoldenData = new String(TestFileUtil.readFile(cppGoldenFile), Charset.forName("UTF-8")); + assertEquals(javaGoldenData, cppGoldenData); + } + + /** + * Emits all name to integral code value mappings for error codes that exist + * in the Document protocol. + * + * This list must be updated (here and in the C++ equivalent) whenever a new + * code is added, and the resulting file must be checked in. + */ + private void enumerateAllDocumentProtocolErrorCodes(NamedErrorCodes codes) { + codes.put("ERROR_MESSAGE_IGNORED", DocumentProtocol.ERROR_MESSAGE_IGNORED); + codes.put("ERROR_POLICY_FAILURE", DocumentProtocol.ERROR_POLICY_FAILURE); + codes.put("ERROR_DOCUMENT_NOT_FOUND", DocumentProtocol.ERROR_DOCUMENT_NOT_FOUND); + codes.put("ERROR_DOCUMENT_EXISTS", DocumentProtocol.ERROR_DOCUMENT_EXISTS); + codes.put("ERROR_REJECTED", DocumentProtocol.ERROR_REJECTED); + codes.put("ERROR_NOT_IMPLEMENTED", DocumentProtocol.ERROR_NOT_IMPLEMENTED); + codes.put("ERROR_ILLEGAL_PARAMETERS", DocumentProtocol.ERROR_ILLEGAL_PARAMETERS); + codes.put("ERROR_UNKNOWN_COMMAND", DocumentProtocol.ERROR_UNKNOWN_COMMAND); + codes.put("ERROR_NO_SPACE", DocumentProtocol.ERROR_NO_SPACE); + codes.put("ERROR_IGNORED", DocumentProtocol.ERROR_IGNORED); + codes.put("ERROR_INTERNAL_FAILURE", DocumentProtocol.ERROR_INTERNAL_FAILURE); + codes.put("ERROR_TEST_AND_SET_CONDITION_FAILED", DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED); + codes.put("ERROR_PROCESSING_FAILURE", DocumentProtocol.ERROR_PROCESSING_FAILURE); + codes.put("ERROR_TIMESTAMP_EXIST", DocumentProtocol.ERROR_TIMESTAMP_EXIST); // (sic) + codes.put("ERROR_NODE_NOT_READY", DocumentProtocol.ERROR_NODE_NOT_READY); + codes.put("ERROR_WRONG_DISTRIBUTION", DocumentProtocol.ERROR_WRONG_DISTRIBUTION); + codes.put("ERROR_ABORTED", DocumentProtocol.ERROR_ABORTED); + codes.put("ERROR_BUSY", DocumentProtocol.ERROR_BUSY); + codes.put("ERROR_NOT_CONNECTED", DocumentProtocol.ERROR_NOT_CONNECTED); + codes.put("ERROR_DISK_FAILURE", DocumentProtocol.ERROR_DISK_FAILURE); + codes.put("ERROR_IO_FAILURE", DocumentProtocol.ERROR_IO_FAILURE); + codes.put("ERROR_BUCKET_NOT_FOUND", DocumentProtocol.ERROR_BUCKET_NOT_FOUND); + codes.put("ERROR_BUCKET_DELETED", DocumentProtocol.ERROR_BUCKET_DELETED); + codes.put("ERROR_STALE_TIMESTAMP", DocumentProtocol.ERROR_STALE_TIMESTAMP); + codes.put("ERROR_SUSPENDED", DocumentProtocol.ERROR_SUSPENDED); + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java new file mode 100644 index 00000000000..3f92aaa323f --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test; + +import com.yahoo.documentapi.messagebus.protocol.LoadBalancer; +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.text.XMLWriter; +import org.junit.Test; + +import java.io.PrintWriter; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class LoadBalancerTestCase { + + @Test + public void requireThatParseExceptionIsReadable() { + assertIllegalArgument("foo", "bar", "Expected recipient on the form 'foo/x/[y.]number/z', got 'bar'."); + assertIllegalArgument("foo", "foobar", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foobar'."); + assertIllegalArgument("foo", "foo", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo'."); + assertIllegalArgument("foo", "foo/", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/'."); + assertIllegalArgument("foo", "foo/0", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/0'."); + assertIllegalArgument("foo", "foo/0.", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/0.'."); + assertIllegalArgument("foo", "foo/0.bar", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/0.bar'."); + assertIllegalArgument("foo", "foo/bar", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/bar'."); + assertIllegalArgument("foo", "foo/bar.", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/bar.'."); + assertIllegalArgument("foo", "foo/bar.0", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/bar.0'."); + } + + private static void assertIllegalArgument(String clusterName, String recipient, String expectedMessage) { + LoadBalancer.Metrics metric = new LoadBalancer.Metrics(""); + LoadBalancer policy = new LoadBalancer(clusterName, "", metric); + try { + fail("Expected exception, got index " + policy.getIndex(recipient) + "."); + } catch (IllegalArgumentException e) { + assertEquals(expectedMessage, e.getMessage()); + } + } + + @Test + public void testLoadBalancer() { + LoadBalancer.Metrics m = new LoadBalancer.Metrics(""); + LoadBalancer lb = new LoadBalancer("foo", "", m); + + Mirror.Entry[] entries = new Mirror.Entry[]{ new Mirror.Entry("foo/0/default", "tcp/bar:1"), + new Mirror.Entry("foo/1/default", "tcp/bar:2"), + new Mirror.Entry("foo/2/default", "tcp/bar:3") }; + List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); + + { + for (int i = 0; i < 99; i++) { + LoadBalancer.Node node = lb.getRecipient(entries); + assertEquals("foo/" + (i % 3) + "/default" , node.entry.getName()); + } + + assertEquals(33, weights.get(0).sent.get().intValue()); + assertEquals(33, weights.get(1).sent.get().intValue()); + assertEquals(33, weights.get(2).sent.get().intValue()); + + weights.get(0).sent.set(new AtomicLong(0)); + weights.get(1).sent.set(new AtomicLong(0)); + weights.get(2).sent.set(new AtomicLong(0)); + } + + { + // Simulate that one node is overloaded. It returns busy twice as often as the others. + for (int i = 0; i < 100; i++) { + lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/0/default", "tcp/bar:1"), weights.get(0)), true); + lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/0/default", "tcp/bar:1"), weights.get(0)), false); + lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/0/default", "tcp/bar:1"), weights.get(0)), false); + + lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/2/default", "tcp/bar:3"), weights.get(2)), true); + lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/2/default", "tcp/bar:3"), weights.get(2)), false); + lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/2/default", "tcp/bar:3"), weights.get(2)), false); + + lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/1/default", "tcp/bar:2"), weights.get(1)), true); + lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/1/default", "tcp/bar:2"), weights.get(1)), true); + lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/1/default", "tcp/bar:2"), weights.get(1)), false); + } + + PrintWriter writer = new PrintWriter(System.out); + m.toXML(new XMLWriter(writer)); + writer.flush(); + + assertEquals(421, (int)(100 * weights.get(0).weight.get() / weights.get(1).weight.get())); + assertEquals(100, (int)(100 * weights.get(1).weight.get())); + assertEquals(421, (int)(100 * weights.get(2).weight.get() / weights.get(1).weight.get())); + } + + + assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); + assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); + assertEquals("foo/1/default" , lb.getRecipient(entries).entry.getName()); + assertEquals("foo/2/default" , lb.getRecipient(entries).entry.getName()); + assertEquals("foo/2/default" , lb.getRecipient(entries).entry.getName()); + assertEquals("foo/2/default" , lb.getRecipient(entries).entry.getName()); + assertEquals("foo/2/default" , lb.getRecipient(entries).entry.getName()); + assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); + assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); + assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); + } + + @Test + public void testLoadBalancerOneItemOnly() { + LoadBalancer.Metrics m = new LoadBalancer.Metrics(""); + LoadBalancer lb = new LoadBalancer("foo", "", m); + + Mirror.Entry[] entries = new Mirror.Entry[]{ new Mirror.Entry("foo/0/default", "tcp/bar:1") }; + List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); + + assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); + + lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/0/default", "tcp/bar:1"), weights.get(0)), true); // busy + + assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); + + } + +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java new file mode 100644 index 00000000000..226f96f6553 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java @@ -0,0 +1,975 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.component.Version;
+import com.yahoo.document.*;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+import com.yahoo.document.idstring.IdString;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.documentapi.messagebus.protocol.*;
+import com.yahoo.messagebus.Routable;
+import com.yahoo.text.Utf8;
+import com.yahoo.vdslib.DocumentList;
+import com.yahoo.vdslib.Entry;
+import com.yahoo.vdslib.SearchResult;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Messages50TestCase extends MessagesTestBase {
+
+ @Override
+ protected void registerTests(Map<Integer, RunnableTest> out) {
+ // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
+ // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
+ out.put(DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE, new testBatchDocumentUpdateMessage());
+ out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage());
+ out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage());
+ out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage());
+ out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage());
+ out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage());
+ out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage());
+ out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage());
+ out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage());
+ out.put(DocumentProtocol.MESSAGE_MAPVISITOR, new testMapVisitorMessage());
+ out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage());
+ out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage());
+ out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage());
+ out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage());
+ out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage());
+ out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage());
+ out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage());
+ out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage());
+ out.put(DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE, new testBatchDocumentUpdateReply());
+ out.put(DocumentProtocol.REPLY_CREATEVISITOR, new testCreateVisitorReply());
+ out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply());
+ out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply());
+ out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply());
+ out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply());
+ out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply());
+ out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply());
+ out.put(DocumentProtocol.REPLY_GETDOCUMENT, new testGetDocumentReply());
+ out.put(DocumentProtocol.REPLY_MAPVISITOR, new testMapVisitorReply());
+ out.put(DocumentProtocol.REPLY_PUTDOCUMENT, new testPutDocumentReply());
+ out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply());
+ out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply());
+ out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply());
+ out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply());
+ out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply());
+ out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply());
+ out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply());
+ out.put(DocumentProtocol.REPLY_WRONGDISTRIBUTION, new testWrongDistributionReply());
+ }
+
+ @Override
+ protected Version version() {
+ return new Version(5, 0);
+ }
+
+ @Override
+ protected boolean shouldTestCoverage() {
+ return false;
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Tests
+ //
+ ////////////////////////////////////////////////////////////////////////////////
+
+ private static int BASE_MESSAGE_LENGTH = 5;
+
+ public class testRemoveLocationMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ {
+ RemoveLocationMessage msg = new RemoveLocationMessage("id.group == \"mygroup\"");
+ assertEquals(BASE_MESSAGE_LENGTH + 29, serialize("RemoveLocationMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (RemoveLocationMessage)deserialize("RemoveLocationMessage", DocumentProtocol.MESSAGE_REMOVELOCATION, lang);
+ assertEquals("id.group == \"mygroup\"", msg.getDocumentSelection());
+ }
+ }
+ }
+ }
+
+ public class testGetBucketListMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123));
+ msg.setLoadType(loadTypes.getNameMap().get("foo"));
+ assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketListMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang);
+ assertEquals(new BucketId(16, 123), msg.getBucketId());
+ assertEquals("foo", msg.getLoadType().getName());
+ }
+ }
+ }
+
+
+ public class testStatBucketMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123");
+ msg.setLoadType(null);
+ assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("StatBucketMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang);
+ assertEquals(new BucketId(16, 123), msg.getBucketId());
+ assertEquals("id.user=123", msg.getDocumentSelection());
+ assertEquals("default", msg.getLoadType().getName());
+ }
+ }
+ }
+
+ public class testGetBucketStateMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ GetBucketStateMessage msg = new GetBucketStateMessage(new BucketId(16, 666));
+ assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketStateMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (GetBucketStateMessage)deserialize("GetBucketStateMessage", DocumentProtocol.MESSAGE_GETBUCKETSTATE, lang);
+ assertEquals(16, msg.getBucketId().getUsedBits());
+ assertEquals(4611686018427388570l, msg.getBucketId().getId());
+ }
+ }
+ }
+
+ public class testCreateVisitorMessage implements RunnableTest {
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void run() {
+ CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london");
+ msg.setDocumentSelection("true and false or true");
+ msg.getParameters().put("myvar", Utf8.toBytes("somevalue"));
+ msg.getParameters().put("anothervar", Utf8.toBytes("34"));
+ msg.getBuckets().add(new BucketId(16, 1234));
+ msg.setVisitRemoves(true);
+ msg.setVisitorOrdering(OrderingSpecification.DESCENDING);
+ msg.setMaxBucketsPerVisitor(2);
+ assertEquals(BASE_MESSAGE_LENGTH + 168, serialize("CreateVisitorMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang);
+ assertEquals("SomeLibrary", msg.getLibraryName());
+ assertEquals("myvisitor", msg.getInstanceId());
+ assertEquals("newyork", msg.getControlDestination());
+ assertEquals("london", msg.getDataDestination());
+ assertEquals("true and false or true", msg.getDocumentSelection());
+ assertEquals(8, msg.getMaxPendingReplyCount());
+ assertEquals(true, msg.getVisitRemoves());
+ assertEquals(false, msg.getVisitInconsistentBuckets());
+ assertEquals(1, msg.getBuckets().size());
+ assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next());
+ assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar")));
+ assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar")));
+ assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering());
+ assertEquals(2, msg.getMaxBucketsPerVisitor());
+ }
+
+ msg.getBuckets().clear();
+
+ assertEquals("CreateVisitorMessage(" +
+ "No buckets, " +
+ "selection 'true and false or true', " +
+ "library SomeLibrary, including removes, " +
+ "get fields: [all]" +
+ ")",
+ msg.toString());
+
+ msg.getBuckets().add(new BucketId(16, 1234));
+
+ assertEquals("CreateVisitorMessage(" +
+ "Bucket BucketId(0x40000000000004d2), " +
+ "selection 'true and false or true', " +
+ "library SomeLibrary, including removes, " +
+ "get fields: [all]" +
+ ")",
+ msg.toString());
+
+ msg.getBuckets().add(new BucketId(16, 1235));
+ msg.getBuckets().add(new BucketId(16, 1236));
+ msg.getBuckets().add(new BucketId(16, 1237));
+ msg.getBuckets().add(new BucketId(16, 1238));
+ msg.setFromTimestamp(10001);
+ msg.setToTimestamp(20002);
+ msg.setVisitInconsistentBuckets(true);
+ assertEquals("CreateVisitorMessage(" +
+ "5 buckets: BucketId(0x40000000000004d2) BucketId(0x40000000000004d3) BucketId(0x40000000000004d4) ..., " +
+ "time 10001-20002, " +
+ "selection 'true and false or true', " +
+ "library SomeLibrary, including removes, " +
+ "get fields: [all], " +
+ "visit inconsistent buckets" +
+ ")",
+ msg.toString());
+ }
+ }
+
+ public class testCreateVisitorReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR);
+ reply.setLastBucket(new BucketId(16, 123));
+ reply.getVisitorStatistics().setBucketsVisited(3);
+ reply.getVisitorStatistics().setDocumentsVisited(1000);
+ reply.getVisitorStatistics().setBytesVisited(1024000);
+ reply.getVisitorStatistics().setDocumentsReturned(123);
+ reply.getVisitorStatistics().setBytesReturned(512000);
+ reply.getVisitorStatistics().setSecondPassDocumentsReturned(456);
+ reply.getVisitorStatistics().setSecondPassBytesReturned(789100);
+
+ assertEquals(65, serialize("CreateVisitorReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (CreateVisitorReply)deserialize("CreateVisitorReply", DocumentProtocol.REPLY_CREATEVISITOR, lang);
+ assertNotNull(reply);
+ assertEquals(new BucketId(16, 123), reply.getLastBucket());
+ assertEquals(3, reply.getVisitorStatistics().getBucketsVisited());
+ assertEquals(1000, reply.getVisitorStatistics().getDocumentsVisited());
+ assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited());
+ assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned());
+ assertEquals(512000, reply.getVisitorStatistics().getBytesReturned());
+ assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned());
+ assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned());
+ }
+ }
+ }
+
+ public class testDestroyVisitorReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("DestroyVisitorReply", DocumentProtocol.REPLY_DESTROYVISITOR);
+ }
+ }
+
+ public class testDocumentListReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("DocumentListReply", DocumentProtocol.REPLY_DOCUMENTLIST);
+ }
+ }
+
+ public class testDocumentSummaryReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY);
+ }
+ }
+
+ public class testEmptyBucketsReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("EmptyBucketsReply", DocumentProtocol.REPLY_EMPTYBUCKETS);
+ }
+ }
+
+ public class testDestroyVisitorMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ DestroyVisitorMessage msg = new DestroyVisitorMessage("myvisitor");
+ assertEquals(BASE_MESSAGE_LENGTH + 17, serialize("DestroyVisitorMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (DestroyVisitorMessage)deserialize("DestroyVisitorMessage", DocumentProtocol.MESSAGE_DESTROYVISITOR, lang);
+ assertEquals("myvisitor", msg.getInstanceId());
+ }
+ }
+ }
+
+ public class testDocumentListMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ DocumentListMessage msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.CPP);
+ assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString());
+ assertEquals(1234, msg.getDocuments().get(0).getTimestamp());
+ assertFalse(msg.getDocuments().get(0).isRemoveEntry());
+
+ assertEquals(BASE_MESSAGE_LENGTH + 63, serialize("DocumentListMessage", msg));
+ msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.JAVA);
+ assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString());
+ assertEquals(1234, msg.getDocuments().get(0).getTimestamp());
+ assertFalse(msg.getDocuments().get(0).isRemoveEntry());
+
+ }
+ }
+
+ public class testEmptyBucketsMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ List<BucketId> bids = new ArrayList<>();
+ for (int i = 0; i < 13; ++i) {
+ bids.add(new BucketId(16, i));
+ }
+
+ EmptyBucketsMessage ebm = new EmptyBucketsMessage(bids);
+ assertEquals(BASE_MESSAGE_LENGTH + 112, serialize("EmptyBucketsMessage", ebm));
+ for (Language lang : LANGUAGES) {
+ ebm = (EmptyBucketsMessage)deserialize("EmptyBucketsMessage", DocumentProtocol.MESSAGE_EMPTYBUCKETS, lang);
+ for (int i = 0; i < 13; ++i) {
+ assertEquals(new BucketId(16, i), ebm.getBucketIds().get(i));
+ }
+ }
+ }
+ }
+
+ public class testDocumentSummaryMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ try {
+ FileInputStream stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-1.dat"));
+ byte[] data = new byte[stream.available()];
+ assertEquals(data.length, stream.read(data));
+
+ Routable routable = decode(data);
+ assertTrue(routable instanceof DocumentSummaryMessage);
+
+ DocumentSummaryMessage msg = (DocumentSummaryMessage)routable;
+ assertEquals(0, msg.getResult().getSummaryCount());
+
+ stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-2.dat"));
+ data = new byte[stream.available()];
+ assertEquals(data.length, stream.read(data));
+
+ routable = decode(data);
+ assertTrue(routable instanceof DocumentSummaryMessage);
+
+ msg = (DocumentSummaryMessage)routable;
+ assertEquals(2, msg.getResult().getSummaryCount());
+ com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0);
+ assertEquals("doc1", s.getDocId());
+ byte[] b = s.getSummary();
+ assertEquals(8, b.length);
+ byte[] c = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(c[i], b[i]);
+ }
+
+ s = msg.getResult().getSummary(1);
+ assertEquals("aoc17", s.getDocId());
+ b = s.getSummary();
+ assertEquals(9, b.length);
+ byte[] d = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(d[i], b[i]);
+ }
+
+ stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-3.dat"));
+ data = new byte[stream.available()];
+ assertEquals(data.length, stream.read(data));
+
+ routable = decode(data);
+ assertTrue(routable instanceof DocumentSummaryMessage);
+
+ msg = (DocumentSummaryMessage)routable;
+ assertEquals(2, msg.getResult().getSummaryCount());
+
+ s = msg.getResult().getSummary(0);
+ assertEquals("aoc17", s.getDocId());
+ b = s.getSummary();
+ assertEquals(9, b.length);
+ byte[] e = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(e[i], b[i]);
+ }
+
+ s = msg.getResult().getSummary(1);
+ assertEquals("doc1", s.getDocId());
+ b = s.getSummary();
+ assertEquals(8, b.length);
+ byte[] f = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(f[i], b[i]);
+ }
+ } catch (IOException e) {
+ fail(e.toString());
+ }
+ }
+ }
+
+
+ public class testGetDocumentMessage implements RunnableTest {
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void run() {
+ GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"));
+ assertEquals(BASE_MESSAGE_LENGTH + 20, serialize("GetDocumentMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang);
+ assertEquals("doc:scheme:", msg.getDocumentId().toString());
+ }
+ }
+ }
+
+
+ public class testRemoveDocumentMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+ assertEquals(BASE_MESSAGE_LENGTH + 16, serialize("RemoveDocumentMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang);
+ assertEquals("doc:scheme:", msg.getDocumentId().toString());
+ }
+ }
+ }
+
+ public class testMapVisitorMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ MapVisitorMessage msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.CPP);
+ assertEquals("3", msg.getData().get("foo"));
+ assertEquals("5", msg.getData().get("bar"));
+
+ assertEquals(BASE_MESSAGE_LENGTH + 32, serialize("MapVisitorMessage", msg));
+
+ msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.JAVA);
+ assertEquals("3", msg.getData().get("foo"));
+ assertEquals("5", msg.getData().get("bar"));
+ }
+ }
+
+
+ public class testVisitorInfoMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ VisitorInfoMessage msg = new VisitorInfoMessage();
+ msg.getFinishedBuckets().add(new BucketId(16, 1));
+ msg.getFinishedBuckets().add(new BucketId(16, 2));
+ msg.getFinishedBuckets().add(new BucketId(16, 4));
+ msg.setErrorMessage("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6");
+ assertEquals(BASE_MESSAGE_LENGTH + 67, serialize("VisitorInfoMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (VisitorInfoMessage)deserialize("VisitorInfoMessage", DocumentProtocol.MESSAGE_VISITORINFO, lang);
+ assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 1)));
+ assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 2)));
+ assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 4)));
+ assertEquals("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6", msg.getErrorMessage());
+ }
+ }
+ }
+
+ public class testSearchResultMessage implements RunnableTest {
+
+ @Override
+ public void run() throws Exception {
+ FileInputStream stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-1.dat"));
+ byte[] data = new byte[stream.available()];
+ assertEquals(data.length, stream.read(data));
+
+ Routable routable = decode(data);
+ assertTrue(routable instanceof SearchResultMessage);
+
+ SearchResultMessage msg = (SearchResultMessage)routable;
+ assertEquals(0, msg.getResult().getHitCount());
+
+ stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-2.dat"));
+ data = new byte[stream.available()];
+ assertEquals(data.length, stream.read(data));
+
+ routable = decode(data);
+ assertTrue(routable instanceof SearchResultMessage);
+
+ msg = (SearchResultMessage)routable;
+ assertEquals(2, msg.getResult().getHitCount());
+ com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0);
+ assertEquals(89.0, h.getRank(), 1E-6);
+ assertEquals("doc1", h.getDocId());
+ h = msg.getResult().getHit(1);
+ assertEquals(109.0, h.getRank(), 1E-6);
+ assertEquals("doc17", h.getDocId());
+
+ stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-3.dat"));
+ data = new byte[stream.available()];
+ assertEquals(data.length, stream.read(data));
+
+ routable = decode(data);
+ assertTrue(routable instanceof SearchResultMessage);
+
+ msg = (SearchResultMessage)routable;
+ assertEquals(2, msg.getResult().getHitCount());
+ h = msg.getResult().getHit(0);
+ assertEquals(109.0, h.getRank(), 1E-6);
+ assertEquals("doc17", h.getDocId());
+ h = msg.getResult().getHit(1);
+ assertEquals(89.0, h.getRank(), 1E-6);
+ assertEquals("doc1", h.getDocId());
+
+ stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-4.dat"));
+ data = new byte[stream.available()];
+ assertEquals(data.length, stream.read(data));
+
+ routable = decode(data);
+ assertTrue(routable instanceof SearchResultMessage);
+
+ msg = (SearchResultMessage)routable;
+ assertEquals(3, msg.getResult().getHitCount());
+ h = msg.getResult().getHit(0);
+ assertTrue(h instanceof SearchResult.HitWithSortBlob);
+ assertEquals(89.0, h.getRank(), 1E-6);
+ assertEquals("doc1", h.getDocId());
+ byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+ assertEquals(9, b.length);
+ byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(e[i], b[i]);
+ }
+ h = msg.getResult().getHit(1);
+ assertTrue(h instanceof SearchResult.HitWithSortBlob);
+ assertEquals(109.0, h.getRank(), 1E-6);
+ assertEquals("doc17", h.getDocId());
+ b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+ assertEquals(9, b.length);
+ byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(d[i], b[i]);
+ }
+ h = msg.getResult().getHit(2);
+ assertTrue(h instanceof SearchResult.HitWithSortBlob);
+ assertEquals(90.0, h.getRank(), 1E-6);
+ assertEquals("doc18", h.getDocId());
+ b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+ assertEquals(9, b.length);
+ byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(c[i], b[i]);
+ }
+ }
+ }
+
+ public class testPutDocumentMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")));
+ msg.setTimestamp(666);
+ assertEquals(BASE_MESSAGE_LENGTH + 41, serialize("PutDocumentMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);
+ assertEquals("testdoc", msg.getDocumentPut().getDocument().getDataType().getName());
+ assertEquals("doc:scheme:", msg.getDocumentPut().getDocument().getId().toString());
+ assertEquals(666, msg.getTimestamp());
+ }
+ }
+ }
+
+ public class testPutDocumentReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ WriteDocumentReply reply = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT);
+ reply.setHighestModificationTimestamp(30);
+
+ assertEquals(13, serialize("PutDocumentReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ WriteDocumentReply obj = (WriteDocumentReply)deserialize("PutDocumentReply", DocumentProtocol.REPLY_PUTDOCUMENT, lang);
+ assertNotNull(obj);
+ assertEquals(30, obj.getHighestModificationTimestamp());
+ }
+ }
+ }
+
+ public class testUpdateDocumentMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");
+ DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:"));
+ update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
+ UpdateDocumentMessage msg = new UpdateDocumentMessage(update);
+ msg.setNewTimestamp(777);
+ msg.setOldTimestamp(666);
+
+ assertEquals(BASE_MESSAGE_LENGTH + 89, serialize("UpdateDocumentMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (UpdateDocumentMessage)deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang);
+ assertEquals(update, msg.getDocumentUpdate());
+ assertEquals(777, msg.getNewTimestamp());
+ assertEquals(666, msg.getOldTimestamp());
+ }
+ }
+ }
+
+ public class testUpdateDocumentReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ UpdateDocumentReply reply = new UpdateDocumentReply();
+ reply.setHighestModificationTimestamp(30);
+ reply.setWasFound(false);
+
+ assertEquals(14, serialize("UpdateDocumentReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ UpdateDocumentReply obj = (UpdateDocumentReply)deserialize("UpdateDocumentReply", DocumentProtocol.REPLY_UPDATEDOCUMENT, lang);
+ assertNotNull(obj);
+ assertEquals(30, reply.getHighestModificationTimestamp());
+ assertEquals(false, obj.wasFound());
+ }
+ }
+ }
+
+ public class testVisitorInfoReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("VisitorInfoReply", DocumentProtocol.REPLY_VISITORINFO);
+ }
+ }
+
+ public class testWrongDistributionReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ WrongDistributionReply reply = new WrongDistributionReply("distributor:3 storage:2");
+ assertEquals(32, serialize("WrongDistributionReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (WrongDistributionReply)deserialize("WrongDistributionReply", DocumentProtocol.REPLY_WRONGDISTRIBUTION, lang);
+ assertEquals("distributor:3 storage:2", reply.getSystemState());
+ }
+ }
+ }
+
+ public class testRemoveDocumentReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ RemoveDocumentReply reply = new RemoveDocumentReply();
+ reply.setHighestModificationTimestamp(30);
+ reply.setWasFound(false);
+
+ assertEquals(14, serialize("RemoveDocumentReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ RemoveDocumentReply obj = (RemoveDocumentReply)deserialize("RemoveDocumentReply", DocumentProtocol.REPLY_REMOVEDOCUMENT, lang);
+ assertNotNull(obj);
+ assertEquals(30, obj.getHighestModificationTimestamp());
+ assertEquals(false, obj.wasFound());
+ }
+ }
+ }
+
+ public class testRemoveLocationReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testDocumentReply("RemoveLocationReply", DocumentProtocol.REPLY_REMOVELOCATION);
+ }
+ }
+
+ public class testSearchResultReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT);
+ }
+ }
+
+ public class testStatBucketReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ StatBucketReply msg = new StatBucketReply();
+ msg.setResults("These are the votes of the Norwegian jury");
+
+ assertEquals(50, serialize("StatBucketReply", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (StatBucketReply)deserialize("StatBucketReply", DocumentProtocol.REPLY_STATBUCKET, lang);
+ assertEquals("These are the votes of the Norwegian jury", msg.getResults());
+ }
+ }
+ }
+
+ public class testBatchDocumentUpdateMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");
+ BatchDocumentUpdateMessage msg = new BatchDocumentUpdateMessage(1234);
+
+ {
+ DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("userdoc:footype:1234:foo"));
+ update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
+ msg.addUpdate(update);
+ }
+ {
+ DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("orderdoc(32,17):footype:1234:123456789:foo"));
+ update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
+ msg.addUpdate(update);
+ }
+
+ try {
+ DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("orderdoc:footype:5678:foo"));
+ update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
+ msg.addUpdate(update);
+ fail();
+ } catch (Exception e) {
+
+ }
+
+ try {
+ DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("groupdoc:footype:hable:foo"));
+ update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
+ msg.addUpdate(update);
+ fail();
+ } catch (Exception e) {
+
+ }
+
+ assertEquals(2, msg.getUpdates().size());
+
+ assertEquals(BASE_MESSAGE_LENGTH + 202, serialize("BatchDocumentUpdateMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (BatchDocumentUpdateMessage)deserialize("BatchDocumentUpdateMessage", DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE, lang);
+ assertEquals(2, msg.getUpdates().size());
+ }
+ }
+ }
+
+ public class testBatchDocumentUpdateReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ BatchDocumentUpdateReply reply = new BatchDocumentUpdateReply();
+ reply.setHighestModificationTimestamp(30);
+ reply.getDocumentsNotFound().add(false);
+ reply.getDocumentsNotFound().add(true);
+ reply.getDocumentsNotFound().add(true);
+
+ assertEquals(20, serialize("BatchDocumentUpdateReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ BatchDocumentUpdateReply obj = (BatchDocumentUpdateReply)deserialize("BatchDocumentUpdateReply", DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE, lang);
+ assertNotNull(obj);
+ assertEquals(30, obj.getHighestModificationTimestamp());
+ assertEquals(3, obj.getDocumentsNotFound().size());
+ assertFalse(obj.getDocumentsNotFound().get(0));
+ assertTrue(obj.getDocumentsNotFound().get(1));
+ assertTrue(obj.getDocumentsNotFound().get(2));
+ }
+ }
+ }
+
+
+ public class testQueryResultReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ testVisitorReply("QueryResultReply", DocumentProtocol.REPLY_QUERYRESULT);
+ }
+ }
+
+ public class testQueryResultMessage implements RunnableTest {
+
+ @Override
+ public void run() throws Exception {
+ FileInputStream stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-1.dat"));
+ byte[] data = new byte[stream.available()];
+ assertEquals(data.length, stream.read(data));
+
+ Routable routable = decode(data);
+ assertTrue(routable instanceof QueryResultMessage);
+
+ QueryResultMessage msg = (QueryResultMessage)routable;
+ assertEquals(0, msg.getResult().getHitCount());
+
+ stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-2.dat"));
+ data = new byte[stream.available()];
+ assertEquals(data.length, stream.read(data));
+
+ routable = decode(data);
+ assertTrue(routable instanceof QueryResultMessage);
+
+ msg = (QueryResultMessage)routable;
+ assertEquals(2, msg.getResult().getHitCount());
+ com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0);
+ assertEquals(89.0, h.getRank(), 1E-6);
+ assertEquals("doc1", h.getDocId());
+ h = msg.getResult().getHit(1);
+ assertEquals(109.0, h.getRank(), 1E-6);
+ assertEquals("doc17", h.getDocId());
+
+ stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-3.dat"));
+ data = new byte[stream.available()];
+ assertEquals(data.length, stream.read(data));
+
+ routable = decode(data);
+ assertTrue(routable instanceof QueryResultMessage);
+
+ msg = (QueryResultMessage)routable;
+ assertEquals(2, msg.getResult().getHitCount());
+ h = msg.getResult().getHit(0);
+ assertEquals(109.0, h.getRank(), 1E-6);
+ assertEquals("doc17", h.getDocId());
+ h = msg.getResult().getHit(1);
+ assertEquals(89.0, h.getRank(), 1E-6);
+ assertEquals("doc1", h.getDocId());
+
+ stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-4.dat"));
+ data = new byte[stream.available()];
+ assertEquals(data.length, stream.read(data));
+
+ routable = decode(data);
+ assertTrue(routable instanceof QueryResultMessage);
+
+ msg = (QueryResultMessage)routable;
+ assertEquals(3, msg.getResult().getHitCount());
+ h = msg.getResult().getHit(0);
+ assertTrue(h instanceof SearchResult.HitWithSortBlob);
+ assertEquals(89.0, h.getRank(), 1E-6);
+ assertEquals("doc1", h.getDocId());
+ byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+ assertEquals(9, b.length);
+ byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(e[i], b[i]);
+ }
+ h = msg.getResult().getHit(1);
+ assertTrue(h instanceof SearchResult.HitWithSortBlob);
+ assertEquals(109.0, h.getRank(), 1E-6);
+ assertEquals("doc17", h.getDocId());
+ b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+ assertEquals(9, b.length);
+ byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(d[i], b[i]);
+ }
+ h = msg.getResult().getHit(2);
+ assertTrue(h instanceof SearchResult.HitWithSortBlob);
+ assertEquals(90.0, h.getRank(), 1E-6);
+ assertEquals("doc18", h.getDocId());
+ b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
+ assertEquals(9, b.length);
+ byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' };
+ for (int i = 0; i < b.length; i++) {
+ assertEquals(c[i], b[i]);
+ }
+ }
+ }
+
+ public class testGetBucketListReply implements RunnableTest {
+
+ public void run() {
+ GetBucketListReply reply = new GetBucketListReply();
+ reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo"));
+ reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar"));
+ reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink"));
+
+ assertEquals(56, serialize("GetBucketListReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (GetBucketListReply)deserialize("GetBucketListReply", DocumentProtocol.REPLY_GETBUCKETLIST, lang);
+ assertEquals(reply.getBuckets().get(0), new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo"));
+ assertEquals(reply.getBuckets().get(1), new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar"));
+ assertEquals(reply.getBuckets().get(2), new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink"));
+ }
+ }
+ }
+
+ public class testGetBucketStateReply implements RunnableTest {
+
+ public void run() {
+ GlobalId foo = new GlobalId(IdString.createIdString("doc:scheme:foo"));
+ GlobalId bar = new GlobalId(IdString.createIdString("doc:scheme:bar"));
+
+ GetBucketStateReply reply = new GetBucketStateReply();
+ List<DocumentState> state = new ArrayList<>(2);
+ state.add(new DocumentState(foo, 777, false));
+ state.add(new DocumentState(bar, 888, true));
+ reply.setBucketState(state);
+ assertEquals(53, serialize("GetBucketStateReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (GetBucketStateReply)deserialize("GetBucketStateReply", DocumentProtocol.REPLY_GETBUCKETSTATE, lang);
+ assertEquals(777, reply.getBucketState().get(0).getTimestamp());
+ assertEquals(foo, reply.getBucketState().get(0).getGid());
+ assertEquals(false, reply.getBucketState().get(0).isRemoveEntry());
+ assertEquals(888, reply.getBucketState().get(1).getTimestamp());
+ assertEquals(bar, reply.getBucketState().get(1).getGid());
+ assertEquals(true, reply.getBucketState().get(1).isRemoveEntry());
+ }
+ }
+ }
+
+ public class testGetDocumentReply implements RunnableTest {
+
+ public void run() {
+ GetDocumentReply reply = new GetDocumentReply(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:"));
+ assertEquals(43, serialize("GetDocumentReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (GetDocumentReply)deserialize("GetDocumentReply", DocumentProtocol.REPLY_GETDOCUMENT, lang);
+ assertEquals("testdoc", reply.getDocument().getDataType().getName());
+ assertEquals("doc:scheme:", reply.getDocument().getId().toString());
+ }
+ }
+ }
+
+ public class testMapVisitorReply implements RunnableTest {
+
+ public void run() {
+ testVisitorReply("MapVisitorReply", DocumentProtocol.REPLY_MAPVISITOR);
+ }
+ }
+
+ protected void testDocumentReply(String filename, int type) {
+ DocumentReply reply = new DocumentReply(type);
+ assertEquals(5, serialize(filename, reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (DocumentReply)deserialize(filename, type, lang);
+ assertNotNull(reply);
+ }
+ }
+
+ protected void testVisitorReply(String filename, int type) {
+ VisitorReply reply = new VisitorReply(type);
+ assertEquals(5, serialize(filename, reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (VisitorReply)deserialize(filename, type, lang);
+ assertNotNull(reply);
+ }
+ }
+
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java new file mode 100644 index 00000000000..68448adb8fc --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.component.Version;
+import com.yahoo.document.BucketId;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.documentapi.messagebus.protocol.CreateVisitorMessage;
+import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
+import com.yahoo.text.Utf8;
+
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Messages51TestCase extends Messages50TestCase {
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Setup
+ //
+ ///////////////////////////////////////////////////////////////////////////////
+
+ @Override
+ protected void registerTests(Map<Integer, RunnableTest> out) {
+ super.registerTests(out);
+
+ // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
+ // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
+ out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage());
+ out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage());
+ out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply());
+ }
+
+ @Override
+ protected Version version() {
+ return new Version(5, 1);
+ }
+
+ @Override
+ protected boolean shouldTestCoverage() {
+ return true;
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Tests
+ //
+ ////////////////////////////////////////////////////////////////////////////////
+
+ private static int BASE_MESSAGE_LENGTH = 5;
+
+ public class testCreateVisitorMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london");
+ msg.setDocumentSelection("true and false or true");
+ msg.getParameters().put("myvar", Utf8.toBytes("somevalue"));
+ msg.getParameters().put("anothervar", Utf8.toBytes("34"));
+ msg.getBuckets().add(new BucketId(16, 1234));
+ msg.setVisitRemoves(true);
+ msg.setFieldSet("foo bar");
+ msg.setVisitorOrdering(OrderingSpecification.DESCENDING);
+ msg.setMaxBucketsPerVisitor(2);
+ assertEquals(BASE_MESSAGE_LENGTH + 178, serialize("CreateVisitorMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang);
+ assertEquals("SomeLibrary", msg.getLibraryName());
+ assertEquals("myvisitor", msg.getInstanceId());
+ assertEquals("newyork", msg.getControlDestination());
+ assertEquals("london", msg.getDataDestination());
+ assertEquals("true and false or true", msg.getDocumentSelection());
+ assertEquals(8, msg.getMaxPendingReplyCount());
+ assertEquals(true, msg.getVisitRemoves());
+ assertEquals("foo bar", msg.getFieldSet());
+ assertEquals(false, msg.getVisitInconsistentBuckets());
+ assertEquals(1, msg.getBuckets().size());
+ assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next());
+ assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar")));
+ assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar")));
+ assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering());
+ assertEquals(2, msg.getMaxBucketsPerVisitor());
+ }
+ }
+ }
+
+ public class testGetDocumentMessage implements RunnableTest {
+
+ @Override
+ public void run() {
+ GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo bar");
+ assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("GetDocumentMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang);
+ assertEquals("doc:scheme:", msg.getDocumentId().toString());
+ assertEquals("foo bar", msg.getFieldSet());
+ }
+ }
+ }
+
+ public class testDocumentIgnoredReply implements RunnableTest {
+
+ @Override
+ public void run() {
+ DocumentIgnoredReply reply = new DocumentIgnoredReply();
+ assertEquals(BASE_MESSAGE_LENGTH, serialize("DocumentIgnoredReply", reply));
+
+ for (Language lang : LANGUAGES) {
+ reply = (DocumentIgnoredReply)deserialize("DocumentIgnoredReply", DocumentProtocol.REPLY_DOCUMENTIGNORED, lang);
+ }
+ }
+ }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java new file mode 100644 index 00000000000..8198044a25e --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.google.common.annotations.Beta;
+import com.yahoo.component.Version;
+import com.yahoo.document.*;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+import com.yahoo.documentapi.messagebus.protocol.*;
+
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author Vegard Sjonfjell
+ */
+
+@Beta
+public class Messages52TestCase extends Messages51TestCase {
+
+ @Override
+ protected Version version() {
+ return new Version(5, 115, 0);
+ }
+
+ @Override
+ protected boolean shouldTestCoverage() {
+ return true;
+ }
+
+ @Override
+ protected void registerTests(Map<Integer, RunnableTest> out) {
+ super.registerTests(out);
+
+ // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
+ // version 5.2. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
+
+ out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage());
+ out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage());
+ out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage());
+ }
+
+ private static int BASE_MESSAGE_LENGTH = 5;
+ private static String CONDITION_STRING = "There's just one condition";
+
+ public class testPutDocumentMessage implements RunnableTest {
+ @Override
+ public void run() {
+ PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")));
+
+ msg.setTimestamp(666);
+ msg.setCondition(new TestAndSetCondition(CONDITION_STRING));
+
+ assertEquals(BASE_MESSAGE_LENGTH + 41 + serializedLength(msg.getCondition().getSelection()), serialize("PutDocumentMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ final PutDocumentMessage deserializedMsg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);
+ assertEquals(msg.getDocumentPut().getDocument().getDataType().getName(), deserializedMsg.getDocumentPut().getDocument().getDataType().getName());
+ assertEquals(msg.getDocumentPut().getDocument().getId().toString(), deserializedMsg.getDocumentPut().getDocument().getId().toString());
+ assertEquals(msg.getTimestamp(), deserializedMsg.getTimestamp());
+ assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection());
+ }
+ }
+ }
+
+ public class testUpdateDocumentMessage implements RunnableTest {
+ @Override
+ public void run() {
+ DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");
+ DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:"));
+ update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
+
+ final UpdateDocumentMessage msg = new UpdateDocumentMessage(update);
+ msg.setNewTimestamp(777);
+ msg.setOldTimestamp(666);
+ msg.setCondition(new TestAndSetCondition(CONDITION_STRING));
+
+ assertEquals(BASE_MESSAGE_LENGTH + 89 + serializedLength(msg.getCondition().getSelection()), serialize("UpdateDocumentMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ final UpdateDocumentMessage deserializedMsg = (UpdateDocumentMessage) deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang);
+ assertEquals(msg.getDocumentUpdate(), deserializedMsg.getDocumentUpdate());
+ assertEquals(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp());
+ assertEquals(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp());
+ assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection());
+ }
+ }
+ }
+
+ public class testRemoveDocumentMessage implements RunnableTest {
+ @Override
+ public void run() {
+ final RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+ msg.setCondition(new TestAndSetCondition(CONDITION_STRING));
+
+ assertEquals(BASE_MESSAGE_LENGTH + 16 + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg));
+
+ for (Language lang : LANGUAGES) {
+ final RemoveDocumentMessage deserializedMsg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang);
+ assertEquals(deserializedMsg.getDocumentId().toString(), msg.getDocumentId().toString());
+ }
+ }
+ }
+
+ static int serializedLength(String str) {
+ return 4 + str.length();
+ }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java new file mode 100755 index 00000000000..f69454a99f4 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.component.Version;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.messagebus.Routable;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class MessagesTestBase {
+
+ protected enum Language {
+ JAVA,
+ CPP
+ }
+ protected static final Set<Language> LANGUAGES = EnumSet.allOf(Language.class);
+
+ protected final DocumentTypeManager docMan = new DocumentTypeManager();
+ protected final LoadTypeSet loadTypes = new LoadTypeSet();
+ protected final DocumentProtocol protocol = new DocumentProtocol(docMan, null, loadTypes);
+
+ public MessagesTestBase() {
+ DocumentTypeManagerConfigurer.configure(docMan, "file:./test/cfg/testdoc.cfg");
+ loadTypes.addLoadType(34, "foo", DocumentProtocol.Priority.NORMAL_2);
+ }
+
+ @Test
+ public void requireThatTestsPass() throws Exception {
+ Map<Integer, RunnableTest> tests = new TreeMap<>();
+ registerTests(tests);
+ for (Map.Entry<Integer, RunnableTest> entry : tests.entrySet()) {
+ entry.getValue().run();
+ }
+ if (shouldTestCoverage()) {
+ assertCoverage(protocol.getRoutableTypes(version()), new ArrayList<>(tests.keySet()));
+ }
+ }
+
+ /**
+ * Returns the version to use for serialization.
+ *
+ * @return The version.
+ */
+ protected abstract Version version();
+
+ /**
+ * Registers the tests to run.
+ */
+ protected abstract void registerTests(Map<Integer, RunnableTest> out);
+
+ /**
+ * Returns whether or not to test message test coverage.
+ */
+ protected abstract boolean shouldTestCoverage();
+
+ /**
+ * Encodes the given routable using the current version of the test case.
+ *
+ * @param routable The routable to encode.
+ * @return The encoded data.
+ */
+ public byte[] encode(Routable routable) {
+ return protocol.encode(version(), routable);
+ }
+
+ /**
+ * Decodes the given byte array using the current version of the test case.
+ *
+ * @param data The data to decode.
+ * @return The decoded routable.
+ */
+ public Routable decode(byte[] data) {
+ return protocol.decode(version(), data);
+ }
+
+ public String getPath(String filename) {
+ return TestFileUtil.getPath(filename);
+ }
+
+ /**
+ * Writes the content of the given routable to the given file.
+ *
+ * @param filename The name of the file to write to.
+ * @param routable The routable to serialize.
+ * @return The size of the written file.
+ */
+ public int serialize(String filename, Routable routable) {
+ Version version = version();
+ String path = getPath(version + "-java-" + filename + ".dat");
+ System.out.println("Serializing to '" + path + "'..");
+ byte[] data = protocol.encode(version, routable);
+ assertNotNull(data);
+ assertTrue(data.length > 0);
+ try {
+ TestFileUtil.writeToFile(path, data);
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ }
+ assertEquals(routable.getType(), protocol.decode(version, data).getType());
+ return data.length;
+ }
+
+ /**
+ * Reads the content of the given file and creates a corresponding routable.
+ *
+ * @param filename The name of the file to read from.
+ * @param classId The type that the routable must decode as.
+ * @param lang The language constant that dictates what file format to read from.
+ * @return The decoded routable.
+ */
+ public Routable deserialize(String filename, int classId, Language lang) {
+ Version version = version();
+ String path = getPath(version + "-" + (lang == Language.JAVA ? "java" : "cpp") + "-" + filename + ".dat");
+ System.out.println("Deserializing from '" + path + "'..");
+ byte[] data;
+ try {
+ data = TestFileUtil.readFile(path);
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ }
+ Routable ret = protocol.decode(version, data);
+ assertNotNull(ret);
+ assertEquals(classId, ret.getType());
+ return ret;
+ }
+
+ private static void assertCoverage(List<Integer> registered, List<Integer> tested) {
+ boolean ok = true;
+ List<Integer> lst = new ArrayList<>(tested);
+ for (Integer type : registered) {
+ if (!lst.contains(type)) {
+ System.err.println("Routable type " + type + " is registered in DocumentProtocol but not tested.");
+ ok = false;
+ } else {
+ lst.remove(type);
+ }
+ }
+ if (!lst.isEmpty()) {
+ for (Integer type : lst) {
+ System.err.println("Routable type " + type + " is tested but not registered in DocumentProtocol.");
+ }
+ ok = false;
+ }
+ assertTrue(ok);
+ }
+
+ protected static interface RunnableTest {
+
+ public void run() throws Exception;
+ }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java new file mode 100755 index 00000000000..3bbba1c0984 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolRoutingPolicy;
+import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactory;
+import com.yahoo.jrt.ListenFailedException;
+import com.yahoo.jrt.slobrok.server.Slobrok;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.metrics.MetricSet;
+import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
+import com.yahoo.messagebus.network.rpc.test.TestServer;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.messagebus.routing.RoutingContext;
+import com.yahoo.messagebus.test.Receptor;
+import com.yahoo.text.Utf8Array;
+import junit.framework.TestCase;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class PolicyFactoryTestCase extends TestCase {
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Setup
+ //
+ ////////////////////////////////////////////////////////////////////////////////
+
+ private Slobrok slobrok;
+ private TestServer srv;
+ private SourceSession src;
+
+ @Override
+ public void setUp() throws ListenFailedException {
+ slobrok = new Slobrok();
+ srv = new TestServer(new MessageBusParams().addProtocol(new DocumentProtocol(new DocumentTypeManager())),
+ new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));
+ src = srv.mb.createSourceSession(new SourceSessionParams().setReplyHandler(new Receptor()));
+ }
+
+ @Override
+ public void tearDown() {
+ slobrok.stop();
+ src.destroy();
+ srv.destroy();
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Tests
+ //
+ ////////////////////////////////////////////////////////////////////////////////
+
+ public void testFactory() {
+ Route route = Route.parse("[MyPolicy]");
+ assertTrue(src.send(createMessage(), route).isAccepted());
+ Reply reply = ((Receptor)src.getReplyHandler()).getReply(60);
+ assertNotNull(reply);
+ System.out.println(reply.getTrace());
+ assertEquals(1, reply.getNumErrors());
+ assertEquals(ErrorCode.UNKNOWN_POLICY, reply.getError(0).getCode());
+
+ Protocol obj = srv.mb.getProtocol(DocumentProtocol.NAME);
+ assertTrue(obj instanceof DocumentProtocol);
+ DocumentProtocol protocol = (DocumentProtocol)obj;
+ protocol.putRoutingPolicyFactory("MyPolicy", new MyFactory());
+
+ assertTrue(src.send(createMessage(), route).isAccepted());
+ assertNotNull(reply = ((Receptor)src.getReplyHandler()).getReply(60));
+ System.out.println(reply.getTrace());
+ assertEquals(1, reply.getNumErrors());
+ assertEquals(DocumentProtocol.ERROR_POLICY_FAILURE, reply.getError(0).getCode());
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Utilities
+ //
+ ////////////////////////////////////////////////////////////////////////////////
+
+ private static Message createMessage() {
+ Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+ msg.getTrace().setLevel(9);
+ return msg;
+ }
+
+ private static class MyFactory implements RoutingPolicyFactory {
+
+ public DocumentProtocolRoutingPolicy createPolicy(String param) {
+ return new MyPolicy(param);
+ }
+
+ public void destroy() {
+ }
+ }
+
+ private static class MyPolicy implements DocumentProtocolRoutingPolicy {
+
+ private final String param;
+
+ private MyPolicy(String param) {
+ this.param = param;
+ }
+
+ public void select(RoutingContext ctx) {
+ ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, param);
+ }
+
+ public void merge(RoutingContext ctx) {
+ throw new AssertionError("Routing passed terminated select.");
+ }
+
+ public void destroy() {
+ }
+
+ public MetricSet getMetrics() {
+ return null;
+ }
+ }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java new file mode 100755 index 00000000000..ef1cdd2818e --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java @@ -0,0 +1,902 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.document.*;
+import com.yahoo.documentapi.messagebus.protocol.*;
+import com.yahoo.jrt.ListenFailedException;
+import com.yahoo.jrt.slobrok.api.IMirror;
+import com.yahoo.jrt.slobrok.api.Mirror;
+import com.yahoo.jrt.slobrok.server.Slobrok;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.Error;
+import com.yahoo.messagebus.network.rpc.test.TestServer;
+import com.yahoo.messagebus.routing.*;
+import com.yahoo.messagebus.test.Receptor;
+import com.yahoo.vdslib.DocumentList;
+import com.yahoo.vdslib.Entry;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@SuppressWarnings("deprecation")
+public class PolicyTestCase {
+
+ private static final int TIMEOUT = 300;
+ private static final TimeUnit TIMEOUT_UNIT = TimeUnit.SECONDS;
+ private static final long TIMEOUT_MILLIS = TIMEOUT_UNIT.toMillis(TIMEOUT);
+ private final DocumentTypeManager manager = new DocumentTypeManager();
+
+ @Before
+ public void setUp() {
+ DocumentTypeManagerConfigurer.configure(manager, "file:./test/cfg/testdoc.cfg");
+ }
+
+ @Test
+ public void testProtocol() {
+ DocumentProtocol protocol = new DocumentProtocol(manager);
+
+ RoutingPolicy policy = protocol.createPolicy("AND", null);
+ assertTrue(policy instanceof ANDPolicy);
+
+ policy = new DocumentProtocol(manager).createPolicy("DocumentRouteSelector", "raw:route[0]\n");
+ assertTrue(policy instanceof DocumentRouteSelectorPolicy);
+
+ policy = new DocumentProtocol(manager).createPolicy("Extern", "foo;bar/baz");
+ assertTrue(policy instanceof ExternPolicy);
+
+ policy = new DocumentProtocol(manager).createPolicy("LocalService", null);
+ assertTrue(policy instanceof LocalServicePolicy);
+
+ policy = new DocumentProtocol(manager).createPolicy("RoundRobin", null);
+ assertTrue(policy instanceof RoundRobinPolicy);
+
+ policy = new DocumentProtocol(manager).createPolicy("SearchRow", null);
+ assertTrue(policy instanceof SearchRowPolicy);
+
+ policy = new DocumentProtocol(manager).createPolicy("SearchColumn", null);
+ assertTrue(policy instanceof SearchColumnPolicy);
+
+ policy = new DocumentProtocol(manager).createPolicy("SubsetService", null);
+ assertTrue(policy instanceof SubsetServicePolicy);
+
+ policy = new DocumentProtocol(manager).createPolicy("LoadBalancer", null);
+ assertTrue(policy instanceof LoadBalancerPolicy);
+ }
+
+ @Test
+ public void testAND() {
+ PolicyTestFrame frame = new PolicyTestFrame(manager);
+ frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:")))));
+ frame.setHop(new HopSpec("test", "[AND]")
+ .addRecipient("foo")
+ .addRecipient("bar"));
+ frame.assertSelect(Arrays.asList("foo", "bar"));
+
+ frame.setHop(new HopSpec("test", "[AND:baz]")
+ .addRecipient("foo")
+ .addRecipient("bar"));
+ frame.assertSelect(Arrays.asList("baz")); // param precedes recipients
+
+ frame.setHop(new HopSpec("test", "[AND:foo]"));
+ frame.assertMergeOneReply("foo");
+
+ frame.setHop(new HopSpec("test", "[AND:foo bar]"));
+ frame.assertMergeTwoReplies("foo", "bar");
+ frame.destroy();
+ }
+
+ @Test
+ public void requireThatExternPolicyWithIllegalParamIsAnErrorPolicy() throws ListenFailedException {
+ Slobrok slobrok = new Slobrok();
+ String spec = "tcp/localhost:" + slobrok.port();
+ assertTrue(new DocumentProtocol(manager).createPolicy("Extern", null) instanceof ErrorPolicy);
+ assertTrue(new DocumentProtocol(manager).createPolicy("Extern", "") instanceof ErrorPolicy);
+ assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec) instanceof ErrorPolicy);
+ assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec + ";") instanceof ErrorPolicy);
+ assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec + ";bar") instanceof ErrorPolicy);
+ }
+
+ @Test
+ public void requireThatExternPolicyWithUnknownPatternSelectsNone() throws Exception {
+ PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:");
+ setupExternPolicy(frame, new Slobrok(), "foo/bar");
+ frame.assertSelect(null);
+ }
+
+ @Test
+ public void requireThatExternPolicySelectsFromExternSlobrok() throws Exception {
+ PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:");
+ Slobrok slobrok = new Slobrok();
+ List<TestServer> servers = new ArrayList<>();
+ for (int i = 0; i < 10; ++i) {
+ TestServer server = new TestServer("docproc/cluster.default/" + i, null, slobrok, null,
+ new DocumentProtocol(manager));
+ server.net.registerSession("chain.default");
+ servers.add(server);
+ }
+ setupExternPolicy(frame, slobrok, "docproc/cluster.default/*/chain.default", 10);
+ Set<String> lst = new HashSet<>();
+ for (int i = 0; i < 10; ++i) {
+ RoutingNode leaf = frame.select(1).get(0);
+ String recipient = leaf.getRoute().toString();
+ lst.add(recipient);
+
+ leaf.handleReply(new EmptyReply());
+ assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+ }
+ assertEquals(10, lst.size());
+ for (TestServer server : servers) {
+ server.destroy();
+ }
+ frame.destroy();
+ }
+
+ @Test
+ public void requireThatExternPolicyMergesOneReplyAsProtocol() throws Exception {
+ PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:");
+ Slobrok slobrok = new Slobrok();
+ TestServer server = new TestServer("docproc/cluster.default/0", null, slobrok, null,
+ new DocumentProtocol(manager));
+ server.net.registerSession("chain.default");
+ setupExternPolicy(frame, slobrok, "docproc/cluster.default/*/chain.default", 1);
+ frame.assertMergeOneReply(server.net.getConnectionSpec() + "/chain.default");
+ server.destroy();
+ frame.destroy();
+ }
+
+ @Test
+ public void testExternSend() throws Exception {
+ // Setup local source node.
+ Slobrok local = new Slobrok();
+ TestServer src = new TestServer("src", null, local, null, new DocumentProtocol(manager));
+ SourceSession ss = src.mb.createSourceSession(new Receptor(), new SourceSessionParams().setTimeout(TIMEOUT));
+
+ // Setup remote cluster with routing config.
+ Slobrok slobrok = new Slobrok();
+ TestServer itr = new TestServer("itr",
+ new RoutingTableSpec(DocumentProtocol.NAME)
+ .addRoute(new RouteSpec("default").addHop("dst"))
+ .addHop(new HopSpec("dst", "dst/session")),
+ slobrok, null, new DocumentProtocol(manager));
+ IntermediateSession is = itr.mb.createIntermediateSession("session", true, new Receptor(), new Receptor());
+ TestServer dst = new TestServer("dst", null, slobrok, null, new DocumentProtocol(manager));
+ DestinationSession ds = dst.mb.createDestinationSession("session", true, new Receptor());
+
+ // Send message from local node to remote cluster and resolve route there.
+ Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+ msg.getTrace().setLevel(9);
+ msg.setRoute(Route.parse("[Extern:tcp/localhost:" + slobrok.port() + ";itr/session] default"));
+
+ assertTrue(ss.send(msg).isAccepted());
+ assertNotNull(msg = ((Receptor)is.getMessageHandler()).getMessage(TIMEOUT));
+ is.forward(msg);
+ assertNotNull(msg = ((Receptor)ds.getMessageHandler()).getMessage(TIMEOUT));
+ ds.acknowledge(msg);
+ Reply reply = ((Receptor)is.getReplyHandler()).getReply(TIMEOUT);
+ assertNotNull(reply);
+ is.forward(reply);
+ assertNotNull(reply = ((Receptor)ss.getReplyHandler()).getReply(TIMEOUT));
+
+ System.out.println(reply.getTrace().toString());
+
+ // Perform necessary cleanup.
+ src.destroy();
+ itr.destroy();
+ dst.destroy();
+ slobrok.stop();
+ local.stop();
+ }
+
+ @Test
+ public void testExternMultipleSlobroks() throws ListenFailedException {
+ Slobrok local = new Slobrok();
+ TestServer srcServer = new TestServer("src", null, local, null, new DocumentProtocol(manager));
+ SourceSession srcSession =
+ srcServer.mb.createSourceSession(new Receptor(), new SourceSessionParams().setTimeout(TIMEOUT));
+
+ Slobrok extern = new Slobrok();
+ String spec = "tcp/localhost:" + extern.port();
+
+ TestServer dstServer = new TestServer("dst", null, extern, null, new DocumentProtocol(manager));
+ Receptor dstHandler = new Receptor();
+ DestinationSession dstSession = dstServer.mb.createDestinationSession("session", true, dstHandler);
+
+ Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+ msg.setRoute(Route.parse("[Extern:" + spec + ";dst/session]"));
+ assertTrue(srcSession.send(msg).isAccepted());
+ assertNotNull(msg = dstHandler.getMessage(TIMEOUT));
+ dstSession.acknowledge(msg);
+ Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(TIMEOUT);
+ assertNotNull(reply);
+
+ extern.stop();
+ dstSession.destroy();
+ dstServer.destroy();
+ dstHandler.reset();
+ assertNull(dstHandler.getMessage(0));
+
+ extern = new Slobrok();
+ spec += ",tcp/localhost:" + extern.port();
+
+ dstServer = new TestServer("dst", null, extern, null, new DocumentProtocol(manager));
+ dstHandler = new Receptor();
+ dstSession = dstServer.mb.createDestinationSession("session", true, dstHandler);
+
+ msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
+ msg.setRoute(Route.parse("[Extern:" + spec + ";dst/session]"));
+ assertTrue(srcSession.send(msg).isAccepted());
+ assertNotNull(msg = dstHandler.getMessage(TIMEOUT));
+ dstSession.acknowledge(msg);
+ reply = ((Receptor)srcSession.getReplyHandler()).getReply(TIMEOUT);
+ assertNotNull(reply);
+
+ extern.stop();
+ dstSession.destroy();
+ dstServer.destroy();
+
+ local.stop();
+ srcSession.destroy();
+ srcServer.destroy();
+ }
+
+ @Test
+ public void testLocalService() {
+ // Test select with proper address.
+ PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);
+ frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:0")))));
+ for (int i = 0; i < 10; ++i) {
+ frame.getNetwork().registerSession(i + "/chain.default");
+ }
+ assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10));
+ frame.setHop(new HopSpec("test", "docproc/cluster.default/[LocalService]/chain.default"));
+
+ Set<String> lst = new HashSet<>();
+ for (int i = 0; i < 10; ++i) {
+ RoutingNode leaf = frame.select(1).get(0);
+ String recipient = leaf.getRoute().toString();
+ lst.add(recipient);
+
+ leaf.handleReply(new EmptyReply());
+ assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+ }
+ assertEquals(10, lst.size());
+
+ // Test select with broken address.
+ lst.clear();
+ frame.setHop(new HopSpec("test", "docproc/cluster.default/[LocalService:broken]/chain.default"));
+ for (int i = 0; i < 10; ++i) {
+ RoutingNode leaf = frame.select(1).get(0);
+ String recipient = leaf.getRoute().toString();
+ assertTrue(recipient.equals("docproc/cluster.default/*/chain.default"));
+ lst.add(recipient);
+
+ leaf.handleReply(new EmptyReply());
+ assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+ }
+ assertEquals(1, lst.size());
+
+ // Test merge behavior.
+ frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:")))));
+ frame.setHop(new HopSpec("test", "[LocalService]"));
+ frame.assertMergeOneReply("*");
+
+ frame.destroy();
+ }
+
+ @Test
+ public void testLocalServiceCache() {
+ PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager);
+ HopSpec fooHop = new HopSpec("foo", "docproc/cluster.default/[LocalService]/chain.foo");
+ fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo")));
+ fooFrame.setHop(fooHop);
+
+ PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame);
+ HopSpec barHop = new HopSpec("bar", "docproc/cluster.default/[LocalService]/chain.bar");
+ barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar")));
+ barFrame.setHop(barHop);
+
+ fooFrame.getMessageBus().setupRouting(
+ new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME)
+ .addHop(fooHop)
+ .addHop(barHop)));
+
+ fooFrame.getNetwork().registerSession("0/chain.foo");
+ fooFrame.getNetwork().registerSession("0/chain.bar");
+ assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2));
+
+ RoutingNode fooChild = fooFrame.select(1).get(0);
+ assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().getHop(0).toString());
+ RoutingNode barChild = barFrame.select(1).get(0);
+ assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().getHop(0).toString());
+
+ barChild.handleReply(new EmptyReply());
+ fooChild.handleReply(new EmptyReply());
+
+ assertNotNull(barFrame.getReceptor().getReply(TIMEOUT));
+ assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT));
+ }
+
+ @Test
+ public void testSearchRow() {
+ PolicyTestFrame frame = new PolicyTestFrame(manager);
+ frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:")))));
+ frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo"));
+ frame.assertMergeOneReply("foo");
+ frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar"));
+ frame.assertMergeTwoReplies("foo", "bar");
+
+ frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo"));
+ Map<String, Integer> replies = new HashMap<>();
+ replies.put("foo", ErrorCode.SERVICE_OOS);
+ frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS));
+
+ frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo").addRecipient("bar"));
+ replies.put("foo", ErrorCode.SERVICE_OOS);
+ replies.put("bar", ErrorCode.NONE);
+ frame.assertMergeOk(replies, Arrays.asList("bar"));
+
+ replies.put("foo", ErrorCode.SERVICE_OOS);
+ replies.put("bar", ErrorCode.SERVICE_OOS);
+ frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));
+
+ frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo").addRecipient("bar").addRecipient("baz"));
+ replies.put("foo", ErrorCode.SERVICE_OOS);
+ replies.put("bar", ErrorCode.NONE);
+ replies.put("baz", ErrorCode.NONE);
+ frame.assertMergeOk(replies, Arrays.asList("bar", "baz"));
+
+ replies.put("foo", ErrorCode.SERVICE_OOS);
+ replies.put("bar", ErrorCode.SERVICE_OOS);
+ replies.put("baz", ErrorCode.NONE);
+ frame.assertMergeOk(replies, Arrays.asList("baz"));
+
+ replies.put("foo", ErrorCode.SERVICE_OOS);
+ replies.put("bar", ErrorCode.SERVICE_OOS);
+ replies.put("baz", ErrorCode.SERVICE_OOS);
+ frame.assertMergeError(replies,
+ Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));
+
+ frame.setHop(new HopSpec("test", "[SearchRow:2]").addRecipient("foo").addRecipient("bar").addRecipient("baz"));
+ replies.put("foo", ErrorCode.SERVICE_OOS);
+ replies.put("bar", ErrorCode.NONE);
+ replies.put("baz", ErrorCode.NONE);
+ frame.assertMergeOk(replies, Arrays.asList("bar", "baz"));
+
+ replies.put("foo", ErrorCode.SERVICE_OOS);
+ replies.put("bar", ErrorCode.SERVICE_OOS);
+ replies.put("baz", ErrorCode.NONE);
+ frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));
+
+ replies.put("foo", ErrorCode.SERVICE_OOS);
+ replies.put("bar", ErrorCode.SERVICE_OOS);
+ replies.put("baz", ErrorCode.SERVICE_OOS);
+ frame.assertMergeError(replies,
+ Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));
+
+ frame.destroy();
+ }
+
+ @Test
+ public void testSearchRowMerge() {
+ PolicyTestFrame frame = new PolicyTestFrame(manager);
+ frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo"));
+ tryWasFound(frame, 1, 0x0, false);
+ tryWasFound(frame, 1, 0x1, true);
+
+ frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar"));
+ tryWasFound(frame, 2, 0x0, false);
+ tryWasFound(frame, 2, 0x1, true);
+ tryWasFound(frame, 2, 0x2, true);
+ tryWasFound(frame, 2, 0x3, true);
+
+ frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar").addRecipient("baz"));
+ tryWasFound(frame, 3, 0x0, false);
+ tryWasFound(frame, 3, 0x1, true);
+ tryWasFound(frame, 3, 0x2, true);
+ tryWasFound(frame, 3, 0x3, true);
+ tryWasFound(frame, 3, 0x4, true);
+ tryWasFound(frame, 3, 0x5, true);
+ tryWasFound(frame, 3, 0x6, true);
+ tryWasFound(frame, 3, 0x7, true);
+ frame.destroy();
+ }
+
+ private void tryWasFound(PolicyTestFrame frame, int expectedRecipients,
+ int foundMask, boolean expectedFound)
+ {
+ {
+ frame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:69")));
+ List<RoutingNode> selected = frame.select(expectedRecipients);
+ for (int i = 0, len = selected.size(); i < len; ++i) {
+ RemoveDocumentReply reply = new RemoveDocumentReply();
+ reply.setWasFound(((1 << i) & foundMask) != 0);
+ selected.get(i).handleReply(reply);
+ }
+ Reply reply = frame.getReceptor().getReply(TIMEOUT);
+ assertNotNull(reply);
+ assertEquals(DocumentProtocol.REPLY_REMOVEDOCUMENT, reply.getType());
+ assertEquals(expectedFound, ((RemoveDocumentReply)reply).wasFound());
+ }
+ {
+ DocumentUpdate upd = new DocumentUpdate(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:"));
+ frame.setMessage(new UpdateDocumentMessage(upd));
+ List<RoutingNode> selected = frame.select(expectedRecipients);
+ for (int i = 0, len = selected.size(); i < len; ++i) {
+ UpdateDocumentReply reply = new UpdateDocumentReply();
+ reply.setWasFound(((1 << i) & foundMask) != 0);
+ selected.get(i).handleReply(reply);
+ }
+ Reply reply = frame.getReceptor().getReply(TIMEOUT);
+ assertNotNull(reply);
+ assertEquals(DocumentProtocol.REPLY_UPDATEDOCUMENT, reply.getType());
+ assertEquals(expectedFound, ((UpdateDocumentReply)reply).wasFound());
+ }
+ }
+
+ @Test
+ public void multipleGetRepliesAreMergedToFoundDocument() {
+ PolicyTestFrame frame = new PolicyTestFrame(manager);
+ frame.setHop(new HopSpec("test", getDocumentRouteSelectorRawConfig())
+ .addRecipient("foo").addRecipient("bar"));
+ frame.setMessage(new GetDocumentMessage(new DocumentId("doc:scheme:yarn"), "[all]"));
+ List<RoutingNode> selected = frame.select(2);
+ for (int i = 0, len = selected.size(); i < len; ++i) {
+ Document doc = null;
+ if (i == 0) {
+ doc = new Document(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:yarn"));
+ doc.setLastModified(123456L);
+ }
+ GetDocumentReply reply = new GetDocumentReply(null);
+ reply.setDocument(doc);
+ selected.get(i).handleReply(reply);
+ }
+ Reply reply = frame.getReceptor().getReply(TIMEOUT);
+ assertNotNull(reply);
+ assertEquals(DocumentProtocol.REPLY_GETDOCUMENT, reply.getType());
+ assertEquals(123456L, ((GetDocumentReply)reply).getLastModified());
+ }
+
+ private String getDocumentRouteSelectorRawConfig() {
+ return "[DocumentRouteSelector:raw:" +
+ "route[2]\n" +
+ "route[0].name \"foo\"\n" +
+ "route[0].selector \"testdoc\"\n" +
+ "route[0].feed \"myfeed\"\n" +
+ "route[1].name \"bar\"\n" +
+ "route[1].selector \"other\"\n" +
+ "route[1].feed \"myfeed\"\n]";
+ }
+
+ @Test
+ public void testSearchColumn() {
+ PolicyTestFrame frame = new PolicyTestFrame(manager);
+ frame.setHop(new HopSpec("test", "[SearchColumn]")
+ .addRecipient("c0").addRecipient("c1")
+ .addRecipient("c2").addRecipient("c3"));
+
+ // Test hash distribution.
+ assertDistribution(frame, "doc:ns:3", "c0");
+ assertDistribution(frame, "doc:ns:18", "c1");
+ assertDistribution(frame, "doc:ns:0", "c2");
+ assertDistribution(frame, "doc:ns:4", "c3");
+
+ assertDistribution(frame, "userdoc:ns:49152:0", "c0");
+ assertDistribution(frame, "userdoc:ns:49152:1", "c0");
+ assertDistribution(frame, "userdoc:ns:16384:2", "c1");
+ assertDistribution(frame, "userdoc:ns:16384:3", "c1");
+ assertDistribution(frame, "userdoc:ns:5461:4", "c2");
+ assertDistribution(frame, "userdoc:ns:5461:5", "c2");
+ assertDistribution(frame, "userdoc:ns:0:6", "c3");
+ assertDistribution(frame, "userdoc:ns:0:7", "c3");
+
+ assertDistribution(frame, "groupdoc:ns:0:0", "c0");
+ assertDistribution(frame, "groupdoc:ns:0:1", "c0");
+ assertDistribution(frame, "groupdoc:ns:4:2", "c1");
+ assertDistribution(frame, "groupdoc:ns:4:3", "c1");
+ assertDistribution(frame, "groupdoc:ns:2:4", "c2");
+ assertDistribution(frame, "groupdoc:ns:2:5", "c2");
+ assertDistribution(frame, "groupdoc:ns:7:6", "c3");
+ assertDistribution(frame, "groupdoc:ns:7:7", "c3");
+
+ // Test routing based on message type.
+ Message put = new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:"))));
+ frame.setHop(new HopSpec("test", "[SearchColumn]").addRecipient("c0").addRecipient("c1"));
+ frame.setMessage(put);
+ frame.assertMergeOneReply("c0");
+
+ // Test allowed bad parts.
+ frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0"));
+ frame.setMessage(put);
+ Map<String, Integer> replies = new HashMap<>();
+ replies.put("c0", ErrorCode.SERVICE_OOS);
+ frame.assertMergeOk(replies, null);
+
+ replies.put("c0", ErrorCode.SERVICE_OOS);
+ frame.assertMergeOk(replies, null);
+
+ frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0").addRecipient("c1"));
+ frame.setMessage(put);
+ replies.put("c0", ErrorCode.SERVICE_OOS);
+ frame.assertMergeOk(replies, null);
+
+ frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0").addRecipient("c1").addRecipient("c2"));
+ frame.setMessage(put);
+ replies.clear();
+ replies.put("c0", ErrorCode.SERVICE_OOS);
+ frame.assertMergeOk(replies, null);
+
+ frame.setHop(new HopSpec("test", "[SearchColumn:2]").addRecipient("c0").addRecipient("c1").addRecipient("c2"));
+ frame.setMessage(put);
+ replies.clear();
+ replies.put("c0", ErrorCode.SERVICE_OOS);
+ frame.assertMergeOk(replies, null);
+
+ frame.destroy();
+ }
+
+ private void assertDistribution(PolicyTestFrame frame, String id, String expected) {
+ frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+ new DocumentId(id)))));
+ frame.assertSelect(Arrays.asList(expected));
+ }
+
+ @Test
+ public void testSubsetService() {
+ PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);
+ frame.setMessage(new PutDocumentMessage(new DocumentPut(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:"))))));
+
+ // Test requerying for adding nodes.
+ frame.setHop(new HopSpec("test", "docproc/cluster.default/[SubsetService:2]/chain.default"));
+ Set<String> lst = new HashSet<>();
+ for (int i = 1; i <= 10; ++i) {
+ frame.getNetwork().registerSession(i + "/chain.default");
+ assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", i));
+
+ RoutingNode leaf = frame.select(1).get(0);
+ lst.add(leaf.getRoute().toString());
+ leaf.handleReply(new EmptyReply());
+ assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+ }
+ assertTrue(lst.size() > 1); // must have requeried
+
+ // Test load balancing.
+ String prev = null;
+ for (int i = 1; i <= 10; ++i) {
+ RoutingNode leaf = frame.select(1).get(0);
+ String next = leaf.getRoute().toString();
+ if (prev == null) {
+ assertNotNull(next);
+ } else {
+ assertFalse(prev.equals(next));
+ }
+ prev = next;
+ leaf.handleReply(new EmptyReply());
+ assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+ }
+
+ // Test requerying for dropping nodes.
+ lst.clear();
+ for (int i = 1; i <= 10; ++i) {
+ RoutingNode leaf = frame.select(1).get(0);
+ String route = leaf.getRoute().toString();
+ lst.add(route);
+
+ frame.getNetwork().unregisterSession(route.substring(frame.getIdentity().length() + 1));
+ assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10 - i));
+
+ Reply reply = new EmptyReply();
+ reply.addError(new Error(ErrorCode.NO_ADDRESS_FOR_SERVICE, route));
+ leaf.handleReply(reply);
+ assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+ }
+ assertEquals(10, lst.size());
+
+ // Test merge behavior.
+ frame.setHop(new HopSpec("test", "[SubsetService]"));
+ frame.assertMergeOneReply("*");
+
+ frame.destroy();
+ }
+
+ @Test
+ public void testSubsetServiceCache() {
+ PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager);
+ HopSpec fooHop = new HopSpec("foo", "docproc/cluster.default/[SubsetService:2]/chain.foo");
+ fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo")));
+ fooFrame.setHop(fooHop);
+
+ PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame);
+ HopSpec barHop = new HopSpec("bar", "docproc/cluster.default/[SubsetService:2]/chain.bar");
+ barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar")));
+ barFrame.setHop(barHop);
+
+ fooFrame.getMessageBus().setupRouting(
+ new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME)
+ .addHop(fooHop)
+ .addHop(barHop)));
+
+ fooFrame.getNetwork().registerSession("0/chain.foo");
+ fooFrame.getNetwork().registerSession("0/chain.bar");
+ assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2));
+
+ RoutingNode fooChild = fooFrame.select(1).get(0);
+ assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().getHop(0).toString());
+ RoutingNode barChild = barFrame.select(1).get(0);
+ assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().getHop(0).toString());
+
+ barChild.handleReply(new EmptyReply());
+ fooChild.handleReply(new EmptyReply());
+
+ assertNotNull(barFrame.getReceptor().getReply(TIMEOUT));
+ assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT));
+ }
+
+ @Test
+ public void testDocumentRouteSelector() {
+ // Test policy usage safeguard.
+ String okConfig = "raw:route[0]\n";
+ String errConfig = "raw:" +
+ "route[1]\n" +
+ "route[0].name \"foo\"\n" +
+ "route[0].selector \"foo bar\"\n" +
+ "route[0].feed \"baz\"\n";
+
+ DocumentProtocol protocol = new DocumentProtocol(manager, okConfig);
+ assertTrue(protocol.createPolicy("DocumentRouteSelector", null) instanceof DocumentRouteSelectorPolicy);
+ assertTrue(protocol.createPolicy("DocumentRouteSelector", "") instanceof DocumentRouteSelectorPolicy);
+ assertTrue(protocol.createPolicy("DocumentRouteSelector", errConfig) instanceof ErrorPolicy);
+
+ protocol = new DocumentProtocol(manager, errConfig);
+ assertTrue(protocol.createPolicy("DocumentRouteSelector", null) instanceof ErrorPolicy);
+ assertTrue(protocol.createPolicy("DocumentRouteSelector", "") instanceof ErrorPolicy);
+ assertTrue(protocol.createPolicy("DocumentRouteSelector", okConfig) instanceof DocumentRouteSelectorPolicy);
+
+ // Test policy with proper config.
+ PolicyTestFrame frame = new PolicyTestFrame(manager);
+ frame.setHop(new HopSpec("test", "[DocumentRouteSelector:raw:" +
+ "route[2]\n" +
+ "route[0].name \"foo\"\n" +
+ "route[0].selector \"testdoc\"\n" +
+ "route[0].feed \"myfeed\"\n" +
+ "route[1].name \"bar\"\n" +
+ "route[1].selector \"other\"\n" +
+ "route[1].feed \"myfeed\"\n]").addRecipient("foo").addRecipient("bar"));
+
+ frame.setMessage(new GetDocumentMessage(new DocumentId("doc:scheme:"), "fieldSet"));
+ frame.assertSelect(Arrays.asList("bar", "foo"));
+
+ Message put = new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:"))));
+ frame.setMessage(put);
+ frame.assertSelect(Arrays.asList("foo"));
+
+ frame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:")));
+ frame.assertSelect(Arrays.asList("bar", "foo"));
+
+ frame.setMessage(new UpdateDocumentMessage(new DocumentUpdate(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:"))));
+ frame.assertSelect(Arrays.asList("foo"));
+
+ frame.setMessage(put);
+ frame.assertMergeOneReply("foo");
+
+ frame.destroy();
+ }
+
+
+ @Test
+ public void testDocumentRouteSelectorIgnore() {
+ PolicyTestFrame frame = new PolicyTestFrame(manager);
+ frame.setHop(new HopSpec("test", "[DocumentRouteSelector:raw:" +
+ "route[1]\n" +
+ "route[0].name \"docproc/cluster.foo\"\n" +
+ "route[0].selector \"testdoc and testdoc.stringfield == 'foo'\"\n" +
+ "route[0].feed \"myfeed\"\n]").addRecipient("docproc/cluster.foo"));
+
+ frame.setMessage(new PutDocumentMessage(
+ new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+ new DocumentId("id:yarn:testdoc:n=1234:fluff")))));
+ frame.select(0);
+ Reply reply = frame.getReceptor().getReply(TIMEOUT);
+ assertNotNull(reply);
+ assertEquals(DocumentProtocol.REPLY_DOCUMENTIGNORED, reply.getType());
+ assertEquals(0, reply.getNumErrors());
+
+ frame.setMessage(new UpdateDocumentMessage(new DocumentUpdate(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:"))));
+ frame.assertSelect(Arrays.asList("docproc/cluster.foo"));
+
+ frame.destroy();
+ }
+
+ @Test
+ public void testLoadBalancer() {
+ PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);
+ frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:")))));
+ frame.getNetwork().registerSession("0/chain.default");
+ assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 1));
+ frame.setHop(new HopSpec("test", "[LoadBalancer:cluster=docproc/cluster.default;session=chain.default]"));
+
+ assertSelect(frame, 1, Arrays.asList(frame.getNetwork().getConnectionSpec() + "/chain.default"));
+ }
+
+ @Test
+ public void testRoundRobin() {
+ // Test select with proper address.
+ PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);
+ frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+ new DocumentId("doc:scheme:")))));
+ for (int i = 0; i < 10; ++i) {
+ frame.getNetwork().registerSession(i + "/chain.default");
+ }
+ assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10));
+ frame.setHop(new HopSpec("test", "[RoundRobin]")
+ .addRecipient("docproc/cluster.default/3/chain.default")
+ .addRecipient("docproc/cluster.default/6/chain.default")
+ .addRecipient("docproc/cluster.default/9/chain.default"));
+ assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/3/chain.default",
+ "docproc/cluster.default/6/chain.default",
+ "docproc/cluster.default/9/chain.default"));
+ frame.getNetwork().unregisterSession("6/chain.default");
+ assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 9));
+ assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/3/chain.default",
+ "docproc/cluster.default/9/chain.default"));
+ frame.getNetwork().unregisterSession("3/chain.default");
+ assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 8));
+ assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/9/chain.default"));
+ frame.getNetwork().unregisterSession("9/chain.default");
+ assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 7));
+ assertSelect(frame, 32, new ArrayList<String>());
+
+ // Test merge behavior.
+ frame.setHop(new HopSpec("test", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.default"));
+ frame.assertMergeOneReply("docproc/cluster.default/0/chain.default");
+
+ frame.destroy();
+ }
+
+ @Test
+ public void testRoundRobinCache() {
+ PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager);
+ HopSpec fooHop = new HopSpec("foo", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.foo");
+ fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo")));
+ fooFrame.setHop(fooHop);
+
+ PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame);
+ HopSpec barHop = new HopSpec("bar", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.bar");
+ barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar")));
+ barFrame.setHop(barHop);
+
+ fooFrame.getMessageBus().setupRouting(
+ new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME)
+ .addHop(fooHop)
+ .addHop(barHop)));
+
+ fooFrame.getNetwork().registerSession("0/chain.foo");
+ fooFrame.getNetwork().registerSession("0/chain.bar");
+ assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2));
+
+ RoutingNode fooChild = fooFrame.select(1).get(0);
+ assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().toString());
+ RoutingNode barChild = barFrame.select(1).get(0);
+ assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().toString());
+
+ barChild.handleReply(new EmptyReply());
+ fooChild.handleReply(new EmptyReply());
+
+ assertNotNull(barFrame.getReceptor().getReply(TIMEOUT));
+ assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT));
+ }
+
+ /**
+ * Ensures that the given number of select passes on the given frame produces an expected list of recipients.
+ *
+ * @param frame The frame to select on.
+ * @param numSelects The number of selects to perform.
+ * @param expected The list of expected recipients.
+ */
+ private static void assertSelect(PolicyTestFrame frame, int numSelects, List<String> expected) {
+ Set<String> lst = new TreeSet<>();
+
+ for (int i = 0; i < numSelects; ++i) {
+ if (!expected.isEmpty()) {
+ RoutingNode leaf = frame.select(1).get(0);
+ String recipient = leaf.getRoute().toString();
+ lst.add(recipient);
+ leaf.handleReply(new EmptyReply());
+ } else {
+ frame.select(0);
+ }
+ assertNotNull(frame.getReceptor().getReply(TIMEOUT));
+ }
+
+ assertEquals(expected.size(), lst.size());
+ Iterator<String> it = lst.iterator();
+ for (String recipient : expected) {
+ assertEquals(recipient, it.next());
+ }
+ }
+
+ private static void assertMirrorReady(Mirror slobrok)
+ throws InterruptedException, TimeoutException
+ {
+ for (int i = 0; i < TIMEOUT_MILLIS / 10; ++i) {
+ if (slobrok.ready()) {
+ return;
+ }
+ Thread.sleep(10);
+ }
+ throw new TimeoutException();
+ }
+
+ private static void assertMirrorContains(IMirror slobrok, String pattern, int numEntries)
+ throws InterruptedException, TimeoutException
+ {
+ for (int i = 0; i < TIMEOUT_MILLIS / 10; ++i) {
+ if (slobrok.lookup(pattern).length == numEntries) {
+ return;
+ }
+ Thread.sleep(10);
+ }
+ throw new TimeoutException();
+ }
+
+ private void setupExternPolicy(PolicyTestFrame frame, Slobrok slobrok, String pattern)
+ throws InterruptedException, TimeoutException
+ {
+ setupExternPolicy(frame, slobrok, pattern, -1);
+ }
+
+ private void setupExternPolicy(PolicyTestFrame frame, Slobrok slobrok, String pattern, int numEntries)
+ throws InterruptedException, TimeoutException
+ {
+ String param = "tcp/localhost:" + slobrok.port() + ";" + pattern;
+ frame.setHop(new HopSpec("test", "[Extern:" + param + "]"));
+ MessageBus mbus = frame.getMessageBus();
+ HopBlueprint hop = mbus.getRoutingTable(DocumentProtocol.NAME).getHop("test");
+ PolicyDirective dir = (PolicyDirective)hop.getDirective(0);
+ ExternPolicy policy = (ExternPolicy)mbus.getRoutingPolicy(DocumentProtocol.NAME, dir.getName(), dir.getParam());
+ assertMirrorReady(policy.getMirror());
+ if (numEntries >= 0) {
+ assertMirrorContains(policy.getMirror(), pattern, numEntries);
+ }
+ }
+
+ private PolicyTestFrame newFrame() {
+ return new PolicyTestFrame(manager);
+ }
+
+ private PolicyTestFrame newFrame(Message msg) {
+ PolicyTestFrame frame = newFrame();
+ frame.setMessage(msg);
+ return frame;
+ }
+
+ private PutDocumentMessage newPutDocument(String documentId) {
+ return new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
+ new DocumentId(documentId))));
+ }
+
+ private PolicyTestFrame newPutDocumentFrame(String documentId) {
+ return newFrame(newPutDocument(documentId));
+ }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java new file mode 100755 index 00000000000..37b6c1a9068 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java @@ -0,0 +1,385 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.jrt.ListenFailedException;
+import com.yahoo.jrt.slobrok.api.Mirror;
+import com.yahoo.jrt.slobrok.server.Slobrok;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.network.Identity;
+import com.yahoo.messagebus.network.Network;
+import com.yahoo.messagebus.network.ServiceAddress;
+import com.yahoo.messagebus.network.rpc.RPCNetwork;
+import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
+import com.yahoo.messagebus.network.rpc.test.TestServer;
+import com.yahoo.messagebus.routing.*;
+import com.yahoo.messagebus.test.Receptor;
+import com.yahoo.messagebus.test.SimpleProtocol;
+import com.yahoo.messagebus.test.SimpleReply;
+
+import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * This is a utility class to allow easier policy test cases. The most important reason to use this is to make sure that
+ * each test uses a "clean" mbus and slobrok instance.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class PolicyTestFrame extends junit.framework.Assert {
+
+ private final AtomicBoolean destroyed = new AtomicBoolean(false);
+ private String identity;
+ private Slobrok slobrok;
+ private MessageBus mbus;
+ private MyNetwork net;
+ private Message msg = null;
+ private HopSpec hop = null;
+ private Receptor handler = new Receptor();
+
+ /**
+ * Create an anonymous test frame.
+ *
+ * @param documentMgr The document manager to use.
+ */
+ public PolicyTestFrame(DocumentTypeManager documentMgr) {
+ this("anonymous", documentMgr);
+ }
+
+ /**
+ * Create a named test frame.
+ *
+ * @param identity The identity to use for the server.
+ * @param documentMgr The document manager to use.
+ */
+ public PolicyTestFrame(String identity, DocumentTypeManager documentMgr) {
+ this.identity = identity;
+ try {
+ slobrok = new Slobrok();
+ } catch (ListenFailedException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ net = new MyNetwork(new RPCNetworkParams()
+ .setIdentity(new Identity(identity))
+ .setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));
+ mbus = new MessageBus(net, new MessageBusParams()
+ .addProtocol(new DocumentProtocol(documentMgr)));
+ }
+
+ /**
+ * Create a test frame running on the same slobrok and mbus as another.
+ *
+ * @param frame The frame whose internals to share.
+ */
+ public PolicyTestFrame(PolicyTestFrame frame) {
+ identity = frame.identity;
+ slobrok = frame.slobrok;
+ net = frame.net;
+ mbus = frame.mbus;
+ }
+
+ // Inherit doc from Object.
+ @Override
+ public void finalize() throws Throwable {
+ destroy();
+ super.finalize();
+ }
+
+ /**
+ * Sets the destroyed flag to true. The very first time this method is called, it cleans up all its dependencies.
+ * Even if you retain a reference to this object, all of its content is allowed to be garbage collected.
+ */
+ public void destroy() {
+ if (!destroyed.getAndSet(true)) {
+ mbus.destroy();
+ mbus = null;
+ net = null;
+
+ slobrok.stop();
+ slobrok = null;
+ }
+ }
+
+ /**
+ * Routes the contained message based on the current setup, and returns the leaf send contexts.
+ *
+ * @param numExpected The expected number of leaf nodes.
+ * @return The list of selected send contexts.
+ */
+ public List<RoutingNode> select(int numExpected) {
+ msg.setRoute(Route.parse(hop.getName()));
+ new RoutingNode(mbus, net, null, handler, msg).send();
+ List<RoutingNode> ret = net.removeNodes();
+ assertEquals(numExpected, ret.size());
+ return ret;
+ }
+
+ /**
+ * Ensures that the current setup selects a given set of routes.
+ *
+ * @param expected A list of expected route leaf nodes.
+ */
+ public void assertSelect(List<String> expected) {
+ if (expected == null) {
+ assertEquals(0, select(0).size());
+ } else {
+ List<RoutingNode> selected = select(expected.size());
+ for (RoutingNode node : selected) {
+ assertTrue("Route '" + node.getRoute() + "' not selected.",
+ expected.contains(node.getRoute().toString()));
+ node.handleReply(new EmptyReply());
+ }
+ }
+ assertNotNull(handler.getReply(60));
+ }
+
+ /**
+ * This is a convenience method for invoking {@link #assertMerge(Map,List,List)} with no expected value.
+ *
+ * @param replies The errors to set in the leaf node replies.
+ * @param expectedErrors The list of expected errors in the merged reply.
+ */
+ public void assertMergeError(Map<String, Integer> replies, List<Integer> expectedErrors) {
+ assertMerge(replies, expectedErrors, null);
+ }
+
+ /**
+ * This is a convenience method for invoking {@link this#assertMerge(Map,List,List)} with no expected errors.
+ *
+ * @param replies The errors to set in the leaf node replies.
+ * @param allowedValues The list of allowed values in the final reply.
+ */
+ public void assertMergeOk(Map<String, Integer> replies, List<String> allowedValues) {
+ assertMerge(replies, null, allowedValues);
+ }
+
+ /**
+ * Ensures that the current setup generates as many leaf nodes as there are members of the errors argument. Each
+ * error is then given one of these errors, and the method then ensures that the single returned reply contains the
+ * given list of expected errors. Finally, if the expected value argument is non-null, this method ensures that the
+ * reply is a SimpleReply whose string value exists in the allowed list.
+ *
+ * @param replies The errors to set in the leaf node replies.
+ * @param expectedErrors The list of expected errors in the merged reply.
+ * @param allowedValues The list of allowed values in the final reply.
+ */
+ public void assertMerge(Map<String, Integer> replies, List<Integer> expectedErrors, List<String> allowedValues) {
+ List<RoutingNode> selected = select(replies.size());
+
+ for (RoutingNode node : selected) {
+ String route = node.getRoute().toString();
+ assertTrue(replies.containsKey(route));
+ Reply ret = new SimpleReply(route);
+ if (replies.get(route) != ErrorCode.NONE) {
+ ret.addError(new com.yahoo.messagebus.Error(replies.get(route), route));
+ }
+ node.handleReply(ret);
+ }
+
+ Reply reply = handler.getReply(60);
+ assertNotNull(reply);
+ if (expectedErrors != null) {
+ assertEquals(expectedErrors.size(), reply.getNumErrors());
+ for (int i = 0; i < expectedErrors.size(); ++i) {
+ assertTrue(expectedErrors.contains(reply.getError(i).getCode()));
+ }
+ } else if (reply.hasErrors()) {
+ StringBuilder err = new StringBuilder("Got unexpected error(s):\n");
+ for (int i = 0; i < reply.getNumErrors(); ++i) {
+ err.append("\t").append(reply.getError(i).toString());
+ if (i < reply.getNumErrors() - 1) {
+ err.append("\n");
+ }
+ }
+ fail(err.toString());
+ }
+ if (allowedValues != null) {
+ assertEquals(SimpleProtocol.REPLY, reply.getType());
+ assertTrue(allowedValues.contains(((SimpleReply)reply).getValue()));
+ } else {
+ assertEquals(0, reply.getType());
+ }
+ }
+
+ /**
+ * Ensures that the current setup chooses a single recipient, and that it merges similarly to how the
+ * {@link DocumentProtocol} would merge these.
+ *
+ * @param recipient The expected recipient.
+ */
+ public void assertMergeOneReply(String recipient) {
+ assertSelect(Arrays.asList(recipient));
+
+ Map<String, Integer> replies = new HashMap<>();
+ replies.put(recipient, ErrorCode.NONE);
+ assertMergeOk(replies, Arrays.asList(recipient));
+
+ replies.put(recipient, ErrorCode.TRANSIENT_ERROR);
+ assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR));
+ }
+
+ /**
+ * Ensures that the current setup will choose the two given recipients, and that it merges similarly to how the
+ * {@link DocumentProtocol} would merge these.
+ *
+ * @param recipientOne The first expected recipient.
+ * @param recipientTwo The second expected recipient.
+ */
+ public void assertMergeTwoReplies(String recipientOne, String recipientTwo) {
+ assertSelect(Arrays.asList(recipientOne, recipientTwo));
+
+ Map<String, Integer> replies = new HashMap<>();
+ replies.put(recipientOne, ErrorCode.NONE);
+ replies.put(recipientTwo, ErrorCode.NONE);
+ assertMergeOk(replies, Arrays.asList(recipientOne, recipientTwo));
+
+ replies.put(recipientOne, ErrorCode.TRANSIENT_ERROR);
+ replies.put(recipientTwo, ErrorCode.NONE);
+ assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR));
+
+ replies.put(recipientOne, ErrorCode.TRANSIENT_ERROR);
+ replies.put(recipientTwo, ErrorCode.TRANSIENT_ERROR);
+ assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR, ErrorCode.TRANSIENT_ERROR));
+
+ replies.put(recipientOne, ErrorCode.NONE);
+ replies.put(recipientTwo, DocumentProtocol.ERROR_MESSAGE_IGNORED);
+ assertMergeOk(replies, Arrays.asList(recipientOne));
+
+ replies.put(recipientOne, DocumentProtocol.ERROR_MESSAGE_IGNORED);
+ replies.put(recipientTwo, ErrorCode.NONE);
+ assertMergeOk(replies, Arrays.asList(recipientTwo));
+
+ replies.put(recipientOne, DocumentProtocol.ERROR_MESSAGE_IGNORED);
+ replies.put(recipientTwo, DocumentProtocol.ERROR_MESSAGE_IGNORED);
+ assertMergeError(replies, Arrays.asList(DocumentProtocol.ERROR_MESSAGE_IGNORED,
+ DocumentProtocol.ERROR_MESSAGE_IGNORED));
+ }
+
+ /**
+ * Waits for a given service pattern to resolve to the given number of hits in the local slobrok.
+ *
+ * @param pattern The pattern to lookup.
+ * @param cnt The number of entries to wait for.
+ * @return True if the expected number of entries was found.
+ */
+ public boolean waitSlobrok(String pattern, int cnt) {
+ for (int i = 0; i < 1000 && !Thread.currentThread().isInterrupted(); ++i) {
+ Mirror.Entry[] res = net.getMirror().lookup(pattern);
+ if (res.length == cnt) {
+ return true;
+ }
+ try { Thread.sleep(10); } catch (InterruptedException e) { /* ignore */ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns the identity of this frame.
+ *
+ * @return The ident string.
+ */
+ public String getIdentity() {
+ return identity;
+ }
+
+ /**
+ * Returns the private slobrok server.
+ *
+ * @return The slobrok.
+ */
+ public Slobrok getSlobrok() {
+ return slobrok;
+ }
+
+ /**
+ * Returns the private message bus.
+ *
+ * @return The bus.
+ */
+ public MessageBus getMessageBus() {
+ return mbus;
+ }
+
+ /**
+ * Returns the private network layer.
+ *
+ * @return The network.
+ */
+ public Network getNetwork() {
+ return net;
+ }
+
+ /**
+ * Returns the message being tested.
+ *
+ * @return The message.
+ */
+ public Message getMessage() {
+ return msg;
+ }
+
+ /**
+ * Sets the message being tested.
+ *
+ * @param msg The message to set.
+ */
+ public void setMessage(Message msg) {
+ this.msg = msg;
+ }
+
+ /**
+ * Sets the spec of the hop to test with.
+ *
+ * @param hop The spec to set.
+ */
+ public void setHop(HopSpec hop) {
+ this.hop = hop;
+ mbus.setupRouting(new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME).addHop(hop)));
+ }
+
+ /**
+ * Returns the reply receptor used by this frame. All messages tested are tagged with this receptor, so after a
+ * successful select, the receptor should contain a non-null reply.
+ *
+ * @return The reply receptor.
+ */
+ public Receptor getReceptor() {
+ return handler;
+ }
+
+ /**
+ * Implements a dummy network.
+ */
+ private class MyNetwork extends RPCNetwork {
+
+ private List<RoutingNode> nodes = new ArrayList<>();
+
+ public MyNetwork(RPCNetworkParams params) {
+ super(params);
+ }
+
+ @Override
+ public boolean allocServiceAddress(RoutingNode recipient) {
+ recipient.setServiceAddress(new ServiceAddress() { });
+ return true;
+ }
+
+ @Override
+ public void freeServiceAddress(RoutingNode recipient) {
+ recipient.setServiceAddress(null);
+ }
+
+ @Override
+ public void send(Message msg, List<RoutingNode> recipients) {
+ this.nodes.addAll(recipients);
+ }
+
+ public List<RoutingNode> removeNodes() {
+ List<RoutingNode> ret = nodes;
+ nodes = new ArrayList<>();
+ return ret;
+ }
+ }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PriorityTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PriorityTestCase.java new file mode 100644 index 00000000000..86964d65100 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PriorityTestCase.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test; + +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol.Priority; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class PriorityTestCase { + + @Test + public void requireThat51PriorityValuesAreCorrect() throws IOException { + String path = "test/crosslanguagefiles/5.1-Priority.txt"; + BufferedReader in = new BufferedReader(new FileReader(path)); + + List<Priority> expected = new LinkedList<Priority>(Arrays.asList(Priority.values())); + String str; + while ((str = in.readLine()) != null) { + String arr[] = str.split(":", 2); + Priority pri = Priority.valueOf(arr[0]); + assertEquals(pri.toString(), pri.getValue(), Integer.valueOf(arr[1]).intValue()); + assertTrue("Unexpected priority '" + str + "'.", expected.remove(pri)); + } + assertTrue("Expected priorities " + expected + ".", expected.isEmpty()); + } + + @Test + public void requireThatUnknownPriorityThrowsException() { + try { + DocumentProtocol.getPriority(Priority.HIGHEST.getValue() - 1); + fail(); + } catch (IllegalArgumentException e) { + + } + try { + DocumentProtocol.getPriority(Priority.LOWEST.getValue() + 1); + fail(); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void requireThatUnknownPriorityNameThrowsException() { + try { + DocumentProtocol.getPriorityByName("FOO"); + fail(); + } catch (IllegalArgumentException e) { + + } + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java new file mode 100755 index 00000000000..8ba217ce15d --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java @@ -0,0 +1,188 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test;
+
+import com.yahoo.component.VersionSpecification;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.serialization.DocumentDeserializer;
+import com.yahoo.document.serialization.DocumentSerializer;
+import com.yahoo.documentapi.messagebus.protocol.DocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
+import com.yahoo.documentapi.messagebus.protocol.DocumentReply;
+import com.yahoo.documentapi.messagebus.protocol.RoutableFactories50;
+import com.yahoo.jrt.ListenFailedException;
+import com.yahoo.jrt.slobrok.server.Slobrok;
+import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.network.Identity;
+import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
+import com.yahoo.messagebus.network.rpc.test.TestServer;
+import com.yahoo.messagebus.routing.Route;
+import com.yahoo.messagebus.test.Receptor;
+import junit.framework.TestCase;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class RoutableFactoryTestCase extends TestCase {
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Setup
+ //
+ ////////////////////////////////////////////////////////////////////////////////
+
+ private Slobrok slobrok;
+ private DocumentProtocol srcProtocol, dstProtocol;
+ private TestServer srcServer, dstServer;
+ private SourceSession srcSession;
+ private DestinationSession dstSession;
+
+ @Override
+ public void setUp() throws ListenFailedException {
+ slobrok = new Slobrok();
+ DocumentTypeManager docMan = new DocumentTypeManager();
+ dstProtocol = new DocumentProtocol(docMan);
+ dstServer = new TestServer(new MessageBusParams().addProtocol(dstProtocol),
+ new RPCNetworkParams().setIdentity(new Identity("dst")).setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));
+ dstSession = dstServer.mb.createDestinationSession(new DestinationSessionParams().setName("session").setMessageHandler(new Receptor()));
+ srcProtocol = new DocumentProtocol(docMan);
+ srcServer = new TestServer(new MessageBusParams().addProtocol(srcProtocol),
+ new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));
+ srcSession = srcServer.mb.createSourceSession(new SourceSessionParams().setReplyHandler(new Receptor()));
+ assertTrue(srcServer.waitSlobrok("dst/session", 1));
+ }
+
+ @Override
+ public void tearDown() {
+ slobrok.stop();
+ dstSession.destroy();
+ dstServer.destroy();
+ srcSession.destroy();
+ srcServer.destroy();
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Tests
+ //
+ ////////////////////////////////////////////////////////////////////////////////
+
+ public void testFactory() {
+ Route route = Route.parse("dst/session");
+
+ // Source should fail to encode the message.
+ assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
+ Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(60);
+ assertNotNull(reply);
+ System.out.println(reply.getTrace());
+ assertTrue(reply.hasErrors());
+ assertEquals(ErrorCode.ENCODE_ERROR, reply.getError(0).getCode());
+ assertNull(reply.getError(0).getService());
+
+ // Destination should fail to decode the message.
+ srcProtocol.putRoutableFactory(MyMessage.TYPE, new MyMessageFactory(), new VersionSpecification());
+ assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
+ assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));
+ System.out.println(reply.getTrace());
+ assertTrue(reply.hasErrors());
+ assertEquals(ErrorCode.DECODE_ERROR, reply.getError(0).getCode());
+ assertEquals("dst/session", reply.getError(0).getService());
+
+ // Destination should fail to encode the reply.
+ dstProtocol.putRoutableFactory(MyMessage.TYPE, new MyMessageFactory(), new VersionSpecification());
+ assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
+ Message msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60);
+ assertNotNull(msg);
+ reply = new MyReply();
+ reply.swapState(msg);
+ dstSession.reply(reply);
+ assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));
+ System.out.println(reply.getTrace());
+ assertTrue(reply.hasErrors());
+ assertEquals(ErrorCode.ENCODE_ERROR, reply.getError(0).getCode());
+ assertEquals("dst/session", reply.getError(0).getService());
+
+ // Source should fail to decode the reply.
+ dstProtocol.putRoutableFactory(MyReply.TYPE, new MyReplyFactory(), new VersionSpecification());
+ assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
+ assertNotNull(msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60));
+ reply = new MyReply();
+ reply.swapState(msg);
+ dstSession.reply(reply);
+ assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));
+ System.out.println(reply.getTrace());
+ assertTrue(reply.hasErrors());
+ assertEquals(ErrorCode.DECODE_ERROR, reply.getError(0).getCode());
+ assertNull(reply.getError(0).getService());
+
+ // All should succeed.
+ srcProtocol.putRoutableFactory(MyReply.TYPE, new MyReplyFactory(), new VersionSpecification());
+ assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
+ assertNotNull(msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60));
+ reply = new MyReply();
+ reply.swapState(msg);
+ dstSession.reply(reply);
+ assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));
+ System.out.println(reply.getTrace());
+ assertFalse(reply.hasErrors());
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ //
+ // Utilities
+ //
+ ////////////////////////////////////////////////////////////////////////////////
+
+ private static class MyMessageFactory extends RoutableFactories50.DocumentMessageFactory {
+
+ @Override
+ protected DocumentMessage doDecode(DocumentDeserializer buf) {
+ return new MyMessage();
+ }
+
+ @Override
+ protected boolean doEncode(DocumentMessage msg, DocumentSerializer buf) {
+ return true;
+ }
+ }
+
+ private static class MyReplyFactory extends RoutableFactories50.DocumentReplyFactory {
+
+ @Override
+ protected DocumentReply doDecode(DocumentDeserializer buf) {
+ return new MyReply();
+ }
+
+ @Override
+ protected boolean doEncode(DocumentReply msg, DocumentSerializer buf) {
+ return true;
+ }
+ }
+
+ private static class MyMessage extends DocumentMessage {
+
+ final static int TYPE = 666;
+
+ MyMessage() {
+ getTrace().setLevel(9);
+ }
+
+ @Override
+ public DocumentReply createReply() {
+ return new MyReply();
+ }
+
+ @Override
+ public int getType() {
+ return TYPE;
+ }
+ }
+
+ private static class MyReply extends DocumentReply {
+
+ final static int TYPE = 777;
+
+ public MyReply() {
+ super(TYPE);
+ }
+ }
+}
diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/TestFileUtil.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/TestFileUtil.java new file mode 100644 index 00000000000..fa5e54d981f --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/TestFileUtil.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.Charset; + +public class TestFileUtil { + protected static final String DATA_PATH = "./test/crosslanguagefiles"; + + public static void writeToFile(String path, byte[] data) throws IOException { + try (FileOutputStream stream = new FileOutputStream(path)) { + stream.write(data); + } + } + + /** + * Write `data` to `path` using UTF-8 as binary encoding format. + */ + public static void writeToFile(String path, String data) throws IOException { + writeToFile(path, data.getBytes(Charset.forName("UTF-8"))); + } + + /** + * Returns the path to use for data files. + * + * @param filename The name of the file to include in the path. + * @return The data file path. + */ + public static String getPath(String filename) { + return DATA_PATH + "/" + filename; + } + + public static byte[] readFile(String path) throws IOException { + try (FileInputStream stream = new FileInputStream(path)) { + byte[] data = new byte[stream.available()]; + int read = stream.read(data); + if (read != data.length) { + throw new IOException(String.format("Truncated read (expected %d bytes, read %d)", data.length, read)); + } + return data; + } + } + +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/BasicTests.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/BasicTests.java new file mode 100644 index 00000000000..4673061954c --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/BasicTests.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test.storagepolicy; + +import com.yahoo.collections.Pair; +import com.yahoo.documentapi.messagebus.protocol.WrongDistributionReply; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.routing.RoutingNode; + +import java.util.Arrays; + +public class BasicTests extends StoragePolicyTestEnvironment { + + /** Test that we can send a message through the policy. */ + public void testNormalUsage() { + setClusterNodes(new int[]{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + // First we want a wrong distribution reply, so make sure we don't try correct node on random + policyFactory.avoidPickingAtRandom(bucketOneNodePreference[0]); + RoutingNode target = select(); + replyWrongDistribution(target, "foo", 5, "version:1 bits:16 distributor:10 storage:10"); + // Then send to correct node and verify that + sendToCorrectNode("foo", bucketOneNodePreference[0]); + } + + /** Test that we can identify newest cluster state and hang on to correct one. */ + public void testRepliesWrongOrderDuringStateChange() throws Exception{ + { + setClusterNodes(new int[]{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + RoutingNode target1 = select(); + RoutingNode target2 = select(); + replyWrongDistribution(target2, "foo", 0, "version:2 bits:16 distributor:10"); + replyWrongDistribution(target1, "foo", 5, "version:1 bits:16 distributor:10 ." + bucketOneNodePreference[0] + ".s:d"); + sendToCorrectNode("foo", bucketOneNodePreference[0]); + } + tearDown(); + setUp(); + { + setClusterNodes(new int[]{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + RoutingNode target1 = select(); + RoutingNode target2 = select(); + replyWrongDistribution(target2, "foo", 0, "version:1 bits:16 distributor:10"); + replyWrongDistribution(target1, "foo", 5, "version:2 bits:16 distributor:10 ." + bucketOneNodePreference[0] + ".s:d"); + sendToCorrectNode("foo", bucketOneNodePreference[1]); + } + } + /** + * To be independent of changes in distribution algorithm, we programmatically calculate preferred order of + * bucket 1, which we will be using in the tests. To avoid doing this ahead of every test, we still hardcode the + * values, though only one place, and have this test to verify they are correct, and make it easy to update the values. + */ + public void testVerifyBucketOneNodePreferenceInTenNodeDefaultCluster() { + int result[] = new int[10]; + setClusterNodes(new int[]{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + String clusterState = " bits:16 storage:10 distributor:10"; + for (int i=0; i<10; ++i) { + // Update cached cluster state, to reflect which node we want to find + RoutingNode target = select(); + target.handleReply(new WrongDistributionReply("version:" + (i + 1) + clusterState)); + Reply reply = frame.getReceptor().getReply(60); + assertNotNull(reply); + assertFalse(reply.hasErrors()); + // Find correct target + target = select(); + Pair<String, Integer> address = getAddress(target); + result[i] = address.getSecond(); + removeNode(address.getSecond()); + clusterState += " ." + result[i] + ".s:d"; + } + assertEquals(Arrays.toString(bucketOneNodePreference), Arrays.toString(result)); + } + +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java new file mode 100644 index 00000000000..0e365ef11f6 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java @@ -0,0 +1,216 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test.storagepolicy; + +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.DocumentId; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.StoragePolicy; +import com.yahoo.messagebus.routing.RoutingNode; +import com.yahoo.vdslib.distribution.RandomGen; +import com.yahoo.vdslib.state.*; + +import java.util.Map; +import java.util.TreeMap; +import java.util.regex.Pattern; + +public abstract class Simulator extends StoragePolicyTestEnvironment { + + enum FailureType { + TRANSIENT_ERROR, + FATAL_ERROR, + OLD_CLUSTER_STATE, + RESET_CLUSTER_STATE, + RESET_CLUSTER_STATE_NO_GOOD_NODES, + NODE_NOT_IN_SLOBROK + }; + private Integer getIdealTarget(String idString, String clusterState) { + DocumentId did = new DocumentId(idString); + BucketIdFactory factory = new BucketIdFactory(); + BucketId bid = factory.getBucketId(did); + try{ + return policyFactory.getLastParameters().createDistribution(null).getIdealDistributorNode(new ClusterState(clusterState), bid, "uim"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public class BadNode { + private int index; + private double failureRate = 1.0; + private FailureType failureType; + private ClusterState badClusterState; + private boolean downInCurrentState = false; + + public BadNode(int index, FailureType type) { + this.index = index; + this.failureType = type; + } + + public BadNode setFailureRate(double rate) { failureRate = rate; return this; } + public BadNode setBadClusterState(ClusterState state) { + badClusterState = state; + if (debug) System.err.println("Setting bad cluster state in distributor " + index + ": " + state); + return this; + } + public BadNode setDownInCurrentState() { downInCurrentState = true; return this; } + + public int getIndex() { return index; } + public FailureType getFailureType() { return failureType; } + public double getFailureRate() { return failureRate; } + public ClusterState getBadClusterState() { return badClusterState; } + public boolean isSetDownInCurrentState() { return downInCurrentState; } + } + + public class PersistentFailureTestParameters { + private ClusterState initialClusterState; + private ClusterState currentClusterState; + private int totalRequests = 200; + private int parallellRequests = 10; + private Map<Integer, BadNode> badnodes = new TreeMap<Integer, BadNode>(); + private boolean newNode = false; + + public PersistentFailureTestParameters() { + try{ + initialClusterState = new ClusterState("version:2 bits:16 distributor:9"); + currentClusterState = initialClusterState.clone(); + currentClusterState.setVersion(3); + } catch (Exception e) { throw new RuntimeException(e); } + } + + public PersistentFailureTestParameters setTotalRequests(int count) { totalRequests = count; return this; } + public PersistentFailureTestParameters setParallellRequests(int count) { parallellRequests = count; return this; } + public PersistentFailureTestParameters addBadNode(BadNode node) { badnodes.put(node.getIndex(), node); return this; } + public PersistentFailureTestParameters newNodeAdded() { + currentClusterState.setNodeState(new Node(NodeType.DISTRIBUTOR, 9), new NodeState(NodeType.DISTRIBUTOR, State.UP)); + newNode = true; + return this; + } + + public void validate() { + // To simplify looping, ensure we do complete loops + assertTrue(totalRequests % parallellRequests == 0); + // Node that in some tests are used as new node up, cannot be a bad node + assertTrue(!badnodes.containsKey(9)); + for (BadNode node : badnodes.values()) { + ClusterState badClusterState = currentClusterState.clone(); + int nodesToSetDown = 0; + if (node.getFailureType() == FailureType.OLD_CLUSTER_STATE) { + badClusterState.setVersion(1); + nodesToSetDown = 4; + } else if (node.getFailureType() == FailureType.RESET_CLUSTER_STATE) { + badClusterState.setVersion(5); + nodesToSetDown = 4; + } else if (node.getFailureType() == FailureType.RESET_CLUSTER_STATE_NO_GOOD_NODES) { + badClusterState.setVersion(5); + nodesToSetDown = -1; + } else { + badClusterState = null; + } + if (badClusterState != null) { + int setDown = 0; + for (int i=0; i<10; ++i) { + if (nodesToSetDown != -1 && setDown >= nodesToSetDown) break; + if (badnodes.containsKey(i)) continue; + badClusterState.setNodeState(new Node(NodeType.DISTRIBUTOR, i), new NodeState(NodeType.DISTRIBUTOR, State.DOWN)); + ++setDown; + } + if (nodesToSetDown > 0 && nodesToSetDown != setDown) throw new IllegalStateException("Failed to set down " + nodesToSetDown + " nodes"); + node.setBadClusterState(badClusterState); + } + if (node.isSetDownInCurrentState()) { + currentClusterState.setNodeState(new Node(NodeType.DISTRIBUTOR, node.getIndex()), new NodeState(NodeType.DISTRIBUTOR, State.DOWN)); + } + } + if (debug) System.err.println("Using initial state " + initialClusterState); + if (debug) System.err.println("Using current state " + currentClusterState); + } + + public int getTotalRequests() { return totalRequests; } + public int getParallellRequests() { return parallellRequests; } + public Map<Integer, BadNode> getBadNodes() { return badnodes; } + public boolean isNewNodeAdded() { return newNode; } + public ClusterState getInitialClusterState() { return initialClusterState; } + public ClusterState getCurrentClusterState(Integer distributor) { + if (distributor != null && badnodes.containsKey(distributor) && badnodes.get(distributor).getBadClusterState() != null) { + return badnodes.get(distributor).getBadClusterState(); + } + return currentClusterState; + } + } + public void runSimulation(String expected, PersistentFailureTestParameters params) { + params.validate(); + // Set nodes in slobrok + setClusterNodes(new int[]{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + for (BadNode node : params.getBadNodes().values()) { + if (node.getFailureType() == FailureType.NODE_NOT_IN_SLOBROK) removeNode(node.getIndex()); + } + { + RoutingNode target = select(); + replyWrongDistribution(target, "foo", null, params.getInitialClusterState().toString()); + } + RandomGen randomizer = new RandomGen(432121); + int correctnode[] = new int[2], + wrongnode[] = new int[2], + failed[] = new int[2], + worked[] = new int[2], + downnode[] = new int[2]; + for (int step = 0, steps = (params.getTotalRequests() / params.getParallellRequests()); step < steps; ++step) { + int half = (step < steps / 2 ? 0 : 1); + if (debug) System.err.println("Starting step " + step + " in half " + half); + String docId[] = new String[params.getParallellRequests()]; + RoutingNode targets[] = new RoutingNode[params.getParallellRequests()]; + for (int i=0; i<params.getParallellRequests(); ++i) { + docId[i] = "doc:ns:" + (step * params.getParallellRequests() + i); + frame.setMessage(createMessage(docId[i])); + targets[i] = select(); + } + for (int i=0; i<params.getParallellRequests(); ++i) { + RoutingNode target = targets[i]; + int index = getAddress(target).getSecond(); + if (!params.getCurrentClusterState(null).getNodeState(new Node(NodeType.DISTRIBUTOR, index)).getState().oneOf(StoragePolicy.owningBucketStates)) { + ++downnode[half]; + } + BadNode badNode = params.getBadNodes().get(index); + if (getAddress(target).getSecond() == getIdealTarget(docId[i], params.getCurrentClusterState(null).toString())) { + ++correctnode[half]; + } else { + ++wrongnode[half]; + } + if (badNode != null && randomizer.nextDouble() < badNode.getFailureRate()) { + ++failed[half]; + switch (badNode.getFailureType()) { + case TRANSIENT_ERROR: replyError(target, new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_BUSY, "Transient error")); break; + case FATAL_ERROR: replyError(target, new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_UNPARSEABLE, "Fatal error")); break; + case OLD_CLUSTER_STATE: + case RESET_CLUSTER_STATE: + case RESET_CLUSTER_STATE_NO_GOOD_NODES: replyWrongDistribution(target, "foo", null, params.getCurrentClusterState(index).toString()); break; + case NODE_NOT_IN_SLOBROK: throw new IllegalStateException("This point in code should not be reachable"); + } + } else { + ++worked[half]; + boolean correctTarget = (getAddress(target).getSecond() == getIdealTarget(docId[i], params.getCurrentClusterState(index).toString())); + if (correctTarget) { + replyOk(target); + } else { + replyWrongDistribution(target, "foo", null, params.getCurrentClusterState(index).toString()); + } + } + } + } + StringBuilder actual = new StringBuilder(); + String result[][] = new String[2][]; + for (int i=0; i<2; ++i) { + actual.append(i == 0 ? "First " : " Last ") + .append("correctnode ").append(correctnode[i]) + .append(", wrongnode ").append(wrongnode[i]) + .append(", downnode ").append(downnode[i]) + .append(", worked ").append(worked[i]) + .append(", failed ").append(failed[i]); + } + if (!Pattern.matches(expected, actual.toString())) { + assertEquals(expected, actual.toString()); + } + } + +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/StoragePolicyTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/StoragePolicyTest.java new file mode 100644 index 00000000000..de27fd22e95 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/StoragePolicyTest.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test.storagepolicy; + +public class StoragePolicyTest extends Simulator { + /** + * Verify that a resent message with failures doesn't ruin overall performance. (By dumping the cached state too often + * so other requests are sent to wrong target) + * Lets one node always fail message with transient error. + */ + public void testPersistentFailureTransientError() { + runSimulation("First correctnode 99, wrongnode 1, downnode 0, worked 90, failed 10 " + + "Last correctnode 99, wrongnode 1, downnode 0, worked 92, failed 8", + new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.TRANSIENT_ERROR))); + } + /** + * Verify that a resent message with failures doesn't ruin overall performance. (By dumping the cached state too often + * so other requests are sent to wrong target) + * Lets one node always fail message with fatal error. + */ + public void testPersistentFailureFatalError() { + runSimulation("First correctnode 99, wrongnode 1, downnode 0, worked 90, failed 10 " + + "Last correctnode 99, wrongnode 1, downnode 0, worked 92, failed 8", + new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.FATAL_ERROR))); + } + /** + * Verify that a node responding with old cluster state doesn't ruin overall performance (By dumping/switching cached + * state too often) + * Let one node reporting an old cluster state (but node is still set up in fleetcontroller state). + * We expect some requests to go to wrong node due to this issue, but the majority of requests should be unaffected. + */ + public void testPersistentFailureOldClusterState() { + runSimulation("First correctnode .*, wrongnode .*, downnode .*, worked .*, failed .* " + + "Last correctnode 100, wrongnode 0, downnode 0, worked 100, failed 0", + new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.OLD_CLUSTER_STATE).setDownInCurrentState())); + } + /** + * Verify that a reset cluster state version doesn't keep sending requests to the wrong node. + * We expect a few failures in first half. We should have detected the issue before second half, so there all should be fine. + */ + public void testPersistentFailureResetClusterState() { + // If reset detection works (within the few messages sent in test), we should not fail any requests or send to wrong nodes in second half + runSimulation("First correctnode .*, wrongnode .*, downnode .*, worked .*, failed .* " + + "Last correctnode .*, wrongnode 0, downnode 0, worked .*, failed 0", + new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.RESET_CLUSTER_STATE).setDownInCurrentState())); + } + /** + * Verify that a reset cluster state version doesn't keep sending requests to the wrong node. + * We expect a few failures in first half. We should have detected the issue before second half, so there all should be fine. + */ + public void testPersistentFailureResetClusterStateNoGoodNodes() { + // If reset detection works (within the few messages sent in test), we should not fail any requests in second half. + + // Current problem here, is that even though we from time to time will send requests to other nodes, and will eventually throw the faulty cluster state, + // we will have pending operations towards this distributor when it happens, so it very quickly returns into a bad state. + + // This issue should hopefully not be that critical as we don't expect nodes to stay up and report erronious states. Even nodes that are down do get the + // cluster states sent to them, and if that doesn't work, how do the client manage to talk to them? + + runSimulation("First correctnode .*, wrongnode .*, downnode .*, worked .*, failed .* " + + "Last correctnode .*, wrongnode 100, downnode 100, worked 0, failed 100", + new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.RESET_CLUSTER_STATE_NO_GOOD_NODES).setDownInCurrentState())); + } + /** + * Verify that a reset cluster state version doesn't keep sending requests to the wrong node. + * We expect a few failures in first half. We should have detected the issue before second half, so there all should be fine. + */ + public void testPersistentFailureResetClusterStateNoGoodNodesNotMarkedDown() { + // If reset detection works (within the few messages sent in test), we should not fail any requests in second half. + + // This is just as sad as the above. Even if the node got detected to be screwed, we'd still be in the setting above. We don't expect nodes + // to get into this state however. + + runSimulation("First correctnode .*, wrongnode .*, downnode .*, worked .*, failed .* " + + "Last correctnode .*, wrongnode 91, downnode 0, worked 0, failed 100", + new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.RESET_CLUSTER_STATE_NO_GOOD_NODES))); + } + /** + * Verify that a reset cluster state version doesn't keep sending requests to the wrong node. + * Another scenario where we have a node coming up in correct state. + * We expect a few failures in first half. We should have detected the issue before second half, so there all should be fine. + */ + public void testPersistentFailureResetClusterStateNewNodeUp() { + // If we handled this well, we should have no failing requests, and no requests to down node in second half + runSimulation("First correctnode .*, wrongnode .*, downnode .*, worked .*, failed .* " + + "Last correctnode .*, wrongnode 0, downnode 0, worked .*, failed 0", + new PersistentFailureTestParameters().newNodeAdded().addBadNode(new BadNode(3, FailureType.RESET_CLUSTER_STATE).setDownInCurrentState())); + } + /** Test node that is not in slobrok. Until fleetcontroller detects this, we expect 10% of the requests to go to wrong node. */ + public void testPersistentFailureNodeNotInSlobrok() { + runSimulation("First correctnode .*, wrongnode 11, downnode 0, worked .*, failed .* " + + "Last correctnode .*, wrongnode 9, downnode 0, worked 100, failed 0", + new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.NODE_NOT_IN_SLOBROK))); + } + + /** With two failures, one marked down, hopefully the one not marked down doesn't lead us to use the one marked down. */ + public void testPersistentFailureTwoNodesFailingOneMarkedDown() { + // We see that we don't send to down nodes in second half. We still fail requests towards the one not marked down, + // and occasionally send to random due to this + runSimulation("First correctnode .*, wrongnode 23, downnode .*, worked .*, failed .* " + + "Last correctnode .*, wrongnode 4, downnode 0, worked 219, failed 31", + new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.TRANSIENT_ERROR)) + .addBadNode(new BadNode(4, FailureType.TRANSIENT_ERROR).setDownInCurrentState()) + .setTotalRequests(500)); + // Note that we use extra requests here as with only 200 requests there was a pretty good chance of not going to any down node on random anyhow. + } + + // Left to test? + + // Cluster state down - Not overwrite last good nodes to send random to? + // Overwrite cached state or not? +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/StoragePolicyTestEnvironment.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/StoragePolicyTestEnvironment.java new file mode 100644 index 00000000000..4dae5b072fb --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/StoragePolicyTestEnvironment.java @@ -0,0 +1,219 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test.storagepolicy; + +import com.yahoo.collections.Pair; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.documentapi.messagebus.protocol.test.PolicyTestFrame; +import com.yahoo.messagebus.EmptyReply; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.routing.HopSpec; +import com.yahoo.messagebus.routing.RoutingContext; +import com.yahoo.messagebus.routing.RoutingNode; +import com.yahoo.text.Utf8Array; +import com.yahoo.vdslib.distribution.Distribution; +import com.yahoo.vdslib.distribution.RandomGen; +import junit.framework.TestCase; + +import java.util.*; + +public abstract class StoragePolicyTestEnvironment extends TestCase { + protected StoragePolicyTestFactory policyFactory; + protected PolicyTestFrame frame; + private Set<Integer> nodes; + protected static int[] bucketOneNodePreference = new int[]{ 3, 5, 7, 6, 8, 0, 9, 2, 1, 4 }; + protected boolean debug = true; + + @Override + public void setUp() throws Exception { + DocumentTypeManager manager = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(manager, "file:./test/cfg/testdoc.cfg"); + frame = new PolicyTestFrame(manager); + nodes = new TreeSet<>(); + DocumentProtocol protocol = (DocumentProtocol) frame.getMessageBus().getProtocol((Utf8Array)DocumentProtocol.NAME); + policyFactory = new StoragePolicyTestFactory(nodes); + protocol.putRoutingPolicyFactory("storage", policyFactory); + frame.setMessage(createMessage("userdoc:ns:1:foo")); + frame.setHop(new HopSpec("test", "[storage:cluster=foo]")); + } + + @Override + public void tearDown() { + frame.destroy(); + } + + protected static Message createMessage(String id) { + Message msg = new RemoveDocumentMessage(new DocumentId(id)); + msg.getTrace().setLevel(9); + return msg; + } + + protected void setClusterNodes(int[] ints) { + Set<Integer> clusterNodes = new TreeSet<>(); + for (int i=0; i<ints.length; ++i) clusterNodes.add(ints[i]); + nodes.clear(); + nodes.addAll(clusterNodes); + } + private static Pair<String, String> extractClusterAndIndexFromPattern(String pattern) { + String[] bits = pattern.split("/"); + if (bits.length < 4) throw new IllegalStateException("Invalid pattern '" + pattern + "'. Expected more parts in it."); + String distributor = bits[3]; + String cluster = bits[1]; + if (cluster.indexOf('.') < 0) throw new IllegalStateException("Expected . in cluster spec '" + cluster + "'."); + cluster = cluster.substring(cluster.indexOf('.') + 1); + return new Pair<>(cluster, distributor); + } + protected static Pair<String, Integer> getAddress(RoutingNode node) { + Pair<String, String> pair = extractClusterAndIndexFromPattern(node.getRoute().getHop(0).toString()); + return new Pair<>(pair.getFirst(), Integer.valueOf(pair.getSecond())); + } + + protected RoutingNode select() { + List<RoutingNode> result = frame.select(1); + assertEquals(1, result.size()); + return result.get(0); + } + + protected void addNode(int index) { + nodes.add(index); + } + protected void removeNode(int second) { + assertTrue(nodes.remove(second)); + } + + public static class TestHostFetcher extends StoragePolicy.HostFetcher { + private final String clusterName; + private RandomGen randomizer = new RandomGen(1234); + private final Set<Integer> nodes; + private Integer avoidPickingAtRandom = null; + + public TestHostFetcher(String clusterName, Set<Integer> nodes) { + this.clusterName = clusterName; + this.nodes = nodes; + } + + public void setAvoidPickingAtRandom(Integer index) { avoidPickingAtRandom = index; } + + @Override + public String getTargetSpec(Integer distributor, RoutingContext context) { + try{ + if (distributor == null) { + if (nodes.size() == 1) { + assertTrue(avoidPickingAtRandom != nodes.iterator().next()); + distributor = nodes.iterator().next(); + } else { + Iterator<Integer> it = nodes.iterator(); + for (int i = 0, n = randomizer.nextInt(nodes.size() - 1); i<n; ++i) it.next(); + distributor = it.next(); + if (avoidPickingAtRandom != null && distributor == avoidPickingAtRandom) distributor = it.next(); + } + } + if (nodes.contains(distributor)) { + return "storage/cluster." + clusterName + "/distributor/" + distributor; + } else { + return null; + } + } catch (RuntimeException e) { + e.printStackTrace(); + assertTrue(e.getMessage(), false); + throw e; + } + } + } + + public static class TestParameters extends StoragePolicy.Parameters { + private final TestHostFetcher hostFetcher; + private final Distribution distribution; + + public TestParameters(String parameters, Set<Integer> nodes) { + super(AsyncInitializationPolicy.parse(parameters)); + hostFetcher = new TestHostFetcher(getClusterName(), nodes); + distribution = new Distribution(Distribution.getDefaultDistributionConfig(2, 10)); + } + + @Override + public StoragePolicy.HostFetcher createHostFetcher(ExternalSlobrokPolicy policy) { return hostFetcher; } + + @Override + public Distribution createDistribution(ExternalSlobrokPolicy policy) { return distribution; } + } + + public static class StoragePolicyTestFactory implements RoutingPolicyFactory { + private Set<Integer> nodes; + private final LinkedList<TestParameters> parameterInstances = new LinkedList<TestParameters>(); + private Integer avoidPickingAtRandom = null; + + public StoragePolicyTestFactory(Set<Integer> nodes) { + this.nodes = nodes; + } + public DocumentProtocolRoutingPolicy createPolicy(String parameters) { + parameterInstances.addLast(new TestParameters(parameters, nodes)); + ((TestHostFetcher) parameterInstances.getLast().createHostFetcher(null)).setAvoidPickingAtRandom(avoidPickingAtRandom); + return new StoragePolicy(parameterInstances.getLast(), AsyncInitializationPolicy.parse(parameters)); + } + public void avoidPickingAtRandom(Integer distributor) { + avoidPickingAtRandom = distributor; + for (TestParameters params : parameterInstances) { + ((TestHostFetcher) params.createHostFetcher(null)).setAvoidPickingAtRandom(avoidPickingAtRandom); + } + } + public TestParameters getLastParameters() { return parameterInstances.getLast(); } + public void destroy() { + } + } + + private int findPreferredAvailableNodeForTestBucket() { + for (int i=0; i<10; ++i) { + if (nodes.contains(bucketOneNodePreference[i])) return bucketOneNodePreference[i]; + } + throw new IllegalStateException("Found no node available"); + } + + protected void sendToCorrectNode(String cluster, int correctNode) { + RoutingNode target = select(); + target.handleReply(new EmptyReply()); + Reply reply = frame.getReceptor().getReply(60); + assertNotNull(reply); + assertFalse(reply.hasErrors()); + assertEquals(reply.getTrace().toString(), "storage/cluster." + cluster + "/distributor/" + correctNode, target.getRoute().getHop(0).toString()); + } + + protected void replyWrongDistribution(RoutingNode target, String cluster, Integer randomNode, String clusterState) { + // We want test to send to wrong node when sending to random. If distribution changes so the first random + // node picked is the same node we should alter test + if (randomNode != null) { + assertFalse(randomNode == findPreferredAvailableNodeForTestBucket()); + } + target.handleReply(new WrongDistributionReply(clusterState)); + Reply reply = frame.getReceptor().getReply(60); + assertNotNull(reply); + assertFalse(reply.hasErrors()); + + // Verify that we sent to expected node + if (randomNode != null) { + assertEquals(reply.getTrace().toString(), "storage/cluster." + cluster + "/distributor/" + randomNode, target.getRoute().getHop(0).toString()); + } + if (debug) System.err.println("WRONG DISTRIBUTION: " + reply.getTrace()); + } + + protected void replyOk(RoutingNode target) { + target.handleReply(new EmptyReply()); + Reply reply = frame.getReceptor().getReply(60); + assertNotNull(reply); + assertFalse(reply.hasErrors()); + if (debug) System.err.println("OK: " + reply.getTrace()); + } + + protected void replyError(RoutingNode target, com.yahoo.messagebus.Error error) { + EmptyReply reply = new EmptyReply(); + reply.addError(error); + target.handleReply(reply); + assertTrue(reply == frame.getReceptor().getReply(60)); + assertTrue(reply.hasErrors()); + if (debug) System.err.println("ERROR: " + reply.getTrace()); + } + +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java new file mode 100644 index 00000000000..515b1f45538 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.test; + +import com.yahoo.document.DocumentRemove; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentAccessParams; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; +import com.yahoo.documentapi.local.LocalDocumentAccess; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.network.Identity; +import com.yahoo.messagebus.network.rpc.RPCNetworkParams; + +import java.util.Arrays; + +/** + * Mock-up destination used for testing. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class Destination implements MessageHandler { + + private final DestinationSession session; + private final DocumentAccess access; + private final SyncSession local; + private final RPCMessageBus bus; + + public Destination(String slobrokConfigId, String documentManagerConfigId) { + + DocumentAccessParams params = new DocumentAccessParams(); + params.setDocumentManagerConfigId(documentManagerConfigId); + access = new LocalDocumentAccess(params); + local = access.createSyncSession(new SyncParameters()); + bus = new RPCMessageBus(Arrays.asList((Protocol)new DocumentProtocol(access.getDocumentTypeManager())), + new RPCNetworkParams() + .setIdentity(new Identity("test/destination")) + .setSlobrokConfigId(slobrokConfigId), + "file:src/test/cfg/messagebus.cfg"); + session = bus.getMessageBus().createDestinationSession("session", true, this); + } + + protected void sendReply(Reply reply) { + session.reply(reply); + } + + public void handleMessage(Message msg) { + Reply reply = ((DocumentMessage)msg).createReply(); + try { + switch (msg.getType()) { + + case DocumentProtocol.MESSAGE_GETDOCUMENT: + reply = new GetDocumentReply(local.get(((GetDocumentMessage)msg).getDocumentId())); + break; + + case DocumentProtocol.MESSAGE_PUTDOCUMENT: + local.put(((PutDocumentMessage)msg).getDocumentPut()); + break; + + case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: + local.remove(new DocumentRemove(((RemoveDocumentMessage)msg).getDocumentId())); + break; + + case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: + local.update(((UpdateDocumentMessage)msg).getDocumentUpdate()); + break; + + default: + throw new UnsupportedOperationException("Unsupported message type '" + msg.getType() + "'."); + } + } catch (Exception e) { + reply = new EmptyReply(); + reply.addError(new Error(ErrorCode.APP_FATAL_ERROR, e.toString())); + } + msg.swapState(reply); + session.reply(reply); + } + + public void shutdown() { + local.destroy(); + access.shutdown(); + session.destroy(); + bus.getMessageBus().destroy(); + } + +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusDocumentApiTestCase.java new file mode 100644 index 00000000000..48eba5bdaa8 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusDocumentApiTestCase.java @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.test; + +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.protocol.CreateVisitorReply; +import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.test.AbstractDocumentApiTestCase; +import com.yahoo.jrt.ListenFailedException; +import com.yahoo.jrt.slobrok.server.Slobrok; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.network.Identity; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class MessageBusDocumentApiTestCase extends AbstractDocumentApiTestCase { + + private Slobrok slobrok; + private Destination destination; + private DocumentAccess access; + + @Override + protected DocumentAccess access() { + return access; + } + + @Before + public void setUp() throws ListenFailedException { + slobrok = new Slobrok(); + String slobrokConfigId = + "raw:slobrok[1]\n" + "slobrok[0].connectionspec tcp/localhost:" + slobrok.port() + "\n"; + + MessageBusParams params = new MessageBusParams(); + params.getRPCNetworkParams().setIdentity(new Identity("test/feeder")); + params.getRPCNetworkParams().setSlobrokConfigId(slobrokConfigId); + params.setDocumentManagerConfigId("file:src/test/cfg/documentmanager.cfg"); + params.setRouteName("Route"); + params.setRoutingConfigId("file:src/test/cfg/messagebus.cfg"); + params.setTraceLevel(9); + params.setSourceSessionParams(new SourceSessionParams().setThrottlePolicy(null)); + access = new MessageBusDocumentAccess(params); + + destination = new VisitableDestination(slobrokConfigId, params.getDocumentManagerConfigId()); + } + + @After + public void tearDown() { + access.shutdown(); + destination.shutdown(); + slobrok.stop(); + } + + private static class VisitableDestination extends Destination { + private VisitableDestination(String slobrokConfigId, String documentManagerConfigId) { + super(slobrokConfigId, documentManagerConfigId); + } + + public void handleMessage(Message msg) { + if (msg.getType() == DocumentProtocol.MESSAGE_CREATEVISITOR) { + Reply reply = ((DocumentMessage)msg).createReply(); + msg.swapState(reply); + CreateVisitorReply visitorReply = (CreateVisitorReply)reply; + visitorReply.setLastBucket(ProgressToken.FINISHED_BUCKET); + sendReply(reply); + } else { + super.handleMessage(msg); + } + } + } + + + @Test + public void requireThatVisitorSessionWorksWithMessageBus() throws ParseException, InterruptedException { + VisitorParameters parameters = new VisitorParameters("id.user==1234"); + parameters.setRoute("Route"); + VisitorSession session = ((MessageBusDocumentAccess)access).createVisitorSession(parameters); + boolean ok = session.waitUntilDone(60*5*1000); + assertTrue(ok); + session.destroy(); + + // TODO(vekterli): test remote-to-local message sending as well? + // TODO(vekterli): test DocumentAccess shutdown during active ession? + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java new file mode 100755 index 00000000000..3bb61a5e887 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java @@ -0,0 +1,2508 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.test; + +import com.yahoo.document.BucketId; +import com.yahoo.document.DocumentId; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.messagebus.MessageBusVisitorSession; +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.Result; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RouteSpec; +import com.yahoo.messagebus.routing.RoutingTable; +import com.yahoo.messagebus.routing.RoutingTableSpec; +import com.yahoo.vdslib.VisitorStatistics; +import org.junit.Test; + +import java.nio.charset.Charset; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; + +public class MessageBusVisitorSessionTestCase { + private class MockSender implements MessageBusVisitorSession.Sender { + private int maxPending = 1000; + private int pendingCount = 0; + private ArrayList<Message> messages = new ArrayList<Message>(); + private ReplyHandler replyHandler = null; + private boolean destroyed = false; + private RuntimeException exceptionOnSend = null; + + @Override + public Result send(Message msg) { + synchronized (this) { + // Used to force failure during create visitors task processing + if (exceptionOnSend != null) { + throw exceptionOnSend; + } + if (pendingCount < maxPending) { + messages.add(msg); + ++pendingCount; + notifyAll(); + return Result.ACCEPTED; + } else { + return new Result(1234, "too many pending messages"); + } + } + } + + @Override + public void destroy() { + synchronized (this) { + destroyed = true; + } + } + + @Override + public int getPendingCount() { + synchronized (this) { + return pendingCount; + } + } + + public boolean isDestroyed() { + synchronized (this) { + return destroyed; + } + } + + public void setExceptionOnSend(RuntimeException exceptionOnSend) { + this.exceptionOnSend = exceptionOnSend; + } + + public void waitForMessages(int count, long timeout) throws IllegalStateException { + long timeoutAt = System.currentTimeMillis() + timeout; + synchronized (this) { + while (messages.size() < count) { + if (System.currentTimeMillis() >= timeoutAt) { + throw new IllegalStateException("Timed out waiting for " + count + " messages"); + } + try { + this.wait(timeout); + } catch (InterruptedException e) { + } + } + } + } + + public int getMessageCount() { + synchronized (this) { + return messages.size(); + } + } + + public Message getAndRemoveMessage(int index) { + synchronized (this) { + if (index >= messages.size()) { + throw new IllegalArgumentException("Bad message index"); + } + return messages.remove(index); + } + } + + public void setReplyHandler(ReplyHandler replyHandler) { + synchronized (this) { + this.replyHandler = replyHandler; + } + } + + public void setMaxPending(int maxPending) { + synchronized (this) { + this.maxPending = maxPending; + } + } + + public void reply(Reply reply) { + synchronized (this) { + if (replyHandler == null) { + throw new IllegalArgumentException("Reply handler has not been set"); + } + --pendingCount; + assert(pendingCount >= 0); + } + replyHandler.handleReply(reply); + } + } + + private class MockSenderFactory implements MessageBusVisitorSession.SenderFactory { + private MockSender sender; + + public MockSenderFactory(MockSender sender) { + this.sender = sender; + } + + @Override + public MessageBusVisitorSession.Sender createSender(ReplyHandler replyHandler, VisitorParameters visitorParameters) { + MockSender ret = sender; + if (ret == null) { + throw new IllegalStateException("Attempted to create mock sender twice"); + } + ret.setReplyHandler(replyHandler); + sender = null; + return ret; + } + } + + private class MockReceiver implements MessageBusVisitorSession.Receiver { + private ArrayList<Reply> replies = new ArrayList<Reply>(); + private MessageHandler messageHandler = null; + private boolean destroyed = false; + private String connectionSpec = "receiver/connection/spec"; + + public ArrayList<Reply> getReplies() { + return replies; + } + + public void setMessageHandler(MessageHandler messageHandler) { + this.messageHandler = messageHandler; + } + + public boolean isDestroyed() { + return destroyed; + } + + @Override + public void reply(Reply reply) { + replies.add(reply); + } + + public int getReplyCount() { + return replies.size(); + } + + @Override + public void destroy() { + destroyed = true; + } + + @Override + public String getConnectionSpec() { + return connectionSpec; + } + + public void setConnectionSpec(String connectionSpec) { + this.connectionSpec = connectionSpec; + } + + /** + * Invoke registered MessageHandler with message + * @param message message to "send" + */ + public void send(Message message) { + messageHandler.handleMessage(message); + } + + public Reply getAndRemoveReply(int index) { + if (index >= replies.size()) { + throw new IllegalArgumentException("Bad reply index"); + } + return replies.remove(index); + } + + public String repliesToString() { + StringBuilder builder = new StringBuilder(); + for (Reply reply : replies) { + builder.append(reply.getClass().getSimpleName()); + if (reply.hasErrors()) { + builder.append('('); + for (int i = 0; i < reply.getNumErrors(); ++i) { + if (i > 0) { + builder.append(", "); + } + Error err = reply.getError(i); + builder.append(DocumentProtocol.getErrorName(err.getCode())); + builder.append(": "); + builder.append(err.getMessage()); + } + builder.append(')'); + } + builder.append('\n'); + } + return builder.toString(); + } + } + + private class MockReceiverFactory implements MessageBusVisitorSession.ReceiverFactory { + private MockReceiver receiver; + + private MockReceiverFactory(MockReceiver receiver) { + this.receiver = receiver; + } + + @Override + public MessageBusVisitorSession.Receiver createReceiver(MessageHandler messageHandler, + String sessionName) { + MockReceiver ret = receiver; + if (ret == null) { + throw new IllegalStateException("Attempted to create mock receiver twice"); + } + ret.setMessageHandler(messageHandler); + receiver = null; + return ret; + } + } + + public static class TaskDescriptor implements Comparable<TaskDescriptor> { + private Runnable task; + private long timestamp; + private long sequenceId; + + public TaskDescriptor(Runnable task, long timestamp, long sequenceId) { + this.task = task; + this.timestamp = timestamp; + this.sequenceId = sequenceId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TaskDescriptor td = (TaskDescriptor) o; + + if (sequenceId != td.sequenceId) return false; + if (timestamp != td.timestamp) return false; + if (!task.equals(td.task)) return false; + + return true; + } + + public int compareTo(TaskDescriptor o) { + if (timestamp < o.timestamp) return -1; + if (timestamp > o.timestamp) return 1; + if (sequenceId < o.sequenceId) return -1; + if (sequenceId > o.sequenceId) return 1; + return 0; + } + + public Runnable getTask() { + return task; + } + + public long getTimestamp() { + return timestamp; + } + + public long getSequenceId() { + return sequenceId; + } + } + + /** + * Mock the executor to keep things nicely single threaded for the testing. + * No need to synchronize things here since we don't use multiple threads. + */ + public class MockAsyncTaskExecutor implements MessageBusVisitorSession.AsyncTaskExecutor { + private long sequenceCounter = 0; + private long timeMs = 0; + private Set<TaskDescriptor> tasks = new TreeSet<TaskDescriptor>(); + private int rejectTasksAfter = -1; + + public void setRejectTasksAfter(int rejectTasksAfter) { + this.rejectTasksAfter = rejectTasksAfter; + } + + private void checkTaskAcceptance() { + if (rejectTasksAfter == 0) { + throw new RejectedExecutionException("rejectTasksAfter is 0; rejecting task"); + } else if (rejectTasksAfter > 0) { + --rejectTasksAfter; + } + } + + @Override + public void submitTask(Runnable task) { + checkTaskAcceptance(); + tasks.add(new TaskDescriptor(task, 0, ++sequenceCounter)); + } + + @Override + public void scheduleTask(Runnable task, long delay, TimeUnit unit) { + checkTaskAcceptance(); + tasks.add(new TaskDescriptor(task, timeMs + unit.toMillis(delay), ++sequenceCounter)); + } + + public Set<TaskDescriptor> getTasks() { + return tasks; + } + + public int getScheduledTaskCount() { + return tasks.size(); + } + + public void setMockTimeMs(long timeMs) { + this.timeMs = timeMs; + } + + public void expectAndProcessTasks(int expectedTaskCount, + int processCount, + long[] taskRunAtTime) + { + if (tasks.size() != expectedTaskCount) { + throw new IllegalStateException("Expected " + expectedTaskCount + + " queued tasks, found " + tasks.size()); + } + if (taskRunAtTime != null && taskRunAtTime.length != tasks.size()) { + throw new IllegalStateException("Task time array must be equal in size to number of tasks"); + } + for (int i = 0; i < processCount; ++i) { + Iterator<TaskDescriptor> iter = tasks.iterator(); + TaskDescriptor td = iter.next(); + if (taskRunAtTime != null) { + if (taskRunAtTime[i] != td.getTimestamp()) { + throw new IllegalStateException( + "Expected task with scheduled execution time " + + taskRunAtTime[i] + ", was " + td.getTimestamp()); + } + } + iter.remove(); + td.getTask().run(); + } + } + + public void expectAndProcessTasks(int expectedTaskCount, int processCount) { + expectAndProcessTasks(expectedTaskCount, processCount, null); + } + + public void expectAndProcessTasks(int expectedTaskCount) { + expectAndProcessTasks(expectedTaskCount, expectedTaskCount); + } + + public void expectAndProcessTasks(int expectedTaskCount, long[] taskRunAtTime) { + expectAndProcessTasks(expectedTaskCount, expectedTaskCount, taskRunAtTime); + } + + public void expectNoTasks() { + if (!tasks.isEmpty()) { + StringBuilder sb = new StringBuilder(); + sb.append("Expected no tasks, but found these: "); + for (TaskDescriptor td : tasks) { + sb.append(td.getTask()).append(" "); + } + throw new IllegalStateException(sb.toString()); + } + } + } + + private MessageBusVisitorSession createVisitorSession(MockSender sender, + MockReceiver receiver, + MockAsyncTaskExecutor executor, + VisitorParameters visitorParameters, + RoutingTable routingTable) + { + if (routingTable == null) { + routingTable = new RoutingTable(new RoutingTableSpec(DocumentProtocol.NAME)); + } + try { + return new MessageBusVisitorSession( + visitorParameters, + executor, + new MockSenderFactory(sender), + new MockReceiverFactory(receiver), + routingTable); + } catch (ParseException e) { + throw new IllegalArgumentException("Bad document selection", e); + } + } + + private MessageBusVisitorSession createVisitorSession(MockSender sender, + MockReceiver receiver, + MockAsyncTaskExecutor executor, + VisitorParameters visitorParameters) + { + return createVisitorSession(sender, receiver, executor, visitorParameters, null); + } + + VisitorParameters createVisitorParameters(String selection) { + VisitorParameters params = new VisitorParameters(selection); + params.setRoute("storage"); // cannot be null by default + // TODO: skip the above and rather mock cluster route resolution, since + // this must be supported anyway! + return params; + } + + private String createVisitorToString(CreateVisitorMessage msg) { + StringBuilder sb = new StringBuilder(); + sb.append("CreateVisitorMessage(buckets=[\n"); + for (BucketId id : msg.getBuckets()) { + sb.append(id).append("\n"); + } + sb.append("]\n"); + if (!"".equals(msg.getDocumentSelection())) { + sb.append("selection='").append(msg.getDocumentSelection()).append("'\n"); + } + if (msg.getTimeRemaining() != 5 * 60 * 1000) { + sb.append("time remaining=").append(msg.getTimeRemaining()).append("\n"); + } + if (msg.getFromTimestamp() != 0) { + sb.append("from timestamp=").append(msg.getFromTimestamp()).append("\n"); + } + if (msg.getToTimestamp() != 0) { + sb.append("to timestamp=").append(msg.getToTimestamp()).append("\n"); + } + if (msg.getMaxPendingReplyCount() != 32) { + sb.append("max pending=").append(msg.getMaxPendingReplyCount()).append("\n"); + } + if (!"[all]".equals(msg.getFieldSet())) { + sb.append("fieldset=").append(msg.getFieldSet()).append("\n"); + } + if (msg.getVisitInconsistentBuckets()) { + sb.append("visit inconsistent=").append(msg.getVisitInconsistentBuckets()).append("\n"); + } + if (msg.getVisitRemoves()) { + sb.append("visit removes=").append(msg.getVisitRemoves()).append("\n"); + } + if (!msg.getParameters().isEmpty()) { + sb.append("parameters=[\n"); + for (Map.Entry<String, byte[]> kv : msg.getParameters().entrySet()) { + sb.append(kv.getKey()).append(" -> "); + sb.append(new String(kv.getValue(), Charset.defaultCharset())); + sb.append("\n"); + } + sb.append("]\n"); + } + if (msg.getRoute() != null && !"storage".equals(msg.getRoute().toString())) { + sb.append("route=").append(msg.getRoute()).append("\n"); + } + if (msg.getVisitorOrdering() != 0) { + sb.append("ordering=").append(msg.getVisitorOrdering()).append("\n"); + } + if (msg.getMaxBucketsPerVisitor() != 1) { + sb.append("max buckets per visitor=").append(msg.getMaxBucketsPerVisitor()).append("\n"); + } + if (msg.getLoadType() != LoadType.DEFAULT) { + sb.append("load type=").append(msg.getLoadType().getName()).append("\n"); + } + if (msg.getPriority() != DocumentProtocol.Priority.NORMAL_3) { + sb.append("priority=").append(msg.getPriority()).append("\n"); + } + if (!"DumpVisitor".equals(msg.getLibraryName())) { + sb.append("visitor library=").append(msg.getLibraryName()).append("\n"); + } + if (msg.getTrace().getLevel() != 0) { + sb.append("trace level=").append(msg.getTrace().getLevel()).append("\n"); + } + sb.append(")"); + return sb.toString(); + } + + private CreateVisitorReply createReply(CreateVisitorMessage msg) { + CreateVisitorReply reply = (CreateVisitorReply)msg.createReply(); + reply.setMessage(msg); + return reply; + } + + private String replyToCreateVisitor(MockSender sender, BucketId progress) { + CreateVisitorMessage msg = (CreateVisitorMessage)sender.getAndRemoveMessage(0); + CreateVisitorReply reply = createReply(msg); + reply.setLastBucket(progress); + sender.reply(reply); + return createVisitorToString(msg); + } + + private interface ReplyModifier { + public void modify(CreateVisitorReply reply); + } + + private String replyToCreateVisitor(MockSender sender, ReplyModifier modifier) { + CreateVisitorMessage msg = (CreateVisitorMessage)sender.getAndRemoveMessage(0); + CreateVisitorReply reply = createReply(msg); + modifier.modify(reply); + sender.reply(reply); + return createVisitorToString(msg); + } + + private String replyWrongDistributionToCreateVisitor(MockSender sender, + String clusterState) { + CreateVisitorMessage msg = (CreateVisitorMessage)sender.getAndRemoveMessage(0); + WrongDistributionReply reply = new WrongDistributionReply(clusterState); + reply.setMessage(msg); + reply.addError( + new com.yahoo.messagebus.Error( + DocumentProtocol.ERROR_WRONG_DISTRIBUTION, + "i pity the fool who uses 1 distribution bit!")); + sender.reply(reply); + return createVisitorToString(msg); + } + + private String replyErrorToCreateVisitor(MockSender sender, Error error) { + CreateVisitorMessage msg = (CreateVisitorMessage)sender.getAndRemoveMessage(0); + CreateVisitorReply reply = createReply(msg); + reply.setMessage(msg); + reply.addError(error); + sender.reply(reply); + return createVisitorToString(msg); + } + + private class MockComponents { + public MockSender sender; + public MockReceiver receiver; + public MockAsyncTaskExecutor executor; + public VisitorParameters params; + public MockControlHandler controlHandler; + public MockDataHandler dataHandler; + public MessageBusVisitorSession visitorSession; + + public MockComponents(VisitorParameters visitorParameters) { + this(visitorParameters, null); + } + + public MockComponents(VisitorParameters visitorParameters, RoutingTable routingTable) { + sender = new MockSender(); + receiver = new MockReceiver(); + executor = new MockAsyncTaskExecutor(); + params = visitorParameters; + controlHandler = new MockControlHandler(); + dataHandler = new MockDataHandler(); + params.setControlHandler(controlHandler); + params.setLocalDataHandler(dataHandler); + visitorSession = createVisitorSession(sender, receiver, executor, params, routingTable); + } + + public MockComponents() { + this(createVisitorParameters("")); + } + + public MockComponents(String selection) { + this(createVisitorParameters(selection)); + } + + // This seems a bit anti-pattern-ish in terms of builder usage... + public MockComponents(MockComponentsBuilder builder) { + sender = builder.sender; + receiver = builder.receiver; + executor = builder.executor; + params = builder.params; + controlHandler = builder.controlHandler; + dataHandler = builder.dataHandler; + visitorSession = createVisitorSession(sender, receiver, executor, params, builder.routingTable); + } + } + + private class MockComponentsBuilder { + public MockSender sender = new MockSender(); + public MockReceiver receiver = new MockReceiver(); + public MockAsyncTaskExecutor executor = new MockAsyncTaskExecutor(); + public VisitorParameters params = createVisitorParameters(""); + public MockControlHandler controlHandler = new MockControlHandler(); + public MockDataHandler dataHandler = new MockDataHandler(); + public RoutingTable routingTable = null; + + public MockComponents createMockComponents() { + return new MockComponents(this); + } + } + + private MockComponents createDefaultMock() { + return new MockComponents(); + } + + private MockComponents createDefaultMock(String selection) { + return new MockComponents(selection); + } + + private MockComponents createDefaultMock(VisitorParameters visitorParameters) { + return new MockComponents(visitorParameters); + } + + private MockComponents createDefaultMock(VisitorParameters visitorParameters, + RoutingTable routingTable) { + return new MockComponents(visitorParameters, routingTable); + } + + private void doTestSingleBucketVisit(VisitorParameters params, + String expectedMessage) + { + MockSender sender = new MockSender(); + MockReceiver receiver = new MockReceiver(); + MockAsyncTaskExecutor executor = new MockAsyncTaskExecutor(); + + MessageBusVisitorSession visitorSession = createVisitorSession( + sender, receiver, executor, params); + visitorSession.start(); + + // Process initial task which sends a single CreateVisitor. + executor.expectAndProcessTasks(1); + assertEquals(expectedMessage, replyToCreateVisitor(sender, ProgressToken.FINISHED_BUCKET)); + assertFalse(visitorSession.isDone()); + + // Single task for handling CreateVisitorReply. + executor.expectAndProcessTasks(1); + executor.expectNoTasks(); + assertTrue(visitorSession.isDone()); + } + + @Test + public void testSendSingleCreateVisitor() { + VisitorParameters params = createVisitorParameters(""); + Set<BucketId> bucketsToVisit = new TreeSet<BucketId>(); + BucketId bid = new BucketId(16, 1234); + bucketsToVisit.add(bid); + params.setBucketsToVisit(bucketsToVisit); + + String expected = "CreateVisitorMessage(buckets=[\n" + + bid + "\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)"; + + doTestSingleBucketVisit(params, expected); + } + + /** + * Test that using an id.user=foo selection only tries to visit a single + * superbucket for that user. + */ + @Test + public void testIdUserSelection() { + VisitorParameters params = createVisitorParameters("id.user=1234"); + String expected = "CreateVisitorMessage(buckets=[\n" + + new BucketId(32, 1234) + "\n" + + "BucketId(0x0000000000000000)\n" + + "]\n" + + "selection='id.user=1234'\n)"; + doTestSingleBucketVisit(params, expected); + } + + @Test + public void testMessageParameters() { + MockSender sender = new MockSender(); + MockReceiver receiver = new MockReceiver(); + MockAsyncTaskExecutor executor = new MockAsyncTaskExecutor(); + // Test all parameters that can be forwarded except bucketsToVisit, + // which is already explicitly tested in testSendSingleCreateVisitor(). + VisitorParameters params = new VisitorParameters(""); + params.setDocumentSelection("id.user=5678"); + params.setFromTimestamp(9001); + params.setToTimestamp(10001); + params.setVisitorLibrary("CoolVisitor"); + params.setLibraryParameter("groovy", "dudes"); + params.setLibraryParameter("ninja", "turtles"); + params.setMaxBucketsPerVisitor(55); + params.setPriority(DocumentProtocol.Priority.HIGHEST); + params.setRoute("extraterrestrial/highway"); + params.setTimeoutMs(1337); + params.setMaxPending(111); + params.setFieldSet("[header]"); + params.setVisitorOrdering(123); + params.setLoadType(new LoadType(3, "samnmax", DocumentProtocol.Priority.HIGH_3)); + params.setVisitRemoves(true); + params.setVisitInconsistentBuckets(true); + params.setTraceLevel(9); + + MessageBusVisitorSession visitorSession = createVisitorSession( + sender, receiver, executor, params); + visitorSession.start(); + + // Process initial task which sends a single CreateVisitor. + executor.expectAndProcessTasks(1); + + CreateVisitorMessage msg = (CreateVisitorMessage)sender.getAndRemoveMessage(0); + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x800000000000162e)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n" + + "selection='id.user=5678'\n" + + "time remaining=1337\n" + + "from timestamp=9001\n" + + "to timestamp=10001\n" + + "max pending=111\n" + + "fieldset=[header]\n" + + "visit inconsistent=true\n" + + "visit removes=true\n" + + "parameters=[\n" + + "groovy -> dudes\n" + + "ninja -> turtles\n" + + "]\n" + + "route=extraterrestrial/highway\n" + + "ordering=123\n" + + "max buckets per visitor=55\n" + + "load type=samnmax\n" + + "priority=HIGHEST\n" + + "visitor library=CoolVisitor\n" + + "trace level=9\n" + + ")", + createVisitorToString(msg)); + + assertFalse(msg.getRetryEnabled()); + } + + @Test + public void testBucketProgress() { + MockComponents mc = createDefaultMock("id.user==1234"); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n" + + "selection='id.user==1234'\n)", + replyToCreateVisitor(mc.sender, new BucketId(33, 1234 | (1L << 32)))); + + // Reply task + mc.executor.expectAndProcessTasks(1); + assertFalse(mc.visitorSession.isDone()); + // Should get new CreateVisitor task for sub-bucket continuation + mc.executor.expectAndProcessTasks(1); + CreateVisitorMessage msg2 = (CreateVisitorMessage)mc.sender.getAndRemoveMessage(0); + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x84000001000004d2)\n" + + "]\n" + + "selection='id.user==1234'\n)", + createVisitorToString(msg2)); + + assertEquals(mc.controlHandler.getProgress(), mc.visitorSession.getProgress()); + } + + @Test + public void testMaxPendingVisitorsForSender() { + MockSender sender = new MockSender(); + MockReceiver receiver = new MockReceiver(); + sender.setMaxPending(1); + MockAsyncTaskExecutor executor = new MockAsyncTaskExecutor(); + // Visit-all will normally start with 1 distribution bit and send + // to 2 superbuckets if allowed to do so. + VisitorParameters params = createVisitorParameters(""); + MessageBusVisitorSession visitorSession = createVisitorSession( + sender, receiver, executor, params); + + visitorSession.start(); + executor.expectAndProcessTasks(1); + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000000)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyToCreateVisitor(sender, ProgressToken.FINISHED_BUCKET)); + executor.expectAndProcessTasks(1); // Reply + executor.expectAndProcessTasks(1); // New visitor + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000001)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyToCreateVisitor(sender, ProgressToken.FINISHED_BUCKET)); + } + + @Test + public void testVisitAll() { + MockSender sender = new MockSender(); + MockReceiver receiver = new MockReceiver(); + sender.setMaxPending(1000); + MockAsyncTaskExecutor executor = new MockAsyncTaskExecutor(); + VisitorParameters params = createVisitorParameters(""); + MessageBusVisitorSession visitorSession = createVisitorSession( + sender, receiver, executor, params); + + visitorSession.start(); + executor.expectAndProcessTasks(1); + assertEquals(2, sender.getMessageCount()); + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000000)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyToCreateVisitor(sender, ProgressToken.FINISHED_BUCKET)); + + executor.expectAndProcessTasks(1); + executor.expectNoTasks(); // No new visitors yet. + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000001)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyToCreateVisitor(sender, new BucketId(8, 1 | (1 << 8)))); + + executor.expectAndProcessTasks(1); + // Send new visitor for bucket 1 + executor.expectAndProcessTasks(1); + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000001)\n" + + "BucketId(0x2000000000000001)\n" + + "]\n)", + replyToCreateVisitor(sender, ProgressToken.FINISHED_BUCKET)); + + executor.expectAndProcessTasks(1); // Reply task + executor.expectNoTasks(); // Visiting complete + + assertTrue(visitorSession.isDone()); + } + + @Test + public void testWrongDistributionAdjustsDistributionBits() { + MockSender sender = new MockSender(); + MockReceiver receiver = new MockReceiver(); + sender.setMaxPending(2); + MockAsyncTaskExecutor executor = new MockAsyncTaskExecutor(); + VisitorParameters params = createVisitorParameters(""); + MessageBusVisitorSession visitorSession = createVisitorSession( + sender, receiver, executor, params); + + visitorSession.start(); + executor.expectAndProcessTasks(1); + assertEquals(2, sender.getMessageCount()); + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000000)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyWrongDistributionToCreateVisitor( + sender, "version:2 storage:100 distributor:100 bits:16")); + executor.expectAndProcessTasks(1); // WDR reply + // Replying with WRONG_DISTRIBUTION when there are active visitors + // should not send any new visitors until all active have returned. + // This allows the visitor iterator to consistently adjust the visiting + // progress based on the distribution bit change. + executor.expectNoTasks(); + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000001)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyWrongDistributionToCreateVisitor( + sender, "version:2 storage:100 distributor:100 bits:16")); + executor.expectAndProcessTasks(1); // WDR reply + executor.expectAndProcessTasks(1, new long[] { 0 }); // Send new visitors, no delay + + // Now with 16 distribution bits. + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x4000000000000000)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyToCreateVisitor(sender, ProgressToken.FINISHED_BUCKET)); + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x4000000000008000)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyToCreateVisitor(sender, ProgressToken.FINISHED_BUCKET)); + // .... and 65533 more + } + + private class MockControlHandler extends VisitorControlHandler { + private StringBuilder builder = new StringBuilder(); + private RuntimeException exceptionOnProgress = null; + private boolean synchronousWaitUntilDone = false; + private boolean waiting = false; + + public void setExceptionOnProgress(RuntimeException exceptionOnProgress) { + this.exceptionOnProgress = exceptionOnProgress; + } + + public void setSynchronousWaitUntilDone(boolean synchronousWaitUntilDone) { + this.synchronousWaitUntilDone = synchronousWaitUntilDone; + } + + @Override + public void onProgress(ProgressToken token) { + super.onProgress(token); + builder.append("onProgress : "); + builder.append(token.getActiveBucketCount()).append(" active, "); + builder.append(token.getPendingBucketCount()).append(" pending, "); + builder.append(token.getFinishedBucketCount()).append(" finished, "); + builder.append(token.getTotalBucketCount()).append(" total\n"); + if (exceptionOnProgress != null) { + throw exceptionOnProgress; + } + } + + @Override + public void onVisitorError(String message) { + super.onVisitorError(message); + builder.append("onVisitorError : ").append(message).append("\n"); + } + + @Override + public void onVisitorStatistics(VisitorStatistics vs) { + super.onVisitorStatistics(vs); + builder.append("onVisitorStatistics : "); + // Only bother with a couple of fields. + builder.append(vs.getBucketsVisited()).append(" buckets visited, "); + builder.append(vs.getDocumentsReturned() + vs.getSecondPassDocumentsReturned()).append(" docs returned\n"); + } + + @Override + public void onDone(CompletionCode code, String message) { + super.onDone(code, message); + builder.append("onDone : ").append(code).append( " - "); + builder.append("'").append(message).append("'\n"); + } + + @Override + public void setSession(VisitorControlSession session) { + super.setSession(session); + builder.append("setSession\n"); + } + + @Override + public void reset() { + super.reset(); + builder.append("reset\n"); + } + + @Override + public boolean waitUntilDone(long timeoutMs) throws InterruptedException { + builder.append("waitUntilDone : " + timeoutMs + "\n"); + if (synchronousWaitUntilDone) { + synchronized (this) { + waiting = true; + } + return super.waitUntilDone(timeoutMs); + } + return isDone(); + } + + public synchronized boolean isWaiting() { + return waiting; + } + + public String toString() { + return builder.toString(); + } + + public void resetMock() { + builder = new StringBuilder(); + } + } + + private class MockDataHandler extends VisitorDataHandler { + + public class MessageWrapper { + private Message message; + private AckToken ackToken; + + public MessageWrapper(Message message, AckToken ackToken) { + this.message = message; + this.ackToken = ackToken; + } + + public Message getMessage() { + return message; + } + + public AckToken getAckToken() { + return ackToken; + } + } + + private ArrayList<MessageWrapper> messages = new ArrayList<MessageWrapper>(); + private StringBuilder builder = new StringBuilder(); + private RuntimeException exceptionOnMessage = null; + + public void setExceptionOnMessage(RuntimeException exceptionOnMessage) { + this.exceptionOnMessage = exceptionOnMessage; + } + + @Override + public void setSession(VisitorControlSession session) { + builder.append("setSession\n"); + super.setSession(session); + } + + @Override + public void reset() { + builder.append("reset\n"); + super.reset(); + } + + @Override + public VisitorResponse getNext() { + builder.append("getNext\n"); + return new VisitorResponse(null); + } + + @Override + public VisitorResponse getNext(int timeoutMilliseconds) throws InterruptedException { + builder.append("getNext : ").append(timeoutMilliseconds).append('\n'); + return new VisitorResponse(null); + } + + @Override + public void onDone() { + builder.append("onDone\n"); + super.onDone(); + } + + @Override + public void onMessage(Message m, AckToken token) { + builder.append("onMessage\n"); + messages.add(new MessageWrapper(m, token)); + if (exceptionOnMessage != null) { + throw exceptionOnMessage; + } + } + + public ArrayList<MessageWrapper> getMessages() { + return messages; + } + + public String toString() { + return builder.toString(); + } + + public void resetMock() { + builder = new StringBuilder(); + } + } + + @Test + public void testControlHandlerInvocationNormal() { + MockComponents mc = createDefaultMock("id.user=1234"); + assertEquals("reset\nsetSession\n", mc.controlHandler.toString()); + mc.controlHandler.resetMock(); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + replyToCreateVisitor(mc.sender, new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + reply.setLastBucket(ProgressToken.FINISHED_BUCKET); + VisitorStatistics stats = new VisitorStatistics(); + stats.setBucketsVisited(11); + stats.setDocumentsReturned(22); + reply.setVisitorStatistics(stats); + } + }); + mc.executor.expectAndProcessTasks(1); + assertEquals("onProgress : 0 active, 0 pending, 1 finished, 1 total\n" + + "onVisitorStatistics : 11 buckets visited, 22 docs returned\n" + + "onDone : SUCCESS - ''\n", + mc.controlHandler.toString()); + assertTrue(mc.visitorSession.isDone()); + } + + @Test + public void testLocalDataHandlerInvocationWithAck() { + MockComponents mc = createDefaultMock("id.user=1234"); + assertEquals("reset\nsetSession\n", mc.dataHandler.toString()); + mc.dataHandler.resetMock(); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + + // Send a remove (so we don't have to create a new doc instance) + mc.receiver.send(new RemoveDocumentMessage(new DocumentId("doc:foo:bar"))); + mc.executor.expectAndProcessTasks(1); + + // Not yet ACKed + assertEquals("", mc.receiver.repliesToString()); + + assertEquals(1, mc.dataHandler.getMessages().size()); + MockDataHandler.MessageWrapper msg = mc.dataHandler.getMessages().get(0); + mc.dataHandler.ack(msg.getAckToken()); + + assertEquals("RemoveDocumentReply\n", mc.receiver.repliesToString()); + + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET); + mc.executor.expectAndProcessTasks(1); + assertEquals( + "onMessage\n" + + "onDone\n", + mc.dataHandler.toString()); + assertTrue(mc.visitorSession.isDone()); + } + + @Test + public void testCreateDefaultVisitorControlHandlerIfNoneGiven() { + MockSender sender = new MockSender(); + MockReceiver receiver = new MockReceiver(); + MockAsyncTaskExecutor executor = new MockAsyncTaskExecutor(); + VisitorParameters params = createVisitorParameters(""); + MessageBusVisitorSession visitorSession = createVisitorSession( + sender, receiver, executor, params); + assertNotNull(params.getControlHandler()); + } + + @Test + public void testNoDataHandlersImpliesVisitorDataQueue() { + MockSender sender = new MockSender(); + MockReceiver receiver = new MockReceiver(); + MockAsyncTaskExecutor executor = new MockAsyncTaskExecutor(); + VisitorParameters params = createVisitorParameters(""); + MessageBusVisitorSession visitorSession = createVisitorSession( + sender, receiver, executor, params); + assertNotNull(params.getLocalDataHandler()); + assertTrue(params.getLocalDataHandler() instanceof VisitorDataQueue); + } + + @Test + public void testAbortVisiting() { + MockComponents mc = createDefaultMock(); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + assertEquals(2, mc.sender.getMessageCount()); + mc.controlHandler.resetMock(); + // While we have active visitors, abort visiting. Completion function + // should not be called until we have no pending messages. + mc.visitorSession.abort(); + assertFalse(mc.visitorSession.isDone()); + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000000)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET)); + + mc.executor.expectAndProcessTasks(1); + assertEquals("onProgress : 1 active, 0 pending, 1 finished, 2 total\n" + + "onVisitorStatistics : 0 buckets visited, 0 docs returned\n", + mc.controlHandler.toString()); + assertFalse(mc.visitorSession.isDone()); + mc.controlHandler.resetMock(); + + // When aborted, no new visitors should be sent. + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000001)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyToCreateVisitor(mc.sender, new BucketId(0x8400000100000001L))); + + mc.executor.expectAndProcessTasks(1); + mc.executor.expectAndProcessTasks(0); + assertEquals(0, mc.sender.getMessageCount()); + assertTrue(mc.visitorSession.isDone()); + + assertEquals("onProgress : 0 active, 1 pending, 1 finished, 2 total\n" + + "onVisitorStatistics : 0 buckets visited, 0 docs returned\n" + + "onDone : ABORTED - 'Visitor aborted by user'\n", + mc.controlHandler.toString()); + assertEquals("ABORTED: Visitor aborted by user", + mc.controlHandler.getResult().toString()); + } + + /** + * Test that different sessions get different visitor names. + */ + @Test + public void testUniqueSessionNames() { + MockComponents mc1 = createDefaultMock(); + MockComponents mc2 = createDefaultMock(); + assert(!mc1.visitorSession.getSessionName().equals( + mc2.visitorSession.getSessionName())); + } + + /** + * Test that different visitors within the same session get different + * names. + */ + @Test + public void testUniqueVisitorNames() { + MockComponents mc = createDefaultMock(); + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + assertEquals(2, mc.sender.getMessageCount()); + + CreateVisitorMessage msg1 = (CreateVisitorMessage)mc.sender.getAndRemoveMessage(0); + CreateVisitorMessage msg2 = (CreateVisitorMessage)mc.sender.getAndRemoveMessage(0); + assert(!msg1.getInstanceId().equals(msg2.getInstanceId())); + } + + @Test + public void testMax1ConcurrentSendCreateVisitorsTask() { + MockComponents mc = createDefaultMock(); + + mc.executor.setMockTimeMs(1000); + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + assertEquals(2, mc.sender.getMessageCount()); + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000000)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyToCreateVisitor(mc.sender, new BucketId(0x8400000100000000L))); + + // Execute reply task which will schedule a SendCreateVisitors task. + mc.executor.expectAndProcessTasks(1); + assertEquals(1, mc.executor.getScheduledTaskCount()); + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000001)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyToCreateVisitor(mc.sender, new BucketId(0x8400000100000001L))); + // Execute reply task which should _not_ schedule a SendCreateVisitors task + // since one has already been scheduled. Note that since the second reply + // task was directly submitted rather than scheduled, it should always be + // executed before the SendCreateVisitors task in our deterministic test + // environment. + mc.executor.expectAndProcessTasks(2, 1); + // Finally execute scheduled SendCreateVisitors task. + mc.executor.expectAndProcessTasks(1); + mc.executor.expectNoTasks(); + assertEquals(2, mc.sender.getMessageCount()); + } + + @Test + public void testRetryVisitorOnTransientError() { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.visitorSession.start(); + mc.controlHandler.resetMock(); + mc.executor.expectAndProcessTasks(1); + replyToCreateVisitor(mc.sender, new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + reply.addError(new Error( + DocumentProtocol.ERROR_ABORTED, + "bucket fell down a well")); + } + }); + mc.executor.expectAndProcessTasks(1); // reply + // Must have a 100ms delay + mc.executor.expectAndProcessTasks(1, new long[] { 100 }); // send + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n" + + "selection='id.user==1234'\n)", + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET)); + mc.executor.expectAndProcessTasks(1); + mc.executor.expectNoTasks(); + assertTrue(mc.visitorSession.isDone()); + assertEquals("onVisitorError : ABORTED: bucket fell down a well\n" + + "onProgress : 0 active, 0 pending, 1 finished, 1 total\n" + + "onVisitorStatistics : 0 buckets visited, 0 docs returned\n" + + "onDone : SUCCESS - ''\n", + mc.controlHandler.toString()); + } + + @Test + public void testFailVisitingOnFatalError() { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.visitorSession.start(); + mc.controlHandler.resetMock(); + mc.executor.expectAndProcessTasks(1); + replyToCreateVisitor(mc.sender, new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + reply.addError(new Error( + DocumentProtocol.ERROR_INTERNAL_FAILURE, + "node caught fire")); + } + }); + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectNoTasks(); + assertEquals(0, mc.sender.getMessageCount()); + assertTrue(mc.visitorSession.isDone()); + + assertEquals("onVisitorError : INTERNAL_FAILURE: node caught fire\n" + + "onDone : FAILURE - 'INTERNAL_FAILURE: node caught fire'\n", + mc.controlHandler.toString()); + } + + /** + * Do not complete visiting upon fatal error until all replies have + * been received. + */ + @Test + public void testWaitUntilVisitorsDoneOnFatalError() { + MockComponents mc = createDefaultMock(); + mc.visitorSession.start(); + mc.controlHandler.resetMock(); // clear messages + mc.executor.expectAndProcessTasks(1); + assertEquals(2, mc.sender.getMessageCount()); + replyToCreateVisitor(mc.sender, new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + reply.addError(new Error( + DocumentProtocol.ERROR_INTERNAL_FAILURE, + "node fell down a well")); + } + }); + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectNoTasks(); + assertEquals(1, mc.sender.getMessageCount()); // no resending + assertFalse(mc.visitorSession.isDone()); // not done yet + + replyToCreateVisitor(mc.sender, new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + reply.addError(new Error( + DocumentProtocol.ERROR_INTERNAL_FAILURE, + "node got hit by a falling brick")); + } + }); + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectNoTasks(); + assertEquals(0, mc.sender.getMessageCount()); // no resending + assertTrue(mc.visitorSession.isDone()); + + // should get first received failure message as completion failure message + assertEquals("onVisitorError : INTERNAL_FAILURE: node fell down a well\n" + + "onVisitorError : INTERNAL_FAILURE: node got hit by a falling brick\n" + + "onDone : FAILURE - 'INTERNAL_FAILURE: node fell down a well'\n", + mc.controlHandler.toString()); + } + + private void doTestEarlyCompletion(VisitorParameters visitorParameters, + ReplyModifier replyModifier1, + ReplyModifier replyModifier2) + { + MockComponents mc = createDefaultMock(visitorParameters); + mc.controlHandler.resetMock(); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + // First reply gives only 9 hits, so must send another visitor + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n" + + "selection='id.user==1234'\n)", + replyToCreateVisitor(mc.sender, replyModifier1)); + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectAndProcessTasks(1); // new visitor + mc.controlHandler.resetMock(); + assertEquals(1, mc.sender.getMessageCount()); + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x84000001000004d2)\n" + + "]\n" + + "selection='id.user==1234'\n)", + replyToCreateVisitor(mc.sender, replyModifier2)); + // we've now got enough total hits; session should be marked as + // completed and no further visitors should be sent. + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectNoTasks(); + assertEquals(0, mc.sender.getMessageCount()); + + assertEquals("onProgress : 0 active, 1 pending, 0 finished, 1 total\n" + + "onVisitorStatistics : 2 buckets visited, 10 docs returned\n" + + "onDone : SUCCESS - ''\n", mc.controlHandler.toString()); + assertEquals("OK: ", mc.controlHandler.getResult().toString()); + } + + /** + * Test visitor "prematurely" completing due to max total hits being + * reached when no other visitors are currently pending. + */ + @Test + public void testMaxTotalHitsEarlyCompletion() { + VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); + visitorParameters.setMaxTotalHits(10); + ReplyModifier replyModifier1 = new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + VisitorStatistics stats = new VisitorStatistics(); + stats.setBucketsVisited(1); + stats.setDocumentsReturned(9); + reply.setVisitorStatistics(stats); + reply.setLastBucket(new BucketId(33, 1234 | (1L << 32))); + } + }; + ReplyModifier replyModifier2 = new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + VisitorStatistics stats = new VisitorStatistics(); + stats.setBucketsVisited(1); + stats.setDocumentsReturned(1); + reply.setVisitorStatistics(stats); + reply.setLastBucket(new BucketId(34, 1234 | (1L << 33))); + } + }; + doTestEarlyCompletion(visitorParameters, replyModifier1, replyModifier2); + } + + @Test + public void testVisitingCompletedFromSufficientFirstPassHits() { + VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); + visitorParameters.setMaxFirstPassHits(10); + ReplyModifier replyModifier1 = new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + VisitorStatistics stats = new VisitorStatistics(); + stats.setBucketsVisited(1); + stats.setDocumentsReturned(9); + reply.setVisitorStatistics(stats); + reply.setLastBucket(new BucketId(33, 1234 | (1L << 32))); + } + }; + ReplyModifier replyModifier2 = new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + VisitorStatistics stats = new VisitorStatistics(); + stats.setBucketsVisited(1); + stats.setDocumentsReturned(1); + reply.setVisitorStatistics(stats); + reply.setLastBucket(new BucketId(34, 1234 | (1L << 33))); + } + }; + doTestEarlyCompletion(visitorParameters, replyModifier1, replyModifier2); + } + + @Test + public void testVisitingCompletedFromSecondPassHits() { + VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); + visitorParameters.setMaxTotalHits(10); + ReplyModifier replyModifier1 = new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + VisitorStatistics stats = new VisitorStatistics(); + stats.setBucketsVisited(1); + stats.setDocumentsReturned(5); + stats.setSecondPassDocumentsReturned(4); + reply.setVisitorStatistics(stats); + reply.setLastBucket(new BucketId(33, 1234 | (1L << 32))); + } + }; + ReplyModifier replyModifier2 = new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + VisitorStatistics stats = new VisitorStatistics(); + stats.setBucketsVisited(1); + stats.setSecondPassDocumentsReturned(1); + reply.setVisitorStatistics(stats); + reply.setLastBucket(new BucketId(34, 1234 | (1L << 33))); + } + }; + doTestEarlyCompletion(visitorParameters, replyModifier1, replyModifier2); + } + + /** + * Test that waitUntilDone on the session is forwarded to the control handler. + */ + @Test + public void testControlHandlerWaitUntilDone() throws Exception { + MockComponents mc = createDefaultMock(); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + mc.controlHandler.resetMock(); + + assertFalse(mc.visitorSession.waitUntilDone(1234)); // not completed + assertEquals("waitUntilDone : 1234\n", mc.controlHandler.toString()); + } + + @Test + public void testDataHandlerGetNext() throws Exception { + MockComponents mc = createDefaultMock(); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + mc.dataHandler.resetMock(); + + assertNotNull(mc.visitorSession.getNext()); + assertNotNull(mc.visitorSession.getNext(1234)); + assertEquals("getNext\ngetNext : 1234\n", mc.dataHandler.toString()); + } + + @Test + public void testNoLocalDataHandlerGetNext() throws Exception { + MockSender sender = new MockSender(); + MockReceiver receiver = new MockReceiver(); + MockAsyncTaskExecutor executor = new MockAsyncTaskExecutor(); + VisitorParameters params = createVisitorParameters(""); + params.setRemoteDataHandler("the/moon"); + MessageBusVisitorSession visitorSession = createVisitorSession( + sender, receiver, executor, params); + + visitorSession.start(); + executor.expectAndProcessTasks(1); + + try { + assertNotNull(visitorSession.getNext()); + fail("No exception thrown on getNext()"); + } catch (IllegalStateException e) { + assertEquals("Data has been routed to external source for this visitor", e.getMessage()); + } + try { + assertNotNull(visitorSession.getNext(1234)); + fail("No exception thrown on getNext(int)"); + } catch (IllegalStateException e) { + assertEquals("Data has been routed to external source for this visitor", e.getMessage()); + } + } + + private static class SharedValue<T> { + private T value = null; + + public T getValue() { + return value; + } + + public void setValue(T value) { + this.value = value; + } + } + + void waitUntilTrue(long timeoutMs, Callable<Boolean> callable) throws Exception { + long timeStart = System.currentTimeMillis(); + while (!callable.call()) { + if (System.currentTimeMillis() - timeStart >= timeoutMs) { + throw new RuntimeException("Timeout while waiting for callable to yield true"); + } + Thread.sleep(10); + } + } + + /** + * Test that calling waitUntilDone waits until session has completed. + * Test that destroy() destroys the communication interfaces it uses. + * @throws Exception + */ + @Test + public void testSynchronousWaitUntilDoneAndDestroy() throws Exception { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + mc.controlHandler.setSynchronousWaitUntilDone(true); + mc.controlHandler.resetMock(); + final MockControlHandler controlHandler = mc.controlHandler; + final MessageBusVisitorSession session = mc.visitorSession; + final SharedValue<Exception> exceptionPropagator = new SharedValue<Exception>(); + final CyclicBarrier barrier = new CyclicBarrier(2); + + // Have to do this multi-threaded for once since waitUntilDone/destroy + // are both synchronous and will not return before session is complete, + // either through success or failure. + Thread t = new Thread(new Runnable() { + @Override + public void run() { + try { + boolean ok = session.waitUntilDone(20000); + if (!session.isDone()) { + throw new IllegalStateException("waitUntilDone returned, but session is not marked as done"); + } + assertTrue(ok); + session.destroy(); + barrier.await(20000, TimeUnit.MILLISECONDS); + } catch (Exception e) { + exceptionPropagator.setValue(e); + } + } + }); + t.start(); + + try { + waitUntilTrue(20000, new Callable<Boolean>() { + @Override + public Boolean call() throws Exception { + return controlHandler.isWaiting(); + } + }); + + // Reply to visitor, causing session to complete + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n" + + "selection='id.user==1234'\n)", + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET)); + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectNoTasks(); + + barrier.await(20000, TimeUnit.MILLISECONDS); + } catch (Exception e) { + t.interrupt(); + throw e; + } finally { + t.join(); + } + + if (exceptionPropagator.getValue() != null) { + throw new IllegalStateException( + "Exception thrown in destruction thread", + exceptionPropagator.getValue()); + } + + assertTrue(mc.sender.isDestroyed()); + assertTrue(mc.receiver.isDestroyed()); + + assertEquals( + "waitUntilDone : 20000\n" + + "onProgress : 0 active, 0 pending, 1 finished, 1 total\n" + + "onVisitorStatistics : 0 buckets visited, 0 docs returned\n" + + "onDone : SUCCESS - ''\n", + mc.controlHandler.toString()); + } + + @Test + public void testDestroyAbortsSessionIfNotDone() throws Exception { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + mc.controlHandler.setSynchronousWaitUntilDone(true); + mc.controlHandler.resetMock(); + final MessageBusVisitorSession session = mc.visitorSession; + final SharedValue<Exception> exceptionPropagator = new SharedValue<Exception>(); + final CyclicBarrier barrier = new CyclicBarrier(2); + + // Have to do this multi-threaded for once since destroy is + // synchronous and any code logic bug could otherwise cause the + // test (and thus the build) to hang indefinitely. + // NOTE: even though the MockControlHandler itself is not thread safe, + // the control flow of the test should guarantee there is no concurrent + // access to it. + Thread t = new Thread(new Runnable() { + @Override + public void run() { + try { + session.destroy(); + if (!session.isDone()) { + throw new IllegalStateException("Session is not marked as done after destroy()"); + } + barrier.await(20000, TimeUnit.MILLISECONDS); + } catch (Exception e) { + exceptionPropagator.setValue(e); + } + } + }); + t.start(); + + try { + waitUntilTrue(20000, new Callable<Boolean>() { + @Override + public Boolean call() throws Exception { + return session.isDestroying(); + } + }); + + // Reply to visitor. Normally, the visitor would be resent, but + // since destroy aborts the session, this won't happen and the + // session will be marked as completed instead. + replyErrorToCreateVisitor(mc.sender, new Error(DocumentProtocol.ERROR_BUCKET_DELETED, "goner")); + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectNoTasks(); + + barrier.await(20000, TimeUnit.MILLISECONDS); + } catch (Exception e) { + t.interrupt(); + throw e; + } finally { + t.join(); + } + + if (exceptionPropagator.getValue() != null) { + throw new IllegalStateException( + "Exception thrown in destruction thread", + exceptionPropagator.getValue()); + } + + assertTrue(mc.sender.isDestroyed()); + assertTrue(mc.receiver.isDestroyed()); + + assertEquals( + "onDone : ABORTED - 'Session explicitly destroyed before completion'\n", + mc.controlHandler.toString()); + } + + /** + * Test that receiving a WrongDistributionReply with a cluster state + * we cannot parse fails the visiting session. We cannot visit anything + * if we don't have a proper state anyway, so might as well fail fast. + */ + @Test + public void testClusterStateParseFailure() { + MockComponents mc = createDefaultMock(); + mc.visitorSession.start(); + mc.controlHandler.resetMock(); // clear messages + mc.executor.expectAndProcessTasks(1); + assertEquals(2, mc.sender.getMessageCount()); + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000000)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyWrongDistributionToCreateVisitor( + mc.sender, "one:bad cluster:state")); + mc.executor.expectAndProcessTasks(1); // WDR reply + // no resending since visiting has failed + mc.executor.expectNoTasks(); + assertFalse(mc.controlHandler.isDone()); + + // Complete visiting + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000001)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyWrongDistributionToCreateVisitor( + mc.sender, "another:bad cluster:state")); + mc.executor.expectAndProcessTasks(1); // WDR reply + assertTrue(mc.controlHandler.isDone()); + assertEquals("onDone : FAILURE - 'Failed to parse cluster state 'one:bad cluster:state''\n", + mc.controlHandler.toString()); + } + + @Test + public void testReceiveVisitorInfoMessage() { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.visitorSession.start(); + mc.controlHandler.resetMock(); + mc.executor.expectAndProcessTasks(1); + + // Send a VisitorInfo back without any errors. This should trigger + // the control handler's onProgress routine (at least this is what + // the legacy code does, so let's go with that). + mc.receiver.send(new VisitorInfoMessage()); + mc.executor.expectAndProcessTasks(1); // Message handler task + + assertEquals("onProgress : 1 active, 0 pending, 0 finished, 1 total\n", + mc.controlHandler.toString()); + assertEquals("VisitorReply\n", mc.receiver.repliesToString()); + mc.receiver.getAndRemoveReply(0); + + // Send VisitorInfo with error. This should invoke the control + // handler's onVisitorError method. + VisitorInfoMessage errMsg = new VisitorInfoMessage(); + errMsg.setErrorMessage("bears! bears everywhere!"); + + mc.receiver.send(errMsg); + mc.controlHandler.resetMock(); + mc.executor.expectAndProcessTasks(1); // Message handler task + + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET); + mc.executor.expectAndProcessTasks(1); // Reply handler task + + // Visitor info with error should not fail visiting itself, this + // is only done for _replies_ with errors. + assertEquals( + "onVisitorError : bears! bears everywhere!\n" + + "onProgress : 1 active, 0 pending, 0 finished, 1 total\n" + + "onProgress : 0 active, 0 pending, 1 finished, 1 total\n" + + "onVisitorStatistics : 0 buckets visited, 0 docs returned\n" + + "onDone : SUCCESS - ''\n", + mc.controlHandler.toString()); + assertEquals("VisitorReply\n", mc.receiver.repliesToString()); + } + + RoutingTable createDummyRoutingTable() { + RoutingTableSpec spec = new RoutingTableSpec(DocumentProtocol.NAME); + spec.addRoute(new RouteSpec("storage/badger.bar")); + RouteSpec storageCluster = new RouteSpec("storage/cluster.foo"); + storageCluster.addHop("bunnies"); + spec.addRoute(storageCluster); + spec.addRoute(new RouteSpec("storage/otters.baz")); + return new RoutingTable(spec); + } + + /** + * Test that we try to get a route to the storage cluster automatically if + * the provided visitor parameter route is null. + */ + @Test + public void testDefaultClusterRouteResolutionNullRoute() { + VisitorParameters visitorParameters = createVisitorParameters(""); + visitorParameters.setRoute((Route)null); // ensure route is null + RoutingTable table = createDummyRoutingTable(); + + createDefaultMock(visitorParameters, table); + assertEquals("storage/cluster.foo", visitorParameters.getRoute().toString()); + } + + /** + * Test that we try to get a route to the storage cluster automatically if + * the provided route has no hops. + */ + @Test + public void testDefaultClusterRouteResolutionNoHops() { + VisitorParameters visitorParameters = createVisitorParameters(""); + visitorParameters.setRoute(new Route()); + RoutingTable table = createDummyRoutingTable(); + + createDefaultMock(visitorParameters, table); + assertEquals("storage/cluster.foo", visitorParameters.getRoute().toString()); + } + + /** + * Test that we don't try to override a valid route in the parameters. + */ + @Test + public void testExplicitRouteNotOverridden() { + VisitorParameters visitorParameters = createVisitorParameters(""); + visitorParameters.setRoute("mars"); + RoutingTable table = createDummyRoutingTable(); + + createDefaultMock(visitorParameters, table); + assertEquals("mars", visitorParameters.getRoute().toString()); + } + + @Test + public void testRoutingTableHasMultipleStorageClusters() { + VisitorParameters visitorParameters = createVisitorParameters(""); + visitorParameters.setRoute(new Route()); + RoutingTableSpec spec = new RoutingTableSpec(DocumentProtocol.NAME); + spec.addRoute(new RouteSpec("storage/cluster.foo")); + spec.addRoute(new RouteSpec("storage/cluster.bar")); + RoutingTable table = new RoutingTable(spec); + + try { + createDefaultMock(visitorParameters, table); + fail("No exception thrown on multiple storage clusters"); + } catch (IllegalArgumentException e) { + assertEquals("There are multiple storage clusters in your application, " + + "please specify which one to visit.", + e.getMessage()); + } + } + + @Test + public void testRoutingTableHasNoStorageClusters() { + VisitorParameters visitorParameters = createVisitorParameters(""); + visitorParameters.setRoute(new Route()); + RoutingTableSpec spec = new RoutingTableSpec(DocumentProtocol.NAME); + spec.addRoute(new RouteSpec("storage/lobster.foo")); + RoutingTable table = new RoutingTable(spec); + + try { + createDefaultMock(visitorParameters, table); + fail("No exception thrown on zero storage clusters"); + } catch (IllegalArgumentException e) { + assertEquals("No storage cluster found in your application.", + e.getMessage()); + } + } + + @Test + public void testExecutionErrorDuringReplyHandling() { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.visitorSession.start(); + mc.controlHandler.resetMock(); + mc.executor.expectAndProcessTasks(1); + + // Slightly dirty; since there aren't really many paths during + // reply handling where we can reliably force an exception to + // happen, send a bogus visitor reply with a null result bucket which + // will trigger NPE when the progress token tries to access it. + replyToCreateVisitor(mc.sender, new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + reply.setLastBucket(null); + } + }); + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectNoTasks(); + // Session shall now have failed (and completed) + assertEquals(0, mc.sender.getMessageCount()); + assertTrue(mc.visitorSession.isDone()); + + assertEquals("onDone : FAILURE - 'Got exception of type java.lang.NullPointerException " + + "with message 'null' while processing reply in visitor session'\n", + mc.controlHandler.toString()); + } + + /** + * Test branch where we don't know how to handle a certain reply type. + * This should never happen (since we only get replies for messages we've + * already sent) but deal with it anyway! + */ + @Test + public void testFailureOnUnknownReplyType() { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.visitorSession.start(); + mc.controlHandler.resetMock(); + mc.executor.expectAndProcessTasks(1); + + mc.sender.getAndRemoveMessage(0); + // Make a bogus reply that we never asked for + RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:foo:bar")); + DocumentReply reply = msg.createReply(); + mc.sender.reply(reply); + + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectNoTasks(); + assertEquals(0, mc.sender.getMessageCount()); + assertTrue(mc.visitorSession.isDone()); + + assertEquals("onDone : FAILURE - 'Received reply we do not know how to " + + "handle: com.yahoo.documentapi.messagebus.protocol.RemoveDocumentReply'\n", + mc.controlHandler.toString()); + } + + @Test + public void testExecutionErrorInSendCreateVisitorsTask() { + MockComponents mc = createDefaultMock(); + mc.sender.setExceptionOnSend(new IllegalArgumentException("closed, come back tomorrow")); + mc.visitorSession.start(); + mc.controlHandler.resetMock(); // clear messages + mc.executor.expectAndProcessTasks(1); + assertEquals(0, mc.sender.getMessageCount()); + + assertTrue(mc.controlHandler.isDone()); + assertEquals("onDone : FAILURE - 'Got exception of type java.lang.IllegalArgumentException " + + "with message 'closed, come back tomorrow' while attempting to send visitors'\n", + mc.controlHandler.toString()); + } + + @Test + public void testExceptionInHandleVisitorInfoMessage() { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.visitorSession.start(); + mc.controlHandler.resetMock(); + mc.controlHandler.setExceptionOnProgress(new IllegalArgumentException("failed bigtime")); + mc.executor.expectAndProcessTasks(1); + + mc.receiver.send(new VisitorInfoMessage()); + mc.executor.expectAndProcessTasks(1); // Message handler task + + // Reply with OK; session should still have failed due to the processing error + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET); + mc.executor.expectAndProcessTasks(1); + mc.executor.expectNoTasks(); + assertTrue(mc.controlHandler.isDone()); + + // NOTE: 1st onProgress is invoked from VisitorInfo task. + // No onVisitorStatistics since that happens after onProgress, which throws + assertEquals("onProgress : 1 active, 0 pending, 0 finished, 1 total\n" + + "onProgress : 0 active, 0 pending, 1 finished, 1 total\n" + + "onDone : FAILURE - 'Got exception of type java.lang.IllegalArgumentException " + + "with message 'failed bigtime' while processing VisitorInfoMessage'\n", + mc.controlHandler.toString()); + assertEquals("VisitorReply(APP_FATAL_ERROR: Got exception of type java.lang.IllegalArgumentException " + + "with message 'failed bigtime' while processing VisitorInfoMessage)\n", + mc.receiver.repliesToString()); + } + + @Test + public void testExceptionInHandleDocumentMessage() { + MockComponents mc = createDefaultMock("id.user=1234"); + mc.dataHandler.resetMock(); + mc.controlHandler.resetMock(); + mc.dataHandler.setExceptionOnMessage(new IllegalArgumentException("oh no")); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + + mc.receiver.send(new RemoveDocumentMessage(new DocumentId("doc:foo:bar"))); + mc.executor.expectAndProcessTasks(1); + assertEquals(1, mc.dataHandler.getMessages().size()); + + // Reply with OK; session should still have failed due to the processing error + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET); + mc.executor.expectAndProcessTasks(1); + mc.executor.expectNoTasks(); + assertTrue(mc.controlHandler.isDone()); + + assertEquals("RemoveDocumentReply(APP_FATAL_ERROR: Got exception of type java.lang.IllegalArgumentException " + + "with message 'oh no' while processing DocumentMessage)\n", + mc.receiver.repliesToString()); + + assertEquals("onProgress : 0 active, 0 pending, 1 finished, 1 total\n" + + "onVisitorStatistics : 0 buckets visited, 0 docs returned\n" + + "onDone : FAILURE - 'Got exception of type java.lang.IllegalArgumentException " + + "with message 'oh no' while processing DocumentMessage'\n", + mc.controlHandler.toString()); + } + + @Test + public void testSilentlyIgnoreBucketDeletedNotFoundErrors() { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.controlHandler.resetMock(); + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + replyErrorToCreateVisitor(mc.sender, new Error( + DocumentProtocol.ERROR_BUCKET_NOT_FOUND, + "dave's not here, maaan")); + mc.executor.expectAndProcessTasks(1); // reply + // Should just resend with a 100ms delay + mc.executor.expectAndProcessTasks(1, new long[] { 100 }); + + // Now hit it with a BUCKET_DELETED error, which is also silent + replyErrorToCreateVisitor(mc.sender, new Error( + DocumentProtocol.ERROR_BUCKET_DELETED, + "dave's not here either, maaan!")); + mc.executor.expectAndProcessTasks(1); // reply + // Should also resend with a 100ms delay + mc.executor.expectAndProcessTasks(1, new long[] { 100 }); + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n" + + "selection='id.user==1234'\n)", + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET)); + mc.executor.expectAndProcessTasks(1); + + assertTrue(mc.controlHandler.isDone()); + assertEquals("onProgress : 0 active, 0 pending, 1 finished, 1 total\n" + + "onVisitorStatistics : 0 buckets visited, 0 docs returned\n" + + "onDone : SUCCESS - ''\n", + mc.controlHandler.toString()); + } + + private String dumpProgressToken(ProgressToken token) { + StringBuilder builder = new StringBuilder(); + builder.append("#total: ").append(token.getTotalBucketCount()).append('\n'); + builder.append("#finished: ").append(token.getFinishedBucketCount()).append('\n'); + if (token.containsFailedBuckets()) { + builder.append("failed:\n"); + Map<BucketId, BucketId> failed = token.getFailedBuckets(); + for (Map.Entry<BucketId, BucketId> kv : failed.entrySet()) { + builder.append(kv.getKey()).append(" : ").append(kv.getValue()).append('\n'); + } + } + return builder.toString(); + } + + @Test + public void testSkipBucketOnFatalErrorReply() { + VisitorParameters visitorParameters = createVisitorParameters(""); + visitorParameters.skipBucketsOnFatalErrors(true); + MockComponents mc = createDefaultMock(visitorParameters); + mc.controlHandler.resetMock(); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); // create visitors + assertEquals(2, mc.sender.getMessageCount()); + + replyErrorToCreateVisitor(mc.sender, new Error( + DocumentProtocol.ERROR_INTERNAL_FAILURE, + "borked")); + mc.executor.expectAndProcessTasks(1); + mc.executor.expectNoTasks(); // no more buckets to send for--all either failed or active + assertEquals(1, mc.sender.getMessageCount()); + assertFalse(mc.controlHandler.isDone()); + + // partial bucket progress which must be remembered + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x0400000000000001)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n)", + replyToCreateVisitor(mc.sender, new BucketId(33, 1L | (1L << 32)))); + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectAndProcessTasks(1); // create visitors + assertEquals(1, mc.sender.getMessageCount()); + assertFalse(mc.controlHandler.isDone()); + + // then fail bucket #2 + replyErrorToCreateVisitor(mc.sender, new Error( + DocumentProtocol.ERROR_INTERNAL_FAILURE, + "more borked")); + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectNoTasks(); + assertEquals(0, mc.sender.getMessageCount()); + + assertTrue(mc.controlHandler.isDone()); + + // make sure progress token was updated with bad buckets and + // remembers the initial error message + assertNotNull(mc.controlHandler.getProgress()); + assertTrue(mc.controlHandler.getProgress().containsFailedBuckets()); + assertEquals("INTERNAL_FAILURE: borked", + mc.controlHandler.getProgress().getFirstErrorMsg()); + + assertEquals("#total: 2\n" + + "#finished: 2\n" + + "failed:\n" + + "BucketId(0x0400000000000000) : BucketId(0x0000000000000000)\n" + + "BucketId(0x0400000000000001) : BucketId(0x8400000100000001)\n", + dumpProgressToken(mc.controlHandler.getProgress())); + + assertEquals( + "onVisitorError : INTERNAL_FAILURE: borked\n" + + "onProgress : 0 active, 1 pending, 1 finished, 2 total\n" + + "onVisitorStatistics : 0 buckets visited, 0 docs returned\n" + + "onVisitorError : INTERNAL_FAILURE: more borked\n" + + "onDone : FAILURE - 'INTERNAL_FAILURE: borked'\n", + mc.controlHandler.toString()); + } + + @Test + public void testSkipBucketOnFatalMessageProcessingError() { + VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); + visitorParameters.skipBucketsOnFatalErrors(true); + MockComponents mc = createDefaultMock(visitorParameters); + mc.controlHandler.resetMock(); + mc.dataHandler.resetMock(); + mc.dataHandler.setExceptionOnMessage(new IllegalArgumentException("oh no")); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + + mc.receiver.send(new RemoveDocumentMessage(new DocumentId("doc:foo:bar"))); + mc.executor.expectAndProcessTasks(1); + assertEquals(1, mc.dataHandler.getMessages().size()); + + // NOTE: current behavior does _not_ fail the session at the end of + // visiting if the CreateVisitor replies do not also return with failure + // since this is tied to the ProgressToken and its failed buckets list. + // We make the simplifying assumption that failing a visitor _message_ + // will subsequently cause its reply to fail back to us, allowing us to + // handle this as a regular skippable bucket. + // TODO: reconsider this? + + replyErrorToCreateVisitor(mc.sender, new Error( + DocumentProtocol.ERROR_INTERNAL_FAILURE, + "The Borkening")); + mc.executor.expectAndProcessTasks(1); + mc.executor.expectNoTasks(); + assertTrue(mc.controlHandler.isDone()); + + // Get UNPARSEABLE rather than APP_FATAL_ERROR if skip buckets is set + assertEquals("RemoveDocumentReply(UNPARSEABLE: Got exception of type java.lang.IllegalArgumentException " + + "with message 'oh no' while processing DocumentMessage)\n", + mc.receiver.repliesToString()); + + assertEquals("onVisitorError : INTERNAL_FAILURE: The Borkening\n" + + "onDone : FAILURE - 'INTERNAL_FAILURE: The Borkening'\n", + mc.controlHandler.toString()); + assertEquals("FAILURE: INTERNAL_FAILURE: The Borkening", + mc.controlHandler.getResult().toString()); + } + + /** + * Test assembly of message traces in session. Trace level propagation + * is already tested elsewhere. + */ + @Test + public void testMessageTracing() { + VisitorParameters visitorParameters = createVisitorParameters(""); + visitorParameters.setTraceLevel(7); + MockComponents mc = createDefaultMock(visitorParameters); + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + + final TraceNode traceNodes[] = { + new TraceNode().addChild("hello"), + new TraceNode().addChild("world") + }; + + for (int i = 0; i < 2; ++i) { + final int idx = i; + replyToCreateVisitor(mc.sender, new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + reply.getTrace().getRoot().addChild(traceNodes[idx]); + } + }); + } + mc.executor.expectAndProcessTasks(2); + mc.executor.expectNoTasks(); + assertTrue(mc.controlHandler.isDone()); + + Trace trace = mc.visitorSession.getTrace(); + assertNotNull(trace); + assertEquals(7, trace.getLevel()); + assertEquals( + "<trace>\n" + + " <trace>\n" + + " <trace>\n" + + " hello\n" + + " </trace>\n" + + " </trace>\n" + + " <trace>\n" + + " <trace>\n" + + " world\n" + + " </trace>\n" + + " </trace>\n" + + "</trace>\n", + trace.toString()); + } + + @Test + public void testResumeVisitingProgress() { + MockComponents mc = createDefaultMock("id.user==1234"); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n" + + "selection='id.user==1234'\n)", + replyToCreateVisitor(mc.sender, new BucketId(33, 1234 | (1L << 32)))); + + // Abort session to stop sending visitors. Progress should still + // be recorded. + mc.visitorSession.abort(); + mc.executor.expectAndProcessTasks(1); + mc.executor.expectNoTasks(); + assertTrue(mc.controlHandler.isDone()); + + VisitorParameters params = createVisitorParameters("id.user==1234"); + params.setResumeToken(mc.controlHandler.getProgress()); + mc = createDefaultMock(params); + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x84000001000004d2)\n" + + "]\n" + + "selection='id.user==1234'\n)", + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET)); + mc.executor.expectAndProcessTasks(1); + mc.executor.expectNoTasks(); + assertTrue(mc.controlHandler.isDone()); + } + + @Test + public void testResumeVisitingAlreadyCompleted() { + ProgressToken token; + // First, get a finished token + { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET); + mc.executor.expectAndProcessTasks(1); + assertTrue(mc.controlHandler.isDone()); + token = mc.controlHandler.getProgress(); + } + assertTrue(token.isFinished()); + + VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); + visitorParameters.setResumeToken(token); + MockComponents mc = createDefaultMock(visitorParameters); + + mc.visitorSession.start(); + mc.executor.expectNoTasks(); + assertTrue(mc.controlHandler.isDone()); + } + + @Test + public void testLocalDataAndControlDestinations() { + MockComponentsBuilder builder = new MockComponentsBuilder(); + builder.receiver.setConnectionSpec("foo/bar"); + builder.params = createVisitorParameters("id.user==1234"); + MockComponents mc = builder.createMockComponents(); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + + CreateVisitorMessage msg = (CreateVisitorMessage)mc.sender.getAndRemoveMessage(0); + // Local connection spec will be used for both control and data destinations + assertEquals("foo/bar", msg.getControlDestination()); + assertEquals("foo/bar", msg.getDataDestination()); + } + + @Test + public void testRemoteDataDestination() { + MockComponentsBuilder builder = new MockComponentsBuilder(); + builder.receiver.setConnectionSpec("curiosity"); + builder.params = createVisitorParameters("id.user==1234"); + builder.params.setLocalDataHandler(null); + builder.params.setRemoteDataHandler("odyssey"); + MockComponents mc = builder.createMockComponents(); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + + CreateVisitorMessage msg = (CreateVisitorMessage)mc.sender.getAndRemoveMessage(0); + assertEquals("curiosity", msg.getControlDestination()); + assertEquals("odyssey", msg.getDataDestination()); + } + + @Test + public void testExceptionIfNoDataDestinationSet() { + MockComponentsBuilder builder = new MockComponentsBuilder(); + builder.receiver.setConnectionSpec(null); + builder.params = createVisitorParameters("id.user==1234"); + builder.params.setLocalDataHandler(null); + builder.params.setRemoteDataHandler(null); + try { + builder.createMockComponents(); + fail("No exception thrown on missing data destination"); + } catch (IllegalStateException e) { + assertEquals("No data destination specified", e.getMessage()); + } + } + + /** + * Test that failing to submit a new message handling task causes + * a reply to immediately generated and sent. This must happen or + * the other endpoint will never receive a reply (until the local + * node's process/message bus goes down). + */ + @Test + public void testImmediatelyReplyIfMessageTaskSubmitFails() { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + mc.executor.setRejectTasksAfter(0); + + mc.receiver.send(new VisitorInfoMessage()); + mc.executor.expectNoTasks(); + + assertEquals("VisitorReply(ABORTED: Visitor session has been aborted)\n", + mc.receiver.repliesToString()); + } + + /** + * We cannot reliably handle reply tasks failing to be submitted, since + * the reply task performs all our internal state handling logic. As such, + * we just immediately go into a failure destruction mode as soon as this + * happens, in which we do not wait for any active messages to be replied + * to. + */ + @Test + public void testImmediatelyDestroySessionIfReplyTaskSubmitFails() { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.visitorSession.start(); + mc.controlHandler.resetMock(); + mc.executor.expectAndProcessTasks(1); + mc.executor.setRejectTasksAfter(0); + + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET); + mc.executor.expectNoTasks(); + assertTrue(mc.controlHandler.isDone()); + assertEquals("onDone : FAILURE - 'Failed to submit reply task to executor service: rejectTasksAfter is 0; rejecting task'\n", + mc.controlHandler.toString()); + } + + @Test + public void testDynamicallyIncreaseMaxBucketsPerVisitorOption() { + VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); + visitorParameters.setDynamicallyIncreaseMaxBucketsPerVisitor(true); + visitorParameters.setMaxBucketsPerVisitor(2); + visitorParameters.setDynamicMaxBucketsIncreaseFactor(10); + visitorParameters.setMaxFirstPassHits(10); + MockComponents mc = createDefaultMock(visitorParameters); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n" + + "selection='id.user==1234'\n" + + "max buckets per visitor=2\n)", + replyToCreateVisitor(mc.sender, new BucketId(33, 1234 | (1L << 32)))); + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectAndProcessTasks(1); // send create visitors + + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x84000001000004d2)\n" + + "]\n" + + "selection='id.user==1234'\n" + + "max buckets per visitor=20\n)", + replyToCreateVisitor(mc.sender, new BucketId(34, 1234 | (1L << 33)))); + + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectAndProcessTasks(1); // send create visitors + + // Saturate at 128 + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x88000002000004d2)\n" + + "]\n" + + "selection='id.user==1234'\n" + + "max buckets per visitor=128\n)", + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET)); + } + + @Test + public void testVisitorTimeoutsNotConsideredFatal() { + VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); + MockComponents mc = createDefaultMock(visitorParameters); + mc.controlHandler.resetMock(); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); // create visitors + assertEquals(1, mc.sender.getMessageCount()); + + replyErrorToCreateVisitor(mc.sender, new Error(ErrorCode.TIMEOUT, "out of time!")); + mc.executor.expectAndProcessTasks(1); // reply + mc.executor.expectAndProcessTasks(1, new long[] { 100 }); // delayed create visitors + assertEquals("CreateVisitorMessage(buckets=[\n" + + "BucketId(0x80000000000004d2)\n" + + "BucketId(0x0000000000000000)\n" + + "]\n" + + "selection='id.user==1234'\n)", + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET)); + mc.executor.expectAndProcessTasks(1); // reply + } + + /** + * Test that there is no race condition between a reply is handed off + * to the executor service via a task (thus decrementing the pending count + * for the sender) and the session checking for completion early, e.g. + * because of an error transitioning it into a failure state. + */ + @Test + public void testNoRaceConditionForPendingReplyTasks() { + MockComponents mc = createDefaultMock(); + mc.visitorSession.start(); + mc.controlHandler.resetMock(); // clear messages + mc.executor.expectAndProcessTasks(1); + assertEquals(2, mc.sender.getMessageCount()); + replyToCreateVisitor(mc.sender, new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + reply.addError(new Error( + DocumentProtocol.ERROR_INTERNAL_FAILURE, + "node fell down a well")); + } + }); + replyToCreateVisitor(mc.sender, new ReplyModifier() { + @Override + public void modify(CreateVisitorReply reply) { + reply.addError(new Error( + DocumentProtocol.ERROR_INTERNAL_FAILURE, + "node got hit by a falling brick")); + } + }); + + // Now 2 pending reply tasks, but 0 pending messages. Ergo, using + // the sender as a ground truth to determine whether or not we have + // completed will cause a race condition. + mc.executor.expectAndProcessTasks(2); + mc.executor.expectNoTasks(); + assertEquals(0, mc.sender.getMessageCount()); // no resending + assertTrue(mc.visitorSession.isDone()); + + // should get first received failure message as completion failure message + assertEquals("onVisitorError : INTERNAL_FAILURE: node fell down a well\n" + + "onVisitorError : INTERNAL_FAILURE: node got hit by a falling brick\n" + + "onDone : FAILURE - 'INTERNAL_FAILURE: node fell down a well'\n", + mc.controlHandler.toString()); + } + + @Test + public void testReplyErrorIfInfoMessageArrivesAfterDone() { + MockComponents mc = createDefaultMock("id.user==1234"); + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + + replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET); + mc.executor.expectAndProcessTasks(1); + + mc.receiver.send(new VisitorInfoMessage()); + mc.executor.expectAndProcessTasks(1); + // Should not be passed on to data handler + assertEquals(0, mc.dataHandler.getMessages().size()); + + assertEquals("VisitorReply(APP_FATAL_ERROR: Visitor has been shut down)\n", + mc.receiver.repliesToString()); + } + + @Test + public void testReplyErrorIfLocalDataHandlerIsNull() { + MockComponentsBuilder builder = new MockComponentsBuilder(); + builder.params = createVisitorParameters("id.user==1234"); + builder.params.setLocalDataHandler(null); + builder.params.setRemoteDataHandler("odyssey"); + MockComponents mc = builder.createMockComponents(); + + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + + mc.receiver.send(new RemoveDocumentMessage(new DocumentId("doc:foo:bar"))); + mc.executor.expectAndProcessTasks(1); + + assertEquals("RemoveDocumentReply(APP_FATAL_ERROR: Visitor data with no local data destination)\n", + mc.receiver.repliesToString()); + } + + /** + * TODOs: + * - parameter validation (max pending, ...) + * - thread safety stress test + * - [add percent finished to progress file; ticket 5360824] + */ + + // TODO: what about session-wide timeouts? + // TODO: consider refactoring locking granularity + // TODO: figure out if we risk a re-run of the "too many tasks" issue +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/ScheduledEventQueueTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/ScheduledEventQueueTestCase.java new file mode 100755 index 00000000000..f315f386f60 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/ScheduledEventQueueTestCase.java @@ -0,0 +1,221 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.test; + +import com.yahoo.documentapi.messagebus.ScheduledEventQueue; +import com.yahoo.concurrent.Timer; +import junit.framework.TestCase; +import org.junit.Test; + +import java.util.concurrent.RejectedExecutionException; + +public class ScheduledEventQueueTestCase extends TestCase { + + class TestTask implements Runnable { + public long timestamp = 0; + + public void run() { + + } + } + + @Test + public void testPushTask() { + ScheduledEventQueue queue = new ScheduledEventQueue(); + TestTask task = new TestTask(); + queue.pushTask(task); + assertEquals(task, queue.popTask()); + } + + @Test + public void testPushTwoTasks() { + ScheduledEventQueue queue = new ScheduledEventQueue(); + TestTask task1 = new TestTask(); + TestTask task2 = new TestTask(); + queue.pushTask(task1); + queue.pushTask(task2); + assertEquals(task1, queue.popTask()); + assertEquals(task2, queue.popTask()); + } + + @Test + public void testNullWhenPoppingNonexistantTask() { + ScheduledEventQueue queue = new ScheduledEventQueue(); + assertNull(queue.popTask()); + } + + class TestTimer implements Timer { + public long milliTime = 0; + + public long milliTime() { + return milliTime; + } + } + + @Test + public void testPushTaskWithTime() { + TestTimer timer = new TestTimer(); + ScheduledEventQueue queue = new ScheduledEventQueue(timer); + TestTask task = new TestTask(); + queue.pushTask(task, 1000); + assertNull(queue.popTask()); + timer.milliTime = 1000; + assertEquals(task, queue.popTask()); + } + + @Test + public void testTwoTasksWithSameTime() { + TestTimer timer = new TestTimer(); + ScheduledEventQueue queue = new ScheduledEventQueue(timer); + TestTask task1 = new TestTask(); + queue.pushTask(task1, 1000); + TestTask task2 = new TestTask(); + queue.pushTask(task2, 1000); + assertNull(queue.popTask()); + timer.milliTime = 1000; + assertEquals(task1, queue.popTask()); + assertEquals(task2, queue.popTask()); + } + + @Test + public void testThreeTasksWithDifferentTime() { + TestTimer timer = new TestTimer(); + ScheduledEventQueue queue = new ScheduledEventQueue(timer); + TestTask task1 = new TestTask(); + queue.pushTask(task1, 1000); + TestTask task2 = new TestTask(); + queue.pushTask(task2, 500); + TestTask task3 = new TestTask(); + queue.pushTask(task3); + assertEquals(task3, queue.popTask()); + assertNull(queue.popTask()); + timer.milliTime = 1000; + assertEquals(task2, queue.popTask()); + assertEquals(task1, queue.popTask()); + } + + class ClockSetterThread implements Runnable { + ScheduledEventQueue queue; + TestTimer timer; + long newTime; + + public ClockSetterThread(ScheduledEventQueue queue, TestTimer timer, long newTime) { + this.queue = queue; + this.timer = timer; + this.newTime = newTime; + } + + public void run() { + try { + while (!queue.isWaiting()) { + Thread.sleep(1); + } + } catch (InterruptedException e) { + } + timer.milliTime = newTime; + queue.wakeTasks(); + } + } + + @Test + public void testPushAndWaitForTask() { + TestTimer timer = new TestTimer(); + ScheduledEventQueue queue = new ScheduledEventQueue(timer); + TestTask task = new TestTask(); + queue.pushTask(task, 50); + assertNull(queue.popTask()); + new Thread(new ClockSetterThread(queue, timer, 50)).start(); + assertEquals(task, queue.getNextTask()); + assertEquals(50, timer.milliTime()); + } + + class TaskPusherThread implements Runnable { + ScheduledEventQueue queue; + TestTask task; + + public TaskPusherThread(ScheduledEventQueue queue, TestTask task) { + this.queue = queue; + this.task = task; + } + + public void run() { + try { + while (!queue.isWaiting()) { + Thread.sleep(1); + } + } catch (InterruptedException e) { + } + queue.pushTask(task); + } + } + + @Test + public void testPushAndWaitSingle() { + ScheduledEventQueue queue = new ScheduledEventQueue(); + TestTask task = new TestTask(); + new Thread(new TaskPusherThread(queue, task)).start(); + assertNull(queue.popTask()); + assertEquals(task, queue.getNextTask()); + } + + @Test + public void testPushAndWaitMultiple() { + TestTimer timer = new TestTimer(); + ScheduledEventQueue queue = new ScheduledEventQueue(timer); + TestTask lastTask = new TestTask(); + queue.pushTask(lastTask, 250); + TestTask task = new TestTask(); + new Thread(new TaskPusherThread(queue, task)).start(); + assertNull(queue.popTask()); + assertEquals(task, queue.getNextTask()); + new Thread(new ClockSetterThread(queue, timer, 250)).start(); + assertEquals(lastTask, queue.getNextTask()); + assertEquals(250, timer.milliTime()); + } + + @Test + public void testPushTaskRejectedAfterShutdown() { + ScheduledEventQueue queue = new ScheduledEventQueue(); + TestTask task = new TestTask(); + queue.shutdown(); + assertTrue(queue.isShutdown()); + try { + queue.pushTask(task); + fail(); + } catch (RejectedExecutionException e) { + } + } + + class ShutdownThread implements Runnable { + ScheduledEventQueue queue; + TestTimer timer; + + public ShutdownThread(ScheduledEventQueue queue, TestTimer timer) { + this.queue = queue; + this.timer = timer; + } + + public void run() { + try { + while (!queue.isWaiting()) { + Thread.sleep(1); + } + } catch (InterruptedException e) { + } + queue.shutdown(); + timer.milliTime = 100; + queue.wakeTasks(); + } + } + + @Test + public void testShutdownInGetNext() { + TestTimer timer = new TestTimer(); + ScheduledEventQueue queue = new ScheduledEventQueue(timer); + TestTask task = new TestTask(); + queue.pushTask(task, 100); + new Thread(new ShutdownThread(queue, timer)).start(); + assertEquals(task, queue.getNextTask()); + assertEquals(100, timer.milliTime()); + } + +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/ThroughputLimitQueueTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/ThroughputLimitQueueTestCase.java new file mode 100644 index 00000000000..73749b4813c --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/ThroughputLimitQueueTestCase.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.test; + +import com.yahoo.documentapi.ThroughputLimitQueue; +import com.yahoo.concurrent.Timer; +import junit.framework.TestCase; + +/** + * @author thomasg + */ +public class ThroughputLimitQueueTestCase extends TestCase { + class TestTimer implements Timer { + public long milliTime = 0; + + public long milliTime() { + return milliTime; + } + } + + + public void setUp() { + + } + + public void tearDown() { + + } + + public void testCapacity() { + TestTimer t = new TestTimer(); + t.milliTime = 10; + ThroughputLimitQueue<Object> q = new ThroughputLimitQueue<Object>(t, 2000); + + q.add(new Object()); + q.add(new Object()); + q.remove(); + t.milliTime += 10; + q.remove(); + + assertEquals(200, q.capacity()); + + for (int i = 0; i < 1000; i++) { + q.add(new Object()); + q.add(new Object()); + t.milliTime += 100; + q.remove(); + t.milliTime += 100; + q.remove(); + } + + assertEquals(20, q.capacity()); + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java new file mode 100644 index 00000000000..a7d2d9e3c66 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java @@ -0,0 +1,148 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.test; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentType; +import com.yahoo.documentapi.*; +import org.junit.Test; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.*; + +/** + * These tests should work with all implementations (who choose to implement these features) To test a certain + * implementation subclass this test and make the subclass setup assign the implementation class to the + * <code>access</code> member variable (make sure it also calls super.setUp()). Override tests of nonsupported features + * to do nothing. + * + * @author bratseth + */ +@SuppressWarnings("deprecation") +public abstract class AbstractDocumentApiTestCase { + + protected abstract DocumentAccess access(); + + @Test + public void requireThatSyncSessionWorks() { + SyncSession session = access().createSyncSession(new SyncParameters()); + + DocumentType type = access().getDocumentTypeManager().getDocumentType("music"); + Document doc1 = new Document(type, new DocumentId("doc:music:1")); + Document doc2 = new Document(type, new DocumentId("doc:music:2")); + + session.put(new DocumentPut(doc1)); + session.put(new DocumentPut(doc2)); + assertEquals(doc1, session.get(new DocumentId("doc:music:1"))); + assertEquals(doc2, session.get(new DocumentId("doc:music:2"))); + + session.remove(new DocumentRemove(new DocumentId("doc:music:1"))); + assertNull(session.get(new DocumentId("doc:music:1"))); + assertEquals(doc2, session.get(new DocumentId("doc:music:2"))); + + session.remove(new DocumentRemove(new DocumentId("doc:music:2"))); + assertNull(session.get(new DocumentId("doc:music:1"))); + assertNull(session.get(new DocumentId("doc:music:2"))); + + session.destroy(); + } + + @Test + public void requireThatAsyncSessionWorks() throws InterruptedException { + AsyncSession session = access().createAsyncSession(new AsyncParameters()); + HashMap<Long, Response> results = new LinkedHashMap<>(); + Result result; + DocumentType type = access().getDocumentTypeManager().getDocumentType("music"); + Document doc1 = new Document(type, new DocumentId("doc:music:1")); + Document doc2 = new Document(type, new DocumentId("doc:music:2")); + + result = session.put(doc1); + assertTrue(result.isSuccess()); + results.put(result.getRequestId(), new Response(result.getRequestId())); + result = session.put(doc2); + assertTrue(result.isSuccess()); + results.put(result.getRequestId(), new Response(result.getRequestId())); + + result = session.get(new DocumentId("doc:music:1")); + assertTrue(result.isSuccess()); + results.put(result.getRequestId(), new DocumentResponse(result.getRequestId(), doc1)); + result = session.get(new DocumentId("doc:music:2")); + assertTrue(result.isSuccess()); + results.put(result.getRequestId(), new DocumentResponse(result.getRequestId(), doc2)); + + result = session.remove(new DocumentId("doc:music:1")); + assertTrue(result.isSuccess()); + results.put(result.getRequestId(), new Response(result.getRequestId())); + result = session.get(new DocumentId("doc:music:1")); + assertTrue(result.isSuccess()); + results.put(result.getRequestId(), new DocumentResponse(result.getRequestId())); + result = session.get(new DocumentId("doc:music:2")); + assertTrue(result.isSuccess()); + results.put(result.getRequestId(), new DocumentResponse(result.getRequestId(), doc2)); + + result = session.remove(new DocumentId("doc:music:2")); + assertTrue(result.isSuccess()); + results.put(result.getRequestId(), new Response(result.getRequestId())); + result = session.get(new DocumentId("doc:music:1")); + assertTrue(result.isSuccess()); + results.put(result.getRequestId(), new DocumentResponse(result.getRequestId())); + result = session.get(new DocumentId("doc:music:2")); + assertTrue(result.isSuccess()); + results.put(result.getRequestId(), new DocumentResponse(result.getRequestId())); + + for (int i = 0; i < 4; i++) { + Response response; + if (i % 2 == 0) { + response = pollNext(session); + } else { + response = session.getNext(10000); + } + assertTrue(response.isSuccess()); + assertEquals(results.get(response.getRequestId()), response); + } + + session.destroy(); + } + + @Test + public void requireThatAsyncHandlerWorks() throws InterruptedException { + MyHandler handler = new MyHandler(); + AsyncSession session = access().createAsyncSession(new AsyncParameters().setResponseHandler(handler)); + DocumentType type = access().getDocumentTypeManager().getDocumentType("music"); + Document doc1 = new Document(type, new DocumentId("doc:music:1")); + assertTrue(session.put(doc1).isSuccess()); + assertTrue(handler.latch.await(60, TimeUnit.SECONDS)); + assertNotNull(handler.response); + session.destroy(); + } + + private static Response pollNext(AsyncSession session) throws InterruptedException { + for (int i = 0; i < 600; ++i) { + Response response = session.getNext(); + if (response != null) { + return response; + } + Thread.sleep(100); + } + return null; + } + + private static class MyHandler implements ResponseHandler { + + final CountDownLatch latch = new CountDownLatch(1); + Response response = null; + + @Override + public void handleResponse(Response response) { + this.response = response; + latch.countDown(); + } + } +} diff --git a/documentapi/src/testlist.txt b/documentapi/src/testlist.txt new file mode 100644 index 00000000000..851741bd24a --- /dev/null +++ b/documentapi/src/testlist.txt @@ -0,0 +1,9 @@ +tests/messagebus +tests/messages +tests/policies +tests/policyfactory +tests/priority +tests/routablefactory +tests/systemstate +tests/loadtypes +tests/replymerger diff --git a/documentapi/src/testrun/.gitignore b/documentapi/src/testrun/.gitignore new file mode 100644 index 00000000000..b29b0c6486c --- /dev/null +++ b/documentapi/src/testrun/.gitignore @@ -0,0 +1,9 @@ +test-report.html +test-report.html.* +test.*.*.desc +test.*.*.file.* +test.*.*.files.html +test.*.*.log +tmp.* +xsync.log +/test.*.*.result diff --git a/documentapi/src/tests/.gitignore b/documentapi/src/tests/.gitignore new file mode 100644 index 00000000000..a3e9c375723 --- /dev/null +++ b/documentapi/src/tests/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +*_test diff --git a/documentapi/src/tests/create-test.sh b/documentapi/src/tests/create-test.sh new file mode 100755 index 00000000000..5debc5f635a --- /dev/null +++ b/documentapi/src/tests/create-test.sh @@ -0,0 +1,75 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +gen_ignore_file() { + echo "generating '$1' ..." + echo ".depend" > $1 + echo "Makefile" >> $1 + echo "${test}_test" >> $1 +} + +gen_project_file() { + echo "generating '$1' ..." + echo "APPLICATION ${test}_test" > $1 + echo "OBJS $test" >> $1 + echo "LIBS documentapi/documentapi" >> $1 + echo "EXTERNALLIBS vespalib vespalog document config messagebus-test" >> $1 + echo "EXTERNALLIBS messagebus config slobrokserver vespalib" >> $1 + echo "" >> $1 + echo "CUSTOMMAKE" >> $1 + echo "test: depend ${test}_test" >> $1 + echo -e "\t@./${test}_test" >> $1 +} + +gen_source() { + echo "generating '$1' ..." + echo "#include <vespa/log/log.h>" > $1 + echo "LOG_SETUP(\"${test}_test\");" >> $1 + echo "" >> $1 + echo "#include <vespa/fastos/fastos.h>" >> $1 + echo "#include <vespa/vespalib/testkit/testapp.h>" >> $1 + echo "" >> $1 + echo "//using namespace documentapi;" >> $1 + echo "" >> $1 + echo "TEST_SETUP(Test);" >> $1 + echo "" >> $1 + echo "int" >> $1 + echo "Test::Main()" >> $1 + echo "{" >> $1 + echo " TEST_INIT(\"${test}_test\");" >> $1 + echo " TEST_DONE();" >> $1 + echo "}" >> $1 +} + +gen_desc() { + echo "generating '$1' ..." + echo "$test test. Take a look at $test.cpp for details." > $1 +} + +gen_file_list() { + echo "generating '$1' ..." + echo "$test.cpp" > $1 +} + +if [ $# -ne 1 ]; then + echo "usage: $0 <name>" + echo " name: name of the test to create" + exit 1 +fi + +test=$1 +if [ -e $test ]; then + echo "$test already present, don't want to mess it up..." + exit 1 +fi + +echo "creating directory '$test' ..." +mkdir -p $test || exit 1 +cd $test || exit 1 +test=`basename $test` + +gen_ignore_file .cvsignore +gen_project_file fastos.project +gen_source $test.cpp +gen_desc DESC +gen_file_list FILES diff --git a/documentapi/src/tests/loadtypes/.gitignore b/documentapi/src/tests/loadtypes/.gitignore new file mode 100644 index 00000000000..497fe4d4b3f --- /dev/null +++ b/documentapi/src/tests/loadtypes/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +documentapi_loadtype_test_app diff --git a/documentapi/src/tests/loadtypes/CMakeLists.txt b/documentapi/src/tests/loadtypes/CMakeLists.txt new file mode 100644 index 00000000000..7c1e92b087f --- /dev/null +++ b/documentapi/src/tests/loadtypes/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(documentapi_loadtype_test_app + SOURCES + loadtypetest.cpp + testrunner.cpp + DEPENDS + documentapi + vdstestlib +) +vespa_add_test(NAME documentapi_loadtype_test_app COMMAND documentapi_loadtype_test_app) diff --git a/documentapi/src/tests/loadtypes/loadtypetest.cpp b/documentapi/src/tests/loadtypes/loadtypetest.cpp new file mode 100644 index 00000000000..42b58b1a9c9 --- /dev/null +++ b/documentapi/src/tests/loadtypes/loadtypetest.cpp @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> + +namespace documentapi { + +struct LoadTypeTest : public CppUnit::TestFixture { + + void testConfig(); + + CPPUNIT_TEST_SUITE(LoadTypeTest); + CPPUNIT_TEST(testConfig); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(LoadTypeTest); + +#define ASSERT_CONFIG_FAILURE(configId, error) \ + try { \ + LoadTypeSet createdFromConfigId(configId); \ + CPPUNIT_FAIL("Config was expected to fail with error: " \ + + string(error)); \ + } catch (config::InvalidConfigException& e) { \ + CPPUNIT_ASSERT_CONTAIN(string(error), e.getMessage()); \ + } + +void +LoadTypeTest::testConfig() +{ + // Using id 0 is illegal. Reserved for default type. + ASSERT_CONFIG_FAILURE( + "raw:" + "type[1]\n" + "type[0].id 0\n" + "type[0].name \"foo\"\n" + "type[0].priority \"\"", + "Load type identifiers need to be"); + // Using name "default" is illegal. Reserved for default type. + ASSERT_CONFIG_FAILURE( + "raw:" + "type[1]\n" + "type[0].id 1\n" + "type[0].name \"default\"\n" + "type[0].priority \"\"", "Load type names need to be"); + // Identifiers need to be unique. + ASSERT_CONFIG_FAILURE( + "raw:" + "type[2]\n" + "type[0].id 1\n" + "type[0].name \"test\"\n" + "type[0].priority \"\"\n" + "type[1].id 1\n" + "type[1].name \"testa\"\n" + "type[1].priority \"\"", "Load type identifiers need to be"); + // Names need to be unique. + ASSERT_CONFIG_FAILURE( + "raw:" + "type[2]\n" + "type[0].id 1\n" + "type[0].name \"test\"\n" + "type[0].priority \"\"\n" + "type[1].id 2\n" + "type[1].name \"test\"\n" + "type[1].priority \"\"" , "Load type names need to be"); + LoadTypeSet set("raw:" + "type[3]\n" + "type[0].id 1\n" + "type[0].name \"user\"\n" + "type[0].priority \"\"\n" + "type[1].id 2\n" + "type[1].name \"maintenance\"\n" + "type[1].priority \"\"\n" + "type[2].id 3\n" + "type[2].name \"put\"\n" + "type[2].priority \"\"" + ); +} + +} // documentapi diff --git a/documentapi/src/tests/loadtypes/testrunner.cpp b/documentapi/src/tests/loadtypes/testrunner.cpp new file mode 100644 index 00000000000..71200f84224 --- /dev/null +++ b/documentapi/src/tests/loadtypes/testrunner.cpp @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/vdstestlib/cppunit/cppunittestrunner.h> + +LOG_SETUP("storagecppunittests"); + +int +main(int argc, char **argv) +{ + vdstestlib::CppUnitTestRunner testRunner; + return testRunner.run(argc, argv); +} diff --git a/documentapi/src/tests/messagebus/.gitignore b/documentapi/src/tests/messagebus/.gitignore new file mode 100644 index 00000000000..e409c623d1b --- /dev/null +++ b/documentapi/src/tests/messagebus/.gitignore @@ -0,0 +1,5 @@ +*_test +.depend +Makefile +log +documentapi_messagebus_test_app diff --git a/documentapi/src/tests/messagebus/CMakeLists.txt b/documentapi/src/tests/messagebus/CMakeLists.txt new file mode 100644 index 00000000000..0f0760c1b57 --- /dev/null +++ b/documentapi/src/tests/messagebus/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(documentapi_messagebus_test_app + SOURCES + messagebus_test.cpp + DEPENDS + documentapi +) +vespa_add_test(NAME documentapi_messagebus_test_app COMMAND documentapi_messagebus_test_app) diff --git a/documentapi/src/tests/messagebus/documentrouteselectorpolicy.cfg b/documentapi/src/tests/messagebus/documentrouteselectorpolicy.cfg new file mode 100644 index 00000000000..9f56aa29020 --- /dev/null +++ b/documentapi/src/tests/messagebus/documentrouteselectorpolicy.cfg @@ -0,0 +1,4 @@ +route[1] +route[0].name foo +route[0].selector testdoc +route[0].feed bigmac diff --git a/documentapi/src/tests/messagebus/messagebus_test.cpp b/documentapi/src/tests/messagebus/messagebus_test.cpp new file mode 100644 index 00000000000..b55d35825fe --- /dev/null +++ b/documentapi/src/tests/messagebus/messagebus_test.cpp @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/document/config/config-documenttypes.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/log/log.h> +#include <vespa/messagebus/message.h> +#include <vespa/messagebus/routable.h> +#include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vespalib/testkit/testapp.h> + +LOG_SETUP("messages_test"); + +using document::DocumentTypeRepo; +using document::readDocumenttypesConfig; +using namespace documentapi; +using mbus::Blob; +using mbus::Routable; +using mbus::IRoutingPolicy; + +class Test : public vespalib::TestApp { + DocumentTypeRepo::SP _repo; + +public: + int Main(); + +private: + void testMessage(); + void testProtocol(); +}; + +TEST_APPHOOK(Test); + +int +Test::Main() +{ + TEST_INIT(_argv[0]); + _repo.reset(new DocumentTypeRepo(readDocumenttypesConfig("../../../test/cfg/testdoctypes.cfg"))); + + testMessage(); TEST_FLUSH(); + testProtocol(); TEST_FLUSH(); + + TEST_DONE(); +} + +void Test::testMessage() { + const document::DataType *testdoc_type = _repo->getDocumentType("testdoc"); + + // Test one update. + UpdateDocumentMessage upd1( + document::DocumentUpdate::SP( + new document::DocumentUpdate(*testdoc_type, + document::DocumentId(document::DocIdString( + "testdoc", "testme1"))))); + + EXPECT_TRUE(upd1.getType() == DocumentProtocol::MESSAGE_UPDATEDOCUMENT); + EXPECT_TRUE(upd1.getProtocol() == "document"); + + LoadTypeSet set; + DocumentProtocol protocol(set, _repo); + + Blob blob = protocol.encode(vespalib::Version(5,0), upd1); + EXPECT_TRUE(blob.size() > 0); + + Routable::UP dec1 = protocol.decode(vespalib::Version(5,0), blob); + EXPECT_TRUE(dec1.get() != NULL); + EXPECT_TRUE(dec1->isReply() == false); + EXPECT_TRUE(dec1->getType() == DocumentProtocol::MESSAGE_UPDATEDOCUMENT); + + // Compare to another. + UpdateDocumentMessage upd2( + document::DocumentUpdate::SP( + new document::DocumentUpdate(*testdoc_type, + document::DocumentId(document::DocIdString( + "testdoc", "testme2"))))); + EXPECT_TRUE(!(upd1.getDocumentUpdate()->getId() == upd2.getDocumentUpdate()->getId())); + + DocumentMessage& msg2 = static_cast<DocumentMessage&>(upd2); + EXPECT_TRUE(msg2.getType() == DocumentProtocol::MESSAGE_UPDATEDOCUMENT); +} + +void Test::testProtocol() { + LoadTypeSet set; + DocumentProtocol protocol(set, _repo); + EXPECT_TRUE(protocol.getName() == "document"); + + IRoutingPolicy::UP policy = protocol.createPolicy(string("SearchRow"),string("")); + EXPECT_TRUE(policy.get() != NULL); + + policy = protocol.createPolicy(string("SearchColumn"),string("")); + EXPECT_TRUE(policy.get() != NULL); + + policy = protocol.createPolicy(string("DocumentRouteSelector"), string("file:documentrouteselectorpolicy.cfg")); + EXPECT_TRUE(policy.get() != NULL); + + policy = protocol.createPolicy(string(""),string("")); + EXPECT_TRUE(policy.get() == NULL); + + policy = protocol.createPolicy(string("Balle"),string("")); + EXPECT_TRUE(policy.get() == NULL); +} + + diff --git a/documentapi/src/tests/messages/.gitignore b/documentapi/src/tests/messages/.gitignore new file mode 100644 index 00000000000..298c9cd21c6 --- /dev/null +++ b/documentapi/src/tests/messages/.gitignore @@ -0,0 +1,5 @@ +*_test +*_app +.depend +Makefile +log diff --git a/documentapi/src/tests/messages/CMakeLists.txt b/documentapi/src/tests/messages/CMakeLists.txt new file mode 100644 index 00000000000..bfffd0d0502 --- /dev/null +++ b/documentapi/src/tests/messages/CMakeLists.txt @@ -0,0 +1,38 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(documentapi_messages50_test_app + SOURCES + testbase.cpp + messages50test.cpp + messages50app.cpp + DEPENDS + documentapi +) +vespa_add_test(NAME documentapi_messages50_test_app COMMAND documentapi_messages50_test_app) +vespa_add_executable(documentapi_messages51_test_app + SOURCES + testbase.cpp + messages50test.cpp + messages51test.cpp + messages51app.cpp + DEPENDS + documentapi +) +vespa_add_test(NAME documentapi_messages51_test_app COMMAND documentapi_messages51_test_app) +vespa_add_executable(documentapi_messages52_test_app + SOURCES + testbase.cpp + messages50test.cpp + messages51test.cpp + messages52test.cpp + messages52app.cpp + DEPENDS + documentapi +) +vespa_add_test(NAME documentapi_messages52_test_app COMMAND documentapi_messages52_test_app) +vespa_add_executable(documentapi_error_codes_test_app_app + SOURCES + error_codes_test.cpp + DEPENDS + documentapi +) +vespa_add_test(NAME documentapi_error_codes_test_app_app COMMAND documentapi_error_codes_test_app_app) diff --git a/documentapi/src/tests/messages/error_codes_test.cpp b/documentapi/src/tests/messages/error_codes_test.cpp new file mode 100644 index 00000000000..1714fb70a04 --- /dev/null +++ b/documentapi/src/tests/messages/error_codes_test.cpp @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2015 Yahoo Technologies Norway AS +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <iostream> +#include <fstream> +#include <sstream> +#include <string> +#include <exception> +#include <map> + +using NamedErrorCodes = std::map<std::string, uint32_t>; + +// DocumentAPI C++ module uses Ye Olde Test Framework. +class ErrorCodesTest : public vespalib::TestApp { + int Main() override; + void error_codes_match_java_definitions(); + NamedErrorCodes all_document_protocol_error_codes(); +}; + +TEST_APPHOOK(ErrorCodesTest); + +// ERROR_CODE_KV(FOO) -> {"FOO", DocumentProtocol::FOO} +#define ERROR_CODE_KV(code_name) \ + {#code_name, DocumentProtocol::code_name} + +NamedErrorCodes +ErrorCodesTest::all_document_protocol_error_codes() +{ + using documentapi::DocumentProtocol; + return { + ERROR_CODE_KV(ERROR_MESSAGE_IGNORED), + ERROR_CODE_KV(ERROR_POLICY_FAILURE), + ERROR_CODE_KV(ERROR_DOCUMENT_NOT_FOUND), + // Error code not consistently named between languages! + // Java: ERROR_DOCUMENT_EXISTS, C++: ERROR_EXISTS + // Names must be consistent in test or checking will fail. + {"ERROR_DOCUMENT_EXISTS", DocumentProtocol::ERROR_EXISTS}, + ERROR_CODE_KV(ERROR_REJECTED), + ERROR_CODE_KV(ERROR_NOT_IMPLEMENTED), + ERROR_CODE_KV(ERROR_ILLEGAL_PARAMETERS), + ERROR_CODE_KV(ERROR_UNKNOWN_COMMAND), + ERROR_CODE_KV(ERROR_NO_SPACE), + ERROR_CODE_KV(ERROR_IGNORED), + ERROR_CODE_KV(ERROR_INTERNAL_FAILURE), + ERROR_CODE_KV(ERROR_TEST_AND_SET_CONDITION_FAILED), + ERROR_CODE_KV(ERROR_PROCESSING_FAILURE), + ERROR_CODE_KV(ERROR_TIMESTAMP_EXIST), + ERROR_CODE_KV(ERROR_NODE_NOT_READY), + ERROR_CODE_KV(ERROR_WRONG_DISTRIBUTION), + ERROR_CODE_KV(ERROR_ABORTED), + ERROR_CODE_KV(ERROR_BUSY), + ERROR_CODE_KV(ERROR_NOT_CONNECTED), + ERROR_CODE_KV(ERROR_DISK_FAILURE), + ERROR_CODE_KV(ERROR_IO_FAILURE), + ERROR_CODE_KV(ERROR_BUCKET_NOT_FOUND), + ERROR_CODE_KV(ERROR_BUCKET_DELETED), + ERROR_CODE_KV(ERROR_STALE_TIMESTAMP), + ERROR_CODE_KV(ERROR_SUSPENDED) + }; +} + +#undef ERROR_CODE_KV + +namespace { + +std::string read_file(const std::string& file_name) { + std::ifstream ifs(file_name); + if (!ifs.is_open()) { + throw std::runtime_error("file '" + file_name + "' does not exist"); + } + std::ostringstream oss; + oss << ifs.rdbuf(); + return oss.str(); +} + +void write_file(const std::string& file_name, + const std::string& content) +{ + std::ofstream ofs(file_name, std::ios_base::trunc); + ofs << content; +} + +std::string to_sorted_key_value_string(const NamedErrorCodes& codes) { + std::ostringstream os; + bool emit_newline = false; + for (auto& kv : codes) { + if (emit_newline) { + os << '\n'; + } + os << kv.first << ' ' << kv.second; + emit_newline = true; + } + return os.str(); +} + +std::string path_prefixed(const std::string& file_name) { + return "../../../test/crosslanguagefiles/" + file_name; +} + +} // anon ns + +void +ErrorCodesTest::error_codes_match_java_definitions() +{ + NamedErrorCodes codes(all_document_protocol_error_codes()); + auto cpp_golden_file = path_prefixed("HEAD-cpp-golden-error-codes.txt"); + auto cpp_golden_data = to_sorted_key_value_string(codes); + write_file(cpp_golden_file, cpp_golden_data); + + auto java_golden_file = path_prefixed("HEAD-java-golden-error-codes.txt"); + auto java_golden_data = read_file(java_golden_file); + EXPECT_EQUAL(cpp_golden_data, java_golden_data); +} + +int +ErrorCodesTest::Main() +{ + TEST_INIT("error_codes_test"); + error_codes_match_java_definitions(); + TEST_FLUSH(); + TEST_DONE(); +} + diff --git a/documentapi/src/tests/messages/messages50app.cpp b/documentapi/src/tests/messages/messages50app.cpp new file mode 100644 index 00000000000..64532d4fd14 --- /dev/null +++ b/documentapi/src/tests/messages/messages50app.cpp @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("messages50"); + +#include "messages50test.h" + +TEST_APPHOOK(Messages50Test); diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp new file mode 100644 index 00000000000..1e0069d50b9 --- /dev/null +++ b/documentapi/src/tests/messages/messages50test.cpp @@ -0,0 +1,1225 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".test"); + +#include "messages50test.h" +#include <vespa/document/datatype/datatype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/update/fieldpathupdates.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/vdslib/container/writabledocumentlist.h> + +using document::DataType; +using document::DocumentTypeRepo; + +/////////////////////////////////////////////////////////////////////////////// +// +// Setup +// +/////////////////////////////////////////////////////////////////////////////// + +Messages50Test::Messages50Test() +{ + // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support + // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. + putTest(DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE, TEST_METHOD(Messages50Test::testBatchDocumentUpdateMessage)); + putTest(DocumentProtocol::MESSAGE_CREATEVISITOR, TEST_METHOD(Messages50Test::testCreateVisitorMessage)); + putTest(DocumentProtocol::MESSAGE_DESTROYVISITOR, TEST_METHOD(Messages50Test::testDestroyVisitorMessage)); + putTest(DocumentProtocol::MESSAGE_DOCUMENTLIST, TEST_METHOD(Messages50Test::testDocumentListMessage)); + putTest(DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, TEST_METHOD(Messages50Test::testDocumentSummaryMessage)); + putTest(DocumentProtocol::MESSAGE_EMPTYBUCKETS, TEST_METHOD(Messages50Test::testEmptyBucketsMessage)); + putTest(DocumentProtocol::MESSAGE_GETBUCKETLIST, TEST_METHOD(Messages50Test::testGetBucketListMessage)); + putTest(DocumentProtocol::MESSAGE_GETBUCKETSTATE, TEST_METHOD(Messages50Test::testGetBucketStateMessage)); + putTest(DocumentProtocol::MESSAGE_GETDOCUMENT, TEST_METHOD(Messages50Test::testGetDocumentMessage)); + putTest(DocumentProtocol::MESSAGE_MAPVISITOR, TEST_METHOD(Messages50Test::testMapVisitorMessage)); + putTest(DocumentProtocol::MESSAGE_MULTIOPERATION, TEST_METHOD(Messages50Test::testMultiOperationMessage)); + putTest(DocumentProtocol::MESSAGE_PUTDOCUMENT, TEST_METHOD(Messages50Test::testPutDocumentMessage)); + putTest(DocumentProtocol::MESSAGE_QUERYRESULT, TEST_METHOD(Messages50Test::testQueryResultMessage)); + putTest(DocumentProtocol::MESSAGE_REMOVEDOCUMENT, TEST_METHOD(Messages50Test::testRemoveDocumentMessage)); + putTest(DocumentProtocol::MESSAGE_REMOVELOCATION, TEST_METHOD(Messages50Test::testRemoveLocationMessage)); + putTest(DocumentProtocol::MESSAGE_SEARCHRESULT, TEST_METHOD(Messages50Test::testSearchResultMessage)); + putTest(DocumentProtocol::MESSAGE_STATBUCKET, TEST_METHOD(Messages50Test::testStatBucketMessage)); + putTest(DocumentProtocol::MESSAGE_UPDATEDOCUMENT, TEST_METHOD(Messages50Test::testUpdateDocumentMessage)); + putTest(DocumentProtocol::MESSAGE_VISITORINFO, TEST_METHOD(Messages50Test::testVisitorInfoMessage)); + + putTest(DocumentProtocol::REPLY_BATCHDOCUMENTUPDATE, TEST_METHOD(Messages50Test::testBatchDocumentUpdateReply)); + putTest(DocumentProtocol::REPLY_CREATEVISITOR, TEST_METHOD(Messages50Test::testCreateVisitorReply)); + putTest(DocumentProtocol::REPLY_DESTROYVISITOR, TEST_METHOD(Messages50Test::testDestroyVisitorReply)); + putTest(DocumentProtocol::REPLY_DOCUMENTLIST, TEST_METHOD(Messages50Test::testDocumentListReply)); + putTest(DocumentProtocol::REPLY_DOCUMENTSUMMARY, TEST_METHOD(Messages50Test::testDocumentSummaryReply)); + putTest(DocumentProtocol::REPLY_EMPTYBUCKETS, TEST_METHOD(Messages50Test::testEmptyBucketsReply)); + putTest(DocumentProtocol::REPLY_GETBUCKETLIST, TEST_METHOD(Messages50Test::testGetBucketListReply)); + putTest(DocumentProtocol::REPLY_GETBUCKETSTATE, TEST_METHOD(Messages50Test::testGetBucketStateReply)); + putTest(DocumentProtocol::REPLY_GETDOCUMENT, TEST_METHOD(Messages50Test::testGetDocumentReply)); + putTest(DocumentProtocol::REPLY_MAPVISITOR, TEST_METHOD(Messages50Test::testMapVisitorReply)); + putTest(DocumentProtocol::REPLY_MULTIOPERATION, TEST_METHOD(Messages50Test::testMultiOperationReply)); + putTest(DocumentProtocol::REPLY_PUTDOCUMENT, TEST_METHOD(Messages50Test::testPutDocumentReply)); + putTest(DocumentProtocol::REPLY_QUERYRESULT, TEST_METHOD(Messages50Test::testQueryResultReply)); + putTest(DocumentProtocol::REPLY_REMOVEDOCUMENT, TEST_METHOD(Messages50Test::testRemoveDocumentReply)); + putTest(DocumentProtocol::REPLY_REMOVELOCATION, TEST_METHOD(Messages50Test::testRemoveLocationReply)); + putTest(DocumentProtocol::REPLY_SEARCHRESULT, TEST_METHOD(Messages50Test::testSearchResultReply)); + putTest(DocumentProtocol::REPLY_STATBUCKET, TEST_METHOD(Messages50Test::testStatBucketReply)); + putTest(DocumentProtocol::REPLY_UPDATEDOCUMENT, TEST_METHOD(Messages50Test::testUpdateDocumentReply)); + putTest(DocumentProtocol::REPLY_VISITORINFO, TEST_METHOD(Messages50Test::testVisitorInfoReply)); + putTest(DocumentProtocol::REPLY_WRONGDISTRIBUTION, TEST_METHOD(Messages50Test::testWrongDistributionReply)); +} + + + +/////////////////////////////////////////////////////////////////////////////// +// +// Tests +// +/////////////////////////////////////////////////////////////////////////////// + +static const int MESSAGE_BASE_LENGTH = 5; + +namespace { + +document::Document::SP +createDoc(const DocumentTypeRepo &repo, const string &type_name, const string &id) +{ + return document::Document::SP(new document::Document( + *repo.getDocumentType(type_name), + document::DocumentId(id))); +} + +} // namespace + +bool +Messages50Test::testGetBucketListMessage() +{ + GetBucketListMessage msg(document::BucketId(16, 123)); + msg.setLoadType(_loadTypes["foo"]); + EXPECT_EQUAL(string("foo"), msg.getLoadType().getName()); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 12u, serialize("GetBucketListMessage", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetBucketListMessage", DocumentProtocol::MESSAGE_GETBUCKETLIST, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetBucketListMessage &ref = static_cast<GetBucketListMessage&>(*obj); + EXPECT_EQUAL(string("foo"), ref.getLoadType().getName()); + EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucketId()); + } + } + return true; +} + +bool +Messages50Test::testEmptyBucketsMessage() +{ + std::vector<document::BucketId> bids; + for (size_t i=0; i < 13; ++i) { + bids.push_back(document::BucketId(16, i)); + } + + EmptyBucketsMessage msg(bids); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 112u, serialize("EmptyBucketsMessage", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("EmptyBucketsMessage", DocumentProtocol::MESSAGE_EMPTYBUCKETS, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + EmptyBucketsMessage &ref = static_cast<EmptyBucketsMessage&>(*obj); + for (size_t i=0; i < 13; ++i) { + EXPECT_EQUAL(document::BucketId(16, i), ref.getBucketIds()[i]); + } + } + } + return true; +} + + +bool +Messages50Test::testStatBucketMessage() +{ + StatBucketMessage msg(document::BucketId(16, 123), "id.user=123"); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 27u, serialize("StatBucketMessage", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("StatBucketMessage", DocumentProtocol::MESSAGE_STATBUCKET, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + StatBucketMessage &ref = static_cast<StatBucketMessage&>(*obj); + EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucketId()); + EXPECT_EQUAL("id.user=123", ref.getDocumentSelection()); + } + } + return true; +} + +bool +Messages50Test::testCreateVisitorMessage() { + CreateVisitorMessage tmp("SomeLibrary", "myvisitor", "newyork", "london"); + tmp.setDocumentSelection("true and false or true"); + tmp.getParameters().set("myvar", "somevalue"); + tmp.getParameters().set("anothervar", uint64_t(34)); + tmp.getBuckets().push_back(document::BucketId(16, 1234)); + tmp.setVisitRemoves(true); + tmp.setVisitorOrdering(document::OrderingSpecification::DESCENDING); + tmp.setMaxBucketsPerVisitor(2); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)168, serialize("CreateVisitorMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("CreateVisitorMessage", DocumentProtocol::MESSAGE_CREATEVISITOR, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + CreateVisitorMessage &ref = static_cast<CreateVisitorMessage&>(*obj); + + EXPECT_EQUAL(string("SomeLibrary"), ref.getLibraryName()); + EXPECT_EQUAL(string("myvisitor"), ref.getInstanceId()); + EXPECT_EQUAL(string("newyork"), ref.getControlDestination()); + EXPECT_EQUAL(string("london"), ref.getDataDestination()); + EXPECT_EQUAL(string("true and false or true"), ref.getDocumentSelection()); + EXPECT_EQUAL(uint32_t(8), ref.getMaximumPendingReplyCount()); + EXPECT_EQUAL(true, ref.visitRemoves()); + EXPECT_EQUAL(false, ref.visitHeadersOnly()); + EXPECT_EQUAL(false, ref.visitInconsistentBuckets()); + EXPECT_EQUAL(size_t(1), ref.getBuckets().size()); + EXPECT_EQUAL(document::BucketId(16, 1234), ref.getBuckets()[0]); + EXPECT_EQUAL(string("somevalue"), ref.getParameters().get("myvar")); + EXPECT_EQUAL(uint64_t(34), ref.getParameters().get("anothervar", uint64_t(1))); + EXPECT_EQUAL(document::OrderingSpecification::DESCENDING, ref.getVisitorOrdering()); + EXPECT_EQUAL(uint32_t(2), ref.getMaxBucketsPerVisitor()); + } + } + return true; +} + +bool +Messages50Test::testDestroyVisitorMessage() +{ + DestroyVisitorMessage tmp("myvisitor"); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)17, serialize("DestroyVisitorMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("DestroyVisitorMessage", DocumentProtocol::MESSAGE_DESTROYVISITOR, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + DestroyVisitorMessage &ref = static_cast<DestroyVisitorMessage&>(*obj); + EXPECT_EQUAL(string("myvisitor"), ref.getInstanceId()); + } + } + return true; +} + +bool +Messages50Test::testDocumentListMessage() +{ + document::Document::SP doc = + createDoc(getTypeRepo(), "testdoc", "userdoc:scheme:1234:"); + DocumentListMessage::Entry entry(1234, doc, false); + + DocumentListMessage tmp(document::BucketId(16, 1234)); + tmp.getDocuments().push_back(entry); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)63, serialize("DocumentListMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("DocumentListMessage", DocumentProtocol::MESSAGE_DOCUMENTLIST, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + DocumentListMessage &ref = static_cast<DocumentListMessage&>(*obj); + + EXPECT_EQUAL("userdoc:scheme:1234:", ref.getDocuments()[0].getDocument()->getId().toString()); + EXPECT_EQUAL(1234, ref.getDocuments()[0].getTimestamp()); + EXPECT_TRUE(!ref.getDocuments()[0].isRemoveEntry()); + } + } + return true; +} + + +bool +Messages50Test::testRemoveLocationMessage() +{ + { + document::BucketIdFactory factory; + document::select::Parser parser(getTypeRepo(), factory); + RemoveLocationMessage msg(factory, parser, "id.group == \"mygroup\""); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 29u, serialize("RemoveLocationMessage", msg)); + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("RemoveLocationMessage", DocumentProtocol::MESSAGE_REMOVELOCATION, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + RemoveLocationMessage &ref = static_cast<RemoveLocationMessage&>(*obj); + EXPECT_EQUAL(string("id.group == \"mygroup\""), ref.getDocumentSelection()); + } + } + } + + return true; +} + + + +bool +Messages50Test::testDocumentSummaryMessage() +{ + DocumentSummaryMessage srm; + EXPECT_EQUAL(srm.hasSequenceId(), false); + EXPECT_EQUAL(srm.getSummaryCount(), size_t(0)); + + mbus::Blob data = encode(srm); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(12), data.size()); + + writeFile(getPath("5-cpp-DocumentSummaryMessage-1.dat"), data); + // print(data); + + mbus::Routable::UP routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_DOCUMENTSUMMARY); + DocumentSummaryMessage * dm = static_cast<DocumentSummaryMessage *>(routable.get()); + EXPECT_EQUAL(dm->getSummaryCount(), size_t(0)); + + srm.addSummary("doc1", "summary1", 8); + srm.addSummary("aoc17", "summary45", 9); + + data = encode(srm); + //print(data); + + const void *summary(NULL); + const char *docId(NULL); + size_t sz(0); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, data.size()); + writeFile(getPath("5-cpp-DocumentSummaryMessage-2.dat"), data); + routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_DOCUMENTSUMMARY); + dm = static_cast<DocumentSummaryMessage *>(routable.get()); + EXPECT_EQUAL(dm->getSummaryCount(), size_t(2)); + dm->getSummary(0, docId, summary, sz); + EXPECT_EQUAL(sz, 8u); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + EXPECT_EQUAL(memcmp("summary1", summary, sz), 0); + dm->getSummary(1, docId, summary, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(strcmp("aoc17", docId), 0); + EXPECT_EQUAL(memcmp("summary45", summary, sz), 0); + + srm.sort(); + + data = encode(srm); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, data.size()); + writeFile(getPath("5-cpp-DocumentSummaryMessage-3.dat"), data); + routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_DOCUMENTSUMMARY); + dm = static_cast<DocumentSummaryMessage *>(routable.get()); + EXPECT_EQUAL(dm->getSummaryCount(), size_t(2)); + dm->getSummary(0, docId, summary, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(strcmp("aoc17", docId), 0); + EXPECT_EQUAL(memcmp("summary45", summary, sz), 0); + dm->getSummary(1, docId, summary, sz); + EXPECT_EQUAL(sz, 8u); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + EXPECT_EQUAL(memcmp("summary1", summary, sz), 0); + return true; +} + +bool +Messages50Test::testMultiOperationMessage() +{ + document::Document::SP doc = + createDoc(getTypeRepo(), "testdoc", "doc:scheme:foo"); + std::vector<char> buffer(1024); + document::BucketIdFactory factory; + + vdslib::WritableDocumentList doclist(getTypeRepoSp(), + &buffer[0], buffer.size()); + ASSERT_TRUE(doclist.addPut(*doc)); + + size_t n = MESSAGE_BASE_LENGTH; + n += sizeof(uint32_t); // routable object type + n += sizeof(uint64_t); // bucket id + n += sizeof(uint32_t); // bytes in docblock + n += sizeof(uint32_t); // num operations + n += sizeof(vdslib::DocumentList::MetaEntry); + n += doc->serialize()->getLength(); + n += 1; // boolean keepTimeStamps + + MultiOperationMessage msg(document::BucketId(16, factory.getBucketId(doc->getId()).getRawId()), doclist); + EXPECT_EQUAL(n, serialize("MultiOperationMessage", msg)); + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("MultiOperationMessage", DocumentProtocol::MESSAGE_MULTIOPERATION, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + MultiOperationMessage &ref = static_cast<MultiOperationMessage&>(*obj); + EXPECT_EQUAL((uint32_t)1, ref.getOperations().size()); + EXPECT_EQUAL(*doc, *dynamic_cast<document::Document*>(ref.getOperations().begin()->getDocument().get())); + EXPECT_EQUAL(document::BucketId(16, factory.getBucketId(doc->getId()).getRawId()), ref.getBucketId()); + } + } + return true; +} + +bool +Messages50Test::testGetDocumentMessage() +{ + GetDocumentMessage tmp(document::DocumentId("doc:scheme:"), 0); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)20, serialize("GetDocumentMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetDocumentMessage", DocumentProtocol::MESSAGE_GETDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetDocumentMessage &ref = static_cast<GetDocumentMessage&>(*obj); + EXPECT_EQUAL(string("doc:scheme:"), ref.getDocumentId().toString()); + } + } + return true; +} + +bool +Messages50Test::testMapVisitorMessage() +{ + MapVisitorMessage tmp; + tmp.getData().set("foo", 3); + tmp.getData().set("bar", 5); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)32, serialize("MapVisitorMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("MapVisitorMessage", DocumentProtocol::MESSAGE_MAPVISITOR, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + MapVisitorMessage &ref = static_cast<MapVisitorMessage&>(*obj); + EXPECT_EQUAL(3, ref.getData().get("foo", 0)); + EXPECT_EQUAL(5, ref.getData().get("bar", 0)); + } + } + return true; +} + +bool +Messages50Test::testCreateVisitorReply() +{ + CreateVisitorReply reply(DocumentProtocol::REPLY_CREATEVISITOR); + reply.setLastBucket(document::BucketId(16, 123)); + vdslib::VisitorStatistics vs; + vs.setBucketsVisited(3); + vs.setDocumentsVisited(1000); + vs.setBytesVisited(1024000); + vs.setDocumentsReturned(123); + vs.setBytesReturned(512000); + vs.setSecondPassDocumentsReturned(456); + vs.setSecondPassBytesReturned(789100); + reply.setVisitorStatistics(vs); + + EXPECT_EQUAL(65u, serialize("CreateVisitorReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("CreateVisitorReply", DocumentProtocol::REPLY_CREATEVISITOR, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + CreateVisitorReply &ref = static_cast<CreateVisitorReply&>(*obj); + + EXPECT_EQUAL(ref.getLastBucket(), document::BucketId(16, 123)); + EXPECT_EQUAL(ref.getVisitorStatistics().getBucketsVisited(), (uint32_t)3); + EXPECT_EQUAL(ref.getVisitorStatistics().getDocumentsVisited(), (uint64_t)1000); + EXPECT_EQUAL(ref.getVisitorStatistics().getBytesVisited(), (uint64_t)1024000); + EXPECT_EQUAL(ref.getVisitorStatistics().getDocumentsReturned(), (uint64_t)123); + EXPECT_EQUAL(ref.getVisitorStatistics().getBytesReturned(), (uint64_t)512000); + EXPECT_EQUAL(ref.getVisitorStatistics().getSecondPassDocumentsReturned(), (uint64_t)456); + EXPECT_EQUAL(ref.getVisitorStatistics().getSecondPassBytesReturned(), (uint64_t)789100); + } + } + return true; +} + +bool +Messages50Test::testPutDocumentMessage() +{ + document::Document::SP doc = + createDoc(getTypeRepo(), "testdoc", "doc:scheme:"); + PutDocumentMessage msg(doc); + + msg.setTimestamp(666); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 41u, serialize("PutDocumentMessage", msg)); + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("PutDocumentMessage", DocumentProtocol::MESSAGE_PUTDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + PutDocumentMessage &ref = static_cast<PutDocumentMessage&>(*obj); + EXPECT_TRUE(ref.getDocument()->getType().getName() == "testdoc"); + EXPECT_TRUE(ref.getDocument()->getId().toString() == "doc:scheme:"); + EXPECT_EQUAL(666u, ref.getTimestamp()); + EXPECT_EQUAL(37u, ref.getApproxSize()); + } + } + return true; +} + +bool +Messages50Test::testGetBucketStateMessage() +{ + GetBucketStateMessage tmp; + tmp.setBucketId(document::BucketId(16, 666)); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 12u, serialize("GetBucketStateMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetBucketStateMessage", DocumentProtocol::MESSAGE_GETBUCKETSTATE, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetBucketStateMessage &ref = static_cast<GetBucketStateMessage&>(*obj); + + EXPECT_EQUAL(16u, ref.getBucketId().getUsedBits()); + EXPECT_EQUAL(4611686018427388570ull, ref.getBucketId().getId()); + } + } + return true; +} + +bool +Messages50Test::testPutDocumentReply() +{ + WriteDocumentReply reply(DocumentProtocol::REPLY_PUTDOCUMENT); + reply.setHighestModificationTimestamp(30); + + EXPECT_EQUAL(13u, serialize("PutDocumentReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("PutDocumentReply", DocumentProtocol::REPLY_PUTDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + WriteDocumentReply &ref = static_cast<WriteDocumentReply&>(*obj); + EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp()); + } + } + return true; +} + +bool +Messages50Test::testUpdateDocumentReply() +{ + UpdateDocumentReply reply; + reply.setWasFound(false); + reply.setHighestModificationTimestamp(30); + + EXPECT_EQUAL(14u, serialize("UpdateDocumentReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("UpdateDocumentReply", DocumentProtocol::REPLY_UPDATEDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + UpdateDocumentReply &ref = static_cast<UpdateDocumentReply&>(*obj); + EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp()); + EXPECT_EQUAL(false, ref.wasFound()); + } + } + return true; +} + +bool +Messages50Test::testRemoveDocumentMessage() +{ + RemoveDocumentMessage tmp(document::DocumentId("doc:scheme:")); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)16, serialize("RemoveDocumentMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("RemoveDocumentMessage", DocumentProtocol::MESSAGE_REMOVEDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + RemoveDocumentMessage &ref = static_cast<RemoveDocumentMessage&>(*obj); + EXPECT_EQUAL(string("doc:scheme:"), ref.getDocumentId().toString()); + } + } + return true; +} + +bool +Messages50Test::testRemoveDocumentReply() +{ + RemoveDocumentReply reply; + std::vector<uint64_t> ts; + reply.setWasFound(false); + reply.setHighestModificationTimestamp(30); + + EXPECT_EQUAL(14u, serialize("RemoveDocumentReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("RemoveDocumentReply", DocumentProtocol::REPLY_REMOVEDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + RemoveDocumentReply &ref = static_cast<RemoveDocumentReply&>(*obj); + EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp()); + EXPECT_EQUAL(false, ref.wasFound()); + } + } + return true; +} + +bool +Messages50Test::testSearchResultMessage() +{ + SearchResultMessage srm; + EXPECT_EQUAL(srm.getSequenceId(), 0u); + EXPECT_EQUAL(srm.getHitCount(), 0u); + EXPECT_EQUAL(srm.getAggregatorList().getSerializedSize(), 4u); + EXPECT_EQUAL(srm.vdslib::SearchResult::getSerializedSize(), 20u); + EXPECT_EQUAL(srm.getSerializedSize(), 20u); + + mbus::Blob data = encode(srm); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(24), data.size()); + + writeFile(getPath("5-cpp-SearchResultMessage-1.dat"), data); + // print(data); + + mbus::Routable::UP routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT); + SearchResultMessage * dm = static_cast<SearchResultMessage *>(routable.get()); + EXPECT_EQUAL(dm->getSequenceId(), size_t(0)); + EXPECT_EQUAL(dm->getHitCount(), size_t(0)); + + srm.addHit(0, "doc1", 89); + srm.addHit(1, "doc17", 109); + //srm.setSequenceId(567); + + data = encode(srm); + //EXPECT_EQUAL(srm.getSequenceId(), size_t(567)); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, data.size()); + writeFile(getPath("5-cpp-SearchResultMessage-2.dat"), data); + routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT); + dm = static_cast<SearchResultMessage *>(routable.get()); +// EXPECT_EQUAL(dm->getSequenceId(), size_t(567)); + EXPECT_EQUAL(dm->getHitCount(), size_t(2)); + const char *docId; + SearchResultMessage::RankType rank; + dm->getHit(0, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + dm->getHit(1, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + + srm.sort(); + + data = encode(srm); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, data.size()); + writeFile(getPath("5-cpp-SearchResultMessage-3.dat"), data); + routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT); + dm = static_cast<SearchResultMessage *>(routable.get()); +// EXPECT_EQUAL(dm->getSequenceId(), size_t(567)); + EXPECT_EQUAL(dm->getHitCount(), size_t(2)); + dm->getHit(0, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + dm->getHit(1, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + + SearchResultMessage srm2; + srm2.addHit(0, "doc1", 89, "sortdata2", 9); + srm2.addHit(1, "doc17", 109, "sortdata1", 9); + srm2.addHit(2, "doc18", 90, "sortdata3", 9); + //srm2.setSequenceId(567); + data = encode(srm2); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, data.size()); + writeFile(getPath("5-cpp-SearchResultMessage-4.dat"), data); + routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT); + dm = static_cast<SearchResultMessage *>(routable.get()); + //EXPECT_EQUAL(dm->getSequenceId(), size_t(567)); + EXPECT_EQUAL(dm->getHitCount(), size_t(3)); + dm->getHit(0, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + dm->getHit(1, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + dm->getHit(2, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); + EXPECT_EQUAL(strcmp("doc18", docId), 0); + + srm2.sort(); + const void *buf; + size_t sz; + srm2.getHit(0, docId, rank); + srm2.getSortBlob(0, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + srm2.getHit(1, docId, rank); + srm2.getSortBlob(1, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + srm2.getHit(2, docId, rank); + srm2.getSortBlob(2, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); + EXPECT_EQUAL(strcmp("doc18", docId), 0); + + data = encode(srm2); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, data.size()); + writeFile(getPath("5-cpp-SearchResultMessage-5.dat"), data); + routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT); + dm = static_cast<SearchResultMessage *>(routable.get()); +// EXPECT_EQUAL(dm->getSequenceId(), size_t(567)); + EXPECT_EQUAL(dm->getHitCount(), size_t(3)); + dm->getHit(0, docId, rank); + dm->getSortBlob(0, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + dm->getHit(1, docId, rank); + dm->getSortBlob(1, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + dm->getHit(2, docId, rank); + dm->getSortBlob(2, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); + EXPECT_EQUAL(strcmp("doc18", docId), 0); + return true; +} + + +bool +Messages50Test::testMultiOperationReply() +{ + WriteDocumentReply reply(DocumentProtocol::REPLY_MULTIOPERATION); + reply.setHighestModificationTimestamp(30); + + EXPECT_EQUAL(13u, serialize("MultiOperationReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("MultiOperationReply", DocumentProtocol::REPLY_MULTIOPERATION, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + WriteDocumentReply &ref = static_cast<WriteDocumentReply&>(*obj); + EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp()); + } + } + return true; +} + +bool +Messages50Test::testUpdateDocumentMessage() +{ + const DocumentTypeRepo &repo = getTypeRepo(); + const document::DocumentType &docType = + *repo.getDocumentType("testdoc"); + document::DocumentUpdate::SP + upd(new document::DocumentUpdate(docType, + document::DocumentId("doc:scheme:"))); + upd->addFieldPathUpdate(document::FieldPathUpdate::CP( + new document::RemoveFieldPathUpdate(repo, docType, "intfield", "testdoc.intfield > 0"))); + UpdateDocumentMessage msg(upd); + msg.setOldTimestamp(666u); + msg.setNewTimestamp(777u); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 89u, serialize("UpdateDocumentMessage", msg)); + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("UpdateDocumentMessage", DocumentProtocol::MESSAGE_UPDATEDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + UpdateDocumentMessage &ref = static_cast<UpdateDocumentMessage&>(*obj); + EXPECT_EQUAL(*upd, *ref.getDocumentUpdate()); + EXPECT_EQUAL(666u, ref.getOldTimestamp()); + EXPECT_EQUAL(777u, ref.getNewTimestamp()); + EXPECT_EQUAL(85u, ref.getApproxSize()); + } + } + return true; +} + +bool +Messages50Test::testBatchDocumentUpdateMessage() +{ + const DocumentTypeRepo &repo = getTypeRepo(); + const document::DocumentType &docType = *repo.getDocumentType("testdoc"); + + BatchDocumentUpdateMessage msg(1234); + + { + document::DocumentUpdate::SP upd; + upd.reset(new document::DocumentUpdate(docType, document::DocumentId("userdoc:footype:1234:foo"))); + upd->addFieldPathUpdate(document::FieldPathUpdate::CP( + new document::RemoveFieldPathUpdate(repo, docType, "intfield", "testdoc.intfield > 0"))); + msg.addUpdate(upd); + } + { + document::DocumentUpdate::SP upd; + upd.reset(new document::DocumentUpdate(docType, document::DocumentId("orderdoc(32,17):footype:1234:123456789:foo"))); + upd->addFieldPathUpdate(document::FieldPathUpdate::CP( + new document::RemoveFieldPathUpdate(repo, docType, "intfield", "testdoc.intfield > 0"))); + msg.addUpdate(upd); + } + try { + document::DocumentUpdate::SP upd; + upd.reset(new document::DocumentUpdate(docType, document::DocumentId("userdoc:footype:5678:foo"))); + upd->addFieldPathUpdate(document::FieldPathUpdate::CP( + new document::RemoveFieldPathUpdate(repo, docType, "intfield", "testdoc.intfield > 0"))); + msg.addUpdate(upd); + EXPECT_TRUE(false); + } catch (...) { + } + try { + document::DocumentUpdate::SP upd; + upd.reset(new document::DocumentUpdate(docType, document::DocumentId("groupdoc:footype:hable:foo"))); + upd->addFieldPathUpdate(document::FieldPathUpdate::CP( + new document::RemoveFieldPathUpdate(repo, docType, "intfield", "testdoc.intfield > 0"))); + msg.addUpdate(upd); + EXPECT_TRUE(false); + } catch (...) { + } + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 202u, serialize("BatchDocumentUpdateMessage", msg)); + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("BatchDocumentUpdateMessage", DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + BatchDocumentUpdateMessage &ref = static_cast<BatchDocumentUpdateMessage&>(*obj); + EXPECT_EQUAL(2u, ref.getUpdates().size()); + } + } + + return true; +} + +bool +Messages50Test::testBatchDocumentUpdateReply() +{ + BatchDocumentUpdateReply reply; + reply.setHighestModificationTimestamp(30); + { + std::vector<bool> notFound(3); + notFound[0] = false; + notFound[1] = true; + notFound[2] = true; + reply.getDocumentsNotFound() = notFound; + } + + EXPECT_EQUAL(20u, serialize("BatchDocumentUpdateReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("BatchDocumentUpdateReply", DocumentProtocol::REPLY_BATCHDOCUMENTUPDATE, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + BatchDocumentUpdateReply &ref = dynamic_cast<BatchDocumentUpdateReply&>(*obj); + EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp()); + { + const std::vector<bool>& notFound = ref.getDocumentsNotFound(); + EXPECT_TRUE(notFound[0] == false); + EXPECT_TRUE(notFound[1] == true); + EXPECT_TRUE(notFound[2] == true); + } + } + } + return true; +} + +bool +Messages50Test::testQueryResultMessage() +{ + QueryResultMessage srm; + vdslib::SearchResult & sr(srm.getSearchResult()); + EXPECT_EQUAL(srm.getSequenceId(), 0u); + EXPECT_EQUAL(sr.getHitCount(), 0u); + EXPECT_EQUAL(sr.getAggregatorList().getSerializedSize(), 4u); + EXPECT_EQUAL(sr.getSerializedSize(), 20u); + EXPECT_EQUAL(srm.getApproxSize(), 28u); + + mbus::Blob data = encode(srm); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(32), data.size()); + + writeFile(getPath("5-cpp-QueryResultMessage-1.dat"), data); + // print(data); + + mbus::Routable::UP routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT); + QueryResultMessage * dm = static_cast<QueryResultMessage *>(routable.get()); + vdslib::SearchResult * dr(&dm->getSearchResult()); + EXPECT_EQUAL(dm->getSequenceId(), size_t(0)); + EXPECT_EQUAL(dr->getHitCount(), size_t(0)); + + sr.addHit(0, "doc1", 89); + sr.addHit(1, "doc17", 109); + + data = encode(srm); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, data.size()); + writeFile(getPath("5-cpp-QueryResultMessage-2.dat"), data); + routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT); + dm = static_cast<QueryResultMessage *>(routable.get()); + dr = &dm->getSearchResult(); + EXPECT_EQUAL(dr->getHitCount(), size_t(2)); + const char *docId; + vdslib::SearchResult::RankType rank; + dr->getHit(0, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + dr->getHit(1, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + + sr.sort(); + + data = encode(srm); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, data.size()); + writeFile(getPath("5-cpp-QueryResultMessage-3.dat"), data); + routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT); + dm = static_cast<QueryResultMessage *>(routable.get()); + dr = &dm->getSearchResult(); + EXPECT_EQUAL(dr->getHitCount(), size_t(2)); + dr->getHit(0, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + dr->getHit(1, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + + QueryResultMessage srm2; + vdslib::SearchResult & sr2(srm2.getSearchResult()); + sr2.addHit(0, "doc1", 89, "sortdata2", 9); + sr2.addHit(1, "doc17", 109, "sortdata1", 9); + sr2.addHit(2, "doc18", 90, "sortdata3", 9); + data = encode(srm2); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, data.size()); + writeFile(getPath("5-cpp-QueryResultMessage-4.dat"), data); + routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT); + dm = static_cast<QueryResultMessage *>(routable.get()); + dr = &dm->getSearchResult(); + EXPECT_EQUAL(dr->getHitCount(), size_t(3)); + dr->getHit(0, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + dr->getHit(1, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + dr->getHit(2, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(90)); + EXPECT_EQUAL(strcmp("doc18", docId), 0); + + sr2.sort(); + const void *buf; + size_t sz; + sr2.getHit(0, docId, rank); + sr2.getSortBlob(0, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + sr2.getHit(1, docId, rank); + sr2.getSortBlob(1, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + sr2.getHit(2, docId, rank); + sr2.getSortBlob(2, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(90)); + EXPECT_EQUAL(strcmp("doc18", docId), 0); + + data = encode(srm2); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, data.size()); + writeFile(getPath("5-cpp-QueryResultMessage-5.dat"), data); + routable = decode(data); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT); + dm = static_cast<QueryResultMessage *>(routable.get()); + dr = &dm->getSearchResult(); + EXPECT_EQUAL(dr->getHitCount(), size_t(3)); + dr->getHit(0, docId, rank); + dr->getSortBlob(0, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + dr->getHit(1, docId, rank); + dr->getSortBlob(1, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + dr->getHit(2, docId, rank); + dr->getSortBlob(2, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(90)); + EXPECT_EQUAL(strcmp("doc18", docId), 0); + return true; +} + +bool +Messages50Test::testQueryResultReply() +{ + return tryVisitorReply("QueryResultReply", DocumentProtocol::REPLY_QUERYRESULT); +} + +bool +Messages50Test::testVisitorInfoMessage() +{ + + VisitorInfoMessage tmp; + tmp.getFinishedBuckets().push_back(document::BucketId(16, 1)); + tmp.getFinishedBuckets().push_back(document::BucketId(16, 2)); + tmp.getFinishedBuckets().push_back(document::BucketId(16, 4)); + string utf8 = "error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6"; + tmp.setErrorMessage(utf8); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 67u, serialize("VisitorInfoMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("VisitorInfoMessage", DocumentProtocol::MESSAGE_VISITORINFO, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + VisitorInfoMessage &ref = static_cast<VisitorInfoMessage&>(*obj); + EXPECT_EQUAL(document::BucketId(16, 1), ref.getFinishedBuckets()[0]); + EXPECT_EQUAL(document::BucketId(16, 2), ref.getFinishedBuckets()[1]); + EXPECT_EQUAL(document::BucketId(16, 4), ref.getFinishedBuckets()[2]); + EXPECT_EQUAL(utf8, ref.getErrorMessage()); + } + } + return true; +} + +bool +Messages50Test::testDestroyVisitorReply() +{ + return tryDocumentReply("DestroyVisitorReply", DocumentProtocol::REPLY_DESTROYVISITOR); +} + +bool +Messages50Test::testDocumentListReply() +{ + return tryVisitorReply("DocumentListReply", DocumentProtocol::REPLY_DOCUMENTLIST); +} + +bool +Messages50Test::testDocumentSummaryReply() +{ + return tryVisitorReply("DocumentSummaryReply", DocumentProtocol::REPLY_DOCUMENTSUMMARY); +} + +bool +Messages50Test::testGetDocumentReply() +{ + document::Document::SP doc = + createDoc(getTypeRepo(), "testdoc", "doc:scheme:"); + GetDocumentReply tmp(doc); + + EXPECT_EQUAL((size_t)43, serialize("GetDocumentReply", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetDocumentReply", DocumentProtocol::REPLY_GETDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetDocumentReply &ref = static_cast<GetDocumentReply&>(*obj); + + EXPECT_EQUAL(string("testdoc"), ref.getDocument()->getType().getName()); + EXPECT_EQUAL(string("doc:scheme:"), ref.getDocument()->getId().toString()); + } + } + return true; +} + +bool +Messages50Test::testMapVisitorReply() +{ + return tryVisitorReply("MapVisitorReply", DocumentProtocol::REPLY_MAPVISITOR); +} + +bool +Messages50Test::testSearchResultReply() +{ + return tryVisitorReply("SearchResultReply", DocumentProtocol::REPLY_SEARCHRESULT); +} + +bool +Messages50Test::testStatBucketReply() +{ + StatBucketReply msg; + msg.setResults("These are the votes of the Norwegian jury"); + + EXPECT_EQUAL(50u, serialize("StatBucketReply", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("StatBucketReply", DocumentProtocol::REPLY_STATBUCKET, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + StatBucketReply &ref = static_cast<StatBucketReply&>(*obj); + EXPECT_EQUAL("These are the votes of the Norwegian jury", ref.getResults()); + } + } + return true; +} + +bool +Messages50Test::testVisitorInfoReply() +{ + return tryVisitorReply("VisitorInfoReply", DocumentProtocol::REPLY_VISITORINFO); +} + +bool +Messages50Test::testWrongDistributionReply() +{ + WrongDistributionReply tmp("distributor:3 storage:2"); + + serialize("WrongDistributionReply", tmp); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("WrongDistributionReply", DocumentProtocol::REPLY_WRONGDISTRIBUTION, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + WrongDistributionReply &ref = static_cast<WrongDistributionReply&>(*obj); + EXPECT_EQUAL(string("distributor:3 storage:2"), ref.getSystemState()); + } + } + return true; +} + +bool +Messages50Test::testGetBucketListReply() +{ + GetBucketListReply reply; + reply.getBuckets().push_back(GetBucketListReply::BucketInfo(document::BucketId(16, 123), "foo")); + reply.getBuckets().push_back(GetBucketListReply::BucketInfo(document::BucketId(17, 1123), "bar")); + reply.getBuckets().push_back(GetBucketListReply::BucketInfo(document::BucketId(18, 11123), "zoink")); + + EXPECT_EQUAL(56u, serialize("GetBucketListReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetBucketListReply", DocumentProtocol::REPLY_GETBUCKETLIST, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetBucketListReply &ref = static_cast<GetBucketListReply&>(*obj); + + EXPECT_EQUAL(ref.getBuckets()[0], GetBucketListReply::BucketInfo(document::BucketId(16, 123), "foo")); + EXPECT_EQUAL(ref.getBuckets()[1], GetBucketListReply::BucketInfo(document::BucketId(17, 1123), "bar")); + EXPECT_EQUAL(ref.getBuckets()[2], GetBucketListReply::BucketInfo(document::BucketId(18, 11123), "zoink")); + } + } + return true; +} + +bool +Messages50Test::testGetBucketStateReply() +{ + document::GlobalId foo = document::DocumentId("doc:scheme:foo").getGlobalId(); + document::GlobalId bar = document::DocumentId("doc:scheme:bar").getGlobalId(); + + GetBucketStateReply reply; + reply.getBucketState().push_back(DocumentState(foo, 777, false)); + reply.getBucketState().push_back(DocumentState(bar, 888, true)); + EXPECT_EQUAL(53u, serialize("GetBucketStateReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetBucketStateReply", DocumentProtocol::REPLY_GETBUCKETSTATE, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetBucketStateReply &ref = static_cast<GetBucketStateReply&>(*obj); + + EXPECT_EQUAL(777u, ref.getBucketState()[0].getTimestamp()); + EXPECT_EQUAL(foo, ref.getBucketState()[0].getGlobalId()); + EXPECT_EQUAL(false, ref.getBucketState()[0].isRemoveEntry()); + EXPECT_EQUAL(888u, ref.getBucketState()[1].getTimestamp()); + EXPECT_EQUAL(bar, ref.getBucketState()[1].getGlobalId()); + EXPECT_EQUAL(true, ref.getBucketState()[1].isRemoveEntry()); + } + } + return true; +} + +bool +Messages50Test::testEmptyBucketsReply() +{ + return tryVisitorReply("EmptyBucketsReply", DocumentProtocol::REPLY_EMPTYBUCKETS); +} + +bool +Messages50Test::testRemoveLocationReply() +{ + DocumentReply tmp(DocumentProtocol::REPLY_REMOVELOCATION); + + EXPECT_EQUAL((uint32_t)5, serialize("RemoveLocationReply", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("RemoveLocationReply", DocumentProtocol::REPLY_REMOVELOCATION, lang); + EXPECT_TRUE(obj.get() != NULL); + } + return true; +} + + + +//////////////////////////////////////////////////////////////////////////////// +// +// Utilities +// +//////////////////////////////////////////////////////////////////////////////// + +bool +Messages50Test::tryDocumentReply(const string &filename, uint32_t type) +{ + DocumentReply tmp(type); + + EXPECT_EQUAL((uint32_t)5, serialize(filename, tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize(filename, type, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + DocumentReply *ref = dynamic_cast<DocumentReply*>(obj.get()); + EXPECT_TRUE(ref != NULL); + } + } + return true; +} + +bool +Messages50Test::tryVisitorReply(const string &filename, uint32_t type) +{ + VisitorReply tmp(type); + + EXPECT_EQUAL((uint32_t)5, serialize(filename, tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize(filename, type, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + VisitorReply *ref = dynamic_cast<VisitorReply*>(obj.get()); + EXPECT_TRUE(ref != NULL); + } + } + return true; +} diff --git a/documentapi/src/tests/messages/messages50test.h b/documentapi/src/tests/messages/messages50test.h new file mode 100644 index 00000000000..515a61e59fc --- /dev/null +++ b/documentapi/src/tests/messages/messages50test.h @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "testbase.h" + +class Messages50Test : public TestBase { +protected: + const vespalib::Version getVersion() const { return vespalib::Version(5, 0); } + bool shouldTestCoverage() const { return FALSE; } + bool tryDocumentReply(const string &filename, uint32_t type); + bool tryVisitorReply(const string &filename, uint32_t type); + +public: + Messages50Test(); + + bool testBatchDocumentUpdateMessage(); + bool testBatchDocumentUpdateReply(); + bool testCreateVisitorMessage(); + bool testCreateVisitorReply(); + bool testDestroyVisitorMessage(); + bool testDestroyVisitorReply(); + bool testDocumentListMessage(); + bool testDocumentListReply(); + bool testDocumentSummaryMessage(); + bool testDocumentSummaryReply(); + bool testEmptyBucketsMessage(); + bool testEmptyBucketsReply(); + bool testGetBucketListMessage(); + bool testGetBucketListReply(); + bool testGetBucketStateMessage(); + bool testGetBucketStateReply(); + bool testGetDocumentMessage(); + bool testGetDocumentReply(); + bool testMapVisitorMessage(); + bool testMapVisitorReply(); + bool testMultiOperationMessage(); + bool testMultiOperationReply(); + bool testPutDocumentMessage(); + bool testPutDocumentReply(); + bool testQueryResultMessage(); + bool testQueryResultReply(); + bool testRemoveDocumentMessage(); + bool testRemoveDocumentReply(); + bool testRemoveLocationMessage(); + bool testRemoveLocationReply(); + bool testSearchResultMessage(); + bool testSearchResultReply(); + bool testStatBucketMessage(); + bool testStatBucketReply(); + bool testUpdateDocumentMessage(); + bool testUpdateDocumentReply(); + bool testVisitorInfoMessage(); + bool testVisitorInfoReply(); + bool testWrongDistributionReply(); +}; + diff --git a/documentapi/src/tests/messages/messages51app.cpp b/documentapi/src/tests/messages/messages51app.cpp new file mode 100644 index 00000000000..6d68774f679 --- /dev/null +++ b/documentapi/src/tests/messages/messages51app.cpp @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("messages51"); + +#include "messages51test.h" + +TEST_APPHOOK(Messages51Test); diff --git a/documentapi/src/tests/messages/messages51test.cpp b/documentapi/src/tests/messages/messages51test.cpp new file mode 100644 index 00000000000..06a6becc45b --- /dev/null +++ b/documentapi/src/tests/messages/messages51test.cpp @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".test"); + +#include "messages51test.h" +#include <vespa/document/datatype/datatype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/update/fieldpathupdates.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/vdslib/container/writabledocumentlist.h> + +using document::DataType; +using document::DocumentTypeRepo; + +/////////////////////////////////////////////////////////////////////////////// +// +// Setup +// +/////////////////////////////////////////////////////////////////////////////// + +Messages51Test::Messages51Test() +{ + // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support + // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. + putTest(DocumentProtocol::MESSAGE_CREATEVISITOR, TEST_METHOD(Messages51Test::testCreateVisitorMessage)); + putTest(DocumentProtocol::MESSAGE_GETDOCUMENT, TEST_METHOD(Messages51Test::testGetDocumentMessage)); + putTest(DocumentProtocol::REPLY_DOCUMENTIGNORED, TEST_METHOD(Messages51Test::testDocumentIgnoredReply)); +} + + +/////////////////////////////////////////////////////////////////////////////// +// +// Tests +// +/////////////////////////////////////////////////////////////////////////////// + +static const int MESSAGE_BASE_LENGTH = 5; + +bool +Messages51Test::testCreateVisitorMessage() { + CreateVisitorMessage tmp("SomeLibrary", "myvisitor", "newyork", "london"); + tmp.setDocumentSelection("true and false or true"); + tmp.getParameters().set("myvar", "somevalue"); + tmp.getParameters().set("anothervar", uint64_t(34)); + tmp.getBuckets().push_back(document::BucketId(16, 1234)); + tmp.setVisitRemoves(true); + tmp.setFieldSet("foo bar"); + tmp.setVisitorOrdering(document::OrderingSpecification::DESCENDING); + tmp.setMaxBucketsPerVisitor(2); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)178, serialize("CreateVisitorMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("CreateVisitorMessage", DocumentProtocol::MESSAGE_CREATEVISITOR, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + CreateVisitorMessage &ref = static_cast<CreateVisitorMessage&>(*obj); + + EXPECT_EQUAL(string("SomeLibrary"), ref.getLibraryName()); + EXPECT_EQUAL(string("myvisitor"), ref.getInstanceId()); + EXPECT_EQUAL(string("newyork"), ref.getControlDestination()); + EXPECT_EQUAL(string("london"), ref.getDataDestination()); + EXPECT_EQUAL(string("true and false or true"), ref.getDocumentSelection()); + EXPECT_EQUAL(string("foo bar"), ref.getFieldSet()); + EXPECT_EQUAL(uint32_t(8), ref.getMaximumPendingReplyCount()); + EXPECT_EQUAL(true, ref.visitRemoves()); + EXPECT_EQUAL(false, ref.visitHeadersOnly()); + EXPECT_EQUAL(false, ref.visitInconsistentBuckets()); + EXPECT_EQUAL(size_t(1), ref.getBuckets().size()); + EXPECT_EQUAL(document::BucketId(16, 1234), ref.getBuckets()[0]); + EXPECT_EQUAL(string("somevalue"), ref.getParameters().get("myvar")); + EXPECT_EQUAL(uint64_t(34), ref.getParameters().get("anothervar", uint64_t(1))); + EXPECT_EQUAL(document::OrderingSpecification::DESCENDING, ref.getVisitorOrdering()); + EXPECT_EQUAL(uint32_t(2), ref.getMaxBucketsPerVisitor()); + } + } + return true; +} + +bool +Messages51Test::testGetDocumentMessage() +{ + GetDocumentMessage tmp(document::DocumentId("doc:scheme:"), "foo bar"); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)27, serialize("GetDocumentMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetDocumentMessage", DocumentProtocol::MESSAGE_GETDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetDocumentMessage &ref = static_cast<GetDocumentMessage&>(*obj); + EXPECT_EQUAL(string("doc:scheme:"), ref.getDocumentId().toString()); + EXPECT_EQUAL(string("foo bar"), ref.getFieldSet()); + } + } + return true; +} + +bool +Messages51Test::testDocumentIgnoredReply() +{ + DocumentIgnoredReply tmp; + serialize("DocumentIgnoredReply", tmp); + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj( + deserialize("DocumentIgnoredReply", + DocumentProtocol::REPLY_DOCUMENTIGNORED, lang)); + EXPECT_TRUE(obj.get() != NULL); + } + return true; +} diff --git a/documentapi/src/tests/messages/messages51test.h b/documentapi/src/tests/messages/messages51test.h new file mode 100644 index 00000000000..9cf57a44b29 --- /dev/null +++ b/documentapi/src/tests/messages/messages51test.h @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "messages50test.h" + +class Messages51Test : public Messages50Test { +protected: + const vespalib::Version getVersion() const { return vespalib::Version(5, 1); } + bool shouldTestCoverage() const { return TRUE; } + +public: + Messages51Test(); + + bool testCreateVisitorMessage(); + bool testGetDocumentMessage(); + bool testDocumentIgnoredReply(); +}; + diff --git a/documentapi/src/tests/messages/messages52app.cpp b/documentapi/src/tests/messages/messages52app.cpp new file mode 100644 index 00000000000..15fb603524b --- /dev/null +++ b/documentapi/src/tests/messages/messages52app.cpp @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("messages52"); + +#include "messages52test.h" + +TEST_APPHOOK(Messages52Test); diff --git a/documentapi/src/tests/messages/messages52test.cpp b/documentapi/src/tests/messages/messages52test.cpp new file mode 100644 index 00000000000..f3625150511 --- /dev/null +++ b/documentapi/src/tests/messages/messages52test.cpp @@ -0,0 +1,122 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".test"); + +#include "messages52test.h" +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/update/fieldpathupdates.h> +#include <vespa/documentapi/documentapi.h> + +using document::DocumentTypeRepo; + +namespace { + +document::Document::SP +createDoc(const DocumentTypeRepo &repo, const string &type_name, const string &id) +{ + return document::Document::SP(new document::Document( + *repo.getDocumentType(type_name), + document::DocumentId(id))); +} + +} + +static const int MESSAGE_BASE_LENGTH = 5; + +Messages52Test::Messages52Test() +{ + // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support + // version 5.2. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. + + putTest(DocumentProtocol::MESSAGE_PUTDOCUMENT, TEST_METHOD(Messages52Test::testPutDocumentMessage)); + putTest(DocumentProtocol::MESSAGE_REMOVEDOCUMENT, TEST_METHOD(Messages52Test::testRemoveDocumentMessage)); + putTest(DocumentProtocol::MESSAGE_UPDATEDOCUMENT, TEST_METHOD(Messages52Test::testUpdateDocumentMessage)); +} + +bool +Messages52Test::testPutDocumentMessage() +{ + auto doc = createDoc(getTypeRepo(), "testdoc", "doc:scheme:"); + PutDocumentMessage msg(doc); + + msg.setTimestamp(666); + msg.setCondition(TestAndSetCondition("There's just one condition")); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + + 41u + + serializedLength(msg.getCondition().getSelection()), + serialize("PutDocumentMessage", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + auto routableUp = deserialize("PutDocumentMessage", DocumentProtocol::MESSAGE_PUTDOCUMENT, lang); + if (EXPECT_TRUE(routableUp.get() != nullptr)) { + auto & deserializedMsg = static_cast<PutDocumentMessage &>(*routableUp); + + EXPECT_EQUAL(msg.getDocument()->getType().getName(), deserializedMsg.getDocument()->getType().getName()); + EXPECT_EQUAL(msg.getDocument()->getId().toString(), deserializedMsg.getDocument()->getId().toString()); + EXPECT_EQUAL(msg.getTimestamp(), deserializedMsg.getTimestamp()); + EXPECT_EQUAL(67u, deserializedMsg.getApproxSize()); + EXPECT_EQUAL(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); + } + } + + return true; +} + +bool +Messages52Test::testRemoveDocumentMessage() +{ + RemoveDocumentMessage msg(document::DocumentId("doc:scheme:")); + + msg.setCondition(TestAndSetCondition("There's just one condition")); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(16) + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + auto routablePtr = deserialize("RemoveDocumentMessage", DocumentProtocol::MESSAGE_REMOVEDOCUMENT, lang); + + if (EXPECT_TRUE(routablePtr.get() != nullptr)) { + auto & ref = static_cast<RemoveDocumentMessage &>(*routablePtr); + EXPECT_EQUAL(string("doc:scheme:"), ref.getDocumentId().toString()); + EXPECT_EQUAL(msg.getCondition().getSelection(), ref.getCondition().getSelection()); + } + } + return true; +} + +bool +Messages52Test::testUpdateDocumentMessage() +{ + const DocumentTypeRepo & repo = getTypeRepo(); + const document::DocumentType & docType = *repo.getDocumentType("testdoc"); + + document::DocumentUpdate::SP docUpdate(new document::DocumentUpdate(docType, + document::DocumentId("doc:scheme:"))); + + docUpdate->addFieldPathUpdate(document::FieldPathUpdate::CP( + new document::RemoveFieldPathUpdate(repo, docType, "intfield", "testdoc.intfield > 0"))); + + UpdateDocumentMessage msg(docUpdate); + msg.setOldTimestamp(666u); + msg.setNewTimestamp(777u); + msg.setCondition(TestAndSetCondition("There's just one condition")); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 89u + serializedLength(msg.getCondition().getSelection()), serialize("UpdateDocumentMessage", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + auto routableUp = deserialize("UpdateDocumentMessage", DocumentProtocol::MESSAGE_UPDATEDOCUMENT, lang); + + if (EXPECT_TRUE(routableUp.get() != nullptr)) { + auto & deserializedMsg = static_cast<UpdateDocumentMessage &>(*routableUp); + EXPECT_EQUAL(*msg.getDocumentUpdate(), *deserializedMsg.getDocumentUpdate()); + EXPECT_EQUAL(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp()); + EXPECT_EQUAL(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp()); + EXPECT_EQUAL(115u, deserializedMsg.getApproxSize()); + EXPECT_EQUAL(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); + } + } + return true; +} diff --git a/documentapi/src/tests/messages/messages52test.h b/documentapi/src/tests/messages/messages52test.h new file mode 100644 index 00000000000..71e3d54902c --- /dev/null +++ b/documentapi/src/tests/messages/messages52test.h @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell +#pragma once + +#include "messages51test.h" + +class Messages52Test : public Messages51Test { +protected: + const vespalib::Version getVersion() const override { return vespalib::Version(5, 115, 0); } + +public: + Messages52Test(); + + bool testPutDocumentMessage(); + bool testUpdateDocumentMessage(); + bool testRemoveDocumentMessage(); + +private: + static size_t serializedLength(const string & str) { + return sizeof(int32_t) + str.size(); + } +}; + diff --git a/documentapi/src/tests/messages/testbase.cpp b/documentapi/src/tests/messages/testbase.cpp new file mode 100644 index 00000000000..a6aeefd883f --- /dev/null +++ b/documentapi/src/tests/messages/testbase.cpp @@ -0,0 +1,197 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".testbase"); + +#include "testbase.h" +#include <vespa/document/config/config-documenttypes.h> +#include <vespa/document/repo/documenttyperepo.h> + +using document::DocumentTypeRepo; +using document::readDocumenttypesConfig; + +TestBase::TestBase() : + _repo(new DocumentTypeRepo(readDocumenttypesConfig("../../../test/cfg/testdoctypes.cfg"))), + _dataPath("../../../test/crosslanguagefiles"), + _loadTypes(), + _protocol(_loadTypes, _repo), + _tests() +{ + _loadTypes.addLoadType(34, "foo", Priority::PRI_NORMAL_2); +} + +int +TestBase::Main() +{ + TEST_INIT("messages_test"); + + // Retrieve version number to test for. + LOG(info, "Running tests for version %s.", getVersion().toString().c_str()); + + // Run registered tests. + for (std::map<uint32_t, TEST_METHOD_PT>::iterator it = _tests.begin(); + it != _tests.end(); ++it) + { + LOG(info, "Running test for routable type %d.", it->first); + EXPECT_TRUE( (this->*(it->second))() ); + TEST_FLUSH(); + } + + // Test routable type coverage. + std::vector<uint32_t> expected, actual; + EXPECT_TRUE(testCoverage(expected, actual)); + expected.push_back(0); + EXPECT_TRUE(!testCoverage(expected, actual)); + actual.push_back(1); + EXPECT_TRUE(!testCoverage(expected, actual)); + actual.push_back(0); + EXPECT_TRUE(!testCoverage(expected, actual)); + expected.push_back(1); + EXPECT_TRUE(testCoverage(expected, actual)); + + expected.clear(); + _protocol.getRoutableTypes(getVersion(), expected); + + actual.clear(); + for (std::map<uint32_t, TEST_METHOD_PT>::iterator it = _tests.begin(); + it != _tests.end(); ++it) + { + actual.push_back(it->first); + } + if (shouldTestCoverage()) { + EXPECT_TRUE(testCoverage(expected, actual, true)); + } + TEST_DONE(); +} + +TestBase & +TestBase::putTest(uint32_t type, TEST_METHOD_PT test) +{ + _tests[type] = test; + return *this; +} + +bool +TestBase::testCoverage(const std::vector<uint32_t> &expected, const std::vector<uint32_t> &actual, bool report) const +{ + bool ret = true; + + std::vector<uint32_t> lst(actual); + for (std::vector<uint32_t>::const_iterator it = expected.begin(); + it != expected.end(); ++it) + { + std::vector<uint32_t>::iterator occ = std::find(lst.begin(), lst.end(), *it); + if (occ == lst.end()) { + if (report) { + LOG(error, "Routable type %d is registered in DocumentProtocol but not tested.", *it); + } + ret = false; + } else { + lst.erase(occ); + } + } + if (!lst.empty()) { + if (report) { + for (std::vector<uint32_t>::iterator it = lst.begin(); + it != lst.end(); ++it) + { + LOG(error, "Routable type %d is tested but not registered in DocumentProtocol.", *it); + } + } + ret = false; + } + + return ret; +} + +uint32_t +TestBase::serialize(const string &filename, const mbus::Routable &routable) +{ + const vespalib::Version version = getVersion(); + string path = getPath(version.toString() + "-cpp-" + filename + ".dat"); + LOG(info, "Serializing to '%s'..", path.c_str()); + + mbus::Blob blob = _protocol.encode(version, routable); + if (!EXPECT_TRUE(writeFile(path, blob))) { + LOG(error, "Could not open file '%s' for writing.", path.c_str()); + return 0; + } + mbus::Routable::UP obj = _protocol.decode(version, blob); + if (!EXPECT_TRUE(obj.get() != NULL)) { + LOG(error, "Protocol failed to decode serialized data."); + return 0; + } + if (!EXPECT_TRUE(routable.getType() == obj->getType())) { + LOG(error, "Expected class %d, got %d.", routable.getType(), obj->getType()); + return 0; + } + return blob.size(); +} + +mbus::Routable::UP +TestBase::deserialize(const string &filename, uint32_t classId, uint32_t lang) +{ + const vespalib::Version version = getVersion(); + string path = getPath(version.toString() + (lang == LANG_JAVA ? "-java" : "-cpp") + "-" + filename + ".dat"); + LOG(info, "Deserializing from '%s'..", path.c_str()); + + mbus::Blob blob = readFile(path); + if (!EXPECT_TRUE(blob.size() != 0)) { + LOG(error, "Could not open file '%s' for reading.", path.c_str()); + return mbus::Routable::UP(); + } + mbus::Routable::UP ret = _protocol.decode(version, blob); + + if (!EXPECT_TRUE(ret.get())) { + LOG(error, "Unable to decode class %d", classId); + } else if (!EXPECT_TRUE(classId == ret->getType())) { + LOG(error, "Expected class %d, got %d.", classId, ret->getType()); + return mbus::Routable::UP(); + } + return ret; +} + +void +TestBase::dump(const mbus::Blob& blob) const +{ + fprintf(stderr, "[%ld]: ", blob.size()); + for(size_t i = 0; i < blob.size(); i++) { + if (blob.data()[i] > 32 && blob.data()[i] < 126) { + fprintf(stderr, "%c ", blob.data()[i]); + } + else { + fprintf(stderr, "%d ", blob.data()[i]); + } + } + fprintf(stderr, "\n"); +} + + +bool +TestBase::writeFile(const string &filename, const mbus::Blob& blob) const +{ + int file = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (file == -1) { + return false; + } + write(file, blob.data(), blob.size()); + close(file); + return true; +} + +mbus::Blob +TestBase::readFile(const string &filename) const +{ + int file = open(filename.c_str(), O_RDONLY); + int len = (file == -1) ? 0 : lseek(file, 0, SEEK_END); + mbus::Blob blob(len); + if (file != -1) { + lseek(file, 0, SEEK_SET); + read(file, blob.data(), len); + close(file); + } + + return blob; +} + + diff --git a/documentapi/src/tests/messages/testbase.h b/documentapi/src/tests/messages/testbase.h new file mode 100644 index 00000000000..5ebe154cb94 --- /dev/null +++ b/documentapi/src/tests/messages/testbase.h @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/messagebus/routable.h> +#include <vespa/vespalib/component/version.h> +#include <vespa/vespalib/testkit/testapp.h> + +using namespace documentapi; + +/** + * Declare the signature of the test method. + */ +class TestBase; +typedef bool (TestBase::*TEST_METHOD_PT)(); +#define TEST_METHOD(pt) ((TEST_METHOD_PT)&pt) + +/** + * This is the test base itself. It offers a set of utility functions that reflect on the version returned by + * the pure virtual getVersion() function. You need to inherit this and assign a version and a set of message + * tests to it. + */ +class TestBase : public vespalib::TestApp { + const document::DocumentTypeRepo::SP _repo; +protected: + const string _dataPath; + LoadTypeSet _loadTypes; + DocumentProtocol _protocol; + std::map<uint32_t, TEST_METHOD_PT> _tests; + + // Declares what languages share serialization. + enum { + LANG_CPP = 0, + LANG_JAVA, + NUM_LANGUAGES + }; + + TestBase(); + virtual ~TestBase() { /* empty */ } + virtual const vespalib::Version getVersion() const = 0; + virtual bool shouldTestCoverage() const = 0; + TestBase &putTest(uint32_t type, TEST_METHOD_PT test); + int Main(); + +public: + const document::DocumentTypeRepo &getTypeRepo() { return *_repo; } + const document::DocumentTypeRepo::SP &getTypeRepoSp() { return _repo; } + + bool testCoverage(const std::vector<uint32_t> &expected, const std::vector<uint32_t> &actual, bool report = false) const; + bool writeFile(const string &filename, const mbus::Blob& blob) const; + mbus::Blob readFile(const string &filename) const; + uint32_t serialize(const string &filename, const mbus::Routable &routable); + mbus::Routable::UP deserialize(const string &filename, uint32_t classId, uint32_t lang); + void dump(const mbus::Blob &blob) const; + + string getPath(const string &filename) const { return _dataPath + "/" + filename; } + mbus::Blob encode(const mbus::Routable &obj) const { return _protocol.encode(getVersion(), obj); } + mbus::Routable::UP decode(mbus::BlobRef data) const { return _protocol.decode(getVersion(), data); } +}; + diff --git a/documentapi/src/tests/policies/.gitignore b/documentapi/src/tests/policies/.gitignore new file mode 100644 index 00000000000..c92767a6536 --- /dev/null +++ b/documentapi/src/tests/policies/.gitignore @@ -0,0 +1,5 @@ +*_test +.depend +Makefile +log +documentapi_policies_test_app diff --git a/documentapi/src/tests/policies/CMakeLists.txt b/documentapi/src/tests/policies/CMakeLists.txt new file mode 100644 index 00000000000..dcf0f4bef4b --- /dev/null +++ b/documentapi/src/tests/policies/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(documentapi_policies_test_app + SOURCES + testframe.cpp + policies_test.cpp + DEPENDS + documentapi +) +vespa_add_test(NAME documentapi_policies_test_app COMMAND documentapi_policies_test_app) diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp new file mode 100644 index 00000000000..1cab15a325f --- /dev/null +++ b/documentapi/src/tests/policies/policies_test.cpp @@ -0,0 +1,1252 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("policies_test"); + +#include <vespa/document/config/config-documenttypes.h> +#include <vespa/document/datatype/datatype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/fieldvalue/longfieldvalue.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/documentapi/messagebus/policies/andpolicy.h> +#include <vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h> +#include <vespa/documentapi/messagebus/policies/errorpolicy.h> +#include <vespa/documentapi/messagebus/policies/externpolicy.h> +#include <vespa/documentapi/messagebus/policies/loadbalancerpolicy.h> +#include <vespa/documentapi/messagebus/policies/localservicepolicy.h> +#include <vespa/documentapi/messagebus/policies/roundrobinpolicy.h> +#include <vespa/documentapi/messagebus/policies/searchcolumnpolicy.h> +#include <vespa/documentapi/messagebus/policies/searchrowpolicy.h> +#include <vespa/documentapi/messagebus/policies/storagepolicy.h> +#include <vespa/documentapi/messagebus/policies/subsetservicepolicy.h> +#include <vespa/documentapi/messagebus/systemstate/systemstatehandle.h> +#include <limits> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/messagebus/testlib/testserver.h> +#include <vespa/vdslib/container/mutabledocumentlist.h> +#include <vespa/vespalib/testkit/testapp.h> +#include "testframe.h" + +using document::DataType; +using document::Document; +using document::DocumentId; +using document::DocumentTypeRepo; +using document::DocumentUpdate; +using document::readDocumenttypesConfig; +using slobrok::api::MirrorAPI; +using namespace documentapi; + +class Test : public vespalib::TestApp { +private: + LoadTypeSet _loadTypes; + DocumentTypeRepo::SP _repo; + const DataType *_docType; + +private: + bool trySelect(TestFrame &frame, uint32_t numSelects, const std::vector<string> &expected); + bool tryDistribution(TestFrame &frame, const string &id, const string &expected); + void tryWasFound(TestFrame &frame, uint32_t expectedRecipients, + uint32_t foundMask, bool expectedFound); + void setupExternPolicy(TestFrame &frame, mbus::Slobrok &slobrok, const string &pattern, + int32_t numEntries = -1); + StoragePolicy &setupStoragePolicy(TestFrame &frame, const string ¶m, + const string &pattern = "", int32_t numEntries = -1); + bool isErrorPolicy(const string &name, const string ¶m); + void assertMirrorReady(const slobrok::api::MirrorAPI &mirror); + void assertMirrorContains(const slobrok::api::MirrorAPI &mirror, const string &pattern, + uint32_t numEntries); + mbus::Message::UP newPutDocumentMessage(const string &documentId); + +public: + int Main(); + void testAND(); + void testDocumentRouteSelector(); + void testDocumentRouteSelectorIgnore(); + void testExternSend(); + void testExternMultipleSlobroks(); + void testLoadBalancer(); + void testLocalService(); + void testLocalServiceCache(); + void testProtocol(); + void testRoundRobin(); + void testRoundRobinCache(); + void testSearchColumn(); + void testSearchRow(); + void testSearchRowMerge(); + void multipleGetRepliesAreMergedToFoundDocument(); + void testSubsetService(); + void testSubsetServiceCache(); + + void requireThatExternPolicyWithIllegalParamIsAnErrorPolicy(); + void requireThatExternPolicyWithUnknownPatternSelectsNone(); + void requireThatExternPolicySelectsFromExternSlobrok(); + void requireThatExternPolicyMergesOneReplyAsProtocol(); + void requireThatStoragePolicyWithIllegalParamIsAnErrorPolicy(); + void requireThatStoragePolicyIsRandomWithoutState(); + void requireThatStoragePolicyIsTargetedWithState(); + void requireThatStoragePolicyCombinesSystemAndSlobrokState(); +}; + +TEST_APPHOOK(Test); + +int +Test::Main() { + TEST_INIT(_argv[0]); + + _repo.reset(new DocumentTypeRepo(readDocumenttypesConfig( + "../../../test/cfg/testdoctypes.cfg"))); + _docType = _repo->getDocumentType("testdoc"); + + testProtocol(); TEST_FLUSH(); + + testAND(); TEST_FLUSH(); + testDocumentRouteSelector(); TEST_FLUSH(); + testDocumentRouteSelectorIgnore(); TEST_FLUSH(); + testExternSend(); TEST_FLUSH(); + testExternMultipleSlobroks(); TEST_FLUSH(); + testLoadBalancer(); TEST_FLUSH(); + testLocalService(); TEST_FLUSH(); + testLocalServiceCache(); TEST_FLUSH(); + testRoundRobin(); TEST_FLUSH(); + testRoundRobinCache(); TEST_FLUSH(); + testSearchColumn(); TEST_FLUSH(); + testSearchRow(); TEST_FLUSH(); + testSearchRowMerge(); TEST_FLUSH(); + testSubsetService(); TEST_FLUSH(); + testSubsetServiceCache(); TEST_FLUSH(); + + multipleGetRepliesAreMergedToFoundDocument(); TEST_FLUSH(); + + requireThatExternPolicyWithIllegalParamIsAnErrorPolicy(); TEST_FLUSH(); + requireThatExternPolicyWithUnknownPatternSelectsNone(); TEST_FLUSH(); + requireThatExternPolicySelectsFromExternSlobrok(); TEST_FLUSH(); + requireThatExternPolicyMergesOneReplyAsProtocol(); TEST_FLUSH(); + + requireThatStoragePolicyWithIllegalParamIsAnErrorPolicy(); TEST_FLUSH(); + requireThatStoragePolicyIsRandomWithoutState(); TEST_FLUSH(); + requireThatStoragePolicyIsTargetedWithState(); TEST_FLUSH(); + requireThatStoragePolicyCombinesSystemAndSlobrokState(); TEST_FLUSH(); + + TEST_DONE(); +} + +void +Test::testProtocol() +{ + mbus::IProtocol::SP protocol(new DocumentProtocol(_loadTypes, _repo)); + + mbus::IRoutingPolicy::UP policy = protocol->createPolicy("AND", ""); + ASSERT_TRUE(dynamic_cast<ANDPolicy*>(policy.get()) != NULL); + + policy = protocol->createPolicy("DocumentRouteSelector", "raw:route[0]\n"); + ASSERT_TRUE(dynamic_cast<DocumentRouteSelectorPolicy*>(policy.get()) != NULL); + + policy = protocol->createPolicy("Extern", "foo;bar/baz"); + ASSERT_TRUE(dynamic_cast<ExternPolicy*>(policy.get()) != NULL); + + policy = protocol->createPolicy("LoadBalancer", + "cluster=docproc/cluster.default;" + "session=chain.default;syncinit"); + ASSERT_TRUE(dynamic_cast<LoadBalancerPolicy*>(policy.get()) != NULL); + + policy = protocol->createPolicy("LocalService", ""); + ASSERT_TRUE(dynamic_cast<LocalServicePolicy*>(policy.get()) != NULL); + + policy = protocol->createPolicy("RoundRobin", ""); + ASSERT_TRUE(dynamic_cast<RoundRobinPolicy*>(policy.get()) != NULL); + + policy = protocol->createPolicy("SearchRow", ""); + ASSERT_TRUE(dynamic_cast<SearchRowPolicy*>(policy.get()) != NULL); + + policy = protocol->createPolicy("SearchColumn", ""); + ASSERT_TRUE(dynamic_cast<SearchColumnPolicy*>(policy.get()) != NULL); + + policy = protocol->createPolicy("SubsetService", ""); + ASSERT_TRUE(dynamic_cast<SubsetServicePolicy*>(policy.get()) != NULL); +} + +void +Test::testAND() +{ + TestFrame frame(_repo); + frame.setMessage(mbus::Message::UP(new PutDocumentMessage( + document::Document::SP( + new document::Document(*_docType, + DocumentId("doc:scheme:")))))); + frame.setHop(mbus::HopSpec("test", "[AND]") + .addRecipient("foo") + .addRecipient("bar")); + EXPECT_TRUE(frame.testSelect(StringList().add("foo").add("bar"))); + + frame.setHop(mbus::HopSpec("test", "[AND:baz]") + .addRecipient("foo").addRecipient("bar")); + EXPECT_TRUE(frame.testSelect(StringList().add("baz"))); // param precedes recipients + + frame.setHop(mbus::HopSpec("test", "[AND:foo]")); + EXPECT_TRUE(frame.testMergeOneReply("foo")); + + frame.setHop(mbus::HopSpec("test", "[AND:foo bar]")); + EXPECT_TRUE(frame.testMergeTwoReplies("foo", "bar")); +} + +void +Test::requireThatExternPolicyWithIllegalParamIsAnErrorPolicy() +{ + mbus::Slobrok slobrok; + string spec = vespalib::make_string("tcp/localhost:%d", slobrok.port()); + + EXPECT_TRUE(isErrorPolicy("Extern", "")); + EXPECT_TRUE(isErrorPolicy("Extern", spec)); + EXPECT_TRUE(isErrorPolicy("Extern", spec + ";")); + EXPECT_TRUE(isErrorPolicy("Extern", spec + ";bar")); +} + +void +Test::requireThatExternPolicyWithUnknownPatternSelectsNone() +{ + TestFrame frame(_repo); + frame.setMessage(newPutDocumentMessage("doc:scheme:")); + + mbus::Slobrok slobrok; + setupExternPolicy(frame, slobrok, "foo/bar"); + EXPECT_TRUE(frame.testSelect(StringList())); +} + +void +Test::requireThatExternPolicySelectsFromExternSlobrok() +{ + TestFrame frame(_repo); + frame.setMessage(newPutDocumentMessage("doc:scheme:")); + mbus::Slobrok slobrok; + std::vector<mbus::TestServer*> servers; + for (uint32_t i = 0; i < 10; ++i) { + mbus::TestServer *server = new mbus::TestServer( + mbus::Identity(vespalib::make_string("docproc/cluster.default/%d", i)), + mbus::RoutingSpec(), slobrok, "", + mbus::IProtocol::SP(new DocumentProtocol(_loadTypes, _repo))); + servers.push_back(server); + server->net.registerSession("chain.default"); + } + setupExternPolicy(frame, slobrok, "docproc/cluster.default/*/chain.default", 10); + std::set<string> lst; + for (uint32_t i = 0; i < servers.size(); ++i) { + std::vector<mbus::RoutingNode*> leaf; + ASSERT_TRUE(frame.select(leaf, 1)); + lst.insert(leaf[0]->getRoute().toString()); + + leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + ASSERT_TRUE(frame.getReceptor().getReply(600).get() != NULL); + } + EXPECT_EQUAL(servers.size(), lst.size()); + for (uint32_t i = 0; i < servers.size(); ++i) { + delete servers[i]; + } +} + +void +Test::requireThatExternPolicyMergesOneReplyAsProtocol() +{ + TestFrame frame(_repo); + frame.setMessage(newPutDocumentMessage("doc:scheme:")); + mbus::Slobrok slobrok; + mbus::TestServer server(mbus::Identity("docproc/cluster.default/0"), + mbus::RoutingSpec(), slobrok, "", + mbus::IProtocol::SP(new DocumentProtocol(_loadTypes, _repo))); + server.net.registerSession("chain.default"); + setupExternPolicy(frame, slobrok, "docproc/cluster.default/0/chain.default", 1); + EXPECT_TRUE(frame.testMergeOneReply(server.net.getConnectionSpec() + "/chain.default")); +} + +mbus::Message::UP +Test::newPutDocumentMessage(const string &documentId) +{ + Document::SP doc(new Document(*_docType, DocumentId(documentId))); + return mbus::Message::UP(new PutDocumentMessage(doc)); +} + +void +Test::setupExternPolicy(TestFrame &frame, mbus::Slobrok &slobrok, const string &pattern, + int32_t numEntries) +{ + string param = vespalib::make_string("tcp/localhost:%d;%s", + slobrok.port(), pattern.c_str()); + frame.setHop(mbus::HopSpec("test", vespalib::make_string("[Extern:%s]", param.c_str()))); + mbus::MessageBus &mbus = frame.getMessageBus(); + const mbus::HopBlueprint *hop = mbus.getRoutingTable(DocumentProtocol::NAME)->getHop("test"); + const mbus::PolicyDirective dir = static_cast<mbus::PolicyDirective&>(*hop->getDirective(0)); + ExternPolicy &policy = static_cast<ExternPolicy&>(*mbus.getRoutingPolicy( + DocumentProtocol::NAME, + dir.getName(), + dir.getParam())); + assertMirrorReady(policy.getMirror()); + if (numEntries >= 0) { + assertMirrorContains(policy.getMirror(), pattern, numEntries); + } +} + +void +Test::assertMirrorReady(const slobrok::api::MirrorAPI &mirror) +{ + for (uint32_t i = 0; i < 6000; ++i) { + if (mirror.ready()) { + return; + } + FastOS_Thread::Sleep(10); + } + ASSERT_TRUE(false); +} + +void +Test::assertMirrorContains(const slobrok::api::MirrorAPI &mirror, const string &pattern, + uint32_t numEntries) +{ + for (uint32_t i = 0; i < 6000; ++i) { + if (mirror.lookup(pattern).size() == numEntries) { + return; + } + FastOS_Thread::Sleep(10); + } + ASSERT_TRUE(false); +} + +void +Test::testExternSend() +{ + // Setup local source node. + mbus::Slobrok local; + mbus::TestServer src(mbus::Identity("src"), mbus::RoutingSpec(), local, "", + mbus::IProtocol::SP( + new DocumentProtocol(_loadTypes, _repo))); + mbus::Receptor sr; + mbus::SourceSession::UP ss = src.mb.createSourceSession(sr, mbus::SourceSessionParams().setTimeout(60)); + + mbus::Slobrok slobrok; + mbus::TestServer itr(mbus::Identity("itr"), mbus::RoutingSpec() + .addTable(mbus::RoutingTableSpec(DocumentProtocol::NAME) + .addRoute(mbus::RouteSpec("default").addHop("dst")) + .addHop(mbus::HopSpec("dst", "dst/session"))), + slobrok, "", mbus::IProtocol::SP( + new DocumentProtocol(_loadTypes, _repo))); + mbus::Receptor ir; + mbus::IntermediateSession::UP is = itr.mb.createIntermediateSession("session", true, ir, ir); + + mbus::TestServer dst(mbus::Identity("dst"), mbus::RoutingSpec(), slobrok, "", + mbus::IProtocol::SP(new DocumentProtocol(_loadTypes, _repo))); + mbus::Receptor dr; + mbus::DestinationSession::UP ds = dst.mb.createDestinationSession("session", true, dr); + + // Send message from local node to remote cluster and resolve route there. + mbus::Message::UP msg(new GetDocumentMessage(document::DocumentId("doc:scheme:"), 0)); + msg->getTrace().setLevel(9); + msg->setRoute(mbus::Route::parse(vespalib::make_string("[Extern:tcp/localhost:%d;itr/session] default", slobrok.port()))); + + ASSERT_TRUE(ss->send(std::move(msg)).isAccepted()); + ASSERT_TRUE((msg = ir.getMessage(600)).get() != NULL); + is->forward(std::move(msg)); + ASSERT_TRUE((msg = dr.getMessage(600)).get() != NULL); + ds->acknowledge(std::move(msg)); + mbus::Reply::UP reply = ir.getReply(600); + ASSERT_TRUE(reply.get() != NULL); + is->forward(std::move(reply)); + ASSERT_TRUE((reply = sr.getReply(600)).get() != NULL); + + fprintf(stderr, "%s", reply->getTrace().toString().c_str()); +} + +void +Test::testExternMultipleSlobroks() +{ + mbus::Slobrok local; + mbus::TestServer src(mbus::Identity("src"), mbus::RoutingSpec(), local, "", + mbus::IProtocol::SP(new DocumentProtocol(_loadTypes, _repo))); + mbus::Receptor sr; + mbus::SourceSession::UP ss = src.mb.createSourceSession(sr, mbus::SourceSessionParams().setTimeout(60)); + + string spec; + mbus::Receptor dr; + { + mbus::Slobrok ext; + spec.append(vespalib::make_string("tcp/localhost:%d", ext.port())); + + mbus::TestServer dst(mbus::Identity("dst"), mbus::RoutingSpec(), ext, "", + mbus::IProtocol::SP(new DocumentProtocol(_loadTypes, _repo))); + mbus::DestinationSession::UP ds = dst.mb.createDestinationSession("session", true, dr); + + mbus::Message::UP msg(new GetDocumentMessage(document::DocumentId("doc:scheme:"), 0)); + msg->setRoute(mbus::Route::parse(vespalib::make_string("[Extern:%s;dst/session]", spec.c_str()))); + ASSERT_TRUE(ss->send(std::move(msg)).isAccepted()); + ASSERT_TRUE((msg = dr.getMessage(600)).get() != NULL); + ds->acknowledge(std::move(msg)); + mbus::Reply::UP reply = sr.getReply(600); + ASSERT_TRUE(reply.get() != NULL); + } + { + mbus::Slobrok ext; + spec.append(vespalib::make_string(",tcp/localhost:%d", ext.port())); + + mbus::TestServer dst(mbus::Identity("dst"), mbus::RoutingSpec(), ext, "", + mbus::IProtocol::SP(new DocumentProtocol(_loadTypes, _repo))); + mbus::DestinationSession::UP ds = dst.mb.createDestinationSession("session", true, dr); + + mbus::Message::UP msg(new GetDocumentMessage(document::DocumentId("doc:scheme:"), 0)); + msg->setRoute(mbus::Route::parse(vespalib::make_string("[Extern:%s;dst/session]", spec.c_str()))); + ASSERT_TRUE(ss->send(std::move(msg)).isAccepted()); + ASSERT_TRUE((msg = dr.getMessage(600)).get() != NULL); + ds->acknowledge(std::move(msg)); + mbus::Reply::UP reply = sr.getReply(600); + ASSERT_TRUE(reply.get() != NULL); + } +} + +void +Test::testLocalService() +{ + // Prepare message. + TestFrame frame(_repo, "docproc/cluster.default"); + frame.setMessage(mbus::Message::UP(new PutDocumentMessage(Document::SP( + new Document(*_docType, + DocumentId("doc:scheme:")))))); + + // Test select with proper address. + for (uint32_t i = 0; i < 10; ++i) { + frame.getNetwork().registerSession(vespalib::make_string("%d/chain.default", i)); + } + ASSERT_TRUE(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10)); + frame.setHop(mbus::HopSpec("test", "docproc/cluster.default/[LocalService]/chain.default")); + + std::set<string> lst; + for (uint32_t i = 0; i < 10; ++i) { + std::vector<mbus::RoutingNode*> leaf; + ASSERT_TRUE(frame.select(leaf, 1)); + lst.insert(leaf[0]->getRoute().toString()); + + leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + ASSERT_TRUE(frame.getReceptor().getReply(600).get() != NULL); + } + EXPECT_EQUAL(10u, lst.size()); + + // Test select with broken address. + lst.clear(); + frame.setHop(mbus::HopSpec("test", "docproc/cluster.default/[LocalService:broken]/chain.default")); + for (uint32_t i = 0; i < 10; ++i) { + std::vector<mbus::RoutingNode*> leaf; + ASSERT_TRUE(frame.select(leaf, 1)); + lst.insert(leaf[0]->getRoute().toString()); + + leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + ASSERT_TRUE(frame.getReceptor().getReply(600).get() != NULL); + } + EXPECT_EQUAL(1u, lst.size()); + EXPECT_EQUAL("docproc/cluster.default/*/chain.default", *lst.begin()); + + // Test merge behavior. + frame.setHop(mbus::HopSpec("test", "[LocalService]")); + EXPECT_TRUE(frame.testMergeOneReply("*")); +} + +void +Test::testLocalServiceCache() +{ + TestFrame fooFrame(_repo, "docproc/cluster.default"); + mbus::HopSpec fooHop("foo", "docproc/cluster.default/[LocalService]/chain.foo"); + fooFrame.setMessage(mbus::Message::UP(new GetDocumentMessage(document::DocumentId("doc:scheme:foo")))); + fooFrame.setHop(fooHop); + + TestFrame barFrame(fooFrame); + mbus::HopSpec barHop("test", "docproc/cluster.default/[LocalService]/chain.bar"); + barFrame.setMessage(mbus::Message::UP(new GetDocumentMessage(document::DocumentId("doc:scheme:bar")))); + barFrame.setHop(barHop); + + fooFrame.getMessageBus().setupRouting( + mbus::RoutingSpec().addTable(mbus::RoutingTableSpec(DocumentProtocol::NAME) + .addHop(fooHop) + .addHop(barHop))); + + fooFrame.getNetwork().registerSession("0/chain.foo"); + fooFrame.getNetwork().registerSession("0/chain.bar"); + ASSERT_TRUE(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2)); + + std::vector<mbus::RoutingNode*> fooSelected; + ASSERT_TRUE(fooFrame.select(fooSelected, 1)); + EXPECT_EQUAL("docproc/cluster.default/0/chain.foo", fooSelected[0]->getRoute().getHop(0).toString()); + + std::vector<mbus::RoutingNode*> barSelected; + ASSERT_TRUE(barFrame.select(barSelected, 1)); + EXPECT_EQUAL("docproc/cluster.default/0/chain.bar", barSelected[0]->getRoute().getHop(0).toString()); + + barSelected[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + fooSelected[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + + ASSERT_TRUE(barFrame.getReceptor().getReply(600).get() != NULL); + ASSERT_TRUE(fooFrame.getReceptor().getReply(600).get() != NULL); +} + +void +Test::testRoundRobin() +{ + // Prepare message. + TestFrame frame(_repo, "docproc/cluster.default"); + frame.setMessage(mbus::Message::UP(new PutDocumentMessage(Document::SP( + new Document(*_docType, + DocumentId("doc:scheme:")))))); + + // Test select with proper address. + for (uint32_t i = 0; i < 10; ++i) { + frame.getNetwork().registerSession(vespalib::make_string("%d/chain.default", i)); + } + ASSERT_TRUE(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10)); + frame.setHop(mbus::HopSpec("test", "[RoundRobin]") + .addRecipient("docproc/cluster.default/3/chain.default") + .addRecipient("docproc/cluster.default/6/chain.default") + .addRecipient("docproc/cluster.default/9/chain.default")); + EXPECT_TRUE(trySelect(frame, 32, StringList() + .add("docproc/cluster.default/3/chain.default") + .add("docproc/cluster.default/6/chain.default") + .add("docproc/cluster.default/9/chain.default"))); + frame.getNetwork().unregisterSession("6/chain.default"); + ASSERT_TRUE(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 9)); + EXPECT_TRUE(trySelect(frame, 32, StringList() + .add("docproc/cluster.default/3/chain.default") + .add("docproc/cluster.default/9/chain.default"))); + frame.getNetwork().unregisterSession("3/chain.default"); + ASSERT_TRUE(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 8)); + EXPECT_TRUE(trySelect(frame, 32, StringList() + .add("docproc/cluster.default/9/chain.default"))); + frame.getNetwork().unregisterSession("9/chain.default"); + ASSERT_TRUE(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 7)); + EXPECT_TRUE(trySelect(frame, 32, StringList())); + + // Test merge behavior. + frame.setHop(mbus::HopSpec("test", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.default")); + EXPECT_TRUE(frame.testMergeOneReply("docproc/cluster.default/0/chain.default")); +} + +void +Test::testRoundRobinCache() +{ + TestFrame fooFrame(_repo, "docproc/cluster.default"); + mbus::HopSpec fooHop("foo", "[RoundRobin]"); + fooHop.addRecipient("docproc/cluster.default/0/chain.foo"); + fooFrame.setMessage(mbus::Message::UP(new GetDocumentMessage(document::DocumentId("doc:scheme:foo")))); + fooFrame.setHop(fooHop); + + TestFrame barFrame(fooFrame); + mbus::HopSpec barHop("bar", "[RoundRobin]"); + barHop.addRecipient("docproc/cluster.default/0/chain.bar"); + barFrame.setMessage(mbus::Message::UP(new GetDocumentMessage(document::DocumentId("doc:scheme:bar")))); + barFrame.setHop(barHop); + + fooFrame.getMessageBus().setupRouting( + mbus::RoutingSpec().addTable(mbus::RoutingTableSpec(DocumentProtocol::NAME) + .addHop(fooHop) + .addHop(barHop))); + + fooFrame.getNetwork().registerSession("0/chain.foo"); + fooFrame.getNetwork().registerSession("0/chain.bar"); + ASSERT_TRUE(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2)); + + std::vector<mbus::RoutingNode*> fooSelected; + ASSERT_TRUE(fooFrame.select(fooSelected, 1)); + EXPECT_EQUAL("docproc/cluster.default/0/chain.foo", fooSelected[0]->getRoute().getHop(0).toString()); + + std::vector<mbus::RoutingNode*> barSelected; + ASSERT_TRUE(barFrame.select(barSelected, 1)); + EXPECT_EQUAL("docproc/cluster.default/0/chain.bar", barSelected[0]->getRoute().getHop(0).toString()); + + barSelected[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + fooSelected[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + + ASSERT_TRUE(barFrame.getReceptor().getReply(600).get() != NULL); + ASSERT_TRUE(fooFrame.getReceptor().getReply(600).get() != NULL); +} + +void +Test::testSearchRow() +{ + TestFrame frame(_repo); + frame.setMessage(mbus::Message::UP(new PutDocumentMessage(Document::SP( + new Document(*_docType, + DocumentId("doc:scheme:")))))); + frame.setHop(mbus::HopSpec("test", "[SearchRow]") + .addRecipient("foo")); + EXPECT_TRUE(frame.testMergeOneReply("foo")); + frame.setHop(mbus::HopSpec("test", "[SearchRow]") + .addRecipient("foo") + .addRecipient("bar")); + EXPECT_TRUE(frame.testMergeTwoReplies("foo", "bar")); + + frame.setHop(mbus::HopSpec("test", "[SearchRow:1]") + .addRecipient("foo")); + TestFrame::ReplyMap replies; + replies["foo"] = mbus::ErrorCode::SERVICE_OOS; + EXPECT_TRUE(frame.testMergeError(replies, UIntList().add(mbus::ErrorCode::SERVICE_OOS))); + + frame.setHop(mbus::HopSpec("test", "[SearchRow:1]") + .addRecipient("foo") + .addRecipient("bar")); + replies["foo"] = mbus::ErrorCode::SERVICE_OOS; + replies["bar"] = mbus::ErrorCode::NONE; + EXPECT_TRUE(frame.testMergeOk(replies, StringList().add("bar"))); + + replies["foo"] = mbus::ErrorCode::SERVICE_OOS; + replies["bar"] = mbus::ErrorCode::SERVICE_OOS; + EXPECT_TRUE(frame.testMergeError(replies, UIntList() + .add(mbus::ErrorCode::SERVICE_OOS) + .add(mbus::ErrorCode::SERVICE_OOS))); + + frame.setHop(mbus::HopSpec("test", "[SearchRow:1]") + .addRecipient("foo") + .addRecipient("bar") + .addRecipient("baz")); + replies["foo"] = mbus::ErrorCode::SERVICE_OOS; + replies["bar"] = mbus::ErrorCode::NONE; + replies["baz"] = mbus::ErrorCode::NONE; + EXPECT_TRUE(frame.testMergeOk(replies, StringList().add("bar").add("baz"))); + + replies["foo"] = mbus::ErrorCode::SERVICE_OOS; + replies["bar"] = mbus::ErrorCode::SERVICE_OOS; + replies["baz"] = mbus::ErrorCode::NONE; + EXPECT_TRUE(frame.testMergeOk(replies, StringList().add("baz"))); + + replies["foo"] = mbus::ErrorCode::SERVICE_OOS; + replies["bar"] = mbus::ErrorCode::SERVICE_OOS; + replies["baz"] = mbus::ErrorCode::SERVICE_OOS; + EXPECT_TRUE(frame.testMergeError(replies, UIntList() + .add(mbus::ErrorCode::SERVICE_OOS) + .add(mbus::ErrorCode::SERVICE_OOS) + .add(mbus::ErrorCode::SERVICE_OOS))); + + frame.setHop(mbus::HopSpec("test", "[SearchRow:2]") + .addRecipient("foo") + .addRecipient("bar") + .addRecipient("baz")); + replies["foo"] = mbus::ErrorCode::SERVICE_OOS; + replies["bar"] = mbus::ErrorCode::NONE; + replies["baz"] = mbus::ErrorCode::NONE; + EXPECT_TRUE(frame.testMergeOk(replies, StringList().add("bar").add("baz"))); + + replies["foo"] = mbus::ErrorCode::SERVICE_OOS; + replies["bar"] = mbus::ErrorCode::SERVICE_OOS; + replies["baz"] = mbus::ErrorCode::NONE; + EXPECT_TRUE(frame.testMergeError(replies, UIntList() + .add(mbus::ErrorCode::SERVICE_OOS) + .add(mbus::ErrorCode::SERVICE_OOS))); + + replies["foo"] = mbus::ErrorCode::SERVICE_OOS; + replies["bar"] = mbus::ErrorCode::SERVICE_OOS; + replies["baz"] = mbus::ErrorCode::SERVICE_OOS; + EXPECT_TRUE(frame.testMergeError(replies, UIntList() + .add(mbus::ErrorCode::SERVICE_OOS) + .add(mbus::ErrorCode::SERVICE_OOS) + .add(mbus::ErrorCode::SERVICE_OOS))); +} + +void +Test::testSearchRowMerge() +{ + TestFrame frame(_repo); + frame.setHop(mbus::HopSpec("test", "[SearchRow]") + .addRecipient("foo")); + tryWasFound(frame, 1, 0x0, false); + tryWasFound(frame, 1, 0x1, true); + + frame.setHop(mbus::HopSpec("test", "[SearchRow]") + .addRecipient("foo") + .addRecipient("bar")); + tryWasFound(frame, 2, 0x0, false); + tryWasFound(frame, 2, 0x1, true); + tryWasFound(frame, 2, 0x2, true); + tryWasFound(frame, 2, 0x3, true); + + frame.setHop(mbus::HopSpec("test", "[SearchRow]") + .addRecipient("foo") + .addRecipient("bar") + .addRecipient("baz")); + tryWasFound(frame, 3, 0x0, false); + tryWasFound(frame, 3, 0x1, true); + tryWasFound(frame, 3, 0x2, true); + tryWasFound(frame, 3, 0x3, true); + tryWasFound(frame, 3, 0x4, true); + tryWasFound(frame, 3, 0x5, true); + tryWasFound(frame, 3, 0x6, true); + tryWasFound(frame, 3, 0x7, true); +} + +void +Test::tryWasFound(TestFrame &frame, uint32_t expectedRecipients, + uint32_t foundMask, bool expectedFound) +{ + { + frame.setMessage(mbus::Message::UP(new RemoveDocumentMessage(DocumentId("doc:scheme:69")))); + std::vector<mbus::RoutingNode*> selected; + EXPECT_TRUE(frame.select(selected, expectedRecipients)); + for (uint32_t i = 0, len = selected.size(); i < len; ++i) { + mbus::Reply::UP reply(new RemoveDocumentReply()); + static_cast<RemoveDocumentReply&>(*reply).setWasFound((1 << i) & foundMask); + selected[i]->handleReply(std::move(reply)); + } + mbus::Reply::UP reply = frame.getReceptor().getReply(600); + EXPECT_TRUE(reply.get() != NULL); + EXPECT_EQUAL((uint32_t)DocumentProtocol::REPLY_REMOVEDOCUMENT, reply->getType()); + EXPECT_EQUAL(expectedFound, static_cast<RemoveDocumentReply&>(*reply).wasFound()); + } + { + DocumentUpdate::SP upd(new DocumentUpdate(*_docType, DocumentId("doc:scheme:"))); + frame.setMessage(mbus::Message::UP(new UpdateDocumentMessage(upd))); + std::vector<mbus::RoutingNode*> selected; + EXPECT_TRUE(frame.select(selected, expectedRecipients)); + for (uint32_t i = 0, len = selected.size(); i < len; ++i) { + mbus::Reply::UP reply(new UpdateDocumentReply()); + static_cast<UpdateDocumentReply&>(*reply).setWasFound((1 << i) & foundMask); + selected[i]->handleReply(std::move(reply)); + } + mbus::Reply::UP reply = frame.getReceptor().getReply(600); + EXPECT_TRUE(reply.get() != NULL); + EXPECT_EQUAL((uint32_t)DocumentProtocol::REPLY_UPDATEDOCUMENT, reply->getType()); + EXPECT_EQUAL(expectedFound, static_cast<UpdateDocumentReply&>(*reply).wasFound()); + } +} + +void +Test::multipleGetRepliesAreMergedToFoundDocument() +{ + TestFrame frame(_repo); + frame.setHop(mbus::HopSpec("test", "[DocumentRouteSelector:raw:" + "route[2]\n" + "route[0].name \"foo\"\n" + "route[0].selector \"testdoc\"\n" + "route[0].feed \"myfeed\"\n" + "route[1].name \"bar\"\n" + "route[1].selector \"other\"\n" + "route[1].feed \"myfeed\"\n]") + .addRecipient("foo") + .addRecipient("bar")); + frame.setMessage(mbus::Message::UP(new GetDocumentMessage(DocumentId("doc:scheme:yarn")))); + std::vector<mbus::RoutingNode*> selected; + EXPECT_TRUE(frame.select(selected, 2)); + for (uint32_t i = 0, len = selected.size(); i < len; ++i) { + document::Document::SP doc; + if (i == 0) { + doc.reset(new Document(*_docType, DocumentId("doc:scheme:yarn"))); + doc->setLastModified(123456ULL); + } + mbus::Reply::UP reply(new GetDocumentReply(doc)); + selected[i]->handleReply(std::move(reply)); + } + mbus::Reply::UP reply = frame.getReceptor().getReply(600); + EXPECT_TRUE(reply.get() != NULL); + EXPECT_EQUAL(static_cast<uint32_t>(DocumentProtocol::REPLY_GETDOCUMENT), + reply->getType()); + EXPECT_EQUAL(123456ULL, static_cast<GetDocumentReply&>(*reply).getLastModified()); +} + +void +Test::testSearchColumn() +{ + TestFrame frame(_repo); + frame.setHop(mbus::HopSpec("test", "[SearchColumn]") + .addRecipient("c0") + .addRecipient("c1") + .addRecipient("c2") + .addRecipient("c3")); + + // Test hash distribution. + EXPECT_TRUE(tryDistribution(frame, "doc:ns:3", "c0")); + EXPECT_TRUE(tryDistribution(frame, "doc:ns:18", "c1")); + EXPECT_TRUE(tryDistribution(frame, "doc:ns:0", "c2")); + EXPECT_TRUE(tryDistribution(frame, "doc:ns:4", "c3")); + + EXPECT_TRUE(tryDistribution(frame, "userdoc:ns:49152:0", "c0")); + EXPECT_TRUE(tryDistribution(frame, "userdoc:ns:49152:1", "c0")); + EXPECT_TRUE(tryDistribution(frame, "userdoc:ns:16384:2", "c1")); + EXPECT_TRUE(tryDistribution(frame, "userdoc:ns:16384:3", "c1")); + EXPECT_TRUE(tryDistribution(frame, "userdoc:ns:5461:4", "c2")); + EXPECT_TRUE(tryDistribution(frame, "userdoc:ns:5461:5", "c2")); + EXPECT_TRUE(tryDistribution(frame, "userdoc:ns:0:6", "c3")); + EXPECT_TRUE(tryDistribution(frame, "userdoc:ns:0:7", "c3")); + + EXPECT_TRUE(tryDistribution(frame, "groupdoc:ns:0:0", "c0")); + EXPECT_TRUE(tryDistribution(frame, "groupdoc:ns:0:1", "c0")); + EXPECT_TRUE(tryDistribution(frame, "groupdoc:ns:4:2", "c1")); + EXPECT_TRUE(tryDistribution(frame, "groupdoc:ns:4:3", "c1")); + EXPECT_TRUE(tryDistribution(frame, "groupdoc:ns:2:4", "c2")); + EXPECT_TRUE(tryDistribution(frame, "groupdoc:ns:2:5", "c2")); + EXPECT_TRUE(tryDistribution(frame, "groupdoc:ns:7:6", "c3")); + EXPECT_TRUE(tryDistribution(frame, "groupdoc:ns:7:7", "c3")); + + // Test routing based on message type. + mbus::Message::UP put(new PutDocumentMessage(Document::SP( + new Document(*_docType, + DocumentId("doc:scheme:"))))); +} + +bool +Test::tryDistribution(TestFrame &frame, const string &id, const string &expected) +{ + Document::SP doc(new Document(*_docType, DocumentId(id))); + mbus::Message::UP msg(new PutDocumentMessage(doc)); + frame.setMessage(std::move(msg)); + return frame.testSelect(StringList().add(expected)); +} + +void +Test::testDocumentRouteSelector() +{ + // Test policy with usage safeguard. + string okConfig = "raw:route[0]\n"; + string errConfig = "raw:" + "route[1]\n" + "route[0].name \"foo\"\n" + "route[0].selector \"foo bar\"\n" + "route[0].feed \"baz\"\n"; + { + DocumentProtocol protocol(_loadTypes, _repo, okConfig); + EXPECT_TRUE(dynamic_cast<DocumentRouteSelectorPolicy*>(protocol.createPolicy("DocumentRouteSelector", "").get()) != NULL); + EXPECT_TRUE(dynamic_cast<ErrorPolicy*>(protocol.createPolicy("DocumentRouteSelector", errConfig).get()) != NULL); + } + { + DocumentProtocol protocol(_loadTypes, _repo, errConfig); + EXPECT_TRUE(dynamic_cast<ErrorPolicy*>(protocol.createPolicy("DocumentRouteSelector", "").get()) != NULL); + EXPECT_TRUE(dynamic_cast<DocumentRouteSelectorPolicy*>(protocol.createPolicy("DocumentRouteSelector", okConfig).get()) != NULL); + } + + // Test policy with proper config. + TestFrame frame(_repo); + frame.setHop(mbus::HopSpec("test", "[DocumentRouteSelector:raw:" + "route[2]\n" + "route[0].name \"foo\"\n" + "route[0].selector \"testdoc\"\n" + "route[0].feed \"myfeed\"\n" + "route[1].name \"bar\"\n" + "route[1].selector \"other\"\n" + "route[1].feed \"myfeed\"\n]") + .addRecipient("foo") + .addRecipient("bar")); + + frame.setMessage(mbus::Message::UP(new GetDocumentMessage(document::DocumentId("doc:scheme:"), 0))); + EXPECT_TRUE(frame.testSelect(StringList().add("foo").add("bar"))); + + mbus::Message::UP put(new PutDocumentMessage(Document::SP( + new Document(*_docType, + DocumentId("doc:scheme:"))))); + frame.setMessage(std::move(put)); + EXPECT_TRUE(frame.testSelect( StringList().add("foo"))); + + { + vdslib::OperationList opList; + + document::DocumentId id("doc:scheme:"); + Document::UP doc(new Document(*_docType, id)); + opList.addPut(std::move(doc)); + + document::BucketIdFactory factory; + put = frame.setMessage(MultiOperationMessage::create(_repo, factory.getBucketId(id), opList)); + EXPECT_TRUE(frame.testSelect(StringList().add("foo"))); + } + + { + vdslib::OperationList opList; + document::DocumentId id("doc:scheme:"); + Document::UP doc(new Document(*_repo->getDocumentType("other"), id)); + opList.addPut(std::move(doc)); + + document::BucketIdFactory factory; + put = frame.setMessage(MultiOperationMessage::create(_repo, + factory.getBucketId(id), opList)); + EXPECT_TRUE(frame.testSelect(StringList().add("bar"))); + } + + frame.setMessage(mbus::Message::UP(new RemoveDocumentMessage(document::DocumentId("doc:scheme:")))); + EXPECT_TRUE(frame.testSelect(StringList().add("foo").add("bar"))); + + frame.setMessage(mbus::Message::UP(new UpdateDocumentMessage( + document::DocumentUpdate::SP( + new document::DocumentUpdate( + *_docType, + DocumentId("doc:scheme:")))))); + EXPECT_TRUE(frame.testSelect(StringList().add("foo"))); + + frame.setMessage(std::move(put)); + EXPECT_TRUE(frame.testMergeOneReply("foo")); +} + +void +Test::testDocumentRouteSelectorIgnore() +{ + TestFrame frame(_repo); + frame.setHop(mbus::HopSpec("test", "[DocumentRouteSelector:raw:" + "route[1]\n" + "route[0].name \"docproc/cluster.foo\"\n" + "route[0].selector \"testdoc and testdoc.stringfield == 'foo'\"\n" + "route[0].feed \"myfeed\"\n]") + .addRecipient("docproc/cluster.foo")); + + frame.setMessage(mbus::Message::UP(new PutDocumentMessage( + document::Document::SP( + new document::Document(*_docType, + DocumentId("id:yarn:testdoc:n=1234:fluff")))))); + std::vector<mbus::RoutingNode*> leaf; + ASSERT_TRUE(frame.select(leaf, 0)); + mbus::Reply::UP reply = frame.getReceptor().getReply(600); + ASSERT_TRUE(reply.get() != NULL); + EXPECT_EQUAL(uint32_t(DocumentProtocol::REPLY_DOCUMENTIGNORED), reply->getType()); + EXPECT_EQUAL(0u, reply->getNumErrors()); + + frame.setMessage(mbus::Message::UP(new UpdateDocumentMessage( + document::DocumentUpdate::SP( + new document::DocumentUpdate( + *_docType, + DocumentId("doc:scheme:")))))); + EXPECT_TRUE(frame.testSelect(StringList().add("docproc/cluster.foo"))); +} + +namespace { + string getDefaultDistributionConfig( + uint16_t redundancy = 2, uint16_t nodeCount = 10, + vespa::config::content::StorDistributionConfig::DiskDistribution distr + = vespa::config::content::StorDistributionConfig::MODULO_BID) + { + std::ostringstream ost; + ost << "raw:redundancy " << redundancy << "\n" + << "group[1]\n" + << "group[0].index \"invalid\"\n" + << "group[0].name \"invalid\"\n" + << "group[0].partitions \"*\"\n" + << "group[0].nodes[" << nodeCount << "]\n"; + for (uint16_t i=0; i<nodeCount; ++i) { + ost << "group[0].nodes[" << i << "].index " << i << "\n"; + } + ost << "disk_distribution " + << vespa::config::content::StorDistributionConfig::getDiskDistributionName(distr) + << "\n"; + return ost.str(); + } +} + +void Test::testLoadBalancer() { + LoadBalancer lb("foo", ""); + + MirrorAPI::SpecList entries; + entries.push_back(MirrorAPI::Spec("foo/0/default", "tcp/bar:1")); + entries.push_back(MirrorAPI::Spec("foo/1/default", "tcp/bar:2")); + entries.push_back(MirrorAPI::Spec("foo/2/default", "tcp/bar:3")); + + const std::vector<LoadBalancer::NodeInfo>& nodeInfo = lb.getNodeInfo(); + + for (int i = 0; i < 99; i++) { + std::pair<string, int> recipient = lb.getRecipient(entries); + EXPECT_EQUAL((i % 3), recipient.second); + } + + // Simulate that one node is overloaded. It returns busy twice as often as the others. + for (int i = 0; i < 100; i++) { + lb.received(0, true); + lb.received(0, false); + lb.received(0, false); + lb.received(2, true); + lb.received(2, false); + lb.received(2, false); + lb.received(1, true); + lb.received(1, true); + lb.received(1, false); + } + + EXPECT_EQUAL(421, (int)(100 * nodeInfo[0].weight / nodeInfo[1].weight)); + EXPECT_EQUAL(421, (int)(100 * nodeInfo[2].weight / nodeInfo[1].weight)); + + EXPECT_EQUAL(0 , lb.getRecipient(entries).second); + EXPECT_EQUAL(0 , lb.getRecipient(entries).second); + EXPECT_EQUAL(1 , lb.getRecipient(entries).second); + EXPECT_EQUAL(2 , lb.getRecipient(entries).second); + EXPECT_EQUAL(2 , lb.getRecipient(entries).second); + EXPECT_EQUAL(2 , lb.getRecipient(entries).second); + EXPECT_EQUAL(2 , lb.getRecipient(entries).second); + EXPECT_EQUAL(0 , lb.getRecipient(entries).second); + EXPECT_EQUAL(0 , lb.getRecipient(entries).second); + EXPECT_EQUAL(0 , lb.getRecipient(entries).second); +} + +void +Test::requireThatStoragePolicyWithIllegalParamIsAnErrorPolicy() +{ + EXPECT_TRUE(isErrorPolicy("Storage", "")); + EXPECT_TRUE(isErrorPolicy("Storage", "config=foo;slobroks=foo")); + EXPECT_TRUE(isErrorPolicy("Storage", "slobroks=foo")); +} + +void +Test::requireThatStoragePolicyIsRandomWithoutState() +{ + TestFrame frame(_repo); + frame.setMessage(newPutDocumentMessage("doc:scheme:")); + + mbus::Slobrok slobrok; + std::vector<mbus::TestServer*> servers; + for (uint32_t i = 0; i < 5; ++i) { + mbus::TestServer *srv = new mbus::TestServer( + mbus::Identity(vespalib::make_string("storage/cluster.mycluster/distributor/%d", i)), + mbus::RoutingSpec(), slobrok, "", + mbus::IProtocol::SP(new DocumentProtocol(_loadTypes, _repo))); + servers.push_back(srv); + srv->net.registerSession("default"); + } + string param = vespalib::make_string( + "cluster=mycluster;slobroks=tcp/localhost:%d;clusterconfigid=%s;syncinit", + slobrok.port(), getDefaultDistributionConfig(2, 5).c_str()); + StoragePolicy &policy = setupStoragePolicy( + frame, param, + "storage/cluster.mycluster/distributor/*/default", 5); + ASSERT_TRUE(policy.getSystemState() == NULL); + + std::set<string> lst; + for (uint32_t i = 0; i < 666; i++) { + std::vector<mbus::RoutingNode*> leaf; + ASSERT_TRUE(frame.select(leaf, 1)); + lst.insert(leaf[0]->getRoute().toString()); + leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + } + EXPECT_EQUAL(servers.size(), lst.size()); + for (uint32_t i = 0; i < servers.size(); ++i) { + delete servers[i]; + } +} + +StoragePolicy & +Test::setupStoragePolicy(TestFrame &frame, const string ¶m, + const string &pattern, int32_t numEntries) +{ + frame.setHop(mbus::HopSpec("test", vespalib::make_string("[Storage:%s]", param.c_str()))); + mbus::MessageBus &mbus = frame.getMessageBus(); + const mbus::HopBlueprint *hop = mbus.getRoutingTable(DocumentProtocol::NAME)->getHop("test"); + const mbus::PolicyDirective dir = static_cast<mbus::PolicyDirective&>(*hop->getDirective(0)); + StoragePolicy &policy = static_cast<StoragePolicy&>(*mbus.getRoutingPolicy( + DocumentProtocol::NAME, + dir.getName(), + dir.getParam())); + policy.initSynchronous(); + assertMirrorReady(*policy.getMirror()); + if (numEntries >= 0) { + assertMirrorContains(*policy.getMirror(), pattern, numEntries); + } + return policy; +} + +void +Test::requireThatStoragePolicyIsTargetedWithState() +{ + TestFrame frame(_repo); + frame.setMessage(newPutDocumentMessage("doc:scheme:")); + + mbus::Slobrok slobrok; + std::vector<mbus::TestServer*> servers; + for (uint32_t i = 0; i < 5; ++i) { + mbus::TestServer *srv = new mbus::TestServer( + mbus::Identity(vespalib::make_string("storage/cluster.mycluster/distributor/%d", i)), + mbus::RoutingSpec(), slobrok, "", + mbus::IProtocol::SP(new DocumentProtocol(_loadTypes, _repo))); + servers.push_back(srv); + srv->net.registerSession("default"); + } + string param = vespalib::make_string( + "cluster=mycluster;slobroks=tcp/localhost:%d;clusterconfigid=%s;syncinit", + slobrok.port(), getDefaultDistributionConfig(2, 5).c_str()); + StoragePolicy &policy = setupStoragePolicy( + frame, param, + "storage/cluster.mycluster/distributor/*/default", 5); + ASSERT_TRUE(policy.getSystemState() == NULL); + { + std::vector<mbus::RoutingNode*> leaf; + ASSERT_TRUE(frame.select(leaf, 1)); + leaf[0]->handleReply(mbus::Reply::UP(new WrongDistributionReply("distributor:5 storage:5"))); + ASSERT_TRUE(policy.getSystemState() != NULL); + EXPECT_EQUAL(policy.getSystemState()->toString(), "distributor:5 storage:5"); + } + std::set<string> lst; + for (int i = 0; i < 666; i++) { + std::vector<mbus::RoutingNode*> leaf; + ASSERT_TRUE(frame.select(leaf, 1)); + lst.insert(leaf[0]->getRoute().toString()); + leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + } + EXPECT_EQUAL(1u, lst.size()); + for (uint32_t i = 0; i < servers.size(); ++i) { + delete servers[i]; + } +} + +void +Test::requireThatStoragePolicyCombinesSystemAndSlobrokState() +{ + TestFrame frame(_repo); + frame.setMessage(newPutDocumentMessage("doc:scheme:")); + + mbus::Slobrok slobrok; + mbus::TestServer server(mbus::Identity("storage/cluster.mycluster/distributor/0"), + mbus::RoutingSpec(), slobrok, "", + mbus::IProtocol::SP(new DocumentProtocol(_loadTypes, _repo))); + server.net.registerSession("default"); + + string param = vespalib::make_string( + "cluster=mycluster;slobroks=tcp/localhost:%d;clusterconfigid=%s;syncinit", + slobrok.port(), getDefaultDistributionConfig(2, 5).c_str()); + StoragePolicy &policy = setupStoragePolicy( + frame, param, + "storage/cluster.mycluster/distributor/*/default", 1); + ASSERT_TRUE(policy.getSystemState() == NULL); + { + std::vector<mbus::RoutingNode*> leaf; + ASSERT_TRUE(frame.select(leaf, 1)); + leaf[0]->handleReply(mbus::Reply::UP(new WrongDistributionReply("distributor:99 storage:99"))); + ASSERT_TRUE(policy.getSystemState() != NULL); + EXPECT_EQUAL(policy.getSystemState()->toString(), "distributor:99 storage:99"); + } + for (int i = 0; i < 666; i++) { + ASSERT_TRUE(frame.testSelect(StringList().add(server.net.getConnectionSpec() + "/default"))); + } +} + +void +Test::testSubsetService() +{ + // Prepare message. + TestFrame frame(_repo, "docproc/cluster.default"); + frame.setMessage(mbus::Message::UP(new PutDocumentMessage(Document::SP( + new Document(*_docType, + DocumentId("doc:scheme:")))))); + + // Test requerying for adding nodes. + frame.setHop(mbus::HopSpec("test", "docproc/cluster.default/[SubsetService:2]/chain.default")); + std::set<string> lst; + for (uint32_t i = 1; i <= 10; ++i) { + frame.getNetwork().registerSession(vespalib::make_string("%d/chain.default", i)); + ASSERT_TRUE(frame.waitSlobrok("docproc/cluster.default/*/chain.default", i)); + + std::vector<mbus::RoutingNode*> leaf; + ASSERT_TRUE(frame.select(leaf, 1)); + lst.insert(leaf[0]->getRoute().toString()); + leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + ASSERT_TRUE(frame.getReceptor().getReply(600).get() != NULL); + } + ASSERT_TRUE(lst.size() > 1); // must have requeried + + // Test load balancing. + string prev = ""; + for (uint32_t i = 1; i <= 10; ++i) { + std::vector<mbus::RoutingNode*> leaf; + ASSERT_TRUE(frame.select(leaf, 1)); + + string next = leaf[0]->getRoute().toString(); + if (prev.empty()) { + ASSERT_TRUE(!next.empty()); + } else { + ASSERT_TRUE(prev != next); + } + + prev = next; + leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + ASSERT_TRUE(frame.getReceptor().getReply(600).get() != NULL); + } + + // Test requerying for dropping nodes. + lst.clear(); + for (uint32_t i = 1; i <= 10; ++i) { + std::vector<mbus::RoutingNode*> leaf; + ASSERT_TRUE(frame.select(leaf, 1)); + string route = leaf[0]->getRoute().toString(); + lst.insert(route); + + frame.getNetwork().unregisterSession(route.substr(frame.getIdentity().length() + 1)); + ASSERT_TRUE(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10 - i)); + + mbus::Reply::UP reply(new mbus::EmptyReply()); + reply->addError(mbus::Error(mbus::ErrorCode::NO_ADDRESS_FOR_SERVICE, route)); + leaf[0]->handleReply(std::move(reply)); + ASSERT_TRUE(frame.getReceptor().getReply(600).get() != NULL); + } + EXPECT_EQUAL(10u, lst.size()); + + // Test merge behavior. + frame.setHop(mbus::HopSpec("test", "[SubsetService]")); + EXPECT_TRUE(frame.testMergeOneReply("*")); +} + +void +Test::testSubsetServiceCache() +{ + TestFrame fooFrame(_repo, "docproc/cluster.default"); + mbus::HopSpec fooHop("foo", "docproc/cluster.default/[SubsetService:2]/chain.foo"); + fooFrame.setMessage(mbus::Message::UP(new GetDocumentMessage(document::DocumentId("doc:scheme:foo")))); + fooFrame.setHop(fooHop); + + TestFrame barFrame(fooFrame); + mbus::HopSpec barHop("bar", "docproc/cluster.default/[SubsetService:2]/chain.bar"); + barFrame.setMessage(mbus::Message::UP(new GetDocumentMessage(document::DocumentId("doc:scheme:bar")))); + barFrame.setHop(barHop); + + fooFrame.getMessageBus().setupRouting( + mbus::RoutingSpec().addTable(mbus::RoutingTableSpec(DocumentProtocol::NAME) + .addHop(fooHop) + .addHop(barHop))); + + fooFrame.getNetwork().registerSession("0/chain.foo"); + fooFrame.getNetwork().registerSession("0/chain.bar"); + ASSERT_TRUE(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2)); + + std::vector<mbus::RoutingNode*> fooSelected; + ASSERT_TRUE(fooFrame.select(fooSelected, 1)); + EXPECT_EQUAL("docproc/cluster.default/0/chain.foo", fooSelected[0]->getRoute().getHop(0).toString()); + + std::vector<mbus::RoutingNode*> barSelected; + ASSERT_TRUE(barFrame.select(barSelected, 1)); + EXPECT_EQUAL("docproc/cluster.default/0/chain.bar", barSelected[0]->getRoute().getHop(0).toString()); + + barSelected[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + fooSelected[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + + ASSERT_TRUE(barFrame.getReceptor().getReply(600).get() != NULL); + ASSERT_TRUE(fooFrame.getReceptor().getReply(600).get() != NULL); +} + +bool +Test::trySelect(TestFrame &frame, uint32_t numSelects, const std::vector<string> &expected) { + std::set<string> lst; + for (uint32_t i = 0; i < numSelects; ++i) { + std::vector<mbus::RoutingNode*> leaf; + if (!expected.empty()) { + frame.select(leaf, 1); + lst.insert(leaf[0]->getRoute().toString()); + leaf[0]->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + } else { + frame.select(leaf, 0); + } + if(frame.getReceptor().getReply(600).get() == NULL) { + LOG(error, "Reply failed to propagate to reply handler."); + return false; + } + } + if (expected.size() != lst.size()) { + LOG(error, "Expected %d recipients, got %d.", (uint32_t)expected.size(), (uint32_t)lst.size()); + return false; + } + std::set<string>::iterator it = lst.begin(); + for (uint32_t i = 0; i < expected.size(); ++i, ++it) { + if (*it != expected[i]) { + LOG(error, "Expected '%s', got '%s'.", expected[i].c_str(), it->c_str()); + return false; + } + } + return true; +} + +bool +Test::isErrorPolicy(const string &name, const string ¶m) +{ + DocumentProtocol protocol(_loadTypes, _repo); + mbus::IRoutingPolicy::UP policy = protocol.createPolicy(name, param); + + return policy.get() != NULL && dynamic_cast<ErrorPolicy*>(policy.get()) != NULL; +} + diff --git a/documentapi/src/tests/policies/testframe.cpp b/documentapi/src/tests/policies/testframe.cpp new file mode 100644 index 00000000000..cb30e5377aa --- /dev/null +++ b/documentapi/src/tests/policies/testframe.cpp @@ -0,0 +1,336 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".testframe"); + +#include "testframe.h" +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/network/rpcnetwork.h> +#include <vespa/messagebus/sendproxy.h> +#include <vespa/messagebus/testlib/simplemessage.h> +#include <vespa/messagebus/testlib/simpleprotocol.h> +#include <vespa/messagebus/testlib/simplereply.h> + +using document::DocumentTypeRepo; +using namespace documentapi; + +class MyServiceAddress : public mbus::IServiceAddress { +private: + string _address; + +public: + MyServiceAddress(const string &address) : + _address(address) { + // empty + } + + const string &getAddress() { + return _address; + } +}; + +class MyNetwork : public mbus::RPCNetwork { +private: + std::vector<mbus::RoutingNode*> _nodes; + +public: + MyNetwork(const mbus::RPCNetworkParams ¶ms) : + mbus::RPCNetwork(params), + _nodes() { + // empty + } + + bool allocServiceAddress(mbus::RoutingNode &recipient) { + string hop = recipient.getRoute().getHop(0).toString(); + recipient.setServiceAddress(mbus::IServiceAddress::UP(new MyServiceAddress(hop))); + return true; + } + + void freeServiceAddress(mbus::RoutingNode &recipient) { + recipient.setServiceAddress(mbus::IServiceAddress::UP()); + } + + void send(const mbus::Message &, const std::vector<mbus::RoutingNode*> &nodes) { + _nodes.insert(_nodes.begin(), nodes.begin(), nodes.end()); + } + + void removeNodes(std::vector<mbus::RoutingNode*> &nodes) { + nodes.insert(nodes.begin(), _nodes.begin(), _nodes.end()); + _nodes.clear(); + } +}; + +TestFrame::TestFrame(const DocumentTypeRepo::SP &repo, const string &ident) : + _identity(ident), + _slobrok(new mbus::Slobrok()), + _set(), + _net(new MyNetwork(mbus::RPCNetworkParams() + .setIdentity(mbus::Identity(ident)) + .setSlobrokConfig(_slobrok->config()))), + _mbus(new mbus::MessageBus(*_net, mbus::MessageBusParams() + .addProtocol(mbus::IProtocol::SP(new DocumentProtocol(_set, repo))))), + _msg(), + _hop(mbus::HopSpec("foo", "bar")), + _handler() +{ + // empty +} + +TestFrame::TestFrame(TestFrame &frame) : + mbus::IReplyHandler(), + _identity(frame._identity), + _slobrok(frame._slobrok), + _net(frame._net), + _mbus(frame._mbus), + _msg(), + _hop(mbus::HopSpec("baz", "cox")), + _handler() +{ + // empty +} + +TestFrame::~TestFrame() +{ + // empty +} + +void +TestFrame::setHop(const mbus::HopSpec &hop) +{ + _hop = hop; + _mbus->setupRouting(mbus::RoutingSpec().addTable(mbus::RoutingTableSpec(DocumentProtocol::NAME).addHop(_hop))); +} + +bool +TestFrame::select(std::vector<mbus::RoutingNode*> &selected, uint32_t numExpected) +{ + _msg->setRoute(mbus::Route::parse(_hop.getName())); + _msg->pushHandler(*this); + mbus::SendProxy &proxy = *(new mbus::SendProxy(*_mbus, *_net, NULL)); // deletes self + proxy.handleMessage(std::move(_msg)); + + static_cast<MyNetwork&>(*_net).removeNodes(selected); + if (selected.size() != numExpected) { + LOG(error, "Expected %d recipients, got %d.", numExpected, (uint32_t)selected.size()); + return false; + } + return true; +} + +bool +TestFrame::testSelect(const std::vector<string> &expected) +{ + std::vector<mbus::RoutingNode*> selected; + if (!select(selected, expected.size())) { + LOG(error, "Failed to select recipients."); + for (size_t i = 0; i < selected.size(); ++i) { + LOG(error, "Selected: %s", + selected[i]->getRoute().toString().c_str()); + } + return false; + } + for (std::vector<mbus::RoutingNode*>::iterator it = selected.begin(); + it != selected.end(); ++it) + { + string route = (*it)->getRoute().toString(); + if (find(expected.begin(), expected.end(), route) == expected.end()) { + LOG(error, "Recipient '%s' not selected.", route.c_str()); + } + (*it)->handleReply(mbus::Reply::UP(new mbus::EmptyReply())); + } + if (_handler.getReply(600).get() == NULL) { + LOG(error, "Reply not propagated to handler."); + return false; + } + return true; +} + +bool +TestFrame::testMergeError(const ReplyMap &replies, const std::vector<uint32_t> &expectedErrors) +{ + return testMerge(replies, expectedErrors, StringList()); +} + +bool +TestFrame::testMergeOk(const ReplyMap &replies, const std::vector<string> &allowedValues) +{ + return testMerge(replies, UIntList(), allowedValues); +} + +bool +TestFrame::testMerge(const ReplyMap &replies, + const std::vector<uint32_t> &expectedErrors, + const std::vector<string> &allowedValues) +{ + std::vector<mbus::RoutingNode*> selected; + if (!select(selected, replies.size())) { + return false; + } + + for (std::vector<mbus::RoutingNode*>::iterator it = selected.begin(); + it != selected.end(); ++it) + { + string route = (*it)->getRoute().toString(); + ReplyMap::const_iterator mip = replies.find(route); + if (mip == replies.end()) { + LOG(error, "Recipient '%s' not expected.", route.c_str()); + return false; + } + + mbus::Reply::UP ret(new mbus::SimpleReply(route)); + if (mip->second != mbus::ErrorCode::NONE) { + ret->addError(mbus::Error(mip->second, route)); + } + (*it)->handleReply(std::move(ret)); + } + + mbus::Reply::UP reply = _handler.getReply(600); + if (reply.get() == NULL) { + LOG(error, "Reply not propagated to handler."); + return false; + } + if (!expectedErrors.empty()) { + if (expectedErrors.size() != reply->getNumErrors()) { + LOG(error, "Expected %d errors, got %d.", (uint32_t)expectedErrors.size(), reply->getNumErrors()); + return false; + } + for (uint32_t i = 0; i < expectedErrors.size(); ++i) { + uint32_t err = reply->getError(i).getCode(); + if (std::find(expectedErrors.begin(), expectedErrors.end(), err) == expectedErrors.end()) { + LOG(error, "Expected error code %d not found.", err); + return false; + } + } + } else if (reply->hasErrors()) { + LOG(error, "Got %d unexpected error(s):", reply->getNumErrors()); + for(uint32_t i = 0; i < reply->getNumErrors(); ++i) { + LOG(error, "%d. %s", i + 1, reply->getError(i).toString().c_str()); + } + return false; + } + if (!allowedValues.empty()) { + if (mbus::SimpleProtocol::REPLY != reply->getType()) { + LOG(error, "Expected reply type %d, got %d.", mbus::SimpleProtocol::REPLY, reply->getType()); + return false; + } + string val = static_cast<mbus::SimpleReply&>(*reply).getValue(); + if (std::find(allowedValues.begin(), allowedValues.end(), val) == allowedValues.end()) { + LOG(error, "Value '%s' not allowed.", val.c_str()); + return false; + } + } else { + if (0 != reply->getType()) { + LOG(error, "Expected reply type %d, got %d.", 0, reply->getType()); + return false; + } + } + return true; +} + +bool +TestFrame::testMergeOneReply(const string &recipient) +{ + if (!testSelect(StringList().add(recipient))) { + return false; + } + + ReplyMap replies; + replies[recipient] = mbus::ErrorCode::NONE; + if (!testMergeOk(replies, StringList().add(recipient))) { + LOG(error, "Failed to merge reply with no error."); + return false; + } + + replies[recipient] = mbus::ErrorCode::TRANSIENT_ERROR; + if (!testMergeError(replies, UIntList().add(mbus::ErrorCode::TRANSIENT_ERROR))) { + LOG(error, "Failed to merge reply with transient error."); + return false; + } + + return true; +} + +bool +TestFrame::testMergeTwoReplies(const string &recipientOne, const string &recipientTwo) +{ + if (!testSelect(StringList().add(recipientOne).add(recipientTwo))) { + return false; + } + + ReplyMap replies; + replies[recipientOne] = mbus::ErrorCode::NONE; + replies[recipientTwo] = mbus::ErrorCode::NONE; + if (!testMergeOk(replies, StringList().add(recipientOne).add(recipientTwo))) { + LOG(error, "Failed to merge two replies with no error."); + return false; + } + + replies[recipientOne] = mbus::ErrorCode::TRANSIENT_ERROR; + replies[recipientTwo] = mbus::ErrorCode::NONE; + if (!testMergeError(replies, UIntList().add(mbus::ErrorCode::TRANSIENT_ERROR))) { + LOG(error, "Failed to merge two replies where one has transient error."); + return false; + } + + replies[recipientOne] = mbus::ErrorCode::TRANSIENT_ERROR; + replies[recipientTwo] = mbus::ErrorCode::TRANSIENT_ERROR; + if (!testMergeError(replies, UIntList() + .add(mbus::ErrorCode::TRANSIENT_ERROR) + .add(mbus::ErrorCode::TRANSIENT_ERROR))) { + LOG(error, "Failed to merge two replies where both have transient errors."); + return false; + } + + replies[recipientOne] = mbus::ErrorCode::NONE; + replies[recipientTwo] = DocumentProtocol::ERROR_MESSAGE_IGNORED; + if (!testMergeOk(replies, StringList().add(recipientOne))) { + LOG(error, "Failed to merge two replies where second should be ignored."); + return false; + } + + replies[recipientOne] = DocumentProtocol::ERROR_MESSAGE_IGNORED; + replies[recipientTwo] = mbus::ErrorCode::NONE; + if (!testMergeOk(replies, StringList().add(recipientTwo))) { + LOG(error, "Failed to merge two replies where first should be ignored."); + return false; + } + + replies[recipientOne] = DocumentProtocol::ERROR_MESSAGE_IGNORED; + replies[recipientTwo] = DocumentProtocol::ERROR_MESSAGE_IGNORED; + if (!testMergeError(replies, UIntList() + .add(DocumentProtocol::ERROR_MESSAGE_IGNORED) + .add(DocumentProtocol::ERROR_MESSAGE_IGNORED))) { + LOG(error, "Failed to merge two replies where both can be ignored."); + return false; + } + + return true; +} + +bool +TestFrame::waitSlobrok(const string &pattern, uint32_t cnt) +{ + for (uint32_t i = 0; i < 1000; ++i) { + slobrok::api::MirrorAPI::SpecList res = _net->getMirror().lookup(pattern); + if (res.size() == cnt) { + return true; + } + FastOS_Thread::Sleep(10); + } + LOG(error, "Slobrok failed to resolve '%s' to %d recipients in time.", pattern.c_str(), cnt); + return false; +} + +SystemStateHandle +TestFrame::getSystemState() +{ + mbus::IProtocol::SP protocol = _mbus->getProtocol(DocumentProtocol::NAME); + return SystemStateHandle(static_cast<DocumentProtocol&>(*protocol).getSystemState()); +} + +void +TestFrame::handleReply(mbus::Reply::UP reply) +{ + _msg = reply->getMessage(); + _handler.handleReply(std::move(reply)); +} diff --git a/documentapi/src/tests/policies/testframe.h b/documentapi/src/tests/policies/testframe.h new file mode 100644 index 00000000000..6c3080974d1 --- /dev/null +++ b/documentapi/src/tests/policies/testframe.h @@ -0,0 +1,219 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/systemstate/systemstatehandle.h> +#include <vespa/messagebus/messagebus.h> +#include <vespa/messagebus/network/identity.h> +#include <vespa/messagebus/network/inetwork.h> +#include <vespa/messagebus/testlib/receptor.h> +#include <vespa/messagebus/testlib/slobrok.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> + +using documentapi::string; + +class TestFrame : public mbus::IReplyHandler { +private: + string _identity; + std::shared_ptr<mbus::Slobrok> _slobrok; + documentapi::LoadTypeSet _set; + std::shared_ptr<mbus::INetwork> _net; + std::shared_ptr<mbus::MessageBus> _mbus; + mbus::Message::UP _msg; + mbus::HopSpec _hop; + mbus::Receptor _handler; + + TestFrame &operator=(const TestFrame &); // hide + +public: + /** + * Convenience typedefs. + */ + typedef std::map<string, uint32_t> ReplyMap; + + /** + * Create a named test frame. + * + * @param identity The identity to use for the server. + */ + TestFrame(const document::DocumentTypeRepo::SP &repo, + const string &ident = "anonymous"); + + /** + * Create a test frame running on the same slobrok and mbus as another. + * + * @param frame The frame whose internals to share. + */ + TestFrame(TestFrame &frame); + + /** + * Cleans up allocated resources. + */ + virtual ~TestFrame(); + + /** + * Routes the contained message based on the current setup, and returns the leaf send contexts. + * + * @param selected The list to add the selected recipients to. + * @param numExpected The expected number of contexts. + * @return True if everything was ok. + */ + bool select(std::vector<mbus::RoutingNode*> &selected, uint32_t numExpected); + + /** + * Ensures that the current setup selects a given set of routes. + * + * @param expected A list of expected route leaf nodes. + * @return True if everything was ok. + */ + bool testSelect(const std::vector<string> &expected); + + /** + * This is a convenience method for invoking {@link #assertMerge(std::map,std::vector,std::vector)} with + * no expected value. + * + * @param replies The errors to set in the leaf node replies. + * @param expectedErrors The list of expected errors in the merged reply. + * @return True if everything was ok. + */ + bool testMergeError(const ReplyMap &replies, const std::vector<uint32_t> &expectedErrors); + + /** + * This is a convenience method for invoking {@link #assertMerge(std::map,std::vector,std::vector)} with + * no expected errors. + * + * @param replies The errors to set in the leaf node replies. + * @param allowedValues The list of allowed values in the final reply. + * @return True if everything was ok. + */ + bool testMergeOk(const ReplyMap &replies, const std::vector<string> &allowedValues); + + /** + * Ensures that the current setup generates as many leaf nodes as there are members of the errors argument. Each + * error is then given one of these errors, and the method then ensures that the single returned reply contains the + * given list of expected errors. Finally, if the expected value argument is non-null, this method ensures that the + * reply is a SimpleReply whose string value exists in the allowed list. + * + * @param replies The errors to set in the leaf node replies. + * @param expectedErrors The list of expected errors in the merged reply. + * @param allowedValues The list of allowed values in the final reply. + * @return True if everything was ok. + */ + bool testMerge(const ReplyMap &replies, + const std::vector<uint32_t> &expectedErrors, + const std::vector<string> &allowedValues); + + /** + * Ensures that the current setup chooses a single recipient, and that it merges similarly to how the + * {@link DocumentProtocol} would merge these. + * + * @param recipient The expected recipient. + * @return True if everything was ok. + */ + bool testMergeOneReply(const string &recipient); + + /** + * Ensures that the current setup will choose the two given recipients, and that it merges similarly to how the + * {@link DocumentProtocol} would merge these. + * + * @param recipientOne The first expected recipient. + * @param recipientTwo The second expected recipient. + */ + bool testMergeTwoReplies(const string &recipientOne, const string &recipientTwo); + + /** + * Waits for a given service pattern to resolve to the given number of hits in the local slobrok. + * + * @param pattern The pattern to lookup. + * @param cnt The number of entries to wait for. + * @return True if the expected number of entries was found. + */ + bool waitSlobrok(const string &pattern, uint32_t cnt); + + /** + * Returns the identity of this frame. + * + * @return The ident string. + */ + const string &getIdentity() { return _identity; } + + /** + * Returns the private slobrok server. + * + * @return The slobrok. + */ + mbus::Slobrok &getSlobrok() { return *_slobrok; } + + /** + * Returns the private message bus. + * + * @return The bus. + */ + mbus::MessageBus &getMessageBus() { return *_mbus; } + + /** + * Returns the private network layer. + * + * @return The network. + */ + mbus::INetwork &getNetwork() { return *_net; } + + /** + * Returns the message being tested. + * + * @return The message. + */ + mbus::Message::UP getMessage() { return std::move(_msg); } + + /** + * Sets the message being tested. + * + * @param msg The message to set. + */ + mbus::Message::UP setMessage(mbus::Message::UP msg) { + std::swap(msg, _msg); + return std::move(msg); + } + + /** + * Sets the spec of the hop to test with. + * + * @param hop The spec to set. + */ + void setHop(const mbus::HopSpec &hop); + + /** + * Returns the reply receptor used by this frame. All messages tested are tagged with this receptor, so after a + * successful select, the receptor should contain a non-null reply. + * + * @return The reply receptor. + */ + mbus::Receptor &getReceptor() { return _handler; } + + /** + * Returns the system state from contained document protocol. + * + * @return Handle to the system state. + */ + documentapi::SystemStateHandle getSystemState(); + + // Implements IReplyHandler. + void handleReply(mbus::Reply::UP reply); +}; + +class UIntList : public std::vector<uint32_t> { +public: + UIntList &add(uint32_t err) { + std::vector<uint32_t>::push_back(err); + return *this; + } +}; + +class StringList : public std::vector<string> { +public: + StringList &add(const string &val) { + std::vector<string>::push_back(val); + return *this; + } +}; + diff --git a/documentapi/src/tests/policyfactory/.gitignore b/documentapi/src/tests/policyfactory/.gitignore new file mode 100644 index 00000000000..8ab5fc6c580 --- /dev/null +++ b/documentapi/src/tests/policyfactory/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +policyfactory_test +documentapi_policyfactory_test_app diff --git a/documentapi/src/tests/policyfactory/CMakeLists.txt b/documentapi/src/tests/policyfactory/CMakeLists.txt new file mode 100644 index 00000000000..c533847ff36 --- /dev/null +++ b/documentapi/src/tests/policyfactory/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(documentapi_policyfactory_test_app + SOURCES + policyfactory.cpp + DEPENDS + documentapi +) +vespa_add_test(NAME documentapi_policyfactory_test_app COMMAND documentapi_policyfactory_test_app) diff --git a/documentapi/src/tests/policyfactory/DESC b/documentapi/src/tests/policyfactory/DESC new file mode 100644 index 00000000000..faf40102408 --- /dev/null +++ b/documentapi/src/tests/policyfactory/DESC @@ -0,0 +1 @@ +policyfactory test. Take a look at policyfactory.cpp for details. diff --git a/documentapi/src/tests/policyfactory/FILES b/documentapi/src/tests/policyfactory/FILES new file mode 100644 index 00000000000..744e02bc7d6 --- /dev/null +++ b/documentapi/src/tests/policyfactory/FILES @@ -0,0 +1 @@ +policyfactory.cpp diff --git a/documentapi/src/tests/policyfactory/policyfactory.cpp b/documentapi/src/tests/policyfactory/policyfactory.cpp new file mode 100644 index 00000000000..3caba70496b --- /dev/null +++ b/documentapi/src/tests/policyfactory/policyfactory.cpp @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("policyfactory_test"); + +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <vespa/messagebus/testlib/receptor.h> +#include <vespa/messagebus/testlib/slobrok.h> +#include <vespa/messagebus/testlib/testserver.h> +#include <vespa/vespalib/testkit/testapp.h> + +using document::DocumentTypeRepo; +using namespace documentapi; + +/////////////////////////////////////////////////////////////////////////////// +// +// Utilities +// +/////////////////////////////////////////////////////////////////////////////// + +class MyPolicy : public mbus::IRoutingPolicy { +private: + string _param; +public: + MyPolicy(const string ¶m); + void select(mbus::RoutingContext &ctx); + void merge(mbus::RoutingContext &ctx); +}; + +MyPolicy::MyPolicy(const string ¶m) : + _param(param) +{ + // empty +} + +void +MyPolicy::select(mbus::RoutingContext &ctx) +{ + ctx.setError(DocumentProtocol::ERROR_POLICY_FAILURE, _param); +} + +void +MyPolicy::merge(mbus::RoutingContext &ctx) +{ + (void)ctx; + LOG_ASSERT(false); +} + +class MyFactory : public IRoutingPolicyFactory { +public: + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; +}; + +mbus::IRoutingPolicy::UP +MyFactory::createPolicy(const string ¶m) const +{ + return mbus::IRoutingPolicy::UP(new MyPolicy(param)); +} + +mbus::Message::UP +createMessage() +{ + mbus::Message::UP ret(new RemoveDocumentMessage(document::DocumentId("doc:scheme:"))); + ret->getTrace().setLevel(9); + return ret; +} + +/////////////////////////////////////////////////////////////////////////////// +// +// Tests +// +/////////////////////////////////////////////////////////////////////////////// + +TEST_SETUP(Test); + +int +Test::Main() +{ + TEST_INIT("policyfactory_test"); + + DocumentTypeRepo::SP repo(new DocumentTypeRepo); + mbus::Slobrok slobrok; + LoadTypeSet loadTypes; + mbus::TestServer + srv(mbus::MessageBusParams() + .addProtocol(mbus::IProtocol::SP(new DocumentProtocol( + loadTypes, repo))), + mbus::RPCNetworkParams().setSlobrokConfig(slobrok.config())); + mbus::Receptor handler; + mbus::SourceSession::UP src = srv.mb.createSourceSession(mbus::SourceSessionParams().setReplyHandler(handler)); + + mbus::Route route = mbus::Route::parse("[MyPolicy]"); + ASSERT_TRUE(src->send(createMessage(), route).isAccepted()); + mbus::Reply::UP reply = static_cast<mbus::Receptor&>(src->getReplyHandler()).getReply(600); + ASSERT_TRUE(reply.get() != NULL); + fprintf(stderr, "%s", reply->getTrace().toString().c_str()); + EXPECT_EQUAL(1u, reply->getNumErrors()); + EXPECT_EQUAL((uint32_t)mbus::ErrorCode::UNKNOWN_POLICY, reply->getError(0).getCode()); + + mbus::IProtocol::SP obj = srv.mb.getProtocol(DocumentProtocol::NAME); + DocumentProtocol *protocol = dynamic_cast<DocumentProtocol*>(obj.get()); + ASSERT_TRUE(protocol != NULL); + protocol->putRoutingPolicyFactory("MyPolicy", IRoutingPolicyFactory::SP(new MyFactory())); + + ASSERT_TRUE(src->send(createMessage(), route).isAccepted()); + reply = static_cast<mbus::Receptor&>(src->getReplyHandler()).getReply(600); + ASSERT_TRUE(reply.get() != NULL); + fprintf(stderr, "%s", reply->getTrace().toString().c_str()); + EXPECT_EQUAL(1u, reply->getNumErrors()); + EXPECT_EQUAL((uint32_t)DocumentProtocol::ERROR_POLICY_FAILURE, reply->getError(0).getCode()); + + TEST_DONE(); +} diff --git a/documentapi/src/tests/priority/.gitignore b/documentapi/src/tests/priority/.gitignore new file mode 100644 index 00000000000..b0e5123f142 --- /dev/null +++ b/documentapi/src/tests/priority/.gitignore @@ -0,0 +1 @@ +documentapi_priority_test_app diff --git a/documentapi/src/tests/priority/CMakeLists.txt b/documentapi/src/tests/priority/CMakeLists.txt new file mode 100644 index 00000000000..289ea4b4ebe --- /dev/null +++ b/documentapi/src/tests/priority/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(documentapi_priority_test_app + SOURCES + priority.cpp + DEPENDS + documentapi +) +vespa_add_test(NAME documentapi_priority_test_app COMMAND documentapi_priority_test_app) diff --git a/documentapi/src/tests/priority/DESC b/documentapi/src/tests/priority/DESC new file mode 100644 index 00000000000..d213e9de057 --- /dev/null +++ b/documentapi/src/tests/priority/DESC @@ -0,0 +1 @@ +priority test. Take a look at priority.cpp for details. diff --git a/documentapi/src/tests/priority/FILES b/documentapi/src/tests/priority/FILES new file mode 100644 index 00000000000..4a9bd82566a --- /dev/null +++ b/documentapi/src/tests/priority/FILES @@ -0,0 +1 @@ +priority.cpp diff --git a/documentapi/src/tests/priority/priority.cpp b/documentapi/src/tests/priority/priority.cpp new file mode 100644 index 00000000000..3ad0e2041cf --- /dev/null +++ b/documentapi/src/tests/priority/priority.cpp @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("priority_test"); + +#include <fstream> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/documentapi/messagebus/priority.h> + +using namespace documentapi; + +TEST_SETUP(Test); + +int +Test::Main() +{ + TEST_INIT("priority_test"); + + std::vector<int32_t> expected; + expected.push_back(Priority::PRI_HIGHEST); + expected.push_back(Priority::PRI_VERY_HIGH); + expected.push_back(Priority::PRI_HIGH_1); + expected.push_back(Priority::PRI_HIGH_2); + expected.push_back(Priority::PRI_HIGH_3); + expected.push_back(Priority::PRI_NORMAL_1); + expected.push_back(Priority::PRI_NORMAL_2); + expected.push_back(Priority::PRI_NORMAL_3); + expected.push_back(Priority::PRI_NORMAL_4); + expected.push_back(Priority::PRI_NORMAL_5); + expected.push_back(Priority::PRI_NORMAL_6); + expected.push_back(Priority::PRI_LOW_1); + expected.push_back(Priority::PRI_LOW_2); + expected.push_back(Priority::PRI_LOW_3); + expected.push_back(Priority::PRI_VERY_LOW); + expected.push_back(Priority::PRI_LOWEST); + + std::ifstream in; + in.open("../../../test/crosslanguagefiles/5.1-Priority.txt"); + ASSERT_TRUE(in.good()); + while (in) { + std::string str; + in >> str; + if (str.empty()) { + continue; + } + size_t pos = str.find(":"); + ASSERT_TRUE(pos != std::string::npos); + int32_t pri = atoi(str.substr(pos + 1).c_str()); + ASSERT_EQUAL(Priority::getPriority(str.substr(0, pos)), pri); + + std::vector<int32_t>::iterator it = + std::find(expected.begin(), expected.end(), pri); + ASSERT_TRUE(it != expected.end()); + expected.erase(it); + } + ASSERT_TRUE(expected.empty()); + + TEST_DONE(); +} diff --git a/documentapi/src/tests/replymerger/.gitignore b/documentapi/src/tests/replymerger/.gitignore new file mode 100644 index 00000000000..932bb09e490 --- /dev/null +++ b/documentapi/src/tests/replymerger/.gitignore @@ -0,0 +1 @@ +documentapi_replymerger_test_app diff --git a/documentapi/src/tests/replymerger/CMakeLists.txt b/documentapi/src/tests/replymerger/CMakeLists.txt new file mode 100644 index 00000000000..9faa203fb2b --- /dev/null +++ b/documentapi/src/tests/replymerger/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(documentapi_replymerger_test_app + SOURCES + replymerger_test.cpp + DEPENDS + documentapi +) +vespa_add_test(NAME documentapi_replymerger_test_app COMMAND documentapi_replymerger_test_app) diff --git a/documentapi/src/tests/replymerger/DESC b/documentapi/src/tests/replymerger/DESC new file mode 100644 index 00000000000..ca179fc0da0 --- /dev/null +++ b/documentapi/src/tests/replymerger/DESC @@ -0,0 +1 @@ +replymerger test. Take a look at replymerger.cpp for details. diff --git a/documentapi/src/tests/replymerger/FILES b/documentapi/src/tests/replymerger/FILES new file mode 100644 index 00000000000..5056276d197 --- /dev/null +++ b/documentapi/src/tests/replymerger/FILES @@ -0,0 +1 @@ +replymerger.cpp diff --git a/documentapi/src/tests/replymerger/replymerger_test.cpp b/documentapi/src/tests/replymerger/replymerger_test.cpp new file mode 100644 index 00000000000..2e1551e8dca --- /dev/null +++ b/documentapi/src/tests/replymerger/replymerger_test.cpp @@ -0,0 +1,304 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/log/log.h> +LOG_SETUP("replymerger_test"); + +#include <vespa/fastos/fastos.h> +#include <iostream> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/documentapi/messagebus/replymerger.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/removedocumentreply.h> +#include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> +#include <vespa/documentapi/messagebus/messages/getdocumentreply.h> +#include <vespa/messagebus/emptyreply.h> + +using namespace documentapi; + +class Test : public vespalib::TestApp +{ + static void assertReplyErrorsMatch(const mbus::Reply& r, + const std::vector<mbus::Error>& errors); +public: + int Main(); + + void mergingGenericRepliesWithNoErrorsPicksFirstReply(); + void mergingSingleReplyWithOneErrorReturnsEmptyReplyWithError(); + void mergingSingleReplyWithMultipleErrorsReturnsEmptyReplyWithAllErrors(); + void mergingMultipleRepliesWithMultipleErrorsReturnsEmptyReplyWithAllErrors(); + void returnIgnoredReplyWhenAllRepliesHaveOnlyIgnoredErrors(); + void successfulReplyTakesPrecedenceOverIgnoredReplyWhenNoErrors(); + void nonIgnoredErrorTakesPrecedence(); + void returnRemoveDocumentReplyWhereDocWasFound(); + void returnFirstRemoveDocumentReplyIfNoDocsWereFound(); + void returnUpdateDocumentReplyWhereDocWasFound(); + void returnGetDocumentReplyWhereDocWasFound(); + void mergingZeroRepliesReturnsDefaultEmptyReply(); +}; + +TEST_APPHOOK(Test); + +void +Test::mergingGenericRepliesWithNoErrorsPicksFirstReply() +{ + mbus::EmptyReply r1; + mbus::EmptyReply r2; + mbus::EmptyReply r3; + ReplyMerger merger; + merger.merge(0, r1); + merger.merge(1, r2); + merger.merge(2, r3); + ReplyMerger::Result ret(merger.mergedReply()); + ASSERT_TRUE(ret.isSuccessful()); + ASSERT_FALSE(ret.hasGeneratedReply()); + EXPECT_EQUAL(0u, ret.getSuccessfulReplyIndex()); +} + +void +Test::mergingSingleReplyWithOneErrorReturnsEmptyReplyWithError() +{ + mbus::EmptyReply r1; + std::vector<mbus::Error> errors = { mbus::Error(1234, "oh no!") }; + r1.addError(errors[0]); + ReplyMerger merger; + merger.merge(0, r1); + ReplyMerger::Result ret(merger.mergedReply()); + ASSERT_FALSE(ret.isSuccessful()); + ASSERT_TRUE(ret.hasGeneratedReply()); + std::unique_ptr<mbus::Reply> gen(ret.releaseGeneratedReply()); + assertReplyErrorsMatch(*gen, errors); +} + +void +Test::mergingSingleReplyWithMultipleErrorsReturnsEmptyReplyWithAllErrors() +{ + mbus::EmptyReply r1; + std::vector<mbus::Error> errors = { + mbus::Error(1234, "oh no!"), + mbus::Error(4567, "oh dear!") + }; + r1.addError(errors[0]); + r1.addError(errors[1]); + ReplyMerger merger; + merger.merge(0, r1); + ReplyMerger::Result ret(merger.mergedReply()); + ASSERT_FALSE(ret.isSuccessful()); + ASSERT_TRUE(ret.hasGeneratedReply()); + std::unique_ptr<mbus::Reply> gen(ret.releaseGeneratedReply()); + assertReplyErrorsMatch(*gen, errors); +} + +void +Test::mergingMultipleRepliesWithMultipleErrorsReturnsEmptyReplyWithAllErrors() +{ + mbus::EmptyReply r1; + mbus::EmptyReply r2; + std::vector<mbus::Error> errors = { + mbus::Error(1234, "oh no!"), + mbus::Error(4567, "oh dear!"), + mbus::Error(678, "omg!") + }; + r1.addError(errors[0]); + r1.addError(errors[1]); + r2.addError(errors[2]); + ReplyMerger merger; + merger.merge(0, r1); + merger.merge(1, r2); + ReplyMerger::Result ret(merger.mergedReply()); + ASSERT_FALSE(ret.isSuccessful()); + ASSERT_TRUE(ret.hasGeneratedReply()); + std::unique_ptr<mbus::Reply> gen(ret.releaseGeneratedReply()); + assertReplyErrorsMatch(*gen, errors); +} + +void +Test::returnIgnoredReplyWhenAllRepliesHaveOnlyIgnoredErrors() +{ + mbus::EmptyReply r1; + mbus::EmptyReply r2; + std::vector<mbus::Error> errors = { + mbus::Error(DocumentProtocol::ERROR_MESSAGE_IGNORED, "oh no!"), + mbus::Error(DocumentProtocol::ERROR_MESSAGE_IGNORED, "oh dear!"), + mbus::Error(DocumentProtocol::ERROR_MESSAGE_IGNORED, "omg!") + }; + r1.addError(errors[0]); + r1.addError(errors[1]); + r2.addError(errors[2]); + ReplyMerger merger; + merger.merge(0, r1); + merger.merge(1, r2); + ReplyMerger::Result ret(merger.mergedReply()); + ASSERT_FALSE(ret.isSuccessful()); + ASSERT_TRUE(ret.hasGeneratedReply()); + std::unique_ptr<mbus::Reply> gen(ret.releaseGeneratedReply()); + // Only first ignore error from each reply. + assertReplyErrorsMatch(*gen, { errors[0], errors[2] }); +} + +void +Test::successfulReplyTakesPrecedenceOverIgnoredReplyWhenNoErrors() +{ + mbus::EmptyReply r1; + mbus::EmptyReply r2; + std::vector<mbus::Error> errors = { + mbus::Error(DocumentProtocol::ERROR_MESSAGE_IGNORED, "oh no!"), + }; + r1.addError(errors[0]); + ReplyMerger merger; + merger.merge(0, r1); + merger.merge(1, r2); + ReplyMerger::Result ret(merger.mergedReply()); + ASSERT_TRUE(ret.isSuccessful()); + ASSERT_FALSE(ret.hasGeneratedReply()); + EXPECT_EQUAL(1u, ret.getSuccessfulReplyIndex()); +} + +void +Test::nonIgnoredErrorTakesPrecedence() +{ + mbus::EmptyReply r1; + mbus::EmptyReply r2; + std::vector<mbus::Error> errors = { + mbus::Error(DocumentProtocol::ERROR_MESSAGE_IGNORED, "oh no!"), + mbus::Error(DocumentProtocol::ERROR_ABORTED, "kablammo!"), + mbus::Error(DocumentProtocol::ERROR_MESSAGE_IGNORED, "omg!") + }; + r1.addError(errors[0]); + r1.addError(errors[1]); + r2.addError(errors[2]); + ReplyMerger merger; + merger.merge(0, r1); + merger.merge(1, r2); + ReplyMerger::Result ret(merger.mergedReply()); + ASSERT_FALSE(ret.isSuccessful()); + ASSERT_TRUE(ret.hasGeneratedReply()); + std::unique_ptr<mbus::Reply> gen(ret.releaseGeneratedReply()); + // All errors from replies with errors are included, not those that + // are fully ignored. + assertReplyErrorsMatch(*gen, { errors[0], errors[1] }); +} + +void +Test::returnRemoveDocumentReplyWhereDocWasFound() +{ + RemoveDocumentReply r1; + RemoveDocumentReply r2; + RemoveDocumentReply r3; + r1.setWasFound(false); + r2.setWasFound(true); + r3.setWasFound(false); + + ReplyMerger merger; + merger.merge(0, r1); + merger.merge(1, r2); + merger.merge(2, r3); + ReplyMerger::Result ret(merger.mergedReply()); + ASSERT_TRUE(ret.isSuccessful()); + ASSERT_FALSE(ret.hasGeneratedReply()); + ASSERT_EQUAL(1u, ret.getSuccessfulReplyIndex()); +} + +void +Test::returnFirstRemoveDocumentReplyIfNoDocsWereFound() +{ + RemoveDocumentReply r1; + RemoveDocumentReply r2; + r1.setWasFound(false); + r2.setWasFound(false); + + ReplyMerger merger; + merger.merge(0, r1); + merger.merge(1, r2); + ReplyMerger::Result ret(merger.mergedReply()); + ASSERT_TRUE(ret.isSuccessful()); + ASSERT_FALSE(ret.hasGeneratedReply()); + ASSERT_EQUAL(0u, ret.getSuccessfulReplyIndex()); +} + +void +Test::returnUpdateDocumentReplyWhereDocWasFound() +{ + UpdateDocumentReply r1; + UpdateDocumentReply r2; + UpdateDocumentReply r3; + r1.setWasFound(false); + r2.setWasFound(true); // return first reply + r3.setWasFound(true); + + ReplyMerger merger; + merger.merge(0, r1); + merger.merge(1, r2); + merger.merge(2, r3); + ReplyMerger::Result ret(merger.mergedReply()); + ASSERT_TRUE(ret.isSuccessful()); + ASSERT_FALSE(ret.hasGeneratedReply()); + ASSERT_EQUAL(1u, ret.getSuccessfulReplyIndex()); +} + +void +Test::returnGetDocumentReplyWhereDocWasFound() +{ + GetDocumentReply r1; + GetDocumentReply r2; + GetDocumentReply r3; + r2.setLastModified(12345ULL); + + ReplyMerger merger; + merger.merge(0, r1); + merger.merge(1, r2); + merger.merge(2, r3); + ReplyMerger::Result ret(merger.mergedReply()); + ASSERT_TRUE(ret.isSuccessful()); + ASSERT_FALSE(ret.hasGeneratedReply()); + ASSERT_EQUAL(1u, ret.getSuccessfulReplyIndex()); +} + +void +Test::assertReplyErrorsMatch(const mbus::Reply& r, + const std::vector<mbus::Error>& errors) +{ + ASSERT_EQUAL(r.getNumErrors(), errors.size()); + for (size_t i = 0; i < errors.size(); ++i) { + ASSERT_EQUAL(errors[i].getCode(), r.getError(i).getCode()); + ASSERT_EQUAL(errors[i].getMessage(), r.getError(i).getMessage()); + } +} + +void +Test::mergingZeroRepliesReturnsDefaultEmptyReply() +{ + ReplyMerger merger; + ReplyMerger::Result ret(merger.mergedReply()); + ASSERT_FALSE(ret.isSuccessful()); + ASSERT_TRUE(ret.hasGeneratedReply()); + std::unique_ptr<mbus::Reply> gen(ret.releaseGeneratedReply()); + ASSERT_TRUE(dynamic_cast<mbus::EmptyReply*>(gen.get()) != 0); + assertReplyErrorsMatch(*gen, {}); +} + +#ifdef RUN_TEST +# error Someone defined RUN_TEST already! Oh no! +#endif +#define RUN_TEST(f) \ + std::cerr << "running test case '" #f "'\n"; \ + f(); TEST_FLUSH(); + +int +Test::Main() +{ + TEST_INIT("replymerger_test"); + + RUN_TEST(mergingGenericRepliesWithNoErrorsPicksFirstReply); + RUN_TEST(mergingSingleReplyWithOneErrorReturnsEmptyReplyWithError); + RUN_TEST(mergingSingleReplyWithMultipleErrorsReturnsEmptyReplyWithAllErrors); + RUN_TEST(mergingMultipleRepliesWithMultipleErrorsReturnsEmptyReplyWithAllErrors); + RUN_TEST(returnIgnoredReplyWhenAllRepliesHaveOnlyIgnoredErrors); + RUN_TEST(successfulReplyTakesPrecedenceOverIgnoredReplyWhenNoErrors); + RUN_TEST(nonIgnoredErrorTakesPrecedence); + RUN_TEST(returnRemoveDocumentReplyWhereDocWasFound); + RUN_TEST(returnFirstRemoveDocumentReplyIfNoDocsWereFound); + RUN_TEST(returnUpdateDocumentReplyWhereDocWasFound); + RUN_TEST(returnGetDocumentReplyWhereDocWasFound); + RUN_TEST(mergingZeroRepliesReturnsDefaultEmptyReply); + + TEST_DONE(); +} diff --git a/documentapi/src/tests/routablefactory/.gitignore b/documentapi/src/tests/routablefactory/.gitignore new file mode 100644 index 00000000000..bf482dc22db --- /dev/null +++ b/documentapi/src/tests/routablefactory/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +routablefactory_test +documentapi_routablefactory_test_app diff --git a/documentapi/src/tests/routablefactory/CMakeLists.txt b/documentapi/src/tests/routablefactory/CMakeLists.txt new file mode 100644 index 00000000000..ba6ea53bf8f --- /dev/null +++ b/documentapi/src/tests/routablefactory/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(documentapi_routablefactory_test_app + SOURCES + routablefactory.cpp + DEPENDS + documentapi +) +vespa_add_test(NAME documentapi_routablefactory_test_app COMMAND documentapi_routablefactory_test_app) diff --git a/documentapi/src/tests/routablefactory/DESC b/documentapi/src/tests/routablefactory/DESC new file mode 100644 index 00000000000..11c0cb8c0d3 --- /dev/null +++ b/documentapi/src/tests/routablefactory/DESC @@ -0,0 +1 @@ +routablefactory test. Take a look at routablefactory.cpp for details. diff --git a/documentapi/src/tests/routablefactory/FILES b/documentapi/src/tests/routablefactory/FILES new file mode 100644 index 00000000000..6176a49814e --- /dev/null +++ b/documentapi/src/tests/routablefactory/FILES @@ -0,0 +1 @@ +routablefactory.cpp diff --git a/documentapi/src/tests/routablefactory/routablefactory.cpp b/documentapi/src/tests/routablefactory/routablefactory.cpp new file mode 100644 index 00000000000..5bd7ee48471 --- /dev/null +++ b/documentapi/src/tests/routablefactory/routablefactory.cpp @@ -0,0 +1,242 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("routablefactory_test"); + +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/routablefactories51.h> +#include <vespa/messagebus/testlib/receptor.h> +#include <vespa/messagebus/testlib/slobrok.h> +#include <vespa/messagebus/testlib/testserver.h> +#include <vespa/vespalib/testkit/testapp.h> + +using document::DocumentTypeRepo; +using namespace documentapi; + +/////////////////////////////////////////////////////////////////////////////// +// +// Utilities +// +/////////////////////////////////////////////////////////////////////////////// + +class MyReply : public DocumentReply { +public: + enum { + TYPE = 777 + }; + + MyReply() : + DocumentReply(TYPE) { + // empty + } +}; + +class MyMessage : public DocumentMessage { +public: + enum { + TYPE = 666 + }; + + MyMessage() { + getTrace().setLevel(9); + } + + DocumentReply::UP doCreateReply() const { + return DocumentReply::UP(new MyReply()); + } + + uint32_t getType() const { + return TYPE; + } +}; + +class MyMessageFactory : public RoutableFactories51::DocumentMessageFactory { +protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const { + (void)buf; + return DocumentMessage::UP(new MyMessage()); + } + + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const { + (void)msg; + (void)buf; + return true; + } +}; + +class MyReplyFactory : public RoutableFactories51::DocumentReplyFactory { +protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const { + (void)buf; + return DocumentReply::UP(new MyReply()); + } + + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const { + (void)reply; + (void)buf; + return true; + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// +// Setup +// +/////////////////////////////////////////////////////////////////////////////// + +class TestData { + const DocumentTypeRepo::SP _repo; + +public: + mbus::Slobrok _slobrok; + LoadTypeSet _loadTypes; + DocumentProtocol::SP _srcProtocol; + mbus::TestServer _srcServer; + mbus::SourceSession::UP _srcSession; + mbus::Receptor _srcHandler; + DocumentProtocol::SP _dstProtocol; + mbus::TestServer _dstServer; + mbus::DestinationSession::UP _dstSession; + mbus::Receptor _dstHandler; + +public: + TestData(); + bool start(); +}; + +class Test : public vespalib::TestApp { +protected: + void testFactory(TestData &data); + +public: + int Main(); +}; + +TEST_APPHOOK(Test); + +TestData::TestData() : + _repo(new DocumentTypeRepo), + _slobrok(), + _loadTypes(), + _srcProtocol(new DocumentProtocol(_loadTypes, _repo)), + _srcServer(mbus::MessageBusParams().addProtocol(_srcProtocol), + mbus::RPCNetworkParams().setSlobrokConfig(_slobrok.config())), + _srcSession(), + _srcHandler(), + _dstProtocol(new DocumentProtocol(_loadTypes, _repo)), + _dstServer(mbus::MessageBusParams().addProtocol(_dstProtocol), + mbus::RPCNetworkParams().setIdentity(mbus::Identity("dst")).setSlobrokConfig(_slobrok.config())), + _dstSession(), + _dstHandler() +{ + // empty +} + +bool +TestData::start() +{ + _srcSession = _srcServer.mb.createSourceSession(mbus::SourceSessionParams().setReplyHandler(_srcHandler)); + if (_srcSession.get() == NULL) { + return false; + } + _dstSession = _dstServer.mb.createDestinationSession(mbus::DestinationSessionParams().setName("session").setMessageHandler(_dstHandler)); + if (_dstSession.get() == NULL) { + return false; + } + if (!_srcServer.waitSlobrok("dst/session", 1u)) { + return false; + } + return true; +} + +int +Test::Main() +{ + TEST_INIT("routablefactory_test"); + + TestData data; + ASSERT_TRUE(data.start()); + + testFactory(data); TEST_FLUSH(); + + TEST_DONE(); +} + +/////////////////////////////////////////////////////////////////////////////// +// +// Tests +// +/////////////////////////////////////////////////////////////////////////////// + +void +Test::testFactory(TestData &data) +{ + mbus::Route route = mbus::Route::parse("dst/session"); + + // Source should fail to encode the message. + EXPECT_TRUE(data._srcSession->send(mbus::Message::UP(new MyMessage()), route).isAccepted()); + mbus::Reply::UP reply = data._srcHandler.getReply(600); + ASSERT_TRUE(reply.get() != NULL); + fprintf(stderr, "%s\n", reply->getTrace().toString().c_str()); + ASSERT_TRUE(reply->hasErrors()); + EXPECT_EQUAL((uint32_t)mbus::ErrorCode::ENCODE_ERROR, reply->getError(0).getCode()); + EXPECT_EQUAL("", reply->getError(0).getService()); + + // Destination should fail to decode the message. + data._srcProtocol->putRoutableFactory(MyMessage::TYPE, IRoutableFactory::SP(new MyMessageFactory()), + vespalib::VersionSpecification()); + EXPECT_TRUE(data._srcSession->send(mbus::Message::UP(new MyMessage()), route).isAccepted()); + reply = data._srcHandler.getReply(600); + ASSERT_TRUE(reply.get() != NULL); + fprintf(stderr, "%s\n", reply->getTrace().toString().c_str()); + EXPECT_TRUE(reply->hasErrors()); + EXPECT_EQUAL((uint32_t)mbus::ErrorCode::DECODE_ERROR, reply->getError(0).getCode()); + EXPECT_EQUAL("dst/session", reply->getError(0).getService()); + + // Destination should fail to encode the reply-> + data._dstProtocol->putRoutableFactory(MyMessage::TYPE, IRoutableFactory::SP(new MyMessageFactory()), + vespalib::VersionSpecification()); + EXPECT_TRUE(data._srcSession->send(mbus::Message::UP(new MyMessage()), route).isAccepted()); + mbus::Message::UP msg = data._dstHandler.getMessage(600); + ASSERT_TRUE(msg.get() != NULL); + reply.reset(new MyReply()); + reply->swapState(*msg); + data._dstSession->reply(std::move(reply)); + reply = data._srcHandler.getReply(600); + ASSERT_TRUE(reply.get() != NULL); + fprintf(stderr, "%s\n", reply->getTrace().toString().c_str()); + EXPECT_TRUE(reply->hasErrors()); + EXPECT_EQUAL((uint32_t)mbus::ErrorCode::ENCODE_ERROR, reply->getError(0).getCode()); + EXPECT_EQUAL("dst/session", reply->getError(0).getService()); + + // Source should fail to decode the reply. + data._dstProtocol->putRoutableFactory(MyReply::TYPE, IRoutableFactory::SP(new MyReplyFactory()), + vespalib::VersionSpecification()); + EXPECT_TRUE(data._srcSession->send(mbus::Message::UP(new MyMessage()), route).isAccepted()); + msg = data._dstHandler.getMessage(600); + ASSERT_TRUE(msg.get() != NULL); + reply.reset(new MyReply()); + reply->swapState(*msg); + data._dstSession->reply(std::move(reply)); + reply = data._srcHandler.getReply(600); + ASSERT_TRUE(reply.get() != NULL); + fprintf(stderr, "%s\n", reply->getTrace().toString().c_str()); + EXPECT_TRUE(reply->hasErrors()); + EXPECT_EQUAL((uint32_t)mbus::ErrorCode::DECODE_ERROR, reply->getError(0).getCode()); + EXPECT_EQUAL("", reply->getError(0).getService()); + + // All should succeed. + data._srcProtocol->putRoutableFactory(MyReply::TYPE, IRoutableFactory::SP(new MyReplyFactory()), + vespalib::VersionSpecification()); + EXPECT_TRUE(data._srcSession->send(mbus::Message::UP(new MyMessage()), route).isAccepted()); + msg = data._dstHandler.getMessage(600); + ASSERT_TRUE(msg.get() != NULL); + reply.reset(new MyReply()); + reply->swapState(*msg); + data._dstSession->reply(std::move(reply)); + reply = data._srcHandler.getReply(600); + ASSERT_TRUE(reply.get() != NULL); + fprintf(stderr, "%s\n", reply->getTrace().toString().c_str()); + EXPECT_TRUE(!reply->hasErrors()); +} diff --git a/documentapi/src/tests/systemstate/.gitignore b/documentapi/src/tests/systemstate/.gitignore new file mode 100644 index 00000000000..3f52bc38742 --- /dev/null +++ b/documentapi/src/tests/systemstate/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +systemstate_test +documentapi_systemstate_test_app diff --git a/documentapi/src/tests/systemstate/CMakeLists.txt b/documentapi/src/tests/systemstate/CMakeLists.txt new file mode 100644 index 00000000000..a1a88c66278 --- /dev/null +++ b/documentapi/src/tests/systemstate/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(documentapi_systemstate_test_app + SOURCES + systemstate.cpp + DEPENDS + documentapi +) +vespa_add_test(NAME documentapi_systemstate_test_app COMMAND documentapi_systemstate_test_app) diff --git a/documentapi/src/tests/systemstate/DESC b/documentapi/src/tests/systemstate/DESC new file mode 100644 index 00000000000..19dbc9195f1 --- /dev/null +++ b/documentapi/src/tests/systemstate/DESC @@ -0,0 +1,3 @@ +This is a unit test for the system state parser and the corresponding NodeState class. It mirrors the +StateParserTestCase available in the java implementation of message bus. It consists of tests that verify that parsing +works, pathing works, encoding/decoding works, and finally that the NodeState class works as intended. diff --git a/documentapi/src/tests/systemstate/FILES b/documentapi/src/tests/systemstate/FILES new file mode 100644 index 00000000000..e1d0e026d31 --- /dev/null +++ b/documentapi/src/tests/systemstate/FILES @@ -0,0 +1 @@ +systemstate.cpp diff --git a/documentapi/src/tests/systemstate/systemstate.cpp b/documentapi/src/tests/systemstate/systemstate.cpp new file mode 100644 index 00000000000..b8f6c04fa2b --- /dev/null +++ b/documentapi/src/tests/systemstate/systemstate.cpp @@ -0,0 +1,225 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("systemstate_test"); + +#include <vespa/documentapi/messagebus/systemstate/systemstate.h> +#include <vespa/documentapi/messagebus/systemstate/systemstatehandle.h> +#include <vespa/vespalib/testkit/testapp.h> + +using namespace documentapi; + +class Test : public vespalib::TestApp { +public: + int Main(); + void testParser(); + void testPathing(); + void testState(); + void testEncoding(); + void testHandle(); + void testCompact(); + +private: + void assertParser(const string &state, const string &expected = ""); +}; + +TEST_APPHOOK(Test); + +int +Test::Main() +{ + TEST_INIT("systemstate_test"); + + testParser(); TEST_FLUSH(); + testPathing(); TEST_FLUSH(); + testState(); TEST_FLUSH(); + testEncoding(); TEST_FLUSH(); + testHandle(); TEST_FLUSH(); + testCompact(); TEST_FLUSH(); + + TEST_DONE(); + return 0; +} + +void +Test::testParser() +{ + assertParser("storage"); + assertParser("storage?", "ERROR"); + assertParser("storage?a", "ERROR"); + assertParser("storage?a=", "ERROR"); + assertParser("storage?a=1"); + assertParser("storage?a=1&", "ERROR"); + assertParser("storage?a=1&b", "ERROR"); + assertParser("storage?a=1&b=2"); + assertParser("storage?a=1&b=2 search"); + assertParser("storage?a=1&b=2 search?", "ERROR"); + assertParser("storage?a=1&b=2 search?a", "ERROR"); + assertParser("storage?a=1&b=2 search?a=", "ERROR"); + assertParser("storage?a=1&b=2 search?a=1"); + assertParser("storage?a=1&b=2 search?a=1&", "ERROR"); + assertParser("storage?a=1&b=2 search?a=1&b", "ERROR"); + assertParser("storage?a=1&b=2 search?a=1&b=", "ERROR"); + assertParser("storage?a=1&b=2 search?a=1&b=2"); + + assertParser("storage"); + assertParser("storage/"); + assertParser("storage/?", "ERROR"); + assertParser("storage/?a", "ERROR"); + assertParser("storage/?a=", "ERROR"); + assertParser("storage/?a=1"); + assertParser("storage/cluster.storage"); + assertParser("storage/cluster.storage/"); + + assertParser("storage?a=1"); + assertParser("storage/?a=1"); + assertParser("storage/.?a=1"); + assertParser("storage/./?a=1"); + assertParser("storage/./cluster.storage?a=1"); + assertParser("storage/./cluster.storage/?a=1"); + assertParser("storage/./cluster.storage/..?a=1"); + assertParser("storage/./cluster.storage/../?a=1"); + assertParser("storage/./cluster.storage/../storage?a=1"); + assertParser("storage/./cluster.storage/../storage/?a=1"); +} + +void +Test::testPathing() +{ + assertParser("storage?a=1", "storage?a=1"); + assertParser("storage/?a=1", "storage?a=1"); + assertParser("storage/.?a=1", "storage?a=1"); + assertParser("storage/./?a=1", "storage?a=1"); + assertParser("storage/./cluster.storage?a=1", "storage/cluster.storage?a=1"); + assertParser("storage/./cluster.storage/?a=1", "storage/cluster.storage?a=1"); + assertParser("storage/./cluster.storage/..?a=1", "storage?a=1"); + assertParser("storage/./cluster.storage/../?a=1", "storage?a=1"); + assertParser("storage/./cluster.storage/../storage?a=1", "storage/storage?a=1"); + assertParser("storage/./cluster.storage/../storage/?a=1", "storage/storage?a=1"); + + assertParser("a?p1=1 a/b?p2=2 a/b/c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); + assertParser("a .?p1=1 ./b?p2=2 ./b/c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); + assertParser("a .?p1=1 ./../a/b/ .?p2=2 c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); + assertParser("a/./ .?p1=1 ../a/b/c/.. .?p2=2 ./c/../c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); + assertParser("a/b/c/d/ ../../ ../ ../a .?p1=1 ./b?p2=2 ./ ../a/b/c?p3=3", "a?p1=1 a/b?p2=2 a/b/c?p3=3"); + + assertParser("a/b/c/d?p1=1 a?p2=2", "a?p2=2 a/b/c/d?p1=1"); + assertParser("a/b/c/d/?p1=1 /a?p2=2", "a?p2=2 a/b/c/d?p1=1"); + assertParser("/a/b/c/d/?p1=1 /a?p2=2", "a?p2=2 a/b/c/d?p1=1"); + + assertParser("a .?p1=1", "a?p1=1"); + assertParser("a/b .?p1=1", "a/b?p1=1"); + assertParser("a/b c?p1=1 d?p2=2", "a/b/c?p1=1 a/b/d?p2=2"); +} + +void +Test::testState() +{ + NodeState state; + state + .addChild("distributor", NodeState() + .setState("n", "27")) + .addChild("storage", NodeState() + .setState("n", "170") + .addChild("2", NodeState() + .setState("s", "d")) + .addChild("13", NodeState() + .setState("s", "r") + .setState("c", "0.0"))); + + EXPECT_EQUAL("27", state.getState("distributor/n")); + EXPECT_EQUAL("170", state.getState("storage/n")); + EXPECT_EQUAL("d", state.getState("storage/2/s")); + EXPECT_EQUAL("r", state.getState("storage/13/s")); + EXPECT_EQUAL("0.0", state.getState("storage/13/c")); + + EXPECT_EQUAL("27", state.getChild("distributor")->getState("n")); + EXPECT_EQUAL("170", state.getChild("storage")->getState("n")); + EXPECT_EQUAL("d", state.getChild("storage")->getChild("2")->getState("s")); + EXPECT_EQUAL("r", state.getChild("storage")->getChild("13")->getState("s")); + EXPECT_EQUAL("0.0", state.getChild("storage")->getChild("13")->getState("c")); +} + +void +Test::testEncoding() +{ + NodeState state; + state.setState("foo", "http://search.yahoo.com/?query=bar"); + LOG(info, "'%s'", state.toString().c_str()); + EXPECT_EQUAL(".?foo=http%3A%2F%2Fsearch.yahoo.com%2F%3Fquery%3Dbar", state.toString()); + assertParser(state.toString(), state.toString()); + + state = NodeState() + .addChild("foo:bar", NodeState() + .setState("foo", "http://search.yahoo.com/?query=bar")); + LOG(info, "'%s'", state.toString().c_str()); + EXPECT_EQUAL("foo%3Abar?foo=http%3A%2F%2Fsearch.yahoo.com%2F%3Fquery%3Dbar", state.toString()); + assertParser(state.toString(), state.toString()); + + state = NodeState() + .addChild("foo/bar", NodeState() + .setState("foo", "http://search.yahoo.com/?query=bar")); + LOG(info, "'%s'", state.toString().c_str()); + EXPECT_EQUAL("foo/bar?foo=http%3A%2F%2Fsearch.yahoo.com%2F%3Fquery%3Dbar", state.toString()); + assertParser(state.toString(), state.toString()); +} + +void +Test::testHandle() +{ + SystemState::UP state(SystemState::newInstance("")); + ASSERT_TRUE(state.get() != NULL); + + SystemStateHandle handle(*state); + ASSERT_TRUE(handle.isValid()); + + SystemStateHandle hoe(handle); + ASSERT_TRUE(!handle.isValid()); + ASSERT_TRUE(hoe.isValid()); +} + +void +Test::testCompact() +{ + NodeState state; + state + .setState("a/b0/s", "d") + .setState("a/b0/c0/s", "d") + .setState("a/b0/c1/s", "d") + .setState("a/b1/s", "d") + .setState("a/b1/c0/s", "d") + .setState("a/b1/c1/s", "d"); + EXPECT_EQUAL("a/b0?s=d a/b0/c0?s=d a/b0/c1?s=d a/b1?s=d a/b1/c0?s=d a/b1/c1?s=d", state.toString()); + + state.removeChild("a/b0/c0"); + EXPECT_EQUAL("a/b0?s=d a/b0/c1?s=d a/b1?s=d a/b1/c0?s=d a/b1/c1?s=d", state.toString()); + + state.removeState("a/b0/c1/s"); + EXPECT_EQUAL("a/b0?s=d a/b1?s=d a/b1/c0?s=d a/b1/c1?s=d", state.toString()); + + state.setState("a/b1/c0/s", ""); + EXPECT_EQUAL("a/b0?s=d a/b1?s=d a/b1/c1?s=d", state.toString()); + + state.removeChild("a/b1"); + EXPECT_EQUAL("a/b0?s=d", state.toString()); + + state.removeChild("a"); + EXPECT_EQUAL("", state.toString()); +} + +void +Test::assertParser(const string &state, const string &expected) +{ + SystemState::UP obj = SystemState::newInstance(state); + if (obj.get() == NULL) { + EXPECT_EQUAL("ERROR", expected); + } + else { + SystemStateHandle handle(*obj); + LOG(info, "'%s' => '%s'", state.c_str(), handle.getRoot().toString().c_str()); + if (!expected.empty()) { + EXPECT_EQUAL(expected, handle.getRoot().toString()); + } + } +} + diff --git a/documentapi/src/vespa/binref/.gitignore b/documentapi/src/vespa/binref/.gitignore new file mode 100644 index 00000000000..cfb0e619824 --- /dev/null +++ b/documentapi/src/vespa/binref/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +testrun.sh diff --git a/documentapi/src/vespa/binref/CMakeLists.txt b/documentapi/src/vespa/binref/CMakeLists.txt new file mode 100644 index 00000000000..5c90dd5bfcc --- /dev/null +++ b/documentapi/src/vespa/binref/CMakeLists.txt @@ -0,0 +1 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. diff --git a/documentapi/src/vespa/documentapi/.gitignore b/documentapi/src/vespa/documentapi/.gitignore new file mode 100644 index 00000000000..8e4505e0af2 --- /dev/null +++ b/documentapi/src/vespa/documentapi/.gitignore @@ -0,0 +1,5 @@ +.depend +Makefile +config-*.cpp +config-*.h +/libdocumentapi.so.5.1 diff --git a/documentapi/src/vespa/documentapi/CMakeLists.txt b/documentapi/src/vespa/documentapi/CMakeLists.txt new file mode 100644 index 00000000000..890cc2905c2 --- /dev/null +++ b/documentapi/src/vespa/documentapi/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(documentapi + SOURCES + $<TARGET_OBJECTS:documentapi_documentapimessagebus> + $<TARGET_OBJECTS:documentapi_documentapimessages> + $<TARGET_OBJECTS:documentapi_documentapipolicies> + $<TARGET_OBJECTS:documentapi_documentapisystemstate> + $<TARGET_OBJECTS:documentapi_documentapiloadtypes> + INSTALL lib64 + DEPENDS +) diff --git a/documentapi/src/vespa/documentapi/common.h b/documentapi/src/vespa/documentapi/common.h new file mode 100644 index 00000000000..f329929ce7e --- /dev/null +++ b/documentapi/src/vespa/documentapi/common.h @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/vstringfmt.h> + +namespace documentapi { + +// Decide the type of string used once +typedef vespalib::string string; + +} // namespace mbus + diff --git a/documentapi/src/vespa/documentapi/documentapi.h b/documentapi/src/vespa/documentapi/documentapi.h new file mode 100644 index 00000000000..e1b9b5e4b03 --- /dev/null +++ b/documentapi/src/vespa/documentapi/documentapi.h @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h> +#include <vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h> +#include <vespa/documentapi/messagebus/messages/getbucketstatemessage.h> +#include <vespa/documentapi/messagebus/messages/getbucketstatereply.h> +#include <vespa/documentapi/messagebus/messages/getdocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/getdocumentreply.h> +#include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/removedocumentreply.h> +#include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> +#include <vespa/documentapi/messagebus/messages/feedreply.h> +#include <vespa/documentapi/messagebus/messages/updatedocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/searchresultmessage.h> +#include <vespa/documentapi/messagebus/messages/visitor.h> +#include <vespa/documentapi/messagebus/messages/multioperationmessage.h> +#include <vespa/documentapi/messagebus/messages/documentsummarymessage.h> +#include <vespa/documentapi/messagebus/messages/wrongdistributionreply.h> +#include <vespa/documentapi/messagebus/messages/getbucketlistmessage.h> +#include <vespa/documentapi/messagebus/messages/getbucketlistreply.h> +#include <vespa/documentapi/messagebus/messages/statbucketmessage.h> +#include <vespa/documentapi/messagebus/messages/statbucketreply.h> +#include <vespa/documentapi/messagebus/messages/documentstate.h> +#include <vespa/documentapi/messagebus/messages/emptybucketsmessage.h> +#include <vespa/documentapi/messagebus/messages/removelocationmessage.h> +#include <vespa/documentapi/messagebus/messages/queryresultmessage.h> +#include <vespa/documentapi/messagebus/messages/documentignoredreply.h> + + +namespace documentapi { + +} + diff --git a/documentapi/src/vespa/documentapi/loadtypes/CMakeLists.txt b/documentapi/src/vespa/documentapi/loadtypes/CMakeLists.txt new file mode 100644 index 00000000000..9be6458174a --- /dev/null +++ b/documentapi/src/vespa/documentapi/loadtypes/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(documentapi_documentapiloadtypes OBJECT + SOURCES + loadtype.cpp + DEPENDS +) diff --git a/documentapi/src/vespa/documentapi/loadtypes/loadtype.cpp b/documentapi/src/vespa/documentapi/loadtypes/loadtype.cpp new file mode 100644 index 00000000000..6cdccc6ad21 --- /dev/null +++ b/documentapi/src/vespa/documentapi/loadtypes/loadtype.cpp @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/loadtypes/loadtype.h> + +namespace documentapi { + +const LoadType LoadType::DEFAULT(0, "default", Priority::PRI_NORMAL_3); + +void +LoadType::print(vespalib::asciistream & os) const +{ + os << "LoadType(" << getId() << ": " << getName() << ")"; +} + +} // documentapi diff --git a/documentapi/src/vespa/documentapi/loadtypes/loadtype.h b/documentapi/src/vespa/documentapi/loadtypes/loadtype.h new file mode 100644 index 00000000000..fb00c5805b6 --- /dev/null +++ b/documentapi/src/vespa/documentapi/loadtypes/loadtype.h @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class LoadType + * \ingroup loadtype + * + * \brief Class used to identify a given load type. + * + * A load type is a type given to a Vespa operation that is independent of the + * message type, priority or such information. Load types are given by clients + * to external load (if not given, default load type is used), and might also be + * set by the system itself for maintenance load. + */ + +#pragma once + +#include <vespa/metrics/loadmetric.h> +#include <string> +#include <vespa/vespalib/util/linkedptr.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/documentapi/messagebus/priority.h> + +namespace documentapi { + +class LoadTypeSet; + +// Inherit metrics loadtype so it is easy to use load types in load metrics. +class LoadType : public metrics::LoadType { + Priority::Value _priority; + +public: + typedef vespalib::LinkedPtr<LoadType> LP; + + LoadType(uint32_t id, const string& name, Priority::Value priority) + : metrics::LoadType(id, name), _priority(priority) {} + static const LoadType DEFAULT; + + Priority::Value getPriority() const { return _priority; } +private: + void print(vespalib::asciistream & os) const; +}; + +} // documentapi + diff --git a/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.h b/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.h new file mode 100644 index 00000000000..19b8ce61dbf --- /dev/null +++ b/documentapi/src/vespa/documentapi/loadtypes/loadtypeset.h @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class LoadTypeSet + * \ingroup loadtype + * + * \brief Class containing all the various load types that have been configured. + * + * The load type set makes configured load types available in an easy way for + * different parts of Vespa to access it. + */ +#pragma once + +#include <vespa/config/config.h> +#include <vespa/documentapi/loadtypes/loadtype.h> +#include <vespa/metrics/loadmetric.h> +#include <vector> +#include <vespa/config-load-type.h> +#include <vespa/vespalib/stllike/hash_map.h> + +namespace documentapi { + +class LoadTypeSet +{ + vespalib::hash_map<uint32_t, LoadType::LP> _types; + // Want order to be ~ alphabetical. + std::map<string, LoadType*> _nameMap; + + // This object cannot be copied + LoadTypeSet(const LoadTypeSet&); + LoadTypeSet& operator=(const LoadTypeSet&); + + void configure(const vespa::config::content::LoadTypeConfig& config) { + // This configure does not support live reconfig + if (!_types.empty()) return; + + addLoadType(0, LoadType::DEFAULT.getName(), LoadType::DEFAULT.getPriority()); + + for (uint32_t i=0; i<config.type.size(); ++i) { + addLoadType(config.type[i].id, config.type[i].name, Priority::getPriority(config.type[i].priority)); + } + } + +public: + typedef std::unique_ptr<LoadTypeSet> UP; + typedef std::shared_ptr<LoadTypeSet> SP; + + LoadTypeSet() { + addLoadType(0, LoadType::DEFAULT.getName(), LoadType::DEFAULT.getPriority()); + } + + LoadTypeSet(const config::ConfigUri & configUri) { + std::unique_ptr<vespa::config::content::LoadTypeConfig> cfg = + config::ConfigGetter<vespa::config::content::LoadTypeConfig>::getConfig(configUri.getConfigId(), configUri.getContext()); + configure(*cfg); + } + + LoadTypeSet(const vespa::config::content::LoadTypeConfig& config) { + configure(config); + } + + void addLoadType(uint32_t id, const string& name, Priority::Value priority) { + vespalib::hash_map<uint32_t, LoadType::LP>::iterator it( + _types.find(id)); + if (it != _types.end()) { + throw config::InvalidConfigException( + "Load type identifiers need to be non-overlapping, 1+ " + "and without gaps.\n", VESPA_STRLOC); + } + if (_nameMap.find(name) != _nameMap.end()) { + throw config::InvalidConfigException( + "Load type names need to be unique and different from " + "the reserved name \"default\".", VESPA_STRLOC); + } + _types[id] = LoadType::LP(new LoadType(id, name, priority)); + _nameMap[name] = _types[id].get(); + } + + const std::map<string, LoadType*>& getLoadTypes() const + { return _nameMap; } + metrics::LoadTypeSet getMetricLoadTypes() const { + metrics::LoadTypeSet result; + for (vespalib::hash_map<uint32_t, LoadType::LP>::const_iterator it + = _types.begin(); it != _types.end(); ++it) + { + result.push_back(metrics::LoadType( + it->first, it->second->getName())); + } + return result; + } + + const LoadType& operator[](uint32_t id) const { + vespalib::hash_map<uint32_t, LoadType::LP>::const_iterator it( + _types.find(id)); + return (it == _types.end() ? LoadType::DEFAULT : *it->second); + } + const LoadType& operator[](const string& name) const { + std::map<string, LoadType*>::const_iterator it( + _nameMap.find(name)); + + return (it == _nameMap.end() ? LoadType::DEFAULT : *it->second); + } + + uint32_t size() const { return uint32_t(_types.size()); } + + /** + * Attempts to locate a load type with given name. Returns 0 if none found. + */ + const LoadType* findLoadType(const string& name) const { + std::map<string, LoadType*>::const_iterator it( + _nameMap.find(name)); + return (it == _nameMap.end() ? 0 : it->second); + } +}; + +} // documentapi + diff --git a/documentapi/src/vespa/documentapi/messagebus/.gitignore b/documentapi/src/vespa/documentapi/messagebus/.gitignore new file mode 100644 index 00000000000..d58390943e2 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +config-*.cpp +config-*.h diff --git a/documentapi/src/vespa/documentapi/messagebus/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/CMakeLists.txt new file mode 100644 index 00000000000..81dcbea1bbe --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(documentapi_documentapimessagebus OBJECT + SOURCES + documentprotocol.cpp + replymerger.cpp + routablefactories41.cpp + routablefactories42.cpp + routablefactories50.cpp + routablefactories51.cpp + routablefactories52.cpp + routablerepository.cpp + routingpolicyfactories.cpp + routingpolicyrepository.cpp + DEPENDS + documentapi_documentapipolicies +) diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp new file mode 100644 index 00000000000..127235211d6 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -0,0 +1,254 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".documentprotocol"); + +#include "routablefactories50.h" +#include "routablefactories51.h" +#include "routablefactories52.h" +#include "routingpolicyfactories.h" +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/util/stringutil.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/documentapi/messagebus/replymerger.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/vespalib/component/versionspecification.h> +#include <vespa/vespalib/util/exceptions.h> + +using document::DocumentTypeRepo; + +namespace documentapi { + +const mbus::string DocumentProtocol::NAME = "document"; + +DocumentProtocol::DocumentProtocol(const LoadTypeSet& loadTypes, + DocumentTypeRepo::SP repo, + const string &configId) : + _routingPolicyRepository(), + _routableRepository(loadTypes), + _systemState(SystemState::newInstance("")), + _repo(repo) +{ + // Prepare config string for routing policy factories. + string cfg = (configId.empty() ? "client" : configId); + + // When adding factories to this list, please KEEP THEM ORDERED alphabetically like they are now. + putRoutingPolicyFactory("AND", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::AndPolicyFactory())); + putRoutingPolicyFactory("Content", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::ContentPolicyFactory())); + putRoutingPolicyFactory("MessageType", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::MessageTypePolicyFactory())); + putRoutingPolicyFactory("DocumentRouteSelector", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::DocumentRouteSelectorPolicyFactory(*_repo, cfg))); + putRoutingPolicyFactory("Extern", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::ExternPolicyFactory())); + putRoutingPolicyFactory("LocalService", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::LocalServicePolicyFactory())); + putRoutingPolicyFactory("RoundRobin", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::RoundRobinPolicyFactory())); + putRoutingPolicyFactory("SearchColumn", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::SearchColumnPolicyFactory())); + putRoutingPolicyFactory("SearchRow", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::SearchRowPolicyFactory())); + putRoutingPolicyFactory("Storage", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::StoragePolicyFactory())); + putRoutingPolicyFactory("SubsetService", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::SubsetServicePolicyFactory())); + putRoutingPolicyFactory("LoadBalancer", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::LoadBalancerPolicyFactory())); + + // Prepare version specifications to use when adding routable factories. + vespalib::VersionSpecification version50(5, 0); + vespalib::VersionSpecification version51(5, 1); + vespalib::VersionSpecification version52(5, 115); + + std::vector<vespalib::VersionSpecification> from50 = { version50, version51, version52 }; + std::vector<vespalib::VersionSpecification> from51 = { version51, version52 }; + std::vector<vespalib::VersionSpecification> from52 = { version52 }; + + // Add 5.0 serialization + putRoutableFactory(MESSAGE_BATCHDOCUMENTUPDATE, IRoutableFactory::SP(new RoutableFactories50::BatchDocumentUpdateMessageFactory(*_repo)), from50); + putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories50::CreateVisitorMessageFactory(*_repo)), from50); + putRoutableFactory(MESSAGE_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories50::DestroyVisitorMessageFactory()), from50); + putRoutableFactory(MESSAGE_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories50::DocumentListMessageFactory(*_repo)), from50); + putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories50::DocumentSummaryMessageFactory()), from50); + putRoutableFactory(MESSAGE_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories50::EmptyBucketsMessageFactory()), from50); + putRoutableFactory(MESSAGE_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories50::GetBucketListMessageFactory()), from50); + putRoutableFactory(MESSAGE_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories50::GetBucketStateMessageFactory()), from50); + putRoutableFactory(MESSAGE_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::GetDocumentMessageFactory()), from50); + putRoutableFactory(MESSAGE_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories50::MapVisitorMessageFactory(*_repo)), from50); + putRoutableFactory(MESSAGE_MULTIOPERATION, IRoutableFactory::SP(new RoutableFactories50::MultiOperationMessageFactory(_repo)), from50); + putRoutableFactory(MESSAGE_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::PutDocumentMessageFactory(*_repo)), from50); + putRoutableFactory(MESSAGE_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories50::QueryResultMessageFactory()), from50); + putRoutableFactory(MESSAGE_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::RemoveDocumentMessageFactory()), from50); + putRoutableFactory(MESSAGE_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories50::RemoveLocationMessageFactory(*_repo)), from50); + putRoutableFactory(MESSAGE_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories50::SearchResultMessageFactory()), from50); + putRoutableFactory(MESSAGE_STATBUCKET, IRoutableFactory::SP(new RoutableFactories50::StatBucketMessageFactory()), from50); + putRoutableFactory(MESSAGE_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::UpdateDocumentMessageFactory(*_repo)), from50); + putRoutableFactory(MESSAGE_VISITORINFO, IRoutableFactory::SP(new RoutableFactories50::VisitorInfoMessageFactory()), from50); + putRoutableFactory(REPLY_BATCHDOCUMENTUPDATE, IRoutableFactory::SP(new RoutableFactories50::BatchDocumentUpdateReplyFactory()), from50); + putRoutableFactory(REPLY_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories50::CreateVisitorReplyFactory()), from50); + putRoutableFactory(REPLY_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories50::DestroyVisitorReplyFactory()), from50); + putRoutableFactory(REPLY_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories50::DocumentListReplyFactory()), from50); + putRoutableFactory(REPLY_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories50::DocumentSummaryReplyFactory()), from50); + putRoutableFactory(REPLY_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories50::EmptyBucketsReplyFactory()), from50); + putRoutableFactory(REPLY_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories50::GetBucketListReplyFactory()), from50); + putRoutableFactory(REPLY_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories50::GetBucketStateReplyFactory()), from50); + putRoutableFactory(REPLY_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::GetDocumentReplyFactory(*_repo)), from50); + putRoutableFactory(REPLY_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories50::MapVisitorReplyFactory()), from50); + putRoutableFactory(REPLY_MULTIOPERATION, IRoutableFactory::SP(new RoutableFactories50::MultiOperationReplyFactory()), from50); + putRoutableFactory(REPLY_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::PutDocumentReplyFactory()), from50); + putRoutableFactory(REPLY_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories50::QueryResultReplyFactory()), from50); + putRoutableFactory(REPLY_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::RemoveDocumentReplyFactory()), from50); + putRoutableFactory(REPLY_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories50::RemoveLocationReplyFactory()), from50); + putRoutableFactory(REPLY_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories50::SearchResultReplyFactory()), from50); + putRoutableFactory(REPLY_STATBUCKET, IRoutableFactory::SP(new RoutableFactories50::StatBucketReplyFactory()), from50); + putRoutableFactory(REPLY_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::UpdateDocumentReplyFactory()), from50); + putRoutableFactory(REPLY_VISITORINFO, IRoutableFactory::SP(new RoutableFactories50::VisitorInfoReplyFactory()), from50); + putRoutableFactory(REPLY_WRONGDISTRIBUTION, IRoutableFactory::SP(new RoutableFactories50::WrongDistributionReplyFactory()), from50); + + // Add 5.1 serialization + putRoutableFactory(MESSAGE_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories51::GetDocumentMessageFactory()), from51); + putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories51::CreateVisitorMessageFactory(*_repo)), from51); + putRoutableFactory(REPLY_DOCUMENTIGNORED, IRoutableFactory::SP(new RoutableFactories51::DocumentIgnoredReplyFactory()), from51); + + // Add 5.2 serialization + putRoutableFactory(MESSAGE_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::PutDocumentMessageFactory(*_repo)), from52); + putRoutableFactory(MESSAGE_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::UpdateDocumentMessageFactory(*_repo)), from52); + putRoutableFactory(MESSAGE_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::RemoveDocumentMessageFactory()), from52); +} + +mbus::IRoutingPolicy::UP +DocumentProtocol::createPolicy(const mbus::string &name, const mbus::string ¶m) const +{ + return _routingPolicyRepository.createPolicy(name, param); +} + +DocumentProtocol & +DocumentProtocol::putRoutingPolicyFactory(const string &name, IRoutingPolicyFactory::SP factory) +{ + _routingPolicyRepository.putFactory(name, factory); + return *this; +} + +mbus::Blob +DocumentProtocol::encode(const vespalib::Version &version, const mbus::Routable &routable) const +{ + mbus::Blob blob(_routableRepository.encode(version, routable)); + // When valgrind reports errors of uninitialized data being written to + // the network, it is useful to be able to see the serialized data to + // try to identify what bits are uninitialized. + if (LOG_WOULD_LOG(spam)) { + std::ostringstream message; + document::StringUtil::printAsHex( + message, blob.data(), blob.size()); + LOG(spam, "Encoded message of protocol %s type %u using version " + "%s serialization:\n%s", + routable.getProtocol().c_str(), routable.getType(), + version.toString().c_str(), message.str().c_str()); + } + return blob; +} + +mbus::Routable::UP +DocumentProtocol::decode(const vespalib::Version &version, mbus::BlobRef data) const +{ + try { + return _routableRepository.decode(version, data); + } catch (vespalib::Exception &e) { + LOG(warning, "%s", e.getMessage().c_str()); + return mbus::Routable::UP(); + } +} + +uint32_t +DocumentProtocol::getRoutableTypes(const vespalib::Version &version, std::vector<uint32_t> &out) const +{ + return _routableRepository.getRoutableTypes(version, out); +} + +DocumentProtocol & +DocumentProtocol::putRoutableFactory(uint32_t type, IRoutableFactory::SP factory, + const vespalib::VersionSpecification &version) +{ + _routableRepository.putFactory(version, type, factory); + return *this; +} + +DocumentProtocol & +DocumentProtocol::putRoutableFactory(uint32_t type, IRoutableFactory::SP factory, + const std::vector<vespalib::VersionSpecification> &versions) +{ + for (std::vector<vespalib::VersionSpecification>::const_iterator it = versions.begin(); + it != versions.end(); ++it) + { + putRoutableFactory(type, factory, *it); + } + return *this; +} + +string +DocumentProtocol::getErrorName(uint32_t errorCode) { + switch (errorCode) { + case ERROR_MESSAGE_IGNORED: return "MESSAGE_IGNORED"; + case ERROR_DOCUMENT_NOT_FOUND: return "DOCUMENT_NOT_FOUND"; + case ERROR_EXISTS: return "EXISTS"; + case ERROR_BUCKET_NOT_FOUND: return "BUCKET_NOT_FOUND"; + case ERROR_BUCKET_DELETED: return "BUCKET_DELETED"; + case ERROR_NOT_IMPLEMENTED: return "NOT_IMPLEMENTED"; + case ERROR_ILLEGAL_PARAMETERS: return "ILLEGAL_PARAMETERS"; + case ERROR_IGNORED: return "IGNORED"; + case ERROR_UNKNOWN_COMMAND: return "UNKNOWN_COMMAND"; + case ERROR_UNPARSEABLE: return "UNPARSEABLE"; + case ERROR_NO_SPACE: return "NO_SPACE"; + case ERROR_INTERNAL_FAILURE: return "INTERNAL_FAILURE"; + case ERROR_PROCESSING_FAILURE: return "PROCESSING_FAILURE"; + case ERROR_TIMESTAMP_EXIST: return "TIMESTAMP_EXIST"; + case ERROR_NODE_NOT_READY: return "NODE_NOT_READY"; + case ERROR_WRONG_DISTRIBUTION: return "WRONG_DISTRIBUTION"; + case ERROR_REJECTED: return "REJECTED"; + case ERROR_ABORTED: return "ABORTED"; + case ERROR_BUSY: return "BUSY"; + case ERROR_NOT_CONNECTED: return "NOT_CONNECTED"; + case ERROR_DISK_FAILURE: return "DISK_FAILURE"; + case ERROR_IO_FAILURE: return "IO_FAILURE"; + case ERROR_SUSPENDED: return "SUSPENDED"; + case ERROR_TEST_AND_SET_CONDITION_FAILED: return "TEST_AND_SET_CONDITION_FAILED"; + } + return mbus::ErrorCode::getName(errorCode); +} + +void +DocumentProtocol::merge(mbus::RoutingContext &ctx) +{ + std::set<uint32_t> mask; + merge(ctx, mask); +} + +void +DocumentProtocol::merge(mbus::RoutingContext& ctx, + const std::set<uint32_t>& mask) +{ + ReplyMerger rm; + uint32_t idx = 0; + for (mbus::RoutingNodeIterator it = ctx.getChildIterator(); + it.isValid(); it.next(), ++idx) + { + if (mask.find(idx) != mask.end()) { + continue; + } + rm.merge(idx, it.getReplyRef()); + } + assert(idx != 0); + ReplyMerger::Result res(rm.mergedReply()); + if (res.isSuccessful()) { + const uint32_t okIdx = res.getSuccessfulReplyIndex(); + ctx.setReply(ctx.getChildIterator().skip(okIdx).removeReply()); + } else { + assert(res.hasGeneratedReply()); + ctx.setReply(mbus::Reply::UP(res.releaseGeneratedReply().release())); + } +} + +bool +DocumentProtocol::hasOnlyErrorsOfType(const mbus::Reply &reply, uint32_t errCode) +{ + for (uint32_t i = 0; i < reply.getNumErrors(); ++i) { + if (reply.getError(i).getCode() != errCode) { + return false; + } + } + return true; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h new file mode 100644 index 00000000000..16225d96122 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h @@ -0,0 +1,311 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "iroutablefactory.h" +#include "routablerepository.h" +#include "routingpolicyrepository.h" +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/documentapi/messagebus/systemstate/systemstate.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/iprotocol.h> +#include <vespa/messagebus/reply.h> +#include <vespa/messagebus/routing/routingcontext.h> + +namespace documentapi { + +class LoadTypeSet; + +class DocumentProtocol : public mbus::IProtocol { +private: + RoutingPolicyRepository _routingPolicyRepository; + RoutableRepository _routableRepository; + SystemState::UP _systemState; + document::DocumentTypeRepo::SP _repo; + +public: + /** + * Convenience typedef. + */ + typedef std::unique_ptr<DocumentProtocol> UP; + typedef std::shared_ptr<DocumentProtocol> SP; + + /** + * The name of this protocol is public static so it can be referenced by all of this protocol's messages + * and replies instead of hard coding the string in every class. + */ + static const mbus::string NAME; + + /** + * Defines all message and reply types that are implemented by this protocol. + */ + enum MessageType { + DOCUMENT_MESSAGE = 100000, +// MESSAGE_STARTOFFEED = DOCUMENT_MESSAGE + 1, +// MESSAGE_ENDOFFEED = DOCUMENT_MESSAGE + 2, + MESSAGE_GETDOCUMENT = DOCUMENT_MESSAGE + 3, + MESSAGE_PUTDOCUMENT = DOCUMENT_MESSAGE + 4, + MESSAGE_REMOVEDOCUMENT = DOCUMENT_MESSAGE + 5, + MESSAGE_UPDATEDOCUMENT = DOCUMENT_MESSAGE + 6, + MESSAGE_CREATEVISITOR = DOCUMENT_MESSAGE + 7, + MESSAGE_DESTROYVISITOR = DOCUMENT_MESSAGE + 8, + MESSAGE_VISITORINFO = DOCUMENT_MESSAGE + 9, + MESSAGE_SEARCHRESULT = DOCUMENT_MESSAGE + 11, + MESSAGE_MULTIOPERATION = DOCUMENT_MESSAGE + 13, + MESSAGE_DOCUMENTSUMMARY = DOCUMENT_MESSAGE + 14, + MESSAGE_MAPVISITOR = DOCUMENT_MESSAGE + 15, + MESSAGE_GETBUCKETSTATE = DOCUMENT_MESSAGE + 18, + MESSAGE_STATBUCKET = DOCUMENT_MESSAGE + 19, + MESSAGE_GETBUCKETLIST = DOCUMENT_MESSAGE + 20, + MESSAGE_DOCUMENTLIST = DOCUMENT_MESSAGE + 21, + MESSAGE_EMPTYBUCKETS = DOCUMENT_MESSAGE + 23, + MESSAGE_REMOVELOCATION = DOCUMENT_MESSAGE + 24, + MESSAGE_QUERYRESULT = DOCUMENT_MESSAGE + 25, + MESSAGE_BATCHDOCUMENTUPDATE = DOCUMENT_MESSAGE + 26, +// MESSAGE_GARBAGECOLLECT = DOCUMENT_MESSAGE + 27, + + DOCUMENT_REPLY = 200000, +// REPLY_STARTOFFEED = DOCUMENT_REPLY + 1, +// REPLY_ENDOFFEED = DOCUMENT_REPLY + 2, + REPLY_GETDOCUMENT = DOCUMENT_REPLY + 3, + REPLY_PUTDOCUMENT = DOCUMENT_REPLY + 4, + REPLY_REMOVEDOCUMENT = DOCUMENT_REPLY + 5, + REPLY_UPDATEDOCUMENT = DOCUMENT_REPLY + 6, + REPLY_CREATEVISITOR = DOCUMENT_REPLY + 7, + REPLY_DESTROYVISITOR = DOCUMENT_REPLY + 8, + REPLY_VISITORINFO = DOCUMENT_REPLY + 9, + REPLY_SEARCHRESULT = DOCUMENT_REPLY + 11, + REPLY_MULTIOPERATION = DOCUMENT_REPLY + 13, + REPLY_DOCUMENTSUMMARY = DOCUMENT_REPLY + 14, + REPLY_MAPVISITOR = DOCUMENT_REPLY + 15, + REPLY_GETBUCKETSTATE = DOCUMENT_REPLY + 18, + REPLY_STATBUCKET = DOCUMENT_REPLY + 19, + REPLY_GETBUCKETLIST = DOCUMENT_REPLY + 20, + REPLY_DOCUMENTLIST = DOCUMENT_REPLY + 21, + REPLY_EMPTYBUCKETS = DOCUMENT_REPLY + 23, + REPLY_REMOVELOCATION = DOCUMENT_REPLY + 24, + REPLY_QUERYRESULT = DOCUMENT_REPLY + 25, + REPLY_BATCHDOCUMENTUPDATE = DOCUMENT_REPLY + 26, +// REPLY_GARBAGECOLLECT = DOCUMENT_REPLY + 27, + REPLY_WRONGDISTRIBUTION = DOCUMENT_REPLY + 1000, + REPLY_DOCUMENTIGNORED = DOCUMENT_REPLY + 1001 + }; + + /** + * Defines all extended errors that are used by this protocol. + */ + enum { + /** Used by policies to indicate an inappropriate message. */ + ERROR_MESSAGE_IGNORED = mbus::ErrorCode::APP_FATAL_ERROR + 1, + + /** Used for error policy when policy creation failed. */ + ERROR_POLICY_FAILURE = mbus::ErrorCode::APP_FATAL_ERROR + 2, + + // Error codes to represent various failures that can come from VDS. All + // indexed from fatal error or transient failure plus 1000-1999 + + /** Document in operation cannot be found. (VDS Get and Remove) */ + ERROR_DOCUMENT_NOT_FOUND = mbus::ErrorCode::APP_FATAL_ERROR + 1001, + /** + * Operation cannot be performed because token already exist. + * (Create bucket, create visitor) + */ + ERROR_EXISTS = mbus::ErrorCode::APP_FATAL_ERROR + 1002, + + ERROR_NOT_IMPLEMENTED = mbus::ErrorCode::APP_FATAL_ERROR + 1004, + /** Parameters given in request is illegal. */ + ERROR_ILLEGAL_PARAMETERS = mbus::ErrorCode::APP_FATAL_ERROR + 1005, + /** Unknown request received. (New client requesting from old server) */ + ERROR_UNKNOWN_COMMAND = mbus::ErrorCode::APP_FATAL_ERROR + 1007, + /** Request cannot be decoded. */ + ERROR_UNPARSEABLE = mbus::ErrorCode::APP_FATAL_ERROR + 1008, + /** Not enough free space on disk to perform operation. */ + ERROR_NO_SPACE = mbus::ErrorCode::APP_FATAL_ERROR + 1009, + /** Request was not handled correctly. */ + ERROR_IGNORED = mbus::ErrorCode::APP_FATAL_ERROR + 1010, + /** We failed in some way we didn't expect to fail. */ + ERROR_INTERNAL_FAILURE = mbus::ErrorCode::APP_FATAL_ERROR + 1011, + /** Node refuse to perform operation. (Illegally formed message?) */ + ERROR_REJECTED = mbus::ErrorCode::APP_FATAL_ERROR + 1012, + /** Test and set condition (selection) failed. */ + ERROR_TEST_AND_SET_CONDITION_FAILED = mbus::ErrorCode::APP_FATAL_ERROR + 1013, + + /** Node not ready to perform operation. (Initializing VDS nodes) */ + ERROR_NODE_NOT_READY = mbus::ErrorCode::APP_TRANSIENT_ERROR + 1001, + /** + * Wrong node to talk to in current state. + * (VDS system state disagreement) + */ + ERROR_WRONG_DISTRIBUTION = mbus::ErrorCode::APP_TRANSIENT_ERROR + 1002, + /** Operation cut short and aborted. (Destroy visitor, node stopping) */ + ERROR_ABORTED = mbus::ErrorCode::APP_TRANSIENT_ERROR + 1004, + /** Node too busy to process request (Typically full queues) */ + ERROR_BUSY = mbus::ErrorCode::APP_TRANSIENT_ERROR + 1005, + /** Lost connection with the node we requested something from. */ + ERROR_NOT_CONNECTED = mbus::ErrorCode::APP_TRANSIENT_ERROR + 1006, + + /** Node have not implemented support for the given operation. */ + /** + * We failed accessing the disk, which we think is a disk hardware + * problem. + */ + ERROR_DISK_FAILURE = mbus::ErrorCode::APP_TRANSIENT_ERROR + 1007, + /** + * We failed during an IO operation, we dont think is a specific disk + * hardware problem. + */ + ERROR_IO_FAILURE = mbus::ErrorCode::APP_TRANSIENT_ERROR + 1008, + /** + * Bucket given in operation not found due to bucket database + * inconsistencies between storage and distributor nodes. + */ + ERROR_BUCKET_NOT_FOUND = mbus::ErrorCode::APP_TRANSIENT_ERROR + 1009, + /** + * Bucket recently removed, such that operation cannot be performed. + * Differs from BUCKET_NOT_FOUND in that there is no db inconsistency. + */ + ERROR_BUCKET_DELETED = mbus::ErrorCode::APP_TRANSIENT_ERROR + 1012, + /** + * Storage node received a timestamp that is stale. Likely clock skew. + */ + ERROR_STALE_TIMESTAMP = mbus::ErrorCode::APP_TRANSIENT_ERROR + 1013, + + // Error codes for docproc + + /** Failed to process the given request. (Used by docproc) */ + ERROR_PROCESSING_FAILURE = mbus::ErrorCode::APP_FATAL_ERROR + 2001, + /** Unique timestamp specified for new operation is already in use. */ + ERROR_TIMESTAMP_EXIST = mbus::ErrorCode::APP_FATAL_ERROR + 2002, + + /** + * The given node have gotten a critical error and have suspended + * itself. (Docproc nodes do this then they can't function anymore) + */ + ERROR_SUSPENDED = mbus::ErrorCode::APP_TRANSIENT_ERROR + 2001 + }; + +public: + /** + * Constructs a new document protocol using the given id for config subscription. + * + * @param configId The id to use when subscribing to config. + */ + DocumentProtocol(const LoadTypeSet& loadTypes, + document::DocumentTypeRepo::SP repo, + const string &configId = ""); + + /** + * Adds a new routable factory to this protocol. This method is thread-safe, and may be invoked on a + * protocol object that is already in use by a message bus instance. Notice that the name you supply for a + * factory is the case-sensitive name that will be referenced by routes. + * + * @param name The name of the factory to add. + * @param factory The factory to add. + * @return This, to allow chaining. + */ + DocumentProtocol &putRoutingPolicyFactory(const string &name, IRoutingPolicyFactory::SP factory); + + /** + * Adds a new routable factory to this protocol. This method is thread-safe, and may be invoked on a + * protocol object that is already in use by a message bus instance. Notice that you must explicitly + * register a factory for each supported version. You can always bypass this by passing a default version + * specification object to this function, because that object will match any version. + * + * @param type The routable type to assign a factory to. + * @param factory The factory to add. + * @param version The version for which this factory can be used. + * @return This, to allow chaining. + */ + DocumentProtocol &putRoutableFactory(uint32_t type, IRoutableFactory::SP factory, + const vespalib::VersionSpecification &version); + + /** + * Convenience method to call {@link #putRoutableFactory(int, RoutableFactory, + * com.yahoo.component.VersionSpecification)} for multiple version specifications. + * + * @param type The routable type to assign a factory to. + * @param factory The factory to add. + * @param versions The versions for which this factory can be used. + * @return This, to allow chaining. + */ + DocumentProtocol &putRoutableFactory(uint32_t type, IRoutableFactory::SP factory, + const std::vector<vespalib::VersionSpecification> &versions); + + /** + * Returns a list of routable types that support the given version. + * + * @param version The version to return types for. + * @param out The list to write to. + * @return The number of supported types. + */ + uint32_t getRoutableTypes(const vespalib::Version &version, std::vector<uint32_t> &out) const; + + /** + * Returns a string representation of the given error code. + * + * @param errorCode The code whose string symbol to return. + * @return The error string. + */ + static string getErrorName(uint32_t errorCode); + + /** + * Deserialized the given type of routable from the given byte buffer. + * + * @param type The type of routable. + * @param buf A byte buffer that contains a serialized routable. + * @return The deserialized routable. + */ + mbus::Routable::UP deserialize(uint32_t type, + document::ByteBuffer &buf) const; + + /** + * This is a convenient entry to the {@link #merge(RoutingContext,std::set)} method by way of a routing + * context object. The replies of all child contexts are merged and stored in the context. + * + * @param ctx The context whose children to merge. + */ + static void merge(mbus::RoutingContext &ctx); + + /** + * This method implements the common way to merge document replies for whatever routing policy. In case of + * an error in any of the replies, it will prepare an EmptyReply() and add all errors to it. If there are + * no errors, this method will use the first reply in the list and transfer whatever feed answers might + * exist in the replies to it. + * + * @param ctx The context whose children to merge. + * @param mask The indexes of the children to skip. + */ + static void merge(mbus::RoutingContext &ctx, const std::set<uint32_t> &mask); + + /** + * Returns true if the given reply has at least one error, and all errors + * are of the given type. + * + * @param reply The reply to check for error. + * @param errCode The error code to check for. + * @return Whether or not the reply has only the given error code. + */ + static bool hasOnlyErrorsOfType(const mbus::Reply &reply, uint32_t errCode); + + /** + * Returns the curren state of the system, as observed by this protocol. This state object may be freely + * modified by the caller. + * + * @return The system state. + */ + SystemState &getSystemState() { return *_systemState; } + + // Implements IProtocol. + const mbus::string &getName() const { return NAME; } + + // Implements IProtocol. + mbus::IRoutingPolicy::UP createPolicy(const mbus::string &name, const mbus::string ¶m) const; + + // Implements IProtocol. + mbus::Blob encode(const vespalib::Version &version, const mbus::Routable &routable) const; + + // Implements IProtocol. + mbus::Routable::UP decode(const vespalib::Version &version, mbus::BlobRef data) const; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/iroutablefactory.h b/documentapi/src/vespa/documentapi/messagebus/iroutablefactory.h new file mode 100644 index 00000000000..0ca3bc10531 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/iroutablefactory.h @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <boost/shared_ptr.hpp> +#include <vespa/document/util/bytebuffer.h> +#include <vespa/messagebus/routable.h> +#include <vespa/vespalib/component/version.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> + +namespace documentapi { + +class LoadTypeSet; + +/** + * This interface defines the necessary methods of a routable factory that can be plugged into a {@link + * DocumentProtocol} using the {@link DocumentProtocol#putRoutableFactory(int, RoutableFactory, + * com.yahoo.component.VersionSpecification)} method. Consider extending {@link DocumentMessageFactory} or + * {@link DocumentReplyFactory} instead of implementing this interface. + * + * Notice that no routable type is passed to the {@link #decode(ByteBuffer)} method, so you may NOT share a + * factory across multiple routable types. To share serialization logic between factory use a common + * superclass or composition with a common serialization utility. + */ +class IRoutableFactory : public boost::noncopyable { +public: + /** + * Convenience typedefs. + */ + typedef std::unique_ptr<IRoutableFactory> UP; + typedef std::shared_ptr<IRoutableFactory> SP; + + /** + * Virtual destructor required for inheritance. + */ + virtual ~IRoutableFactory() { } + + /** + * This method encodes the content of the given routable into a byte buffer that can later be decoded + * using the {@link #decode(ByteBuffer)} method. + * + * This method is NOT exception safe. Return false to signal failure. + * + * @param obj The routable to encode. + * @param out The buffer to write into. + * @return True if the routable could be encoded. + */ + virtual bool encode(const mbus::Routable &obj, + vespalib::GrowableByteBuffer &out) const = 0; + + /** + * This method decodes the given byte bufer to a routable. + * + * This method is NOT exception safe. Return null to signal failure. + * + * @param in The buffer to read from. + * @param loadTypes The set of configured load types. + * @return The decoded routable. + */ + virtual mbus::Routable::UP decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const = 0; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/iroutingpolicyfactory.h b/documentapi/src/vespa/documentapi/messagebus/iroutingpolicyfactory.h new file mode 100644 index 00000000000..08bad22b916 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/iroutingpolicyfactory.h @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/messagebus/routing/iroutingpolicy.h> +#include <vespa/documentapi/common.h> + +namespace documentapi { + +/** + * This interface defines the necessary methods of a routing policy factory that can be plugged into a {@link + * DocumentProtocol} using the {@link DocumentProtocol#putRoutingPolicyFactory(String, RoutingPolicyFactory)} + * method. + */ +class IRoutingPolicyFactory { +public: + /** + * Convenience typedefs. + */ + typedef std::unique_ptr<IRoutingPolicyFactory> UP; + typedef std::shared_ptr<IRoutingPolicyFactory> SP; + + /** + * Virtual destructor required for inheritance. + */ + virtual ~IRoutingPolicyFactory() { } + + /** + * This method creates and returns a routing policy that corresponds to the implementing class, using the + * given parameter string. There is only ever one instance of a routing policy for a given name and + * parameter combination, and because of this the policies must be state-less beyond what can be derived + * from the parameter string. Because there is only a single thread running route resolution within + * message bus, it is not necessary to make policies thread-safe. For more information see {@link + * RoutingPolicy}. + * + * Do NOT throw exceptions out of this method because that will cause the running thread to die, just + * return null to signal failure instead. + * + * @param param The parameter to use when creating the policy. + * @return The created routing policy. + */ + virtual mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const = 0; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/.gitignore b/documentapi/src/vespa/documentapi/messagebus/messages/.gitignore new file mode 100644 index 00000000000..d58390943e2 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +config-*.cpp +config-*.h diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt new file mode 100644 index 00000000000..943c32ee108 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt @@ -0,0 +1,35 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(documentapi_documentapimessages OBJECT + SOURCES + documentmessage.cpp + documentreply.cpp + documentstate.cpp + documentsummarymessage.cpp + emptybucketsmessage.cpp + feedanswer.cpp + feedmessage.cpp + feedreply.cpp + getbucketlistmessage.cpp + getbucketlistreply.cpp + getbucketstatemessage.cpp + getbucketstatereply.cpp + getdocumentmessage.cpp + getdocumentreply.cpp + multioperationmessage.cpp + putdocumentmessage.cpp + queryresultmessage.cpp + removedocumentmessage.cpp + removedocumentreply.cpp + removelocationmessage.cpp + searchresultmessage.cpp + statbucketmessage.cpp + statbucketreply.cpp + updatedocumentmessage.cpp + visitor.cpp + wrongdistributionreply.cpp + updatedocumentreply.cpp + batchdocumentupdatemessage.cpp + batchdocumentupdatereply.cpp + documentignoredreply.cpp + DEPENDS +) diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp new file mode 100644 index 00000000000..4d874f2637c --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h> +#include <vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h> +#include <vespa/document/base/idstring.h> +#include <vespa/document/select/parser.h> +#include <vespa/document/bucket/bucketselector.h> + +namespace documentapi { + +BatchDocumentUpdateMessage::BatchDocumentUpdateMessage(uint64_t userId) + : _userId(userId) +{ + setBucketId(document::UserDocIdString(vespalib::make_string("userdoc:foo:%lu:bar", _userId))); +} + +BatchDocumentUpdateMessage::BatchDocumentUpdateMessage(const string& group) + : _userId(0), + _group(group) +{ + setBucketId(document::GroupDocIdString("groupdoc:foo:" + _group + ":bar")); +} + +void +BatchDocumentUpdateMessage::setBucketId(const document::IdString& idString) +{ + document::BucketIdFactory factory; + _bucketId = factory.getBucketId(document::DocumentId(idString)); +} + +void +BatchDocumentUpdateMessage::addUpdate(document::DocumentUpdate::SP update) +{ + verifyUpdate(*update); + _updates.push_back(update); +} + +void +BatchDocumentUpdateMessage::verifyUpdate(const document::DocumentUpdate& update) { + const document::IdString& idString = update.getId().getScheme(); + + if (_group.length()) { + string group; + + if (idString.hasGroup()) { + group = idString.getGroup(); + } else { + throw vespalib::IllegalArgumentException("Batch update message can only contain groupdoc or orderdoc items"); + } + + if (group != _group) { + throw vespalib::IllegalArgumentException(vespalib::make_string("Batch update message can not contain messages from group %s, only group %s", group.c_str(), _group.c_str())); + } + } else { + uint64_t userId; + + if (idString.hasNumber()) { + userId = idString.getNumber(); + } else { + throw vespalib::IllegalArgumentException("Batch update message can only contain userdoc or orderdoc items"); + } + + if (userId != _userId) { + throw vespalib::IllegalArgumentException(vespalib::make_string("Batch update message can not contain messages from user %llu, only user %llu", (long long unsigned)userId, (long long unsigned)_userId)); + } + } +} + +DocumentReply::UP +BatchDocumentUpdateMessage::doCreateReply() const +{ + return DocumentReply::UP(new BatchDocumentUpdateReply()); +} + +uint32_t +BatchDocumentUpdateMessage::getType() const { + return DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h new file mode 100644 index 00000000000..c2e1e599ee8 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/update/documentupdate.h> +#include <vespa/documentapi/messagebus/messages/documentmessage.h> +#include <vespa/documentapi/messagebus/messages/writedocumentreply.h> +#include <vespa/document/bucket/bucketid.h> +#include <vespa/document/base/idstring.h> + +namespace documentapi { + +/** + Message to use to send multiple updates for documents + belonging to the same user or group to Vespa. Using this + message improves performance in VDS mainly. +*/ +class BatchDocumentUpdateMessage : public DocumentMessage +{ +public: + typedef std::vector<document::DocumentUpdate::SP > UpdateList; + + /** + Creates a batch update message that can contain only updates + for documents belonging to the given user. + */ + BatchDocumentUpdateMessage(uint64_t userId); + + /** + Creates a batch update message that can contain only updates + for documents belonging to the given group. + */ + BatchDocumentUpdateMessage(const string& group); + + /** + @return Returns a list of the updates to be performed. + */ + const UpdateList& getUpdates() const { return _updates; }; + + /** + Adds an update to be performed. + */ + void addUpdate(document::DocumentUpdate::SP update); + + /** + Returns the user id that this batch can contain. + Only valid if this object was created with the first constructor. + */ + uint64_t getUserId() const { return _userId; }; + + /** + Returns the grouo that this batch can contain. + Only valid if this object was created with the second constructor. + */ + const string& getGroup() const { return _group; } + + // Implements DocumentMessage. + uint32_t getType() const; + + /** + Returns a bucket id suitable for routing this message. + */ + const document::BucketId& getBucketId() const { return _bucketId; } + + string toString() const { return "batchdocumentupdatemessage"; } + +protected: + // Implements DocumentMessage. + DocumentReply::UP doCreateReply() const; + +private: + uint64_t _userId; + string _group; + + UpdateList _updates; + document::BucketId _bucketId; + + void verifyUpdate(const document::DocumentUpdate& update); + void setBucketId(const document::IdString& idString); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.cpp new file mode 100644 index 00000000000..533559a36c3 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.cpp @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h> +#include <vespa/documentapi/messagebus/messages/writedocumentreply.h> + +namespace documentapi { + +BatchDocumentUpdateReply::BatchDocumentUpdateReply() + : WriteDocumentReply(DocumentProtocol::REPLY_BATCHDOCUMENTUPDATE) +{ +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h new file mode 100644 index 00000000000..ec0aef31a5d --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/messages/writedocumentreply.h> + +namespace documentapi { + +class BatchDocumentUpdateReply : public WriteDocumentReply +{ + /** + * If all documents to update are found, this vector will be empty. If + * one or more documents are not found, this vector will have the size of + * the initial number of updates, with entries set to true where the + * corresponding update was not found. + */ + std::vector<bool> _documentsNotFound; +public: + typedef std::unique_ptr<BatchDocumentUpdateReply> UP; + typedef std::shared_ptr<BatchDocumentUpdateReply> SP; + + BatchDocumentUpdateReply(); + + const std::vector<bool>& getDocumentsNotFound() const { return _documentsNotFound; } + std::vector<bool>& getDocumentsNotFound() { return _documentsNotFound; } + + string toString() const { return "batchdocumentupdatereply"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentacceptedreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/documentacceptedreply.h new file mode 100644 index 00000000000..58b766f2d81 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentacceptedreply.h @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/messages/documentreply.h> + +namespace documentapi { + +/** + * Common base class for replies that indicate that a document was routed + * to some recipient. Does not imply that the reply contains no errors! + */ +class DocumentAcceptedReply : public DocumentReply { +public: + DocumentAcceptedReply(uint32_t type) + : DocumentReply(type) + {} +}; + +} // ns documentapi + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentignoredreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentignoredreply.cpp new file mode 100644 index 00000000000..acccf0dffa9 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentignoredreply.cpp @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/documentignoredreply.h> + +namespace documentapi { + +DocumentIgnoredReply::DocumentIgnoredReply() + : DocumentReply(DocumentProtocol::REPLY_DOCUMENTIGNORED) +{ +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentignoredreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/documentignoredreply.h new file mode 100644 index 00000000000..ea1ba31b9ef --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentignoredreply.h @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/messages/documentreply.h> + +namespace documentapi { + +class DocumentIgnoredReply : public DocumentReply { +public: + typedef std::unique_ptr<DocumentIgnoredReply> UP; + typedef std::shared_ptr<DocumentIgnoredReply> SP; + + DocumentIgnoredReply(); + + string toString() const { return "DocumentIgnoredReply"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp new file mode 100644 index 00000000000..1097beb8746 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.cpp @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".documentmessage"); + +#include <vespa/documentapi/messagebus/messages/documentmessage.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/priority.h> + +namespace documentapi { + +DocumentMessage::DocumentMessage() : + mbus::Message(), + _priority(Priority::PRI_NORMAL_3), + _loadType(LoadType::DEFAULT), + _approxSize(1024) +{ + // empty +} + +mbus::Reply::UP +DocumentMessage::createReply() const +{ + mbus::Reply::UP ret(doCreateReply().release()); + LOG_ASSERT(ret.get() != NULL); + return ret; +} + +const mbus::string& +DocumentMessage::getProtocol() const +{ + return DocumentProtocol::NAME; +} + +uint32_t +DocumentMessage::getApproxSize() const +{ + return _approxSize; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.h new file mode 100644 index 00000000000..3a626f966d8 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentmessage.h @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/util/bytebuffer.h> +#include <vespa/messagebus/message.h> +#include <vespa/messagebus/reply.h> +#include "documentreply.h" +#include <vespa/documentapi/loadtypes/loadtype.h> +#include <vespa/documentapi/messagebus/priority.h> + +namespace documentapi { + +class DocumentMessage : public mbus::Message { +private: + Priority::Value _priority; + LoadType _loadType; + uint32_t _approxSize; // Not sent on wire; set by deserializer or by caller. + +protected: + /** + * This method is used by {@link #createReply()} to ensure that all document messages return document type + * replies. This method may NOT return null as that will cause an assertion error. + * + * @return A document reply that corresponds to this message. + */ + virtual DocumentReply::UP doCreateReply() const = 0; + +public: + /** + * Convenience typedefs. + */ + typedef std::unique_ptr<DocumentMessage> UP; + typedef std::shared_ptr<DocumentMessage> SP; + + /** + * Constructs a new document message with no content. + */ + DocumentMessage(); + + /** + * Virtual destructor required for inheritance. + */ + virtual ~DocumentMessage() { } + + /** + * Creates and returns a reply to this message. This method uses the internal {@link #doCreateReply()} to + * guarantee that the reply is a {@link DocumentReply}, and casts it to a message bus type reply for + * convenience. + * + * @return The created reply. + */ + mbus::Reply::UP createReply() const; + + /** + * Returns the priority of this message. + * + * @return The priority. + */ + Priority::Value getPriority() const { return _priority; }; + + uint8_t priority() const { return (uint8_t)_priority; }; + + /** + * Sets the priority tag for this message. + * + * @param priority The priority to set. + */ + void setPriority(Priority::Value p) { _priority = p; }; + + /** + * @return Returns the load type for this message. + */ + const LoadType& getLoadType() const { return _loadType; } + + /** + * Sets the load type for this message. + */ + void setLoadType(const LoadType& loadType) { _loadType = loadType; } + + uint32_t getApproxSize() const override; + + void setApproxSize(uint32_t approxSize) { + _approxSize = approxSize; + } + + // Implements mbus::Message. + const mbus::string& getProtocol() const; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.cpp new file mode 100644 index 00000000000..044f99409bf --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.cpp @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/priority.h> +#include <vespa/documentapi/messagebus/messages/documentreply.h> + +namespace documentapi { + +DocumentReply::DocumentReply(uint32_t type) : + mbus::Reply(), + _type(type), + _priority(Priority::PRI_NORMAL_3) +{ + // empty +} + +const mbus::string& +DocumentReply::getProtocol() const +{ + return DocumentProtocol::NAME; +} + +} + + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.h new file mode 100644 index 00000000000..addc3890d7e --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentreply.h @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/messagebus/reply.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/priority.h> + +namespace documentapi { + +/** + * This class implements a generic document protocol reply that can be reused by document messages that require no + * special reply implementation while still allowing applications to distinguish between types. + */ +class DocumentReply : public mbus::Reply { +private: + uint32_t _type; + Priority::Value _priority; + +public: + /** + * Convenience typedef. + */ + typedef std::unique_ptr<DocumentReply> UP; + typedef std::shared_ptr<DocumentReply> SP; + + /** + * Constructs a new reply of given type. + * + * @param type The type code to assign to this. + */ + DocumentReply(uint32_t type); + + /** + * Virtual destructor required for inheritance. + */ + virtual ~DocumentReply() { } + + /** + * Returns the priority tag for this message. This is an optional tag added for VDS that is not interpreted by the + * document protocol. + * + * @return The priority. + */ + Priority::Value getPriority() const { return _priority; } + + uint8_t priority() const { return (uint8_t)_priority; }; + + /** + * Sets the priority tag for this message. + * + * @param priority The priority to set. + */ + void setPriority(Priority::Value p) { _priority = p; } + + // Implements mbus::Reply. + const mbus::string& getProtocol() const; + + // Implements mbus::Reply. + uint32_t getType() const { return _type; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp new file mode 100644 index 00000000000..4b073afbb6f --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "documentstate.h" + +#include <vespa/document/util/bytebuffer.h> +#include <vespa/documentapi/common.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + +namespace documentapi { + +DocumentState::DocumentState() + : _timestamp(0), _removeEntry(false) {} + +DocumentState::DocumentState(const DocumentState& o) + : _gid(o._gid), _timestamp(o._timestamp), _removeEntry(o._removeEntry) +{ + if (o._docId.get() != 0) { + _docId.reset(new document::DocumentId(*o._docId)); + } +} + +DocumentState::DocumentState(const document::DocumentId& id, + uint64_t timestamp, bool removeEntry) + : _docId(new document::DocumentId(id)), + _gid(_docId->getGlobalId()), + _timestamp(timestamp), + _removeEntry(removeEntry) +{ +} + +DocumentState::DocumentState(const document::GlobalId& gid, + uint64_t timestamp, bool removeEntry) + : _gid(gid), _timestamp(timestamp), _removeEntry(removeEntry) {} + +DocumentState::DocumentState(document::ByteBuffer& buf) + : _docId(), _gid(), _timestamp(0), _removeEntry(false) +{ + uint8_t hasDocId; + buf.getByte(hasDocId); + if (hasDocId) { + vespalib::nbostream stream( + buf.getBufferAtPos(), buf.getRemaining(), false); + _docId.reset(new document::DocumentId(stream)); + buf.incPos(stream.rp()); + } + char* gid = buf.getBufferAtPos(); + buf.incPos(document::GlobalId::LENGTH); + _gid.set(gid); + buf.getLongNetwork((int64_t&) _timestamp); + uint8_t b; + buf.getByte(b); + _removeEntry = b > 0; +} + +DocumentState& +DocumentState::operator=(const DocumentState& other) +{ + _docId.reset(); + if (other._docId.get() != 0) { + _docId.reset(new document::DocumentId(*other._docId)); + } + _gid = other._gid; + _timestamp = other._timestamp; + _removeEntry = other._removeEntry; + return *this; +} + +void DocumentState::serialize(vespalib::GrowableByteBuffer &buf) const +{ + if (_docId.get()) { + buf.putByte(1); + string str = _docId->toString(); + buf.putBytes(str.c_str(), str.length() + 1); + } else { + buf.putByte(0); + } + char* buffer = buf.allocate(document::GlobalId::LENGTH); + memcpy(buffer, _gid.get(), document::GlobalId::LENGTH); + + buf.putLong(_timestamp); + buf.putByte(_removeEntry ? 1 : 0); +} + +} // documentapi diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.h b/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.h new file mode 100644 index 00000000000..7b1d9332a00 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.h @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/document/base/documentid.h> + +namespace document { + class ByteBuffer; +} +namespace vespalib { + class GrowableByteBuffer; +} + +namespace documentapi { + +class DocumentState { + std::unique_ptr<document::DocumentId> _docId; + document::GlobalId _gid; + uint64_t _timestamp; + bool _removeEntry; + +public: + DocumentState(); + DocumentState(const DocumentState&); + DocumentState(const document::DocumentId&, + uint64_t timestamp, bool removeEntry); + DocumentState(const document::GlobalId&, + uint64_t timestamp, bool removeEntry); + DocumentState(document::ByteBuffer &buf); + + DocumentState& operator=(const DocumentState&); + + void serialize(vespalib::GrowableByteBuffer &buf) const; + + const document::GlobalId& getGlobalId() const { return _gid; } + const document::DocumentId* getDocumentId() const { return _docId.get(); } + uint64_t getTimestamp() const { return _timestamp; } + bool isRemoveEntry() const { return _removeEntry; } +}; + +} // documentapi diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp new file mode 100644 index 00000000000..5416b73d548 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.cpp @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/documentapi/messagebus/messages/documentsummarymessage.h> + +using vdslib::DocumentSummary; + +namespace documentapi { + +DocumentSummaryMessage::DocumentSummaryMessage(const DocumentSummary & sr) : + VisitorMessage(), + DocumentSummary(sr) +{ + // empty +} + +DocumentSummaryMessage::DocumentSummaryMessage() : + VisitorMessage(), + DocumentSummary() +{ + // empty +} + +DocumentReply::UP +DocumentSummaryMessage::doCreateReply() const +{ + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTSUMMARY)); +} + +uint32_t +DocumentSummaryMessage::getApproxSize() const +{ + return DocumentSummary::getSerializedSize(); +} + +uint32_t +DocumentSummaryMessage::getType() const +{ + return DocumentProtocol::MESSAGE_DOCUMENTSUMMARY; +} + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.h new file mode 100644 index 00000000000..cac631a3f1e --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentsummarymessage.h @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vdslib/container/documentsummary.h> +#include <vespa/documentapi/messagebus/messages/visitor.h> + +namespace documentapi { + +class DocumentSummaryMessage : public VisitorMessage, + public vdslib::DocumentSummary { +protected: + // Implements VisitorMessage. + DocumentReply::UP doCreateReply() const; + +public: + /** + * Convenience typedef. + */ + typedef std::unique_ptr<DocumentSummaryMessage> UP; + typedef std::shared_ptr<DocumentSummaryMessage> SP; + + /** + * Constructs a new document message with no content. + */ + DocumentSummaryMessage(); + + /** + * Constructs a new document message with summary comment. + * + * @param summary The document summary to contain. + */ + DocumentSummaryMessage(const vdslib::DocumentSummary &summary); + + // Overrides VisitorMessage. + uint32_t getApproxSize() const; + + // Implements VisitorMessage. + uint32_t getType() const; + + string toString() const { return "documentsummarymessage"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp new file mode 100644 index 00000000000..5d1d1ed00e5 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.cpp @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/messages/emptybucketsmessage.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> + +namespace documentapi { + +EmptyBucketsMessage::EmptyBucketsMessage() : + _bucketIds() +{ + // empty +} + +EmptyBucketsMessage::EmptyBucketsMessage(const std::vector<document::BucketId> &bucketIds) : + _bucketIds(bucketIds) +{ + // empty +} + +void +EmptyBucketsMessage::setBucketIds(const std::vector<document::BucketId> &bucketIds) +{ + _bucketIds = bucketIds; +} + +void +EmptyBucketsMessage::resize(uint32_t size) +{ + _bucketIds.resize(size); +} + +DocumentReply::UP +EmptyBucketsMessage::doCreateReply() const +{ + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_EMPTYBUCKETS)); +} + +uint32_t +EmptyBucketsMessage::getType() const +{ + return DocumentProtocol::MESSAGE_EMPTYBUCKETS; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.h new file mode 100644 index 00000000000..cbd0d7c35ab --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/emptybucketsmessage.h @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/bucket/bucketid.h> +#include <vespa/documentapi/messagebus/messages/visitor.h> + +namespace documentapi { + +/** + * @class EmptyBucketsMessage + * @ingroup message + * + * @brief Encapsulates a set of Empty bucket ids. + */ +class EmptyBucketsMessage : public VisitorMessage { +private: + std::vector<document::BucketId> _bucketIds; + +protected: + DocumentReply::UP doCreateReply() const; + +public: + EmptyBucketsMessage(); // must be serialized into + + EmptyBucketsMessage(const std::vector<document::BucketId> &bucketIds); + + std::vector<document::BucketId> &getBucketIds() { return _bucketIds; } + const std::vector<document::BucketId> &getBucketIds() const { return _bucketIds; } + + void setBucketIds(const std::vector<document::BucketId> &bucketIds); + + void resize(uint32_t size); + + uint32_t getType() const; + + string toString() const { return "emptybucketsmessage"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/feedanswer.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/feedanswer.cpp new file mode 100644 index 00000000000..2cf6fc10dae --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/feedanswer.cpp @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/messages/feedanswer.h> + +namespace documentapi { + +FeedAnswer::FeedAnswer() : + _answerCode(0), + _wantedIncrement(0), + _recipient(""), + _moreInfo("") +{ + // empty +} + +FeedAnswer::FeedAnswer(int answerCode, + int wantedIncrement, + const string& recipient, + const string& moreInfo) : + _answerCode(answerCode), + _wantedIncrement(wantedIncrement), + _recipient(recipient), + _moreInfo(moreInfo) +{ + // empty +} + +FeedAnswer::~FeedAnswer() +{ + // empty +} + +int +FeedAnswer::getAnswerCode() const +{ + return _answerCode; +} + +int +FeedAnswer::getWantedIncrement() const +{ + return _wantedIncrement; +} + +const string& +FeedAnswer::getRecipient() const +{ + return _recipient; +} + +const string& +FeedAnswer::getMoreInfo() const +{ + return _moreInfo; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/feedanswer.h b/documentapi/src/vespa/documentapi/messagebus/messages/feedanswer.h new file mode 100644 index 00000000000..4664659aaa4 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/feedanswer.h @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/common.h> + +namespace documentapi { + +/** + * This class contains the response to a feed command from a single + * RTC node. + */ +class FeedAnswer { +public: + /** + * How the feed command was handled. Be careful about enum + * ordering as this will be serialized. Add new values at the end. + */ + enum Handling { + UNKNOWN = 0, + HANDLED_OK, + HANDLED_MISSED_PREV, + HANDLED_AS_HINT, + IGNORED_DUP, + IGNORED_SOF_REALTIME, + IGNORED_LABEL_MISMATCH, + ERROR_TOOLOW_INCREMENT, + ERROR_TOOHIGH_INCREMENT, + ERROR_INCREMENT_IN_BATCHMODE, + ERROR_MISSING_SOF_FOR_EOF, + ERROR_WRONG_SOF_FOR_EOF, + ERROR_WRITING_LABEL, + HANDLED_AS_PROBE + }; + +public: + /** + * Constructs an empty feed answer. + */ + FeedAnswer(); + + /** + * Constructs a complete feed answer. + * + * @param answerCode The code per the enum above. + * @param wantedIncrement The increment of the current feed transaction. + * @param recipient The name of the RTC node. + * @param moreInfo Arbitrary additional info. + */ + FeedAnswer(int answerCode, int wantedIncrement, + const string& recipient, + const string& moreInfo); + + /** + * Destructor. Frees any allocated resources. + */ + virtual ~FeedAnswer(); + + /** + * Returns the numerical code of this answer. + * + * @return The code. + */ + int getAnswerCode() const; + + /** + * Returns the increment of the feed transaction that the RTC is + * currently processing. + * + * @param The increment. + */ + int getWantedIncrement() const; + + /** + * Returns the name of the RTC node whose answer this is. + * + * @return The recipient name. + */ + const string& getRecipient() const; + + /** + * Returns any additional info added to the answer. + * + * @return The additional data. + */ + const string& getMoreInfo() const; + +private: + int _answerCode; // The code of this answer. + int _wantedIncrement; // The increment of the current feed. + string _recipient; // The name of the RTC node. + string _moreInfo; // Any additional data. +}; + +} + + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/feedmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/feedmessage.cpp new file mode 100644 index 00000000000..e3e61f888fb --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/feedmessage.cpp @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/messages/feedmessage.h> + +namespace documentapi { + +FeedMessage::FeedMessage() : + DocumentMessage(), + _name(), + _generation(0), + _increment(0) +{ + // empty +} + +FeedMessage::FeedMessage(const string& name, int generation, int increment) : + DocumentMessage(), + _name(name), + _generation(generation), + _increment(increment) +{ + // empty +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/feedmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/feedmessage.h new file mode 100644 index 00000000000..11291359333 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/feedmessage.h @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/util/bytebuffer.h> +#include <vespa/documentapi/messagebus/messages/documentmessage.h> + +namespace documentapi { + +class FeedMessage : public DocumentMessage { +private: + string _name; + int _generation; + int _increment; + +public: + /** + * Convenience typedef. + */ + typedef std::unique_ptr<FeedMessage> UP; + typedef std::shared_ptr<FeedMessage> SP; + +public: + /** + * Constructs a new document message for deserialization. + */ + FeedMessage(); + + /** + * Constructs a new feed message. + * + * @param name The feed label. + * @param generation The feed generation. + * @param increment The feed increment. + */ + FeedMessage(const string& name, int generation, int increment); + + /** + * Returns the name of this feed. + * + * @return The name. + */ + const string& getName() const { return _name; } + + /** + * Sets the name of this feed. + * + * @param name The name to set. + */ + void setName(const string& name) { _name = name; } + + /** + * Returns the generation of this feed. + * + * @return The generation. + */ + int getGeneration() const { return _generation; } + + /** + * Sets the generation of this feed. + * + * @param generation The generation to set. + */ + void setGeneration(int generation) { _generation = generation; } + + /** + * Returns the increment of this feed. + * + * @return The increment. + */ + int getIncrement() const { return _increment; }; + + /** + * Sets the increment of this feed. + * + * @param increment The increment to set. + */ + void setIncrement(int increment) { _increment = increment; }; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/feedreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/feedreply.cpp new file mode 100644 index 00000000000..98e90c22367 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/feedreply.cpp @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/messages/feedreply.h> + +namespace documentapi { + +FeedReply::FeedReply(uint32_t type) : + DocumentReply(type), + _feedAnswers() +{ + // empty +} + +FeedReply::FeedReply(uint32_t type, const std::vector<FeedAnswer> &feedAnswers) : + DocumentReply(type), + _feedAnswers(feedAnswers) +{ + // empty +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/feedreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/feedreply.h new file mode 100644 index 00000000000..1b61278bf49 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/feedreply.h @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/util/bytebuffer.h> +#include <vespa/documentapi/messagebus/messages/documentreply.h> +#include <vespa/documentapi/messagebus/messages/feedanswer.h> + +namespace documentapi { + +class FeedReply : public DocumentReply { +private: + std::vector<FeedAnswer> _feedAnswers; + +public: + /** + * Convenience typedef. + */ + typedef std::unique_ptr<FeedReply> UP; + typedef std::shared_ptr<FeedReply> SP; + + /** + * Constructs a new reply for deserialization. + * + * @param type The type to assign to this. + */ + FeedReply(uint32_t type); + + /** + * Constructs a new feed reply. + * + * @param type The type to assign to this. + * @param feedAnswers The list of answers given by the search nodes. + */ + FeedReply(uint32_t type, const std::vector<FeedAnswer> &feedAnswers); + + /** + * Returns the list of answers given by the search nodes. + * + * @return The list of answers. + */ + std::vector<FeedAnswer> &getFeedAnswers() { return _feedAnswers; } + + /** + * Returns the list of answers given by the search nodes. + * + * @return The list of answers. + */ + const std::vector<FeedAnswer> &getFeedAnswers() const { return _feedAnswers; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp new file mode 100644 index 00000000000..ad7a44c1cc9 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.cpp @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/getbucketlistmessage.h> +#include <vespa/documentapi/messagebus/messages/getbucketlistreply.h> + +namespace documentapi { + +GetBucketListMessage::GetBucketListMessage() : + DocumentMessage(), + _bucketId() +{ + // empty +} + +GetBucketListMessage::GetBucketListMessage(const document::BucketId &bucketId) : + DocumentMessage(), + _bucketId(bucketId) +{ + // empty +} + +DocumentReply::UP +GetBucketListMessage::doCreateReply() const +{ + return DocumentReply::UP(new GetBucketListReply()); +} + +bool +GetBucketListMessage::hasSequenceId() const +{ + return true; +} + +uint64_t +GetBucketListMessage::getSequenceId() const +{ + return _bucketId.getRawId(); +} + +uint32_t +GetBucketListMessage::getType() const +{ + return DocumentProtocol::MESSAGE_GETBUCKETLIST; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.h new file mode 100644 index 00000000000..8d5fa587ffd --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistmessage.h @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/bucket/bucketid.h> +#include <vespa/documentapi/messagebus/messages/documentmessage.h> + +namespace documentapi { + +class GetBucketListMessage : public DocumentMessage { +private: + document::BucketId _bucketId; + +protected: + // Implements DocumentMessage. + DocumentReply::UP doCreateReply() const; + +public: + /** + * Constructs a new message for deserialization. + */ + GetBucketListMessage(); + + /** + * Constructs a new message with initial content. + * + * @param bucketId The bucket whose list to retrieve. + */ + GetBucketListMessage(const document::BucketId &bucketId); + + /** + * Returns the bucket whose list to retrieve. + * + * @return The bucket id. + */ + const document::BucketId& getBucketId() const { return _bucketId; } + + /** + * Sets the bucket whose list to retrieve. + * + * @param id The bucket id to set. + */ + void setBucketId(const document::BucketId& id) { _bucketId = id; } + + // Overrides DocumentMessage. + bool hasSequenceId() const; + + // Overrides DocumentMessage. + uint64_t getSequenceId() const; + + // Implements DocumentMessage. + uint32_t getType() const; + + string toString() const { return "getbucketlistmessage"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp new file mode 100644 index 00000000000..dd44786bd03 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.cpp @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/getbucketlistreply.h> + +namespace documentapi { + +GetBucketListReply::BucketInfo::BucketInfo() : + _bucket(), + _bucketInformation() +{ + // empty +} + +GetBucketListReply::BucketInfo::BucketInfo(const document::BucketId &bucketId, + const string &bucketInformation) : + _bucket(bucketId), + _bucketInformation(bucketInformation) +{ + // empty +} + +bool +GetBucketListReply::BucketInfo::operator==(const GetBucketListReply::BucketInfo &rhs) const +{ + return _bucket == rhs._bucket && _bucketInformation == rhs._bucketInformation; +} + +GetBucketListReply::GetBucketListReply() : + DocumentReply(DocumentProtocol::REPLY_GETBUCKETLIST), + _buckets() +{ + // empty +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.h new file mode 100644 index 00000000000..ebd9d151447 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketlistreply.h @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/messages/documentreply.h> +#include <vespa/document/bucket/bucketid.h> + +namespace documentapi { + +class GetBucketListReply : public DocumentReply { +public: + class BucketInfo { + public: + document::BucketId _bucket; + string _bucketInformation; + + BucketInfo(); + BucketInfo(const document::BucketId &bucketId, + const string &bucketInformation); + bool operator==(const BucketInfo &rhs) const; + }; + +private: + std::vector<BucketInfo> _buckets; + +public: + /** + * Constructs a new reply with no content. + */ + GetBucketListReply(); + + /** + * Returns the bucket state contained in this. + * + * @return The state object. + */ + std::vector<BucketInfo> &getBuckets() { return _buckets; } + + /** + * Returns a const reference to the bucket state contained in this. + * + * @return The state object. + */ + const std::vector<BucketInfo> &getBuckets() const { return _buckets; } + + string toString() const { return "getbucketlistreply"; } +}; + +inline std::ostream & +operator<<(std::ostream &out, const GetBucketListReply::BucketInfo &info) +{ + out << "BucketInfo(" << info._bucket << ": " << info._bucketInformation << ")"; + return out; +} + +} // documentapi + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatemessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatemessage.cpp new file mode 100644 index 00000000000..93c4f261fc5 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatemessage.cpp @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/getbucketstatemessage.h> +#include <vespa/documentapi/messagebus/messages/getbucketstatereply.h> + +namespace documentapi { + +GetBucketStateMessage::GetBucketStateMessage() : + DocumentMessage(), + _bucket() +{ + // empty +} + +GetBucketStateMessage::GetBucketStateMessage(const document::BucketId &bucket) : + DocumentMessage(), + _bucket(bucket) +{ + // empty +} + +DocumentReply::UP +GetBucketStateMessage::doCreateReply() const +{ + return DocumentReply::UP(new GetBucketStateReply()); +} + +bool +GetBucketStateMessage::hasSequenceId() const +{ + return true; +} + +uint64_t +GetBucketStateMessage::getSequenceId() const +{ + return _bucket.getRawId(); +} + +uint32_t +GetBucketStateMessage::getType() const +{ + return DocumentProtocol::MESSAGE_GETBUCKETSTATE; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatemessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatemessage.h new file mode 100644 index 00000000000..b34b7cc476b --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatemessage.h @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/bucket/bucketid.h> +#include <vespa/documentapi/messagebus/messages/documentmessage.h> + +namespace documentapi { + +class GetBucketStateMessage : public DocumentMessage { +private: + document::BucketId _bucket; + +protected: + // Implements DocumentMessage. + DocumentReply::UP doCreateReply() const; + +public: + /** + * Constructs a new message for deserialization. + */ + GetBucketStateMessage(); + + /** + * Constructs a new message with initial content. + * + * @param bucket The bucket whose state to retrieve. + */ + GetBucketStateMessage(const document::BucketId &bucket); + + /** + * Returns the bucket whose state to retrieve. + * + * @return The bucket id. + */ + document::BucketId getBucketId() const { return _bucket; } + + /** + * Sets the bucket whose state to retrieve. + * + * @param bucket The bucket id to set. + */ + void setBucketId(document::BucketId bucket) { _bucket = bucket; } + + // Overrides DocumentMessage. + bool hasSequenceId() const; + + // Overrides DocumentMessage. + uint64_t getSequenceId() const; + + // Implements DocumentMessage. + uint32_t getType() const; + + string toString() const { return "getbucketstatemessage"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.cpp new file mode 100644 index 00000000000..f0ce18f66b7 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.cpp @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/getbucketstatereply.h> + +namespace documentapi { + +GetBucketStateReply::GetBucketStateReply() : + DocumentReply(DocumentProtocol::REPLY_GETBUCKETSTATE), + _state() +{ + // empty +} + +GetBucketStateReply::GetBucketStateReply(std::vector<DocumentState> &state) : + DocumentReply(DocumentProtocol::REPLY_GETBUCKETSTATE), + _state() +{ + _state.swap(state); +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.h b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.h new file mode 100644 index 00000000000..4de839d5e6c --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getbucketstatereply.h @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/messages/documentreply.h> +#include "documentstate.h" + +namespace documentapi { + +class GetBucketStateReply : public DocumentReply { +private: + std::vector<DocumentState> _state; + +public: + /** + * Constructs a new reply with no content. + */ + GetBucketStateReply(); + + /** + * Constructs a new reply with initial content. This method takes ownership of the provided state, i.e. it + * swaps the content of the argument into self. + * + * @param state The state to swap. + */ + GetBucketStateReply(std::vector<DocumentState> &state); + + /** + * Sets the bucket state of this by swapping the content of the provided state object. + * + * @param state The state to swap. + */ + void setBucketState(std::vector<DocumentState> &state) { _state.swap(state); } + + /** + * Returns the bucket state contained in this. + * + * @return The state object. + */ + std::vector<DocumentState> &getBucketState() { return _state; } + + /** + * Returns the bucket state contained in this. + * + * @return The state object. + */ + const std::vector<DocumentState> &getBucketState() const { return _state; } + + string toString() const { return "getbucketstatereply"; } +}; + +} // documentapi + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentmessage.cpp new file mode 100644 index 00000000000..08df8a88c50 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentmessage.cpp @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/getdocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/getdocumentreply.h> + +namespace documentapi { + +GetDocumentMessage::GetDocumentMessage() : + DocumentMessage(), + _documentId(), + _fieldSet("[all]") +{ + // empty +} + +GetDocumentMessage::GetDocumentMessage(const document::DocumentId &documentId, int flags) : + DocumentMessage(), + _documentId(documentId) +{ + setFlags(flags); +} + +GetDocumentMessage::GetDocumentMessage(const document::DocumentId &documentId, + const vespalib::stringref & fieldSet) : + DocumentMessage(), + _documentId(documentId), + _fieldSet(fieldSet) +{ +} + +DocumentReply::UP +GetDocumentMessage::doCreateReply() const +{ + return DocumentReply::UP(new GetDocumentReply()); +} + +bool +GetDocumentMessage::hasSequenceId() const +{ + return true; +} + +uint64_t +GetDocumentMessage::getSequenceId() const +{ + return *reinterpret_cast<const uint64_t*>(_documentId.getGlobalId().get()); +} + +uint32_t +GetDocumentMessage::getType() const +{ + return DocumentProtocol::MESSAGE_GETDOCUMENT; +} + +const document::DocumentId & +GetDocumentMessage::getDocumentId() const +{ + return _documentId; +} + +void +GetDocumentMessage::setDocumentId(const document::DocumentId &documentId) +{ + _documentId = documentId; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentmessage.h new file mode 100644 index 00000000000..b174c729cae --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentmessage.h @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/base/documentid.h> +#include <vespa/documentapi/messagebus/messages/documentmessage.h> + +namespace documentapi { + +class GetDocumentMessage : public DocumentMessage { +private: + document::DocumentId _documentId; // The identifier of the document to retrieve. + string _fieldSet; // Comma-separated list of fields to return + +protected: + // Implements DocumentMessage. + DocumentReply::UP doCreateReply() const; + +public: + /** + * Convenience typedef. + */ + typedef std::unique_ptr<GetDocumentMessage> UP; + typedef std::shared_ptr<GetDocumentMessage> SP; + + enum { + FLAG_NONE = 0, + FLAG_ONLY_HEADER = 1 + }; + + /** + * Constructs a new message for deserialization. + */ + GetDocumentMessage(); + + /** + * Constructs a new document get message. + * + * @param documentId The identifier of the document to retrieve. + * @param flags How to retrieve the document. + */ + GetDocumentMessage(const document::DocumentId &documentId, int flags = 0); + + /** + * Constructs a new document get message. + * + * @param documentId The identifier of the document to retrieve. + * @param fieldSet The fields to retrieve (comma-separated) + */ + GetDocumentMessage(const document::DocumentId &documentId, + const vespalib::stringref & fieldSet); + + /** + * Returns the identifier of the document to retrieve. + * + * @return The document id. + */ + const document::DocumentId &getDocumentId() const; + + /** + * Sets the identifier of the document to retrieve. + * + * @param documentId The document id to set. + */ + void setDocumentId(const document::DocumentId &documentId); + + /** + * Returs the storage flags of this message. + * + * @return The storage flags. + */ + int getFlags() const { return (_fieldSet == "[header]" ? FLAG_ONLY_HEADER : + FLAG_NONE); }; + + /** + * Sets the storage flags of this message. + * + * @param flags The flags to set. + */ + void setFlags(int flags) { + _fieldSet = (flags == FLAG_ONLY_HEADER) ? "[header]" : "[all]"; + } + + /** + * Returns the fields to be retrieved by the get. + */ + const string& getFieldSet() const { return _fieldSet; } + + // Overrides DocumentMessage. + bool hasSequenceId() const; + + // Overrides DocumentMessage. + uint64_t getSequenceId() const; + + // Implements DocumentMessage. + uint32_t getType() const; + + string toString() const { return "getdocumentmessage"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.cpp new file mode 100644 index 00000000000..b8a952bd421 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.cpp @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/messages/getdocumentreply.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> + +namespace documentapi { + +GetDocumentReply::GetDocumentReply() : + DocumentAcceptedReply(DocumentProtocol::REPLY_GETDOCUMENT), + _document(), + _lastModified(0) +{ + // empty +} + +GetDocumentReply::GetDocumentReply(document::Document::SP document) : + DocumentAcceptedReply(DocumentProtocol::REPLY_GETDOCUMENT), + _document(document), + _lastModified(0) +{ + if (_document.get()) { + _lastModified = _document->getLastModified(); + } +} + +document::Document::SP +GetDocumentReply::getDocument() +{ + return _document; +} + +std::shared_ptr<const document::Document> +GetDocumentReply::getDocument() const +{ + return _document; +} + +void +GetDocumentReply::setDocument(document::Document::SP document) +{ + _document = document; + if (document.get()) { + _lastModified = document->getLastModified(); + } else { + _lastModified = 0u; + } +} + +void +GetDocumentReply::setLastModified(uint64_t lastModified) +{ + _lastModified = lastModified; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.h new file mode 100644 index 00000000000..295f1b3a84c --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.h @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/fieldvalue/document.h> +#include <vespa/documentapi/messagebus/messages/documentacceptedreply.h> + +namespace documentapi { + +class GetDocumentReply : public DocumentAcceptedReply { +private: + document::Document::SP _document; + uint64_t _lastModified; + +public: + /** + * Convenience typedef. + */ + typedef std::unique_ptr<GetDocumentReply> UP; + typedef std::shared_ptr<GetDocumentReply> SP; + + /** + * Constructs a new reply for deserialization. + */ + GetDocumentReply(); + + /** + * Constructs a new document get reply. + * + * @param document The document requested. + */ + GetDocumentReply(document::Document::SP document); + + /** + * Returns the document retrieved. + * + * @return The document. + */ + document::Document::SP getDocument(); + + /** + * Returns the document retrieved. + * + * @return The document. + */ + std::shared_ptr<const document::Document> getDocument() const; + + /** + * Sets the document retrieved. + * + * @param document The document. + */ + void setDocument(document::Document::SP document); + + /** + * Returns the date the document was last modified. + * + * @return The date. + */ + uint64_t getLastModified() const { return _lastModified; }; + + /** + * Set the date the document was last modified. + * + * @param lastModified The date. + */ + void setLastModified(uint64_t lastModified); + + string toString() const { return "getdocumentreply"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/multioperationmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/multioperationmessage.cpp new file mode 100644 index 00000000000..58cda975336 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/multioperationmessage.cpp @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/messages/multioperationmessage.h> +#include <vespa/vdslib/container/mutabledocumentlist.h> + +namespace documentapi { + +MultiOperationMessage::MultiOperationMessage(const document::DocumentTypeRepo::SP & repo, const document::BucketId& bucketId) : + VisitorMessage(), + _bucketId(bucketId), + _buffer(0), + _operations(repo, 0, 0), + _keepTimeStamps(false) +{ + // empty +} + +MultiOperationMessage::MultiOperationMessage(const document::DocumentTypeRepo::SP & repo, const document::BucketId& bucketId, int bufferSize) : + VisitorMessage(), + _bucketId(bucketId), + _buffer(bufferSize), + _operations(repo, &_buffer[0], _buffer.size(), false), + _keepTimeStamps(false) +{ +} + + +MultiOperationMessage::MultiOperationMessage(const document::DocumentTypeRepo::SP & repo, + const document::BucketId& bucketId, + const std::vector<char> &buffer, + bool timeStamps) : + VisitorMessage(), + _bucketId(bucketId), + _operations(repo, 0, 0), + _keepTimeStamps(timeStamps) +{ + setOperations(repo, buffer); +} + +MultiOperationMessage::MultiOperationMessage(const document::BucketId& bucketId, + vdslib::DocumentList &operations, + bool timeStamps) : + VisitorMessage(), + _bucketId(bucketId), + _buffer(), + _operations(operations.getTypeRepo(), 0, 0), + _keepTimeStamps(timeStamps) +{ + _buffer.resize(operations.getBufferSize()); + memcpy(&_buffer[0], operations.getBuffer(), _buffer.size()); + _operations = vdslib::DocumentList(operations.getTypeRepo(), &_buffer[0], _buffer.size(), true); +} + +void +MultiOperationMessage::setOperations(const document::DocumentTypeRepo::SP & repo, const std::vector<char> &buffer) +{ + _buffer = buffer; + if (_buffer.size() > 0) { + _operations = vdslib::DocumentList(repo, &_buffer[0], _buffer.size(), true); + } + verifyBucketId(); +} + +void +MultiOperationMessage::setOperations(vdslib::DocumentList &operations) +{ + if (&_buffer[0] == operations.getBuffer()) { + _buffer.resize(operations.getBufferSize()); + } else { + _buffer.resize(operations.getBufferSize()); + memcpy(&_buffer[0], operations.getBuffer(), _buffer.size()); + } + _operations = vdslib::DocumentList(operations.getTypeRepo(), &_buffer[0], _buffer.size(), true); + verifyBucketId(); +} + +void +MultiOperationMessage::verifyBucketId() const { + document::BucketIdFactory fac; + + for (vdslib::DocumentList::const_iterator iter = _operations.begin(); + iter != _operations.end(); + iter++) { + document::DocumentId docId = iter->getDocumentId(); + document::BucketId bucketId = fac.getBucketId(docId); + bucketId.setUsedBits(_bucketId.getUsedBits()); + if (bucketId != _bucketId) { + throw vespalib::IllegalArgumentException(vespalib::make_string("Operations added to a MultiOperationMessage must belong to the specified bucketId. Document %s with bucket id %s does not match bucket id %s", docId.toString().c_str(), bucketId.toString().c_str(), _bucketId.toString().c_str())); + } + } +} + +mbus::Message::UP +MultiOperationMessage::create(const document::DocumentTypeRepo::SP & repo, const document::BucketId& bucketId, const vdslib::OperationList &opl) { + std::unique_ptr<MultiOperationMessage> msg( + new MultiOperationMessage(repo, bucketId, opl.getRequiredBufferSize())); + std::vector<char> &buf = msg->getBuffer(); + vdslib::MutableDocumentList mdl(repo, &(buf[0]), buf.size()); + if (! mdl.addOperationList(opl)) { + abort(); + } + msg->setOperations(mdl); + return mbus::Message::UP(msg.release()); +} + +uint32_t +MultiOperationMessage::getApproxSize() const +{ + return _operations.getBufferSize(); +} + +DocumentReply::UP +MultiOperationMessage::doCreateReply() const +{ + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_MULTIOPERATION)); +} + +uint32_t MultiOperationMessage::getType() const +{ + return DocumentProtocol::MESSAGE_MULTIOPERATION; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/multioperationmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/multioperationmessage.h new file mode 100644 index 00000000000..5b116ab81e6 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/multioperationmessage.h @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vdslib/container/parameters.h> +#include <vespa/vdslib/container/documentlist.h> +#include <vespa/vdslib/container/operationlist.h> +#include <vespa/document/bucket/bucketid.h> +#include <vespa/documentapi/messagebus/messages/visitor.h> +#include <vespa/documentapi/messagebus/messages/documentreply.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> + +namespace documentapi { + +/** + * @class MultiOperationMessage + * @ingroup message + * + * @brief Encapsulates a set of operations (PUT, REMOVE, UPDATE). + */ +class MultiOperationMessage : public VisitorMessage { +private: + document::BucketId _bucketId; + std::vector<char> _buffer; + vdslib::DocumentList _operations; + bool _keepTimeStamps; + +protected: + DocumentReply::UP doCreateReply() const; + +public: + typedef std::unique_ptr<MultiOperationMessage> UP; + + MultiOperationMessage(const document::DocumentTypeRepo::SP & repo, const document::BucketId& bucketId); + MultiOperationMessage(const document::DocumentTypeRepo::SP & repo, const document::BucketId& bucketId, int bufferSize); + MultiOperationMessage(const document::DocumentTypeRepo::SP & repo, const document::BucketId& bucketId, + const std::vector<char>& buffer, bool keepTimeStamps = false); + MultiOperationMessage(const document::BucketId& bucketId, vdslib::DocumentList& docList, bool keepTimeStamps = false); + + static mbus::Message::UP create(const document::DocumentTypeRepo::SP & repo, const document::BucketId& bucketId, const vdslib::OperationList& operations); + + std::vector<char>& getBuffer() { return _buffer; } + const std::vector<char>& getBuffer() const { return _buffer; } + + void setOperations(const document::DocumentTypeRepo::SP & repo, const std::vector<char>& buffer); + void setOperations(vdslib::DocumentList& operations); + const vdslib::DocumentList& getOperations() const { return _operations; } + + void serialize(document::ByteBuffer& buf) const; + + uint32_t getApproxSize() const; + + uint32_t getType() const; + const document::BucketId& getBucketId() const { return _bucketId; } + + bool keepTimeStamps() const { return _keepTimeStamps;} + void keepTimeStamps(bool b) { _keepTimeStamps = b;} + + string toString() const { return "multioperationmessage"; } + +private: + void verifyBucketId() const; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp new file mode 100644 index 00000000000..a1df5176dd8 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/writedocumentreply.h> + +namespace documentapi { + +PutDocumentMessage::PutDocumentMessage() : + TestAndSetMessage(), + _document(), + _time(0) +{ + // empty +} + +PutDocumentMessage::PutDocumentMessage(document::Document::SP document) : + TestAndSetMessage(), + _document(), + _time(0) +{ + setDocument(document); +} + +DocumentReply::UP +PutDocumentMessage::doCreateReply() const +{ + return DocumentReply::UP(new WriteDocumentReply(DocumentProtocol::REPLY_PUTDOCUMENT)); +} + +bool +PutDocumentMessage::hasSequenceId() const +{ + return true; +} + +uint64_t +PutDocumentMessage::getSequenceId() const +{ + return *reinterpret_cast<const uint64_t*>(_document->getId().getGlobalId().get()); +} + +uint32_t +PutDocumentMessage::getType() const +{ + return DocumentProtocol::MESSAGE_PUTDOCUMENT; +} + +document::Document::SP +PutDocumentMessage::getDocument() +{ + return _document; +} + +std::shared_ptr<const document::Document> +PutDocumentMessage::getDocument() const +{ + return _document; +} + +void +PutDocumentMessage::setDocument(document::Document::SP document) +{ + if (document.get() == NULL) { + throw vespalib::IllegalArgumentException("Document can not be null.", VESPA_STRLOC); + } + _document = document; +} + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h new file mode 100644 index 00000000000..f216c4c2955 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/fieldvalue/document.h> +#include <vespa/documentapi/messagebus/messages/testandsetmessage.h> + +namespace documentapi { + +class PutDocumentMessage : public TestAndSetMessage { +private: + document::Document::SP _document; + uint64_t _time; + +protected: + // Implements DocumentMessage. + DocumentReply::UP doCreateReply() const override; + +public: + /** + * Convenience typedef. + */ + typedef std::unique_ptr<PutDocumentMessage> UP; + typedef std::shared_ptr<PutDocumentMessage> SP; + + /** + * Constructs a new document message for deserialization. + */ + PutDocumentMessage(); + + /** + * Constructs a new document put message. + * + * @param document The document to put. + */ + PutDocumentMessage(document::Document::SP document); + + /** + * Returns the document to put. + * + * @return The document. + */ + document::Document::SP getDocument(); + + /** + * Returns the document to put. + * + * @return The document. + */ + std::shared_ptr<const document::Document> getDocument() const; + + /** + * Sets the document to put. + * + * @param document The document to set. + */ + void setDocument(document::Document::SP document); + + /** + * Returns the timestamp of the document to put. + * + * @return The document timestamp. + */ + uint64_t getTimestamp() const { return _time; } + + /** + * Sets the timestamp of the document to put. + * + * @param time The timestamp to set. + */ + void setTimestamp(uint64_t time) { _time = time; } + + bool hasSequenceId() const override; + + uint64_t getSequenceId() const override; + + uint32_t getType() const override; + + string toString() const { return "putdocumentmessage"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp new file mode 100644 index 00000000000..656d11b769e --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/documentapi/messagebus/messages/queryresultmessage.h> + +namespace documentapi { + +QueryResultMessage::QueryResultMessage() : + VisitorMessage(), + _searchResult(), + _summary() +{ + // empty +} + +QueryResultMessage::QueryResultMessage(const vdslib::SearchResult & result, const vdslib::DocumentSummary & summary) : + VisitorMessage(), + _searchResult(result), + _summary(summary) +{ + // empty +} + +DocumentReply::UP +QueryResultMessage::doCreateReply() const +{ + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_QUERYRESULT)); +} + +uint32_t +QueryResultMessage::getApproxSize() const +{ + return getSearchResult().getSerializedSize() + getDocumentSummary().getSerializedSize(); +} + +uint32_t +QueryResultMessage::getType() const +{ + return DocumentProtocol::MESSAGE_QUERYRESULT; +} + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h new file mode 100644 index 00000000000..c399501dfee --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vdslib/container/searchresult.h> +#include <vespa/vdslib/container/documentsummary.h> +#include <vespa/documentapi/messagebus/messages/visitor.h> + +namespace documentapi { + +class QueryResultMessage : public VisitorMessage { +private: + vdslib::SearchResult _searchResult; + vdslib::DocumentSummary _summary; +protected: + // Implements VisitorMessage. + DocumentReply::UP doCreateReply() const; + +public: + /** + * Convenience typedefs. + */ + typedef std::unique_ptr<QueryResultMessage> UP; + typedef std::shared_ptr<QueryResultMessage> SP; + + /** + * Constructs a new search result message for deserialization. + */ + QueryResultMessage(); + + /** + * Constructs a new search result message for the given search result. + * + * @param result The result to set. + */ + QueryResultMessage(const vdslib::SearchResult & result, const vdslib::DocumentSummary & summary); + + // Overrides VisitorMessage. + uint32_t getApproxSize() const; + + // Implements VisitorMessage. + uint32_t getType() const; + + // Accessors + const vdslib::SearchResult & getSearchResult() const { return _searchResult; } + vdslib::SearchResult & getSearchResult() { return _searchResult; } + const vdslib::DocumentSummary & getDocumentSummary() const { return _summary; } + vdslib::DocumentSummary & getDocumentSummary() { return _summary; } + + string toString() const { return "queryresultmessage"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentmessage.cpp new file mode 100644 index 00000000000..f93405fdae0 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentmessage.cpp @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/removedocumentreply.h> + +namespace documentapi { + +RemoveDocumentMessage::RemoveDocumentMessage() : + TestAndSetMessage(), + _documentId() +{ + // empty +} + +RemoveDocumentMessage::RemoveDocumentMessage(const document::DocumentId& documentId) : + TestAndSetMessage(), + _documentId(documentId) +{ + // empty +} + +DocumentReply::UP +RemoveDocumentMessage::doCreateReply() const +{ + return DocumentReply::UP(new RemoveDocumentReply()); +} + +bool +RemoveDocumentMessage::hasSequenceId() const +{ + return true; +} + +uint64_t +RemoveDocumentMessage::getSequenceId() const +{ + return *reinterpret_cast<const uint64_t*>(_documentId.getGlobalId().get()); +} + +uint32_t +RemoveDocumentMessage::getType() const +{ + return DocumentProtocol::MESSAGE_REMOVEDOCUMENT; +} + +const document::DocumentId& +RemoveDocumentMessage::getDocumentId() const +{ + return _documentId; +} + +void +RemoveDocumentMessage::setDocumentId(const document::DocumentId& documentId) +{ + _documentId = documentId; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentmessage.h new file mode 100644 index 00000000000..72c277ba54c --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentmessage.h @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/base/documentid.h> +#include <vespa/documentapi/messagebus/messages/testandsetmessage.h> + +namespace documentapi { + +class RemoveDocumentMessage : public TestAndSetMessage { +private: + document::DocumentId _documentId; // The identifier of the document to remove. + +protected: + // Implements DocumentMessage. + DocumentReply::UP doCreateReply() const; + +public: + /** + * Convenience typedef. + */ + typedef std::unique_ptr<RemoveDocumentMessage> UP; + typedef std::shared_ptr<RemoveDocumentMessage> SP; + + /** + * Constructs a new remove document message with no content. + */ + RemoveDocumentMessage(); + + /** + * Constructs a new remove document message with a given document id. + * + * @param id The identifier of the document to remove. + */ + RemoveDocumentMessage(const document::DocumentId& id); + + /** + * Returns the identifier of the document to remove. + * + * @return The document id. + */ + const document::DocumentId& getDocumentId() const; + + /** + * Sets the identifier of the document to remove. + * + * @param documentId The document id to set. + */ + void setDocumentId(const document::DocumentId& documentId); + + // Overrides DocumentMessage. + bool hasSequenceId() const; + + // Overrides DocumentMessage. + uint64_t getSequenceId() const; + + // Implements DocumentMessage. + uint32_t getType() const; + + string toString() const { return "removedocumentmessage"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentreply.cpp new file mode 100644 index 00000000000..336aa7fb73b --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentreply.cpp @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/removedocumentreply.h> + +namespace documentapi { + +RemoveDocumentReply::RemoveDocumentReply() : + WriteDocumentReply(DocumentProtocol::REPLY_REMOVEDOCUMENT), + _found(true) +{ + // empty +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentreply.h new file mode 100644 index 00000000000..5e02fe336c9 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removedocumentreply.h @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/messages/writedocumentreply.h> + +namespace documentapi { + +class RemoveDocumentReply : public WriteDocumentReply { +private: + bool _found; + +public: + /** + * Convenience typedef. + */ + typedef std::unique_ptr<RemoveDocumentReply> UP; + typedef std::shared_ptr<RemoveDocumentReply> SP; + +public: + /** + * Constructs a new reply with no content. + */ + RemoveDocumentReply(); + + /** + * Set whether or not the document was found and removed. + * + * @param found True if the document was found. + */ + void setWasFound(bool found) { _found = found; } + + /** + * Returns whether or not the document was found and removed. + * + * @return True if document was found. + */ + bool wasFound() const { return getWasFound(); } + + /** + * Returns whether or not the document was found and removed. + * + * @return True if document was found. + */ + bool getWasFound() const { return _found; } + + string toString() const { return "removedocumentreply"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp new file mode 100644 index 00000000000..29c5d40ca7e --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.cpp @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/messages/removelocationmessage.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/document/select/parser.h> +#include <vespa/document/bucket/bucketselector.h> + +namespace documentapi { + +RemoveLocationMessage::RemoveLocationMessage( + const document::BucketIdFactory& factory, + document::select::Parser& parser, + const string& documentSelection) + : _documentSelection(documentSelection) +{ + document::BucketSelector bucketSel(factory); + std::unique_ptr<document::BucketSelector::BucketVector> exprResult( + bucketSel.select(*parser.parse(documentSelection))); + + if (exprResult.get() && exprResult->size() == 1) { + _bucketId = (*exprResult)[0]; + } else { + throw vespalib::IllegalArgumentException( + "Document selection doesn't map to a single bucket!", + VESPA_STRLOC); + } +} + +DocumentReply::UP +RemoveLocationMessage::doCreateReply() const { + return DocumentReply::UP( + new DocumentReply(DocumentProtocol::REPLY_REMOVELOCATION)); +} + +uint32_t +RemoveLocationMessage::getType() const { + return DocumentProtocol::MESSAGE_REMOVELOCATION; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h new file mode 100644 index 00000000000..8d5bfbae1f8 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/removelocationmessage.h @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/messages/documentmessage.h> +#include <vespa/document/bucket/bucketid.h> +#include <vespa/document/bucket/bucketselector.h> +#include <vespa/document/select/parser.h> + +namespace documentapi { + +/** + * Message (VDS only) to remove an entire location for users using userdoc or groupdoc schemes for their documents. + * A location in this context is either a user id or a group name. + */ +class RemoveLocationMessage : public DocumentMessage { +public: + RemoveLocationMessage(const document::BucketIdFactory& factory, document::select::Parser& parser, const string& documentSelection); + + const string& getDocumentSelection() const { return _documentSelection; } + + uint32_t getType() const; + + const document::BucketId& getBucketId() const { return _bucketId; }; + + string toString() const { return "removelocationmessage"; } + +protected: + DocumentReply::UP doCreateReply() const; + +private: + string _documentSelection; + document::BucketId _bucketId; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp new file mode 100644 index 00000000000..818f085c35b --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.cpp @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/documentapi/messagebus/messages/searchresultmessage.h> + +using vdslib::SearchResult; + +namespace documentapi { + +SearchResultMessage::SearchResultMessage() : + VisitorMessage(), + SearchResult() +{ + // empty +} + +SearchResultMessage::SearchResultMessage(const SearchResult &result) : + VisitorMessage(), + SearchResult(result) +{ + // empty +} + +DocumentReply::UP +SearchResultMessage::doCreateReply() const +{ + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_SEARCHRESULT)); +} + +uint32_t +SearchResultMessage::getApproxSize() const +{ + return SearchResult::getSerializedSize(); +} + +uint32_t +SearchResultMessage::getType() const +{ + return DocumentProtocol::MESSAGE_SEARCHRESULT; +} + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h new file mode 100644 index 00000000000..2a2d2cfe201 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/searchresultmessage.h @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vdslib/container/searchresult.h> +#include <vespa/documentapi/messagebus/messages/visitor.h> + +namespace documentapi { + +class SearchResultMessage : public VisitorMessage, + public vdslib::SearchResult { +protected: + // Implements VisitorMessage. + DocumentReply::UP doCreateReply() const; + +public: + /** + * Convenience typedefs. + */ + typedef std::unique_ptr<SearchResultMessage> UP; + typedef std::shared_ptr<SearchResultMessage> SP; + + /** + * Constructs a new search result message for deserialization. + */ + SearchResultMessage(); + + /** + * Constructs a new search result message for the given search result. + * + * @param result The result to set. + */ + SearchResultMessage(const vdslib::SearchResult &result); + + // Overrides VisitorMessage. + uint32_t getApproxSize() const; + + // Implements VisitorMessage. + uint32_t getType() const; + + string toString() const { return "searchresultmessage"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.cpp new file mode 100644 index 00000000000..615eb45cbb1 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.cpp @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/statbucketmessage.h> +#include <vespa/documentapi/messagebus/messages/statbucketreply.h> + +namespace documentapi { + +StatBucketMessage::StatBucketMessage() : + DocumentMessage(), + _bucket(), + _documentSelection() +{ + // empty +} + +StatBucketMessage::StatBucketMessage(document::BucketId bucket, const string& documentSelection) : + DocumentMessage(), + _bucket(bucket), + _documentSelection(documentSelection) +{ + // empty +} + +DocumentReply::UP +StatBucketMessage::doCreateReply() const +{ + return DocumentReply::UP(new StatBucketReply()); +} + +bool +StatBucketMessage::hasSequenceId() const +{ + return true; +} + +uint64_t +StatBucketMessage::getSequenceId() const +{ + return _bucket.getRawId(); +} + +uint32_t +StatBucketMessage::getType() const +{ + return DocumentProtocol::MESSAGE_STATBUCKET; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.h new file mode 100644 index 00000000000..818aaa6a9f1 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/statbucketmessage.h @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/bucket/bucketid.h> +#include <vespa/documentapi/messagebus/messages/documentmessage.h> + +namespace documentapi { + +class StatBucketMessage : public DocumentMessage { +private: + document::BucketId _bucket; + string _documentSelection; + +protected: + // Implements DocumentMessage. + DocumentReply::UP doCreateReply() const; + +public: + /** + * Constructs a new message with no content. + */ + StatBucketMessage(); + + /** + * Constructs a new message with initial content. + * + * @param bucket The bucket whose list to retrieve. + */ + StatBucketMessage(document::BucketId bucket, const string& documentSelection); + + /** + * Returns the bucket to stat. + * + * @return The bucket id. + */ + document::BucketId getBucketId() const { return _bucket; } + + /** + * Set the bucket to stat. + * + * @param id The identifier to set. + */ + void setBucketId(document::BucketId id) { _bucket = id; }; + + /** + * Returns the document selection used to filter the documents + * returned. + * + * @return The selection string. + */ + const string &getDocumentSelection() const { return _documentSelection; }; + + /** + * Sets the document selection used to filter the documents returned. + * + * @param value The selection string to set. + */ + void setDocumentSelection(const string &value) { _documentSelection = value; }; + + // Overrides DocumentMessage. + bool hasSequenceId() const; + + // Overrides DocumentMessage. + uint64_t getSequenceId() const; + + // Implements DocumentMessage. + uint32_t getType() const; + + string toString() const { return "statbucketmessage"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/statbucketreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/statbucketreply.cpp new file mode 100644 index 00000000000..13e58830139 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/statbucketreply.cpp @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/statbucketreply.h> + +namespace documentapi { + +StatBucketReply::StatBucketReply() : + DocumentReply(DocumentProtocol::REPLY_STATBUCKET), + _results() +{ + // empty +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/statbucketreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/statbucketreply.h new file mode 100644 index 00000000000..551b455a9c2 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/statbucketreply.h @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/messages/documentreply.h> + +namespace documentapi { + +class StatBucketReply : public DocumentReply { +private: + string _results; + +public: + StatBucketReply(); + + void setResults(const string& results) { _results = results; } + + const string& getResults() const { return _results; } + + string toString() const { return "statbucketreply"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/testandsetcondition.h b/documentapi/src/vespa/documentapi/messagebus/messages/testandsetcondition.h new file mode 100644 index 00000000000..54800b96743 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/testandsetcondition.h @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell +#pragma once +#include <vespa/fastos/fastos.h> + +namespace documentapi { + +class TestAndSetCondition { +private: + string _selection; + +public: + TestAndSetCondition() + : _selection() + {} + + TestAndSetCondition(vespalib::stringref selection) + : _selection(selection) + {} + + TestAndSetCondition(const TestAndSetCondition &) = default; + TestAndSetCondition & operator=(const TestAndSetCondition &) = default; + + TestAndSetCondition(TestAndSetCondition &&) = default; + TestAndSetCondition & operator=(TestAndSetCondition &&) = default; + + const string & getSelection() const { return _selection; } + bool isPresent() const { return !_selection.empty(); } +}; + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/testandsetmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/testandsetmessage.h new file mode 100644 index 00000000000..21d92385a64 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/testandsetmessage.h @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell +#pragma once + +#include <vespa/documentapi/messagebus/messages/documentmessage.h> +#include <vespa/documentapi/messagebus/messages/testandsetcondition.h> +#include <string> + +namespace documentapi { + +class TestAndSetMessage : public DocumentMessage { +private: + TestAndSetCondition _condition; + +public: + void setCondition(const TestAndSetCondition & condition) { _condition = condition; } + const TestAndSetCondition & getCondition() const { return _condition; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp new file mode 100644 index 00000000000..7f226f94bbc --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/updatedocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> + +namespace documentapi { + +UpdateDocumentMessage::UpdateDocumentMessage() : + TestAndSetMessage(), + _documentUpdate(), + _oldTime(0), + _newTime(0) +{ + // empty +} + +UpdateDocumentMessage::UpdateDocumentMessage(document::DocumentUpdate::SP documentUpdate) : + TestAndSetMessage(), + _documentUpdate(), + _oldTime(0), + _newTime(0) +{ + setDocumentUpdate(documentUpdate); +} + +DocumentReply::UP +UpdateDocumentMessage::doCreateReply() const +{ + return DocumentReply::UP(new UpdateDocumentReply()); +} + +bool +UpdateDocumentMessage::hasSequenceId() const +{ + return true; +} + +uint64_t +UpdateDocumentMessage::getSequenceId() const +{ + return *reinterpret_cast<const uint64_t*>(_documentUpdate->getId().getGlobalId().get()); +} + +uint32_t +UpdateDocumentMessage::getType() const +{ + return DocumentProtocol::MESSAGE_UPDATEDOCUMENT; +} + +document::DocumentUpdate::SP +UpdateDocumentMessage::getDocumentUpdate() +{ + return _documentUpdate; +} + +std::shared_ptr<const document::DocumentUpdate> +UpdateDocumentMessage::getDocumentUpdate() const +{ + return _documentUpdate; +} + +void +UpdateDocumentMessage::setDocumentUpdate(document::DocumentUpdate::SP documentUpdate) +{ + if (documentUpdate.get() == NULL) { + throw vespalib::IllegalArgumentException("Document update can not be null.", VESPA_STRLOC); + } + _documentUpdate = documentUpdate; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h new file mode 100644 index 00000000000..d924c514d60 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/update/documentupdate.h> +#include <vespa/documentapi/messagebus/messages/testandsetmessage.h> + +namespace documentapi { + +class UpdateDocumentMessage : public TestAndSetMessage { +private: + document::DocumentUpdate::SP _documentUpdate; + uint64_t _oldTime; + uint64_t _newTime; + +protected: + // Implements DocumentMessage. + DocumentReply::UP doCreateReply() const; + +public: + /** + * Convenience typedef. + */ + typedef std::unique_ptr<UpdateDocumentMessage> UP; + typedef std::shared_ptr<UpdateDocumentMessage> SP; + + /** + * Constructs a new document message for deserialization. + */ + UpdateDocumentMessage(); + + /** + * Constructs a new document update message. + * + * @param documentUpdate The document update to perform. + */ + UpdateDocumentMessage(document::DocumentUpdate::SP documentUpdate); + + /** + * Returns the document update to perform. + * + * @return The update. + */ + document::DocumentUpdate::SP getDocumentUpdate(); + + /** + * Returns the document update to perform. + * + * @return The update. + */ + std::shared_ptr<const document::DocumentUpdate> getDocumentUpdate() const; + + /** + * Sets the document update to perform. + * + * @param documentUpdate The document update to set. + */ + void setDocumentUpdate(document::DocumentUpdate::SP documentUpdate); + + /** + * Returns the timestamp required for this update to be applied. + * + * @return The document timestamp. + */ + uint64_t getOldTimestamp() const { return _oldTime; } + + /** + * Sets the timestamp required for this update to be applied. + * + * @param time The timestamp to set. + */ + void setOldTimestamp(uint64_t time) { _oldTime = time; } + + /** + * Returns the timestamp to assign to the updated document. + * + * @return The document timestamp. + */ + uint64_t getNewTimestamp() const { return _newTime; } + + /** + * Sets the timestamp to assign to the updated document. + * + * @param time The timestamp to set. + */ + void setNewTimestamp(uint64_t time) { _newTime = time; } + + // Overrides DocumentMessage. + bool hasSequenceId() const; + + // Overrides DocumentMessage. + uint64_t getSequenceId() const; + + // Implements DocumentMessage. + uint32_t getType() const; + + string toString() const { return "updatedocumentmessage"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentreply.cpp new file mode 100644 index 00000000000..a38ecdb7539 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentreply.cpp @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> + +namespace documentapi { + +UpdateDocumentReply::UpdateDocumentReply() : + WriteDocumentReply(DocumentProtocol::REPLY_UPDATEDOCUMENT), + _found(true) +{ + // empty +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentreply.h new file mode 100644 index 00000000000..918aebb0ea3 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentreply.h @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/messages/writedocumentreply.h> + +namespace documentapi { + +class UpdateDocumentReply : public WriteDocumentReply { +private: + bool _found; + +public: + /** + * Convenience typedef. + */ + typedef std::unique_ptr<UpdateDocumentReply> UP; + typedef std::shared_ptr<UpdateDocumentReply> SP; + +public: + /** + * Constructs a new reply with no content. + */ + UpdateDocumentReply(); + + /** + * Set whether or not the document was found and updated. + * + * @param found True if the document was found. + */ + void setWasFound(bool found) { _found = found; } + + /** + * Returns whether or not the document was found and updated. + * + * @return True if document was found. + */ + bool wasFound() const { return getWasFound(); } + + /** + * Returns whether or not the document was found and updated. + * + * @return True if document was found. + */ + bool getWasFound() const { return _found; } + + string toString() const { return "updatedocumentreply"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp new file mode 100644 index 00000000000..ecdc190ba93 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp @@ -0,0 +1,221 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".visitor.messages"); + +#include <vespa/document/bucket/bucketid.h> +#include "visitor.h" + +namespace documentapi { + +CreateVisitorMessage::CreateVisitorMessage() : + DocumentMessage(), + _docSelection(), + _maxPendingReplyCount(8), + _buckets(), + _fromTime(0), + _toTime(0), + _visitRemoves(false), + _fieldSet("[all]"), + _visitInconsistentBuckets(false), + _params(), + _version(42), + _ordering(document::OrderingSpecification::ASCENDING), + _maxBucketsPerVisitor(1) +{ + // empty +} + +CreateVisitorMessage::CreateVisitorMessage(const string& libraryName, + const string& instanceId, + const string& controlDestination, + const string& dataDestination) : + DocumentMessage(), + _libName(libraryName), + _instanceId(instanceId), + _controlDestination(controlDestination), + _dataDestination(dataDestination), + _docSelection(), + _maxPendingReplyCount(8), + _buckets(), + _fromTime(0), + _toTime(0), + _visitRemoves(false), + _fieldSet("[all]"), + _visitInconsistentBuckets(false), + _params(), + _version(42), + _ordering(document::OrderingSpecification::ASCENDING), + _maxBucketsPerVisitor(1) +{ + // empty +} + +DocumentReply::UP +CreateVisitorMessage::doCreateReply() const +{ + return DocumentReply::UP(new CreateVisitorReply(DocumentProtocol::REPLY_CREATEVISITOR)); +} + +uint32_t +CreateVisitorMessage::getType() const +{ + return DocumentProtocol::MESSAGE_CREATEVISITOR; +} + +DestroyVisitorMessage::DestroyVisitorMessage() : + DocumentMessage(), + _instanceId() +{ + // empty +} + +DestroyVisitorMessage::DestroyVisitorMessage(const string& instanceId) : + DocumentMessage(), + _instanceId(instanceId) +{ + // empty +} + +DocumentReply::UP +DestroyVisitorMessage::doCreateReply() const +{ + return DocumentReply::UP(new DocumentReply(DocumentProtocol::REPLY_DESTROYVISITOR)); +} + +uint32_t +DestroyVisitorMessage::getType() const +{ + return DocumentProtocol::MESSAGE_DESTROYVISITOR; +} + +VisitorReply::VisitorReply(uint32_t type) : + WriteDocumentReply(type) +{ + // empty +} + +CreateVisitorReply::CreateVisitorReply(uint32_t type) : + DocumentReply(type), + _lastBucket(document::BucketId(INT_MAX)) +{ +} + +VisitorInfoMessage::VisitorInfoMessage() : + VisitorMessage(), + _finishedBuckets(), + _errorMessage() +{ + // empty +} + +DocumentReply::UP +VisitorInfoMessage::doCreateReply() const +{ + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_VISITORINFO)); +} + +uint32_t +VisitorInfoMessage::getType() const +{ + return DocumentProtocol::MESSAGE_VISITORINFO; +} + +MapVisitorMessage::MapVisitorMessage() : + _data() +{ + // empty +} + +uint32_t +MapVisitorMessage::getApproxSize() const +{ + return _data.getSerializedSize(); +} + +DocumentReply::UP +MapVisitorMessage::doCreateReply() const +{ + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_MAPVISITOR)); +} + +uint32_t MapVisitorMessage::getType() const +{ + return DocumentProtocol::MESSAGE_MAPVISITOR; +} + +DocumentListMessage::Entry::Entry() +{ + // empty +} + +DocumentListMessage::Entry::Entry(int64_t timestamp, + document::Document::SP doc, + bool removeEntry) : + _timestamp(timestamp), + _document(doc), + _removeEntry(removeEntry) +{ + // empty +} + +DocumentListMessage::Entry::Entry(const Entry& other) : + _timestamp(other._timestamp), + _document(other._document), + _removeEntry(other._removeEntry) +{ + // empty +} + +DocumentListMessage::Entry::Entry(const document::DocumentTypeRepo &repo, + document::ByteBuffer& buf) +{ + buf.getLongNetwork(_timestamp); + _document.reset(new document::Document(repo, buf)); + uint8_t b; + buf.getByte(b); + _removeEntry = b>0; +} + +void +DocumentListMessage::Entry::serialize(document::ByteBuffer& buf) const +{ + buf.putLongNetwork(_timestamp); + _document->serialize(buf); + buf.putByte(_removeEntry ? 1 : 0); +} + +uint32_t +DocumentListMessage::Entry::getSerializedSize() const +{ + return sizeof(int64_t) + sizeof(uint8_t) + + _document->serialize()->getLength(); +} + +DocumentListMessage::DocumentListMessage() : + _bucketId(), + _documents() +{ + // empty +} + +DocumentListMessage::DocumentListMessage(document::BucketId bid) : + _bucketId(bid), + _documents() +{ + // empty +} + +DocumentReply::UP +DocumentListMessage::doCreateReply() const +{ + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTLIST)); +} + +uint32_t +DocumentListMessage::getType() const +{ + return DocumentProtocol::MESSAGE_DOCUMENTLIST; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h new file mode 100644 index 00000000000..e63696cd7ea --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h @@ -0,0 +1,301 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @file persistence.h + * + * Persistence related commands, like put, get & remove + */ +#pragma once + +#include <vespa/vdslib/container/parameters.h> +#include <vespa/vdslib/container/documentlist.h> +#include <vespa/vdslib/container/visitorstatistics.h> +#include <vespa/document/bucket/bucketid.h> +#include <vespa/documentapi/messagebus/messages/documentmessage.h> +#include <vespa/documentapi/messagebus/messages/documentreply.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/document/select/orderingspecification.h> +#include <vespa/documentapi/messagebus/messages/writedocumentreply.h> + +namespace documentapi { + +typedef uint64_t Timestamp; + +/** + * @class CreateVisitorMessage + * @ingroup message + * + * @brief Message for creating a visitor. + */ +class CreateVisitorMessage : public DocumentMessage { +private: + string _libName; + string _instanceId; + string _controlDestination; + string _dataDestination; + string _docSelection; + uint32_t _maxPendingReplyCount; + std::vector<document::BucketId> _buckets; + Timestamp _fromTime; + Timestamp _toTime; + bool _visitRemoves; + string _fieldSet; + bool _visitInconsistentBuckets; + vdslib::Parameters _params; + uint32_t _version; + document::OrderingSpecification::Order _ordering; + uint32_t _maxBucketsPerVisitor; + +protected: + DocumentReply::UP doCreateReply() const; + +public: + typedef std::unique_ptr<CreateVisitorMessage> UP; + + CreateVisitorMessage(); // must be deserialized into + + CreateVisitorMessage(const string& libraryName, + const string& instanceId, + const string& controlDestination, + const string& dataDestination); + + const string& getLibraryName() const { return _libName; } + void setLibraryName(const string& value) { _libName = value; } + + const string& getInstanceId() const { return _instanceId; } + void setInstanceId(const string& value) { _instanceId = value; } + + const string& getDocumentSelection() const { return _docSelection; } + void setDocumentSelection(const string& value) { _docSelection = value; } + + const string& getControlDestination() const { return _controlDestination; } + void setControlDestination(const string& value) { _controlDestination = value; }; + + const string& getDataDestination() const { return _dataDestination; } + void setDataDestination(const string& value) { _dataDestination = value; } + + const vdslib::Parameters& getParameters() const { return _params; } + vdslib::Parameters& getParameters() { return _params; } + void setParameters(const vdslib::Parameters& params) { _params = params; } + + uint32_t getMaximumPendingReplyCount() const { return _maxPendingReplyCount; } + void setMaximumPendingReplyCount(uint32_t count) { _maxPendingReplyCount = count; } + + const std::vector<document::BucketId>& getBuckets() const { return _buckets; } + std::vector<document::BucketId>& getBuckets() { return _buckets; } + + const document::BucketId getBucketId() const { return *_buckets.begin(); } + + bool visitRemoves() const { return _visitRemoves; } + void setVisitRemoves(bool val) { _visitRemoves = val; } + + void setVisitHeadersOnly(bool visitHeadersOnly_) { + _fieldSet = (visitHeadersOnly_ ? "[header]" : "[all]"); + } + + bool visitHeadersOnly() const { + return (_fieldSet == "[header]"); + } + + const string & getFieldSet() const { return _fieldSet; } + void setFieldSet(const vespalib::stringref & fieldSet) { _fieldSet = fieldSet; } + + bool visitInconsistentBuckets() const { return _visitInconsistentBuckets; } + void setVisitInconsistentBuckets(bool val) { _visitInconsistentBuckets = val; } + + Timestamp getFromTimestamp() const { return _fromTime; }; + void setFromTimestamp(Timestamp from) { _fromTime = from; }; + + Timestamp getToTimestamp() const { return _toTime; }; + void setToTimestamp(Timestamp to) { _toTime = to; }; + + document::OrderingSpecification::Order getVisitorOrdering() const { return _ordering; } + void setVisitorOrdering(document::OrderingSpecification::Order ordering) { _ordering = ordering; } + + uint32_t getMaxBucketsPerVisitor() const { return _maxBucketsPerVisitor; } + void setMaxBucketsPerVisitor(uint32_t max) { _maxBucketsPerVisitor = max; } + + uint32_t getType() const; + + void setVisitorDispatcherVersion(uint32_t version) { _version = version; }; + uint32_t getVisitorDispatcherVersion() const { return _version; }; + + string toString() const { return "createvisitormessage"; } +}; + +/** + * @class DestroyVisitorMessage + * @ingroup message + * + * @brief Message for removing a visitor. + */ +class DestroyVisitorMessage : public DocumentMessage { +private: + string _instanceId; + +protected: + DocumentReply::UP doCreateReply() const; + +public: + typedef std::unique_ptr<DestroyVisitorMessage> UP; + + DestroyVisitorMessage(); // must be deserialized into + + DestroyVisitorMessage(const string &instanceId); + + const string& getInstanceId() const { return _instanceId; } + void setInstanceId(const string& id) { _instanceId = id; } + + uint32_t getType() const; + + string toString() const { return "destroyvisitormessage"; } +}; + +/** + * Superclass for all commands sent from VisitorManager to a Visitor + * client. + */ +class VisitorMessage : public DocumentMessage { +protected: + VisitorMessage() { } +}; + +/** + * Superclass for all commands sent from VisitorManager to a Visitor + * client. + */ +class VisitorReply : public WriteDocumentReply { +public: + VisitorReply(uint32_t type); +}; + +class CreateVisitorReply : public DocumentReply { +private: + document::BucketId _lastBucket; + vdslib::VisitorStatistics _visitorStatistics; + +public: + CreateVisitorReply(uint32_t type); + + void setLastBucket(document::BucketId lastBucket) { _lastBucket = lastBucket; } + + document::BucketId getLastBucket() const { return _lastBucket; } + + const vdslib::VisitorStatistics& getVisitorStatistics() const { return _visitorStatistics; } + void setVisitorStatistics(const vdslib::VisitorStatistics& stats) { _visitorStatistics = stats; } + + string toString() const { return "createvisitorreply"; } +}; + +/** + * @class VisitorInfoMessage + * @ingroup message + * + * @brief Sends status information of an ongoing visitor. + * + * - Notification when individual buckets have been completely visited. + */ +class VisitorInfoMessage : public VisitorMessage { +private: + std::vector<document::BucketId> _finishedBuckets; + string _errorMessage; + +protected: + DocumentReply::UP doCreateReply() const; + +public: + typedef std::unique_ptr<VisitorInfoMessage> UP; + + VisitorInfoMessage(); + + std::vector<document::BucketId>& getFinishedBuckets() { return _finishedBuckets; } + const std::vector<document::BucketId>& getFinishedBuckets() const { return _finishedBuckets; } + + const string& getErrorMessage() const { return _errorMessage; } + void setErrorMessage(const string& errorMessage) { _errorMessage = errorMessage; }; + + uint32_t getType() const; + + string toString() const { return "visitorinfomessage"; } +}; + +/** + * @class MapVisitorMessage + * @ingroup message + * + * @brief Sends a docblock to a visitor. + */ +class MapVisitorMessage : public VisitorMessage { +private: + vdslib::Parameters _data; + +protected: + DocumentReply::UP doCreateReply() const; + +public: + typedef std::unique_ptr<MapVisitorMessage> UP; + + MapVisitorMessage(); + + vdslib::Parameters& getData() { return _data; }; + const vdslib::Parameters& getData() const { return _data; }; + + uint32_t getApproxSize() const; + uint32_t getType() const; + + string toString() const { return "mapvisitormessage"; } +}; + +/** + * @class DocumentListMessage + * @ingroup message + */ +class DocumentListMessage : public VisitorMessage { +public: + typedef std::unique_ptr<DocumentListMessage> UP; + + class Entry { + public: + Entry(); + Entry(int64_t timestamp, + document::Document::SP doc, + bool removeEntry); + Entry(const Entry& other); + Entry(const document::DocumentTypeRepo &repo, + document::ByteBuffer& buf); + + int64_t getTimestamp() { return _timestamp; } + const document::Document::SP& getDocument() { return _document; } + bool isRemoveEntry() { return _removeEntry; } + + void serialize(document::ByteBuffer& buf) const; + uint32_t getSerializedSize() const; + private: + int64_t _timestamp; + document::Document::SP _document; + bool _removeEntry; + }; + +private: + document::BucketId _bucketId; + std::vector<Entry> _documents; + +protected: + DocumentReply::UP doCreateReply() const; + +public: + DocumentListMessage(); + DocumentListMessage(document::BucketId bid); + + const document::BucketId& getBucketId() const { return _bucketId; }; + void setBucketId(const document::BucketId& id) { _bucketId = id; }; + + std::vector<Entry>& getDocuments() { return _documents; }; + const std::vector<Entry>& getDocuments() const { return _documents; }; + + uint32_t getType() const; + + string toString() const { return "documentlistmessage"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/writedocumentreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/writedocumentreply.h new file mode 100644 index 00000000000..27d229d5f53 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/writedocumentreply.h @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/messages/documentacceptedreply.h> + +namespace documentapi { + +/** + * This reply class is used by operations that perform writes to VDS/search, + * that is: Put, Remove, Update, MultiOperation. + */ +class WriteDocumentReply : public DocumentAcceptedReply { +private: + uint64_t _highestModificationTimestamp; + +public: + WriteDocumentReply(uint32_t type) : + DocumentAcceptedReply(type), + _highestModificationTimestamp(0) + {} + + /** + * Returns a unique VDS timestamp so that visiting up to and including that + * timestamp will return a state including this operation but not any + * operations sent to the same distributor after it. For PUT/UPDATE/REMOVE + * operations this timestamp will be the timestamp of the operation, while + * for MULTIOPERATION, the timestamp will be the highest one generated by + * it. + * + * @return Returns the modification timestamp. + */ + uint64_t getHighestModificationTimestamp() const { + return _highestModificationTimestamp; + } + + /** + * Sets the modification timestamp. + * + * @param timestamp + */ + void setHighestModificationTimestamp(uint64_t ts) { + _highestModificationTimestamp = ts; + } +}; + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/wrongdistributionreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/wrongdistributionreply.cpp new file mode 100644 index 00000000000..b9c5ee824c3 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/wrongdistributionreply.cpp @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/wrongdistributionreply.h> + +namespace documentapi { + +WrongDistributionReply::WrongDistributionReply() : + DocumentReply(DocumentProtocol::REPLY_WRONGDISTRIBUTION), + _systemState() +{ + // empty +} + +WrongDistributionReply::WrongDistributionReply(const string &systemState) : + DocumentReply(DocumentProtocol::REPLY_WRONGDISTRIBUTION), + _systemState(systemState) +{ + // empty +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/wrongdistributionreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/wrongdistributionreply.h new file mode 100644 index 00000000000..d70ab641fba --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/messages/wrongdistributionreply.h @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/messages/documentreply.h> + +namespace documentapi { + +class WrongDistributionReply : public DocumentReply { +private: + string _systemState; + +public: + typedef std::unique_ptr<WrongDistributionReply> UP; + typedef std::shared_ptr<WrongDistributionReply> SP; + + WrongDistributionReply(); + + WrongDistributionReply(const string &systemState); + + const string &getSystemState() const { return _systemState; }; + + void setSystemState(const string &state) { _systemState = state; }; + + string toString() const { return "wrongdistributionreply"; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/.gitignore b/documentapi/src/vespa/documentapi/messagebus/policies/.gitignore new file mode 100644 index 00000000000..3d63e81da12 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/.gitignore @@ -0,0 +1,5 @@ +.depend +Makefile +config-*.cpp +config-*.h +documentrouteselectorpolicy.def diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt new file mode 100644 index 00000000000..e68df872966 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt @@ -0,0 +1,23 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(documentapi_documentapipolicies OBJECT + SOURCES + andpolicy.cpp + externslobrokpolicy.cpp + storagepolicy.cpp + contentpolicy.cpp + messagetypepolicy.cpp + documentrouteselectorpolicy.cpp + errorpolicy.cpp + externpolicy.cpp + localservicepolicy.cpp + roundrobinpolicy.cpp + searchcolumnpolicy.cpp + searchrowpolicy.cpp + subsetservicepolicy.cpp + loadbalancer.cpp + loadbalancerpolicy.cpp + asyncinitializationpolicy.cpp + DEPENDS +) +vespa_generate_config(documentapi_documentapipolicies ../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def) +install(FILES ../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/andpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/andpolicy.cpp new file mode 100644 index 00000000000..79391315cd0 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/andpolicy.cpp @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/messagebus/error.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/policies/andpolicy.h> + +LOG_SETUP(".andpolicy"); + +namespace documentapi { + +ANDPolicy::ANDPolicy(const string ¶m) +{ + if (!param.empty()) { + mbus::Route route = mbus::Route::parse(param); + for (uint32_t i = 0; i < route.getNumHops(); ++i) { + _hops.push_back(route.getHop(i)); + } + } +} + +ANDPolicy::~ANDPolicy() +{ + // empty +} + +void +ANDPolicy::select(mbus::RoutingContext &context) +{ + if (_hops.empty()) { + context.addChildren(context.getAllRecipients()); + } else { + for (std::vector<mbus::Hop>::iterator it = _hops.begin(); + it != _hops.end(); ++it) + { + mbus::Route route = context.getRoute(); + route.setHop(0, *it); + context.addChild(route); + } + } + context.setSelectOnRetry(false); + context.addConsumableError(DocumentProtocol::ERROR_MESSAGE_IGNORED); +} + +void +ANDPolicy::merge(mbus::RoutingContext &context) +{ + DocumentProtocol::merge(context); +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/andpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/andpolicy.h new file mode 100644 index 00000000000..740413a4958 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/andpolicy.h @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vector> +#include <vespa/messagebus/routing/hop.h> +#include <vespa/messagebus/routing/iroutingpolicy.h> +#include <vespa/documentapi/common.h> + +namespace documentapi { + +/** + * An AND policy is a routing policy that can be used to write simple routes that split a message between multiple other + * destinations. It can either be configured in a routing config, which will then produce a policy that always selects + * all configured recipients, or it can be configured using the policy parameter (i.e. a string following the name of + * the policy). Note that configured recipients take precedence over recipients configured in the parameter string. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @version $Id$ + */ +class ANDPolicy : public mbus::IRoutingPolicy { +public: + /** + * Constructs a new AND policy that requires all recipients to be ok for it to merge their replies to an ok reply. + * I.e. all errors in all child replies are copied into the merged reply. + * + * @param param A string of recipients to select unless recipients have been configured. + */ + ANDPolicy(const string& param); + + /** + * Destructor. + * + * Frees all allocated resources. + */ + virtual ~ANDPolicy(); + + // Inherit doc from IRoutingPolicy. + virtual void select(mbus::RoutingContext &context); + + // Inherit doc from IRoutingPolicy. + virtual void merge(mbus::RoutingContext &context); + +private: + ANDPolicy(const ANDPolicy &); // hide + ANDPolicy &operator=(const ANDPolicy &); // hide + +private: + std::vector<mbus::Hop> _hops; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.cpp new file mode 100644 index 00000000000..2a8625f517d --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.cpp @@ -0,0 +1,125 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * File: AsyncInitializationPolicy.cpp + * Author: thomasg + * + * Created on July 17, 2012, 1:43 PM + */ + +#include "asyncinitializationpolicy.h" +#include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/vespalib/text/stringtokenizer.h> + +namespace documentapi { + +std::map<string, string> +AsyncInitializationPolicy::parse(string parameters) { + std::map<string, string> retVal; + + vespalib::StringTokenizer tokenizer(parameters, ";"); + for (uint32_t i = 0; i < tokenizer.size(); i++) { + string keyValue = tokenizer[i]; + vespalib::StringTokenizer keyV(keyValue, "="); + + if (keyV.size() == 1) { + retVal[keyV[0]] = "true"; + } else { + retVal[keyV[0]] = keyV[1]; + } + } + + return retVal; +} + +AsyncInitializationPolicy::AsyncInitializationPolicy( + const std::map<string, string>&) + : _executor(new vespalib::ThreadStackExecutor(1, 1024)), + _state(State::NOT_STARTED), + _syncInit(true) +{ +} + +AsyncInitializationPolicy::~AsyncInitializationPolicy() +{ +} + +void +AsyncInitializationPolicy::initSynchronous() +{ + init(); + _state = State::DONE; +} + +mbus::Error +AsyncInitializationPolicy::currentPolicyInitError() const +{ + // If an init error has been recorded for the last init attempt, report + // it back until we've managed to successfully complete the init step. + if (_error.empty()) { + return mbus::Error(DocumentProtocol::ERROR_NODE_NOT_READY, + "Waiting to initialize policy"); + } else { + return mbus::Error(DocumentProtocol::ERROR_POLICY_FAILURE, + "Error when creating policy: " + _error); + } +} + +void +AsyncInitializationPolicy::select(mbus::RoutingContext& context) +{ + if (_syncInit && _state != State::DONE) { + initSynchronous(); + } + + { + vespalib::MonitorGuard lock(_lock); + + if (_state == State::NOT_STARTED || _state == State::FAILED) { + // Only 1 task may be queued to the executor at any point in time. + // This is maintained by only scheduling a task when either no task + // has been created before or the previous task has signalled it is + // entirely done with accessing the state of this policy (including + // the mutex). After setting _state == RUNNING, only the task + // is allowed to mutate _state. + _executor->execute(vespalib::Executor::Task::UP(new Task(*this))); + _state = State::RUNNING; + } + + if (_state != State::DONE) { + mbus::Reply::UP reply(new mbus::EmptyReply()); + reply->addError(currentPolicyInitError()); + context.setReply(std::move(reply)); + return; + } + // We have the mutex in state DONE and since no task may be queued + // up for execution at this point, it's not possible for this to + // deadlock (executor will stall until all its tasks have finished + // executing, and any queued tasks would attempt to take the mutex + // we're currently holding, deadlocking both threads). + _executor.reset(nullptr); + } + + doSelect(context); +} + +void +AsyncInitializationPolicy::Task::run() +{ + string error; + + try { + error = _owner.init(); + } catch (const std::exception& e) { + error = e.what(); + } + + using State = AsyncInitializationPolicy::State; + + vespalib::MonitorGuard lock(_owner._lock); + _owner._error = error; + _owner._state = error.empty() ? State::DONE : State::FAILED; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.h new file mode 100644 index 00000000000..bf6cde33666 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/asyncinitializationpolicy.h @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/messagebus/routing/iroutingpolicy.h> +#include <vespa/messagebus/error.h> +#include <vespa/vespalib/util/executor.h> +#include <vespa/documentapi/common.h> +#include <vespa/vespalib/util/sync.h> +#include <map> + +namespace documentapi { + +class AsyncInitializationPolicy : public mbus::IRoutingPolicy { +public: + AsyncInitializationPolicy(const std::map<string, string>& parameters); + virtual ~AsyncInitializationPolicy(); + + static std::map<string, string> parse(string parameters); + + /** + * This function is called asynchronously at some point after the + * first select() is called. + * + * @return + */ + virtual string init() = 0; + + void select(mbus::RoutingContext &context) override; + + virtual void doSelect(mbus::RoutingContext& context) = 0; + + const string& getError() { + return _error; + } + + void initSynchronous(); + + /** + * Signal that subclass should be async initialized. Must be called prior + * to the first invocation of select(). + */ + void needAsynchronousInit() { _syncInit = false; } + +private: + mbus::Error currentPolicyInitError() const; + + class Task : public vespalib::Executor::Task + { + public: + Task(AsyncInitializationPolicy& owner) + : _owner(owner) + { + } + + Task(const Task&) = delete; + Task& operator=(const Task&) = delete; + + void run() override; + + private: + AsyncInitializationPolicy& _owner; + }; + + friend class Task; + + std::unique_ptr<vespalib::Executor> _executor; + vespalib::Monitor _lock; + + enum class State { + NOT_STARTED, + RUNNING, + FAILED, + DONE + }; + + State _state; + +protected: + string _error; + bool _syncInit; +}; + +} + + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp new file mode 100644 index 00000000000..33d14aae044 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/policies/contentpolicy.h> + +namespace documentapi { + +ContentPolicy::ContentPolicy(const string& param) + : StoragePolicy(param) +{ +} + +string +ContentPolicy::createConfigId(const string & clusterName) const +{ + return clusterName; +} + +} // documentapi diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.h new file mode 100644 index 00000000000..2009e8b147b --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.h @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/policies/storagepolicy.h> + +namespace documentapi { + +class ContentPolicy : public StoragePolicy +{ +public: + ContentPolicy(const string& param); +private: + virtual string createConfigId(const string & clusterName) const; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp new file mode 100644 index 00000000000..5819e5f6cd3 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp @@ -0,0 +1,167 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".documentrouteselectorpolicy"); + +#include "documentrouteselectorpolicy.h" +#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/document/select/parser.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h> +#include <vespa/documentapi/messagebus/messages/feedmessage.h> +#include <vespa/documentapi/messagebus/messages/multioperationmessage.h> +#include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/updatedocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/documentignoredreply.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/routing/routingcontext.h> + +using document::select::Result; + +namespace documentapi { + +DocumentRouteSelectorPolicy::DocumentRouteSelectorPolicy( + const document::DocumentTypeRepo &repo, const config::ConfigUri & configUri) : + mbus::IRoutingPolicy(), + config::IFetcherCallback<messagebus::protocol::DocumentrouteselectorpolicyConfig>(), + _repo(repo), + _lock(), + _config(), + _error("Not configured."), + _fetcher(configUri.getContext()) +{ + _fetcher.subscribe<messagebus::protocol::DocumentrouteselectorpolicyConfig>(configUri.getConfigId(), this); + _fetcher.start(); +} + +void +DocumentRouteSelectorPolicy::configure(std::unique_ptr<messagebus::protocol::DocumentrouteselectorpolicyConfig> cfg) +{ + string error = ""; + ConfigMap config; + for (uint32_t i = 0; i < cfg->route.size(); i++) { + const messagebus::protocol::DocumentrouteselectorpolicyConfig::Route &route = cfg->route[i]; + if (route.selector.empty()) { + continue; + } + SelectorPtr selector; + try { + document::BucketIdFactory factory; + document::select::Parser parser(_repo, factory); + selector.reset(parser.parse(route.selector).release()); + } + catch (document::select::ParsingFailedException &e) { + error = vespalib::make_string("Error parsing selector '%s' for route '%s'; %s", + route.selector.c_str(), route.name.c_str(), + e.getMessage().c_str()); + break; + } + config[string(route.name)] = selector; + } + vespalib::LockGuard guard(_lock); + _config.swap(config); + _error.swap(error); +} + +const string & +DocumentRouteSelectorPolicy::getError() const +{ + vespalib::LockGuard guard(_lock); + return _error; +} + +void +DocumentRouteSelectorPolicy::select(mbus::RoutingContext &context) +{ + // Require that recipients have been configured. + if (!context.hasRecipients()) { + context.setError(DocumentProtocol::ERROR_POLICY_FAILURE, + "No recipients configured."); + return; + } + + // Invoke private select method for each candidate recipient. + { + vespalib::LockGuard guard(_lock); + if (!_error.empty()) { + context.setError(DocumentProtocol::ERROR_POLICY_FAILURE, _error); + return; + } + for (uint32_t i = 0; i < context.getNumRecipients(); ++i) { + const mbus::Route &recipient = context.getRecipient(i); + vespalib::string routeName = recipient.toString(); + if (select(context, routeName)) { + const mbus::Route *route = context.getMessageBus().getRoutingTable(DocumentProtocol::NAME)->getRoute(routeName); + context.addChild(route != NULL ? *route : recipient); + } + } + } + context.setSelectOnRetry(false); + + // Notify that no children were selected, this is to differentiate this from the NO_RECIPIENTS_FOR_ROUTE error + // that message bus will generate if there are no recipients and no reply. + if (!context.hasChildren()) { + context.setReply(mbus::Reply::UP(new DocumentIgnoredReply())); + } +} + +bool +DocumentRouteSelectorPolicy::select(mbus::RoutingContext &context, const vespalib::string &routeName) +{ + if (_config.empty()) { + LOG(debug, "No config at all, select '%s'.", routeName.c_str()); + return true; + } + ConfigMap::const_iterator it = _config.find(routeName); + if (it == _config.end()) { + LOG(debug, "No config entry for route '%s', select it.", routeName.c_str()); + return true; + } + LOG_ASSERT(it->second.get() != NULL); + + // Select based on message content. + const mbus::Message &msg = context.getMessage(); + switch(msg.getType()) { + case DocumentProtocol::MESSAGE_PUTDOCUMENT: + return it->second->contains(*static_cast<const PutDocumentMessage&>(msg).getDocument()) == + Result::True; + + case DocumentProtocol::MESSAGE_UPDATEDOCUMENT: + return it->second->contains(*static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()) != + Result::False; + + case DocumentProtocol::MESSAGE_MULTIOPERATION: + { + const MultiOperationMessage& mom = static_cast<const MultiOperationMessage&>(msg); + for (vdslib::DocumentList::const_iterator iter = mom.getOperations().begin(); + iter != mom.getOperations().end(); + iter++) { + document::Document::UP doc = iter->getDocument(); + if (it->second->contains(*doc) == Result::False) { + return false; + } + } + return true; + } + case DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE: + { + const BatchDocumentUpdateMessage& mom = static_cast<const BatchDocumentUpdateMessage&>(msg); + for (uint32_t i = 0; i < mom.getUpdates().size(); i++) { + if (it->second->contains(*mom.getUpdates()[i]) == Result::False) { + return false; + } + } + return true; + } + default: + return true; + } +} + +void +DocumentRouteSelectorPolicy::merge(mbus::RoutingContext &context) +{ + DocumentProtocol::merge(context); +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h new file mode 100644 index 00000000000..4bddc6a1ecf --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/select/node.h> +#include <map> +#include <vespa/messagebus/routing/iroutingpolicy.h> +#include <vespa/vespalib/util/sync.h> +#include <vespa/documentapi/messagebus/policies/config-documentrouteselectorpolicy.h> +#include <vespa/documentapi/common.h> +#include <vespa/config/config.h> +#include <vespa/config/helper/configfetcher.h> + +namespace document { class DocumentTypeRepo; } + +namespace mbus { + class Route; + class RoutingContext; +} + +namespace documentapi { + +/** + * This policy is responsible for selecting among the given recipient routes according to the configured document + * selection properties. To factilitate this the "routing" plugin in the vespa model builds a mapping from the route + * names to a document selector and a feed name of every search cluster. This can very well be extended to include + * storage at a later time. + */ +class DocumentRouteSelectorPolicy : public mbus::IRoutingPolicy, + public config::IFetcherCallback<messagebus::protocol::DocumentrouteselectorpolicyConfig> +{ +private: + typedef std::shared_ptr<document::select::Node> SelectorPtr; + typedef std::map<string, SelectorPtr> ConfigMap; + + const document::DocumentTypeRepo &_repo; + vespalib::Lock _lock; + ConfigMap _config; + string _error; + config::ConfigFetcher _fetcher; + + /** + * This method runs the selector associated with the given location on the content of the message. If the selector + * validates the location, this method returns true. + * + * @param context The routing context that contains the necessary data. + * @param routeName The candidate route whose selector to run. + * @return Whether or not to send to the given recipient. + */ + bool select(mbus::RoutingContext &context, const vespalib::string &routeName); + +public: + /** + * This policy is constructed with a configuration uri that is used to subscribe for the document selector + * config. If the string is either null or empty it will default to the proper one. + * + * @param configUri The configuration uri to subscribe with. + */ + DocumentRouteSelectorPolicy(const document::DocumentTypeRepo &repo, + const config::ConfigUri &configUri); + + /** + * This is a safety mechanism to allow the constructor to fail and signal that it can not be used. + * + * @return The error string, or null if no error. + */ + const string &getError() const; + + // Implements Subscriber. + void configure(std::unique_ptr<messagebus::protocol::DocumentrouteselectorpolicyConfig> cfg); + + // Implements IRoutingPolicy. + void select(mbus::RoutingContext &context); + + // Implements IRoutingPolicy. + void merge(mbus::RoutingContext &context); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/errorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/errorpolicy.cpp new file mode 100644 index 00000000000..fccb2f2c3a6 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/errorpolicy.cpp @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".errorpolicy"); + +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include "errorpolicy.h" + +namespace documentapi { + +ErrorPolicy::ErrorPolicy(const string &msg) : + _msg(msg) +{ + // empty +} + +void +ErrorPolicy::select(mbus::RoutingContext &ctx) +{ + ctx.setError(DocumentProtocol::ERROR_POLICY_FAILURE, _msg); +} + +void +ErrorPolicy::merge(mbus::RoutingContext &) +{ + LOG_ASSERT(false); +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/errorpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/errorpolicy.h new file mode 100644 index 00000000000..2790adcc774 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/errorpolicy.h @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <boost/utility.hpp> +#include <vespa/messagebus/routing/iroutingpolicy.h> + +namespace documentapi { + +/** + * This policy assigns an error supplied at constructor time to the routing context when {@link + * #select(RoutingContext)} is invoked. This is useful for returning error states to the client instead of + * those auto-generated by mbus when a routing policy can not be created. + */ +class ErrorPolicy : public boost::noncopyable, public mbus::IRoutingPolicy { +private: + string _msg; + +public: + /** + * Creates a new policy that will assign an {@link EmptyReply} with the given error to all routing + * contexts that invoke {@link #select(RoutingContext)}. + * + * @param msg The message of the error to assign. + */ + ErrorPolicy(const string &msg); + + // Implements IRoutingPolicy. + void select(mbus::RoutingContext &context); + + // Implements IRoutingPolicy. + void merge(mbus::RoutingContext &context); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp new file mode 100644 index 00000000000..74d65d77eda --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp @@ -0,0 +1,146 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".externpolicy"); + +#include <boost/tokenizer.hpp> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/routing/route.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include "externpolicy.h" + +using slobrok::api::IMirrorAPI; +using slobrok::api::MirrorAPI; + +typedef boost::char_separator<char> Separator; +typedef boost::tokenizer<Separator> Tokenizer; + +namespace documentapi { + +ExternPolicy::ExternPolicy(const string ¶m) : + _lock(), + _orb(), + _mirror(), + _pattern(), + _session(), + _error("Not initialized."), + _offset(0), + _gen(0), + _recipients(), + _started(false) +{ + // Parse connection spec. + if (param.empty()) { + _error = "Expected parameter, got empty string."; + return; + } + size_t pos = param.find(';'); + if (pos == string::npos || pos == 0 || pos == param.size() - 1) { + _error = vespalib::make_string("Expected parameter on the form '<spec>;<pattern>', got '%s'.", param.c_str()); + return; + } + + // Activate supervisor and register mirror. + MirrorAPI::StringList spec; + string lst = param.substr(0, pos); + Tokenizer tokens(lst, Separator(",")); + for (Tokenizer::iterator it = tokens.begin(); it != tokens.end(); ++it) { + spec.push_back(*it); + } + + if (spec.size() == 0) { + _error = vespalib::make_string("Extern policy needs at least one slobrok: Slobrok list '%s' resolved to no slobroks", lst.c_str()); + return; + } + + slobrok::ConfiguratorFactory config(spec); + _mirror.reset(new MirrorAPI(_orb, config)); + _started = _orb.Start(); + if (!_started) { + _error = "Failed to start FNET supervisor."; + return; + } else { + LOG(debug, "Connecting to extern slobrok mirror '%s'..", lst.c_str()); + } + + // Parse query pattern. + _pattern = param.substr(pos + 1); + pos = _pattern.find_last_of('/'); + if (pos == string::npos) { + _error = vespalib::make_string("Expected pattern on the form '<service>/<session>', got '%s'.", _pattern.c_str()); + return; + } + _session = _pattern.substr(pos); + + // All ok. + _error.clear(); +} + +ExternPolicy::~ExternPolicy() +{ + _mirror.reset(); + if (_started) { + _orb.ShutDown(true); + } +} + +void +ExternPolicy::select(mbus::RoutingContext &ctx) +{ + if (!_error.empty()) { + ctx.setError(DocumentProtocol::ERROR_POLICY_FAILURE, _error); + } else if (_mirror->ready()) { + mbus::Hop hop = getRecipient(); + if (hop.hasDirectives()) { + mbus::Route route = ctx.getRoute(); + route.setHop(0, hop); + ctx.addChild(route); + } else { + ctx.setError(mbus::ErrorCode::NO_ADDRESS_FOR_SERVICE, + vespalib::make_string("Could not resolve any recipients from '%s'.", _pattern.c_str())); + } + } else { + ctx.setError(mbus::ErrorCode::APP_TRANSIENT_ERROR, "Extern slobrok not ready."); + } + +} + +void +ExternPolicy::merge(mbus::RoutingContext &ctx) +{ + DocumentProtocol::merge(ctx); +} + +mbus::Hop +ExternPolicy::getRecipient() +{ + vespalib::LockGuard guard(_lock); + update(); + if (_recipients.empty()) { + return mbus::Hop(); + } + return _recipients[++_offset % _recipients.size()]; +} + +void +ExternPolicy::update() +{ + uint32_t upd = _mirror->updates(); + if (_gen != upd) { + _gen = upd; + _recipients.clear(); + + IMirrorAPI::SpecList entries = _mirror->lookup(_pattern); + if (!entries.empty()) { + for (IMirrorAPI::SpecList::iterator it = entries.begin(); + it != entries.end(); ++it) + { + _recipients.push_back(mbus::Hop::parse(it->second + _session)); + } + } + } +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.h new file mode 100644 index 00000000000..83b9a64b79e --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.h @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <boost/utility.hpp> +#include <vespa/fnet/frt/frt.h> +#include <vespa/messagebus/routing/hop.h> +#include <vespa/messagebus/routing/iroutingpolicy.h> +#include <vespa/slobrok/sbmirror.h> +#include <string> +#include <vector> +#include <vespa/vespalib/util/sync.h> + +namespace documentapi { + +/** + * This policy implements the necessary logic to communicate with an external Vespa application and resolve its list of + * recipients using that other application's slobrok servers. + */ +class ExternPolicy : public boost::noncopyable, public mbus::IRoutingPolicy { +private: + vespalib::Lock _lock; + FRT_Supervisor _orb; + std::unique_ptr<slobrok::api::MirrorAPI> _mirror; + string _pattern; + string _session; + string _error; + uint32_t _offset; + uint32_t _gen; + std::vector<mbus::Hop> _recipients; + bool _started; + +private: + /** + * Returns the appropriate recipient hop. This method provides synchronized access to the internal mirror. + * + * @return The recipient hop to use. + */ + mbus::Hop getRecipient(); + + /** + * Updates the list of matching recipients by querying the extern slobrok. + */ + void update(); + +public: + /** + * Constructs a policy that will choose local services that match the slobrok pattern in which this policy occured. + * If no local service can be found, this policy simply returns the asterisk to allow the network to choose any. + * + * @param param The address to use for this, if empty this will resolve to hostname. + */ + ExternPolicy(const string ¶m); + + /** + * Destructor. + * + * Frees all allocated resources. + */ + virtual ~ExternPolicy(); + + /** + * This is a safety mechanism to allow the constructor to fail and signal that it can not be used. + * + * @return True if this policy can be used. + */ + const string &getError() const { return _error; } + + /** + * Returns the slobrok mirror api used by this policy to resolve external patterns. This is basically here just to + * enable unit tests. If you rely on this for production code, then you need to reconsider your logic. Furthermore, + * it deref's the content of an auto-pointer that is NULL in case broken() returns true, meaning it _will_ core. + * + * @return The mirror pointer. + */ + slobrok::api::MirrorAPI &getMirror() { return *_mirror; } + + // Overrides IRoutingPolicy. + void select(mbus::RoutingContext &ctx); + + // Overrides IRoutingPolicy. + void merge(mbus::RoutingContext &ctx); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp new file mode 100644 index 00000000000..57103b265a3 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/policies/externslobrokpolicy.h> +#include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/slobrok/cfg.h> + +using slobrok::api::IMirrorAPI; +using slobrok::api::MirrorAPI; + +namespace documentapi { + +ExternSlobrokPolicy::ExternSlobrokPolicy(const std::map<string, string>& param) + : AsyncInitializationPolicy(param), + _firstTry(true), + _slobrokConfigId("admin/slobrok.0") +{ + if (param.find("config") != param.end()) { + vespalib::StringTokenizer configServers(param.find("config")->second, ","); + for (uint32_t j = 0; j < configServers.size(); j++) { + _configSources.push_back(configServers[j]); + } + } + + if (param.find("slobroks") != param.end()) { + vespalib::StringTokenizer slobrokList(param.find("slobroks")->second, ","); + for (uint32_t j = 0; j < slobrokList.size(); j++) { + _slobroks.push_back(slobrokList[j]); + } + } + + if (param.find("slobrokconfigid") != param.end()) { + _slobrokConfigId = param.find("slobrokconfigid")->second; + } + + if (_slobroks.size() || _configSources.size()) { + needAsynchronousInit(); + } +} + +ExternSlobrokPolicy::~ExternSlobrokPolicy() +{ + bool started = _mirror.get() != NULL; + _mirror.reset(); + if (started) { + _orb.ShutDown(true); + } +} + +string ExternSlobrokPolicy::init() { + if (_slobroks.size() != 0) { + slobrok::ConfiguratorFactory config(_slobroks); + _mirror.reset(new MirrorAPI(_orb, config)); + } else if (_configSources.size() != 0) { + slobrok::ConfiguratorFactory config( + config::ConfigUri(_slobrokConfigId, + config::IConfigContext::SP( + new config::ConfigContext(config::ServerSpec(_configSources))))); + _mirror.reset(new MirrorAPI(_orb, config)); + } + + if (_mirror.get()) { + _orb.Start(); + } + + return ""; +} + +IMirrorAPI::SpecList +ExternSlobrokPolicy::lookup(mbus::RoutingContext& context, const string& pattern) { + vespalib::LockGuard guard(_lock); + + const IMirrorAPI& mirror(_mirror.get()? *_mirror : context.getMirror()); + + IMirrorAPI::SpecList entries = mirror.lookup(pattern); + + if (_firstTry) { + int count = 0; + while (entries.empty() && count < 100) { + FastOS_Thread::Sleep(50); + entries = mirror.lookup(pattern); + count++; + } + } + + _firstTry = false; + + return entries; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.h new file mode 100644 index 00000000000..a10b01ba42c --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.h @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/policies/asyncinitializationpolicy.h> +#include <vespa/config-slobroks.h> +#include <vespa/vdslib/distribution/distribution.h> +#include <vespa/slobrok/sbmirror.h> +#include <vespa/documentapi/common.h> + +namespace documentapi { + +/** + Super class for routing policies that allow the user to specify external slobrok lists, + either by supplying external config servers or the slobrok list directly. +*/ +class ExternSlobrokPolicy : public AsyncInitializationPolicy +{ +protected: + bool _firstTry; + config::ServerSpec::HostSpecList _configSources; + vespalib::Lock _lock; + FRT_Supervisor _orb; + std::unique_ptr<slobrok::api::MirrorAPI> _mirror; + slobrok::api::MirrorAPI::StringList _slobroks; + string _slobrokConfigId; + +public: + ExternSlobrokPolicy(const std::map<string, string>& params); + virtual ~ExternSlobrokPolicy(); + + /** + * @return a pointer to the slobrok mirror owned by this policy, if any. + * If the policy uses the default mirror API, NULL is returned. + */ + const slobrok::api::MirrorAPI* getMirror() const { return _mirror.get(); } + + slobrok::api::MirrorAPI::SpecList lookup(mbus::RoutingContext &context, const string& pattern); + + /** + * Initializes the policy + */ + virtual string init(); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancer.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancer.cpp new file mode 100644 index 00000000000..1be17d29ac7 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancer.cpp @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/policies/loadbalancer.h> + +namespace documentapi { + +LoadBalancer::LoadBalancer(const string& cluster, const string& session) + : _cluster(cluster), + _session(session), + _position(0) +{ +}; + +uint32_t +LoadBalancer::getIndex(const string& name) const +{ + size_t lastSlash = name.find('/'); + string idx = name.substr(_cluster.length() + 1, lastSlash); + return atoi(idx.c_str()); +} + +std::pair<string, int> +LoadBalancer::getRecipient(const slobrok::api::IMirrorAPI::SpecList& choices) +{ + std::pair<string, int> retVal("", -1); + + if (choices.size() == 0) { + return retVal; + } + + double weightSum = 0.0; + + for (uint32_t i = 0; i < choices.size(); i++) { + const std::pair<string, string>& curr = choices[i]; + + uint32_t index = getIndex(curr.first); + + if (_nodeInfo.size() < (index + 1)) { + _nodeInfo.resize(index + 1); + } + + NodeInfo& info = _nodeInfo[index]; + info.valid = true; + weightSum += info.weight; + + if (weightSum > _position) { + retVal.first = curr.second; + retVal.second = index; + info.lastSpec = retVal.first; + break; + } + } + + if (retVal.second == -1) { + _position -= weightSum; + return getRecipient(choices); + } else { + _position += 1.0; + } + + return retVal; +} + +void +LoadBalancer::normalizeWeights() { + double lowest = -1.0; + + for (uint32_t i = 0; i < _nodeInfo.size(); i++) { + if (!_nodeInfo[i].valid) { + continue; + } + + if (lowest < 0 || _nodeInfo[i].weight < lowest) { + lowest = _nodeInfo[i].weight; + } + } + + for (uint32_t i = 0; i < _nodeInfo.size(); i++) { + if (!_nodeInfo[i].valid) { + continue; + } + + _nodeInfo[i].weight = _nodeInfo[i].weight / lowest; + } +} + +void +LoadBalancer::received(uint32_t nodeIndex, bool busy) { + if (busy) { + NodeInfo& info = _nodeInfo[nodeIndex]; + + info.weight = info.weight - 0.01; + normalizeWeights(); + } +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancer.h b/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancer.h new file mode 100644 index 00000000000..a689b92730c --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancer.h @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/slobrok/sbmirror.h> +#include <vespa/documentapi/common.h> + +namespace documentapi { + +class LoadBalancer { +public: + class NodeInfo { + public: + NodeInfo() : valid(false), sent(0), busy(0), weight(1.0) {}; + + bool valid; + uint32_t sent; + uint32_t busy; + double weight; + string lastSpec; + }; + + std::vector<NodeInfo> _nodeInfo; + string _cluster; + string _session; + double _position; + + LoadBalancer(const string& cluster, const string& session); + + const std::vector<NodeInfo>& getNodeInfo() const { return _nodeInfo; } + + uint32_t getIndex(const string& name) const; + + /** + Returns the spec and the node index of the node we should send to. + If none are found, node index is -1. + */ + std::pair<string, int> getRecipient(const slobrok::api::IMirrorAPI::SpecList& choices); + + void normalizeWeights(); + + void received(uint32_t nodeIndex, bool busy); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancerpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancerpolicy.cpp new file mode 100644 index 00000000000..f30a6522337 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancerpolicy.cpp @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/policies/loadbalancerpolicy.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/routing/ihopdirective.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/messagebus/routing/verbatimdirective.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/log/log.h> + +LOG_SETUP(".loadbalancerpolicy"); + +namespace documentapi { + +LoadBalancerPolicy::LoadBalancerPolicy(const string& param) + : ExternSlobrokPolicy(parse(param)) +{ + std::map<string, string> params(parse(param)); + + if (params.find("cluster") != params.end()) { + _cluster = params.find("cluster")->second; + } else { + _error = "Required parameter cluster not set"; + return; + } + + if (params.find("session") != params.end()) { + _session = params.find("session")->second; + } else { + _error = "Required parameter session not set"; + return; + } + + _pattern = _cluster + "/*/" + _session; + _loadBalancer.reset(new LoadBalancer(_cluster, _session)); +} + +void +LoadBalancerPolicy::doSelect(mbus::RoutingContext& context) { + std::pair<string, int> node = getRecipient(context); + + if (node.second != -1) { + context.setContext((uint64_t)node.second); + mbus::Route route = context.getRoute(); + route.setHop(0, mbus::Hop::parse(node.first + "/" + _session)); + context.addChild(route); + } else { + context.setError(mbus::ErrorCode::NO_ADDRESS_FOR_SERVICE, + "Could not resolve any nodes to send to in pattern " + _pattern); + } +} + +void +LoadBalancerPolicy::merge(mbus::RoutingContext& context) { + mbus::RoutingNodeIterator it = context.getChildIterator(); + mbus::Reply::UP reply = it.removeReply(); + + uint64_t target = context.getContext().value.UINT64; + + bool busy = false; + for (uint32_t i = 0; i < reply->getNumErrors(); i++) { + if (reply->getError(i).getCode() == mbus::ErrorCode::SESSION_BUSY) { + string lastSpec = _loadBalancer->getNodeInfo()[target].lastSpec; + + if (reply->getError(i).getMessage().find(lastSpec) == string::npos) { + LOG(debug, "Received busy with message %s, doesn't contain target %s so not updating weight.", reply->getError(i).getMessage().c_str(), lastSpec.c_str()); + } else { + LOG(debug, "Received busy for target node %d reducing weight of that node.", (int)target); + busy = true; + } + } + } + + _loadBalancer->received(target, busy); + + context.setReply(std::move(reply)); +} + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancerpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancerpolicy.h new file mode 100644 index 00000000000..8d0eca83ac9 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/loadbalancerpolicy.h @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <boost/noncopyable.hpp> +#include <vespa/documentapi/messagebus/policies/loadbalancer.h> +#include <vespa/documentapi/messagebus/policies/externslobrokpolicy.h> + +namespace documentapi { + +class LoadBalancerPolicy : public boost::noncopyable, + public ExternSlobrokPolicy +{ +public: + LoadBalancerPolicy(const string& param); + + virtual void doSelect(mbus::RoutingContext &context); + + /** + Finds the TCP address of the target docproc. + + @return Returns a hop representing the TCP address of the target docproc, or null if none could be found. + */ + std::pair<string, int> getRecipient(mbus::RoutingContext& context) { + return _loadBalancer->getRecipient(lookup(context, _pattern)); + } + + virtual void merge(mbus::RoutingContext &context); + +private: + string _pattern; + string _cluster; + string _session; + std::unique_ptr<LoadBalancer> _loadBalancer; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.cpp new file mode 100644 index 00000000000..a4cd9f2d25b --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.cpp @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".localservicepolicy"); + +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/routing/route.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/messagebus/routing/verbatimdirective.h> +#include <vespa/vespalib/util/hashmap.h> +#include <vespa/vespalib/util/stringfmt.h> +#include "localservicepolicy.h" + +namespace documentapi { + +LocalServicePolicy::CacheEntry::CacheEntry() : + _offset(0), + _generation(0), + _recipients() +{ + // empty +} + +LocalServicePolicy::LocalServicePolicy(const string ¶m) : + _lock(), + _address(param), + _cache() +{ + // empty +} + +LocalServicePolicy::~LocalServicePolicy() +{ + // empty +} + +void +LocalServicePolicy::select(mbus::RoutingContext &ctx) +{ + mbus::Route route = ctx.getRoute(); + route.setHop(0, getRecipient(ctx)); + ctx.addChild(route); +} + +void +LocalServicePolicy::merge(mbus::RoutingContext &context) +{ + DocumentProtocol::merge(context); +} + +string +LocalServicePolicy::getCacheKey(const mbus::RoutingContext &ctx) const +{ + return ctx.getRoute().getHop(0).toString(); +} + +mbus::Hop +LocalServicePolicy::getRecipient(mbus::RoutingContext &ctx) +{ + vespalib::LockGuard guard(_lock); + CacheEntry &entry = update(ctx); + if (entry._recipients.empty()) { + mbus::Hop hop = ctx.getRoute().getHop(0); + hop.setDirective(ctx.getDirectiveIndex(), + mbus::IHopDirective::SP(new mbus::VerbatimDirective("*"))); + return hop; + } + if (++entry._offset >= entry._recipients.size()) { + entry._offset = 0; + } + return entry._recipients[entry._offset]; +} + +LocalServicePolicy::CacheEntry & +LocalServicePolicy::update(mbus::RoutingContext &ctx) +{ + uint32_t upd = ctx.getMirror().updates(); + CacheEntry &entry = _cache.insert(std::map<string, CacheEntry>::value_type(getCacheKey(ctx), CacheEntry())).first->second; + if (entry._generation != upd) { + entry._generation = upd; + entry._recipients.clear(); + + string pattern = vespalib::make_string("%s*%s", + ctx.getHopPrefix().c_str(), + ctx.getHopSuffix().c_str()); + slobrok::api::IMirrorAPI::SpecList entries = ctx.getMirror().lookup(pattern); + + string self = _address.empty() ? toAddress(ctx.getMessageBus().getConnectionSpec()) : _address; + for (slobrok::api::IMirrorAPI::SpecList::iterator it = entries.begin(); + it != entries.end(); ++it) + { + LOG(debug, "Matching self '%s' to '%s'.", self.c_str(), it->second.c_str()); + if (self == toAddress(it->second)) { + LOG(debug, "Match, add it"); + entry._recipients.push_back(mbus::Hop::parse(it->first)); + } + } + } + return entry; +} + +string +LocalServicePolicy::toAddress(const string &connection) +{ + if (connection.substr(0, 4) == "tcp/") { + uint32_t pos = connection.find_first_of(':', 4); + if (pos > 4) { + return connection.substr(4, pos - 4); + } + } + return ""; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.h new file mode 100644 index 00000000000..85c26794b97 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.h @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <boost/utility.hpp> +#include <vespa/messagebus/routing/hop.h> +#include <vespa/messagebus/routing/iroutingpolicy.h> +#include <string> +#include <vector> +#include <vespa/vespalib/util/sync.h> + +namespace documentapi { + +/** + * This policy implements the logic to prefer local services that matches a slobrok pattern. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @version $Id$ + */ +class LocalServicePolicy : public boost::noncopyable, public mbus::IRoutingPolicy { +private: + struct CacheEntry { + uint32_t _offset; + uint32_t _generation; + std::vector<mbus::Hop> _recipients; + + CacheEntry(); + }; + + vespalib::Lock _lock; + string _address; + std::map<string, CacheEntry> _cache; + + /** + * Returns the appropriate recipient hop for the given routing context. This method provides synchronized access to + * the internal cache. + * + * @param ctx The routing context. + * @return The recipient hop to use. + */ + mbus::Hop getRecipient(mbus::RoutingContext &ctx); + + /** + * Updates and returns the cache entry for the given routing context. This method assumes that synchronization is + * handled outside of it. + * + * @param ctx The routing context. + * @return The updated cache entry. + */ + CacheEntry &update(mbus::RoutingContext &ctx); + + /** + * Returns a cache key for this instance of the policy. Because behaviour is based on the hop in which the policy + * occurs, the cache key is the hop string itself. + * + * @param ctx The routing context. + * @return The cache key. + */ + string getCacheKey(const mbus::RoutingContext &ctx) const; + + /** + * Searches the given connection spec for a hostname or IP address. If an address is not found, this method returns + * null. + * + * @param connection The connection spec to search. + * @return The address, may be null. + */ + static string toAddress(const string &connection); + +public: + /** + * Constructs a policy that will choose local services that match the slobrok pattern in which this policy occured. + * If no local service can be found, this policy simply returns the asterisk to allow the network to choose any. + * + * @param param The address to use for this, if empty this will resolve to hostname. + */ + LocalServicePolicy(const string ¶m); + + /** + * Destructor. + * + * Frees all allocated resources. + */ + virtual ~LocalServicePolicy(); + + // Inherit doc from IRoutingPolicy. + virtual void select(mbus::RoutingContext &context); + + // Inherit doc from IRoutingPolicy. + virtual void merge(mbus::RoutingContext &context); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp new file mode 100644 index 00000000000..e88ac191c26 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.cpp @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include "messagetypepolicy.h" +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/messagebus/routing/routingcontext.h> + +using vespa::config::content::MessagetyperouteselectorpolicyConfig; + +namespace documentapi { + +MessageTypePolicy::MessageTypePolicy(const config::ConfigUri & configUri) : + mbus::IRoutingPolicy(), + config::IFetcherCallback<MessagetyperouteselectorpolicyConfig>(), + _map(), + _defaultRoute(), + _fetcher(configUri.getContext()) +{ + _fetcher.subscribe<MessagetyperouteselectorpolicyConfig>(configUri.getConfigId(), this); + _fetcher.start(); +} + +void +MessageTypePolicy::configure(std::unique_ptr<MessagetyperouteselectorpolicyConfig> cfg) +{ + std::unique_ptr<MessageTypeMap> map(new MessageTypeMap); + for (size_t i(0), m(cfg->route.size()); i < m; i++) { + const MessagetyperouteselectorpolicyConfig::Route & r = cfg->route[i]; + (*map)[r.messagetype] = mbus::Route::parse(r.name); + } + _map.set(map.release()); + _defaultRoute.set(new mbus::Route(mbus::Route::parse(cfg->defaultroute))); + _map.latch(); + _defaultRoute.latch(); +} + +void +MessageTypePolicy::select(mbus::RoutingContext & context) +{ + int messageType = context.getMessage().getType(); + std::shared_ptr<MessageTypeMap> map = _map.get(); + MessageTypeMap::const_iterator found = map->find(messageType); + if (found != map->end()) { + context.addChild(found->second); + } else { + context.addChild(*_defaultRoute.get()); + } +} + +void +MessageTypePolicy::merge(mbus::RoutingContext &context) +{ + DocumentProtocol::merge(context); +} +} // namespace documentapi diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h new file mode 100644 index 00000000000..ade2c78024a --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/messagetypepolicy.h @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/select/node.h> +#include <map> +#include <vespa/messagebus/routing/iroutingpolicy.h> +#include <vespa/messagebus/routing/route.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/vespalib/util/sync.h> +#include <vespa/config-messagetyperouteselectorpolicy.h> +#include <vespa/config/config.h> +#include <vespa/config/helper/configfetcher.h> +#include <vespa/documentapi/common.h> + +namespace documentapi { + +/** + * This policy is responsible for selecting among the given recipient routes + * according to the configured document selection properties. To factilitate + * this the "routing" plugin in the vespa model builds a mapping from the route + * names to a document selector and a feed name of every search cluster. This + * can very well be extended to include storage at a later time. + */ +class MessageTypePolicy : public mbus::IRoutingPolicy, + public config::IFetcherCallback<vespa::config::content::MessagetyperouteselectorpolicyConfig> +{ +private: + typedef vespalib::hash_map<int, mbus::Route> MessageTypeMap; + typedef vespalib::PtrHolder<MessageTypeMap> MessageTypeHolder; + typedef vespalib::PtrHolder<mbus::Route> RouteHolder; + + MessageTypeHolder _map; + RouteHolder _defaultRoute; + config::ConfigFetcher _fetcher; + +public: + /** + * This policy is constructed with a configuration uri that can be used to + * subscribe for the document selector config. If the uri is empty, it will + * default to a proper one. + * + * @param configUri The configuration uri to subscribe with. + */ + MessageTypePolicy(const config::ConfigUri & configUri); + + // Implements Subscriber. + void configure(std::unique_ptr<vespa::config::content::MessagetyperouteselectorpolicyConfig> cfg); + + // Implements IRoutingPolicy. + void select(mbus::RoutingContext &context); + + // Implements IRoutingPolicy. + void merge(mbus::RoutingContext &context); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.cpp new file mode 100644 index 00000000000..3c566d0fca5 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.cpp @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".roundrobinpolicy"); + +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/routing/route.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/messagebus/routing/verbatimdirective.h> +#include "roundrobinpolicy.h" + +namespace documentapi { + +RoundRobinPolicy::CacheEntry::CacheEntry() : + _offset(0), + _generation(), + _recipients() +{ + // empty +} + +RoundRobinPolicy::RoundRobinPolicy(const string &) : + _lock(), + _cache() +{ + // empty +} + +RoundRobinPolicy::~RoundRobinPolicy() +{ + // empty +} + +void +RoundRobinPolicy::select(mbus::RoutingContext &ctx) +{ + mbus::Hop hop = getRecipient(ctx); + if (hop.hasDirectives()) { + mbus::Route route = ctx.getRoute(); + route.setHop(0, hop); + ctx.addChild(route); + } else { + mbus::EmptyReply::UP reply(new mbus::EmptyReply()); + reply->addError(mbus::Error(mbus::ErrorCode::NO_ADDRESS_FOR_SERVICE, + "None of the configured recipients are currently available.")); + ctx.setReply(std::move(reply)); + } +} + +void +RoundRobinPolicy::merge(mbus::RoutingContext &context) +{ + DocumentProtocol::merge(context); +} + +string +RoundRobinPolicy::getCacheKey(const mbus::RoutingContext &ctx) const +{ + string ret; + for (uint32_t i = 0; i < ctx.getNumRecipients(); ++i) { + ret.append(ctx.getRecipient(i).getHop(0).toString()); + ret.append(" "); + } + return ret; +} + +mbus::Hop +RoundRobinPolicy::getRecipient(mbus::RoutingContext &ctx) +{ + vespalib::LockGuard guard(_lock); + CacheEntry &entry = update(ctx); + if (entry._recipients.empty()) { + return mbus::Hop(); + } + if (++entry._offset >= entry._recipients.size()) { + entry._offset = 0; + } + return entry._recipients[entry._offset]; +} + +RoundRobinPolicy::CacheEntry & +RoundRobinPolicy::update(mbus::RoutingContext &ctx) +{ + uint32_t upd = ctx.getMirror().updates(); + CacheEntry &entry = _cache.insert(std::map<string, CacheEntry>::value_type(getCacheKey(ctx), CacheEntry())).first->second; + if (entry._generation != upd) { + entry._generation = upd; + entry._recipients.clear(); + for (uint32_t i = 0; i < ctx.getNumRecipients(); ++i) + { + slobrok::api::IMirrorAPI::SpecList entries = ctx.getMirror().lookup(ctx.getRecipient(i).getHop(0).toString()); + for (slobrok::api::IMirrorAPI::SpecList::iterator it = entries.begin(); + it != entries.end(); ++it) + { + entry._recipients.push_back(mbus::Hop::parse(it->first)); + } + } + } + return entry; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.h new file mode 100644 index 00000000000..e65763a114d --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/roundrobinpolicy.h @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <boost/utility.hpp> +#include <vespa/messagebus/routing/hop.h> +#include <vespa/messagebus/routing/iroutingpolicy.h> +#include <string> +#include <vector> +#include <vespa/vespalib/util/sync.h> + +namespace documentapi { + +/** + * This policy implements the logic to prefer round robins that matches a slobrok pattern. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @version $Id$ + */ +class RoundRobinPolicy : public boost::noncopyable, public mbus::IRoutingPolicy { +private: + struct CacheEntry { + uint32_t _offset; + uint32_t _generation; + std::vector<mbus::Hop> _recipients; + + CacheEntry(); + }; + + vespalib::Lock _lock; + std::map<string, CacheEntry> _cache; + + /** + * Returns the appropriate recipient hop for the given routing context. This method provides synchronized access to + * the internal cache. + * + * @param ctx The routing context. + * @return The recipient hop to use. + */ + mbus::Hop getRecipient(mbus::RoutingContext &ctx); + + /** + * Updates and returns the cache entry for the given routing context. This method assumes that synchronization is + * handled outside of it. + * + * @param ctx The routing context. + * @return The updated cache entry. + */ + CacheEntry &update(mbus::RoutingContext &ctx); + + /** + * Returns a cache key for this instance of the policy. Because behaviour is based on the recipient list of this + * policy, the cache key is the concatenated string of recipient routes. + * + * @param ctx The routing context. + * @return The cache key. + */ + string getCacheKey(const mbus::RoutingContext &ctx) const; + +public: + /** + * Constructs a policy that will round robin among the configured recipients that are currently registered + * in slobrok. + */ + RoundRobinPolicy(const string ¶m); + + /** + * Destructor. + * + * Frees all allocated resources. + */ + virtual ~RoundRobinPolicy(); + + // Inherit doc from IRoutingPolicy. + virtual void select(mbus::RoutingContext &context); + + // Inherit doc from IRoutingPolicy. + virtual void merge(mbus::RoutingContext &context); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.cpp new file mode 100644 index 00000000000..84cbe33f701 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.cpp @@ -0,0 +1,142 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".searchcolumnpolicy"); + +#include <vespa/document/base/documentid.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/getdocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <vespa/documentapi/messagebus/policies/searchcolumnpolicy.h> +#include <vespa/documentapi/messagebus/messages/updatedocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h> +#include <vespa/documentapi/messagebus/messages/multioperationmessage.h> +#include <vespa/messagebus/error.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vdslib/bucketdistribution.h> +#include <vespa/vespalib/util/hashmap.h> + +namespace documentapi { + +SearchColumnPolicy::SearchColumnPolicy(const string ¶m) : + _lock(), + _factory(), + _distributions(), + _maxOOS(0) +{ + if (param.length() > 0) { + int maxOOS = atoi(param.c_str()); + if (maxOOS >= 0) { + _maxOOS = (uint32_t)maxOOS; + } else { + LOG(warning, + "Ignoring a request to set the maximum number of OOS replies to %d because it makes no " + "sense. This routing policy will not allow any recipient to be out of service.", maxOOS); + } + } +} + +SearchColumnPolicy::~SearchColumnPolicy() +{ + // empty +} + +void +SearchColumnPolicy::select(mbus::RoutingContext &context) +{ + std::vector<mbus::Route> recipients; + context.getMatchedRecipients(recipients); + if (recipients.empty()) { + return; + } + const document::DocumentId *id = NULL; + document::BucketId bucketId; + + const mbus::Message &msg = context.getMessage(); + switch(msg.getType()) { + case DocumentProtocol::MESSAGE_PUTDOCUMENT: + id = &static_cast<const PutDocumentMessage&>(msg).getDocument()->getId(); + break; + + case DocumentProtocol::MESSAGE_GETDOCUMENT: + id = &static_cast<const GetDocumentMessage&>(msg).getDocumentId(); + break; + + case DocumentProtocol::MESSAGE_REMOVEDOCUMENT: + id = &static_cast<const RemoveDocumentMessage&>(msg).getDocumentId(); + break; + + case DocumentProtocol::MESSAGE_UPDATEDOCUMENT: + id = &static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()->getId(); + break; + + case DocumentProtocol::MESSAGE_MULTIOPERATION: + bucketId = (static_cast<const MultiOperationMessage&>(msg)).getBucketId(); + break; + + case DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE: + bucketId = (static_cast<const BatchDocumentUpdateMessage&>(msg)).getBucketId(); + break; + + default: + LOG(error, "Message type '%d' not supported.", msg.getType()); + return; + } + if (bucketId.getRawId() == 0) { + bucketId = _factory.getBucketId(*id); + } + uint32_t recipient = getRecipient(bucketId, recipients.size()); + context.addChild(recipients[recipient]); + context.setSelectOnRetry(true); + if (_maxOOS > 0) { + context.addConsumableError(mbus::ErrorCode::SERVICE_OOS); + } +} + +void +SearchColumnPolicy::merge(mbus::RoutingContext &context) +{ + if (_maxOOS > 0) { + if (context.getNumChildren() > 1) { + std::set<uint32_t> oosReplies; + uint32_t idx = 0; + for (mbus::RoutingNodeIterator it = context.getChildIterator(); + it.isValid(); it.next()) + { + const mbus::Reply &ref = it.getReplyRef(); + if (ref.hasErrors() && DocumentProtocol::hasOnlyErrorsOfType(ref, mbus::ErrorCode::SERVICE_OOS)) { + oosReplies.insert(idx); + } + ++idx; + } + if (oosReplies.size() <= _maxOOS) { + DocumentProtocol::merge(context, oosReplies); + return; // may the rtx be with you + } + } else { + const mbus::Reply &ref = context.getChildIterator().getReplyRef(); + if (ref.hasErrors() && DocumentProtocol::hasOnlyErrorsOfType(ref, mbus::ErrorCode::SERVICE_OOS)) { + context.setReply(mbus::Reply::UP(new mbus::EmptyReply())); + return; // god help us all + } + } + } + DocumentProtocol::merge(context); +} + +uint32_t +SearchColumnPolicy::getRecipient(const document::BucketId &bucketId, uint32_t numRecipients) +{ + vespalib::LockGuard guard(_lock); + DistributionCache::iterator it = _distributions.find(numRecipients); + if (it == _distributions.end()) { + it = _distributions.insert(DistributionCache::value_type(numRecipients, vdslib::BucketDistribution(1, 16u))).first; + it->second.setNumColumns(numRecipients); + } + return it->second.getColumn(bucketId); +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.h new file mode 100644 index 00000000000..a7420acf8ef --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.h @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <boost/utility.hpp> +#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/messagebus/routing/iroutingpolicy.h> +#include <vespa/vdslib/bucketdistribution.h> +#include <vespa/vespalib/util/sync.h> + +namespace documentapi { + +/** + * This policy implements the logic to select recipients for a single search column. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @version $Id$ + */ +class SearchColumnPolicy : public boost::noncopyable, public mbus::IRoutingPolicy { +private: + typedef std::map<uint32_t, vdslib::BucketDistribution> DistributionCache; + + vespalib::Lock _lock; + document::BucketIdFactory _factory; + DistributionCache _distributions; + uint32_t _maxOOS; + + /** + * Returns the recipient index for the given bucket id. This updates the shared internal distribution map, so it + * needs to be synchronized. + * + * @param bucketId The bucket whose recipient to return. + * @param numRecipients The number of recipients being distributed to. + * @return The recipient to use. + */ + uint32_t getRecipient(const document::BucketId &bucketId, uint32_t numRecipients); + +public: + /** + * Constructs a new policy object for the given parameter string. The string can be null or empty, which is a + * request to not allow any bad columns. + * + * @param param The maximum number of allowed bad columns. + */ + SearchColumnPolicy(const string ¶m); + + /** + * Destructor. + * + * Frees all allocated resources. + */ + virtual ~SearchColumnPolicy(); + + // Inherit doc from IRoutingPolicy. + virtual void select(mbus::RoutingContext &context); + + // Inherit doc from IRoutingPolicy. + virtual void merge(mbus::RoutingContext &context); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/searchrowpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/searchrowpolicy.cpp new file mode 100644 index 00000000000..a438668a2f4 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/searchrowpolicy.cpp @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/policies/searchrowpolicy.h> + +LOG_SETUP(".searchrowpolicy"); + +namespace documentapi { + +SearchRowPolicy::SearchRowPolicy(const string ¶m) : + _minOk(0) +{ + if (param.length() > 0) { + int minOk = atoi(param.c_str()); + if (minOk > 0) { + _minOk = (uint32_t)minOk; + } else { + LOG(warning, + "Ignoring a request to set the minimum number of OK replies to %d because it makes no sense. " + "This routing policy will not allow any recipient to be out of service.", minOk); + } + } +} + +SearchRowPolicy::~SearchRowPolicy() +{ + // empty +} + +void +SearchRowPolicy::select(mbus::RoutingContext &context) +{ + std::vector<mbus::Route> recipients; + context.getMatchedRecipients(recipients); + context.addChildren(recipients); + context.setSelectOnRetry(false); + if (_minOk > 0) { + context.addConsumableError(mbus::ErrorCode::SERVICE_OOS); + } +} + +void +SearchRowPolicy::merge(mbus::RoutingContext &context) +{ + if (_minOk > 0) { + std::set<uint32_t> oosReplies; + uint32_t idx = 0; + for (mbus::RoutingNodeIterator it = context.getChildIterator(); + it.isValid(); it.next()) + { + const mbus::Reply &ref = it.getReplyRef(); + if (ref.hasErrors() && DocumentProtocol::hasOnlyErrorsOfType(ref, mbus::ErrorCode::SERVICE_OOS)) { + oosReplies.insert(idx); + } + ++idx; + } + if (context.getNumChildren() - oosReplies.size() >= _minOk) { + DocumentProtocol::merge(context, oosReplies); + return; + } + } + DocumentProtocol::merge(context); +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/searchrowpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/searchrowpolicy.h new file mode 100644 index 00000000000..520ca74f3db --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/searchrowpolicy.h @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/messagebus/routing/iroutingpolicy.h> + +namespace documentapi { + +class SearchRowPolicy : public mbus::IRoutingPolicy { +private: + SearchRowPolicy(const SearchRowPolicy &); + SearchRowPolicy &operator=(const SearchRowPolicy &); + +public: + /** + * Creates a search row policy that wraps the underlying search group policy in case the parameter is something + * other than an empty string. + * + * @param param The number of minimum non-OOS replies that this policy requires. + */ + SearchRowPolicy(const string ¶m); + + /** + * Destructor. + * + * Frees all allocated resources. + */ + virtual ~SearchRowPolicy(); + + // Inherit doc from IRoutingPolicy. + virtual void select(mbus::RoutingContext &context); + + // Inherit doc from IRoutingPolicy. + virtual void merge(mbus::RoutingContext &context); + +private: + uint32_t _minOk; // Hide OUT_OF_SERVICE as long as this number of replies are something else. +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp new file mode 100644 index 00000000000..284ec714e82 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp @@ -0,0 +1,261 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/policies/storagepolicy.h> +#include <vespa/log/log.h> +#include <algorithm> +#include <vespa/document/base/documentid.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/routing/ihopdirective.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/messagebus/routing/verbatimdirective.h> +#include <vespa/vespalib/util/random.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> + +LOG_SETUP(".storagepolicy"); + +namespace documentapi { + +StoragePolicy::StoragePolicy(const string& param) + : ExternSlobrokPolicy(parse(param)), + _bucketIdFactory() +{ + std::map<string, string> params(parse(param)); + + if (params.find("cluster") != params.end()) { + _clusterName = params.find("cluster")->second; + } else { + _error = "Required parameter clustername not set"; + } + + if (params.find("clusterconfigid") != params.end()) { + _clusterConfigId = params.find("clusterconfigid")->second; + } +} + +string StoragePolicy::init() +{ + string error = ExternSlobrokPolicy::init(); + if (error.length() > 0) { + return error; + } + + if (!_clusterConfigId.length()) { + _clusterConfigId = createConfigId(_clusterName); + } + + using storage::lib::Distribution; + config::ConfigUri uri(_clusterConfigId); + if (!_configSources.empty()) { + _configFetcher.reset(new config::ConfigFetcher(config::ServerSpec(_configSources))); + } else { + _configFetcher.reset(new config::ConfigFetcher(uri.getContext())); + } + _configFetcher->subscribe<vespa::config::content::StorDistributionConfig>(uri.getConfigId(), this); + _configFetcher->start(); + return ""; +} + +StoragePolicy::~StoragePolicy() +{ +} + +string +StoragePolicy::createConfigId(const string & clusterName) const +{ + return "storage/cluster." + clusterName; +} + +string +StoragePolicy::createPattern(const string & clusterName, int distributor) const +{ + vespalib::asciistream ost; + + ost << "storage/cluster." << clusterName << "/distributor/"; + + if (distributor == -1) { + ost << '*'; + } else { + ost << distributor; + } + ost << "/default"; + return ost.str(); +} + +void +StoragePolicy::configure(std::unique_ptr<vespa::config::content::StorDistributionConfig> config) +{ + try { + _nextDistribution.reset(new storage::lib::Distribution(*config)); + } catch (const std::exception& e) { + LOG(warning, "Got exception when configuring distribution, config id was %s", _clusterConfigId.c_str()); + throw e; + } +} + +void +StoragePolicy::doSelect(mbus::RoutingContext &context) +{ + const mbus::Message &msg = context.getMessage(); + + int distributor = -1; + + if (_state.get()) { + document::BucketId id; + switch(msg.getType()) { + case DocumentProtocol::MESSAGE_PUTDOCUMENT: + id = _bucketIdFactory.getBucketId( + static_cast<const PutDocumentMessage&>(msg).getDocument()->getId()); + break; + + case DocumentProtocol::MESSAGE_GETDOCUMENT: + id = _bucketIdFactory.getBucketId(static_cast<const GetDocumentMessage&>(msg).getDocumentId()); + break; + + case DocumentProtocol::MESSAGE_REMOVEDOCUMENT: + id = _bucketIdFactory.getBucketId(static_cast<const RemoveDocumentMessage&>(msg).getDocumentId()); + break; + + case DocumentProtocol::MESSAGE_UPDATEDOCUMENT: + id = _bucketIdFactory.getBucketId(static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()->getId()); + break; + + case DocumentProtocol::MESSAGE_MULTIOPERATION: + id = (static_cast<const MultiOperationMessage&>(msg)).getBucketId(); + break; + + case DocumentProtocol::MESSAGE_STATBUCKET: + id = static_cast<const StatBucketMessage&>(msg).getBucketId(); + break; + + case DocumentProtocol::MESSAGE_GETBUCKETLIST: + id = static_cast<const GetBucketListMessage&>(msg).getBucketId(); + break; + + case DocumentProtocol::MESSAGE_CREATEVISITOR: + id = static_cast<const CreateVisitorMessage&>(msg).getBuckets()[0]; + break; + + case DocumentProtocol::MESSAGE_REMOVELOCATION: + id = static_cast<const RemoveLocationMessage&>(msg).getBucketId(); + break; + + case DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE: + id = static_cast<const BatchDocumentUpdateMessage&>(msg).getBucketId(); + break; + + default: + LOG(error, "Message type '%d' not supported.", msg.getType()); + return; + } + + // _P_A_R_A_N_O_I_A_ + if (id.getRawId() == 0) { + mbus::Reply::UP reply(new mbus::EmptyReply()); + reply->addError(mbus::Error(mbus::ErrorCode::APP_FATAL_ERROR, + "No bucket id available in message.")); + context.setReply(std::move(reply)); + return; + } + + // Pick a distributor using ideal state algorithm + try { + // Update distribution here, to make it not take lock in average case + if (_nextDistribution.get() != 0) { + _distribution = std::move(_nextDistribution); + _nextDistribution.reset(); + } + assert(_distribution.get()); + distributor = _distribution->getIdealDistributorNode(*_state, id); + } catch (storage::lib::TooFewBucketBitsInUseException& e) { + mbus::Reply::UP reply( + new WrongDistributionReply(_state->toString())); + reply->addError(mbus::Error( + DocumentProtocol::ERROR_WRONG_DISTRIBUTION, + "Too few distribution bits used for given cluster state")); + context.setReply(std::move(reply)); + return; + + } catch (storage::lib::NoDistributorsAvailableException& e) { + // No distributors available in current cluster state. Remove + // cluster state we cannot use and send to random target + _state.reset(); + distributor = -1; + } + } + + mbus::Hop hop = getRecipient(context, distributor); + + if (distributor != -1 && !hop.hasDirectives()) { + hop = getRecipient(context, -1); + } + + if (hop.hasDirectives()) { + mbus::Route route = context.getRoute(); + route.setHop(0, hop); + context.addChild(route); + } else { + context.setError( + mbus::ErrorCode::NO_ADDRESS_FOR_SERVICE, + vespalib::make_string( + "Could not resolve a distributor to send to in cluster %s", + _clusterName.c_str())); + } +} + +mbus::Hop +StoragePolicy::getRecipient(mbus::RoutingContext& context, int distributor) +{ + slobrok::api::IMirrorAPI::SpecList entries = lookup(context, createPattern(_clusterName, distributor)); + + if (!entries.empty()) { + return mbus::Hop::parse( + entries[random() % entries.size()].second + "/default"); + } + + return mbus::Hop(); +} + +void +StoragePolicy::merge(mbus::RoutingContext &context) +{ + mbus::RoutingNodeIterator it = context.getChildIterator(); + mbus::Reply::UP reply = it.removeReply(); + + if (reply->getType() == DocumentProtocol::REPLY_WRONGDISTRIBUTION) { + updateStateFromReply(static_cast<WrongDistributionReply&>(*reply)); + } else if (reply->hasErrors()) { + _state.reset(); + } + + context.setReply(std::move(reply)); +} + +void +StoragePolicy::updateStateFromReply(WrongDistributionReply& wdr) +{ + std::unique_ptr<storage::lib::ClusterState> newState( + new storage::lib::ClusterState(wdr.getSystemState())); + if (_state.get() == 0 || newState->getVersion() >= _state->getVersion()) { + if (_state.get()) { + wdr.getTrace().trace(1, vespalib::make_string( + "System state changed from version %u to %u", + _state->getVersion(), + newState->getVersion())); + } else { + wdr.getTrace().trace(1, vespalib::make_string( + "System state set to version %u", newState->getVersion())); + } + + _state = std::move(newState); + } else { + wdr.getTrace().trace(1, vespalib::make_string( + "System state cleared because system state returned had version %d, while old state had version %d. New states should not have a lower version than the old.", + newState->getVersion(), + _state->getVersion())); + _state.reset(); + } +} + +} // documentapi diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.h new file mode 100644 index 00000000000..096dec568ec --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.h @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/documentapi/messagebus/messages/wrongdistributionreply.h> +#include <vespa/messagebus/reply.h> +#include <vespa/vdslib/distribution/distribution.h> +#include <vespa/vdslib/state/clusterstate.h> +#include <vespa/documentapi/messagebus/policies/externslobrokpolicy.h> + +namespace documentapi { + +class StoragePolicy : public boost::noncopyable, + public ExternSlobrokPolicy, + public config::IFetcherCallback<vespa::config::content::StorDistributionConfig> +{ +private: + document::BucketIdFactory _bucketIdFactory; + std::unique_ptr<storage::lib::ClusterState> _state; + string _clusterName; + string _clusterConfigId; + std::unique_ptr<config::ConfigFetcher> _configFetcher; + std::unique_ptr<storage::lib::Distribution> _distribution; + std::unique_ptr<storage::lib::Distribution> _nextDistribution; + + mbus::Hop getRecipient(mbus::RoutingContext& context, int distributor); + +public: + StoragePolicy(const string& param); + virtual ~StoragePolicy(); + + // Inherit doc from IRoutingPolicy. + virtual void doSelect(mbus::RoutingContext &context); + + // Inherit doc from IRoutingPolicy. + virtual void merge(mbus::RoutingContext &context); + + void updateStateFromReply(WrongDistributionReply& reply); + + /** + * @return a pointer to the system state registered with this policy. If + * we haven't received a system state yet, returns NULL. + */ + const storage::lib::ClusterState* getSystemState() const + { return _state.get(); } + + void configure(std::unique_ptr<vespa::config::content::StorDistributionConfig> config); + + string init(); + +private: + virtual string createConfigId(const string & clusterName) const; + string createPattern(const string & clusterName, int distributor) const; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.cpp new file mode 100644 index 00000000000..634dd9d4d48 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.cpp @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".subsetservicepolicy"); + +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/routing/route.h> +#include <vespa/messagebus/routing/routingcontext.h> +#include <vespa/messagebus/routing/verbatimdirective.h> +#include <vespa/vespalib/util/hashmap.h> +#include <vespa/vespalib/util/stringfmt.h> +#include "subsetservicepolicy.h" + +namespace documentapi { + +SubsetServicePolicy::CacheEntry::CacheEntry() : + _offset(0), + _generation(0), + _recipients() +{ + // empty +} + +SubsetServicePolicy::SubsetServicePolicy(const string ¶m) : + _subsetSize(5), + _cache() +{ + if (param.length() > 0) { + int size = atoi(param.c_str()); + if (size >= 0) { + _subsetSize = (uint32_t)size; + } else { + LOG(warning, + "Ignoring a request to set the subset size to %d because it makes no sense. " + "This routing policy will choose any one matching service.", size); + } + } else { + LOG(warning, "No parameter given to SubsetService policy, using default value %d.", _subsetSize); + } +} + +SubsetServicePolicy::~SubsetServicePolicy() +{ + // empty +} + +void +SubsetServicePolicy::select(mbus::RoutingContext &context) +{ + mbus::Route route = context.getRoute(); + route.setHop(0, getRecipient(context)); + context.addChild(route); +} + +void +SubsetServicePolicy::merge(mbus::RoutingContext &context) +{ + DocumentProtocol::merge(context); +} + +string +SubsetServicePolicy::getCacheKey(const mbus::RoutingContext &ctx) const +{ + return ctx.getRoute().getHop(0).toString(); +} + +mbus::Hop +SubsetServicePolicy::getRecipient(mbus::RoutingContext &ctx) +{ + mbus::Hop hop; + if (_subsetSize > 0) { + vespalib::LockGuard guard(_lock); + CacheEntry &entry = update(ctx); + if (!entry._recipients.empty()) { + if (++entry._offset >= entry._recipients.size()) { + entry._offset = 0; + } + hop = entry._recipients[entry._offset]; + } + } + if (!hop.hasDirectives()) { + hop = ctx.getRoute().getHop(0); + hop.setDirective(ctx.getDirectiveIndex(), + mbus::IHopDirective::SP(new mbus::VerbatimDirective("*"))); + } + return hop; +} + +SubsetServicePolicy::CacheEntry & +SubsetServicePolicy::update(mbus::RoutingContext &ctx) +{ + uint32_t upd = ctx.getMirror().updates(); + CacheEntry &entry = _cache.insert(std::map<string, CacheEntry>::value_type(getCacheKey(ctx), CacheEntry())).first->second; + if (entry._generation != upd) { + entry._generation = upd; + entry._recipients.clear(); + + string pattern = vespalib::make_string("%s*%s", + ctx.getHopPrefix().c_str(), + ctx.getHopSuffix().c_str()); + slobrok::api::IMirrorAPI::SpecList entries = ctx.getMirror().lookup(pattern); + uint32_t pos = vespalib::hashValue(ctx.getMessageBus().getConnectionSpec().c_str()); + for (uint32_t i = 0; i < _subsetSize && i < entries.size(); ++i) { + entry._recipients.push_back(mbus::Hop::parse(entries[(pos + i) % entries.size()].first)); + } + } + return entry; +} + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.h new file mode 100644 index 00000000000..367aae183ff --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.h @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <boost/utility.hpp> +#include <vespa/messagebus/routing/hop.h> +#include <vespa/messagebus/routing/iroutingpolicy.h> +#include <string> +#include <vector> + +namespace documentapi { + +/** + * This policy implements the logic to select a subset of services that matches a slobrok pattern. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @version $Id$ + */ +class SubsetServicePolicy : public boost::noncopyable, public mbus::IRoutingPolicy { +private: + struct CacheEntry { + uint32_t _offset; + uint32_t _generation; + std::vector<mbus::Hop> _recipients; + + CacheEntry(); + }; + + vespalib::Lock _lock; + uint32_t _subsetSize; + std::map<string, CacheEntry> _cache; + + /** + * Returns the appropriate recipient hop for the given routing context. This method provides synchronized access to + * the internal cache. + * + * @param ctx The routing context. + * @return The recipient hop to use. + */ + mbus::Hop getRecipient(mbus::RoutingContext &ctx); + + /** + * Updates and returns the cache entry for the given routing context. This method assumes that synchronization is + * handled outside of it. + * + * @param ctx The routing context. + * @return The updated cache entry. + */ + CacheEntry &update(mbus::RoutingContext &ctx); + + /** + * Returns a cache key for this instance of the policy. Because behaviour is based on the hop in which the policy + * occurs, the cache key is the hop string itself. + * + * @param ctx The routing context. + * @return The cache key. + */ + string getCacheKey(const mbus::RoutingContext &ctx) const; + +public: + /** + * Creates an instance of a subset service policy. The parameter string is parsed as an integer number that is the + * number of services to include in the set to choose from. + * + * @param param The number of services to include in the set. + */ + SubsetServicePolicy(const string ¶m); + + /** + * Destructor. + * + * Frees all allocated resources. + */ + virtual ~SubsetServicePolicy(); + + // Inherit doc from IRoutingPolicy. + virtual void select(mbus::RoutingContext &context); + + // Inherit doc from IRoutingPolicy. + virtual void merge(mbus::RoutingContext &context); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/priority.h b/documentapi/src/vespa/documentapi/messagebus/priority.h new file mode 100644 index 00000000000..49a3e4e868c --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/priority.h @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/common.h> + +namespace documentapi { + +class Priority { +public: +/** + Define the different priorities allowed for document api messages. + Most user traffic should be fit into the NORMAL categories. Traffic + in the HIGH end will be usually be prioritized over important maintenance + operations. Traffic in the LOW end will be prioritized after + these operations. + + These enum values MUST be 0-indexed and continuous. +*/ + enum Value { + PRI_HIGHEST = 0, + PRI_VERY_HIGH = 1, + PRI_HIGH_1 = 2, + PRI_HIGH_2 = 3, + PRI_HIGH_3 = 4, + PRI_NORMAL_1 = 5, + PRI_NORMAL_2 = 6, + PRI_NORMAL_3 = 7, + PRI_NORMAL_4 = 8, + PRI_NORMAL_5 = 9, + PRI_NORMAL_6 = 10, + PRI_LOW_1 = 11, + PRI_LOW_2 = 12, + PRI_LOW_3 = 13, + PRI_VERY_LOW = 14, + PRI_LOWEST = 15, + + PRI_ENUM_SIZE = 16 + }; + + static Value getPriority(const string& priorityName) { + if (priorityName == ("HIGHEST")) { return PRI_HIGHEST; } + if (priorityName == ("VERY_HIGH")) { return PRI_VERY_HIGH; } + if (priorityName == ("HIGH_1")) { return PRI_HIGH_1; } + if (priorityName == ("HIGH_2")) { return PRI_HIGH_2; } + if (priorityName == ("HIGH_3")) { return PRI_HIGH_3; } + if (priorityName == ("NORMAL_1")) { return PRI_NORMAL_1; } + if (priorityName == ("NORMAL_2")) { return PRI_NORMAL_2; } + if (priorityName == ("NORMAL_3")) { return PRI_NORMAL_3; } + if (priorityName == ("NORMAL_4")) { return PRI_NORMAL_4; } + if (priorityName == ("NORMAL_5")) { return PRI_NORMAL_5; } + if (priorityName == ("NORMAL_6")) { return PRI_NORMAL_6; } + if (priorityName == ("LOW_1")) { return PRI_LOW_1; } + if (priorityName == ("LOW_2")) { return PRI_LOW_2; } + if (priorityName == ("LOW_3")) { return PRI_LOW_3; } + if (priorityName == ("VERY_LOW")) { return PRI_VERY_LOW; } + if (priorityName == ("LOWEST")) { return PRI_LOWEST; } + return PRI_LOWEST; + }; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp b/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp new file mode 100644 index 00000000000..fe6a58c1427 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/replymerger.cpp @@ -0,0 +1,181 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <cassert> +#include <vespa/messagebus/reply.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/documentapi/messagebus/replymerger.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/documentapi/messagebus/messages/removedocumentreply.h> +#include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> +#include <vespa/documentapi/messagebus/messages/getdocumentreply.h> + +namespace documentapi { + +ReplyMerger::ReplyMerger() + : _error(), + _ignored(), + _successReply(0), + _successIndex(0) +{ +} + +ReplyMerger::Result::Result(uint32_t successIdx, + std::unique_ptr<mbus::Reply> generatedReply) + : _generatedReply(std::move(generatedReply)), + _successIdx(successIdx) +{ +} + +ReplyMerger::Result::Result(Result&& o) + : _generatedReply(std::move(o._generatedReply)), + _successIdx(o._successIdx) +{ +} + +bool +ReplyMerger::Result::hasGeneratedReply() const +{ + return (_generatedReply.get() != 0); +} + +bool +ReplyMerger::Result::isSuccessful() const +{ + return !hasGeneratedReply(); +} + +std::unique_ptr<mbus::Reply> +ReplyMerger::Result::releaseGeneratedReply() +{ + assert(hasGeneratedReply()); + return std::move(_generatedReply); +} + +uint32_t +ReplyMerger::Result::getSuccessfulReplyIndex() +{ + assert(!hasGeneratedReply()); + return _successIdx; +} + +void +ReplyMerger::merge(uint32_t idx, const mbus::Reply& r) +{ + if (r.hasErrors()) { + mergeAllReplyErrors(r); + } else { + updateStateWithSuccessfulReply(idx, r); + } +} + +bool +ReplyMerger::resourceWasFound(const mbus::Reply& r) const +{ + switch (r.getType()) { + case DocumentProtocol::REPLY_REMOVEDOCUMENT: + return static_cast<const RemoveDocumentReply&>(r).wasFound(); + case DocumentProtocol::REPLY_UPDATEDOCUMENT: + return static_cast<const UpdateDocumentReply&>(r).wasFound(); + case DocumentProtocol::REPLY_GETDOCUMENT: + return static_cast<const GetDocumentReply&>(r).getLastModified() != 0; + default: + return false; + } +} + +// Precondition: _successReply != 0 +bool +ReplyMerger::replyIsBetterThanCurrent(const mbus::Reply& r) const +{ + return resourceWasFound(r) && !resourceWasFound(*_successReply); +} + +void +ReplyMerger::setCurrentBestReply(uint32_t idx, const mbus::Reply& r) +{ + _successIndex = idx; + _successReply = &r; +} + +void +ReplyMerger::updateStateWithSuccessfulReply(uint32_t idx, const mbus::Reply& r) +{ + if (!_successReply || replyIsBetterThanCurrent(r)) { + setCurrentBestReply(idx, r); + } +} + +void +ReplyMerger::mergeAllReplyErrors(const mbus::Reply& r) +{ + if (handleReplyWithOnlyIgnoredErrors(r)) { + return; + } + if (!_error.get()) { + _error.reset(new mbus::EmptyReply()); + } + for (uint32_t i = 0; i < r.getNumErrors(); ++i) { + _error->addError(r.getError(i)); + } +} + +bool +ReplyMerger::handleReplyWithOnlyIgnoredErrors(const mbus::Reply& r) +{ + if (DocumentProtocol::hasOnlyErrorsOfType(r, DocumentProtocol::ERROR_MESSAGE_IGNORED)) { + if (!_ignored.get()) { + _ignored.reset(new mbus::EmptyReply()); + } + _ignored->addError(r.getError(0)); + return true; + } + return false; +} + +bool +ReplyMerger::shouldReturnErrorReply() const +{ + if (_error.get()) { + return true; + } + return (_ignored.get() && !_successReply); +} + +std::unique_ptr<mbus::Reply> +ReplyMerger::releaseGeneratedErrorReply() +{ + if (_error.get()) { + return std::move(_error); + } else { + assert(_ignored.get()); + return std::move(_ignored); + } +} + +bool +ReplyMerger::successfullyMergedAtLeastOneReply() const +{ + return (_successReply != 0); +} + +ReplyMerger::Result +ReplyMerger::createEmptyReplyResult() const +{ + return Result(0u, std::unique_ptr<mbus::Reply>(new mbus::EmptyReply())); +} + +ReplyMerger::Result +ReplyMerger::mergedReply() +{ + std::unique_ptr<mbus::Reply> generated; + if (shouldReturnErrorReply()) { + generated = std::move(releaseGeneratedErrorReply()); + } else if (!successfullyMergedAtLeastOneReply()) { + return createEmptyReplyResult(); + } + return Result(_successIndex, std::move(generated)); +} + +} // documentapi + diff --git a/documentapi/src/vespa/documentapi/messagebus/replymerger.h b/documentapi/src/vespa/documentapi/messagebus/replymerger.h new file mode 100644 index 00000000000..47076dd7b7e --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/replymerger.h @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <utility> +#include <memory> +#include <vespa/messagebus/reply.h> + +namespace documentapi { + +class ReplyMerger +{ +public: + class Result { + friend class ReplyMerger; + + std::unique_ptr<mbus::Reply> _generatedReply; + uint32_t _successIdx; + + Result(uint32_t successIdx, + std::unique_ptr<mbus::Reply> generatedReply); + public: + Result(Result&&); + + bool hasGeneratedReply() const; + bool isSuccessful() const; + std::unique_ptr<mbus::Reply> releaseGeneratedReply(); + uint32_t getSuccessfulReplyIndex(); + }; +private: + std::unique_ptr<mbus::Reply> _error; + std::unique_ptr<mbus::Reply> _ignored; + const mbus::Reply* _successReply; + uint32_t _successIndex; + + void mergeAllReplyErrors(const mbus::Reply&); + bool handleReplyWithOnlyIgnoredErrors(const mbus::Reply& r); + bool shouldReturnErrorReply() const; + std::unique_ptr<mbus::Reply> releaseGeneratedErrorReply(); + bool replyIsBetterThanCurrent(const mbus::Reply& r) const; + void setCurrentBestReply(uint32_t idx, const mbus::Reply& r); + void updateStateWithSuccessfulReply(uint32_t idx, const mbus::Reply& r); + bool successfullyMergedAtLeastOneReply() const; + Result createEmptyReplyResult() const; + bool resourceWasFound(const mbus::Reply& r) const; +public: + ReplyMerger(); + + void merge(uint32_t idx, const mbus::Reply&); + Result mergedReply(); +}; + +} // documentapi + + diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories41.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories41.cpp new file mode 100644 index 00000000000..f07c797fb0f --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories41.cpp @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".routablefactories"); + +#include "routablefactories41.h" +#include <vespa/document/document.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/vespalib/objects/nbostream.h> + +using vespalib::nbostream; + +namespace documentapi { + +string +RoutableFactories41::decodeString(document::ByteBuffer &in) +{ + int32_t len = decodeInt(in); + string ret = string(in.getBufferAtPos(), len); + in.incPos(len); + return ret; +} + +bool +RoutableFactories41::decodeBoolean(document::ByteBuffer &in) +{ + char ret; + in.getBytes(&ret, 1); + return (bool)ret; +} + +int32_t +RoutableFactories41::decodeInt(document::ByteBuffer &in) +{ + int32_t ret; + in.getIntNetwork(ret); + return ret; +} + +int64_t +RoutableFactories41::decodeLong(document::ByteBuffer &in) +{ + int64_t ret; + in.getLongNetwork(ret); + return ret; +} + +document::DocumentId +RoutableFactories41::decodeDocumentId(document::ByteBuffer &in) +{ + nbostream stream(in.getBufferAtPos(), in.getRemaining(), false); + document::DocumentId ret(stream); + in.incPos(stream.rp()); + return ret; +} + +void +RoutableFactories41::encodeDocumentId(const document::DocumentId &id, vespalib::GrowableByteBuffer &out) +{ + string str = id.toString(); + out.putBytes(str.c_str(), str.size() + 1); +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories41.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories41.h new file mode 100644 index 00000000000..283d295dc82 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories41.h @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/base/documentid.h> +#include <vespa/document/util/bytebuffer.h> +#include <vespa/documentapi/messagebus/messages/feedmessage.h> +#include <vespa/documentapi/messagebus/messages/feedreply.h> +#include <vespa/messagebus/routable.h> +#include <vespa/messagebus/blob.h> +#include <vespa/messagebus/blobref.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + +namespace documentapi { + +/** + * This class encapsulates all the {@link RoutableFactory} classes needed to implement factories for the document + * routable. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. + */ +class RoutableFactories41 { +private: + RoutableFactories41() { /* abstract */ } + +public: + + /////////////////////////////////////////////////////////////////////////// + // + // Utilities + // + /////////////////////////////////////////////////////////////////////////// + + /** + * This is a complement for the vespalib::GrowableByteBuffer.putString() method. + * + * @param in The byte buffer to read from. + * @return The decoded string. + */ + static string decodeString(document::ByteBuffer &in); + + /** + * This is a complement for the vespalib::GrowableByteBuffer.putBoolean() method. + * + * @param in The byte buffer to read from. + * @return The decoded bool. + */ + static bool decodeBoolean(document::ByteBuffer &in); + + /** + * Convenience method to decode a 32-bit int from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded int. + */ + static int32_t decodeInt(document::ByteBuffer &in); + + /** + * Convenience method to decode a 64-bit int from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded int. + */ + static int64_t decodeLong(document::ByteBuffer &in); + + /** + * Convenience method to decode a document id from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded document id. + */ + static document::DocumentId decodeDocumentId(document::ByteBuffer &in); + + /** + * Convenience method to encode a document id to the given byte buffer. + * + * @param id The document id to encode. + * @param out The byte buffer to write to. + */ + static void encodeDocumentId(const document::DocumentId &id, vespalib::GrowableByteBuffer &out); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories42.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories42.cpp new file mode 100644 index 00000000000..358de79da8a --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories42.cpp @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".routablefactories42"); + +#include "routablefactories42.h" +#include <vespa/document/document.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/vespalib/objects/nbostream.h> + +using vespalib::nbostream; + +namespace documentapi { + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories42.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories42.h new file mode 100644 index 00000000000..3bf3daadb5e --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories42.h @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "routablefactories41.h" +#include <vespa/document/base/documentid.h> +#include <vespa/document/util/bytebuffer.h> +#include <vespa/documentapi/messagebus/messages/feedmessage.h> +#include <vespa/documentapi/messagebus/messages/feedreply.h> +#include <vespa/messagebus/blob.h> +#include <vespa/messagebus/blobref.h> +#include <vespa/messagebus/routable.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + +namespace document { class DocumentTypeRepo; } + +namespace documentapi { +/** + * This class encapsulates all the {@link RoutableFactory} classes needed to implement factories for the document + * routable. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. + */ +class RoutableFactories42 { +private: + RoutableFactories42() { /* abstract */ } + typedef RoutableFactories41 RF41; + +public: + + /////////////////////////////////////////////////////////////////////////// + // + // Utilities + // + /////////////////////////////////////////////////////////////////////////// + + /** + * This is a complement for the vespalib::GrowableByteBuffer.putString() method. + * + * @param in The byte buffer to read from. + * @return The decoded string. + */ + static string decodeString(document::ByteBuffer &in) + { return RF41::decodeString(in); } + + /** + * This is a complement for the vespalib::GrowableByteBuffer.putBoolean() method. + * + * @param in The byte buffer to read from. + * @return The decoded bool. + */ + static bool decodeBoolean(document::ByteBuffer &in) + { return RF41::decodeBoolean(in); } + + /** + * Convenience method to decode a 32-bit int from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded int. + */ + static int32_t decodeInt(document::ByteBuffer &in) + { return RF41::decodeInt(in); } + + /** + * Convenience method to decode a 64-bit int from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded int. + */ + static int64_t decodeLong(document::ByteBuffer &in) + { return RF41::decodeLong(in); } + + + /** + * Convenience method to decode a document id from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded document id. + */ + static document::DocumentId decodeDocumentId(document::ByteBuffer &in) + { return RF41::decodeDocumentId(in); } + + /** + * Convenience method to encode a document id to the given byte buffer. + * + * @param id The document id to encode. + * @param out The byte buffer to write to. + */ + static void encodeDocumentId(const document::DocumentId &id, + vespalib::GrowableByteBuffer &out) + { return RF41::encodeDocumentId(id, out); } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp new file mode 100644 index 00000000000..e3b17ea1c5c --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp @@ -0,0 +1,1101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".routablefactories50"); + +#include "routablefactories50.h" +#include <vespa/document/document.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <memory> + +using vespalib::nbostream; +using std::make_unique; +using std::make_shared; + +namespace documentapi { + +bool +RoutableFactories50::DocumentMessageFactory::encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const +{ + const DocumentMessage &msg = static_cast<const DocumentMessage&>(obj); + out.putByte(msg.getPriority()); + out.putInt(msg.getLoadType().getId()); + return doEncode(msg, out); +} + +mbus::Routable::UP +RoutableFactories50::DocumentMessageFactory::decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const +{ + uint8_t pri; + in.getByte(pri); + uint32_t loadClass = decodeInt(in); + + DocumentMessage::UP msg = doDecode(in); + if (msg.get() != NULL) { + msg->setPriority((Priority::Value)pri); + msg->setLoadType(loadTypes[loadClass]); + } + + return mbus::Routable::UP(msg.release()); +} + +bool +RoutableFactories50::DocumentReplyFactory::encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const +{ + const DocumentReply &msg = static_cast<const DocumentReply&>(obj); + out.putByte(msg.getPriority()); + return doEncode(msg, out); +} + +mbus::Routable::UP +RoutableFactories50::DocumentReplyFactory::decode(document::ByteBuffer &in, const LoadTypeSet&) const +{ + uint8_t pri; + in.getByte(pri); + DocumentReply::UP reply = doDecode(in); + if (reply.get() != NULL) { + reply->setPriority((Priority::Value)pri); + } + return mbus::Routable::UP(reply.release()); +} + +//////////////////////////////////////////////////////////////////////////////// +// +// Factories +// +//////////////////////////////////////////////////////////////////////////////// + +DocumentMessage::UP +RoutableFactories50::BatchDocumentUpdateMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + uint64_t userId = (uint64_t)decodeLong(buf); + string group = decodeString(buf); + + BatchDocumentUpdateMessage* msg; + if (group.length()) { + msg = new BatchDocumentUpdateMessage(group); + } else { + msg = new BatchDocumentUpdateMessage(userId); + } + DocumentMessage::UP retVal(msg); + + uint32_t len = decodeInt(buf); + for (uint32_t i = 0; i < len; i++) { + document::DocumentUpdate::SP upd; + upd.reset(document::DocumentUpdate::createHEAD(_repo, buf).release()); + msg->addUpdate(upd); + } + + return retVal; +} + +bool +RoutableFactories50::BatchDocumentUpdateMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const BatchDocumentUpdateMessage &msg = static_cast<const BatchDocumentUpdateMessage&>(obj); + + buf.putLong(msg.getUserId()); + buf.putString(msg.getGroup()); + buf.putInt(msg.getUpdates().size()); + + vespalib::nbostream stream; + for (const auto & update : msg.getUpdates()) { + update->serializeHEAD(stream); + } + buf.putBytes(stream.c_str(), stream.size()); + + return true; +} + +DocumentReply::UP +RoutableFactories50::BatchDocumentUpdateReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + BatchDocumentUpdateReply* reply = new BatchDocumentUpdateReply(); + reply->setHighestModificationTimestamp(decodeLong(buf)); + std::vector<bool>& notFound = reply->getDocumentsNotFound(); + notFound.resize(decodeInt(buf)); + for (std::size_t i = 0; i < notFound.size(); ++i) { + notFound[i] = decodeBoolean(buf); + } + return DocumentReply::UP(reply); +} + +bool +RoutableFactories50::BatchDocumentUpdateReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const BatchDocumentUpdateReply& reply = static_cast<const BatchDocumentUpdateReply&>(obj); + buf.putLong(reply.getHighestModificationTimestamp()); + const std::vector<bool>& notFound = reply.getDocumentsNotFound(); + buf.putInt(notFound.size()); + for (std::size_t i = 0; i < notFound.size(); ++i) { + buf.putBoolean(notFound[i]); + } + return true; +} + +DocumentMessage::UP +RoutableFactories50::CreateVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new CreateVisitorMessage()); + CreateVisitorMessage &msg = static_cast<CreateVisitorMessage&>(*ret); + + msg.setLibraryName(decodeString(buf)); + msg.setInstanceId(decodeString(buf)); + msg.setControlDestination(decodeString(buf)); + msg.setDataDestination(decodeString(buf)); + msg.setDocumentSelection(decodeString(buf)); + msg.setMaximumPendingReplyCount(decodeInt(buf)); + + int32_t len = decodeInt(buf); + for (int32_t i = 0; i < len; i++) { + int64_t val; + buf.getLong(val); // NOT using getLongNetwork + msg.getBuckets().push_back(document::BucketId(val)); + } + + msg.setFromTimestamp(decodeLong(buf)); + msg.setToTimestamp(decodeLong(buf)); + msg.setVisitRemoves(decodeBoolean(buf)); + msg.setVisitHeadersOnly(decodeBoolean(buf)); + msg.setVisitInconsistentBuckets(decodeBoolean(buf)); + msg.getParameters().deserialize(_repo, buf); + msg.setVisitorDispatcherVersion(50); + msg.setVisitorOrdering((document::OrderingSpecification::Order)decodeInt(buf)); + msg.setMaxBucketsPerVisitor(decodeInt(buf)); + + return ret; +} + +bool +RoutableFactories50::CreateVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const CreateVisitorMessage &msg = static_cast<const CreateVisitorMessage&>(obj); + + buf.putString(msg.getLibraryName()); + buf.putString(msg.getInstanceId()); + buf.putString(msg.getControlDestination()); + buf.putString(msg.getDataDestination()); + buf.putString(msg.getDocumentSelection()); + buf.putInt(msg.getMaximumPendingReplyCount()); + buf.putInt(msg.getBuckets().size()); + + const std::vector<document::BucketId> &buckets = msg.getBuckets(); + for (std::vector<document::BucketId>::const_iterator it = buckets.begin(); + it != buckets.end(); ++it) + { + uint64_t val = it->getRawId(); + buf.putBytes((const char*)&val, 8); + } + + buf.putLong(msg.getFromTimestamp()); + buf.putLong(msg.getToTimestamp()); + buf.putBoolean(msg.visitRemoves()); + buf.putBoolean(msg.visitHeadersOnly()); + buf.putBoolean(msg.visitInconsistentBuckets()); + + int len = msg.getParameters().getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + msg.getParameters().serialize(dbuf); + + buf.putInt(msg.getVisitorOrdering()); + buf.putInt(msg.getMaxBucketsPerVisitor()); + + return true; +} + +DocumentReply::UP +RoutableFactories50::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentReply::UP ret(new CreateVisitorReply(DocumentProtocol::REPLY_CREATEVISITOR)); + CreateVisitorReply &reply = static_cast<CreateVisitorReply&>(*ret); + reply.setLastBucket(document::BucketId((uint64_t)decodeLong(buf))); + vdslib::VisitorStatistics vs; + vs.setBucketsVisited(decodeInt(buf)); + vs.setDocumentsVisited(decodeLong(buf)); + vs.setBytesVisited(decodeLong(buf)); + vs.setDocumentsReturned(decodeLong(buf)); + vs.setBytesReturned(decodeLong(buf)); + vs.setSecondPassDocumentsReturned(decodeLong(buf)); + vs.setSecondPassBytesReturned(decodeLong(buf)); + reply.setVisitorStatistics(vs); + + return ret; +} + +bool +RoutableFactories50::CreateVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const CreateVisitorReply &reply = static_cast<const CreateVisitorReply&>(obj); + buf.putLong(reply.getLastBucket().getRawId()); + buf.putInt(reply.getVisitorStatistics().getBucketsVisited()); + buf.putLong(reply.getVisitorStatistics().getDocumentsVisited()); + buf.putLong(reply.getVisitorStatistics().getBytesVisited()); + buf.putLong(reply.getVisitorStatistics().getDocumentsReturned()); + buf.putLong(reply.getVisitorStatistics().getBytesReturned()); + buf.putLong(reply.getVisitorStatistics().getSecondPassDocumentsReturned()); + buf.putLong(reply.getVisitorStatistics().getSecondPassBytesReturned()); + return true; +} + +DocumentMessage::UP +RoutableFactories50::DestroyVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new DestroyVisitorMessage()); + DestroyVisitorMessage &msg = static_cast<DestroyVisitorMessage&>(*ret); + msg.setInstanceId(decodeString(buf)); + return ret; +} + +bool +RoutableFactories50::DestroyVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const DestroyVisitorMessage &msg = static_cast<const DestroyVisitorMessage&>(obj); + buf.putString(msg.getInstanceId()); + return true; +} + +DocumentReply::UP +RoutableFactories50::DestroyVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + (void)buf; + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DESTROYVISITOR)); +} + +bool +RoutableFactories50::DestroyVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + (void)obj; + (void)buf; + return true; +} + +DocumentMessage::UP +RoutableFactories50::DocBlockMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + (void)buf; + return DocumentMessage::UP(); // TODO: remove message type +} + +bool +RoutableFactories50::DocBlockMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + (void)obj; + (void)buf; + return false; +} + +DocumentReply::UP +RoutableFactories50::DocBlockReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + (void)buf; + return DocumentReply::UP(); // TODO: remove reply type +} + +bool +RoutableFactories50::DocBlockReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + (void)obj; + (void)buf; + return false; +} + +DocumentMessage::UP +RoutableFactories50::DocumentListMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new DocumentListMessage()); + DocumentListMessage &msg = static_cast<DocumentListMessage&>(*ret); + + msg.setBucketId(document::BucketId(decodeLong(buf))); + + int32_t len = decodeInt(buf); + for (int32_t i = 0; i < len; i++) { + DocumentListMessage::Entry entry(_repo, buf); + msg.getDocuments().push_back(entry); + } + + return ret; +} + +bool +RoutableFactories50::DocumentListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const DocumentListMessage &msg = static_cast<const DocumentListMessage&>(obj); + + buf.putLong(msg.getBucketId().getRawId()); + buf.putInt(msg.getDocuments().size()); + for (uint32_t i = 0; i < msg.getDocuments().size(); i++) { + int len = msg.getDocuments()[i].getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + msg.getDocuments()[i].serialize(dbuf); + } + + return true; +} + +DocumentReply::UP +RoutableFactories50::DocumentListReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + (void)buf; + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTLIST)); +} + +bool +RoutableFactories50::DocumentListReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + (void)obj; + (void)buf; + return true; +} + +DocumentMessage::UP +RoutableFactories50::DocumentSummaryMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new DocumentSummaryMessage()); + DocumentSummaryMessage &msg = static_cast<DocumentSummaryMessage&>(*ret); + + msg.deserialize(buf); + + return ret; +} + +bool +RoutableFactories50::DocumentSummaryMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const DocumentSummaryMessage &msg = static_cast<const DocumentSummaryMessage&>(obj); + + int32_t len = msg.getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + msg.serialize(dbuf); + + return true; +} + +DocumentReply::UP +RoutableFactories50::DocumentSummaryReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + (void)buf; + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTSUMMARY)); +} + +bool +RoutableFactories50::DocumentSummaryReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + (void)obj; + (void)buf; + return true; +} + +DocumentMessage::UP +RoutableFactories50::EmptyBucketsMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new EmptyBucketsMessage()); + EmptyBucketsMessage &msg = static_cast<EmptyBucketsMessage&>(*ret); + + int32_t len = decodeInt(buf); + std::vector<document::BucketId> buckets(len); + for (int32_t i = 0; i < len; ++i) { + buckets[i] = document::BucketId(decodeLong(buf)); + } + msg.getBucketIds().swap(buckets); + + return ret; +} + +bool +RoutableFactories50::EmptyBucketsMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const EmptyBucketsMessage &msg = static_cast<const EmptyBucketsMessage&>(obj); + + buf.putInt(msg.getBucketIds().size()); + const std::vector<document::BucketId> &buckets = msg.getBucketIds(); + for (std::vector<document::BucketId>::const_iterator it = buckets.begin(); + it != buckets.end(); ++it) + { + buf.putLong(it->getRawId()); + } + + return true; +} + +DocumentReply::UP +RoutableFactories50::EmptyBucketsReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + (void)buf; + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_EMPTYBUCKETS)); +} + +bool +RoutableFactories50::EmptyBucketsReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + (void)obj; + (void)buf; + return true; +} + +DocumentMessage::UP +RoutableFactories50::GetBucketListMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new GetBucketListMessage()); + GetBucketListMessage &msg = static_cast<GetBucketListMessage&>(*ret); + + msg.setBucketId(document::BucketId(decodeLong(buf))); + + return ret; +} + +bool +RoutableFactories50::GetBucketListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const GetBucketListMessage &msg = static_cast<const GetBucketListMessage&>(obj); + buf.putLong(msg.getBucketId().getRawId()); + return true; +} + +DocumentReply::UP +RoutableFactories50::GetBucketListReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentReply::UP ret(new GetBucketListReply()); + GetBucketListReply &reply = static_cast<GetBucketListReply&>(*ret); + + int32_t len = decodeInt(buf); + for (int32_t i = 0; i < len; i++) { + GetBucketListReply::BucketInfo info; + info._bucket = document::BucketId((uint64_t)decodeLong(buf)); + info._bucketInformation = decodeString(buf); + reply.getBuckets().push_back(info); + } + + return ret; +} + +bool +RoutableFactories50::GetBucketListReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const GetBucketListReply &reply = static_cast<const GetBucketListReply&>(obj); + + const std::vector<GetBucketListReply::BucketInfo> &buckets = reply.getBuckets(); + buf.putInt(buckets.size()); + for (std::vector<GetBucketListReply::BucketInfo>::const_iterator it = buckets.begin(); + it != buckets.end(); ++it) + { + buf.putLong(it->_bucket.getRawId()); + buf.putString(it->_bucketInformation); + } + + return true; +} + +DocumentMessage::UP +RoutableFactories50::GetBucketStateMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new GetBucketStateMessage()); + GetBucketStateMessage &msg = static_cast<GetBucketStateMessage&>(*ret); + + msg.setBucketId(document::BucketId((uint64_t)decodeLong(buf))); + + return ret; +} + +bool +RoutableFactories50::GetBucketStateMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const GetBucketStateMessage &msg = static_cast<const GetBucketStateMessage&>(obj); + buf.putLong(msg.getBucketId().getRawId()); + return true; +} + +DocumentReply::UP +RoutableFactories50::GetBucketStateReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentReply::UP ret(new GetBucketStateReply()); + GetBucketStateReply &reply = static_cast<GetBucketStateReply&>(*ret); + + int32_t len = decodeInt(buf); + for (int32_t i = 0; i < len; i++) { + DocumentState state(buf); + reply.getBucketState().push_back(state); + } + + return ret; +} + +bool +RoutableFactories50::GetBucketStateReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const GetBucketStateReply &reply = static_cast<const GetBucketStateReply&>(obj); + + buf.putInt(reply.getBucketState().size()); + const std::vector<DocumentState> &state = reply.getBucketState(); + for (std::vector<DocumentState>::const_iterator it = state.begin(); + it != state.end(); ++it) + { + it->serialize(buf); + } + + return true; +} + +DocumentMessage::UP +RoutableFactories50::GetDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new GetDocumentMessage()); + GetDocumentMessage &msg = static_cast<GetDocumentMessage&>(*ret); + + msg.setDocumentId(decodeDocumentId(buf)); + msg.setFlags(decodeInt(buf)); + + return ret; +} + +bool +RoutableFactories50::GetDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const GetDocumentMessage &msg = static_cast<const GetDocumentMessage&>(obj); + + encodeDocumentId(msg.getDocumentId(), buf); + buf.putInt(msg.getFlags()); + + return true; +} + +DocumentReply::UP +RoutableFactories50::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentReply::UP ret(new GetDocumentReply()); + GetDocumentReply &reply = static_cast<GetDocumentReply&>(*ret); + + bool hasDocument = decodeBoolean(buf); + document::Document::SP document; + if (hasDocument) { + document.reset(new document::Document(_repo, buf)); + reply.setDocument(document); + } + int64_t lastModified = decodeLong(buf); + reply.setLastModified(lastModified); + if (document.get()) { + document->setLastModified(lastModified); + } + + return ret; +} + +bool +RoutableFactories50::GetDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const GetDocumentReply &reply = static_cast<const GetDocumentReply&>(obj); + + buf.putByte(reply.getDocument().get() == NULL ? 0 : 1); + if (reply.getDocument().get() != NULL) { + nbostream stream; + reply.getDocument()->serialize(stream); + buf.putBytes(stream.peek(), stream.size()); + } + buf.putLong(reply.getLastModified()); + + return true; +} + +DocumentMessage::UP +RoutableFactories50::MapVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new MapVisitorMessage()); + MapVisitorMessage &msg = static_cast<MapVisitorMessage&>(*ret); + + msg.getData().deserialize(_repo, buf); + + return ret; +} + +bool +RoutableFactories50::MapVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const MapVisitorMessage &msg = static_cast<const MapVisitorMessage&>(obj); + + int32_t len = msg.getData().getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + msg.getData().serialize(dbuf); + + return true; +} + +DocumentReply::UP +RoutableFactories50::MapVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + (void)buf; + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_MAPVISITOR)); +} + +bool +RoutableFactories50::MapVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + (void)obj; + (void)buf; + return true; +} + +DocumentMessage::UP +RoutableFactories50::MultiOperationMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + int64_t bucketId = decodeLong(buf); + + int32_t len = decodeInt(buf); + std::vector<char> tmp(len); + buf.getBytes(&tmp[0], len); + + DocumentMessage::UP ret(new MultiOperationMessage(_repo, document::BucketId(bucketId), tmp, decodeBoolean(buf))); + return ret; +} + +bool +RoutableFactories50::MultiOperationMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const MultiOperationMessage &msg = static_cast<const MultiOperationMessage&>(obj); + buf.putLong(msg.getBucketId().getRawId()); + + uint64_t docBlockSize = msg.getOperations().spaceNeeded(); + buf.putInt(docBlockSize); + char* pos = buf.allocate(docBlockSize); + vdslib::DocumentList copy(msg.getOperations(), pos, docBlockSize); + + buf.putBoolean(msg.keepTimeStamps()); + + return true; +} + +DocumentReply::UP +RoutableFactories50::MultiOperationReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + WriteDocumentReply* reply = new WriteDocumentReply(DocumentProtocol::REPLY_MULTIOPERATION); + reply->setHighestModificationTimestamp(decodeLong(buf)); + return DocumentReply::UP(reply); +} + +bool +RoutableFactories50::MultiOperationReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const WriteDocumentReply& reply = (const WriteDocumentReply&)obj; + buf.putLong(reply.getHighestModificationTimestamp()); + return true; +} + +void +RoutableFactories50::PutDocumentMessageFactory::decodeInto(PutDocumentMessage & msg, document::ByteBuffer & buf) const { + msg.setDocument(make_shared<document::Document>(_repo, buf)); + msg.setTimestamp(static_cast<uint64_t>(decodeLong(buf))); +} + +bool +RoutableFactories50::PutDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + auto & msg = static_cast<const PutDocumentMessage &>(obj); + nbostream stream; + + msg.getDocument()->serialize(stream); + buf.putBytes(stream.peek(), stream.size()); + buf.putLong(static_cast<int64_t>(msg.getTimestamp())); + + return true; +} + +DocumentReply::UP +RoutableFactories50::PutDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + auto reply = make_unique<WriteDocumentReply>(DocumentProtocol::REPLY_PUTDOCUMENT); + reply->setHighestModificationTimestamp(decodeLong(buf)); + + // Doing an explicit move here to force converting result to an rvalue. + // This is done automatically in GCC >= 5. + return std::move(reply); +} + +bool +RoutableFactories50::PutDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const WriteDocumentReply& reply = (const WriteDocumentReply&)obj; + buf.putLong(reply.getHighestModificationTimestamp()); + return true; +} + +void +RoutableFactories50::RemoveDocumentMessageFactory::decodeInto(RemoveDocumentMessage & msg, document::ByteBuffer & buf) const { + msg.setDocumentId(decodeDocumentId(buf)); +} + +bool +RoutableFactories50::RemoveDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const RemoveDocumentMessage &msg = static_cast<const RemoveDocumentMessage&>(obj); + encodeDocumentId(msg.getDocumentId(), buf); + return true; +} + +DocumentReply::UP +RoutableFactories50::RemoveDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentReply::UP ret(new RemoveDocumentReply()); + RemoveDocumentReply &reply = static_cast<RemoveDocumentReply&>(*ret); + reply.setWasFound(decodeBoolean(buf)); + reply.setHighestModificationTimestamp(decodeLong(buf)); + return ret; +} + +bool +RoutableFactories50::RemoveDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const RemoveDocumentReply &reply = static_cast<const RemoveDocumentReply&>(obj); + buf.putBoolean(reply.getWasFound()); + buf.putLong(reply.getHighestModificationTimestamp()); + return true; +} + +DocumentMessage::UP +RoutableFactories50::RemoveLocationMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + string selection = decodeString(buf); + + document::BucketIdFactory factory; + document::select::Parser parser(_repo, factory); + + return DocumentMessage::UP(new RemoveLocationMessage(factory, parser, selection)); +} + +bool +RoutableFactories50::RemoveLocationMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const RemoveLocationMessage &msg = static_cast<const RemoveLocationMessage&>(obj); + buf.putString(msg.getDocumentSelection()); + return true; +} + +DocumentReply::UP +RoutableFactories50::RemoveLocationReplyFactory::doDecode(document::ByteBuffer &) const +{ + return DocumentReply::UP(new DocumentReply(DocumentProtocol::REPLY_REMOVELOCATION)); +} + +bool +RoutableFactories50::RemoveLocationReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const +{ + return true; +} + +DocumentMessage::UP +RoutableFactories50::SearchResultMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new SearchResultMessage()); + SearchResultMessage &msg = static_cast<SearchResultMessage&>(*ret); + + msg.deserialize(buf); + + return ret; +} + +bool +RoutableFactories50::SearchResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const SearchResultMessage &msg = static_cast<const SearchResultMessage&>(obj); + + int len = msg.getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + msg.serialize(dbuf); + + return true; +} + +DocumentMessage::UP +RoutableFactories50::QueryResultMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new QueryResultMessage()); + QueryResultMessage &msg = static_cast<QueryResultMessage&>(*ret); + + msg.getSearchResult().deserialize(buf); + msg.getDocumentSummary().deserialize(buf); + + return ret; +} + +bool +RoutableFactories50::QueryResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const QueryResultMessage &msg = static_cast<const QueryResultMessage&>(obj); + + int len = msg.getSearchResult().getSerializedSize() + msg.getDocumentSummary().getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + msg.getSearchResult().serialize(dbuf); + msg.getDocumentSummary().serialize(dbuf); + + return true; +} + +DocumentReply::UP +RoutableFactories50::SearchResultReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + (void)buf; + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_SEARCHRESULT)); +} + +bool +RoutableFactories50::SearchResultReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + (void)obj; + (void)buf; + return true; +} + +DocumentReply::UP +RoutableFactories50::QueryResultReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + (void)buf; + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_QUERYRESULT)); +} + +bool +RoutableFactories50::QueryResultReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + (void)obj; + (void)buf; + return true; +} + +DocumentMessage::UP +RoutableFactories50::StatBucketMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new StatBucketMessage()); + StatBucketMessage &msg = static_cast<StatBucketMessage&>(*ret); + + msg.setBucketId(document::BucketId(decodeLong(buf))); + msg.setDocumentSelection(decodeString(buf)); + + return ret; +} + +bool +RoutableFactories50::StatBucketMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const StatBucketMessage &msg = static_cast<const StatBucketMessage&>(obj); + + buf.putLong(msg.getBucketId().getRawId()); + buf.putString(msg.getDocumentSelection()); + + return true; +} + +DocumentReply::UP +RoutableFactories50::StatBucketReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentReply::UP ret(new StatBucketReply()); + StatBucketReply &reply = static_cast<StatBucketReply&>(*ret); + + reply.setResults(decodeString(buf)); + + return ret; +} + +bool +RoutableFactories50::StatBucketReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const StatBucketReply &reply = static_cast<const StatBucketReply&>(obj); + buf.putString(reply.getResults()); + return true; +} + +DocumentMessage::UP +RoutableFactories50::StatDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + (void)buf; + return DocumentMessage::UP(); // TODO: remove message type +} + +bool +RoutableFactories50::StatDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + (void)obj; + (void)buf; + return false; +} + +DocumentReply::UP +RoutableFactories50::StatDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + (void)buf; + return DocumentReply::UP(); // TODO: remove reply type +} + +bool +RoutableFactories50::StatDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + (void)obj; + (void)buf; + return false; +} + +void +RoutableFactories50::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const { + msg.setDocumentUpdate(make_shared<document::DocumentUpdate> + (_repo, buf, + document::DocumentUpdate::SerializeVersion:: + SERIALIZE_HEAD)); + msg.setOldTimestamp(static_cast<uint64_t>(decodeLong(buf))); + msg.setNewTimestamp(static_cast<uint64_t>(decodeLong(buf))); +} + +bool +RoutableFactories50::UpdateDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const UpdateDocumentMessage &msg = static_cast<const UpdateDocumentMessage&>(obj); + + vespalib::nbostream stream; + msg.getDocumentUpdate()->serializeHEAD(stream); + buf.putBytes(stream.peek(), stream.size()); + buf.putLong((int64_t)msg.getOldTimestamp()); + buf.putLong((int64_t)msg.getNewTimestamp()); + + return true; +} + +DocumentReply::UP +RoutableFactories50::UpdateDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentReply::UP ret(new UpdateDocumentReply()); + UpdateDocumentReply &reply = static_cast<UpdateDocumentReply&>(*ret); + reply.setWasFound(decodeBoolean(buf)); + reply.setHighestModificationTimestamp(decodeLong(buf)); + return ret; +} + +bool +RoutableFactories50::UpdateDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const UpdateDocumentReply &reply = static_cast<const UpdateDocumentReply&>(obj); + buf.putBoolean(reply.getWasFound()); + buf.putLong(reply.getHighestModificationTimestamp()); + return true; +} + +DocumentMessage::UP +RoutableFactories50::VisitorInfoMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new VisitorInfoMessage()); + VisitorInfoMessage &msg = static_cast<VisitorInfoMessage&>(*ret); + + int32_t len = decodeInt(buf); + for (int32_t i = 0; i < len; i++) { + int64_t val; + buf.getLong(val); // NOT using getLongNetwork + msg.getFinishedBuckets().push_back(document::BucketId(val)); + } + msg.setErrorMessage(decodeString(buf)); + + return ret; +} + +bool +RoutableFactories50::VisitorInfoMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const VisitorInfoMessage &msg = static_cast<const VisitorInfoMessage&>(obj); + + buf.putInt(msg.getFinishedBuckets().size()); + const std::vector<document::BucketId> &buckets = msg.getFinishedBuckets(); + for (std::vector<document::BucketId>::const_iterator it = buckets.begin(); + it != buckets.end(); ++it) + { + uint64_t val = it->getRawId(); + buf.putBytes((const char*)&val, 8); + } + buf.putString(msg.getErrorMessage()); + + return true; +} + +DocumentReply::UP +RoutableFactories50::VisitorInfoReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + (void)buf; + return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_VISITORINFO)); +} + +bool +RoutableFactories50::VisitorInfoReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + (void)obj; + (void)buf; + return true; +} + +DocumentReply::UP +RoutableFactories50::WrongDistributionReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentReply::UP ret(new WrongDistributionReply()); + WrongDistributionReply &reply = static_cast<WrongDistributionReply&>(*ret); + + reply.setSystemState(decodeString(buf)); + + return ret; +} + +bool +RoutableFactories50::WrongDistributionReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const WrongDistributionReply &reply = static_cast<const WrongDistributionReply&>(obj); + buf.putString(reply.getSystemState()); + return true; +} + +void +RoutableFactories50::FeedMessageFactory::myDecode(FeedMessage &msg, document::ByteBuffer &buf) const +{ + msg.setName(decodeString(buf)); + msg.setGeneration(decodeInt(buf)); + msg.setIncrement(decodeInt(buf)); +} + +void +RoutableFactories50::FeedMessageFactory::myEncode(const FeedMessage &msg, vespalib::GrowableByteBuffer &buf) const +{ + buf.putString(msg.getName()); + buf.putInt(msg.getGeneration()); + buf.putInt(msg.getIncrement()); +} + +DocumentReply::UP +RoutableFactories50::FeedReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentReply::UP ret(new FeedReply(getType())); + FeedReply &reply = static_cast<FeedReply&>(*ret); + + std::vector<FeedAnswer> &answers = reply.getFeedAnswers(); + int32_t len = decodeInt(buf); + for (int32_t i = 0; i < len; ++i) { + int32_t typeCode = decodeInt(buf); + int32_t wantedIncrement = decodeInt(buf); + string recipient = decodeString(buf); + string moreInfo = decodeString(buf); + answers.push_back(FeedAnswer(typeCode, wantedIncrement, recipient, moreInfo)); + } + return ret; +} + +bool +RoutableFactories50::FeedReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const FeedReply &reply = static_cast<const FeedReply&>(obj); + buf.putInt(reply.getFeedAnswers().size()); + const std::vector<FeedAnswer> &answers = reply.getFeedAnswers(); + for (std::vector<FeedAnswer>::const_iterator it = answers.begin(); + it != answers.end(); ++it) + { + buf.putInt(it->getAnswerCode()); + buf.putInt(it->getWantedIncrement()); + buf.putString(it->getRecipient()); + buf.putString(it->getMoreInfo()); + } + return true; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.h new file mode 100644 index 00000000000..76cc5a6b2dd --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.h @@ -0,0 +1,501 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "routablefactories42.h" +#include <vespa/document/base/documentid.h> +#include <vespa/document/util/bytebuffer.h> +#include <vespa/documentapi/messagebus/messages/feedmessage.h> +#include <vespa/documentapi/messagebus/messages/feedreply.h> +#include <vespa/messagebus/routable.h> +#include <vespa/messagebus/blob.h> +#include <vespa/messagebus/blobref.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/documentapi/documentapi.h> + +namespace document { class DocumentTypeRepo; } + +/** + * Utility class for invoking setApproxSize on a DocumentMessage with the delta + * between the read position of a ByteBuffer at construction and destruction + * time. The assumption being made is that the in-memory footprint of a message + * is reasonably close to its wire-serialized form. + */ +class ScopedApproxSizeSetter { +public: + ScopedApproxSizeSetter(documentapi::DocumentMessage& msg, + const document::ByteBuffer& buf) + : _msg(msg), + _buf(buf), + _posBefore(_buf.getPos()) + { + } + + ~ScopedApproxSizeSetter() { + _msg.setApproxSize(static_cast<uint32_t>(_buf.getPos() - _posBefore)); + } + +private: + documentapi::DocumentMessage& _msg; + const document::ByteBuffer& _buf; + const size_t _posBefore; +}; + +namespace documentapi { + +template<typename MessageType, typename FactoryType> +DocumentMessage::UP +decodeMessage(const FactoryType * self, document::ByteBuffer & buf) { + auto msg = std::make_unique<MessageType>(); + ScopedApproxSizeSetter sizeSetter(*msg, buf); + + self->decodeInto(*msg, buf); + + // Doing an explicit move here to force converting result to an rvalue. + // This is done automatically in GCC >= 5. + return std::move(msg); +} + +/** + * This class encapsulates all the {@link RoutableFactory} classes needed to implement factories for the document + * routable. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. + */ +class RoutableFactories50 { +public: + RoutableFactories50() = delete; + + /** + * Implements the shared factory logic required for {@link DocumentMessage} objects, and it offers a more + * convenient interface for implementing {@link RoutableFactory}. + */ + class DocumentMessageFactory : public IRoutableFactory { + protected: + /** + * This method encodes the given message into the given byte buffer. You are guaranteed to only receive messages of + * the type that this factory was registered for. + * + * This method is NOT exception safe. Return false to signal failure. + * + * @param msg The message to encode. + * @param buf The byte buffer to write to. + * @return True if the message was encoded. + */ + virtual bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const = 0; + + /** + * This method decodes a message from the given byte buffer. You are guaranteed to only receive byte buffers + * generated by a previous call to {@link #doEncode(DocumentMessage, GrowableByteBuffer)}. + * + * This method is NOT exception safe. Return null to signal failure. + * + * @param buf The byte buffer to read from. + * @return The decoded message. + */ + virtual DocumentMessage::UP doDecode(document::ByteBuffer &buf) const = 0; + + public: + /** + * Convenience typedefs. + */ + typedef std::unique_ptr<IRoutableFactory> UP; + typedef std::shared_ptr<IRoutableFactory> SP; + + // Implements IRoutableFactory. + bool encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const; + + // Implements IRoutableFactory. + mbus::Routable::UP decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const; + }; + + /** + * Implements the shared factory logic required for {@link DocumentReply} objects, and it offers a more + * convenient interface for implementing {@link RoutableFactory}. + */ + class DocumentReplyFactory : public IRoutableFactory { + protected: + /** + * This method encodes the given reply into the given byte buffer. You are guaranteed to only receive + * replies of the type that this factory was registered for. + * + * This method is NOT exception safe. Return false to signal failure. + * + * @param reply The reply to encode. + * @param buf The byte buffer to write to. + * @return True if the message was encoded. + */ + virtual bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const = 0; + + /** + * This method decodes a reply from the given byte buffer. You are guaranteed to only receive byte buffers + * generated by a previous call to {@link #doEncode(DocumentReply, GrowableByteBuffer)}. + * + * This method is NOT exception safe. Return null to signal failure. + * + * @param buf The byte buffer to read from. + * @return The decoded reply. + */ + virtual DocumentReply::UP doDecode(document::ByteBuffer &buf) const = 0; + + public: + /** + * Convenience typedefs. + */ + typedef std::unique_ptr<IRoutableFactory> UP; + typedef std::shared_ptr<IRoutableFactory> SP; + + // Implements IRoutableFactory. + bool encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const; + + // Implements IRoutableFactory. + mbus::Routable::UP decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const; + }; + + /** + * Implements a helper class to do feed message factories. + */ + class FeedMessageFactory : public DocumentMessageFactory { + protected: + void myDecode(FeedMessage &msg, document::ByteBuffer &buf) const; + void myEncode(const FeedMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + + /** + * Implements a helper class to do feed reply factories. + */ + class FeedReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + virtual uint32_t getType() const = 0; + }; + + //////////////////////////////////////////////////////////////////////////////// + // + // Factories + // + //////////////////////////////////////////////////////////////////////////////// + + class BatchDocumentUpdateMessageFactory : public DocumentMessageFactory { + const document::DocumentTypeRepo &_repo; + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + public: + BatchDocumentUpdateMessageFactory(const document::DocumentTypeRepo &r) + : _repo(r) {} + }; + class BatchDocumentUpdateReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class CreateVisitorMessageFactory : public DocumentMessageFactory { + const document::DocumentTypeRepo &_repo; + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + public: + CreateVisitorMessageFactory(const document::DocumentTypeRepo &r) + : _repo(r) {} + }; + class CreateVisitorReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class DestroyVisitorMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + class DestroyVisitorReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class DocBlockMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + class DocBlockReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class DocumentListMessageFactory : public DocumentMessageFactory { + const document::DocumentTypeRepo &_repo; + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + public: + DocumentListMessageFactory(const document::DocumentTypeRepo &r) + : _repo(r) {} + }; + class DocumentListReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class DocumentSummaryMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + class DocumentSummaryReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class EmptyBucketsMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + class EmptyBucketsReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class GetBucketListMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + class GetBucketListReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class GetBucketStateMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + class GetBucketStateReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class GetDocumentMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + class GetDocumentReplyFactory : public DocumentReplyFactory { + const document::DocumentTypeRepo &_repo; + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &msg, vespalib::GrowableByteBuffer &buf) const; + public: + GetDocumentReplyFactory(const document::DocumentTypeRepo &r) + : _repo(r) {} + }; + class MapVisitorMessageFactory : public DocumentMessageFactory { + const document::DocumentTypeRepo &_repo; + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + public: + MapVisitorMessageFactory(const document::DocumentTypeRepo &r) + : _repo(r) {} + }; + class MapVisitorReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class MultiOperationMessageFactory : public DocumentMessageFactory { + document::DocumentTypeRepo::SP _repo; + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + public: + MultiOperationMessageFactory(const document::DocumentTypeRepo::SP &r) + : _repo(r) {} + }; + class MultiOperationReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class PutDocumentMessageFactory : public DocumentMessageFactory { + protected: + const document::DocumentTypeRepo &_repo; + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const { + return decodeMessage<PutDocumentMessage>(this, buf); + } + + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + public: + void decodeInto(PutDocumentMessage & msg, document::ByteBuffer & buf) const; + PutDocumentMessageFactory(const document::DocumentTypeRepo &r) + : _repo(r) {} + }; + class PutDocumentReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class RemoveDocumentMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const { + return decodeMessage<RemoveDocumentMessage>(this, buf); + } + + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + public: + void decodeInto(RemoveDocumentMessage & msg, document::ByteBuffer & buf) const; + }; + class RemoveDocumentReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class RemoveLocationMessageFactory : public DocumentMessageFactory { + const document::DocumentTypeRepo &_repo; + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + public: + RemoveLocationMessageFactory(const document::DocumentTypeRepo &r) + : _repo(r) {} + }; + class RemoveLocationReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class SearchResultMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + class SearchResultReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class StatBucketMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + class StatBucketReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class StatDocumentMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + class StatDocumentReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class UpdateDocumentMessageFactory : public DocumentMessageFactory { + protected: + const document::DocumentTypeRepo &_repo; + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const { + return decodeMessage<UpdateDocumentMessage>(this, buf); + } + + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + public: + void decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const; + UpdateDocumentMessageFactory(const document::DocumentTypeRepo &r) + : _repo(r) {} + }; + class UpdateDocumentReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class VisitorInfoMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + class VisitorInfoReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class WrongDistributionReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + class QueryResultMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + class QueryResultReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + + /////////////////////////////////////////////////////////////////////////// + // + // Utilities + // + /////////////////////////////////////////////////////////////////////////// + + /** + * This is a complement for the vespalib::GrowableByteBuffer.putString() method. + * + * @param in The byte buffer to read from. + * @return The decoded string. + */ + static string decodeString(document::ByteBuffer &in) + { return RoutableFactories42::decodeString(in); } + + /** + * This is a complement for the vespalib::GrowableByteBuffer.putBoolean() method. + * + * @param in The byte buffer to read from. + * @return The decoded bool. + */ + static bool decodeBoolean(document::ByteBuffer &in) + { return RoutableFactories42::decodeBoolean(in); } + + /** + * Convenience method to decode a 32-bit int from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded int. + */ + static int32_t decodeInt(document::ByteBuffer &in) + { return RoutableFactories42::decodeInt(in); } + + /** + * Convenience method to decode a 64-bit int from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded int. + */ + static int64_t decodeLong(document::ByteBuffer &in) + { return RoutableFactories42::decodeLong(in); } + + + /** + * Convenience method to decode a document id from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded document id. + */ + static document::DocumentId decodeDocumentId(document::ByteBuffer &in) + { return RoutableFactories42::decodeDocumentId(in); } + + /** + * Convenience method to encode a document id to the given byte buffer. + * + * @param id The document id to encode. + * @param out The byte buffer to write to. + */ + static void encodeDocumentId(const document::DocumentId &id, + vespalib::GrowableByteBuffer &out) + { return RoutableFactories42::encodeDocumentId(id, out); } +}; + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories51.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories51.cpp new file mode 100644 index 00000000000..2cddda2eecd --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories51.cpp @@ -0,0 +1,168 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".routablefactories51"); + +#include "routablefactories51.h" +#include <vespa/document/document.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/vespalib/objects/nbostream.h> + +using vespalib::nbostream; + +namespace documentapi { + +bool +RoutableFactories51::DocumentMessageFactory::encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const +{ + const DocumentMessage &msg = static_cast<const DocumentMessage&>(obj); + out.putByte(msg.getPriority()); + out.putInt(msg.getLoadType().getId()); + return doEncode(msg, out); +} + +mbus::Routable::UP +RoutableFactories51::DocumentMessageFactory::decode(document::ByteBuffer &in, + const LoadTypeSet& loadTypes) const +{ + uint8_t pri; + in.getByte(pri); + uint32_t loadClass = decodeInt(in); + + DocumentMessage::UP msg = doDecode(in); + if (msg.get() != NULL) { + msg->setPriority((Priority::Value)pri); + msg->setLoadType(loadTypes[loadClass]); + } + + return mbus::Routable::UP(msg.release()); +} + +bool +RoutableFactories51::DocumentReplyFactory::encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const +{ + const DocumentReply &msg = static_cast<const DocumentReply&>(obj); + out.putByte(msg.getPriority()); + return doEncode(msg, out); +} + +mbus::Routable::UP +RoutableFactories51::DocumentReplyFactory::decode(document::ByteBuffer &in, const LoadTypeSet&) const +{ + uint8_t pri; + in.getByte(pri); + DocumentReply::UP reply = doDecode(in); + if (reply.get() != NULL) { + reply->setPriority((Priority::Value)pri); + } + return mbus::Routable::UP(reply.release()); +} + +DocumentMessage::UP +RoutableFactories51::CreateVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + DocumentMessage::UP ret(new CreateVisitorMessage()); + CreateVisitorMessage &msg = static_cast<CreateVisitorMessage&>(*ret); + + msg.setLibraryName(decodeString(buf)); + msg.setInstanceId(decodeString(buf)); + msg.setControlDestination(decodeString(buf)); + msg.setDataDestination(decodeString(buf)); + msg.setDocumentSelection(decodeString(buf)); + msg.setMaximumPendingReplyCount(decodeInt(buf)); + + int32_t len = decodeInt(buf); + for (int32_t i = 0; i < len; i++) { + int64_t val; + buf.getLong(val); // NOT using getLongNetwork + msg.getBuckets().push_back(document::BucketId(val)); + } + + msg.setFromTimestamp(decodeLong(buf)); + msg.setToTimestamp(decodeLong(buf)); + msg.setVisitRemoves(decodeBoolean(buf)); + msg.setFieldSet(decodeString(buf)); + msg.setVisitInconsistentBuckets(decodeBoolean(buf)); + msg.getParameters().deserialize(_repo, buf); + msg.setVisitorDispatcherVersion(50); + msg.setVisitorOrdering((document::OrderingSpecification::Order)decodeInt(buf)); + msg.setMaxBucketsPerVisitor(decodeInt(buf)); + + return ret; +} + +bool +RoutableFactories51::CreateVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const CreateVisitorMessage &msg = static_cast<const CreateVisitorMessage&>(obj); + + buf.putString(msg.getLibraryName()); + buf.putString(msg.getInstanceId()); + buf.putString(msg.getControlDestination()); + buf.putString(msg.getDataDestination()); + buf.putString(msg.getDocumentSelection()); + buf.putInt(msg.getMaximumPendingReplyCount()); + buf.putInt(msg.getBuckets().size()); + + const std::vector<document::BucketId> &buckets = msg.getBuckets(); + for (std::vector<document::BucketId>::const_iterator it = buckets.begin(); + it != buckets.end(); ++it) + { + uint64_t val = it->getRawId(); + buf.putBytes((const char*)&val, 8); + } + + buf.putLong(msg.getFromTimestamp()); + buf.putLong(msg.getToTimestamp()); + buf.putBoolean(msg.visitRemoves()); + buf.putString(msg.getFieldSet()); + buf.putBoolean(msg.visitInconsistentBuckets()); + + int len = msg.getParameters().getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + msg.getParameters().serialize(dbuf); + + buf.putInt(msg.getVisitorOrdering()); + buf.putInt(msg.getMaxBucketsPerVisitor()); + + return true; +} + +DocumentMessage::UP +RoutableFactories51::GetDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + return DocumentMessage::UP( + new GetDocumentMessage(decodeDocumentId(buf), + decodeString(buf))); +} + +bool +RoutableFactories51::GetDocumentMessageFactory::doEncode(const DocumentMessage &obj, + vespalib::GrowableByteBuffer &buf) const +{ + const GetDocumentMessage &msg = static_cast<const GetDocumentMessage&>(obj); + + encodeDocumentId(msg.getDocumentId(), buf); + buf.putString(msg.getFieldSet()); + return true; +} + +DocumentReply::UP +RoutableFactories51::DocumentIgnoredReplyFactory::doDecode(document::ByteBuffer& buf) const +{ + (void) buf; + return DocumentReply::UP(new DocumentIgnoredReply()); +} + +bool +RoutableFactories51::DocumentIgnoredReplyFactory::doEncode( + const DocumentReply& obj, + vespalib::GrowableByteBuffer& buf) const +{ + (void) obj; + (void) buf; + return true; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h new file mode 100644 index 00000000000..7dc51e45a4f --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h @@ -0,0 +1,222 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "routablefactories50.h" +#include <vespa/document/base/documentid.h> +#include <vespa/document/util/bytebuffer.h> +#include <vespa/documentapi/messagebus/messages/feedmessage.h> +#include <vespa/documentapi/messagebus/messages/feedreply.h> +#include <vespa/messagebus/routable.h> +#include <vespa/messagebus/blob.h> +#include <vespa/messagebus/blobref.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + +namespace document { class DocumentTypeRepo; } + +namespace documentapi { +/** + * This class encapsulates all the {@link RoutableFactory} classes needed to implement factories for the document + * routable. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. + */ +class RoutableFactories51 { +public: + RoutableFactories51() = delete; + + /** + * Implements the shared factory logic required for {@link DocumentMessage} objects, and it offers a more + * convenient interface for implementing {@link RoutableFactory}. + */ + class DocumentMessageFactory : public IRoutableFactory { + protected: + /** + * This method encodes the given message into the given byte buffer. You are guaranteed to only receive messages of + * the type that this factory was registered for. + * + * This method is NOT exception safe. Return false to signal failure. + * + * @param msg The message to encode. + * @param buf The byte buffer to write to. + * @return True if the message was encoded. + */ + virtual bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const = 0; + + /** + * This method decodes a message from the given byte buffer. You are guaranteed to only receive byte buffers + * generated by a previous call to {@link #doEncode(DocumentMessage, GrowableByteBuffer)}. + * + * This method is NOT exception safe. Return null to signal failure. + * + * @param buf The byte buffer to read from. + * @return The decoded message. + */ + virtual DocumentMessage::UP doDecode(document::ByteBuffer &buf) const = 0; + + public: + /** + * Convenience typedefs. + */ + typedef std::unique_ptr<IRoutableFactory> UP; + typedef std::shared_ptr<IRoutableFactory> SP; + + // Implements IRoutableFactory. + bool encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const; + + // Implements IRoutableFactory. + mbus::Routable::UP decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const; + }; + + /** + * Implements the shared factory logic required for {@link DocumentReply} objects, and it offers a more + * convenient interface for implementing {@link RoutableFactory}. + */ + class DocumentReplyFactory : public IRoutableFactory { + protected: + /** + * This method encodes the given reply into the given byte buffer. You are guaranteed to only receive + * replies of the type that this factory was registered for. + * + * This method is NOT exception safe. Return false to signal failure. + * + * @param reply The reply to encode. + * @param buf The byte buffer to write to. + * @return True if the message was encoded. + */ + virtual bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const = 0; + + /** + * This method decodes a reply from the given byte buffer. You are guaranteed to only receive byte buffers + * generated by a previous call to {@link #doEncode(DocumentReply, GrowableByteBuffer)}. + * + * This method is NOT exception safe. Return null to signal failure. + * + * @param buf The byte buffer to read from. + * @return The decoded reply. + */ + virtual DocumentReply::UP doDecode(document::ByteBuffer &buf) const = 0; + + public: + /** + * Convenience typedefs. + */ + typedef std::unique_ptr<IRoutableFactory> UP; + typedef std::shared_ptr<IRoutableFactory> SP; + + // Implements IRoutableFactory. + bool encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const; + + // Implements IRoutableFactory. + mbus::Routable::UP decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const; + }; + + class DocumentIgnoredReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + }; + + /** + * Implements a helper class to do feed message factories. + */ + class FeedMessageFactory : public DocumentMessageFactory { + protected: + void myDecode(FeedMessage &msg, document::ByteBuffer &buf) const; + void myEncode(const FeedMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + + /** + * Implements a helper class to do feed reply factories. + */ + class FeedReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const; + virtual uint32_t getType() const = 0; + }; + + //////////////////////////////////////////////////////////////////////////////// + // + // Factories + // + //////////////////////////////////////////////////////////////////////////////// + + class CreateVisitorMessageFactory : public DocumentMessageFactory { + const document::DocumentTypeRepo &_repo; + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + public: + CreateVisitorMessageFactory(const document::DocumentTypeRepo &r) + : _repo(r) {} + }; + + class GetDocumentMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const; + }; + + /////////////////////////////////////////////////////////////////////////// + // + // Utilities + // + /////////////////////////////////////////////////////////////////////////// + protected: + /** + * This is a complement for the vespalib::GrowableByteBuffer.putString() method. + * + * @param in The byte buffer to read from. + * @return The decoded string. + */ + static string decodeString(document::ByteBuffer &in) + { return RoutableFactories50::decodeString(in); } + + /** + * This is a complement for the vespalib::GrowableByteBuffer.putBoolean() method. + * + * @param in The byte buffer to read from. + * @return The decoded bool. + */ + static bool decodeBoolean(document::ByteBuffer &in) + { return RoutableFactories50::decodeBoolean(in); } + + /** + * Convenience method to decode a 32-bit int from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded int. + */ + static int32_t decodeInt(document::ByteBuffer &in) + { return RoutableFactories50::decodeInt(in); } + + /** + * Convenience method to decode a 64-bit int from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded int. + */ + static int64_t decodeLong(document::ByteBuffer &in) + { return RoutableFactories50::decodeLong(in); } + + + /** + * Convenience method to decode a document id from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded document id. + */ + static document::DocumentId decodeDocumentId(document::ByteBuffer &in) + { return RoutableFactories50::decodeDocumentId(in); } + + /** + * Convenience method to encode a document id to the given byte buffer. + * + * @param id The document id to encode. + * @param out The byte buffer to write to. + */ + static void encodeDocumentId(const document::DocumentId &id, + vespalib::GrowableByteBuffer &out) + { return RoutableFactories50::encodeDocumentId(id, out); } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories52.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories52.cpp new file mode 100644 index 00000000000..491be5d30f2 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories52.cpp @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".routablefactories52"); + +#include "routablefactories52.h" +#include <vespa/document/document.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <memory> + +using vespalib::nbostream; +using std::make_shared; +using std::make_unique; + +namespace documentapi { + +void +RoutableFactories52::PutDocumentMessageFactory::decodeInto(PutDocumentMessage & msg, document::ByteBuffer & buf) const { + super::decodeInto(msg, buf); + decodeTasCondition(msg, buf); +} + +bool +RoutableFactories52::PutDocumentMessageFactory::doEncode(const DocumentMessage & msg, vespalib::GrowableByteBuffer & buf) const +{ + if (! super::doEncode(msg, buf)) { + return false; + } + + encodeTasCondition(buf, msg); + return true; +} + +void +RoutableFactories52::RemoveDocumentMessageFactory::decodeInto(RemoveDocumentMessage & msg, document::ByteBuffer & buf) const { + super::decodeInto(msg, buf); + decodeTasCondition(msg, buf); +} + + +bool +RoutableFactories52::RemoveDocumentMessageFactory::doEncode(const DocumentMessage & msg, vespalib::GrowableByteBuffer & buf) const +{ + if (! super::doEncode(msg, buf)) { + return false; + } + + encodeTasCondition(buf, msg); + return true; +} + +void +RoutableFactories52::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const { + super::decodeInto(msg, buf); + decodeTasCondition(msg, buf); +} + +bool +RoutableFactories52::UpdateDocumentMessageFactory::doEncode(const DocumentMessage & msg, vespalib::GrowableByteBuffer & buf) const +{ + if (! super::doEncode(msg, buf)) { + return false; + } + + encodeTasCondition(buf, msg); + return true; +} + +void RoutableFactories52::decodeTasCondition(DocumentMessage & docMsg, document::ByteBuffer & buf) { + auto & msg = static_cast<TestAndSetMessage &>(docMsg); + msg.setCondition(TestAndSetCondition(decodeString(buf))); +} + +void RoutableFactories52::encodeTasCondition(vespalib::GrowableByteBuffer & buf, const DocumentMessage & docMsg) { + auto & msg = static_cast<const TestAndSetMessage &>(docMsg); + buf.putString(msg.getCondition().getSelection()); +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories52.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories52.h new file mode 100644 index 00000000000..281ab899aa6 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories52.h @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell +#pragma once + +#include "routablefactories51.h" +#include <vespa/document/base/documentid.h> +#include <vespa/document/util/bytebuffer.h> +#include <vespa/documentapi/messagebus/messages/feedmessage.h> +#include <vespa/documentapi/messagebus/messages/feedreply.h> +#include <vespa/messagebus/routable.h> +#include <vespa/messagebus/blob.h> +#include <vespa/messagebus/blobref.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/documentapi/messagebus/messages/testandsetmessage.h> + +namespace document { class DocumentTypeRepo; } + +namespace documentapi { +/** + * This class encapsulates all the {@link RoutableFactory} classes needed to implement factories for the document + * routable. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. + */ +class RoutableFactories52 : public RoutableFactories51 { +public: + RoutableFactories52() = delete; + + class PutDocumentMessageFactory : public RoutableFactories50::PutDocumentMessageFactory { + using super = RoutableFactories50::PutDocumentMessageFactory; + protected: + DocumentMessage::UP doDecode(document::ByteBuffer & buf) const override { + return decodeMessage<PutDocumentMessage>(this, buf); + } + + bool doEncode(const DocumentMessage & msg, vespalib::GrowableByteBuffer & buf) const override; + public: + void decodeInto(PutDocumentMessage & msg, document::ByteBuffer & buf) const; + PutDocumentMessageFactory(const document::DocumentTypeRepo & r) : super::PutDocumentMessageFactory(r) {} + }; + + class RemoveDocumentMessageFactory : public RoutableFactories50::RemoveDocumentMessageFactory { + using super = RoutableFactories50::RemoveDocumentMessageFactory; + protected: + DocumentMessage::UP doDecode(document::ByteBuffer & buf) const override { + return decodeMessage<RemoveDocumentMessage>(this, buf); + } + + bool doEncode(const DocumentMessage & msg, vespalib::GrowableByteBuffer & buf) const; + public: + void decodeInto(RemoveDocumentMessage & msg, document::ByteBuffer & buf) const; + }; + + class UpdateDocumentMessageFactory : public RoutableFactories50::UpdateDocumentMessageFactory { + using super = RoutableFactories50::UpdateDocumentMessageFactory; + protected: + DocumentMessage::UP doDecode(document::ByteBuffer & buf) const override { + return decodeMessage<UpdateDocumentMessage>(this, buf); + } + + bool doEncode(const DocumentMessage & msg, vespalib::GrowableByteBuffer & buf) const; + public: + void decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const; + UpdateDocumentMessageFactory(const document::DocumentTypeRepo & r) : super::UpdateDocumentMessageFactory(r) {} + }; + + static void decodeTasCondition(DocumentMessage & docMsg, document::ByteBuffer & buf); + static void encodeTasCondition(vespalib::GrowableByteBuffer & buf, const DocumentMessage & docMsg); +}; + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp new file mode 100644 index 00000000000..dfd959ba2dd --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routablerepository.cpp @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".routablerepository"); + +#include "routablerepository.h" +#include <vespa/documentapi/loadtypes/loadtypeset.h> +#include <vespa/vespalib/util/exceptions.h> + +namespace documentapi { + +RoutableRepository::VersionMap::VersionMap() : + _factoryVersions() +{ + // empty +} + +bool +RoutableRepository::VersionMap::putFactory(const vespalib::VersionSpecification &version, + IRoutableFactory::SP factory) +{ + bool ret = _factoryVersions.find(version) != _factoryVersions.end(); + _factoryVersions[version] = factory; + return ret; + +} + +IRoutableFactory::SP +RoutableRepository::VersionMap::getFactory(const vespalib::Version &version) const +{ + const vespalib::VersionSpecification versionSpec{version.getMajor(), version.getMinor(), version.getMicro()}; + + std::vector< std::pair<vespalib::VersionSpecification, IRoutableFactory::SP> > candidates; + for (auto & entry : _factoryVersions) { + if (entry.first.compareTo(versionSpec) <= 0) { + candidates.push_back(std::make_pair(entry.first, entry.second)); + } + } + if (candidates.empty()) { + return IRoutableFactory::SP(); + } + + return std::max_element(candidates.begin(), candidates.end(), [](auto & lhs, auto & rhs) { return lhs.first.compareTo(rhs.first) <= 0; })->second; +} + +RoutableRepository::RoutableRepository(const LoadTypeSet& loadTypes) : + _lock(), + _factoryTypes(), + _cache(), + _loadTypes(loadTypes) +{ + // empty +} + +mbus::Routable::UP +RoutableRepository::decode(const vespalib::Version &version, mbus::BlobRef data) const +{ + if (data.size() == 0) { + LOG(error, "Received empty byte array for deserialization."); + return mbus::Routable::UP(); + } + + document::ByteBuffer in(data.data(), data.size()); + int type; + in.getIntNetwork(type); + IRoutableFactory::SP factory = getFactory(version, type); + if (factory.get() == NULL) { + LOG(error, "No routable factory found for routable type %d (version %s).", + type, version.toString().c_str()); + return mbus::Routable::UP(); + } + mbus::Routable::UP ret = factory->decode(in, _loadTypes); + if (ret.get() == NULL) { + LOG(error, "Routable factory failed to deserialize routable of type %d (version %s).", + type, version.toString().c_str()); + + std::ostringstream ost; + document::StringUtil::printAsHex(ost, data.data(), data.size()); + LOG(error, "%s", ost.str().c_str()); + return mbus::Routable::UP(); + } + return ret; +} + +mbus::Blob +RoutableRepository::encode(const vespalib::Version &version, const mbus::Routable &obj) const +{ + uint32_t type = obj.getType(); + + IRoutableFactory::SP factory = getFactory(version, type); + if (factory.get() == NULL) { + LOG(error, "No routable factory found for routable type %d (version %s).", + type, version.toString().c_str()); + return mbus::Blob(0); + } + vespalib::GrowableByteBuffer out; + out.putInt(obj.getType()); + if (!factory->encode(obj, out)) { + LOG(error, "Routable factory failed to serialize routable of type %d (version %s).", + type, version.toString().c_str()); + return mbus::Blob(0); + } + mbus::Blob ret(out.position()); + memcpy(ret.data(), out.getBuffer(), out.position()); + return ret; +} + +void +RoutableRepository::putFactory(const vespalib::VersionSpecification &version, + uint32_t type, IRoutableFactory::SP factory) +{ + vespalib::LockGuard guard(_lock); + if (_factoryTypes[type].putFactory(version, factory)) { + _cache.clear(); + } +} + +IRoutableFactory::SP +RoutableRepository::getFactory(const vespalib::Version &version, uint32_t type) const +{ + vespalib::LockGuard guard(_lock); + CacheKey cacheKey(version, type); + FactoryCache::const_iterator cit = _cache.find(cacheKey); + if (cit != _cache.end()) { + return cit->second; + } + TypeMap::const_iterator vit = _factoryTypes.find(type); + if (vit == _factoryTypes.end()) { + return IRoutableFactory::SP(); + } + IRoutableFactory::SP factory = vit->second.getFactory(version); + if (factory.get() == NULL) { + return IRoutableFactory::SP(); + } + _cache[cacheKey] = factory; + return factory; +} + +uint32_t +RoutableRepository::getRoutableTypes(const vespalib::Version &version, std::vector<uint32_t> &out) const +{ + vespalib::LockGuard guard(_lock); + for (TypeMap::const_iterator it = _factoryTypes.begin(); + it != _factoryTypes.end(); ++it) + { + if (it->second.getFactory(version).get() != NULL) { + out.push_back(it->first); + } + } + return _factoryTypes.size(); +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablerepository.h b/documentapi/src/vespa/documentapi/messagebus/routablerepository.h new file mode 100644 index 00000000000..e39eab89507 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routablerepository.h @@ -0,0 +1,105 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <map> +#include <vespa/messagebus/blobref.h> +#include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/component/versionspecification.h> +#include "iroutablefactory.h" + +namespace documentapi { + +class LoadTypeSet; + +/** + * This class encapsulates the logic required to map routable type and version to a corresponding {@link + * RoutableFactory}. It is owned and accessed through a {@link DocumentProtocol} instance. This class uses a + * factory cache to reduce the latency of matching version specifications to actual versions when resolving + * factories. + */ +class RoutableRepository : public boost::noncopyable { +private: + /** + * Internal helper class that implements a map from {@link VersionSpecification} to {@link + * RoutableFactory}. + */ + class VersionMap { + private: + std::map<vespalib::VersionSpecification, IRoutableFactory::SP> _factoryVersions; + + public: + VersionMap(); + bool putFactory(const vespalib::VersionSpecification &version, IRoutableFactory::SP factory); + IRoutableFactory::SP getFactory(const vespalib::Version &version) const; + }; + + typedef std::pair<vespalib::Version, uint32_t> CacheKey; + typedef std::map<CacheKey, IRoutableFactory::SP> FactoryCache; + typedef std::map<uint32_t, VersionMap> TypeMap; + + vespalib::Lock _lock; + TypeMap _factoryTypes; + mutable FactoryCache _cache; + const LoadTypeSet& _loadTypes; + +public: + /** + * Constructs a new routable repository. + */ + RoutableRepository(const LoadTypeSet& loadTypes); + + /** + * Decodes a {@link Routable} from the given byte array. This uses the content of the byte array to + * dispatch the decode request to the appropriate {@link RoutableFactory} that was previously registered. + * + * If a routable can not be decoded, this method returns an empty blob. + * + * @param version The version of the encoded routable. + * @param data The byte array containing the encoded routable. + * @return The decoded routable. + */ + mbus::Routable::UP decode(const vespalib::Version &version, mbus::BlobRef data) const; + + /** + * Encodes a {@link Routable} into a byte array. This dispatches the encode request to the appropriate + * {@link RoutableFactory} that was previously registered. + * + * If a routable can not be encoded, this method returns an empty byte array. + * + * @param version The version to encode the routable as. + * @param obj The routable to encode. + * @return The byte array containing the encoded routable. + */ + mbus::Blob encode(const vespalib::Version &version, const mbus::Routable &obj) const; + + /** + * Registers a routable factory for a given version and routable type. + * + * @param version The version specification that the given factory supports. + * @param type The routable type that the given factory supports. + * @param factory The routable factory to register. + */ + void putFactory(const vespalib::VersionSpecification &version, + uint32_t type, IRoutableFactory::SP factory); + + /** + * Returns the routable factory for a given version and routable type. + * + * @param version The version that the factory must support. + * @param type The routable type that the factory must support. + * @return The routable factory matching the criteria, or null. + */ + IRoutableFactory::SP getFactory(const vespalib::Version &version, uint32_t type) const; + + /** + * Returns a list of routable types that support the given version. + * + * @param version The version to return types for. + * @param out The list to write to. + * @return The number of supported types. + */ + uint32_t getRoutableTypes(const vespalib::Version &version, std::vector<uint32_t> &out) const; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.cpp b/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.cpp new file mode 100644 index 00000000000..902bf006622 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.cpp @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/messagebus/policies/andpolicy.h> +#include <vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.h> +#include <vespa/documentapi/messagebus/policies/errorpolicy.h> +#include <vespa/documentapi/messagebus/policies/externpolicy.h> +#include <vespa/documentapi/messagebus/policies/localservicepolicy.h> +#include <vespa/documentapi/messagebus/policies/roundrobinpolicy.h> +#include <vespa/documentapi/messagebus/policies/searchcolumnpolicy.h> +#include <vespa/documentapi/messagebus/policies/searchrowpolicy.h> +#include <vespa/documentapi/messagebus/policies/subsetservicepolicy.h> +#include <vespa/documentapi/messagebus/policies/storagepolicy.h> +#include <vespa/documentapi/messagebus/policies/contentpolicy.h> +#include <vespa/documentapi/messagebus/policies/messagetypepolicy.h> +#include <vespa/documentapi/messagebus/policies/loadbalancerpolicy.h> +#include "routingpolicyfactories.h" + +using namespace documentapi; + +mbus::IRoutingPolicy::UP +RoutingPolicyFactories::AndPolicyFactory::createPolicy(const string ¶m) const +{ + return mbus::IRoutingPolicy::UP(new ANDPolicy(param)); +} + +mbus::IRoutingPolicy::UP +RoutingPolicyFactories::StoragePolicyFactory::createPolicy(const string ¶m) const +{ + mbus::IRoutingPolicy::UP ret(new StoragePolicy(param)); + string error = static_cast<StoragePolicy&>(*ret).getError(); + if (!error.empty()) { + ret.reset(new ErrorPolicy(error)); + } + return ret; +} + +mbus::IRoutingPolicy::UP +RoutingPolicyFactories::MessageTypePolicyFactory::createPolicy(const string ¶m) const +{ + return mbus::IRoutingPolicy::UP(new MessageTypePolicy(param)); +} + +mbus::IRoutingPolicy::UP +RoutingPolicyFactories::ContentPolicyFactory::createPolicy(const string ¶m) const +{ + mbus::IRoutingPolicy::UP ret(new ContentPolicy(param)); + string error = static_cast<ContentPolicy&>(*ret).getError(); + if (!error.empty()) { + ret.reset(new ErrorPolicy(error)); + } + return ret; +} + +mbus::IRoutingPolicy::UP +RoutingPolicyFactories::LoadBalancerPolicyFactory::createPolicy(const string ¶m) const +{ + mbus::IRoutingPolicy::UP ret(new LoadBalancerPolicy(param)); + string error = static_cast<LoadBalancerPolicy&>(*ret).getError(); + if (!error.empty()) { + fprintf(stderr, "Got error %s\n", error.c_str()); + ret.reset(new ErrorPolicy(error)); + } + return ret; +} + +RoutingPolicyFactories::DocumentRouteSelectorPolicyFactory:: +DocumentRouteSelectorPolicyFactory(const document::DocumentTypeRepo &repo, + const string &configId) : + _repo(repo), + _configId(configId) +{ + // empty +} + +mbus::IRoutingPolicy::UP +RoutingPolicyFactories::DocumentRouteSelectorPolicyFactory::createPolicy(const string ¶m) const +{ + mbus::IRoutingPolicy::UP ret(new DocumentRouteSelectorPolicy( + _repo, param.empty() ? _configId : param)); + string error = static_cast<DocumentRouteSelectorPolicy&>(*ret).getError(); + if (!error.empty()) { + ret.reset(new ErrorPolicy(error)); + } + return ret; +} + +mbus::IRoutingPolicy::UP +RoutingPolicyFactories::ExternPolicyFactory::createPolicy(const string ¶m) const +{ + mbus::IRoutingPolicy::UP ret(new ExternPolicy(param)); + string error = static_cast<ExternPolicy&>(*ret).getError(); + if (!error.empty()) { + ret.reset(new ErrorPolicy(error)); + } + return ret; +} + +mbus::IRoutingPolicy::UP +RoutingPolicyFactories::LocalServicePolicyFactory::createPolicy(const string ¶m) const +{ + return mbus::IRoutingPolicy::UP(new LocalServicePolicy(param)); +} + +mbus::IRoutingPolicy::UP +RoutingPolicyFactories::RoundRobinPolicyFactory::createPolicy(const string ¶m) const +{ + return mbus::IRoutingPolicy::UP(new RoundRobinPolicy(param)); +} + +mbus::IRoutingPolicy::UP +RoutingPolicyFactories::SearchColumnPolicyFactory::createPolicy(const string ¶m) const +{ + return mbus::IRoutingPolicy::UP(new SearchColumnPolicy(param)); +} + +mbus::IRoutingPolicy::UP +RoutingPolicyFactories::SearchRowPolicyFactory::createPolicy(const string ¶m) const +{ + return mbus::IRoutingPolicy::UP(new SearchRowPolicy(param)); +} + +mbus::IRoutingPolicy::UP +RoutingPolicyFactories::SubsetServicePolicyFactory::createPolicy(const string ¶m) const +{ + return mbus::IRoutingPolicy::UP(new SubsetServicePolicy(param)); +} diff --git a/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.h b/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.h new file mode 100644 index 00000000000..75e9ef1ab18 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routingpolicyfactories.h @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/messagebus/systemstate/systemstate.h> +#include "iroutingpolicyfactory.h" + +namespace documentapi { + +class RoutingPolicyFactories { +private: + RoutingPolicyFactories() { /* abstract */ } + +public: + class AndPolicyFactory : public IRoutingPolicyFactory { + public: + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; + }; + class StoragePolicyFactory : public IRoutingPolicyFactory { + public: + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; + }; + class MessageTypePolicyFactory : public IRoutingPolicyFactory { + public: + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; + }; + class ContentPolicyFactory : public IRoutingPolicyFactory { + public: + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; + }; + class LoadBalancerPolicyFactory : public IRoutingPolicyFactory { + public: + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; + }; + class DocumentRouteSelectorPolicyFactory : public IRoutingPolicyFactory { + private: + const document::DocumentTypeRepo &_repo; + string _configId; + public: + DocumentRouteSelectorPolicyFactory( + const document::DocumentTypeRepo &repo, + const string &configId); + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; + }; + class ExternPolicyFactory : public IRoutingPolicyFactory { + public: + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; + }; + class LocalServicePolicyFactory : public IRoutingPolicyFactory { + public: + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; + }; + class RoundRobinPolicyFactory : public IRoutingPolicyFactory { + public: + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; + }; + class SearchColumnPolicyFactory : public IRoutingPolicyFactory { + public: + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; + }; + class SearchRowPolicyFactory : public IRoutingPolicyFactory { + public: + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; + }; + class SubsetServicePolicyFactory : public IRoutingPolicyFactory { + public: + mbus::IRoutingPolicy::UP createPolicy(const string ¶m) const; + }; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.cpp b/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.cpp new file mode 100644 index 00000000000..a3d00c2e96d --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.cpp @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".routingpolicyrepository"); + +#include "routingpolicyrepository.h" + +namespace documentapi { + +RoutingPolicyRepository::RoutingPolicyRepository() : + _lock(), + _factories() +{ + // empty +} + +void +RoutingPolicyRepository::putFactory(const string &name, IRoutingPolicyFactory::SP factory) +{ + vespalib::LockGuard guard(_lock); + _factories[name] = factory; +} + +IRoutingPolicyFactory::SP +RoutingPolicyRepository::getFactory(const string &name) const +{ + vespalib::LockGuard guard(_lock); + FactoryMap::const_iterator it = _factories.find(name); + if (it != _factories.end()) { + return it->second; + } + return IRoutingPolicyFactory::SP(); +} + +mbus::IRoutingPolicy::UP +RoutingPolicyRepository::createPolicy(const string &name, const string ¶m) const +{ + IRoutingPolicyFactory::SP factory = getFactory(name); + if (factory.get() == NULL) { + LOG(error, "No routing policy factory found for name '%s'.", name.c_str()); + return mbus::IRoutingPolicy::UP(); + } + mbus::IRoutingPolicy::UP ret = factory->createPolicy(param); + if (ret.get() == NULL) { + LOG(error, "Routing policy factory failed to create a routing policy for parameter '%s'.", + param.c_str()); + return mbus::IRoutingPolicy::UP(); + } + return ret; +} + +} diff --git a/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.h b/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.h new file mode 100644 index 00000000000..cd19d5f0676 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/routingpolicyrepository.h @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <boost/utility.hpp> +#include <map> +#include <string> +#include <vespa/vespalib/util/sync.h> +#include "iroutingpolicyfactory.h" + +namespace documentapi { + +class RoutingPolicyRepository : public boost::noncopyable { +private: + typedef std::map<string, IRoutingPolicyFactory::SP> FactoryMap; + + vespalib::Lock _lock; + FactoryMap _factories; + +public: + /** + * Constructs a new routing policy repository. + */ + RoutingPolicyRepository(); + + /** + * Registers a routing policy factory for a given name. + * + * @param name The name of the factory to register. + * @param factory The factory to register. + */ + void putFactory(const string &name, IRoutingPolicyFactory::SP factory); + + /** + * Returns the routing policy factory for a given name. + * + * @param name The name of the factory to return. + * @return The routing policy factory matching the criteria, or null. + */ + IRoutingPolicyFactory::SP getFactory(const string &name) const; + + /** + * Creates and returns a routing policy using the named factory and the given parameter. + * + * @param name The name of the factory to use. + * @param param The parameter to pass to the factory. + * @return The craeted policy. + */ + mbus::IRoutingPolicy::UP createPolicy(const string &name, const string ¶m) const; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/.gitignore b/documentapi/src/vespa/documentapi/messagebus/systemstate/.gitignore new file mode 100644 index 00000000000..5dae353d999 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/.gitignore @@ -0,0 +1,2 @@ +.depend +Makefile diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/systemstate/CMakeLists.txt new file mode 100644 index 00000000000..109e181a739 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(documentapi_documentapisystemstate OBJECT + SOURCES + nodestate.cpp + systemstate.cpp + systemstatehandle.cpp + urlencoder.cpp + DEPENDS +) diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.cpp new file mode 100644 index 00000000000..5d671fa90ec --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.cpp @@ -0,0 +1,237 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".nodestate"); + +#include <sstream> +#include "nodestate.h" +#include "urlencoder.h" + +using namespace documentapi; + +NodeState::NodeState() : + _parent(NULL), + _id(""), + _children(), + _state() +{ + // empty +} + +NodeState::NodeState(const NodeState &rhs) : + _parent(rhs._parent), + _id(rhs._id), + _children(rhs._children), + _state(rhs._state) +{ + // empty +} + +NodeState::NodeState(StateMap args) : + _parent(NULL), + _id(""), + _children(), + _state(args) +{ + // empty +} + +NodeState & +NodeState::addChild(const string &key, const NodeState &child) +{ + getChild(key, true)->copy(child); + return *this; +} + +NodeState * +NodeState::getChild(const string &key, bool force) +{ + if (key.empty()) { + return this; + } + + // Find first not-self location item. + size_t from = 0, to = key.find('/'); + while (to != string::npos && key.substr(from, to - from) == ".") { + from = to + 1; + to = key.find('/', from); + } + string arr0 = to != string::npos ? key.substr(from, to - from) : key.substr(from); + string arr1 = to != string::npos ? key.substr(to + 1) : ""; + + // Reference this or parent. + if (arr0 == ".") { + return this; + } + if (arr0 == "..") { + if (_parent == NULL) { + LOG(error, "Location string '%s' requests a parent above the top-most node, returning self to avoid crash.", + key.c_str()); + return this; + } + return _parent->getChild(arr1, force); + } + + // Look for child, forcing it if requested. + ChildMap::iterator it = _children.find(arr0); + if (it == _children.end()) { + if (!force) { + return NULL; + } + _children[arr0] = NodeState::SP(new NodeState()); + _children[arr0]->setParent(*this, arr0); + } + if (to != string::npos) { + return _children[arr0]->getChild(arr1, force); + } + return _children[arr0].get(); +} + +const NodeState::ChildMap & +NodeState::getChildren() const +{ + return _children; +} + +NodeState & +NodeState::removeChild(const string &key) +{ + if (key.empty()) { + return *this; + } + size_t pos = key.find_last_of('/'); + if (pos != string::npos) { + NodeState* parent = getChild(key.substr(0, pos), false); + if (parent != NULL) { + return parent->removeChild(key.substr(pos + 1)); + } + } + else { + _children.erase(key); + } + return compact(); +} + +const string +NodeState::getState(const string &key) +{ + if (key.empty()) { + return ""; + } + size_t pos = key.find_last_of('/'); + if (pos != string::npos) { + NodeState* parent = getChild(key.substr(0, pos), false); + return parent != NULL ? parent->getState(key.substr(pos + 1)) : ""; + } + StateMap::iterator it = _state.find(key); + return it != _state.end() ? it->second : ""; +} + +NodeState & +NodeState::setState(const string &key, const string &value) +{ + if (key.empty()) { + return *this; + } + size_t pos = key.find_last_of('/'); + if (pos != string::npos) { + getChild(key.substr(0, pos), true)->setState(key.substr(pos + 1), value); + } + else { + if (value.empty()) { + return removeState(key); + } + else { + _state[key] = value; + } + } + return *this; +} + +NodeState & +NodeState::removeState(const string &key) +{ + if (key.empty()) { + return *this; + } + size_t pos = key.find_last_of('/'); + if (pos != string::npos) { + NodeState* parent = getChild(key.substr(0, pos), false); + if (parent != NULL) { + return parent->removeState(key.substr(pos + 1)); + } + } + else { + _state.erase(key); + } + return compact(); +} + +NodeState & +NodeState::compact() +{ + if (_state.empty() && _children.empty()) { + if (_parent != NULL) { + return _parent->removeChild(_id); + } + } + return *this; +} + +NodeState & +NodeState::copy(const NodeState &node) +{ + for (StateMap::const_iterator it = node._state.begin(); + it != node._state.end(); ++it) { + _state[it->first] = it->second; + } + for (ChildMap::const_iterator it = node._children.begin(); + it != node._children.end(); ++it) { + getChild(it->first, true)->copy(*it->second); + } + return *this; +} + +NodeState & +NodeState::clear() +{ + _state.clear(); + _children.clear(); + return compact(); +} + +NodeState & +NodeState::setParent(NodeState &parent, const string &id) +{ + _parent = &parent; + _id = id; + return *this; +} + +const string +NodeState::toString() const +{ + const std::string ret = toString(""); + size_t pos = ret.find_last_not_of(' '); + return pos != string::npos ? ret.substr(0, pos + 1) : ret; +} + +const string +NodeState::toString(const string &prefix) const +{ + string ret; + if (!_state.empty()) { + string str; + for (StateMap::const_iterator it = _state.begin(); + it != _state.end(); ++it) { + str += it->first + "=" + URLEncoder::encode(it->second) + "&"; + } + ret += (prefix.empty() ? ".?" : prefix + "?") + str.substr(0, str.size() - 1) + " "; + } + string pre = prefix.empty() ? "" : (prefix + "/"); + for (ChildMap::const_iterator it = _children.begin(); + it != _children.end(); ++it) { + ret += it->second->toString(pre + URLEncoder::encode(it->first)); + } + return ret; +} diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.h new file mode 100644 index 00000000000..c9663fc302b --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/nodestate.h @@ -0,0 +1,166 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <map> +#include <vespa/documentapi/common.h> + + +namespace documentapi { + +/** + * A node state is a single node in an annotatet tree of such nodes. It contains a reference to its parent + * node, a list of named child nodes, as well as a mapping of (key, value) pairs that constitute the annotated + * state of this node. To create an instance of a node state tree, one can either use the {@link SystemState} + * factory class, or one can programmatically construct each node and use the chaining capabilities of its + * set-methods to compact the necessary code. + */ +class NodeState { +public: + typedef std::unique_ptr<NodeState> UP; + typedef std::shared_ptr<NodeState> SP; + typedef std::map<string, string> StateMap; + typedef std::map<string, NodeState::SP> ChildMap; + +private: + NodeState* _parent; + string _id; + ChildMap _children; + StateMap _state; + + /** + * Compacts the system state tree from this node upwards. This will delete itself if it has a parent, but + * no internal state and no children. + * + * @return This or the first non-null ancestor, to allow chaining. + */ + NodeState &compact(); + + /** + * Returns a string representation of this node state. + * + * @param prefix The prefix to use for this string. + * @return A string representation of this. + */ + const string toString(const string &prefix) const; + +public: + /** + * Creates a node state that no internal content. + */ + NodeState(); + + /** + * Creates a node state as a copy of another. + * + * @param rhs The state to copy. + */ + NodeState(const NodeState &rhs); + + /** + * Creates a node state based on a list of argument objects. These arguments are iterated and added to + * this node's internal state map. + * + * @param args The arguments to use as state. + */ + NodeState(StateMap args); + + /** + * Adds a child to this node at the given location. The key can be a location string, in which case the + * necessary intermediate node states are created. + * + * @param key The location at which to add the child. + * @param child The child node to add. + * @return This, to allow chaining. + */ + NodeState &addChild(const string &key, const NodeState &child); + + /** + * Returns the child at the given location relative to this. This method can be forced to return a child + * node even if it does not exist, by adding all intermediate nodes and the target node itself. + * + * @param key The location of the child to return. + * @param force Whether or not to force a return value by creating missing nodes. + * @return The child object, null if not found. + */ + NodeState *getChild(const string &key, bool force = false); + + /** + * Returns the map of child nodes for iteration. + * + * @return The internal child map. + */ + const ChildMap &getChildren() const; + + /** + * Removes the named child node from this node, and attempts to compact the system state from this node + * upwards by removing empty nodes. + * + * @param key The child to remove. + * @return The result of invoking {@link #compact} after the remove. + */ + NodeState &removeChild(const string &key); + + /** + * Retrieves some arbitrary state information for a given key. The key can be a location string, in which + * case the necessary intermediate nodes are traversed. If the key is not found, this method returns + * null. This method can not be const because it uses the non-const method {@link #getChild} to resolve a + * pathed key. + * + * @param key The name of the state information to return. + * @return The value of the state key. + */ + const string getState(const string &key); + + /** + * Sets some arbitrary state data in this node. The key can be a location string, in which case the + * necessary intermediate nodes are traversed and even created if missing. + * + * @param key The key to set. + * @param value The value to assign to the key. + * @return This, to allow chaining. + */ + NodeState &setState(const string &key, const string &value); + + /** + * Removes the named (key, value) state pair from this node, and attempts to compact the system state from + * this node upwards by removing empty nodes. + * + * @param key The state variable to clear. + * @return The result of invoking {@link #compact} after the remove. + */ + NodeState &removeState(const string &key); + + /** + * Copies the state content of another node state object into this. + * + * @param node The node state to copy into this. + * @return This, to allow chaining. + */ + NodeState ©(const NodeState &node); + + /** + * Clears both the internal state and child list, then compacts the tree from this node upwards. + * + * @return The result of invoking {@link #compact} after the remove. + */ + NodeState &clear(); + + /** + * Sets the parent of this node. + * + * @param parent The parent node. + * @param id The identifier of this node as seen in the parent. + * @return This, to allow chaining. + */ + NodeState &setParent(NodeState &parent, const string &id); + + /** + * Returns a string representation of this node state. + * + * @return A string representation of this. + */ + const string toString() const; +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp new file mode 100644 index 00000000000..3177a9f6a98 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp @@ -0,0 +1,303 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".systemstate"); + +#include <boost/spirit/include/classic_core.hpp> +#include <boost/spirit/include/classic_parse_tree.hpp> +#include <boost/spirit/include/classic_tree_to_xml.hpp> +#include <boost/spirit/include/classic_chset.hpp> +#include <boost/spirit/include/classic_escape_char.hpp> +#include <boost/spirit/include/classic_grammar_def.hpp> +#include <vespa/vespalib/util/stringfmt.h> +#include "systemstate.h" + +using namespace documentapi; + +/** + * This class implements a boost::spirit type parser for the system state string. All contained names + * confirm to the boost::spirit naming convention, and care should therefore be taken if one wishes + * to modify any of these. Note that all content is inlined, just as all of boost::spirit, so this is + * therefore contained in the .cpp file instead of a separate .h file. + */ +struct SystemStateGrammar : public boost::spirit::classic::grammar<SystemStateGrammar> { + enum RuleId { + id_hexChar = 1, + id_hexCode, + id_alphaNum, + id_string, + id_argument, + id_argumentList, + id_locationItem, + id_location, + id_systemState + }; + + template <typename Scanner> + struct gram_base { + typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_hexChar> > rule_hexChar; + typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_hexCode> > rule_hexCode; + typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_alphaNum> > rule_alphaNum; + typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_string> > rule_string; + typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_argument> > rule_argument; + typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_argumentList> > rule_argumentList; + typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_locationItem> > rule_locationItem; + typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_location> > rule_location; + typedef typename boost::spirit::classic::rule<Scanner, boost::spirit::classic::parser_tag<id_systemState> > rule_systemState; + typedef boost::spirit::classic::grammar_def<rule_systemState> type; + }; + + template <typename Scanner> + struct definition : gram_base<Scanner>::type { + typename gram_base<Scanner>::rule_hexChar _hexChar; + typename gram_base<Scanner>::rule_hexCode _hexCode; + typename gram_base<Scanner>::rule_alphaNum _alphaNum; + typename gram_base<Scanner>::rule_string _string; + typename gram_base<Scanner>::rule_argument _argument; + typename gram_base<Scanner>::rule_argumentList _argumentList; + typename gram_base<Scanner>::rule_locationItem _locationItem; + typename gram_base<Scanner>::rule_location _location; + typename gram_base<Scanner>::rule_systemState _systemState; + + definition(const SystemStateGrammar &) : + _hexChar(), + _hexCode(), + _alphaNum(), + _string(), + _argument(), + _argumentList(), + _locationItem(), + _location(), + _systemState() { + _hexChar = ( boost::spirit::classic::chset<>("A-Fa-f0-9") ); + _hexCode = ( boost::spirit::classic::ch_p('%') >> _hexChar >> _hexChar); + _alphaNum = ( boost::spirit::classic::chset<>("A-Za-z0-9") | + boost::spirit::classic::ch_p('-') | boost::spirit::classic::ch_p('.') | + boost::spirit::classic::ch_p('_') | boost::spirit::classic::ch_p('~') ); + _string = ( +( boost::spirit::classic::ch_p('+') | _hexCode | _alphaNum ) ); + _argument = ( _string >> boost::spirit::classic::ch_p('=') >> _string ); + _argumentList = ( _argument >> *( boost::spirit::classic::ch_p('&') >> _argument ) ); + _locationItem = ( boost::spirit::classic::str_p("..") | boost::spirit::classic::ch_p('.') | _string ); + _location = ( !boost::spirit::classic::ch_p('/') + >> _locationItem + >> *( boost::spirit::classic::ch_p('/') >> _locationItem ) + >> !boost::spirit::classic::ch_p('/') ); + _systemState = ( +( *boost::spirit::classic::space_p >> + _location >> !( boost::spirit::classic::ch_p('?') >> _argumentList ) ) ); + this->start_parsers(_systemState); + } + }; +}; + +template<typename T> void +debugNode(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node, const string &prefix = "") +{ + std::map<boost::spirit::classic::parser_id, string> names; + names[boost::spirit::classic::parser_id(grammar.id_hexChar)] = "hexChar"; + names[boost::spirit::classic::parser_id(grammar.id_hexCode)] = "hexCode"; + names[boost::spirit::classic::parser_id(grammar.id_alphaNum)] = "alphaNum"; + names[boost::spirit::classic::parser_id(grammar.id_string)] = "string"; + names[boost::spirit::classic::parser_id(grammar.id_argument)] = "argument"; + names[boost::spirit::classic::parser_id(grammar.id_argumentList)] = "argumentList"; + names[boost::spirit::classic::parser_id(grammar.id_locationItem)] = "locationItem"; + names[boost::spirit::classic::parser_id(grammar.id_location)] = "location"; + names[boost::spirit::classic::parser_id(grammar.id_systemState)] = "systemState"; + + std::cout << prefix << names[node.value.id()] << ": " << string(node.value.begin(), node.value.end()) << std::endl; + for (size_t i = 0; i < node.children.size(); i++) { + debugNode(grammar, node.children[i], vespalib::make_string("%s %d.", prefix.c_str(), (int)i)); + } +} + +template<typename T> string +parseHexChar(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) +{ + assert(node.value.id().to_long() == grammar.id_hexChar); + assert(node.children.size() == 1); + assert(node.children[0].value.id().to_long() == grammar.id_hexChar); + (void) grammar; + return string(node.children[0].value.begin(), node.children[0].value.end()); +} + +template<typename T> char +parseHexCode(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) +{ + assert(node.value.id().to_long() == grammar.id_hexCode); + assert(node.children.size() == 3); + assert(node.children[0].value.id().to_long() == grammar.id_hexCode); + assert(node.children[1].value.id().to_long() == grammar.id_hexChar); + assert(node.children[2].value.id().to_long() == grammar.id_hexChar); + string enc = parseHexChar(grammar, node.children[1]) + parseHexChar(grammar, node.children[2]); + return (char)strtol(enc.c_str(), NULL, 16); +} + +template<typename T> string +parseAlphaNum(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) +{ + assert(node.value.id().to_long() == grammar.id_alphaNum); + assert(node.children.size() == 1); + assert(node.children[0].value.id().to_long() == grammar.id_alphaNum); + (void) grammar; + return string(node.children[0].value.begin(), node.children[0].value.end()); +} + +template<typename T> string +parseString(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) +{ + assert(node.value.id().to_long() == grammar.id_string); + string ret; + for (size_t i = 0; i < node.children.size(); ++i) { + boost::spirit::classic::tree_node<T> &child = node.children[i]; + if (child.value.id().to_long() == grammar.id_string) { + ret += " "; + } + else if (child.value.id().to_long() == grammar.id_alphaNum) { + ret += parseAlphaNum(grammar, child); + } + else if (child.value.id().to_long() == grammar.id_hexCode) { + ret += parseHexCode(grammar, child); + } + } + return ret; +} + +template<typename T> void +parseArgument(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node, + std::map<string, string> &arg) +{ + assert(node.value.id().to_long() == grammar.id_argument); + assert(node.children.size() == 3); + assert(node.children[0].value.id().to_long() == grammar.id_string); + assert(node.children[1].value.id().to_long() == grammar.id_argument); + assert(node.children[2].value.id().to_long() == grammar.id_string); + string key = parseString(grammar, node.children[0]); + string val = parseString(grammar, node.children[2]); + arg[key] = val; +} + +template<typename T> void +parseArgumentList(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node, + std::map<string, string> &arg) +{ + assert(node.value.id().to_long() == grammar.id_argumentList); + for (size_t i = 0; i < node.children.size(); ++i) { + boost::spirit::classic::tree_node<T> &child = node.children[i]; + if (child.value.id().to_long() == grammar.id_argument) { + parseArgument(grammar, child, arg); + } + } +} + +template<typename T> string +parseLocationItem(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) +{ + assert(node.value.id().to_long() == grammar.id_locationItem); + assert(node.children.size() == 1); + + string ret; + boost::spirit::classic::tree_node<T> &child = node.children[0]; + if (child.value.id().to_long() == grammar.id_locationItem) { + ret = string(child.value.begin(), child.value.end()); + } + else if (child.value.id().to_long() == grammar.id_string) { + ret = parseString(grammar, child); + } + return ret; +} + +template<typename T> string +parseLocation(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) +{ + assert(node.value.id().to_long() == grammar.id_location); + string ret; + for (size_t i = 0; i < node.children.size(); ++i) { + boost::spirit::classic::tree_node<T> &child = node.children[i]; + if (child.value.id().to_long() == grammar.id_locationItem) { + ret += parseLocationItem(grammar, child) + "/"; + } + } + return ret.substr(0, ret.size() - 1); +} + +template<typename T> NodeState::UP +parseSystemState(SystemStateGrammar &grammar, boost::spirit::classic::tree_node<T> &node) +{ + assert(node.value.id().to_long() == grammar.id_systemState); + NodeState::UP ret(new NodeState()); + string loc, pwd; + std::map<string, string> arg; + for (size_t i = 0; i < node.children.size(); ++i) { + boost::spirit::classic::tree_node<T> &child = node.children[i]; + if (child.value.id().to_long() == grammar.id_systemState) { + if (string(child.value.begin(), child.value.end()) != "?") { + if (!arg.empty()) { + ret->addChild(!loc.empty() ? loc : pwd, NodeState(arg)); + } + else { + pwd = loc; + } + loc.clear(); + arg.clear(); + } + } + else if (child.value.id().to_long() == grammar.id_location) { + if (!pwd.empty()) { + loc = pwd + "/"; + } + loc += parseLocation(grammar, child); + } + else if (child.value.id().to_long() == grammar.id_argumentList) { + parseArgumentList(grammar, child, arg); + } + } + if (!arg.empty()) { + ret->addChild(!loc.empty() ? loc : pwd, NodeState(arg)); + } + return ret; +} + +vespalib::Lock SystemState::_parseLock; + +SystemState::UP +SystemState::newInstance(const string &state) +{ + if (state.empty()) { + return SystemState::UP(new SystemState(NodeState::UP(new NodeState()))); + } + try { + vespalib::LockGuard guard(_parseLock); + SystemStateGrammar grammar; + boost::spirit::classic::tree_parse_info<> info = + boost::spirit::classic::pt_parse(static_cast<const char *>(&*state.begin()), + static_cast<const char *>(&*state.end()), + grammar.use_parser<0>()); + if (!info.full) { + string unexpected(info.stop); + unsigned int position = state.size() - unexpected.size(); + if (unexpected.size() > 10) { + unexpected = unexpected.substr(0, 10); + } + LOG(error, "Unexpected token at position %u ('%s') in query '%s'.", + position, unexpected.c_str(), state.c_str()); + } + else if (info.trees.size() != 1) { + LOG(error, "Parser returned %u trees, expected 1.", + (uint32_t)info.trees.size()); + } + else { + return SystemState::UP(new SystemState(parseSystemState(grammar, info.trees[0]))); + } + } + catch(std::exception& e) { + std::cerr << "SystemState::parse() internal error: " + << e.what() << std::endl; + } + return SystemState::UP(); +} + +SystemState::SystemState(NodeState::UP root) : + _root(std::move(root)), + _lock() { + // empty +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h new file mode 100644 index 00000000000..544a1b6716e --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.h @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <boost/utility.hpp> +#include <string> +#include <vespa/vespalib/util/sync.h> +#include "nodestate.h" + +namespace documentapi { + +/** + * This class is a factory to create a tree of {@link NodeState} objects from a parseable node state + * string. The naming of this class is intended to capture the fact that this annotated service tree actually + * contains the state of each service in the system. + */ +class SystemState : public boost::noncopyable { +private: + static vespalib::Lock _parseLock; + + NodeState::UP _root; + vespalib::Lock _lock; + + friend class SystemStateHandle; + + /** + * Constructs a new system state object to encapsulate a given root node state. This method is private; the only way + * to create a new instance is through the {@link #create} method. + * + * @param root The root node state. + */ + SystemState(NodeState::UP root); + +public: + /** + * Convenience typedefs. + */ + typedef std::unique_ptr<SystemState> UP; + + /** + * Creates a system state expression from a system state string. + * + * @param state The string to parse as a system state. + * @return The created node state tree. + * @throws RuntimeException Thrown if the string could not be parsed. + */ + static SystemState::UP newInstance(const string &state); +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp new file mode 100644 index 00000000000..9049e889691 --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.cpp @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "systemstatehandle.h" + +using namespace documentapi; + +SystemStateHandover::SystemStateHandover(SystemState *state, vespalib::LockGuard &guard) : + _state(state), + _guard(guard) { + // empty +} + +SystemStateHandle::SystemStateHandle(SystemState &state) : + _state(&state), + _guard(state._lock) { + // empty +} + +SystemStateHandle::SystemStateHandle(SystemStateHandle &rhs) : + _state(rhs._state), + _guard(rhs._guard) { + rhs._state = NULL; +} + +SystemStateHandle::SystemStateHandle(const SystemStateHandover &rhs) : + _state(rhs._state), + _guard(rhs._guard) { + // empty +} + +SystemStateHandle::~SystemStateHandle() { + // empty +} + +SystemStateHandle::operator +SystemStateHandover() { + SystemStateHandover ret(_state, _guard); + _state = NULL; + return ret; +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.h new file mode 100644 index 00000000000..6be94dcc04a --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstatehandle.h @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/sync.h> +#include "systemstate.h" + +namespace documentapi { + +/** + * Implements a handover class to enable the system state handler to be perform handover even on const objects + * such as occur when returning a handle by value from a function. + */ +class SystemStateHandover { + friend class SystemStateHandle; +private: + SystemStateHandover(const SystemStateHandover &); + SystemStateHandover &operator=(const SystemStateHandover &); + SystemStateHandover(SystemState *state, vespalib::LockGuard &guard); + +private: + SystemState *_state; + mutable vespalib::LockGuard _guard; +}; + +/** + * Implements a handle to grant synchronized access to the content of a system state object. This needs the + * above handover class to be able to return itself from methods that create it. + */ +class SystemStateHandle { +private: + SystemState *_state; // The associated system state for which this object is a handler. + vespalib::LockGuard _guard; // The lock guard for the system state's lock. + + SystemState &operator=(const SystemStateHandle &rhs); // hide + +public: + /** + * Creates a new system state handler object that grants access to the content of the supplied system + * state object. This handle is required to make sure that all access to the system state content is + * locked. + */ + SystemStateHandle(SystemState &state); + + /** + * Implements the copy constructor. + * + * @param rhs The handle to copy to this. + */ + SystemStateHandle(SystemStateHandle &rhs); + + /** + * Implements the copy constructor for a const handle. + * + * @param rhs The handle to copy to this. + */ + SystemStateHandle(const SystemStateHandover &rhs); + + /** + * Destructor. Releases the contained lock on the associated system state object. There is no unlock() + * mechanism provided, since this will happen automatically as soon as this handle goes out of scope. + */ + ~SystemStateHandle(); + + /** Implements a cast-operator for handover. */ + operator SystemStateHandover(); + + /** Returns whether or not this handle is valid. */ + bool isValid() const { return _state != NULL; } + + /** Returns a reference to the root node of the associated system state. */ + NodeState &getRoot() { return *_state->_root; } + + /** Returns a const reference to the root node of the associated system state. */ + const NodeState &getRoot() const { return *_state->_root; } +}; + +} + diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.cpp new file mode 100644 index 00000000000..94917cc5a1a --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.cpp @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> + +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include "urlencoder.h" + +LOG_SETUP(".urlencoder"); + +using namespace documentapi; + +const string +URLEncoder::encode(const string &str) +{ + vespalib::asciistream out; + for (size_t i = 0; i < str.size(); i++) { + char c = str[i]; + if ((c >= 48 && c <= 57) || // The range '0'-'9'. + (c >= 65 && c <= 90) || // The range 'A'-'Z'. + (c >= 97 && c <= 122) || // The range 'a'-'z'. + (c == '-' || c == '.' || c == '*' || c == '_')) { + out << c; + } + else if (c == ' ') { + out << '+'; + } + else { + out << "%" << vespalib::make_string("%02X", c & 0xff); + } + } + return out.str(); +} diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.h b/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.h new file mode 100644 index 00000000000..3ce88e0fd7d --- /dev/null +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/urlencoder.h @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/documentapi/common.h> + +namespace documentapi { + +/** + * <p>Utility class for HTML form encoding. This class contains static methods for converting a String to the + * application/x-www-form-urlencoded MIME format. For more information about HTML form encoding, consult the + * HTML specification.</p> + * + * <p>When encoding a String, the following rules apply:</p> + * <ul> + * <li>The alphanumeric characters "a" through "z", "A" through "Z" and "0" through "9" remain the + * same.</li> + * <li>The special characters ".", "-", "*", and "_" remain the same.</li> + * <li>The space character " " is converted into a plus sign "+".</li> + * <li>All other characters are unsafe and are first converted into one or more bytes using some encoding + * scheme. Then each byte is represented by the 3-character string "%xy", where xy is the two-digit + * hexadecimal representation of the byte. The recommended encoding scheme to use is UTF-8. However, for + * compatibility reasons, if an encoding is not specified, then the default encoding of the platform is + * used.</li> + * </ul> + * + * <p>For example using UTF-8 as the encoding scheme the string "The string �@foo-bar" would get converted to + * "The+string+%C3%BC%40foo-bar" because in UTF-8 the character � is encoded as two bytes C3 (hex) and BC + * (hex), and the character @ is encoded as one byte 40 (hex).</p> + */ +class URLEncoder { +public: + /** + * Translates a string into application/x-www-form-urlencoded format using a UTF-8 encoding. + * + * @param str The string to be translated. + * @return The translated string. + */ + static const string encode(const string &str); +}; + +} + diff --git a/documentapi/test/cfg/testdoc.cfg b/documentapi/test/cfg/testdoc.cfg new file mode 100644 index 00000000000..89bea273b6e --- /dev/null +++ b/documentapi/test/cfg/testdoc.cfg @@ -0,0 +1,99 @@ +enablecompression false +datatype[7] +datatype[0].id 666999 +datatype[0].arraytype[0] +datatype[0].weightedsettype[0] +datatype[0].structtype[1] +datatype[0].structtype[0].name ronja +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].field[1] +datatype[0].structtype[0].field[0].name longfield +datatype[0].structtype[0].field[0].id[0] +datatype[0].structtype[0].field[0].datatype 4 +datatype[0].documenttype[0] +datatype[1].id -1636745577 +datatype[1].arraytype[0] +datatype[1].weightedsettype[0] +datatype[1].structtype[1] +datatype[1].structtype[0].name testdoc.header +datatype[1].structtype[0].version 0 +datatype[1].structtype[0].field[4] +datatype[1].structtype[0].field[0].name floatfield +datatype[1].structtype[0].field[0].id[0] +datatype[1].structtype[0].field[0].datatype 1 +datatype[1].structtype[0].field[1].name stringfield +datatype[1].structtype[0].field[1].id[0] +datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[2].name longfield +datatype[1].structtype[0].field[2].id[0] +datatype[1].structtype[0].field[2].datatype 4 +datatype[1].structtype[0].field[3].name urifield +datatype[1].structtype[0].field[3].id[0] +datatype[1].structtype[0].field[3].datatype 10 +datatype[1].documenttype[0] +datatype[2].id 1878320748 +datatype[2].arraytype[0] +datatype[2].weightedsettype[0] +datatype[2].structtype[1] +datatype[2].structtype[0].name testdoc.body +datatype[2].structtype[0].version 0 +datatype[2].structtype[0].field[6] +datatype[2].structtype[0].field[0].name intfield +datatype[2].structtype[0].field[0].id[0] +datatype[2].structtype[0].field[0].datatype 0 +datatype[2].structtype[0].field[1].name rawfield +datatype[2].structtype[0].field[1].id[0] +datatype[2].structtype[0].field[1].datatype 3 +datatype[2].structtype[0].field[2].name doublefield +datatype[2].structtype[0].field[2].id[0] +datatype[2].structtype[0].field[2].datatype 5 +datatype[2].structtype[0].field[3].name contentfield +datatype[2].structtype[0].field[3].id[0] +datatype[2].structtype[0].field[3].datatype 2 +datatype[2].structtype[0].field[4].name bytefield +datatype[2].structtype[0].field[4].id[0] +datatype[2].structtype[0].field[4].datatype 16 +datatype[2].structtype[0].field[5].name foo +datatype[2].structtype[0].field[5].id[0] +datatype[2].structtype[0].field[5].datatype 666999 +datatype[2].documenttype[0] +datatype[3].id -1175657560 +datatype[3].arraytype[0] +datatype[3].weightedsettype[0] +datatype[3].structtype[0] +datatype[3].documenttype[1] +datatype[3].documenttype[0].name testdoc +datatype[3].documenttype[0].version 0 +datatype[3].documenttype[0].inherits[0] +datatype[3].documenttype[0].headerstruct -1636745577 +datatype[3].documenttype[0].bodystruct 1878320748 +datatype[4].id 192273965 +datatype[4].arraytype[0] +datatype[4].weightedsettype[0] +datatype[4].structtype[1] +datatype[4].structtype[0].name other.header +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].field[0] +datatype[4].documenttype[0] +datatype[5].id -72846462 +datatype[5].arraytype[0] +datatype[5].weightedsettype[0] +datatype[5].structtype[1] +datatype[5].structtype[0].name other.body +datatype[5].structtype[0].version 0 +datatype[5].structtype[0].field[1] +datatype[5].structtype[0].field[0].name intfield +datatype[5].structtype[0].field[0].id[0] +datatype[5].structtype[0].field[0].datatype 0 +datatype[5].documenttype[0] +datatype[6].id -1146158894 +datatype[6].arraytype[0] +datatype[6].weightedsettype[0] +datatype[6].structtype[0] +datatype[6].documenttype[1] +datatype[6].documenttype[0].name other +datatype[6].documenttype[0].version 0 +datatype[6].documenttype[0].inherits[0] +datatype[6].documenttype[0].headerstruct 192273965 +datatype[6].documenttype[0].bodystruct -72846462 + diff --git a/documentapi/test/cfg/testdoctypes.cfg b/documentapi/test/cfg/testdoctypes.cfg new file mode 100644 index 00000000000..db071076fc4 --- /dev/null +++ b/documentapi/test/cfg/testdoctypes.cfg @@ -0,0 +1,162 @@ +enablecompression false +documenttype[2] +documenttype[0].id -1175657560 +documenttype[0].name "testdoc" +documenttype[0].version 0 +documenttype[0].headerstruct -1636745577 +documenttype[0].bodystruct 1878320748 +documenttype[0].inherits[0] +documenttype[0].datatype[3] +documenttype[0].datatype[0].id 666999 +documenttype[0].datatype[0].type STRUCT +documenttype[0].datatype[0].array.element.id 0 +documenttype[0].datatype[0].map.key.id 0 +documenttype[0].datatype[0].map.value.id 0 +documenttype[0].datatype[0].wset.key.id 0 +documenttype[0].datatype[0].wset.createifnonexistent false +documenttype[0].datatype[0].wset.removeifzero false +documenttype[0].datatype[0].annotationref.annotation.id 0 +documenttype[0].datatype[0].sstruct.name "ronja" +documenttype[0].datatype[0].sstruct.version 0 +documenttype[0].datatype[0].sstruct.compression.type NONE +documenttype[0].datatype[0].sstruct.compression.level 0 +documenttype[0].datatype[0].sstruct.compression.threshold 90 +documenttype[0].datatype[0].sstruct.compression.minsize 0 +documenttype[0].datatype[0].sstruct.field[1] +documenttype[0].datatype[0].sstruct.field[0].name "longfield" +documenttype[0].datatype[0].sstruct.field[0].id 1589309697 +documenttype[0].datatype[0].sstruct.field[0].id_v6 477609536 +documenttype[0].datatype[0].sstruct.field[0].datatype 4 +documenttype[0].datatype[1].id -1636745577 +documenttype[0].datatype[1].type STRUCT +documenttype[0].datatype[1].array.element.id 0 +documenttype[0].datatype[1].map.key.id 0 +documenttype[0].datatype[1].map.value.id 0 +documenttype[0].datatype[1].wset.key.id 0 +documenttype[0].datatype[1].wset.createifnonexistent false +documenttype[0].datatype[1].wset.removeifzero false +documenttype[0].datatype[1].annotationref.annotation.id 0 +documenttype[0].datatype[1].sstruct.name "testdoc.header" +documenttype[0].datatype[1].sstruct.version 0 +documenttype[0].datatype[1].sstruct.compression.type NONE +documenttype[0].datatype[1].sstruct.compression.level 0 +documenttype[0].datatype[1].sstruct.compression.threshold 90 +documenttype[0].datatype[1].sstruct.compression.minsize 0 +documenttype[0].datatype[1].sstruct.field[4] +documenttype[0].datatype[1].sstruct.field[0].name "floatfield" +documenttype[0].datatype[1].sstruct.field[0].id 1055999199 +documenttype[0].datatype[1].sstruct.field[0].id_v6 657399019 +documenttype[0].datatype[1].sstruct.field[0].datatype 1 +documenttype[0].datatype[1].sstruct.field[1].name "longfield" +documenttype[0].datatype[1].sstruct.field[1].id 1589309697 +documenttype[0].datatype[1].sstruct.field[1].id_v6 477609536 +documenttype[0].datatype[1].sstruct.field[1].datatype 4 +documenttype[0].datatype[1].sstruct.field[2].name "stringfield" +documenttype[0].datatype[1].sstruct.field[2].id 1182460484 +documenttype[0].datatype[1].sstruct.field[2].id_v6 779638844 +documenttype[0].datatype[1].sstruct.field[2].datatype 2 +documenttype[0].datatype[1].sstruct.field[3].name "urifield" +documenttype[0].datatype[1].sstruct.field[3].id 628407450 +documenttype[0].datatype[1].sstruct.field[3].id_v6 756285986 +documenttype[0].datatype[1].sstruct.field[3].datatype 10 +documenttype[0].datatype[2].id 1878320748 +documenttype[0].datatype[2].type STRUCT +documenttype[0].datatype[2].array.element.id 0 +documenttype[0].datatype[2].map.key.id 0 +documenttype[0].datatype[2].map.value.id 0 +documenttype[0].datatype[2].wset.key.id 0 +documenttype[0].datatype[2].wset.createifnonexistent false +documenttype[0].datatype[2].wset.removeifzero false +documenttype[0].datatype[2].annotationref.annotation.id 0 +documenttype[0].datatype[2].sstruct.name "testdoc.body" +documenttype[0].datatype[2].sstruct.version 0 +documenttype[0].datatype[2].sstruct.compression.type NONE +documenttype[0].datatype[2].sstruct.compression.level 0 +documenttype[0].datatype[2].sstruct.compression.threshold 90 +documenttype[0].datatype[2].sstruct.compression.minsize 0 +documenttype[0].datatype[2].sstruct.field[5] +documenttype[0].datatype[2].sstruct.field[0].name "bytefield" +documenttype[0].datatype[2].sstruct.field[0].id 1924064342 +documenttype[0].datatype[2].sstruct.field[0].id_v6 2050488857 +documenttype[0].datatype[2].sstruct.field[0].datatype 16 +documenttype[0].datatype[2].sstruct.field[1].name "doublefield" +documenttype[0].datatype[2].sstruct.field[1].id 421343958 +documenttype[0].datatype[2].sstruct.field[1].id_v6 944141865 +documenttype[0].datatype[2].sstruct.field[1].datatype 5 +documenttype[0].datatype[2].sstruct.field[2].name "foo" +documenttype[0].datatype[2].sstruct.field[2].id 1747747633 +documenttype[0].datatype[2].sstruct.field[2].id_v6 2143417350 +documenttype[0].datatype[2].sstruct.field[2].datatype 666999 +documenttype[0].datatype[2].sstruct.field[3].name "intfield" +documenttype[0].datatype[2].sstruct.field[3].id 435380425 +documenttype[0].datatype[2].sstruct.field[3].id_v6 545156740 +documenttype[0].datatype[2].sstruct.field[3].datatype 0 +documenttype[0].datatype[2].sstruct.field[4].name "rawfield" +documenttype[0].datatype[2].sstruct.field[4].id 172982133 +documenttype[0].datatype[2].sstruct.field[4].id_v6 84029972 +documenttype[0].datatype[2].sstruct.field[4].datatype 3 +documenttype[0].annotationtype[0] +documenttype[1].id -1146158894 +documenttype[1].name "other" +documenttype[1].version 0 +documenttype[1].headerstruct 192273965 +documenttype[1].bodystruct -72846462 +documenttype[1].inherits[0] +documenttype[1].datatype[3] +documenttype[1].datatype[0].id 666999 +documenttype[1].datatype[0].type STRUCT +documenttype[1].datatype[0].array.element.id 0 +documenttype[1].datatype[0].map.key.id 0 +documenttype[1].datatype[0].map.value.id 0 +documenttype[1].datatype[0].wset.key.id 0 +documenttype[1].datatype[0].wset.createifnonexistent false +documenttype[1].datatype[0].wset.removeifzero false +documenttype[1].datatype[0].annotationref.annotation.id 0 +documenttype[1].datatype[0].sstruct.name "ronja" +documenttype[1].datatype[0].sstruct.version 0 +documenttype[1].datatype[0].sstruct.compression.type NONE +documenttype[1].datatype[0].sstruct.compression.level 0 +documenttype[1].datatype[0].sstruct.compression.threshold 90 +documenttype[1].datatype[0].sstruct.compression.minsize 0 +documenttype[1].datatype[0].sstruct.field[1] +documenttype[1].datatype[0].sstruct.field[0].name "longfield" +documenttype[1].datatype[0].sstruct.field[0].id 1589309697 +documenttype[1].datatype[0].sstruct.field[0].id_v6 477609536 +documenttype[1].datatype[0].sstruct.field[0].datatype 4 +documenttype[1].datatype[1].id 192273965 +documenttype[1].datatype[1].type STRUCT +documenttype[1].datatype[1].array.element.id 0 +documenttype[1].datatype[1].map.key.id 0 +documenttype[1].datatype[1].map.value.id 0 +documenttype[1].datatype[1].wset.key.id 0 +documenttype[1].datatype[1].wset.createifnonexistent false +documenttype[1].datatype[1].wset.removeifzero false +documenttype[1].datatype[1].annotationref.annotation.id 0 +documenttype[1].datatype[1].sstruct.name "other.header" +documenttype[1].datatype[1].sstruct.version 0 +documenttype[1].datatype[1].sstruct.compression.type NONE +documenttype[1].datatype[1].sstruct.compression.level 0 +documenttype[1].datatype[1].sstruct.compression.threshold 90 +documenttype[1].datatype[1].sstruct.compression.minsize 0 +documenttype[1].datatype[1].sstruct.field[0] +documenttype[1].datatype[2].id -72846462 +documenttype[1].datatype[2].type STRUCT +documenttype[1].datatype[2].array.element.id 0 +documenttype[1].datatype[2].map.key.id 0 +documenttype[1].datatype[2].map.value.id 0 +documenttype[1].datatype[2].wset.key.id 0 +documenttype[1].datatype[2].wset.createifnonexistent false +documenttype[1].datatype[2].wset.removeifzero false +documenttype[1].datatype[2].annotationref.annotation.id 0 +documenttype[1].datatype[2].sstruct.name "other.body" +documenttype[1].datatype[2].sstruct.version 0 +documenttype[1].datatype[2].sstruct.compression.type NONE +documenttype[1].datatype[2].sstruct.compression.level 0 +documenttype[1].datatype[2].sstruct.compression.threshold 90 +documenttype[1].datatype[2].sstruct.compression.minsize 0 +documenttype[1].datatype[2].sstruct.field[1] +documenttype[1].datatype[2].sstruct.field[0].name "intfield" +documenttype[1].datatype[2].sstruct.field[0].id 435380425 +documenttype[1].datatype[2].sstruct.field[0].id_v6 545156740 +documenttype[1].datatype[2].sstruct.field[0].datatype 0 +documenttype[1].annotationtype[0] diff --git a/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateMessage.dat Binary files differnew file mode 100644 index 00000000000..95e663088c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateReply.dat Binary files differnew file mode 100644 index 00000000000..216db17f80e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-BatchMessage.dat Binary files differnew file mode 100644 index 00000000000..c1c43e8081b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-BatchMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-BatchReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-BatchReply.dat Binary files differnew file mode 100644 index 00000000000..be3a9ba1913 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-BatchReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..a506fc760bd --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..e13917227d1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..f39b31217e6 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..1468f027b15 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-DocumentListMessage.dat Binary files differnew file mode 100644 index 00000000000..2d8d12d2704 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-DocumentListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-DocumentListReply.dat Binary files differnew file mode 100644 index 00000000000..c8a1cd888f0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-DocumentListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-1.dat b/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-1.dat Binary files differnew file mode 100644 index 00000000000..0107dd5f350 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-1.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-2.dat b/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-2.dat Binary files differnew file mode 100644 index 00000000000..57187093f28 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-2.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-3.dat b/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-3.dat Binary files differnew file mode 100644 index 00000000000..6a516d38d17 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-3.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryReply.dat Binary files differnew file mode 100644 index 00000000000..16b1e4bc4ef --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsMessage.dat Binary files differnew file mode 100644 index 00000000000..b9df278fd7a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsReply.dat Binary files differnew file mode 100644 index 00000000000..05510dd8c1e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-EndOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-EndOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..6f33a4a0bfb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-EndOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-EndOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-EndOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..6927b7726ec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-EndOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage0.dat b/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage0.dat Binary files differnew file mode 100644 index 00000000000..7ebf108a292 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage0.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage1.dat b/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage1.dat Binary files differnew file mode 100644 index 00000000000..36a2cfd91c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage1.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage2.dat b/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage2.dat Binary files differnew file mode 100644 index 00000000000..10514e98c37 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage2.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage3.dat b/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage3.dat Binary files differnew file mode 100644 index 00000000000..75135b8bb30 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage3.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage4.dat b/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage4.dat Binary files differnew file mode 100644 index 00000000000..5040584a275 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectMessage4.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectReply.dat Binary files differnew file mode 100644 index 00000000000..79c263cb2dc --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-GarbageCollectReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListMessage.dat Binary files differnew file mode 100644 index 00000000000..fa3de45ac5b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListReply.dat Binary files differnew file mode 100644 index 00000000000..830994ed785 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateMessage.dat Binary files differnew file mode 100644 index 00000000000..aa2d206ca3a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateReply.dat Binary files differnew file mode 100644 index 00000000000..0f91e3759f8 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..1444d617c22 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c1ad7920a2e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..1a8a837ea16 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..541cd718a66 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationMessage.dat Binary files differnew file mode 100644 index 00000000000..6efc54a6d09 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationReply.dat Binary files differnew file mode 100644 index 00000000000..8ad31a95bd5 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..9e4a5c5f82f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..480544045bb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-1.dat b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-1.dat Binary files differnew file mode 100644 index 00000000000..dbf830c9365 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-1.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-2.dat b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-2.dat Binary files differnew file mode 100644 index 00000000000..094143cf78d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-2.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-3.dat b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-3.dat Binary files differnew file mode 100644 index 00000000000..3341d74052b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-3.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-4.dat b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-4.dat Binary files differnew file mode 100644 index 00000000000..8aaaefff491 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-4.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-5.dat b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-5.dat Binary files differnew file mode 100644 index 00000000000..e66ed1f07d4 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-5.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultReply.dat Binary files differnew file mode 100644 index 00000000000..003f35d63a7 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..78b7972eb5c --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..bf5db8761e2 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessage.dat Binary files differnew file mode 100644 index 00000000000..16850a6aff3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageGroup.dat Binary files differnew file mode 100644 index 00000000000..abd648184d7 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageGroup.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageUser.dat Binary files differnew file mode 100644 index 00000000000..c2a63cb94c0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageUser.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationReply.dat Binary files differnew file mode 100644 index 00000000000..752c4dba399 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-1.dat b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-1.dat Binary files differnew file mode 100644 index 00000000000..988f9fdab1f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-1.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-2.dat b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-2.dat Binary files differnew file mode 100644 index 00000000000..ac277d09643 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-2.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-3.dat b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-3.dat Binary files differnew file mode 100644 index 00000000000..03b49c8a0ac --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-3.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-4.dat b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-4.dat Binary files differnew file mode 100644 index 00000000000..d52e574ea44 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-4.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-5.dat b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-5.dat Binary files differnew file mode 100644 index 00000000000..e68654e9941 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-5.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultReply.dat Binary files differnew file mode 100644 index 00000000000..cce9c6f8d14 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-StartOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-StartOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..3c51c3c7eec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-StartOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-StartOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-StartOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..ff881679155 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-StartOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-StatBucketMessage.dat Binary files differnew file mode 100644 index 00000000000..1fc2b1cf3c1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-StatBucketMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-StatBucketReply.dat Binary files differnew file mode 100644 index 00000000000..0b98e240018 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-StatBucketReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..f1bca25ceca --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c7151299366 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoMessage.dat Binary files differnew file mode 100644 index 00000000000..80b44e0c6fd --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoReply.dat Binary files differnew file mode 100644 index 00000000000..57a656c9b2d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-WrongDistributionReply.dat Binary files differnew file mode 100644 index 00000000000..0dbe13225ae --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-cpp-WrongDistributionReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateMessage.dat Binary files differnew file mode 100644 index 00000000000..95e663088c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateReply.dat Binary files differnew file mode 100644 index 00000000000..216db17f80e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5-java-BatchMessage.dat Binary files differnew file mode 100644 index 00000000000..c1c43e8081b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-BatchMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-BatchReply.dat b/documentapi/test/crosslanguagefiles/5-java-BatchReply.dat Binary files differnew file mode 100644 index 00000000000..be3a9ba1913 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-BatchReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-java-CreateVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..a506fc760bd --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-CreateVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-java-CreateVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..e13917227d1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-CreateVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..f39b31217e6 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..1468f027b15 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5-java-DocumentListMessage.dat Binary files differnew file mode 100644 index 00000000000..2d8d12d2704 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-DocumentListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5-java-DocumentListReply.dat Binary files differnew file mode 100644 index 00000000000..c8a1cd888f0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-DocumentListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5-java-DocumentSummaryReply.dat Binary files differnew file mode 100644 index 00000000000..16b1e4bc4ef --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-DocumentSummaryReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsMessage.dat Binary files differnew file mode 100644 index 00000000000..b9df278fd7a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsReply.dat Binary files differnew file mode 100644 index 00000000000..05510dd8c1e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-EndOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5-java-EndOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..6f33a4a0bfb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-EndOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-EndOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5-java-EndOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..6927b7726ec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-EndOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage0.dat b/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage0.dat Binary files differnew file mode 100644 index 00000000000..7ebf108a292 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage0.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage1.dat b/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage1.dat Binary files differnew file mode 100644 index 00000000000..36a2cfd91c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage1.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage2.dat b/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage2.dat Binary files differnew file mode 100644 index 00000000000..10514e98c37 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage2.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage3.dat b/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage3.dat Binary files differnew file mode 100644 index 00000000000..75135b8bb30 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage3.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage4.dat b/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage4.dat Binary files differnew file mode 100644 index 00000000000..5040584a275 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-GarbageCollectMessage4.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-GarbageCollectReply.dat b/documentapi/test/crosslanguagefiles/5-java-GarbageCollectReply.dat Binary files differnew file mode 100644 index 00000000000..79c263cb2dc --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-GarbageCollectReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5-java-GetBucketListMessage.dat Binary files differnew file mode 100644 index 00000000000..fa3de45ac5b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-GetBucketListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5-java-GetBucketListReply.dat Binary files differnew file mode 100644 index 00000000000..830994ed785 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-GetBucketListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5-java-GetBucketStateMessage.dat Binary files differnew file mode 100644 index 00000000000..aa2d206ca3a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-GetBucketStateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5-java-GetBucketStateReply.dat Binary files differnew file mode 100644 index 00000000000..0f91e3759f8 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-GetBucketStateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-java-GetDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..1444d617c22 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-GetDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-java-GetDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c1ad7920a2e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-GetDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-java-MapVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..1a8a837ea16 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-MapVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-java-MapVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..541cd718a66 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-MapVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5-java-MultiOperationMessage.dat Binary files differnew file mode 100644 index 00000000000..6efc54a6d09 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-MultiOperationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5-java-MultiOperationReply.dat Binary files differnew file mode 100644 index 00000000000..8ad31a95bd5 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-MultiOperationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-java-PutDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..9e4a5c5f82f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-PutDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-java-PutDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..480544045bb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-PutDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5-java-QueryResultReply.dat Binary files differnew file mode 100644 index 00000000000..003f35d63a7 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-QueryResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..78b7972eb5c --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..bf5db8761e2 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessage.dat Binary files differnew file mode 100644 index 00000000000..16850a6aff3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageGroup.dat Binary files differnew file mode 100644 index 00000000000..9d7901a8b6e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageGroup.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageUser.dat Binary files differnew file mode 100644 index 00000000000..c2a63cb94c0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageUser.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationReply.dat Binary files differnew file mode 100644 index 00000000000..752c4dba399 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5-java-SearchResultReply.dat Binary files differnew file mode 100644 index 00000000000..cce9c6f8d14 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-SearchResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-StartOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5-java-StartOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..3c51c3c7eec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-StartOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-StartOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5-java-StartOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..ff881679155 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-StartOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5-java-StatBucketMessage.dat Binary files differnew file mode 100644 index 00000000000..1fc2b1cf3c1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-StatBucketMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5-java-StatBucketReply.dat Binary files differnew file mode 100644 index 00000000000..0b98e240018 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-StatBucketReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..f1bca25ceca --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c7151299366 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5-java-VisitorInfoMessage.dat Binary files differnew file mode 100644 index 00000000000..80b44e0c6fd --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-VisitorInfoMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5-java-VisitorInfoReply.dat Binary files differnew file mode 100644 index 00000000000..57a656c9b2d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-VisitorInfoReply.dat diff --git a/documentapi/test/crosslanguagefiles/5-java-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5-java-WrongDistributionReply.dat Binary files differnew file mode 100644 index 00000000000..0dbe13225ae --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5-java-WrongDistributionReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-Priority.txt b/documentapi/test/crosslanguagefiles/5.1-Priority.txt new file mode 100644 index 00000000000..5fda195ddaf --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-Priority.txt @@ -0,0 +1,16 @@ +HIGHEST:0 +VERY_HIGH:1 +HIGH_1:2 +HIGH_2:3 +HIGH_3:4 +NORMAL_1:5 +NORMAL_2:6 +NORMAL_3:7 +NORMAL_4:8 +NORMAL_5:9 +NORMAL_6:10 +LOW_1:11 +LOW_2:12 +LOW_3:13 +VERY_LOW:14 +LOWEST:15 diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateMessage.dat Binary files differnew file mode 100644 index 00000000000..95e663088c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateReply.dat Binary files differnew file mode 100644 index 00000000000..216db17f80e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchMessage.dat Binary files differnew file mode 100644 index 00000000000..c1c43e8081b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchReply.dat Binary files differnew file mode 100644 index 00000000000..be3a9ba1913 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..4ca8648e702 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..e13917227d1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..f39b31217e6 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..1468f027b15 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentIgnoredReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentIgnoredReply.dat Binary files differnew file mode 100644 index 00000000000..15a7afe2a59 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentIgnoredReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListMessage.dat Binary files differnew file mode 100644 index 00000000000..2d8d12d2704 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListReply.dat Binary files differnew file mode 100644 index 00000000000..c8a1cd888f0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-1.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-1.dat Binary files differnew file mode 100644 index 00000000000..0107dd5f350 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-1.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-2.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-2.dat Binary files differnew file mode 100644 index 00000000000..57187093f28 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-2.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-3.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-3.dat Binary files differnew file mode 100644 index 00000000000..6a516d38d17 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-3.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryReply.dat Binary files differnew file mode 100644 index 00000000000..16b1e4bc4ef --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsMessage.dat Binary files differnew file mode 100644 index 00000000000..b9df278fd7a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsReply.dat Binary files differnew file mode 100644 index 00000000000..05510dd8c1e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-EndOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-EndOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..6f33a4a0bfb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-EndOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-EndOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-EndOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..6927b7726ec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-EndOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage0.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage0.dat Binary files differnew file mode 100644 index 00000000000..7ebf108a292 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage0.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage1.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage1.dat Binary files differnew file mode 100644 index 00000000000..36a2cfd91c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage1.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage2.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage2.dat Binary files differnew file mode 100644 index 00000000000..10514e98c37 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage2.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage3.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage3.dat Binary files differnew file mode 100644 index 00000000000..75135b8bb30 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage3.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage4.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage4.dat Binary files differnew file mode 100644 index 00000000000..5040584a275 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectMessage4.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectReply.dat Binary files differnew file mode 100644 index 00000000000..79c263cb2dc --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-GarbageCollectReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListMessage.dat Binary files differnew file mode 100644 index 00000000000..fa3de45ac5b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListReply.dat Binary files differnew file mode 100644 index 00000000000..830994ed785 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateMessage.dat Binary files differnew file mode 100644 index 00000000000..aa2d206ca3a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateReply.dat Binary files differnew file mode 100644 index 00000000000..0f91e3759f8 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..3df64ed657f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c1ad7920a2e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..1a8a837ea16 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..541cd718a66 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationMessage.dat Binary files differnew file mode 100644 index 00000000000..6efc54a6d09 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationReply.dat Binary files differnew file mode 100644 index 00000000000..8ad31a95bd5 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..9e4a5c5f82f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..480544045bb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-1.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-1.dat Binary files differnew file mode 100644 index 00000000000..dbf830c9365 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-1.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-2.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-2.dat Binary files differnew file mode 100644 index 00000000000..094143cf78d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-2.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-3.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-3.dat Binary files differnew file mode 100644 index 00000000000..3341d74052b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-3.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-4.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-4.dat Binary files differnew file mode 100644 index 00000000000..8aaaefff491 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-4.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-5.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-5.dat Binary files differnew file mode 100644 index 00000000000..e66ed1f07d4 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-5.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultReply.dat Binary files differnew file mode 100644 index 00000000000..003f35d63a7 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..78b7972eb5c --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..bf5db8761e2 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessage.dat Binary files differnew file mode 100644 index 00000000000..16850a6aff3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageGroup.dat Binary files differnew file mode 100644 index 00000000000..abd648184d7 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageGroup.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageUser.dat Binary files differnew file mode 100644 index 00000000000..c2a63cb94c0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageUser.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationReply.dat Binary files differnew file mode 100644 index 00000000000..752c4dba399 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-1.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-1.dat Binary files differnew file mode 100644 index 00000000000..988f9fdab1f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-1.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-2.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-2.dat Binary files differnew file mode 100644 index 00000000000..ac277d09643 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-2.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-3.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-3.dat Binary files differnew file mode 100644 index 00000000000..03b49c8a0ac --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-3.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-4.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-4.dat Binary files differnew file mode 100644 index 00000000000..d52e574ea44 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-4.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-5.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-5.dat Binary files differnew file mode 100644 index 00000000000..e68654e9941 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-5.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultReply.dat Binary files differnew file mode 100644 index 00000000000..cce9c6f8d14 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-StartOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-StartOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..3c51c3c7eec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-StartOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-StartOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-StartOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..ff881679155 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-StartOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketMessage.dat Binary files differnew file mode 100644 index 00000000000..1fc2b1cf3c1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketReply.dat Binary files differnew file mode 100644 index 00000000000..0b98e240018 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..f1bca25ceca --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c7151299366 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoMessage.dat Binary files differnew file mode 100644 index 00000000000..80b44e0c6fd --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoReply.dat Binary files differnew file mode 100644 index 00000000000..57a656c9b2d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-WrongDistributionReply.dat Binary files differnew file mode 100644 index 00000000000..0dbe13225ae --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-cpp-WrongDistributionReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateMessage.dat Binary files differnew file mode 100644 index 00000000000..95e663088c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateReply.dat Binary files differnew file mode 100644 index 00000000000..216db17f80e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-BatchMessage.dat Binary files differnew file mode 100644 index 00000000000..c1c43e8081b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-BatchMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-BatchReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-BatchReply.dat Binary files differnew file mode 100644 index 00000000000..be3a9ba1913 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-BatchReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..4ca8648e702 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..e13917227d1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..f39b31217e6 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..1468f027b15 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DocumentIgnoredReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-DocumentIgnoredReply.dat Binary files differnew file mode 100644 index 00000000000..15a7afe2a59 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-DocumentIgnoredReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-DocumentListMessage.dat Binary files differnew file mode 100644 index 00000000000..2d8d12d2704 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-DocumentListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-DocumentListReply.dat Binary files differnew file mode 100644 index 00000000000..c8a1cd888f0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-DocumentListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-DocumentSummaryReply.dat Binary files differnew file mode 100644 index 00000000000..16b1e4bc4ef --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-DocumentSummaryReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsMessage.dat Binary files differnew file mode 100644 index 00000000000..b9df278fd7a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsReply.dat Binary files differnew file mode 100644 index 00000000000..05510dd8c1e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-EndOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-EndOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..6f33a4a0bfb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-EndOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-EndOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-EndOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..6927b7726ec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-EndOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage0.dat b/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage0.dat Binary files differnew file mode 100644 index 00000000000..7ebf108a292 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage0.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage1.dat b/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage1.dat Binary files differnew file mode 100644 index 00000000000..36a2cfd91c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage1.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage2.dat b/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage2.dat Binary files differnew file mode 100644 index 00000000000..10514e98c37 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage2.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage3.dat b/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage3.dat Binary files differnew file mode 100644 index 00000000000..75135b8bb30 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage3.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage4.dat b/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage4.dat Binary files differnew file mode 100644 index 00000000000..5040584a275 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectMessage4.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectReply.dat Binary files differnew file mode 100644 index 00000000000..79c263cb2dc --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-GarbageCollectReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListMessage.dat Binary files differnew file mode 100644 index 00000000000..fa3de45ac5b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListReply.dat Binary files differnew file mode 100644 index 00000000000..830994ed785 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateMessage.dat Binary files differnew file mode 100644 index 00000000000..aa2d206ca3a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateReply.dat Binary files differnew file mode 100644 index 00000000000..0f91e3759f8 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..3df64ed657f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c1ad7920a2e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..1a8a837ea16 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..541cd718a66 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationMessage.dat Binary files differnew file mode 100644 index 00000000000..6efc54a6d09 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationReply.dat Binary files differnew file mode 100644 index 00000000000..8ad31a95bd5 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..9e4a5c5f82f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..480544045bb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-QueryResultReply.dat Binary files differnew file mode 100644 index 00000000000..003f35d63a7 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-QueryResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..78b7972eb5c --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..bf5db8761e2 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessage.dat Binary files differnew file mode 100644 index 00000000000..16850a6aff3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageGroup.dat Binary files differnew file mode 100644 index 00000000000..9d7901a8b6e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageGroup.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageUser.dat Binary files differnew file mode 100644 index 00000000000..c2a63cb94c0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageUser.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationReply.dat Binary files differnew file mode 100644 index 00000000000..752c4dba399 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-SearchResultReply.dat Binary files differnew file mode 100644 index 00000000000..cce9c6f8d14 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-SearchResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-StartOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-StartOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..3c51c3c7eec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-StartOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-StartOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-StartOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..ff881679155 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-StartOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-StatBucketMessage.dat Binary files differnew file mode 100644 index 00000000000..1fc2b1cf3c1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-StatBucketMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-StatBucketReply.dat Binary files differnew file mode 100644 index 00000000000..0b98e240018 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-StatBucketReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..f1bca25ceca --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c7151299366 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoMessage.dat Binary files differnew file mode 100644 index 00000000000..80b44e0c6fd --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoReply.dat Binary files differnew file mode 100644 index 00000000000..57a656c9b2d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.1-java-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-WrongDistributionReply.dat Binary files differnew file mode 100644 index 00000000000..0dbe13225ae --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.1-java-WrongDistributionReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateMessage.dat Binary files differnew file mode 100644 index 00000000000..95e663088c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateReply.dat Binary files differnew file mode 100644 index 00000000000..216db17f80e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchMessage.dat Binary files differnew file mode 100644 index 00000000000..c1c43e8081b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchReply.dat Binary files differnew file mode 100644 index 00000000000..be3a9ba1913 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-CreateVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..4ca8648e702 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-CreateVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-CreateVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..e13917227d1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-CreateVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DestroyVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..f39b31217e6 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-DestroyVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DestroyVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..1468f027b15 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-DestroyVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentIgnoredReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentIgnoredReply.dat Binary files differnew file mode 100644 index 00000000000..15a7afe2a59 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentIgnoredReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentListMessage.dat Binary files differnew file mode 100644 index 00000000000..2d8d12d2704 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentListReply.dat Binary files differnew file mode 100644 index 00000000000..c8a1cd888f0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-1.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-1.dat Binary files differnew file mode 100644 index 00000000000..0107dd5f350 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-1.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-2.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-2.dat Binary files differnew file mode 100644 index 00000000000..57187093f28 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-2.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-3.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-3.dat Binary files differnew file mode 100644 index 00000000000..6a516d38d17 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-3.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryReply.dat Binary files differnew file mode 100644 index 00000000000..16b1e4bc4ef --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-EmptyBucketsMessage.dat Binary files differnew file mode 100644 index 00000000000..b9df278fd7a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-EmptyBucketsMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-EmptyBucketsReply.dat Binary files differnew file mode 100644 index 00000000000..05510dd8c1e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-EmptyBucketsReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-EndOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-EndOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..6f33a4a0bfb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-EndOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-EndOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-EndOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..6927b7726ec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-EndOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage0.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage0.dat Binary files differnew file mode 100644 index 00000000000..7ebf108a292 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage0.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage1.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage1.dat Binary files differnew file mode 100644 index 00000000000..36a2cfd91c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage1.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage2.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage2.dat Binary files differnew file mode 100644 index 00000000000..10514e98c37 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage2.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage3.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage3.dat Binary files differnew file mode 100644 index 00000000000..75135b8bb30 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage3.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage4.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage4.dat Binary files differnew file mode 100644 index 00000000000..5040584a275 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectMessage4.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectReply.dat Binary files differnew file mode 100644 index 00000000000..79c263cb2dc --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-GarbageCollectReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketListMessage.dat Binary files differnew file mode 100644 index 00000000000..fa3de45ac5b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketListReply.dat Binary files differnew file mode 100644 index 00000000000..830994ed785 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketStateMessage.dat Binary files differnew file mode 100644 index 00000000000..aa2d206ca3a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketStateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketStateReply.dat Binary files differnew file mode 100644 index 00000000000..0f91e3759f8 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketStateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GetDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..3df64ed657f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-GetDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GetDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c1ad7920a2e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-GetDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-MapVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..1a8a837ea16 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-MapVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-MapVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..541cd718a66 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-MapVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-MultiOperationMessage.dat Binary files differnew file mode 100644 index 00000000000..6efc54a6d09 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-MultiOperationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-MultiOperationReply.dat Binary files differnew file mode 100644 index 00000000000..8ad31a95bd5 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-MultiOperationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-PutDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..13c25f0b729 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-PutDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-PutDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..480544045bb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-PutDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-1.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-1.dat Binary files differnew file mode 100644 index 00000000000..dbf830c9365 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-1.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-2.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-2.dat Binary files differnew file mode 100644 index 00000000000..094143cf78d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-2.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-3.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-3.dat Binary files differnew file mode 100644 index 00000000000..3341d74052b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-3.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-4.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-4.dat Binary files differnew file mode 100644 index 00000000000..8aaaefff491 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-4.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-5.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-5.dat Binary files differnew file mode 100644 index 00000000000..e66ed1f07d4 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-5.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultReply.dat Binary files differnew file mode 100644 index 00000000000..003f35d63a7 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..21f2c9b81c4 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..bf5db8761e2 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessage.dat Binary files differnew file mode 100644 index 00000000000..16850a6aff3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessageGroup.dat Binary files differnew file mode 100644 index 00000000000..abd648184d7 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessageGroup.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessageUser.dat Binary files differnew file mode 100644 index 00000000000..c2a63cb94c0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessageUser.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationReply.dat Binary files differnew file mode 100644 index 00000000000..752c4dba399 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-1.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-1.dat Binary files differnew file mode 100644 index 00000000000..988f9fdab1f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-1.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-2.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-2.dat Binary files differnew file mode 100644 index 00000000000..ac277d09643 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-2.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-3.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-3.dat Binary files differnew file mode 100644 index 00000000000..03b49c8a0ac --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-3.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-4.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-4.dat Binary files differnew file mode 100644 index 00000000000..d52e574ea44 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-4.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-5.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-5.dat Binary files differnew file mode 100644 index 00000000000..e68654e9941 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-5.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultReply.dat Binary files differnew file mode 100644 index 00000000000..cce9c6f8d14 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-StartOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-StartOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..3c51c3c7eec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-StartOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-StartOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-StartOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..ff881679155 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-StartOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-StatBucketMessage.dat Binary files differnew file mode 100644 index 00000000000..1fc2b1cf3c1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-StatBucketMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-StatBucketReply.dat Binary files differnew file mode 100644 index 00000000000..0b98e240018 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-StatBucketReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-UpdateDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..93274279f0d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-UpdateDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-UpdateDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c7151299366 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-UpdateDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-VisitorInfoMessage.dat Binary files differnew file mode 100644 index 00000000000..80b44e0c6fd --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-VisitorInfoMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-VisitorInfoReply.dat Binary files differnew file mode 100644 index 00000000000..57a656c9b2d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-VisitorInfoReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-WrongDistributionReply.dat Binary files differnew file mode 100644 index 00000000000..0dbe13225ae --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-cpp-WrongDistributionReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateMessage.dat Binary files differnew file mode 100644 index 00000000000..95e663088c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateReply.dat Binary files differnew file mode 100644 index 00000000000..216db17f80e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-CreateVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..4ca8648e702 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-CreateVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-CreateVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..e13917227d1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-CreateVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-DestroyVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..f39b31217e6 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-DestroyVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-DestroyVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..1468f027b15 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-DestroyVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-DocumentIgnoredReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-DocumentIgnoredReply.dat Binary files differnew file mode 100644 index 00000000000..15a7afe2a59 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-DocumentIgnoredReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-DocumentListMessage.dat Binary files differnew file mode 100644 index 00000000000..2d8d12d2704 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-DocumentListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-DocumentListReply.dat Binary files differnew file mode 100644 index 00000000000..c8a1cd888f0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-DocumentListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-DocumentSummaryReply.dat Binary files differnew file mode 100644 index 00000000000..16b1e4bc4ef --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-DocumentSummaryReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-EmptyBucketsMessage.dat Binary files differnew file mode 100644 index 00000000000..b9df278fd7a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-EmptyBucketsMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-EmptyBucketsReply.dat Binary files differnew file mode 100644 index 00000000000..05510dd8c1e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-EmptyBucketsReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-EndOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-EndOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..6f33a4a0bfb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-EndOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-EndOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-EndOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..6927b7726ec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-EndOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage0.dat b/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage0.dat Binary files differnew file mode 100644 index 00000000000..7ebf108a292 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage0.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage1.dat b/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage1.dat Binary files differnew file mode 100644 index 00000000000..36a2cfd91c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage1.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage2.dat b/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage2.dat Binary files differnew file mode 100644 index 00000000000..10514e98c37 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage2.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage3.dat b/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage3.dat Binary files differnew file mode 100644 index 00000000000..75135b8bb30 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage3.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage4.dat b/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage4.dat Binary files differnew file mode 100644 index 00000000000..5040584a275 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectMessage4.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectReply.dat Binary files differnew file mode 100644 index 00000000000..79c263cb2dc --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-GarbageCollectReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-GetBucketListMessage.dat Binary files differnew file mode 100644 index 00000000000..fa3de45ac5b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-GetBucketListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-GetBucketListReply.dat Binary files differnew file mode 100644 index 00000000000..830994ed785 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-GetBucketListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-GetBucketStateMessage.dat Binary files differnew file mode 100644 index 00000000000..aa2d206ca3a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-GetBucketStateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-GetBucketStateReply.dat Binary files differnew file mode 100644 index 00000000000..0f91e3759f8 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-GetBucketStateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-GetDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..3df64ed657f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-GetDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-GetDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c1ad7920a2e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-GetDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-MapVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..1a8a837ea16 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-MapVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-MapVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..541cd718a66 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-MapVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-MultiOperationMessage.dat Binary files differnew file mode 100644 index 00000000000..6efc54a6d09 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-MultiOperationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-MultiOperationReply.dat Binary files differnew file mode 100644 index 00000000000..8ad31a95bd5 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-MultiOperationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-PutDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..13c25f0b729 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-PutDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-PutDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..480544045bb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-PutDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-QueryResultReply.dat Binary files differnew file mode 100644 index 00000000000..003f35d63a7 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-QueryResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-RemoveDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..21f2c9b81c4 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-RemoveDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-RemoveDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..bf5db8761e2 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-RemoveDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-RemoveLocationMessage.dat Binary files differnew file mode 100644 index 00000000000..16850a6aff3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-RemoveLocationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-RemoveLocationReply.dat Binary files differnew file mode 100644 index 00000000000..752c4dba399 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-RemoveLocationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-SearchResultReply.dat Binary files differnew file mode 100644 index 00000000000..cce9c6f8d14 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-SearchResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-StartOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-StartOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..3c51c3c7eec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-StartOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-StartOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-StartOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..ff881679155 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-StartOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-StatBucketMessage.dat Binary files differnew file mode 100644 index 00000000000..1fc2b1cf3c1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-StatBucketMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-StatBucketReply.dat Binary files differnew file mode 100644 index 00000000000..0b98e240018 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-StatBucketReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-UpdateDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..93274279f0d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-UpdateDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-UpdateDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c7151299366 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-UpdateDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-VisitorInfoMessage.dat Binary files differnew file mode 100644 index 00000000000..80b44e0c6fd --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-VisitorInfoMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-VisitorInfoReply.dat Binary files differnew file mode 100644 index 00000000000..57a656c9b2d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-VisitorInfoReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.115-java-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-WrongDistributionReply.dat Binary files differnew file mode 100644 index 00000000000..0dbe13225ae --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.115-java-WrongDistributionReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-Priority.txt b/documentapi/test/crosslanguagefiles/5.93.30-Priority.txt new file mode 100644 index 00000000000..5fda195ddaf --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-Priority.txt @@ -0,0 +1,16 @@ +HIGHEST:0 +VERY_HIGH:1 +HIGH_1:2 +HIGH_2:3 +HIGH_3:4 +NORMAL_1:5 +NORMAL_2:6 +NORMAL_3:7 +NORMAL_4:8 +NORMAL_5:9 +NORMAL_6:10 +LOW_1:11 +LOW_2:12 +LOW_3:13 +VERY_LOW:14 +LOWEST:15 diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-BatchDocumentUpdateMessage.dat Binary files differnew file mode 100644 index 00000000000..47a8da92bda --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-BatchDocumentUpdateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-BatchDocumentUpdateReply.dat Binary files differnew file mode 100644 index 00000000000..216db17f80e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-BatchDocumentUpdateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-BatchMessage.dat Binary files differnew file mode 100644 index 00000000000..c1c43e8081b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-BatchMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-BatchReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-BatchReply.dat Binary files differnew file mode 100644 index 00000000000..be3a9ba1913 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-BatchReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-CreateVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..4ca8648e702 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-CreateVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-CreateVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..e13917227d1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-CreateVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-DestroyVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..f39b31217e6 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-DestroyVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-DestroyVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..1468f027b15 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-DestroyVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-DocumentIgnoredReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-DocumentIgnoredReply.dat Binary files differnew file mode 100644 index 00000000000..15a7afe2a59 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-DocumentIgnoredReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-DocumentListMessage.dat Binary files differnew file mode 100644 index 00000000000..2d8d12d2704 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-DocumentListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-DocumentListReply.dat Binary files differnew file mode 100644 index 00000000000..c8a1cd888f0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-DocumentListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-DocumentSummaryReply.dat Binary files differnew file mode 100644 index 00000000000..16b1e4bc4ef --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-DocumentSummaryReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-EmptyBucketsMessage.dat Binary files differnew file mode 100644 index 00000000000..b9df278fd7a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-EmptyBucketsMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-EmptyBucketsReply.dat Binary files differnew file mode 100644 index 00000000000..05510dd8c1e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-EmptyBucketsReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-EndOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-EndOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..6f33a4a0bfb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-EndOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-EndOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-EndOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..6927b7726ec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-EndOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage0.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage0.dat Binary files differnew file mode 100644 index 00000000000..7ebf108a292 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage0.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage1.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage1.dat Binary files differnew file mode 100644 index 00000000000..36a2cfd91c3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage1.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage2.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage2.dat Binary files differnew file mode 100644 index 00000000000..10514e98c37 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage2.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage3.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage3.dat Binary files differnew file mode 100644 index 00000000000..75135b8bb30 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage3.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage4.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage4.dat Binary files differnew file mode 100644 index 00000000000..5040584a275 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectMessage4.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectReply.dat Binary files differnew file mode 100644 index 00000000000..79c263cb2dc --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-GarbageCollectReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-GetBucketListMessage.dat Binary files differnew file mode 100644 index 00000000000..fa3de45ac5b --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-GetBucketListMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-GetBucketListReply.dat Binary files differnew file mode 100644 index 00000000000..830994ed785 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-GetBucketListReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-GetBucketStateMessage.dat Binary files differnew file mode 100644 index 00000000000..aa2d206ca3a --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-GetBucketStateMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-GetBucketStateReply.dat Binary files differnew file mode 100644 index 00000000000..0f91e3759f8 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-GetBucketStateReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-GetDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..3df64ed657f --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-GetDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-GetDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c1ad7920a2e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-GetDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-MapVisitorMessage.dat Binary files differnew file mode 100644 index 00000000000..1a8a837ea16 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-MapVisitorMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-MapVisitorReply.dat Binary files differnew file mode 100644 index 00000000000..541cd718a66 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-MapVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-MultiOperationMessage.dat Binary files differnew file mode 100644 index 00000000000..6efc54a6d09 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-MultiOperationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-MultiOperationReply.dat Binary files differnew file mode 100644 index 00000000000..8ad31a95bd5 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-MultiOperationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-PutDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..13c25f0b729 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-PutDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-PutDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..480544045bb --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-PutDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-QueryResultReply.dat Binary files differnew file mode 100644 index 00000000000..003f35d63a7 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-QueryResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..21f2c9b81c4 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..bf5db8761e2 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveLocationMessage.dat Binary files differnew file mode 100644 index 00000000000..16850a6aff3 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveLocationMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveLocationMessageGroup.dat Binary files differnew file mode 100644 index 00000000000..9d7901a8b6e --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveLocationMessageGroup.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveLocationMessageUser.dat Binary files differnew file mode 100644 index 00000000000..c2a63cb94c0 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveLocationMessageUser.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveLocationReply.dat Binary files differnew file mode 100644 index 00000000000..752c4dba399 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-RemoveLocationReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-SearchResultReply.dat Binary files differnew file mode 100644 index 00000000000..cce9c6f8d14 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-SearchResultReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-StartOfFeedMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-StartOfFeedMessage.dat Binary files differnew file mode 100644 index 00000000000..3c51c3c7eec --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-StartOfFeedMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-StartOfFeedReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-StartOfFeedReply.dat Binary files differnew file mode 100644 index 00000000000..ff881679155 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-StartOfFeedReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-StatBucketMessage.dat Binary files differnew file mode 100644 index 00000000000..1fc2b1cf3c1 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-StatBucketMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-StatBucketReply.dat Binary files differnew file mode 100644 index 00000000000..0b98e240018 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-StatBucketReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-UpdateDocumentMessage.dat Binary files differnew file mode 100644 index 00000000000..93274279f0d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-UpdateDocumentMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-UpdateDocumentReply.dat Binary files differnew file mode 100644 index 00000000000..c7151299366 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-UpdateDocumentReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-VisitorInfoMessage.dat Binary files differnew file mode 100644 index 00000000000..80b44e0c6fd --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-VisitorInfoMessage.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-VisitorInfoReply.dat Binary files differnew file mode 100644 index 00000000000..57a656c9b2d --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-VisitorInfoReply.dat diff --git a/documentapi/test/crosslanguagefiles/5.93.30-java-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5.93.30-java-WrongDistributionReply.dat Binary files differnew file mode 100644 index 00000000000..0dbe13225ae --- /dev/null +++ b/documentapi/test/crosslanguagefiles/5.93.30-java-WrongDistributionReply.dat diff --git a/documentapi/test/crosslanguagefiles/HEAD-cpp-golden-error-codes.txt b/documentapi/test/crosslanguagefiles/HEAD-cpp-golden-error-codes.txt new file mode 100644 index 00000000000..6917c1fa578 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/HEAD-cpp-golden-error-codes.txt @@ -0,0 +1,25 @@ +ERROR_ABORTED 151004 +ERROR_BUCKET_DELETED 151012 +ERROR_BUCKET_NOT_FOUND 151009 +ERROR_BUSY 151005 +ERROR_DISK_FAILURE 151007 +ERROR_DOCUMENT_EXISTS 251002 +ERROR_DOCUMENT_NOT_FOUND 251001 +ERROR_IGNORED 251010 +ERROR_ILLEGAL_PARAMETERS 251005 +ERROR_INTERNAL_FAILURE 251011 +ERROR_IO_FAILURE 151008 +ERROR_MESSAGE_IGNORED 250001 +ERROR_NODE_NOT_READY 151001 +ERROR_NOT_CONNECTED 151006 +ERROR_NOT_IMPLEMENTED 251004 +ERROR_NO_SPACE 251009 +ERROR_POLICY_FAILURE 250002 +ERROR_PROCESSING_FAILURE 252001 +ERROR_REJECTED 251012 +ERROR_STALE_TIMESTAMP 151013 +ERROR_SUSPENDED 152001 +ERROR_TEST_AND_SET_CONDITION_FAILED 251013 +ERROR_TIMESTAMP_EXIST 252002 +ERROR_UNKNOWN_COMMAND 251007 +ERROR_WRONG_DISTRIBUTION 151002
\ No newline at end of file diff --git a/documentapi/test/crosslanguagefiles/HEAD-java-golden-error-codes.txt b/documentapi/test/crosslanguagefiles/HEAD-java-golden-error-codes.txt new file mode 100644 index 00000000000..6917c1fa578 --- /dev/null +++ b/documentapi/test/crosslanguagefiles/HEAD-java-golden-error-codes.txt @@ -0,0 +1,25 @@ +ERROR_ABORTED 151004 +ERROR_BUCKET_DELETED 151012 +ERROR_BUCKET_NOT_FOUND 151009 +ERROR_BUSY 151005 +ERROR_DISK_FAILURE 151007 +ERROR_DOCUMENT_EXISTS 251002 +ERROR_DOCUMENT_NOT_FOUND 251001 +ERROR_IGNORED 251010 +ERROR_ILLEGAL_PARAMETERS 251005 +ERROR_INTERNAL_FAILURE 251011 +ERROR_IO_FAILURE 151008 +ERROR_MESSAGE_IGNORED 250001 +ERROR_NODE_NOT_READY 151001 +ERROR_NOT_CONNECTED 151006 +ERROR_NOT_IMPLEMENTED 251004 +ERROR_NO_SPACE 251009 +ERROR_POLICY_FAILURE 250002 +ERROR_PROCESSING_FAILURE 252001 +ERROR_REJECTED 251012 +ERROR_STALE_TIMESTAMP 151013 +ERROR_SUSPENDED 152001 +ERROR_TEST_AND_SET_CONDITION_FAILED 251013 +ERROR_TIMESTAMP_EXIST 252002 +ERROR_UNKNOWN_COMMAND 251007 +ERROR_WRONG_DISTRIBUTION 151002
\ No newline at end of file diff --git a/documentapi/test/crosslanguagefiles/README b/documentapi/test/crosslanguagefiles/README new file mode 100644 index 00000000000..8089a7cdddd --- /dev/null +++ b/documentapi/test/crosslanguagefiles/README @@ -0,0 +1 @@ +Cross language test files are placed here. Do not remove directory
\ No newline at end of file |