diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /config |
Publish
Diffstat (limited to 'config')
574 files changed, 38627 insertions, 0 deletions
diff --git a/config/.gitignore b/config/.gitignore new file mode 100644 index 00000000000..170b98b585f --- /dev/null +++ b/config/.gitignore @@ -0,0 +1,4 @@ +/pom.xml.build +/target +Makefile +Testing diff --git a/config/CMakeLists.txt b/config/CMakeLists.txt new file mode 100644 index 00000000000..b6e0f7de178 --- /dev/null +++ b/config/CMakeLists.txt @@ -0,0 +1,58 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + DEPENDS + fastos + vespalib + staging_vespalib + vespalog + fnet + + EXTERNAL_DEPENDS + lz4 + + LIBS + src/apps/configproxy-cmd + src/apps/getvespaconfig + src/apps/pingproxy + src/vespa/config + src/vespa/config/common + src/vespa/config/configgen + src/vespa/config/file + src/vespa/config/frt + src/vespa/config/helper + src/vespa/config/print + src/vespa/config/raw + src/vespa/config/retriever + src/vespa/config/set + src/vespa/config/subscription + + APPS + src/tests/configfetcher + src/tests/print + src/tests/payload_converter + src/tests/api + src/tests/configholder + src/tests/unittest + src/tests/frtconnectionpool + src/tests/file_subscription + src/tests/legacysubscriber + src/tests/subscriber + src/tests/configagent + src/tests/getconfig + src/tests/configgen + src/tests/configmanager + src/tests/raw_subscription + src/tests/trace + src/tests/misc + src/tests/configformat + src/tests/frt + src/tests/configretriever + src/tests/functiontest + src/tests/configuri + src/tests/failover + src/tests/subscription + src/tests/configparser +) + +vespa_install_script(src/apps/activate-application/activate-application.sh activate-application bin) +vespa_install_script(src/apps/vespa-config/vespa-config.pl libexec/vespa) diff --git a/config/OWNERS b/config/OWNERS new file mode 100644 index 00000000000..7028eebe31a --- /dev/null +++ b/config/OWNERS @@ -0,0 +1,2 @@ +musum +arnej27959 diff --git a/config/doc/java/batchConfigurer.png b/config/doc/java/batchConfigurer.png Binary files differnew file mode 100644 index 00000000000..18adf8663ba --- /dev/null +++ b/config/doc/java/batchConfigurer.png diff --git a/config/doc/java/classes.vsd b/config/doc/java/classes.vsd Binary files differnew file mode 100644 index 00000000000..9f5fa882790 --- /dev/null +++ b/config/doc/java/classes.vsd diff --git a/config/doc/java/client.png b/config/doc/java/client.png Binary files differnew file mode 100644 index 00000000000..afd84114fa8 --- /dev/null +++ b/config/doc/java/client.png diff --git a/config/doc/java/client_nodes.png b/config/doc/java/client_nodes.png Binary files differnew file mode 100644 index 00000000000..20f29fac093 --- /dev/null +++ b/config/doc/java/client_nodes.png diff --git a/config/doc/java/configHelper.png b/config/doc/java/configHelper.png Binary files differnew file mode 100644 index 00000000000..ef553eee485 --- /dev/null +++ b/config/doc/java/configHelper.png diff --git a/config/doc/java/proxy.png b/config/doc/java/proxy.png Binary files differnew file mode 100644 index 00000000000..91c767f0e4a --- /dev/null +++ b/config/doc/java/proxy.png diff --git a/config/doc/java/proxy_error_configured.png b/config/doc/java/proxy_error_configured.png Binary files differnew file mode 100644 index 00000000000..279a7dac3bb --- /dev/null +++ b/config/doc/java/proxy_error_configured.png diff --git a/config/doc/java/proxy_error_subscribe.png b/config/doc/java/proxy_error_subscribe.png Binary files differnew file mode 100644 index 00000000000..37b9f1d66a2 --- /dev/null +++ b/config/doc/java/proxy_error_subscribe.png diff --git a/config/doc/java/proxy_interaction.png b/config/doc/java/proxy_interaction.png Binary files differnew file mode 100644 index 00000000000..e79dafcd48c --- /dev/null +++ b/config/doc/java/proxy_interaction.png diff --git a/config/doc/java/proxy_reload.png b/config/doc/java/proxy_reload.png Binary files differnew file mode 100644 index 00000000000..0103643f9ab --- /dev/null +++ b/config/doc/java/proxy_reload.png diff --git a/config/doc/library-design.txt b/config/doc/library-design.txt new file mode 100644 index 00000000000..cbea6e283b2 --- /dev/null +++ b/config/doc/library-design.txt @@ -0,0 +1,90 @@ +# Config library + +## Introduction + +The config library is used by Vespa applications to subscribe to +configuration from the Vespa config system. + +The low-level <a href="protocol-design.html">config protocol</a> is +used for communication between the application and a config source. + +The config library has Java and C++ implementations. +Implementation-specific issues are noted at the end of this document. + + +## Config API + +The config API that are used by clients will be mostly unchanged from +previous versions of Vespa. + +## Config subscriptions + +A client application generates config code based on config definition +files as described in the user documentation. +An application will implement a Subscriber +interface with a *configure()* callback and call the generated code's +*subscribe()* method to get a particular config. *subscribe()* will +not return until *configure()* has been called and the application is +configured, or some fatal error occured which will lead to an +exception being thrown. + +The *subscribe()* call will add the client to the list of subscribers +for this config and create a new Subscription object if there does not +exist one already for this config. + +The Subscription object is the central object for communication +between the client and the config source. When such an object is +created it will lead to a *getConfig()* (see <a +href="protocol-design.html">config protocol</a> documentation) call. +The Subscription object will make sure that *getConfig()* is called +and waiting for a response throughout the application's lifetime. +That way, new config will be discovered when this method call returns, +which it will do immediately if the subscribed config changes at the +config source. At the same time, since the server timeout defined in +the protocol can be set to a high number, generating unnecessary +network traffic by polling frequently is avoided. + +When the *getConfig()* call returns, a new *getConfig()* call is +scheduled for execution at a later time. If the response was +successful, this will happen immediately. If there was an error the +delay until the call will be performed depends on the number of times +since last succesful execution, and if the application has been +configured already or not (we want to try more aggrressively if the +aplication has not been configured). There is a maximum delay defined +for this scheduling. + + +## Config sources + +A config source can either be a config server or a config proxy. The +default behavior for applications is to use one local config source, a +config proxy on localhost, port 19090. It is possible to use one or +more other sources too, by setting the environment variable +VESPA\_CONFIG\_SOURCES (a comma-separated list of hostnames, with +optional port number, like _foo,bar_ or _foo:1234,bar:2345_). + +### Selecting config source + +The config library selects a config source when requesting config +(performing the *getConfig()* method call) in a way that makes all +config requests from one paricular Internet host address use the same +config source (unless it is suspended, i.e. down, inaccessible etc.). + +A config source can experience both transient and fatal errors. The +config library (and config proxy) will when configured with several +sources suspend a source for a period of time, where the suspension +time is based on the type of error and the number of times the error +has happened. A suspended config source will not be considered when +doing a new selection of config source, except if it is the only +source configured. + +As an example, a transient failure that happens 5 times will lead to +the config source being suspended for 10 seconds the first time it +happens, 20 seconds the next time and so on. A more permanent error +will lead to similar behavior, except the supension times will be +higher. There is a maximum delay for both types of errors. + + +## C++ library + +## Java library diff --git a/config/doc/protocol-design.txt b/config/doc/protocol-design.txt new file mode 100644 index 00000000000..ba7fd7d7538 --- /dev/null +++ b/config/doc/protocol-design.txt @@ -0,0 +1,85 @@ +# Config protocol + +## Introduction + +This document describes the low-level config protocol based on fnet/rpc. + +## Overview of the protocol + +The protocol is stateless and similiar to HTTP GET and other REST +APIs, but implemented as remote procedure calls (RPC). A client +performs a method call get(), and a server returns a response if the +config has changed since the last time get() was called, or waits a +specified time before returning (unless the config changes before the +timeout is reached, in case it returns immediately). Since the +protocol is stateless, it is easy to use more than one config server +to serve config. It is also possible to use a (cacheing) proxy +between client and server. + +The figure below shows a simplified view of how an application *App* +uses the config library API to subscribe to a config. The protocol is +shown as *get()* call and *ret()* responses. Optionally, a proxy can +be used between client and the server. + +<img src='rpc-config-protocol.png' alt='RPC config protocol' /> + +In the figure above, the first parameter *X* is a designator for the +config to get, the second parameter is a generation number and the +third parameter *T* is a server timeout. + +The server (or proxy, if the client is using one) will generate a +response immediately if the config *X* has another (higher) generation +number than the one requested. If not, the server will create a timer +with timeout *T* and respond when the timer expires or the requested +config changes, whatever comes first. + +If the response was generated because the config changed, the config +payload will be included in the response and a change flag set to mark +that the response contains changed config. Else the change flag will +not be set (and no config payload included in the response). + +The client timeout (how long the client waits for an answer before +giving up), should be longer than the server timeout *T*. + + +## Implementation + +The implementation of get() is an RPC method called getConfig() with +the following signature: + + getConfig(String configId, String defName, String defVersion, String defMD5, String configMD5, long timeout) + +and the parameters: + +* *configId* - config id +* *defName* - config definition name +* *defVersion* - config definition version +* *defMD5* - config definition md5sum +* *configMD5* - config md5sum (the md5sum of the config payload) +* *timeout* - server timeout + +The fifth parameter, the config md5sum, is used instead of a +generation number. If a config is found, the md5sum of the config +payload will be returned in the response, so that subsequent calls to +getConfig() will use this new md5sum as *configMD5*. When getting +config the first time, this parameter will be an empty string. + + +The return parameters are: + + String configId, String defName, String defVersion, String defMD5, String configMD5, int changed, long generation, String payload) + +with the first 5 parameters being the same as the one in the +request. The rest are: + +* *changed* - a flag the will have a value of 1 if there is a config + *payload* in the response, or 0 else. +* *generation* - generation when the config was last changed (in milliseconds since 1970). +* *payload* - config payload. Will only have a value if *changed* is 1. + +The payload is the same as in the previous version of this protocol. + +The *generation* parameter can be used to check how old the config +is. In case of getting a new payload (*changed* is 1, this can be used +to check that the config returned has a newer timstamp thatn the last +returned from a config source (this diff --git a/config/pom.xml b/config/pom.xml new file mode 100755 index 00000000000..4dd4e1a37fd --- /dev/null +++ b/config/pom.xml @@ -0,0 +1,210 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>config</artifactId> + <packaging>container-plugin</packaging> + <version>6-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>annotations</artifactId> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-lib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configgen</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jrt</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>yolean</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>uk.co.datumedge</groupId> + <artifactId>hamcrest-json</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>13.0.1</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava-testlib</artifactId> + <version>17.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>testutil</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </dependency> + <dependency> + <groupId>net.jpountz.lz4</groupId> + <artifactId>lz4</artifactId> + </dependency> + </dependencies> + <profiles> + <profile> + <id>stress-tests</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <includes> + <include>**/ConfigserverSubscriptionTest.java</include> + <include>**/ProxyServerTest.java</include> + </includes> + </configuration> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>factory-tests</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <excludes> + <exclude>**/ConfigserverSubscriptionTest.java</exclude> + <exclude>**/ProxyServerTest.java</exclude> + </excludes> + </configuration> + </plugin> + </plugins> + </build> + </profile> + </profiles> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <id>generate-vtag</id> + <phase>generate-sources</phase> + <goals> + <goal>java</goal> + </goals> + <configuration> + <mainClass>com.yahoo.vespa.VersionTagger</mainClass> + <arguments> + <argument>${project.basedir}/../dist/vtag.map</argument> + <argument>com.yahoo.vespa.config</argument> + <argument>${project.build.directory}/generated-sources/vtag</argument> + </arguments> + <sourceRoot>${project.build.directory}/generated-sources/vtag</sourceRoot> + <classpathScope>compile</classpathScope> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-class-plugin</artifactId> + <version>${project.version}</version> + <executions> + <execution> + <id>configgen-test-defs</id> + <phase>generate-test-sources</phase> + <goals> + <goal>config-gen</goal> + </goals> + <configuration> + <defFilesDirectories>src/test/resources/configs/def-files</defFilesDirectories> + <outputDirectory>target/generated-test-sources/vespa-configgen-plugin</outputDirectory> + <testConfig>true</testConfig> + <requireNamespace>false</requireNamespace> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> + <forkMode>once</forkMode> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/config/src/.gitignore b/config/src/.gitignore new file mode 100644 index 00000000000..528c8f7183d --- /dev/null +++ b/config/src/.gitignore @@ -0,0 +1,4 @@ +/Makefile.ini +/config.mak +/config_command.sh +/project.dsw diff --git a/config/src/Doxyfile b/config/src/Doxyfile new file mode 100644 index 00000000000..19c31437cdb --- /dev/null +++ b/config/src/Doxyfile @@ -0,0 +1,1255 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Doxyfile 1.4.7 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = cloudconfig + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = ../../doc + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, +# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, +# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, +# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, +# Swedish, and Ukrainian. + +OUTPUT_LANGUAGE = English + +# This tag can be used to specify the encoding used in the generated output. +# The encoding is not always determined by the language that is chosen, +# but also whether or not the output is meant for Windows or non-Windows users. +# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES +# forces the Windows encoding (this is the default for the Windows binary), +# whereas setting the tag to NO uses a Unix-style encoding (the default for +# all platforms other than Windows). + +USE_WINDOWS_ENCODING = NO + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = YES + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like the Qt-style comments (thus requiring an +# explicit @brief command for a brief description. + +JAVADOC_AUTOBRIEF = YES + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the DETAILS_AT_TOP tag is set to YES then Doxygen +# will output the detailed description near the top, like JavaDoc. +# If set to NO, the detailed description appears after the member +# documentation. + +DETAILS_AT_TOP = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for Java. +# For instance, namespaces will be presented as packages, qualified scopes +# will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to +# include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = NO + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from the +# version control system). Doxygen will invoke the program by executing (via +# popen()) the command <command> <input-file>, where <command> is the value of +# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = config + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py + +FILE_PATTERNS = *.h \ + *.hpp \ + *.cpp + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command <filter> <input-file>, where <filter> +# is the value of the INPUT_FILTER tag, and <input-file> is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES (the default) +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = YES + +# If the REFERENCES_RELATION tag is set to YES (the default) +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = YES + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. Otherwise they will link to the documentstion. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = NO + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be +# generated containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, +# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are +# probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = NO + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = NO + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = IAM_DOXYGEN + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will +# generate a call dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then doxygen will +# generate a caller dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable caller graphs for selected +# functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width +# (in pixels) of the graphs generated by dot. If a graph becomes larger than +# this value, doxygen will try to truncate the graph, so that it fits within +# the specified constraint. Beware that most browsers cannot cope with very +# large images. + +MAX_DOT_GRAPH_WIDTH = 1024 + +# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height +# (in pixels) of the graphs generated by dot. If a graph becomes larger than +# this value, doxygen will try to truncate the graph, so that it fits within +# the specified constraint. Beware that most browsers cannot cope with very +# large images. + +MAX_DOT_GRAPH_HEIGHT = 1024 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that a graph may be further truncated if the graph's +# image dimensions are not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH +# and MAX_DOT_GRAPH_HEIGHT). If 0 is used for the depth value (the default), +# the graph is not depth-constrained. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, which results in a white background. +# Warning: Depending on the platform used, enabling this option may lead to +# badly anti-aliased labels on the edges of a graph (i.e. they become hard to +# read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to the search engine +#--------------------------------------------------------------------------- + +# The SEARCHENGINE tag specifies whether or not a search engine should be +# used. If set to NO the values of all tags below this one will be ignored. + +SEARCHENGINE = NO diff --git a/config/src/apps/activate-application/activate-application.sh b/config/src/apps/activate-application/activate-application.sh new file mode 100644 index 00000000000..82da8a1ab53 --- /dev/null +++ b/config/src/apps/activate-application/activate-application.sh @@ -0,0 +1,79 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +ROOT=$VESPA_HOME + +if [ "-f" == "$1" ] ; then + $ROOT/bin/deploy activate +else + STATUS=$($ROOT/bin/vespa-status-filedistribution) + if [ $? -eq 0 ] ; then + $ROOT/bin/deploy activate + else + echo "$STATUS" + echo + echo "Files are currently being distributed." + echo "If you want to see the status, call 'vespa-status-filedistribution'." + echo "Otherwise, call 'activate-application -f' to activate the application now; the file transfers will continue in the background." + exit 1 + fi +fi diff --git a/config/src/apps/configproxy-cmd/.gitignore b/config/src/apps/configproxy-cmd/.gitignore new file mode 100644 index 00000000000..e39ac4f0c99 --- /dev/null +++ b/config/src/apps/configproxy-cmd/.gitignore @@ -0,0 +1,3 @@ +/.depend +/Makefile +/configproxy-cmd diff --git a/config/src/apps/configproxy-cmd/CMakeLists.txt b/config/src/apps/configproxy-cmd/CMakeLists.txt new file mode 100644 index 00000000000..17e941cde08 --- /dev/null +++ b/config/src/apps/configproxy-cmd/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_configproxy-cmd_app + SOURCES + main.cpp + methods.cpp + proxycmd.cpp + OUTPUT_NAME configproxy-cmd + INSTALL bin + DEPENDS +) diff --git a/config/src/apps/configproxy-cmd/flags.h b/config/src/apps/configproxy-cmd/flags.h new file mode 100644 index 00000000000..ce64889256b --- /dev/null +++ b/config/src/apps/configproxy-cmd/flags.h @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vector> + +struct Flags { + vespalib::string method; + std::vector<vespalib::string> args; + vespalib::string hostname; + int portnumber; + Flags() : method("cache"), args(), hostname("localhost"), portnumber(19090) {} +}; + diff --git a/config/src/apps/configproxy-cmd/main.cpp b/config/src/apps/configproxy-cmd/main.cpp new file mode 100644 index 00000000000..34f2eeab143 --- /dev/null +++ b/config/src/apps/configproxy-cmd/main.cpp @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <iostream> +#include <vespa/vespalib/stllike/string.h> +#include "flags.h" +#include "proxycmd.h" +#include "methods.h" + +class Application : public FastOS_Application +{ + Flags _flags; + bool parseOpts(); +public: + void usage(void); + int Main(void); + + Application() : _flags() {} +}; + +bool +Application::parseOpts() +{ + char c = '?'; + const char *optArg = NULL; + int optInd = 0; + while ((c = GetOpt("m:s:p:", optArg, optInd)) != -1) { + switch (c) { + case 'm': + _flags.method = optArg; + break; + case 's': + _flags.hostname = optArg; + break; + case 'p': + _flags.portnumber = atoi(optArg); + break; + case 'h': + default: + return false; // triggers usage() + } + } + const Method method = methods::find(_flags.method); + if (optInd + method.args <= _argc) { + for (int i = 0; i < method.args; ++i) { + vespalib::string arg = _argv[optInd++]; + _flags.args.push_back(arg); + } + } else { + std::cerr << "ERROR: method "<< _flags.method << " requires " << method.args + << " arguments, only got " << (_argc - optInd) << std::endl; + return false; + } + if (optInd != _argc) { + std::cerr << "ERROR: "<<(_argc - optInd)<<" extra arguments\n"; + return false; + } + _flags.method = method.rpcMethod; + return true; +} + +void +Application::usage(void) +{ + std::cerr << + "Usage: configproxy-cmd [options]" << std::endl << + " -m <method> method" << std::endl << + " -s <hostname> hostname (default: localhost)" << std::endl << + " -p <port> port number (default: 19090)" << std::endl << + "Available methods for -m option:" << std::endl; + methods::dump(); +} + +int +Application::Main(void) +{ + if (! parseOpts()) { + usage(); + return 1; + } + ProxyCmd client(_flags); + return client.action(); +} + +int +main(int argc, char** argv) +{ + Application app; + return app.Entry(argc, argv); +} diff --git a/config/src/apps/configproxy-cmd/methods.cpp b/config/src/apps/configproxy-cmd/methods.cpp new file mode 100644 index 00000000000..8345316c6a4 --- /dev/null +++ b/config/src/apps/configproxy-cmd/methods.cpp @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "methods.h" +#include <iostream> + +namespace methods { + +const Method methods[] = { + { "cache", "listCachedConfig", 0 }, + { "dumpcache", "dumpCache", 1 }, // filename + { "getConfig", "getConfig", 7 }, // defName defVersion defMD5 configid configMD5 timestamp timeout + { "getmode", "getMode", 0 }, + { "invalidatecache", "invalidateCache", 0 }, + { "cachefull", "listCachedConfigFull", 0 }, + { "sources", "listSourceConnections", 0 }, + { "statistics", "printStatistics", 0 }, + { "setmode", "setMode", 1 }, // { default | memorycache | diskcache } + { "updatesources", "updateSources", 1 }, + { 0, 0, 0} +}; + +const Method find(const vespalib::string &name) { + for (size_t i = 0; methods[i].shortName != 0; ++i) { + if (name == methods[i].shortName) { + return methods[i]; + } + } + Method rv = { name.c_str(), name.c_str(), 0 }; + return rv; +} + +void dump() { + std::cerr << " "; + size_t i = 0; + for (;;) { + std::cerr << methods[i++].shortName; + if (methods[i].shortName == 0) { + break; + } + std::cerr << ","; + } + std::cerr << std::endl; +} + +}; diff --git a/config/src/apps/configproxy-cmd/methods.h b/config/src/apps/configproxy-cmd/methods.h new file mode 100644 index 00000000000..01f54a0a9d9 --- /dev/null +++ b/config/src/apps/configproxy-cmd/methods.h @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +struct Method { + const char *shortName; + const char *rpcMethod; + const int args; +}; + +namespace methods { + +const Method find(const vespalib::string &name); +void dump(); + +}; + diff --git a/config/src/apps/configproxy-cmd/proxycmd.cpp b/config/src/apps/configproxy-cmd/proxycmd.cpp new file mode 100644 index 00000000000..34bd354c79f --- /dev/null +++ b/config/src/apps/configproxy-cmd/proxycmd.cpp @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "proxycmd.h" +#include <iostream> +#include <vespa/vespalib/util/vstringfmt.h> + +ProxyCmd::ProxyCmd(const Flags& flags) + : _supervisor(NULL), + _target(NULL), + _req(NULL), + _flags(flags) +{} + +void ProxyCmd::initRPC() { + _supervisor = new FRT_Supervisor(); + _req = _supervisor->AllocRPCRequest(); + _supervisor->Start(); +} + +void ProxyCmd::invokeRPC() { + if (_req == NULL) return; + _target->InvokeSync(_req, 65.0); +} + +void ProxyCmd::finiRPC() { + if (_req != NULL) { + _req->SubRef(); + _req = NULL; + } + if (_target != NULL) { + _target->SubRef(); + _target = NULL; + } + if (_supervisor != NULL) { + _supervisor->ShutDown(true); + delete _supervisor; + _supervisor = NULL; + } +} + +void ProxyCmd::printArray(FRT_Values *rvals) { + FRT_Value &lines = rvals->GetValue(0); + for (size_t i = 0; i < lines._string_array._len; ++i) { + std::cout << lines._string_array._pt[i]._str << std::endl; + } +} + +vespalib::string ProxyCmd::makeSpec() { + return vespalib::make_vespa_string("tcp/%s:%d", _flags.hostname.c_str(), _flags.portnumber); +} + +void ProxyCmd::autoPrint() { + if (_req->IsError()) { + std::cerr << "FAILURE ["<< _req->GetMethodName() <<"]: " << _req->GetErrorMessage() << std::endl; + return; + } + vespalib::string retspec = _req->GetReturnSpec(); + FRT_Values *rvals = _req->GetReturn(); + if (retspec == "S") { + printArray(rvals); + } else if (retspec == "s") { + std::cout << rvals->GetValue(0)._string._str << std::endl; + } else if (retspec == "i") { + std::cout << rvals->GetValue(0)._intval32 << std::endl; + } else { + _req->Print(); + } +} + +int ProxyCmd::action() { + int errors = 0; + initRPC(); + vespalib::string spec = makeSpec(); + _target = _supervisor->GetTarget(spec.c_str()); + _req->SetMethodName(_flags.method.c_str()); + FRT_Values ¶ms = *_req->GetParams(); + for (size_t i = 0; i < _flags.args.size(); ++i) { + params.AddString(_flags.args[i].c_str(), _flags.args[i].size()); + } + invokeRPC(); + if (_req->IsError()) ++errors; + autoPrint(); + finiRPC(); + return errors; +} diff --git a/config/src/apps/configproxy-cmd/proxycmd.h b/config/src/apps/configproxy-cmd/proxycmd.h new file mode 100644 index 00000000000..6fc8261991f --- /dev/null +++ b/config/src/apps/configproxy-cmd/proxycmd.h @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "flags.h" +#include <vespa/fnet/frt/frt.h> + + +class ProxyCmd +{ +private: + FRT_Supervisor *_supervisor; + FRT_Target *_target; + FRT_RPCRequest *_req; + Flags _flags; + + void initRPC(); + void invokeRPC(); + void finiRPC(); + void printArray(FRT_Values *rvals); + vespalib::string makeSpec(); + void autoPrint(); +public: + ProxyCmd(const Flags& flags); + + virtual ~ProxyCmd() {} + + int action(); +}; + + diff --git a/config/src/apps/getvespaconfig/.gitignore b/config/src/apps/getvespaconfig/.gitignore new file mode 100644 index 00000000000..900ac81ea9b --- /dev/null +++ b/config/src/apps/getvespaconfig/.gitignore @@ -0,0 +1,4 @@ +/.depend +/Makefile +/getvespaconfig +getvespaconfig-bin diff --git a/config/src/apps/getvespaconfig/CMakeLists.txt b/config/src/apps/getvespaconfig/CMakeLists.txt new file mode 100644 index 00000000000..93022581b30 --- /dev/null +++ b/config/src/apps/getvespaconfig/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_getvespaconfig_app + SOURCES + getconfig.cpp + OUTPUT_NAME getvespaconfig-bin + INSTALL bin + DEPENDS + config_cloudconfig +) diff --git a/config/src/apps/getvespaconfig/getconfig.cpp b/config/src/apps/getvespaconfig/getconfig.cpp new file mode 100644 index 00000000000..95e804381d0 --- /dev/null +++ b/config/src/apps/getvespaconfig/getconfig.cpp @@ -0,0 +1,265 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> + +#include <vespa/log/log.h> +LOG_SETUP("getconfig"); + +#include <vespa/fnet/frt/frt.h> +#include <vespa/config/config.h> +#include <vespa/config/frt/protocol.h> +#include <vespa/config/frt/frtconfigrequestfactory.h> +#include <vespa/config/frt/frtconnection.h> +#include <vespa/config/common/payload_converter.h> +#include <vespa/config/common/vespa_version.h> + +#include <string> +#include <sstream> +#include <fstream> + +using namespace config; + +class GetConfig : public FastOS_Application +{ +private: + FRT_Supervisor *_supervisor; + FRT_Target *_target; + + GetConfig(const GetConfig &); + GetConfig &operator=(const GetConfig &); + +public: + GetConfig() : _supervisor(NULL), _target(NULL) {} + virtual ~GetConfig(); + int usage(); + void initRPC(const char *spec); + void finiRPC(); + virtual int Main(); +}; + + +GetConfig::~GetConfig() +{ + LOG_ASSERT(_supervisor == NULL); + LOG_ASSERT(_target == NULL); +} + + +int +GetConfig::usage() +{ + fprintf(stderr, "usage: %s -n name -i configId\n", _argv[0]); + fprintf(stderr, "-n name (config name, including namespace, on the form <namespace>.<name>)\n"); + fprintf(stderr, "-i configId (config id, optional)\n"); + fprintf(stderr, "-j (output config as json)\n"); + fprintf(stderr, "-a schema (config def schema file, optional)\n"); + fprintf(stderr, "-v defVersion (config definition version, optional, deprecated)\n"); + fprintf(stderr, "-m defMd5 (definition md5sum, optional)\n"); + fprintf(stderr, "-c configMd5 (config md5sum, optional)\n"); + fprintf(stderr, "-t serverTimeout (server timeout in seconds, default 3)\n"); + fprintf(stderr, "-w timeout (timeout in seconds, default 10)\n"); + fprintf(stderr, "-s server (server hostname, default localhost)\n"); + fprintf(stderr, "-p port (proxy/server port number, default 19090)\n"); + fprintf(stderr, "-r traceLevel (tracelevel to use in request, default 0\n"); + fprintf(stderr, "-V vespaVersion (vespa version to use in request, optional\n"); + fprintf(stderr, "-d (debug mode)\n"); + fprintf(stderr, "-h (This help text)\n"); + return 1; +} + + +void +GetConfig::initRPC(const char *spec) +{ + _supervisor = new FRT_Supervisor(); + _target = _supervisor->GetTarget(spec); + _supervisor->Start(); +} + + +void +GetConfig::finiRPC() +{ + if (_target != NULL) { + _target->SubRef(); + _target = NULL; + } + if (_supervisor != NULL) { + _supervisor->ShutDown(true); + delete _supervisor; + _supervisor = NULL; + } +} + + +int +GetConfig::Main() +{ + bool debugging = false; + char c = -1; + + std::vector<vespalib::string> defSchema; + const char *schema = NULL; + const char *defName = NULL; + const char *defMD5 = ""; + std::string defNamespace("config"); + const char *serverHost = "localhost"; + const char *configId = getenv("VESPA_CONFIG_ID"); + bool printAsJson = false; + int traceLevel = config::protocol::readTraceLevel(); + const char *vespaVersionString = nullptr; + + if (configId == NULL) { + configId = ""; + } + const char *configMD5 = ""; + int serverTimeout = 3; + int clientTimeout = 10; + + int serverPort = 19090; + + const char *optArg = NULL; + int optInd = 0; + while ((c = GetOpt("a:n:v:i:jm:c:t:V:w:r:s:p:dh", optArg, optInd)) != -1) { + int retval = 1; + switch (c) { + case 'a': + schema = optArg; + break; + case 'n': + defName = optArg; + break; + case 'v': + break; + case 'i': + configId = optArg; + break; + case 'j': + printAsJson = true; + break; + case 'm': + defMD5 = optArg; + break; + case 'c': + configMD5 = optArg; + break; + case 't': + serverTimeout = atoi(optArg); + break; + case 'w': + clientTimeout = atoi(optArg); + break; + case 'r': + traceLevel = atoi(optArg); + break; + case 'V': + vespaVersionString = optArg; + break; + case 's': + serverHost = optArg; + break; + case 'p': + serverPort = atoi(optArg); + break; + case 'd': + debugging = true; + break; + case 'h': + retval = 0; + case '?': + default: + usage(); + return retval; + } + } + + if (defName == NULL || serverPort == 0) { + usage(); + return 1; + } + + if (strchr(defName, '.') != NULL) { + const char *tmp = defName; + defName = strrchr(defName, '.'); + defName++; + defNamespace = std::string(tmp, defName - tmp - 1); + } + + if (schema != NULL) { + std::ifstream is; + is.open(schema); + std::string item; + while (std::getline(is, item)) { + if (item.find("namespace=") == std::string::npos) { + defSchema.push_back(item); + } + } + is.close(); + } + std::ostringstream tmp; + tmp << "tcp/"; + tmp << serverHost; + tmp << ":"; + tmp << serverPort; + std::string sspec = tmp.str(); + const char *spec = sspec.c_str(); + if (debugging) { + printf("connecting to '%s'\n", spec); + } + initRPC(spec); + + auto vespaVersion = VespaVersion::getCurrentVersion(); + if (vespaVersionString != nullptr) { + vespaVersion = VespaVersion::fromString(vespaVersionString); + } + + int protocolVersion = config::protocol::readProtocolVersion(); + FRTConfigRequestFactory requestFactory(protocolVersion, traceLevel, vespaVersion, config::protocol::readProtocolCompressionType()); + FRTConnection connection(spec, *_supervisor, TimingValues()); + ConfigKey key(configId, defName, defNamespace, defMD5, defSchema); + ConfigState state(configMD5, 0); + FRTConfigRequest::UP request = requestFactory.createConfigRequest(key, &connection, state, serverTimeout * 1000); + + _target->InvokeSync(request->getRequest(), clientTimeout); // seconds + + ConfigResponse::UP response = request->createResponse(request->getRequest()); + response->validateResponse(); + if (response->isError()) { + fprintf(stderr, "error %d: %s\n", + response->errorCode(), response->errorMessage().c_str()); + } else { + response->fill(); + ConfigKey rKey(response->getKey()); + ConfigState rState(response->getConfigState()); + ConfigValue rValue(response->getValue()); + if (debugging) { + printf("defName %s\n", rKey.getDefName().c_str()); + printf("defMD5 %s\n", rKey.getDefMd5().c_str()); + printf("defNamespace %s\n", rKey.getDefNamespace().c_str()); + + printf("configID %s\n", rKey.getConfigId().c_str()); + printf("configMD5 %s\n", rState.md5.c_str()); + + printf("generation %" PRId64 "\n", rState.generation); + printf("trace %s\n", response->getTrace().toString().c_str()); + } else if (traceLevel > 0) { + printf("trace %s\n", response->getTrace().toString().c_str()); + } + // TODO: Make printAsJson default + if (printAsJson) { + printf("%s\n", rValue.asJson().c_str()); + } else { + std::vector<vespalib::string> lines = rValue.getLegacyFormat(); + for (uint32_t j = 0; j < lines.size(); j++) { + printf("%s\n", lines[j].c_str()); + } + } + } + finiRPC(); + return 0; +} + +int main(int argc, char **argv) +{ + GetConfig app; + return app.Entry(argc, argv); +} diff --git a/config/src/apps/pingproxy/.gitignore b/config/src/apps/pingproxy/.gitignore new file mode 100644 index 00000000000..778ea35818c --- /dev/null +++ b/config/src/apps/pingproxy/.gitignore @@ -0,0 +1,3 @@ +/.depend +/Makefile +/pingproxy diff --git a/config/src/apps/pingproxy/CMakeLists.txt b/config/src/apps/pingproxy/CMakeLists.txt new file mode 100644 index 00000000000..60f06a4399e --- /dev/null +++ b/config/src/apps/pingproxy/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_pingproxy_app + SOURCES + pingproxy.cpp + OUTPUT_NAME pingproxy + INSTALL bin + DEPENDS +) diff --git a/config/src/apps/pingproxy/pingproxy.cpp b/config/src/apps/pingproxy/pingproxy.cpp new file mode 100644 index 00000000000..444d024e240 --- /dev/null +++ b/config/src/apps/pingproxy/pingproxy.cpp @@ -0,0 +1,157 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> + +#include <vespa/log/log.h> +LOG_SETUP("pingproxy"); + +#include <vespa/fnet/frt/frt.h> + +#include <string> +#include <sstream> + + +class PingProxy : public FastOS_Application +{ +private: + FRT_Supervisor *_supervisor; + FRT_Target *_target; + + PingProxy(const PingProxy &); + PingProxy &operator=(const PingProxy &); + +public: + PingProxy() : _supervisor(NULL), _target(NULL) {} + virtual ~PingProxy(); + int usage(); + void initRPC(const char *spec); + void finiRPC(); + virtual int Main(); +}; + + +PingProxy::~PingProxy() +{ + LOG_ASSERT(_supervisor == NULL); + LOG_ASSERT(_target == NULL); +} + + +int +PingProxy::usage() +{ + fprintf(stderr, "usage: %s\n", _argv[0]); + fprintf(stderr, "-s [server] (server hostname, default localhost)\n"); + fprintf(stderr, "-p [port] (server port number, default 19090)\n"); + return 1; +} + + +void +PingProxy::initRPC(const char *spec) +{ + _supervisor = new FRT_Supervisor(); + _target = _supervisor->GetTarget(spec); + _supervisor->Start(); +} + + +void +PingProxy::finiRPC() +{ + if (_target != NULL) { + _target->SubRef(); + _target = NULL; + } + if (_supervisor != NULL) { + _supervisor->ShutDown(true); + delete _supervisor; + _supervisor = NULL; + } +} + + +int +PingProxy::Main() +{ + int retval = 0; + bool debugging = false; + char c = -1; + + const char *serverHost = "localhost"; + int clientTimeout = 5; + int serverPort = 19090; + + const char *optArg = NULL; + int optInd = 0; + while ((c = GetOpt("w:s:p:dh", optArg, optInd)) != -1) { + switch (c) { + case 'w': + clientTimeout = atoi(optArg); + break; + case 's': + serverHost = optArg; + break; + case 'p': + serverPort = atoi(optArg); + break; + case 'd': + debugging = true; + break; + case '?': + default: + retval = 1; + // fallthrough + case 'h': + usage(); + return retval; + } + } + + if (serverPort == 0) { + usage(); + return 1; + } + + std::ostringstream tmp; + tmp << "tcp/"; + tmp << serverHost; + tmp << ":"; + tmp << serverPort; + std::string sspec = tmp.str(); + const char *spec = sspec.c_str(); + if (debugging) { + printf("connecting to '%s'\n", spec); + } + initRPC(spec); + + FRT_RPCRequest *req = _supervisor->AllocRPCRequest(); + + req->SetMethodName("ping"); + + _target->InvokeSync(req, clientTimeout); // seconds + + if (req->IsError()) { + retval = 1; + fprintf(stderr, "error %d: %s\n", + req->GetErrorCode(), req->GetErrorMessage()); + } else { + FRT_Values &answer = *(req->GetReturn()); + const char *atypes = answer.GetTypeString(); + if (strcmp(atypes, "i") == 0) { + if (debugging) { + printf("ping %d\n", answer[0]._intval32); + } + } else { + fprintf(stderr, "unexpected return types in RPC answer: '%s'\n", atypes); + retval = 1; + } + } + finiRPC(); + return retval; +} + +int main(int argc, char **argv) +{ + PingProxy app; + return app.Entry(argc, argv); +} diff --git a/config/src/apps/vespa-config/vespa-config.pl b/config/src/apps/vespa-config/vespa-config.pl new file mode 100755 index 00000000000..a87e5e52976 --- /dev/null +++ b/config/src/apps/vespa-config/vespa-config.pl @@ -0,0 +1,383 @@ +#!/usr/local/bin/perl -w +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Various small functions used when bootstrapping the config system + +# BEGIN perl environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +use File::Basename; +use File::Path; + +sub findpath { + my $myfullname = ${0}; + my($myname, $mypath) = fileparse($myfullname); + + return $mypath if ( $mypath && -d $mypath ); + $mypath=`pwd`; + + my $pwdfullname = $mypath . "/" . $myname; + return $mypath if ( -f $pwdfullname ); + return 0; +} + +# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise +sub is_vespa_home { + my($VESPA_HOME) = shift; + my $COMMON_ENV="libexec/vespa/common-env.sh"; + if ( $VESPA_HOME && -d $VESPA_HOME ) { + my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; + return $VESPA_HOME if -f $common_env; + } + return 0; +} + +# Returns the home of Vespa, or dies if it cannot +sub findhome { + # Try the VESPA_HOME env variable + return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); + if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly + die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; + } + + # Try the ROOT env variable + $ROOT = $ENV{'ROOT'}; + return $ROOT if is_vespa_home($ROOT); + + # Try the script location or current dir + my $mypath = findpath(); + if ($mypath) { + while ( $mypath =~ s|/[^/]*$|| ) { + return $mypath if is_vespa_home($mypath); + } + } + die "FATAL: Missing VESPA_HOME environment variable\n"; +} + +BEGIN { + my $tmp = findhome(); + if ( $tmp !~ m{[/]$} ) { $tmp .= "/"; } + $ENV{'VESPA_HOME'} = $tmp; +} +my $VESPA_HOME = $ENV{'VESPA_HOME'}; + +# END perl environment bootstrap section + +use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use Yahoo::Vespa::Defaults; +readConfFile(); + +use strict; +use warnings; +use File::Copy; +use File::Temp; + +my $default_configproxy_port = "19090"; +my $default_configserver_port = "19070"; +my $zk_client_port; + +my $base_cfg_dir = $VESPA_HOME . "conf/vespa"; + +# Set this to 1 to look up values (see getValue) in config files instead +# of in environment variables +my $lookupInConfig = 0; + +sub vespa_base_env { + $zk_client_port = getCCSVar('zookeeper_clientPort', 2181); +} + +sub getValue { + my ($varname, $prefix) = @_; + if ($lookupInConfig) { + return getConfigValue($varname, $prefix); + } + else { + return getEnvironmentValue($varname, $prefix); + } +} + +sub getConfigValue { + my ($varname, $config) = @_; + my $path = "$base_cfg_dir/$config.conf"; + if (open(CFG, "<$path")) { + while (<CFG>) { + chomp; + if ( m{^(\w+)\s(.+)} ) { + return $2 if $1 eq $varname; + } + } + close(CFG); + } + return; +} + +sub getEnvironmentValue { + my ($varname, $prefix) = @_; + my $value = $ENV{$prefix . "__" . $varname}; + if (defined $value && $value =~ m{^\s*(\S.*)\s*}) { + return $1; + } + return $value; +} + +sub getCCSVar { + my ($varname, $default) = @_; + my $value = getValue($varname, "cloudconfig_server"); + if (defined($value)) { + return $value; + } + return $default; +} + +sub getVar { + my ($varname, $default, $warn) = @_; + # print "GET var '$varname'\n"; + my $cloud = getValue($varname, "services"); + my $vespa = getValue($varname, "vespa_base"); + if (defined($cloud) && defined($vespa)) { + print STDERR "Found settings for both services.$varname and vespa_base.$varname, using settings from services\n"; + } + if (defined($cloud)) { + return $cloud; + } elsif (defined($vespa)) { + return $vespa; + } elsif ($warn > 0) { + print STDERR "No value found for 'services.$varname' or 'vespa_base.$varname'; using '$default'\n"; + } + return $default; +} + +sub printConfigServerPort { + my $port = getVar('port_configserver_rpc', $default_configserver_port, 0); + print "$port\n"; +} + +sub getConfigServers { + my @ret; + + my $hostname = `hostname`; + chomp $hostname; + + my $addr = getVar('addr_configserver', $hostname, 1); + my $port = getVar('port_configserver_rpc', $default_configserver_port, 0); + + my $h; + foreach $h (split(/,|\s+/, $addr)) { + if ($h =~ m{(\S+:\d+)}) { + push @ret, $1; + } else { + push @ret, "${h}:${port}"; + } + } + return @ret; +} + +sub getZKString { + my $out; + my $addr; + foreach $addr (getConfigServers()) { + $addr =~ s{:\d+}{:$zk_client_port,}; + $out .= $addr; + } + chop($out); # last comma + return $out; +} + +sub printZKString { + my $out = getZKString(); + print $out . "\n"; +} + +sub printAllConfigSourcesWithPort { + my $cfport = getVar('port_configserver_rpc', $default_configserver_port, 0); + my $cpport = getVar('port_configproxy_rpc', $default_configproxy_port, 0); + my $addr = "localhost"; + my $out = "tcp/${addr}:${cpport}"; + foreach $addr (getConfigServers()) { + if ($addr =~ m{\/}) { + if ($addr =~ m{\:}) { + $out .= ",${addr}"; + } else { + $out .= ",${addr}:${cfport}"; + } + } else { + if ($addr =~ m{\:}) { + $out .= ",tcp/${addr}"; + } else { + $out .= ",tcp/${addr}:${cfport}"; + } + } + } + print $out . "\n"; +} + +sub printConfigSources { + my $out; + my $addr; + foreach $addr (getConfigServers()) { + $out .= "tcp/${addr},"; + } + chop($out); # last comma + print $out . "\n"; +} + +sub printConfigHttpSources { + my $out; + my $addr; + foreach $addr (getConfigServers()) { + my $host = ""; + my $port = 0; + if ($addr =~ /(.*):(\d+)$/) { + $host = $1; + $port = $2; + } + $port++; # HTTP is rpc + 1 + $out .= "http://$host:$port "; + } + chop($out); # last space + print $out . "\n"; +} + +sub makeFiledistributorZKConfig { + my $cfgFile = $VESPA_HOME . "conf/filedistributor/zookeepers.cfg"; + open(CFG, "> ${cfgFile}.new") or die "Cannot write to '${cfgFile}.new'"; + my $zkservers = getZKString(); + print CFG "zookeeperserverlist \"$zkservers\"\n"; + close(CFG); + rename("${cfgFile}.new", ${cfgFile}) + or die "Cannot rename '${cfgFile}.new' -> '${cfgFile}': $!\n"; + print STDERR "wrote '${cfgFile}' \n"; +} + +sub makeFiledistributorDistributorConfig { + my $cfgFile = $VESPA_HOME . "conf/filedistributor/filedistributor.cfg"; + open(CFG, "> ${cfgFile}.new") or die "Cannot write to '${cfgFile}.new'"; + print CFG "torrentport 19093\n"; + my $hostname = `hostname`; + chomp $hostname; + print CFG "hostname \"$hostname\"\n"; + print CFG "filedbpath \"$VESPA_HOME" . "var/db/vespa/filedistribution\"\n"; + print CFG "maxdownloadspeed 0\n"; + print CFG "maxuploadspeed 0\n"; + close(CFG); + rename("${cfgFile}.new", ${cfgFile}) + or die "Cannot rename '${cfgFile}.new' -> '${cfgFile}': $!\n"; + print STDERR "wrote '${cfgFile}' \n"; +} + +sub makeFiledistributorRpcConfig { + my $cfgFile = $VESPA_HOME . "conf/filedistributor/filedistributorrpc.cfg"; + open(CFG, "> ${cfgFile}.new") or die "Cannot write to '${cfgFile}.new'"; + my $hostname = `hostname`; + chomp $hostname; + print CFG "connectionspec \"tcp/$hostname:19092\"\n"; + close(CFG); + rename("${cfgFile}.new", ${cfgFile}) + or die "Cannot rename '${cfgFile}.new' -> '${cfgFile}': $!\n"; + print STDERR "wrote '${cfgFile}' \n"; +} + +sub makeFiledistributorReferencesConfig { + my $cfgFile = $VESPA_HOME . "conf/filedistributor/filereferences.cfg"; + open(CFG, "> ${cfgFile}.new") or die "Cannot write to '${cfgFile}.new'"; + close(CFG); + rename("${cfgFile}.new", ${cfgFile}) + or die "Cannot rename '${cfgFile}.new' -> '${cfgFile}': $!\n"; + print STDERR "wrote '${cfgFile}' \n"; +} + +sub makeFiledistributorConfig { + makeFiledistributorZKConfig(); + makeFiledistributorDistributorConfig(); + makeFiledistributorRpcConfig(); + makeFiledistributorReferencesConfig(); +} + +sub isThisAConfigServer { + my $hostnameForThisHost = `hostname`; + chomp $hostnameForThisHost; + my $addr; + foreach $addr (getConfigServers()) { + my $host = ""; + my $port = 0; + if ($addr =~ /(.*):(\d+)$/) { + $host = $1; + } + if ($hostnameForThisHost eq $host or $host eq "localhost") { + print "yes\n"; + exit 0; + } + } + + print "no\n"; + exit 1; +} + +# Perl trim function to remove whitespace from the start and end of the string +sub trim($) { + my $string = shift; + $string =~ s/^\s+//; + $string =~ s/\s+$//; + return $string; +} + +sub getLastLine { + my ($file) = @_; + `grep -v \"\^\$\" $file | tail -n 1` # skip blank lines +} + +sub usage { + print "usage: "; + print "vespa-config [-configsources | -confighttpsources | -zkstring | -mkfiledistributorconfig | -configserverport | -zkclientport | -isthisaconfigserver]\n"; +} + +if ( @ARGV == 0 ) { + usage(); + exit 1; +} + +if ( $ARGV[0] eq "-allconfigsources" ) { + vespa_base_env(); + printAllConfigSourcesWithPort(); + exit 0; +} +if ( $ARGV[0] eq "-configsources" ) { + vespa_base_env(); + printConfigSources(); + exit 0; +} +if ( $ARGV[0] eq "-confighttpsources" ) { + $lookupInConfig = 1; + vespa_base_env(); + printConfigHttpSources(); + exit 0; +} +if ( $ARGV[0] eq "-zkstring" ) { + vespa_base_env(); + printZKString(); + exit 0; +} +if ( $ARGV[0] eq "-configserverport" ) { + $lookupInConfig = 1; + vespa_base_env(); + printConfigServerPort(); + exit 0; +} +if ( $ARGV[0] eq "-mkfiledistributorconfig" ) { + vespa_base_env(); + makeFiledistributorConfig(); + exit 0; +} +if ( $ARGV[0] eq "-zkclientport" ) { + vespa_base_env(); + print "$zk_client_port\n"; + exit 0; +} +if ( $ARGV[0] eq "-isthisaconfigserver" ) { + vespa_base_env(); + isThisAConfigServer(); + exit 0; +} + +usage(); +exit 1; diff --git a/config/src/main/java/.gitignore b/config/src/main/java/.gitignore new file mode 100644 index 00000000000..4cb44b1b2b5 --- /dev/null +++ b/config/src/main/java/.gitignore @@ -0,0 +1 @@ +/deploy diff --git a/config/src/main/java/com/yahoo/config/codegen/package-info.java b/config/src/main/java/com/yahoo/config/codegen/package-info.java new file mode 100644 index 00000000000..b83f806cf87 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/codegen/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.codegen; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java new file mode 100644 index 00000000000..1216efcbec5 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.collections.Pair; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.StringNode; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ConfigPayloadBuilder; + +import java.util.*; + +/** + * Deserializes config payload (cfg format) to a ConfigPayload. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 5.1.6 + */ +public class CfgConfigPayloadBuilder { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(CfgConfigPayloadBuilder.class.getName()); + + /** + * Deserializes a config payload to slime + * + * @param lines a list with config payload strings + * @return an instance of the config class + */ + public ConfigPayload deserialize(List<String> lines) { + return ConfigPayload.fromBuilder(deserializeToBuilder(lines)); + } + + public ConfigPayloadBuilder deserializeToBuilder(List<String> lines) { + int lineNum = 1; + ConfigPayloadBuilder payloadBuilder = new ConfigPayloadBuilder(); + for (String line : lines) { + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, "line " + lineNum + ": '" + line + "'"); + } + parseLine(line, lineNum, payloadBuilder); + lineNum++; + } + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "payload=" + payloadBuilder.toString()); + } + return payloadBuilder; + } + + private void parseLine(final String line, int lineNum, ConfigPayloadBuilder payloadBuilder) { + String trimmedLine = line.trim(); + if (trimmedLine.startsWith("#")) return; + Pair<String, String> fieldAndValue = parseFieldAndValue(trimmedLine); + String field = fieldAndValue.getFirst(); + String value = fieldAndValue.getSecond(); + if (field==null || value==null) { + log.log(LogLevel.DEBUG, "Got field without value in line " + lineNum + ": " + line + ", skipping"); + return; + } + field=field.trim(); + value=value.trim(); + validateField(field, trimmedLine, lineNum); + validateValue(value, trimmedLine, lineNum); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "field=" + field + ",value=" + value); + } + List<String> fields = parseFieldList(field); + ConfigPayloadBuilder currentBuilder = payloadBuilder; + for (int fieldNum = 0; fieldNum < fields.size(); fieldNum++) { + String fieldName = fields.get(fieldNum); + boolean isLeaf = (fieldNum == fields.size() - 1); + if (isLeaf) { + if (isArray(fieldName)) { + // array leaf + ConfigPayloadBuilder.Array array = currentBuilder.getArray(getArrayName(fieldName)); + array.set(getArrayIndex(fieldName), removeQuotes(value)); + } else if (isMap(fieldName)) { + // map leaf + ConfigPayloadBuilder.MapBuilder map = currentBuilder.getMap(getMapName(fieldName)); + map.put(getMapKey(fieldName), removeQuotes(value)); + } else { + // scalar leaf value + currentBuilder.setField(fieldName, removeQuotes(value)); + } + } else { + if (isArray(fieldName)) { + // array of structs + ConfigPayloadBuilder.Array array = currentBuilder.getArray(getArrayName(fieldName)); + currentBuilder = array.get(getArrayIndex(fieldName)); + } else if (isMap(fieldName)) { + // map of structs + ConfigPayloadBuilder.MapBuilder map = currentBuilder.getMap(getMapName(fieldName)); + currentBuilder = map.get(getMapKey(fieldName)); + } else { + // struct + currentBuilder = currentBuilder.getObject(fieldName); + } + } + } + } + + // split on space, but not if inside { } (map key) + Pair<String, String> parseFieldAndValue(String line) { + String field=null; + String value; + StringBuffer sb = new StringBuffer(); + boolean inMapKey = false; + for (char c : line.toCharArray()) { + if (c=='{') inMapKey=true; + if (c=='}') inMapKey=false; + if (c==' ' && !inMapKey) { + if (field==null) { + field = sb.toString(); + sb = new StringBuffer(); + continue; + } + } + sb.append(c); + } + value = sb.toString(); + return new Pair<>(field, value); + } + + // split on dot, but not if inside { } (map key) + List<String> parseFieldList(String field) { + List<String> ret = new ArrayList<>(); + StringBuffer sb = new StringBuffer(); + boolean inMapKey = false; + for (char c : field.toCharArray()) { + if (c=='{') inMapKey=true; + if (c=='}') inMapKey=false; + if (c=='.' && !inMapKey) { + ret.add(sb.toString()); + sb = new StringBuffer(); + continue; + } + sb.append(c); + } + ret.add(sb.toString()); + return ret; + } + + // TODO Need more validation + private void validateField(String field, String line, int lineNum) { + if (field.length() == 0) { + throw new ConfigurationRuntimeException("Error on line " + lineNum + ": " + line + "\n" + + "'" + field + "' is not a valid field name"); + } + } + + // TODO Need more validation + private void validateValue(String value, String line, int lineNum) { + if (value.length() == 0) { + throw new ConfigurationRuntimeException("Error on line " + lineNum + ": " + line + "\n" + + "'" + value + "' is not a valid value"); + } + } + + private boolean isArray(String name) { + return name.endsWith("]"); + } + + private boolean isMap(String name) { + return name.contains("{"); + } + + private String removeQuotes(String s) { + return StringNode.unescapeQuotedString(s); + } + + private String getMapName(String name) { + if (name.contains("{")) { + return name.substring(0, name.indexOf("{")); + } else { + return name; + } + } + + private String getMapKey(String name) { + if (name.contains("{")) { + return removeQuotes(name.substring(name.indexOf("{") + 1, name.indexOf("}"))); + } else { + return ""; + } + } + + private String getArrayName(String name) { + if (name.contains("[")) { + return name.substring(0, name.indexOf("[")); + } else { + return name; + } + } + + private int getArrayIndex(String name) { + if (name.contains("[")) { + return Integer.parseInt(name.substring(name.indexOf("[") + 1, name.indexOf("]"))); + } else { + return 0; + } + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java new file mode 100644 index 00000000000..add088cf349 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.logging.Logger; + +// Debug class that provides useful helper routines +public class ConfigDebug { + public static void logDebug(Logger logger, long timestamp, ConfigKey<?> key, String logmessage) { + if (key.getConfigId().matches(".*container.?\\d+.*") || key.getConfigId().matches(".*doc.api.*")) { + logger.log(LogLevel.INFO, timestamp + " " + key + " " + logmessage); + } + } + + public static void logDebug(Logger log, ConfigInstance.Builder builder, String configId, String logmessage) { + ConfigKey<?> key = new ConfigKey<>(builder.getDefName(), configId, builder.getDefNamespace()); + logDebug(log, 0, key, logmessage); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java b/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java new file mode 100755 index 00000000000..be4ff9f1b79 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + + +import com.yahoo.config.ConfigInstance; + +/** + * This is a simple config getter that retrieves a config with a given class and configId through a + * simple method call. No subscription is retained when the config has been returned to the client. + * + * This class is mainly targeted to unit tests that do not want the extra complexity incurred by setting + * up their own subscriber. Another use-case is clients that get config, do a task, and exit, e.g. + * command-line tools. + * + * @author gjoranv + */ +public class ConfigGetter<T extends ConfigInstance> { + + private final Class<T> clazz; + private final ConfigSource source; + + /** + * Creates a ConfigGetter for class <code>clazz</code> + * + * @param clazz a config class + */ + public ConfigGetter(Class<T> clazz) { + this(null, clazz); + } + + /** + * Creates a ConfigGetter for class <code>clazz</code> with the specified + * {@link ConfigSource}. + * + * @param source a {@link ConfigSource} + * @param clazz a config class + */ + // TODO This is the order of arguments in com.yahoo.config.ConfigGetter and kept here, I would like to switch the order + public ConfigGetter(ConfigSource source, Class<T> clazz) { + this.clazz = clazz; + this.source = source; + } + + /** + * Returns an instance of the config class specified in the constructor. + * + * @param configId a config id to use when getting the config + * @return an instance of a config class + */ + public synchronized T getConfig(String configId) { + ConfigSubscriber subscriber; + ConfigHandle<T> h; + if (source == null) { + subscriber = new ConfigSubscriber(); + } else { + subscriber = new ConfigSubscriber(source); + } + h = subscriber.subscribe(clazz, configId); + subscriber.nextConfig(); + T ret = h.getConfig(); + subscriber.close(); + return ret; + } + + /** + * Creates a ConfigGetter instance and returns an instance of the config class <code>c</code>. + * + * @param c a config class + * @param configId a config id to use when getting the config + * @return an instance of a config class + */ + public static <T extends ConfigInstance> T getConfig(Class<T> c, String configId) { + ConfigGetter<T> getter = new ConfigGetter<T>(c); + return getter.getConfig(configId); + } + + /** + * Creates a ConfigGetter instance and returns an instance of the config class <code>c</code>. + * + * @param c a config class + * @param configId a config id to use when getting the config + * @param source a {@link ConfigSource} + * @return an instance of a config class + */ + public static <T extends ConfigInstance> T getConfig(Class<T> c, String configId, ConfigSource source) { + ConfigGetter<T> getter = new ConfigGetter<T>(source, c); + return getter.getConfig(configId); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java b/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java new file mode 100644 index 00000000000..6cc10be8627 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.impl.ConfigSubscription; + +/** + * A config handle represents one config in the context of one active subscription on a {@link ConfigSubscriber}. + * It will contain meta data of the subscription of that particular config, as well as access to the {@link com.yahoo.config.ConfigInstance} itself. + * + * @param <T> the type of the config + * @author vegardh + * @since 5.1 + */ +public class ConfigHandle<T extends ConfigInstance> { + + private ConfigSubscription<T> sub; + private boolean changed = false; + + protected ConfigHandle(ConfigSubscription<T> sub) { + this.sub = sub; + } + + /** + * Returns true if: + * + * The config generation for the {@link ConfigSubscriber} that produced this is the first one in its life cycle. (Typically first time config.) + * or + * All configs for the subscriber have a new generation since the last time nextConfig() was called + * AND it's the same generation AND there is a change in <strong>this</strong> handle's config. + * (Typically calls for a reconfig.) + * + * @return there is a new config + */ + public boolean isChanged() { + return changed; + } + + void setChanged(boolean changed) { + this.changed = changed; + } + + ConfigSubscription<T> subscription() { + return sub; + } + + /** + * The config of this handle + * + * @return the config that this handle holds + */ + public T getConfig() { + // TODO throw if subscriber not frozen? + return sub.getConfig(); + } + + @Override + public String toString() { + return "Handle changed: " + changed + "\nSub:\n" + sub.toString(); + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java new file mode 100644 index 00000000000..cdb654fcbbb --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.config.Serializer; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; + +/** + * Implements a config instance serializer, serializing a config instance to a slime object. + * + * @author lulf + * @since 5.1.14 + */ +public class ConfigInstanceSerializer implements Serializer { + private final Slime slime; + private final Cursor root; + public ConfigInstanceSerializer(Slime slime) { + this.slime = slime; + root = slime.setObject(); + } + + public ConfigInstanceSerializer(Slime slime, Cursor root) { + this.slime = slime; + this.root = root; + } + + @Override + public Serializer createInner(String name) { + Cursor childRoot = root.setObject(name); + return new ConfigInstanceSerializer(slime, childRoot); + } + + @Override + public Serializer createArray(String name) { + return new ConfigInstanceSerializer(slime, root.setArray(name)); + } + + @Override + public Serializer createInner() { + return new ConfigInstanceSerializer(slime, root.addObject()); + } + + @Override + public Serializer createMap(String name) { + return createInner(name); + } + + public void serialize(String name, boolean value) { + root.setBool(name, value); + } + + public void serialize(String name, double value) { + root.setDouble(name, value); + } + + public void serialize(String name, int value) { + root.setLong(name, value); + } + + public void serialize(String name, long value) { + root.setLong(name, value); + } + + public void serialize(String name, String value) { + root.setString(name, value); + } + + @Override + public void serialize(boolean value) { + root.addBool(value); + } + + @Override + public void serialize(double value) { + root.addDouble(value); + } + + @Override + public void serialize(long value) { + root.addLong(value); + } + + @Override + public void serialize(int value) { + root.addLong(value); + } + + @Override + public void serialize(String value) { + root.addString(value); + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java new file mode 100644 index 00000000000..3c36bf7f105 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +import com.yahoo.config.ConfigBuilder; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.config.*; + +/** + * @author gjoranv + * @since 5.1.6 + */ +public class ConfigInstanceUtil { + + /** + * Copies all values that have been explicitly set on the source to the destination. + * Values that have not been explicitly set in the source builder, will be left unchanged + * in the destination. + * + * @param destination The builder to copy values into. + * @param source The builder to copy values from. Unset values are not copied. + * @param <BUILDER> The builder class. + */ + public static<BUILDER extends ConfigBuilder> void setValues(BUILDER destination, BUILDER source) { + try { + Method setter = destination.getClass().getDeclaredMethod("override", destination.getClass()); + setter.setAccessible(true); + setter.invoke(destination, source); + setter.setAccessible(false); + } catch (Exception e) { + throw new ConfigurationRuntimeException("Could not set values on config builder." + + destination.getClass().getName(), e); + } + } + + public static <T extends ConfigInstance> T getNewInstance(Class<T> type, + String configId, + ConfigPayload payload) { + T instance; + try { + ConfigTransformer<?> transformer = new ConfigTransformer<T>(type); + ConfigBuilder instanceBuilder = transformer.toConfigBuilder(payload); + Constructor<T> constructor = type.getConstructor(instanceBuilder.getClass()); + instance = constructor.newInstance((ConfigInstance.Builder) instanceBuilder); + + // Workaround for JDK7, where compilation fails due to fields being + // private and not accessible from T. Reference it as a + // ConfigInstance to work around it. See + // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7022052 for + // more information. + ConfigInstance i = instance; + i.postInitialize(configId); + setConfigId(i, configId); + + } catch (InstantiationException | InvocationTargetException | NoSuchMethodException | + NoSuchFieldException | IllegalAccessException e) { + throw new IllegalArgumentException("Failed creating new instance of '" + type.getCanonicalName() + + "' for config id '" + configId + "': " + Exceptions.toMessageString(e), e); + } + return instance; + } + + private static void setConfigId(ConfigInstance instance, String configId) + throws NoSuchFieldException, IllegalAccessException { + Field configIdField = ConfigInstance.class.getDeclaredField("configId"); + configIdField.setAccessible(true); + configIdField.set(instance, configId); + configIdField.setAccessible(false); + } + + /** + * Gets the value of a private field on a Builder. + * @param builder a {@link com.yahoo.config.ConfigBuilder} + * @param fieldName a config field name + * @return the value of the private field + */ + public static Object getField(ConfigBuilder builder, String fieldName) { + try { + Field f = builder.getClass().getDeclaredField(fieldName); + f.setAccessible(true); + return f.get(builder); + } catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java new file mode 100644 index 00000000000..ed1dca50096 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +/** + * This exception is thrown when any blocking call within the Config API is interrupted. + * @author lulf + * @since 5.1 + */ +@SuppressWarnings("serial") +public class ConfigInterruptedException extends RuntimeException { + public ConfigInterruptedException(Throwable cause) { + super(cause); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java new file mode 100644 index 00000000000..1b516f333fa --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.vespa.config.ConfigKey; + +/** + * Config source as a programmatically built set of {@link com.yahoo.config.ConfigInstance}s + * + * @author vegardh + * @since 5.1 + */ +public class ConfigSet implements ConfigSource { + private final Map<ConfigKey<?>, ConfigInstance.Builder> configs = new ConcurrentHashMap<>(); + + /** + * Inserts a new builder in this set. If an existing entry exists, it is overwritten. + * + * @param configId The config id for this builder. + * @param builder The builder that will produce config for the particular config id. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public void addBuilder(String configId, ConfigInstance.Builder builder) { + Class<?> configClass = builder.getClass().getDeclaringClass(); + //System.out.println("Declaring class for builder " + builder + " is " + configClass); + ConfigKey<?> key = new ConfigKey(configClass, configId); + configs.put(key, builder); + } + + /** + * Returns a Builder matching the given key, or null if no match + * + * @param key a config key to get a Builder for + * @return a ConfigInstance + */ + public ConfigInstance.Builder get(ConfigKey<?> key) { + return configs.get(key); + } + + /** + * Returns true if this set contains a config instance matching the given key + * + * @param key a config key + * @return a ConfigInstance + */ + public boolean contains(ConfigKey<?> key) { + return configs.containsKey(key); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (Map.Entry<ConfigKey<?>, ConfigInstance.Builder> entry : configs.entrySet()) { + sb.append(entry.getKey()).append("=>").append(entry.getValue()); + } + return sb.toString(); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java new file mode 100644 index 00000000000..44b65e4ba12 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +/** + * A type of source of config + * @author vegardh + * @since 5.1 + * + */ +public interface ConfigSource { + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java new file mode 100755 index 00000000000..4c1757c0ba8 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java @@ -0,0 +1,127 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.log.LogLevel; + +import java.util.*; +import java.util.logging.Logger; + + +/** +* An immutable set of connection endpoints, where each endpoint points to either a + * remote configserver or a configproxy. + * + * Two sets are said to be equal if they contain the same sources, independent of order, + * upper/lower-casing and whitespaces. + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public class ConfigSourceSet implements ConfigSource +{ + private static final Logger log = Logger.getLogger(ConfigSourceSet.class.getName()); + private final Set<String> sources = new LinkedHashSet<String>(); + + /** + * Creates an empty ConfigSourceSet, mostly used for unit testing. + */ + public ConfigSourceSet() { + } + + /** + * Creates a ConfigSourceSet containing all the unique given input addresses. + * Each address is trimmed and lower-cased before adding. + * + * @param addresses Connection endpoints on the format "tcp/host:port". + */ + public ConfigSourceSet(List<String> addresses) { + for (String a : addresses) { + sources.add(a.trim().toLowerCase()); + } + } + + /** + * Creates a ConfigSourceSet containing all the unique given input addresses. + * Each address is trimmed and lower-cased before adding. + * + * @param addresses Connection endpoints on the format "tcp/host:port". + */ + public ConfigSourceSet(String[] addresses) { + this(Arrays.asList(addresses)); + } + + /** + * Convenience constructor to create a ConfigSourceSet with only one input address. + * + * @param address Connection endpoint on the format "tcp/host:port". + */ + public ConfigSourceSet(String address) { + this(new String[] {address}); + } + + /** + * Returns an unmodifiable set containing all sources in this ConfigSourceSet. Iteration order is + * guaranteed to be the same as that of the list or array that was given when this set was created. + * + * @return All sources in this ConfigSourceSet. + */ + public Set<String> getSources() { + return Collections.unmodifiableSet(sources); + } + + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (! (o instanceof ConfigSourceSet)) { + return false; + } + ConfigSourceSet css = (ConfigSourceSet)o; + return sources.equals(css.sources); + } + + public int hashCode() { + return sources.hashCode(); + } + + public String toString() { + return sources.toString(); + } + + /** + * Create a new source set using the environment variables or system properties + * @return a new source set if available, null if not. + */ + public static ConfigSourceSet createDefault() { + String configSources = System.getenv("VESPA_CONFIG_SOURCES"); + if (configSources != null) { + log.log(LogLevel.INFO, "Using config sources from VESPA_CONFIG_SOURCES: " + configSources); + return new ConfigSourceSet(checkSourcesSyntax(configSources)); + } else { + String[] def = {"tcp/localhost:" + System.getProperty("vespa.config.port", "19090")}; + String[] sourceSet = checkSourcesSyntax(System.getProperty("configsources")); + return new ConfigSourceSet(sourceSet == null ? def : sourceSet); + } + } + + /** + * Check sources syntax and convert it to a proper source set by checking if + * sources start with the required "tcp/" prefix and add that prefix if not. + * + * @param sources a source set as a comma-separated string + * @return a String array with sources, or null if the input source set was null + */ + private static String[] checkSourcesSyntax(String sources) { + String[] sourceSet = null; + if (sources != null) { + sourceSet = sources.split(","); + int i = 0; + for (String s : sourceSet) { + if (!s.startsWith("tcp/")) { + sourceSet[i] = "tcp/" + sourceSet[i]; + } + i++; + } + } + return sourceSet; + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java new file mode 100644 index 00000000000..2322726057e --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -0,0 +1,449 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.subscription.impl.ConfigSubscription; +import com.yahoo.config.subscription.impl.JRTConfigRequester; +import com.yahoo.log.LogLevel; +import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.TimingValues; + +/** + * Used for subscribing to one or more configs. Can optionally be given a {@link ConfigSource} for the configs + * that will be used when {@link #subscribe(Class, String)} is called. + * + * {@link #subscribe(Class, String)} on the configs needed, call {@link #nextConfig(long)} and get the config from the + * {@link ConfigHandle} which {@link #subscribe(Class, String)} returned. + * + * @author vegardh + * @since 5.1 + */ +public class ConfigSubscriber { + private Logger log = Logger.getLogger(getClass().getName()); + private State state = State.OPEN; + protected List<ConfigHandle<? extends ConfigInstance>> subscriptionHandles = new ArrayList<>(); + private final ConfigSource source; + private long generation = -1; + + /** + * Reuse requesters for equal source sets, limit number if many subscriptions. + */ + protected Map<ConfigSourceSet, JRTConfigRequester> requesters = new HashMap<>(); + + /** + * The states of the subscriber. Affects the validity of calling certain methods. + * + */ + protected enum State { + OPEN, FROZEN, CLOSED + } + + /** + * Constructs a new subscriber. The default Vespa network config source will be used, which is the address of + * a config proxy (part of vespa_base) running locally. It can also be changed by setting VESPA_CONFIG_SOURCES. + */ + public ConfigSubscriber() { + this(JRTConfigRequester.defaultSourceSet); + } + + /** + * Constructs a new subscriber with the given source. + * + * @param source a {@link ConfigSource} that will be used when {@link #subscribe(Class, String)} is called. + */ + public ConfigSubscriber(ConfigSource source) { + this.source = source; + } + + /** + * Subscribes on the given type of {@link ConfigInstance} with the given config id. + * + * The method blocks until the first config is ready to be fetched with {@link #nextConfig()}. + * + * @param configClass The class, typically generated from a def-file using config-class-plugin + * @param configId Identifies the service in vespa-services.xml, or null if you are using a local {@link ConfigSource} which does not use config id. + * Also supported: raw:, file:, dir: or jar: config id which addresses config locally in the same way. + * + * @return a ConfigHandle + */ + public <T extends ConfigInstance> ConfigHandle<T> subscribe(Class<T> configClass, String configId) { + return subscribe(configClass, configId, source, new TimingValues()); + } + + /** + * Subscribes on the given type of {@link ConfigInstance} with the given config id and subscribe timeout. + * + * The method blocks until the first config is ready to be fetched with {@link #nextConfig()}. + * + * @param configClass The class, typically generated from a def-file using config-class-plugin + * @param configId Identifies the service in vespa-services.xml, or possibly raw:, file:, dir: or jar: type config which addresses config locally. + * @param timeoutMillis The time to wait for a config to become available, in milliseconds + * @return a ConfigHandle + */ + public <T extends ConfigInstance> ConfigHandle<T> subscribe(Class<T> configClass, String configId, long timeoutMillis) { + return subscribe(configClass, configId, source, new TimingValues().setSubscribeTimeout(timeoutMillis)); + } + + // for testing + <T extends ConfigInstance> ConfigHandle<T> subscribe(Class<T> configClass, String configId, ConfigSource source, TimingValues timingValues) { + checkStateBeforeSubscribe(); + final ConfigKey<T> configKey = new ConfigKey<>(configClass, configId); + ConfigSubscription<T> sub = ConfigSubscription.get(configKey, this, source, timingValues); + ConfigHandle<T> handle = new ConfigHandle<>(sub); + subscribeAndHandleErrors(sub, configKey, handle, timingValues); + return handle; + } + + protected void checkStateBeforeSubscribe() { + if (state != State.OPEN) + throw new IllegalStateException("Adding subscription after calling nextConfig() is not allowed"); + } + + protected void subscribeAndHandleErrors(ConfigSubscription<?> sub, ConfigKey<?> configKey, ConfigHandle<?> handle, TimingValues timingValues) { + subscriptionHandles.add(handle); + // Must block here until something available from the subscription, so we know that it offers something when the user calls nextConfig + boolean subOk = sub.subscribe(timingValues.getSubscribeTimeout()); + throwIfExceptionSet(sub); + if (!subOk) { + //sub.close(); + //subscriptionHandles.remove(handle); + throw new ConfigurationRuntimeException("Subscribe for '" + configKey + "' timed out (timeout was " + timingValues.getSubscribeTimeout() + " ms): " + sub); + } + } + + /** + * Use this for waiting for a new config that has changed. + * + * Returns true if: + * + * It is the first time nextConfig() is called on this subscriber, and the framework has fetched config for all subscriptions. (Typically a first time config.) + * + * or + * + * All configs for the subscriber have a new generation since the last time nextConfig() was called, AND they have the same generation AND there is a change in config for at least one + * of the configs. (Typically calls for a reconfig.) + * + * You can check which configs are changed by calling {@link ConfigHandle#isChanged()} on the handle you got from {@link #subscribe(Class, String)}. + * + * If the call times out (timeout 1000 ms), no handle will have the changed flag set. You should not configure anything then. + * + * @return true if a config/reconfig of your system should happen + * @throws ConfigInterruptedException if thread performing this call interrupted. + */ + public boolean nextConfig() { + return nextConfig(TimingValues.defaultNextConfigTimeout); + } + + /** + * Use this for waiting for a new config that has changed, with the given timeout. + * + * Returns true if: + * + * It is the first time nextConfig() is called on this subscriber, and the framework has fetched config for all subscriptions. (Typically a first time config.) + * + * or + * + * All configs for the subscriber have a new generation since the last time nextConfig() was called, AND they have the same generation AND there is a change in config for at least one + * of the configs. (Typically calls for a reconfig.) + * + * You can check which configs are changed by calling {@link ConfigHandle#isChanged()} on the handle you got from {@link #subscribe(Class, String)}. + * + * If the call times out, no handle will have the changed flag set. You should not configure anything then. + * + * @param timeoutMillis timeout in milliseconds + * @return true if a config/reconfig of your system should happen + * @throws ConfigInterruptedException if thread performing this call interrupted. + */ + public boolean nextConfig(long timeoutMillis) { + return acquireSnapshot(timeoutMillis, true); + } + + /** + * Use this for waiting for a new config generation. + * + * Returns true if: + * + * It is the first time nextGeneration() is called on this subscriber, and the framework has fetched config for all subscriptions. (Typically a first time config.) + * + * or + * + * All configs for the subscriber have a new generation since the last time nextGeneration() was called, AND they have the same generation. Note that + * none of the configs have to be changed, but they might be. + * + * + * You can check which configs are changed by calling {@link ConfigHandle#isChanged()} on the handle you got from {@link #subscribe(Class, String)}. + * + * If the call times out (timeout 1000 ms), no handle will have the changed flag set. You should not configure anything then. + * + * @return true if generations for all configs have been updated. + * @throws ConfigInterruptedException if thread performing this call interrupted. + */ + public boolean nextGeneration() { + return nextGeneration(TimingValues.defaultNextConfigTimeout); + } + + /** + * Use this for waiting for a new config generation, with the given timeout + * + * Returns true if: + * + * It is the first time nextGeneration() is called on this subscriber, and the framework has fetched config for all subscriptions. (Typically a first time config.) + * + * or + * + * All configs for the subscriber have a new generation since the last time nextGeneration() was called, AND they have the same generation. Note that + * none of the configs have to be changed, but they might be. + * + * You can check which configs are changed by calling {@link ConfigHandle#isChanged()} on the handle you got from {@link #subscribe(Class, String)}. + * + * If the call times out (timeout 1000 ms), no handle will have the changed flag set. You should not configure anything then. + * + * @param timeoutMillis timeout in milliseconds + * @return true if generations for all configs have been updated. + * @throws ConfigInterruptedException if thread performing this call interrupted. + */ + public boolean nextGeneration(long timeoutMillis) { + return acquireSnapshot(timeoutMillis, false); + } + + /** + * Acquire a snapshot of all configs with the same generation within a timeout. + * @param timeoutInMillis timeout to wait in milliseconds + * @param requireChange if set, at least one config have to change + * @return true, if a new config generation has been found for all configs (additionally requires + * that at lest one of them has changed if <code>requireChange</code> is true), false otherwise + */ + private boolean acquireSnapshot(long timeoutInMillis, boolean requireChange) { + if (state == State.CLOSED) return false; + long started = System.currentTimeMillis(); + long timeLeftMillis = timeoutInMillis; + state = State.FROZEN; + boolean anyConfigChanged = false; + boolean allGenerationsChanged = true; + boolean allGenerationsTheSame = true; + Long currentGenChecker = null; + for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) { + h.setChanged(false); // Reset this flag, if it was set, the user should have acted on it the last time this method returned true. + } + boolean reconfigDue; + do { + // Keep on polling the subscriptions until we have a new generation across the board, or it times out + for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) { + ConfigSubscription<? extends ConfigInstance> subscription = h.subscription(); + if (!subscription.nextConfig(timeLeftMillis)) { + // This subscriber has no new state and we know it has exhausted all time + return false; + } + throwIfExceptionSet(subscription); + if (currentGenChecker == null) currentGenChecker = subscription.getGeneration(); + if (!currentGenChecker.equals(subscription.getGeneration())) allGenerationsTheSame = false; + allGenerationsChanged = allGenerationsChanged && subscription.isGenerationChanged(); + if (subscription.isConfigChanged()) anyConfigChanged = true; + timeLeftMillis = timeLeftMillis - (System.currentTimeMillis() - started); + } + reconfigDue = (anyConfigChanged || !requireChange) && allGenerationsChanged && allGenerationsTheSame; + if (!reconfigDue && timeLeftMillis > 0) { + sleep(10); + } + } while (!reconfigDue && timeLeftMillis > 0); + if (reconfigDue) { + // This indicates the clients will possibly reconfigure their services, so "reset" changed-logic in subscriptions. + // Also if appropriate update the changed flag on the handler, which clients use. + markSubsChangedSeen(); + generation = subscriptionHandles.get(0).subscription().getGeneration(); + } + return reconfigDue; + } + + private void sleep(long i) { + try { + Thread.sleep(i); + } catch (InterruptedException e) { + throw new ConfigInterruptedException(e); + } + } + + /** + * If a {@link ConfigSubscription} has its exception set, reset that field and throw it + * + * @param sub {@link ConfigSubscription} + */ + protected void throwIfExceptionSet(ConfigSubscription<? extends ConfigInstance> sub) { + RuntimeException subThrowable = sub.getException(); + if (subThrowable != null) { + sub.setException(null); + throw subThrowable; + } + } + + private void markSubsChangedSeen() { + for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) { + ConfigSubscription<? extends ConfigInstance> sub = h.subscription(); + h.setChanged(sub.isConfigChanged()); + sub.resetChangedFlags(); + } + } + + /** + * Closes all open {@link ConfigSubscription}s + */ + public void close() { + state = State.CLOSED; + for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) { + h.subscription().close(); + } + closeRequesters(); + log.log(LogLevel.DEBUG, "Config subscriber has been closed."); + } + + /** + * Closes all open requesters + */ + protected void closeRequesters() { + for (JRTConfigRequester requester : requesters.values()) { + requester.close(); + } + } + + @Override + public String toString() { + String ret = "Subscriber state:" + state; + for (ConfigHandle<?> h : subscriptionHandles) { + ret = ret + "\n" + h.toString(); + } + return ret; + } + + /** + * Convenience method to start a daemon thread called "Vespa config thread" with the given runnable. If you want the runnable to + * handle a {@link ConfigSubscriber} or {@link ConfigHandle} you have declared locally outside, declare them as final to make it work. + * + * @param runnable a class implementing {@link java.lang.Runnable} + * @return the newly started thread + */ + public Thread startConfigThread(Runnable runnable) { + Thread t = new Thread(runnable); + t.setDaemon(true); + t.setName("Vespa config thread"); + t.start(); + return t; + } + + protected State state() { + return state; + } + + /** + * Sets all subscriptions under this subscriber to have the given generation. This is intended for testing, to emulate a + * reload-config operation. + * + * @param generation a generation number + */ + public void reload(long generation) { + for (ConfigHandle<?> h : subscriptionHandles) { + h.subscription().reload(generation); + } + } + + /** + * The source used by this subscriber. + * + * @return the {@link ConfigSource} used by this subscriber + */ + public ConfigSource getSource() { + return source; + } + + /** + * Implementation detail, do not use. + * @return requesters + */ + public Map<ConfigSourceSet, JRTConfigRequester> requesters() { + return requesters; + } + + public boolean isClosed() { + return state == State.CLOSED; + } + + /** + * Use this convenience method if you only want to subscribe on <em>one</em> config, and want generic error handling. + * Implement {@link SingleSubscriber} and pass to this method. + * You will get initial config, and a config thread will be started. The method will throw in your thread if initial + * configuration fails, and the config thread will print a generic error message (but continue) if it fails thereafter. The config + * thread will stop if you {@link #close()} this {@link ConfigSubscriber}. + * + * @param <T> ConfigInstance type + * @param singleSubscriber The object to receive config + * @param configClass The class, typically generated from a def-file using config-class-plugin + * @param configId Identifies the service in vespa-services.xml + * @return The handle of the config + * @see #startConfigThread(Runnable) + */ + public <T extends ConfigInstance> ConfigHandle<T> subscribe(final SingleSubscriber<T> singleSubscriber, Class<T> configClass, String configId) { + if (!subscriptionHandles.isEmpty()) + throw new IllegalStateException("Can not start single-subscription because subscriptions were previously opened on this."); + final ConfigHandle<T> handle = subscribe(configClass, configId); + if (!nextConfig()) + throw new ConfigurationRuntimeException("Initial config of " + configClass.getName() + " failed."); + singleSubscriber.configure(handle.getConfig()); + startConfigThread(new Runnable() { + @Override + public void run() { + while (!isClosed()) { + try { + if (nextConfig()) { + if (handle.isChanged()) singleSubscriber.configure(handle.getConfig()); + } + } catch (Exception e) { + log.log(LogLevel.ERROR, "Exception from config system, continuing config thread: " + Exceptions.toMessageString(e)); + } + } + } + }); + return handle; + } + + /** + * The current generation of configs known by this subscriber. + * + * @return the current generation of configs known by this subscriber + */ + public long getGeneration() { + return generation; + } + + /** + * Convenience interface for clients who only subscribe to one config. Implement this, and pass it to {@link ConfigSubscriber#subscribe(SingleSubscriber, Class, String)}. + * + * @author vegardh + */ + public interface SingleSubscriber<T extends ConfigInstance> { + public void configure(T config); + } + + /** + * Finalizer to ensure that we do not leak resources on reconfig. Though finalizers are bad, + * this is not a performance critical object as it will be deconstructed typically container reconfig. + */ + @Override + protected void finalize() throws Throwable { + try { + if (!isClosed()) { + close(); + } + } finally { + super.finalize(); + } + } + + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java b/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java new file mode 100644 index 00000000000..c492a04b1f6 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.io.File; + +import com.yahoo.config.subscription.impl.JRTConfigRequester; + +/** + * A Config URI is a class that can be used to encapsulate a config source and a config id into one + * object to simplify parameter passing. + * + * @author lulf + * @since 5.1 + */ +public class ConfigURI { + private String configId; + private ConfigSource source; + + private ConfigURI(String configId, ConfigSource source) { + this.configId = configId; + this.source = source; + } + + public String getConfigId() { + return configId; + } + + public ConfigSource getSource() { + return source; + } + + public static ConfigURI createFromId(String configId) { + return new ConfigURI(getConfigId(configId), getConfigSource(configId)); + } + + private static ConfigSource getConfigSource(String configId) { + if (configId.startsWith("file:")) { + return new FileSource(new File(configId.substring(5))); + } else if (configId.startsWith("dir:")) { + return new DirSource(new File(configId.substring(4))); + } else { + return JRTConfigRequester.defaultSourceSet; + } + } + + private static String getConfigId(String configId) { + if (configId.startsWith("file:") || configId.startsWith("dir:")) { + return ""; + } else { + return configId; + } + } + + public static ConfigURI createFromIdAndSource(String configId, ConfigSource source) { + return new ConfigURI(configId, source); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/DirSource.java b/config/src/main/java/com/yahoo/config/subscription/DirSource.java new file mode 100644 index 00000000000..d9e425022f0 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/DirSource.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.io.File; + +/** + * Source specifying config from a local directory + * @author vegardh + * @since 5.1 + * + */ +public class DirSource implements ConfigSource { + private final File dir; + + public DirSource(File dir) { + if (!dir.isDirectory()) throw new IllegalArgumentException("Not a directory: "+dir); + this.dir = dir; + } + + public File getDir() { + return dir; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/FileSource.java b/config/src/main/java/com/yahoo/config/subscription/FileSource.java new file mode 100644 index 00000000000..d7634a000b6 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/FileSource.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.io.File; + +/** + * Source specifying config from one local file + * @author vegardh + * @since 5.1 + * + */ +public class FileSource implements ConfigSource { + private final File file; + + public FileSource(File file) { + if (!file.isFile()) throw new IllegalArgumentException("Not an ordinary file: "+file); + this.file = file; + } + + public File getFile() { + return file; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/JarSource.java b/config/src/main/java/com/yahoo/config/subscription/JarSource.java new file mode 100644 index 00000000000..021b3e72025 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/JarSource.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.util.jar.JarFile; + +/** + * Source specifying config as a jar file entry + * @author vegardh + * @since 5.1 + * + */ +public class JarSource implements ConfigSource { + private final String path; + private final JarFile jarFile; + + /** + * Creates a new jar source + * @param jarFile the jar file to use as a source + * @param path the path within the jar file, or null to use the default config/ + */ + public JarSource(JarFile jarFile, String path) { + this.path = path; + this.jarFile = jarFile; + } + + public JarFile getJarFile() { + return jarFile; + } + + public String getPath() { + return path; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/RawSource.java b/config/src/main/java/com/yahoo/config/subscription/RawSource.java new file mode 100644 index 00000000000..74b3d508652 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/RawSource.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +/** + * Source specifying raw config, where payload is given programmatically + * @author vegardh + * @since 5.1 + * + */ +public class RawSource implements ConfigSource { + public final String payload; + + /** + * New source with the given payload on Vespa cfg format + * @param payload config payload + */ + public RawSource(String payload) { + this.payload = payload; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java new file mode 100644 index 00000000000..3778ee38d98 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigSet; +import com.yahoo.config.subscription.ConfigSource; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.vespa.config.ConfigKey; + +import java.lang.reflect.Constructor; + +/** + * Subscription on a programmatically built set of configs + * @author vegardh + * @since 5.1 + */ +public class ConfigSetSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + + private final ConfigSet set; + private final ConfigKey<T> subKey; + + ConfigSetSubscription(ConfigKey<T> key, + ConfigSubscriber subscriber, ConfigSource cset) { + super(key, subscriber); + if (!(cset instanceof ConfigSet)) throw new IllegalArgumentException("Source is not a ConfigSet: "+cset); + this.set=(ConfigSet) cset; + subKey = new ConfigKey<T>(configClass, key.getConfigId()); + if (!set.contains(subKey)) { + throw new IllegalArgumentException("The given ConfigSet "+set+" does not contain a config for "+subKey); + } + setGeneration(0l); + } + + @Override + public boolean nextConfig(long timeout) { + long end = System.currentTimeMillis() + timeout; + do { + ConfigInstance myInstance = getNewInstance(); + // User forced reload + if (checkReloaded()) { + updateInstance(myInstance); + return true; + } + if (!myInstance.equals(config)) { + generation++; + updateInstance(myInstance); + return true; + } + sleep(10); + } while (System.currentTimeMillis() < end); + // These shouldn't be checked anywhere since we return false now, but setting them still + setGenerationChanged(false); + setConfigChanged(false); + return false; + } + + private void sleep(int milliSecondsToSleep) { + try { + Thread.sleep(milliSecondsToSleep); + } catch (InterruptedException e) { + throw new RuntimeException("nextConfig aborted", e); + } + } + + @SuppressWarnings("unchecked") + private void updateInstance(ConfigInstance myInstance) { + if (!myInstance.equals(config)) { + setConfigChanged(true); + } + setConfig((T) myInstance); + setGenerationChanged(true); + } + + @Override + public boolean subscribe(long timeout) { + return true; + } + + public ConfigInstance getNewInstance() { + try { + ConfigInstance.Builder builder = set.get(subKey); + Constructor<?> constructor = builder.getClass().getDeclaringClass().getConstructor(builder.getClass()); + return (ConfigInstance) constructor.newInstance(builder); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java new file mode 100644 index 00000000000..0909dc6e1a2 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java @@ -0,0 +1,310 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.io.File; +import java.util.logging.Logger; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigSet; +import com.yahoo.config.subscription.ConfigSource; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.config.subscription.DirSource; +import com.yahoo.config.subscription.FileSource; +import com.yahoo.config.subscription.JarSource; +import com.yahoo.config.subscription.RawSource; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.TimingValues; +import com.yahoo.vespa.config.protocol.DefContent; + +/** + * Represents one active subscription to one config + * + * @author vegardh + * @since 5.1 + */ +public abstract class ConfigSubscription<T extends ConfigInstance> { + protected static Logger log = Logger.getLogger(ConfigSubscription.class.getName()); + protected ConfigSubscriber subscriber; + protected boolean configChanged = false; + protected boolean generationChanged = false; + protected volatile T config = null; + protected Long generation = null; + protected ConfigKey<T> key; + protected Class<T> configClass; + private volatile RuntimeException exception = null; + private State state = State.OPEN; + /** + * If non-null: The user has set this generation explicitly. nextConfig should take this into account. + * Access to these variables _must_ be synchronized, as nextConfig and reload() is likely to be run from + * independent threads. + */ + private boolean doReload = false; + private long reloadedGeneration = -1; + + enum State { + OPEN, CLOSED + } + + /** + * Initializes one subscription + * + * @param key a {@link ConfigKey} + * @param subscriber the subscriber for this subscription + */ + ConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber) { + this.key = key; + this.configClass = key.getConfigClass(); + this.subscriber = subscriber; + } + + + /** + * Correct type of ConfigSubscription instance based on type of source or form of config id + * + * @param key a {@link ConfigKey} + * @param subscriber the subscriber for this subscription + * @return a subclass of a ConfigsSubscription + */ + public static <T extends ConfigInstance> ConfigSubscription<T> get(ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source, TimingValues timingValues) { + String configId = key.getConfigId(); + if (source instanceof RawSource || configId.startsWith("raw:")) return getRawSub(key, subscriber, source); + if (source instanceof FileSource || configId.startsWith("file:")) return getFileSub(key, subscriber, source); + if (source instanceof DirSource || configId.startsWith("dir:")) return getDirFileSub(key, subscriber, source); + if (source instanceof JarSource || configId.startsWith("jar:")) return getJarSub(key, subscriber, source); + if (source instanceof ConfigSet) return new ConfigSetSubscription<>(key, subscriber, source); + if (source instanceof ConfigSourceSet) return new JRTConfigSubscription<>(key, subscriber, source, timingValues); + throw new IllegalArgumentException("Unknown source type: "+source); + } + + private static <T extends ConfigInstance> JarConfigSubscription<T> getJarSub( + ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source) { + String jarName; + String path="config/"; + if (source instanceof JarSource) { + JarSource js = (JarSource) source; + jarName=js.getJarFile().getName(); + if (js.getPath()!=null) path=js.getPath(); + } else { + jarName=key.getConfigId().replace("jar:", "").replaceFirst("\\!/.*", ""); + if (key.getConfigId().contains("!/")) path = key.getConfigId().replaceFirst(".*\\!/", ""); + } + return new JarConfigSubscription<>(key, subscriber, jarName, path); + } + + private static <T extends ConfigInstance> ConfigSubscription<T> getFileSub( + ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source) { + File file = ((source instanceof FileSource))?((FileSource)source).getFile():new File(key.getConfigId().replace("file:", "")); + return new FileConfigSubscription<>(key, subscriber, file); + } + + private static <T extends ConfigInstance> ConfigSubscription<T> getRawSub( + ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source) { + String payload = ((source instanceof RawSource)?((RawSource)source).payload:key.getConfigId().replace("raw:", "")); + return new RawConfigSubscription<>(key, subscriber,payload); + } + + private static <T extends ConfigInstance> ConfigSubscription<T> getDirFileSub(ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source) { + String dir = key.getConfigId().replace("dir:", ""); + if (source instanceof DirSource) { + dir = ((DirSource)source).getDir().toString(); + } + if (!dir.endsWith(File.separator)) dir = dir + File.separator; + String name = getConfigFilenameNoVersion(key); + File file = new File(dir + name); + if (!file.exists()) { + throw new IllegalArgumentException("Could not find a config file for '" + key.getName() + "' in '" + dir + "'"); + } + return new FileConfigSubscription<>(key, subscriber, file); + } + + @SuppressWarnings("unchecked") + @Override + public boolean equals(Object o) { + if (o instanceof ConfigSubscription) { + ConfigSubscription<T> other = (ConfigSubscription<T>) o; + return key.equals(other.key) && + subscriber.equals(other.subscriber); + } + return false; + } + + void setConfigChanged(boolean changed) { + this.configChanged = changed; + } + + void setGenerationChanged(boolean genChanged) { + this.generationChanged = genChanged; + } + + /** + * Called from {@link ConfigSubscriber} when the changed status of this config is propagated to the clients + */ + public void resetChangedFlags() { + setConfigChanged(false); + setGenerationChanged(false); + } + + public boolean isConfigChanged() { + return configChanged; + } + + public boolean isGenerationChanged() { + return generationChanged; + } + + void setConfig(T config) { + this.config = config; + } + + /** + * The config object of this subscription + * + * @return the ConfigInstance (the config) of this subscription + */ + public T getConfig() { + return config; + } + + /** + * The generation of this subscription + * + * @return the generation of this subscription + */ + public Long getGeneration() { + return generation; + } + + /** + * The class of the subscription's desired {@link ConfigInstance} + * @return the config class + */ + public Class<T> getConfigClass() { + return configClass; + } + + void setGeneration(Long generation) { + this.generation = generation; + } + + @Override + public String toString() { + StringBuilder s = new StringBuilder(key.toString()); + s.append(", Current generation: ").append(generation) + .append(", Generation changed: ").append(generationChanged) + .append(", Config changed: ").append(configChanged); + if (exception != null) + s.append(", Exception: ").append(exception); + return s.toString(); + } + + /** + * The config key which this subscription uses to identify its config + * + * @return the ConfigKey for this subscription + */ + public ConfigKey<T> getKey() { + return key; + } + + /** + * Polls this subscription for a change. The method is guaranteed to use all of the given timeout before returning false. It will also take into account a user-set generation, + * that can be set by {@link ConfigSubscriber#reload(long)}. + * + * @param timeout in milliseconds + * @return false if timed out, true if the state of {@link #configChanged}, {@link #generationChanged} or {@link #exception} changed. If true, the {@link #config} field will be set also. + * has changed + */ + public abstract boolean nextConfig(long timeout); + + /** + * Will block until the next {@link #nextConfig(long)} is guaranteed to return an answer (or throw) immediately (i.e. not block) + * + * @param timeout in milliseconds + * @return false if timed out + */ + public abstract boolean subscribe(long timeout); + + /** + * Called by for example network threads to signal that the user thread should throw this exception immediately + * + * @param e a RuntimeException + */ + public void setException(RuntimeException e) { + this.exception = e; + } + + /** + * Gets an exception set by for example a network thread. If not null, it indicates that it should be + * thrown in the user's thread immediately. + * + * @return a RuntimeException if there exists one + */ + public RuntimeException getException() { + return exception; + } + + /** + * Returns true if an exception set by for example a network thread has been caught. + * + * @return true if there exists an exception for this subscription + */ + boolean hasException() { + return exception != null; + } + + public void close() { + state = State.CLOSED; + } + + State getState() { + return state; + } + + /** + * Returns the file name corresponding to the given key's defName and version. + * + * @param key a {@link ConfigKey} + * @return file name with version number. + */ + static <T extends ConfigInstance> String getConfigFilenameNoVersion(ConfigKey<T> key) { + StringBuilder filename = new StringBuilder(key.getName()); + filename.append(".cfg"); + return filename.toString(); + } + + /** + * Force this into the given generation, used in testing + * @param generation a config generation + */ + public synchronized void reload(long generation) { + this.doReload = true; + this.reloadedGeneration = generation; + } + + /** + * True if someone has set the {@link #reloadedGeneration} number by calling {@link #reload(long)} + * and hence wants to force a given generation programmatically. If that is the case, + * sets the {@link #generation} and {@link #generationChanged} fields accordingly. + * @return true if {@link #reload(long)} has been called, false otherwise + */ + protected synchronized boolean checkReloaded() { + if (doReload) { + // User has called reload + generation = reloadedGeneration; + setGenerationChanged(true); + doReload = false; + return true; + } + return false; + } + + /** + * The config definition schema + * + * @return the config definition for this subscription + */ + public DefContent getDefContent() { + return (DefContent.fromClass(configClass)); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java new file mode 100644 index 00000000000..c1b1e3daaff --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.subscription.CfgConfigPayloadBuilder; +import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.log.LogLevel; + +/** + * Subscription used when config id is file:... + * @author vegardh + * @since 5.1 + * + */ +public class FileConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + + final File file; + long ts; + + FileConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, File f) { + super(key, subscriber); + setGeneration(0l); + file=f; + if (!file.exists() && !file.isFile()) + throw new IllegalArgumentException("Not a file: "+file); + } + + @Override + public boolean nextConfig(long timeout) { + if (!file.exists() && !file.isFile()) throw new IllegalArgumentException("Not a file: "+file); + if (checkReloaded()) { + // TODO: Temporary log messages for debugging. + log.log(LogLevel.INFO, "User forced config reload at " + System.currentTimeMillis()); + // User forced reload + updateConfig(); + log.log(LogLevel.INFO, "Config updated at " + System.currentTimeMillis() + ", changed: " + isConfigChanged()); + log.log(LogLevel.INFO, "Config: " + config.toString()); + return true; + } + if (file.lastModified()!=ts) { + updateConfig(); + generation++; + setGenerationChanged(true); + return true; + } + try { + Thread.sleep(timeout); + } catch (InterruptedException e) { + throw new ConfigInterruptedException(e); + } + // These shouldn't be checked anywhere since we return false now, but setting them still + setGenerationChanged(false); + setConfigChanged(false); + return false; + } + + private void updateConfig() { + ts=file.lastModified(); + ConfigInstance prev = config; + try { + ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(IOUtils.readFile(file).split("\n"))); + config = payload.toInstance(configClass, key.getConfigId()); + } catch (IOException e) { + throw new ConfigurationRuntimeException(e); + } + setConfigChanged(!config.equals(prev)); + } + + @Override + public boolean subscribe(long timeout) { + return true; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java new file mode 100644 index 00000000000..26963428914 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import com.yahoo.config.subscription.ConfigHandle; +import com.yahoo.vespa.config.RawConfig; + +/** + * A config handle which does not use the config class, but payload instead. To be used in proxy? + * + * @author vegardh + */ +@SuppressWarnings({"rawtypes", "unchecked"}) +public class GenericConfigHandle extends ConfigHandle { + + private final GenericJRTConfigSubscription genSub; + + public GenericConfigHandle(GenericJRTConfigSubscription sub) { + super(sub); + genSub = sub; + } + + public RawConfig getRawConfig() { + return genSub.getRawConfig(); + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java new file mode 100644 index 00000000000..5a155e42aca --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.util.List; +import java.util.Map; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigHandle; +import com.yahoo.config.subscription.ConfigSource; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.TimingValues; + +/** + * A subscriber that can subscribe without the class. Used by configproxy. + * + * @author vegardh + */ +public class GenericConfigSubscriber extends ConfigSubscriber { + /** + * Constructs a new subscriber using the given pool of requesters (JRTConfigRequester holds 1 connection which in + * turn is subject to failover across the elems in the source set.) + * The behaviour is undefined if the map key is different from the source set the requester was built with. + * See also {@link JRTConfigRequester#get(com.yahoo.vespa.config.ConnectionPool, com.yahoo.vespa.config.TimingValues)} + * + * @param requesters a map from config source set to config requester + */ + public GenericConfigSubscriber(Map<ConfigSourceSet, JRTConfigRequester> requesters) { + this.requesters = requesters; + } + + public GenericConfigSubscriber() { + super(); + } + + /** + * Subscribes to config without using the class. For internal use in config proxy. + * + * @param key the {@link ConfigKey to subscribe to} + * @param defContent the config definition content for the config to subscribe to + * @param source the config source to use + * @param timingValues {@link TimingValues} + * @return generic handle + */ + public GenericConfigHandle subscribe(ConfigKey<?> key, List<String> defContent, ConfigSource source, TimingValues timingValues) { + checkStateBeforeSubscribe(); + GenericJRTConfigSubscription sub = new GenericJRTConfigSubscription(key, defContent, this, source, timingValues); + GenericConfigHandle handle = new GenericConfigHandle(sub); + subscribeAndHandleErrors(sub, key, handle, timingValues); + return handle; + } + + @Override + public <T extends ConfigInstance> ConfigHandle<T> subscribe(Class<T> configClass, String configId) { + throw new UnsupportedOperationException(); + } + + @Override + public <T extends ConfigInstance> ConfigHandle<T> subscribe(Class<T> configClass, String configId, long timeoutMillis) { + throw new UnsupportedOperationException(); + } + + @Override + public <T extends ConfigInstance> ConfigHandle<T> subscribe(SingleSubscriber<T> singleSubscriber, Class<T> configClass, String configId) { + throw new UnsupportedOperationException(); + } + + /** + * Do nothing, since we share requesters + */ + public void closeRequesters() { + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java new file mode 100644 index 00000000000..5e88e86be71 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.util.List; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigSource; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.TimingValues; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; + +/** + * A JRT subscription which does not use the config class, but {@link com.yahoo.vespa.config.RawConfig} instead. + * Used by config proxy. + * @author vegardh + * + */ +@SuppressWarnings("rawtypes") +public class GenericJRTConfigSubscription extends JRTConfigSubscription { + + private RawConfig config; + private final List<String> defContent; + + @SuppressWarnings("unchecked") + public GenericJRTConfigSubscription(ConfigKey<?> key, + List<String> defContent, + ConfigSubscriber subscriber, + ConfigSource source, + TimingValues timingValues) { + super(key, subscriber, source, timingValues); + this.defContent = defContent; + } + + @Override + protected void setNewConfig(JRTClientConfigRequest jrtReq) { + this.config = RawConfig.createFromResponseParameters(jrtReq); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "in setNewConfig, config=" + this.config); + } + } + + // This method is overridden because config needs to have its generation + // updated if _only_ generation has changed + @Override + void setGeneration(Long generation) { + super.setGeneration(generation); + if (this.config != null) { + this.config.setGeneration(generation); + } + } + + public RawConfig getRawConfig() { + return config; + } + + /** + * The config definition schema + * + * @return the config definition for this subscription + */ + @Override + public DefContent getDefContent() { + return (DefContent.fromList(defContent)); + } + + @Override + public ConfigInstance getConfig() { + return null; + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java new file mode 100644 index 00000000000..2e2e71989a5 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -0,0 +1,362 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.text.SimpleDateFormat; +import java.util.TimeZone; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.RequestWaiter; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; +import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.protocol.JRTConfigRequestFactory; +import com.yahoo.vespa.config.protocol.Trace; + +/** + * This class fetches config payload using JRT, and acts as the callback target. + * It uses the {@link JRTConfigSubscription} and {@link JRTClientConfigRequest} + * as context, and puts the requests objects on a queue on the subscription, + * for handling by the user thread. + * + * @author vegardh + * @since 5.1 + */ +// Note: this is similar to old JRTSource +public class JRTConfigRequester implements RequestWaiter { + private static final Logger log = Logger.getLogger(JRTConfigRequester.class.getName()); + public static final ConfigSourceSet defaultSourceSet = ConfigSourceSet.createDefault(); + private static final int TRACELEVEL = 6; + private final TimingValues timingValues; + private int fatalFailures = 0; // independent of transientFailures + private int transientFailures = 0; // independent of fatalFailures + private final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, new JRTSourceThreadFactory()); + private long suspendWarned; + private long noApplicationWarned; + private static final long delayBetweenWarnings = 60000; //ms + private final ConnectionPool connectionPool; + static final float randomFraction = 0.2f; + /* Time to be added to server timeout to create client timeout. This is the time allowed for the server to respond after serverTimeout has elapsed. */ + private static final Double additionalTimeForClientTimeout = 5.0; + + private static final SimpleDateFormat yyyyMMddz; + + static { + yyyyMMddz = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); + yyyyMMddz.setTimeZone(TimeZone.getTimeZone("GMT")); + } + + /** + * Returns a new requester + * @param connectionPool The connectionPool to use + * @param timingValues The timing values + * @return new requester object + */ + public static JRTConfigRequester get(ConnectionPool connectionPool, TimingValues timingValues) { + return new JRTConfigRequester(connectionPool, timingValues); + } + + /** + * New requester + * @param connectionPool the connectionPool this requester should use + * @param timingValues timeouts and delays used when sending JRT config requests + */ + JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) { + this.connectionPool = connectionPool; + this.timingValues = timingValues; + } + + /** + * Requests the config for the {@link com.yahoo.config.ConfigInstance} on the given {@link ConfigSubscription} + * + * @param sub a subscription + */ + public <T extends ConfigInstance> void request(JRTConfigSubscription<T> sub) { + JRTClientConfigRequest req = JRTConfigRequestFactory.createFromSub(sub); + doRequest(sub, req, timingValues.getSubscribeTimeout()); + } + + private <T extends ConfigInstance> void doRequest(JRTConfigSubscription<T> sub, + JRTClientConfigRequest req, long timeout) { + com.yahoo.vespa.config.Connection connection = connectionPool.getCurrent(); + req.getRequest().setContext(new RequestContext(sub, req, connection)); + boolean reqOK = req.validateParameters(); + if (!reqOK) throw new ConfigurationRuntimeException("Error in parameters for config request: " + req); + // Add some time to the timeout, we never want it to time out in JRT during normal operation + double jrtClientTimeout = getClientTimeout(timeout); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Requesting config for " + sub + " on connection " + connection + " with RPC timeout " + jrtClientTimeout + ",defcontent=" + + req.getDefContent().asString()); + } + connection.invokeAsync(req.getRequest(), jrtClientTimeout, this); + } + + @SuppressWarnings("unchecked") + @Override + public void handleRequestDone(Request req) { + JRTConfigSubscription<ConfigInstance> sub = null; + try { + RequestContext context = (RequestContext) req.getContext(); + sub = context.sub; + doHandle(sub, context.jrtReq, context.connection); + } catch (RuntimeException e) { + if (sub != null) { + // Sets this field, it will get thrown from the user thread + sub.setException(e); + } else { + // Very unlikely + log.log(Level.SEVERE, "Failed to get subscription object from JRT config callback: " + + Exceptions.toMessageString(e)); + } + } + } + + protected void doHandle(JRTConfigSubscription<ConfigInstance> sub, JRTClientConfigRequest jrtReq, Connection connection) { + if (sub.getState() == ConfigSubscription.State.CLOSED) return; // Avoid error messages etc. after closing + boolean validResponse = jrtReq.validateResponse(); + Trace trace = jrtReq.getResponseTrace(); + trace.trace(TRACELEVEL, "JRTConfigRequester.doHandle()"); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, trace.toString()); + } + if (validResponse) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Request callback, OK. Req: " + jrtReq + "\nSpec: " + connection); + } + handleOKRequest(jrtReq, sub, connection); + } else { + logWhenErrorResponse(jrtReq, connection); + handleFailedRequest(jrtReq, sub, connection); + } + } + + private void logWhenErrorResponse(JRTClientConfigRequest jrtReq, Connection connection) { + switch (jrtReq.errorCode()) { + case com.yahoo.jrt.ErrorCode.CONNECTION: + log.log(LogLevel.DEBUG, "Request callback failed: " + jrtReq.errorMessage() + + "\nConnection spec: " + connection); + break; + case ErrorCode.APPLICATION_NOT_LOADED: + case ErrorCode.UNKNOWN_VESPA_VERSION: + final long now = System.currentTimeMillis(); + if (noApplicationWarned < (now - delayBetweenWarnings)) { + log.log(LogLevel.WARNING, "Request callback failed: " + ErrorCode.getName(jrtReq.errorCode()) + + ". Connection spec: " + connection.getAddress()); + noApplicationWarned = now; + } + break; + default: + log.log(LogLevel.WARNING, "Request callback failed. Req: " + jrtReq + "\nSpec: " + connection.getAddress() + + " . Req error message: " + jrtReq.errorMessage()); + break; + } + } + + private void handleFailedRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription<ConfigInstance> sub, Connection connection) { + final boolean configured = (sub.getConfig() != null); + if (configured) { + // The subscription object has an "old" config, which is all we have to offer back now + log.log(LogLevel.INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub); + } + final ErrorType errorType = ErrorType.getErrorType(jrtReq.errorCode()); + connectionPool.setError(connection, jrtReq.errorCode()); + long delay = calculateFailedRequestDelay(errorType, transientFailures, fatalFailures, timingValues, configured); + if (errorType == ErrorType.TRANSIENT) { + handleTransientlyFailed(jrtReq, sub, delay, connection); + } else { + handleFatallyFailed(jrtReq, sub, delay); + } + } + + static long calculateFailedRequestDelay(ErrorType errorCode, int transientFailures, int fatalFailures, + TimingValues timingValues, boolean configured) { + long delay; + if (configured) + delay = timingValues.getConfiguredErrorDelay(); + else + delay = timingValues.getUnconfiguredDelay(); + if (errorCode == ErrorType.TRANSIENT) { + delay = delay * Math.min((transientFailures + 1), timingValues.getMaxDelayMultiplier()); + } else { + delay = timingValues.getFixedDelay() + (delay * Math.min(fatalFailures, timingValues.getMaxDelayMultiplier())); + delay = timingValues.getPlusMinusFractionRandom(delay, randomFraction); + } + return delay; + } + + private void handleTransientlyFailed(JRTClientConfigRequest jrtReq, + JRTConfigSubscription<ConfigInstance> sub, + long delay, + Connection connection) { + long now = System.currentTimeMillis(); + transientFailures++; + if (suspendWarned < (now - delayBetweenWarnings)) { + log.log(LogLevel.INFO, "Connection to " + connection.getAddress() + + " failed or timed out, clients will keep existing config, will keep trying."); + suspendWarned = now; + } + if (sub.getState() != ConfigSubscription.State.OPEN) return; + scheduleNextRequest(jrtReq, sub, delay, calculateErrorTimeout()); + } + + private long calculateErrorTimeout() { + return timingValues.getPlusMinusFractionRandom(timingValues.getErrorTimeout(), randomFraction); + } + + /** + * This handles a fatal error both in the case that the subscriber is configured and not. + * The difference is in the delay (passed from outside) and the log level used for + * error message. + * + * @param jrtReq a JRT config request + * @param sub a config subscription + * @param delay delay before sending a new request + */ + private void handleFatallyFailed(JRTClientConfigRequest jrtReq, + JRTConfigSubscription<ConfigInstance> sub, long delay) { + if (sub.getState() != ConfigSubscription.State.OPEN) return; + fatalFailures++; + // The logging depends on whether we are configured or not. + Level logLevel = sub.getConfig() == null ? LogLevel.DEBUG : LogLevel.INFO; + String logMessage = "Request for config " + jrtReq.getShortDescription() + "' failed with error code " + + jrtReq.errorCode() + " (" + jrtReq.errorMessage() + "), scheduling new connect " + + " in " + delay + " ms"; + log.log(logLevel, logMessage); + scheduleNextRequest(jrtReq, sub, delay, calculateErrorTimeout()); + } + + private void handleOKRequest(JRTClientConfigRequest jrtReq, + JRTConfigSubscription<ConfigInstance> sub, + Connection connection) { + // Reset counters pertaining to error handling here + fatalFailures = 0; + transientFailures = 0; + suspendWarned = 0; + connection.setSuccess(); + sub.setLastCallBackOKTS(System.currentTimeMillis()); + if (jrtReq.hasUpdatedGeneration()) { + // We only want this latest generation to be in the queue, we do not preserve history in this system + handleEmptyPayload(jrtReq, sub); + sub.getReqQueue().clear(); + boolean putOK = sub.getReqQueue().offer(jrtReq); + if (!putOK) { + sub.setException(new ConfigurationRuntimeException("Could not put returned request on queue of subscription " + sub)); + } + } + if (sub.getState() != ConfigSubscription.State.OPEN) return; + scheduleNextRequest(jrtReq, sub, + calculateSuccessDelay(), + calculateSuccessTimeout()); + } + + private long calculateSuccessTimeout() { + return timingValues.getPlusMinusFractionRandom(timingValues.getSuccessTimeout(), randomFraction); + } + + private long calculateSuccessDelay() { + return timingValues.getPlusMinusFractionRandom(timingValues.getFixedDelay(), randomFraction); + } + + /** + * This works around an optimization in the protocol: the payload is not set if it is not changed (seen from the server). + * So, if the sub's queue has a still _unprocessed_ req with payload, and the current one has no payload, + * i.e. it wasn't changed, save the one in the earlier req before clearing the queue. + * + * @param jrtReq a JRT config request + * @param sub a config subscription + */ + private void handleEmptyPayload(JRTClientConfigRequest jrtReq, + JRTConfigSubscription<ConfigInstance> sub) { + if (jrtReq.containsPayload()) { + JRTClientConfigRequest reqInQueue = sub.getReqQueue().poll(); // Just take it out, we were about to clear the queue anyway + if (reqInQueue != null) { + jrtReq.updateRequestPayload(reqInQueue.getNewPayload(), reqInQueue.hasUpdatedConfig()); + } + } + } + + private void scheduleNextRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription<?> sub, long delay, long timeout) { + if (delay < 0) delay = 0; + JRTClientConfigRequest jrtReqNew = jrtReq.nextRequest(timeout); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "My timing values: " + timingValues); + log.log(LogLevel.DEBUG, "Scheduling new request " + delay + " millis from now for " + jrtReqNew.getConfigKey()); + } + scheduler.schedule(new GetConfigTask(jrtReqNew, sub), delay, TimeUnit.MILLISECONDS); + } + + /** + * Task that can be scheduled in a timer for executing a getConfig request + */ + private class GetConfigTask implements Runnable { + private final JRTClientConfigRequest jrtReq; + private final JRTConfigSubscription<?> sub; + + public GetConfigTask(JRTClientConfigRequest jrtReq, + JRTConfigSubscription<?> sub) { + this.jrtReq = jrtReq; + this.sub = sub; + } + + public void run() { + doRequest(sub, jrtReq, jrtReq.getTimeout()); + } + } + + public void close() { + suspendWarned = System.currentTimeMillis(); // Avoid printing warnings after this + connectionPool.close(); + scheduler.shutdown(); + } + + private class JRTSourceThreadFactory implements ThreadFactory { + @SuppressWarnings("NullableProblems") + @Override + public Thread newThread(Runnable runnable) { + ThreadFactory tf = Executors.defaultThreadFactory(); + Thread t = tf.newThread(runnable); + // We want a daemon thread to avoid hanging threads in case something goes wrong in the config system + t.setDaemon(true); + return t; + } + } + + @SuppressWarnings("rawtypes") + private static class RequestContext { + final JRTConfigSubscription sub; + final JRTClientConfigRequest jrtReq; + final Connection connection; + + private RequestContext(JRTConfigSubscription sub, JRTClientConfigRequest jrtReq, Connection connection) { + this.sub = sub; + this.jrtReq = jrtReq; + this.connection = connection; + } + } + + int getTransientFailures() { + return transientFailures; + } + + int getFatalFailures() { + return fatalFailures; + } + + // TODO: Should be package private + public ConnectionPool getConnectionPool() { + return connectionPool; + } + + private Double getClientTimeout(long serverTimeout) { + return (serverTimeout / 1000.0) + additionalTimeForClientTimeout; + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java new file mode 100644 index 00000000000..4405ca2f05c --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -0,0 +1,190 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.config.subscription.ConfigSource; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.JRTConnectionPool; +import com.yahoo.vespa.config.TimingValues; +import com.yahoo.vespa.config.protocol.CompressionType; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; +import com.yahoo.vespa.config.protocol.Payload; + +/** + * A JRT config subscription uses one {@link JRTConfigRequester} to fetch config using Vespa RPC from a config source, typically proxy or server + * + * @author vegardh + * @since 5.1 + */ +public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + private JRTConfigRequester requester; + private TimingValues timingValues; + // Last time we got an OK JRT callback for this + private long lastOK=0; + + /** + * The queue containing either nothing or the one (newest) request that has got callback from JRT, + * but has not yet been handled. + */ + private LinkedBlockingQueue<JRTClientConfigRequest> reqQueue = new LinkedBlockingQueue<>(); + private ConfigSourceSet sources; + + public JRTConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source, TimingValues timingValues) { + super(key, subscriber); + this.timingValues=timingValues; + if (source instanceof ConfigSourceSet) { + this.sources=(ConfigSourceSet) source; + } + } + + @Override + public boolean nextConfig(long timeoutMillis) { + // These flags may have been left true from a previous call, since ConfigSubscriber's nextConfig + // not necessarily returned true and reset the flags then + boolean gotNew = isGenerationChanged() || isConfigChanged() || hasException(); + // Return that now, if there's nothing in queue, so that ConfigSubscriber can move on to other subscriptions to check + if (getReqQueue().peek()==null && gotNew) { + return true; + } + // Otherwise poll the queue for another generation or timeout + // + // Note: since the JRT callback thread will clear the queue first when it inserts a brand new element, + // there is a race here. However: the caller will handle it no matter what it gets from the queue here, + // the important part is that local state on the subscription objects is preserved. + if (!pollQueue(timeoutMillis)) return gotNew; + gotNew = isGenerationChanged() || isConfigChanged() || hasException(); + return gotNew; + } + + /** + * Polls the callback queue and <em>maybe</em> sets the following (caller must check): generation, generation changed, config, config changed + * Important: it never <em>resets</em> those flags, we must persist that state until the {@link ConfigSubscriber} clears it + * @param timeoutMillis timeout when polling (returns after at most this time) + * @return true if it got anything off the queue and <em>maybe</em> changed any state, false if timed out taking from queue + */ + private boolean pollQueue(long timeoutMillis) { + JRTClientConfigRequest jrtReq; + try { + // Only valid responses are on queue, no need to validate + jrtReq = getReqQueue().poll(timeoutMillis, TimeUnit.MILLISECONDS); + } catch (InterruptedException e1) { + throw new ConfigInterruptedException(e1); + } + if (jrtReq == null) { + // timed out, we know nothing new. + return false; + } + if (jrtReq.hasUpdatedGeneration()) { + //printStatus(jrtReq, "Updated generation or config"); + setGeneration(jrtReq.getNewGeneration()); + setGenerationChanged(true); + if (jrtReq.hasUpdatedConfig()) { + // payload changed + setNewConfig(jrtReq); + setConfigChanged(true); + } + } + return true; + } + + protected void setNewConfig(JRTClientConfigRequest jrtReq) { + setConfig(toConfigInstance(jrtReq)); + } + + /** + * This method should ideally throw new MissingConfig/Configuration exceptions and let the caller + * catch them. However, this would make the code in JRT/File/RawSource uglier. + * Alternatively, it could return a SetConfigStatus object with an int and an error message. + * + * @param jrtRequest a config request + * @return an instance of a config class (subclass of ConfigInstance) + */ + T toConfigInstance(JRTClientConfigRequest jrtRequest) { + Payload payload = jrtRequest.getNewPayload(); + ConfigPayload configPayload = ConfigPayload.fromUtf8Array(payload.withCompression(CompressionType.UNCOMPRESSED).getData()); + T configInstance = configPayload.toInstance(configClass, jrtRequest.getConfigKey().getConfigId()); + configInstance.setConfigMd5(jrtRequest.getNewConfigMd5()); + return configInstance; + } + + LinkedBlockingQueue<JRTClientConfigRequest> getReqQueue() { + return reqQueue; + } + + @Override + public boolean subscribe(long timeout) { + lastOK=System.currentTimeMillis(); + requester = getRequester(); + requester.request(this); + JRTClientConfigRequest req = reqQueue.peek(); + while (req == null && (System.currentTimeMillis() - lastOK <= timeout)) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + throw new ConfigInterruptedException(e); + } + req = reqQueue.peek(); + } + return req != null; + } + + private JRTConfigRequester getRequester() { + JRTConfigRequester requester = subscriber.requesters().get(sources); + if (requester==null) { + requester = new JRTConfigRequester(new JRTConnectionPool(sources), timingValues); + subscriber.requesters().put(sources, requester); + } + return requester; + } + + @Override + @SuppressWarnings("serial") + public void close() { + super.close(); + reqQueue = new LinkedBlockingQueue<JRTClientConfigRequest>() { + @Override public void put(JRTClientConfigRequest e) throws InterruptedException { + // When closed, throw away all requests that callbacks try to put + } + }; + } + + /** + * The timing values of this + * @return timing values + */ + public TimingValues timingValues() { + return timingValues; + } + + // Used in integration tests + @SuppressWarnings("UnusedDeclaration") + public JRTConfigRequester requester() { + return requester; + } + + @Override + public void reload(long generation) { + log.log(LogLevel.DEBUG, "reload() is without effect on a JRTConfigSubscription."); + } + + void setLastCallBackOKTS(long lastCallBackOKTS) { + this.lastOK = lastCallBackOKTS; + } + + // For debugging + @SuppressWarnings("UnusedDeclaration") + static void printStatus(JRTClientConfigRequest request, String message) { + final String name = request.getConfigKey().getName(); + if (name.equals("components") || name.equals("chains")) { + log.log(LogLevel.INFO, message + ":" + name + ":" + ", request=" + request); + } + } +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java new file mode 100644 index 00000000000..434b61db918 --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.jar.JarFile; +import java.util.zip.ZipEntry; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.subscription.CfgConfigPayloadBuilder; +import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; + +/** + * Subscription to use when config id is jar:.../foo.jar[!/pathInJar/] + * + * @author vegardh + * @author gjoranv + * @since 5.1 + * + */ +public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + private final String jarName; + private final String path; + private ZipEntry zipEntry = null; + + // jar:configs/app.jar!/configs/ + JarConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, String jarName, String path) { + super(key, subscriber); + this.jarName=jarName; + this.path=path; + } + + @Override + public boolean nextConfig(long timeout) { + if (checkReloaded()) { + // Not supporting changing the payload for jar + return true; + } + if (zipEntry==null) { + // First time polled + JarFile jarFile = null; + try { + jarFile = new JarFile(jarName); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + zipEntry = getEntry(jarFile, path); + if (zipEntry==null) throw new IllegalArgumentException("Config '" + key.getName() + "' not found in '" + jarName + "!/" + path + "'."); + try { + ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(IOUtils.readAll(new InputStreamReader(jarFile.getInputStream(zipEntry), "UTF-8")).split("\n"))); + config = payload.toInstance(configClass, key.getConfigId()); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } catch (IOException e) { + throw new ConfigurationRuntimeException(e); + } + setGeneration(0l); + setGenerationChanged(true); + setConfigChanged(true); + try { + jarFile.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return true; + } + // TODO: Should wait and detect changes + try { + Thread.sleep(timeout); + } catch (InterruptedException e) { + throw new ConfigInterruptedException(e); + } + // These shouldn't be checked anywhere since we return false now, but setting them still + setGenerationChanged(false); + setConfigChanged(false); + return false; + } + /** + * Returns the entry corresponding to the ConfigInstance's defName/Version in the given directory in + * the given JarFile. + * If the file with correct version number does not exist, returns the filename without version number. + * The file's existence is checked elsewhere. + */ + private ZipEntry getEntry(JarFile jarFile, String dir) { + if (!dir.endsWith("/")) { + dir = dir + '/'; + } + return jarFile.getEntry(dir + getConfigFilenameNoVersion(key)); + } + + @Override + public boolean subscribe(long timeout) { + return true; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java new file mode 100644 index 00000000000..3e9047b3bfa --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import com.yahoo.jrt.Request; +import com.yahoo.jrt.RequestWaiter; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.Connection; +import com.yahoo.vespa.config.ConnectionPool; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; +import com.yahoo.vespa.config.protocol.Payload; +import com.yahoo.vespa.config.util.ConfigUtils; + +/** + * For unit testing + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 5.1.11 + */ +public class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { + + private Request lastRequest; + private final ResponseHandler responseHandler; + private int numberOfRequests = 0; + + public int getNumberOfFailovers() { + return numberOfFailovers; + } + + private int numberOfFailovers = 0; + private final int numSpecs; + + public MockConnection() { + this(new OKResponseHandler()); + } + + public MockConnection(ResponseHandler responseHandler) { + this(responseHandler, 1); + } + + public MockConnection(ResponseHandler responseHandler, int numSpecs) { + this.responseHandler = responseHandler; + this.numSpecs = numSpecs; + } + + @Override + public void invokeAsync(Request request, double jrtTimeout, RequestWaiter requestWaiter) { + numberOfRequests++; + lastRequest = request; + responseHandler.requestWaiter(requestWaiter).request(request); + Thread t = new Thread(responseHandler); + t.setDaemon(true); + t.run(); + } + + @Override + public void setError(int errorCode) { + numberOfFailovers++; + } + + @Override + public void setSuccess() { + numberOfFailovers = 0; + } + + @Override + public String getAddress() { + return null; + } + + @Override + public void close() { + + } + + @Override + public void setError(Connection connection, int errorCode) { + connection.setError(errorCode); + } + + @Override + public Connection getCurrent() { + return this; + } + + @Override + public Connection setNewCurrentConnection() { + return this; + } + + @Override + public int getSize() { + return numSpecs; + } + + public int getNumberOfRequests() { + return numberOfRequests; + } + + public Request getRequest() { + return lastRequest; + } + + static class OKResponseHandler extends AbstractResponseHandler { + protected void createResponse() { + JRTServerConfigRequestV3 jrtReq = JRTServerConfigRequestV3.createFromRequest(request); + Payload payload = Payload.from(ConfigPayload.empty()); + long generation = 1; + jrtReq.addOkResponse(payload, generation, ConfigUtils.getMd5(payload.getData())); + } + } + + + public interface ResponseHandler extends Runnable { + + RequestWaiter requestWaiter(); + + Request request(); + + ResponseHandler requestWaiter(RequestWaiter requestWaiter); + + ResponseHandler request(Request request); + } + + public abstract static class AbstractResponseHandler implements ResponseHandler { + private RequestWaiter requestWaiter; + protected Request request; + + @Override + public RequestWaiter requestWaiter() { + return requestWaiter; + } + + @Override + public Request request() { + return request; + } + + @Override + public ResponseHandler requestWaiter(RequestWaiter requestWaiter) { + this.requestWaiter = requestWaiter; + return this; + } + + @Override + public ResponseHandler request(Request request) { + this.request = request; + return this; + } + + @Override + public void run() { + createResponse(); + requestWaiter.handleRequestDone(request); + } + + protected abstract void createResponse(); + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java new file mode 100644 index 00000000000..2c8470cf63c --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import java.util.Arrays; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.CfgConfigPayloadBuilder; +import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; + +/** + * Subscription used when config id is raw:... + * + * Config is the actual text given after the config id, with newlines + * + * @author vegardh + * @since 5.1 + * + */ +public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> { + final String inputPayload; + String payload; + + RawConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, String pl) { + super(key, subscriber); + this.inputPayload=pl; + } + + @Override + public boolean nextConfig(long timeout) { + if (checkReloaded()) { + return true; + } + if (payload==null) { + payload = inputPayload; + setGeneration(0l); + setGenerationChanged(true); + setConfigChanged(true); + + ConfigPayload configPayload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(payload.split("\n"))); + config = configPayload.toInstance(configClass, key.getConfigId()); + return true; + } + try { + Thread.sleep(timeout); + } catch (InterruptedException e) { + throw new ConfigInterruptedException(e); + } + // These shouldn't be checked anywhere since we return false now, but setting them still + setGenerationChanged(false); + setConfigChanged(false); + return false; + } + + @Override + public boolean subscribe(long timeout) { + return true; + } + +} diff --git a/config/src/main/java/com/yahoo/config/subscription/package-info.java b/config/src/main/java/com/yahoo/config/subscription/package-info.java new file mode 100644 index 00000000000..7a77088a43e --- /dev/null +++ b/config/src/main/java/com/yahoo/config/subscription/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +/** + * Classes for subscribing to Vespa config. + */ +package com.yahoo.config.subscription; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/jrt/.gitignore b/config/src/main/java/com/yahoo/jrt/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/main/java/com/yahoo/jrt/.gitignore diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigCacheKey.java b/config/src/main/java/com/yahoo/vespa/config/ConfigCacheKey.java new file mode 100644 index 00000000000..15e67138393 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigCacheKey.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * A ConfigKey that also uses the def MD5 sum. Used for caching when def payload is user provided. + * @author vegardh + * + */ +public class ConfigCacheKey { + private final ConfigKey<?> key; + private final String defMd5; + + /** + * Constructs a new server key based on the contents of the given {@link ConfigKey} and the def md5 sum. + * @param key The key to base on + * @param defMd5 MD5 checksum of the config definition. Never null. + */ + public ConfigCacheKey(ConfigKey<?> key, String defMd5) { + this.key = key; + this.defMd5 = defMd5 == null ? "" : defMd5; + } + + /** + * Constructs new key + * + * @param name config definition name + * @param configIdString Can be null. + * @param namespace namespace for this config definition + * @param defMd5 MD5 checksum of the config definition. Never null. + */ + ConfigCacheKey(String name, String configIdString, String namespace, String defMd5) { + this(new ConfigKey<>(name, configIdString, namespace), defMd5); + } + + @Override + public int hashCode() { + return key.hashCode() + 37 * defMd5.hashCode(); + } + + @Override + public boolean equals(Object o) { + return o instanceof ConfigCacheKey && key.equals(((ConfigCacheKey) o).getKey()) + && defMd5.equals(((ConfigCacheKey)o).defMd5); + } + + /** + * The def md5 sum of this key + * @return md5 sum + */ + public String getDefMd5() { + return defMd5; + } + + public ConfigKey<?> getKey() { + return key; + } + + @Override + public String toString() { + return key + "," + defMd5; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java new file mode 100644 index 00000000000..9b4f8d12756 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java @@ -0,0 +1,1068 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.yolean.Exceptions; + +import java.util.*; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +/** + * Represents one legal def file, or (internally) one array or inner array definition in a def file. + * Definitions are comparable based on version. + * @author vegardh + * + */ +public class ConfigDefinition implements Comparable<ConfigDefinition> { + public static final Pattern namePattern = Pattern.compile("[a-zA-Z][a-zA-Z0-9-_]*"); + public static final Pattern namespacePattern = Pattern.compile("[a-zA-Z][a-zA-Z0-9-\\._]*"); + + public static Logger log = Logger.getLogger(ConfigDefinition.class.getName()); + private final String name; + private final String version; + private final String namespace; + protected ConfigDefinition parent = null; + + // TODO Strings without default are null, could be not OK. + private Map<String, StringDef> stringDefs = new LinkedHashMap<String, StringDef>(); + private Map<String, BoolDef> boolDefs = new LinkedHashMap<String, BoolDef>(); + private Map<String, IntDef> intDefs = new LinkedHashMap<String, IntDef>(); + private Map<String, LongDef> longDefs = new LinkedHashMap<String, LongDef>(); + private Map<String, DoubleDef> doubleDefs = new LinkedHashMap<String, DoubleDef>(); + private Map<String, EnumDef> enumDefs = new LinkedHashMap<String, EnumDef>(); + private Map<String, RefDef> referenceDefs = new LinkedHashMap<String, RefDef>(); + private Map<String, FileDef> fileDefs = new LinkedHashMap<String, FileDef>(); + private Map<String, PathDef> pathDefs = new LinkedHashMap<>(); + private Map<String, StructDef> structDefs = new LinkedHashMap<String, StructDef>(); + private Map<String, InnerArrayDef> innerArrayDefs = new LinkedHashMap<String, InnerArrayDef>(); + private Map<String, ArrayDef> arrayDefs = new LinkedHashMap<String, ArrayDef>(); + private Map<String, LeafMapDef> leafMapDefs = new LinkedHashMap<>(); + private Map<String, StructMapDef> structMapDefs = new LinkedHashMap<>(); + + public static final Integer INT_MIN = -0x80000000; + public static final Integer INT_MAX = 0x7fffffff; + + public static final Long LONG_MIN = -0x8000000000000000L; + public static final Long LONG_MAX = 0x7fffffffffffffffL; + + public static final Double DOUBLE_MIN = -1e308d; + public static final Double DOUBLE_MAX = 1e308d; + + public ConfigDefinition(String name, String version, String namespace) { + this.name = name; + this.version = version; + this.namespace = namespace; + } + + public ConfigDefinition(String name, String version) { + this(name, version, CNode.DEFAULT_NAMESPACE); + } + + public String getName() { + return name; + } + + public String getVersion() { + return version; + } + + public String getNamespace() { + return namespace; + } + + /** @return The parent ConfigDefinition, or null if this is the root. */ + public ConfigDefinition getParent() { + return parent; + } + + /** @return The root ConfigDefinition, might be this. */ + public ConfigDefinition getRoot() { + ConfigDefinition ancestor = this; + while (ancestor.getParent() != null) { + ancestor = ancestor.getParent(); + } + return ancestor; + } + + private static void defFail(String id, String val, String type, Exception e, List<String> warnings) { + defFail("Invalid value '" + val + "' for " + type + " '" + id + "'. "+ Exceptions.toMessageString(e), warnings); + } + + public void verify(String id, String val, List<String> warnings) { + if (stringDefs.containsKey(id)) { + verifyString(id, warnings); + } else if (enumDefs.containsKey(id)) { + verifyEnum(id ,val, warnings); + } else if (referenceDefs.containsKey(id)) { + verifyReference(id, warnings); + } else if (fileDefs.containsKey(id)) { + verifyFile(id, warnings); + } else if (pathDefs.containsKey(id)) { + verifyPath(id, warnings); + } else if (boolDefs.containsKey(id)) { + verifyBool(id, val, warnings); + } else if (intDefs.containsKey(id)) { + verifyInt(id, val, warnings); + } else if (longDefs.containsKey(id)) { + verifyLong(id, val, warnings); + } else if (doubleDefs.containsKey(id)) { + verifyDouble(id, val, warnings); + } else if (structDefs.containsKey(id)) { + verifyStruct(id, warnings); + } else if (arrayDefs.containsKey(id)) { + verifyArray(id, warnings); + } else if (innerArrayDefs.containsKey(id)) { + verifyInnerArray(id, warnings); + } else if (leafMapDefs.containsKey(id)) { + verifyLeafMap(id, warnings); + } else if (structMapDefs.containsKey(id)) { + verifyStructMap(id, warnings); + } else { + defFail("No such field in definition " + getRoot().getNamespace() + "." + getRoot().getName() + + ": " + getAncestorString() + id, warnings); + } + } + + private boolean verifyDouble(String id, String val, List<String> warnings) { + try { + return verifyDouble(id, Double.parseDouble(val), warnings); + } catch (NumberFormatException e) { + defFail(id, val, "double", e, warnings); + return false; + } + } + + private boolean verifyBool(String id, String val, List<String> warnings) { + if ("true".equalsIgnoreCase(val) || "false".equalsIgnoreCase(val)) { + return verifyBool(id, warnings); + } else { + defFail(id, val, "bool", null, warnings); + return false; + } + } + + public void verify(String id, List<String> warnings) { + verify(id, null, warnings); + } + + /** + * Compares def-versions. Examples: 2 is higher than 1, and 2-0-0 is higher than 1-2-2 but the same as 2. + */ + public static class VersionComparator implements Comparator<String> { + int[] parseVersion(String version) { + int[] result = {0, 0, 0}; + String[] v = version.split("-"); + + for (int i = 0; i < 3; i++) { + if (v.length > i) result[i] = Integer.parseInt(v[i]); + } + + return result; + } + + public int compare(String o1, String o2) throws ClassCastException { + int[] version1 = parseVersion(o1); + int[] version2 = parseVersion(o2); + + for (int i = 0; i < 3; i ++) { + int diff = version1[i] - version2[i]; + if (diff != 0) return diff; + } + + return 0; + } + } + + /** + * String based ("untyped") type specification used by parser and arrays. May have the name of the field which it describes. + * The index number is used to export data in correct order. + * @author vegardh + * + */ + public static class TypeSpec { + private String type; // TODO Class? + private Integer index; + private String name; + private Object defVal; + private Object min; + private Object max; + private List<String> enumVals; + + public TypeSpec(String name, String type, Object defVal, String enumValsCommaSep, Object min, Object max) { + this.name=name; + this.type = type; + this.defVal = defVal; + this.enumVals = getEnumVals(enumValsCommaSep); + this.min = min; + this.max = max; + } + + private List<String> getEnumVals(String commaSep) { + if (commaSep==null) { + return null; + } + List<String> in = new ArrayList<String>(); + for (String val: commaSep.split(",")) { + in.add(val.trim()); + } + return in; + } + public String getName() { + return name; + } + public String getType() { + return type; + } + /*public Class getTypeClass() { + return typeClass; + }*/ + public Object getDef() { + return defVal; + } + public Object getMin() { + return min; + } + public Object getMax() { + return max; + } + public List<String> getEnumVals() { + return enumVals; + } + + public boolean checkValue(String id, String val, int index, List<String> warnings) { + if ("int".equals(getType())) { + return checkInt(id, val, index, warnings); + } else if ("long".equals(getType())) { + return checkLong(id, val, index, warnings); + } else if ("double".equals(getType())) { + return checkDouble(id, val, index, warnings); + } else if ("enum".equals(getType())) { + return checkEnum(id, val, index, warnings); + } + return true; + } + + private boolean checkEnum(String id, String val, int index, List<String> warnings) { + if (!getEnumVals().contains(val)) { + ConfigDefinition.failInvalidEnum(val, id, id+"["+index+"]", warnings); + return false; + } + return true; + } + + private boolean checkDouble(String id, String val, int index, List<String> warnings) { + try { + return checkDouble(Double.parseDouble(val), id, index, warnings); + } catch (NumberFormatException e) { + ConfigDefinition.defFail(id, val, "double", e, warnings); + return false; + } + } + + private boolean checkLong(String id, String val, int index, List<String> warnings) { + try { + return checkLong(Long.parseLong(val), id, index, warnings); + } catch (NumberFormatException e) { + ConfigDefinition.defFail(id, val, "long", e, warnings); + return false; + } + } + + private boolean checkInt(String id, String val, int index, List<String> warnings) { + try { + return checkInt(Integer.parseInt(val), id, index, warnings); + } catch (NumberFormatException e) { + ConfigDefinition.defFail(id, val, "int", e, warnings); + return false; + } + } + + private boolean checkInt(Integer theVal, String id, int arrayIndex, List<String> warnings) { + if (!"int".equals(getType())) { + ConfigDefinition.defFail("Illegal value \""+theVal+"\" for array \""+id+"\"", warnings); + return false; + } + if (getMax()!=null && theVal>(Integer)getMax()) { + ConfigDefinition.failTooBig(theVal, getMax(), id, id+"["+arrayIndex+"]", warnings); + return false; + } + if (getMin()!=null && theVal<(Integer)getMin()) { + ConfigDefinition.failTooSmall(theVal, getMin(), id, id+"["+arrayIndex+"]", warnings); + return false; + } + return true; + } + + private boolean checkLong(Long theVal, String id, int arrayIndex, List<String> warnings) { + if (!"long".equals(getType())) { + ConfigDefinition.defFail("Illegal value \""+theVal+"\" for array \""+id+"\"", warnings); + return false; + } + if (getMax()!=null && theVal>(Long)getMax()) { + ConfigDefinition.failTooBig(theVal, getMax(), id, id+"["+arrayIndex+"]", warnings); + return false; + } + if (getMin()!=null && theVal<(Long)getMin()) { + ConfigDefinition.failTooSmall(theVal, getMin(), id, id+"["+arrayIndex+"]", warnings); + return false; + } + return true; + } + + private boolean checkDouble(Double theVal, String id, int arrayIndex, List<String> warnings) { + if (!"double".equals(getType())) { + ConfigDefinition.defFail("Illegal value \""+theVal+"\" for array \""+id+"\", array type is "+getType(), warnings); + return false; + } + if (getMax()!=null && (theVal>(Double)getMax())) { + ConfigDefinition.failTooBig(theVal, getMax(), id, id+"["+arrayIndex+"]", warnings); + return false; + } + if (getMin()!=null && theVal<(Double)getMin()) { + ConfigDefinition.failTooSmall(theVal, getMin(), id, id+"["+arrayIndex+"]", warnings); + return false; + } + return true; + } + + public void setIndex(Integer index) { + this.index = index; + } + public Integer getIndex() { + return index; + } + } + + /** + * A ConfigDefinition that represents a struct, e.g. a.foo, a.bar where 'a' is the struct. Can be thought + * of as an inner array with only one element. + */ + public static class StructDef extends ConfigDefinition { + public StructDef(String name, String version, ConfigDefinition parent) { + super(name, version); + this.parent = parent; + } + } + + /** + * An InnerArray def is a ConfigDefinition with n scalar types of defs, and maybe sub-InnerArrays + * @author vegardh + * + */ + public static class InnerArrayDef extends ConfigDefinition { + public InnerArrayDef(String name, String version, ConfigDefinition parent) { + super(name, version); + this.parent = parent; + } + } + + /** + * An array def is a ConfigDefinition with only one other type of scalar def. + * @author vegardh + * + */ + public static class ArrayDef extends ConfigDefinition { + private TypeSpec typeSpec; + public ArrayDef(String name, String version, ConfigDefinition parent) { + super(name, version); + this.parent = parent; + } + public TypeSpec getTypeSpec() { + return typeSpec; + } + public void setTypeSpec(TypeSpec typeSpec) { + this.typeSpec = typeSpec; + } + + public void verify(String val, int index, List<String> warnings) { + if (val != null && getTypeSpec() != null) { + TypeSpec spec = getTypeSpec(); + spec.checkValue(getName(), val, index, warnings); + } + } + } + + /** + * Def of a myMap{} int + * @author vegardh + * + */ + public static class LeafMapDef extends ConfigDefinition { + private TypeSpec typeSpec; + public LeafMapDef(String name, String version, ConfigDefinition parent) { + super(name, version); + this.parent = parent; + } + public TypeSpec getTypeSpec() { + return typeSpec; + } + public void setTypeSpec(TypeSpec typeSpec) { + this.typeSpec = typeSpec; + } + } + + /** + * Def of a myMap{}.myInt int + * @author vegardh + * + */ + public static class StructMapDef extends ConfigDefinition { + public StructMapDef(String name, String version, ConfigDefinition parent) { + super(name, version); + this.parent = parent; + } + } + + /** + * A Default specification where instances _may_ have a default value + * @author vegardh + */ + public static interface DefaultValued<T> { + public T getDefVal(); + } + + public static class EnumDef implements DefaultValued<String>{ + private List<String> vals; + private String defVal; + public EnumDef(List<String> vals, String defVal) { + if (defVal!=null && !vals.contains(defVal)) { + throw new IllegalArgumentException("Def val "+defVal+" is not in given vals "+vals); + } + this.vals = vals; + this.defVal = defVal; + } + public List<String> getVals() { + return vals; + } + + @Override + public String getDefVal() { + return defVal; + } + } + + public static class StringDef implements DefaultValued<String> { + private String defVal; + + public StringDef(String def) { + this.defVal=def; + } + + @Override + public String getDefVal() { + return defVal; + } + } + + public static class BoolDef implements DefaultValued<Boolean> { + private Boolean defVal; + + public BoolDef(Boolean def) { + this.defVal=def; + } + + @Override + public Boolean getDefVal() { + return defVal; + } + } + + /** The type is called 'double' in .def files, but it is a 64-bit IEE 754 double, + * which means it must be represented as a double in Java + */ + public static class DoubleDef implements DefaultValued<Double> { + private Double defVal; + private Double min; + private Double max; + public DoubleDef(Double defVal, Double min, Double max) { + super(); + this.defVal = defVal; + if (min == null) { + this.min = DOUBLE_MIN; + } else { + this.min = min; + } + if (max == null){ + this.max = DOUBLE_MAX; + } else { + this.max = max; + } + } + + @Override + public Double getDefVal() { + return defVal; + } + public Double getMin() { + return min; + } + public Double getMax() { + return max; + } + } + + public static class IntDef implements DefaultValued<Integer>{ + private Integer defVal; + private Integer min; + private Integer max; + public IntDef(Integer def, Integer min, Integer max) { + super(); + this.defVal = def; + if (min == null) { + this.min = INT_MIN; + } else { + this.min = min; + } + if (max == null) { + this.max = INT_MAX; + } else { + this.max = max; + } + } + + @Override + public Integer getDefVal() { + return defVal; + } + public Integer getMin() { + return min; + } + public Integer getMax() { + return max; + } + } + + public static class LongDef implements DefaultValued<Long>{ + private Long defVal; + private Long min; + private Long max; + public LongDef(Long def, Long min, Long max) { + super(); + this.defVal = def; + if (min == null) { + this.min = LONG_MIN; + } else { + this.min = min; + } + if (max == null) { + this.max = LONG_MAX; + } else { + this.max = max; + } + } + + @Override + public Long getDefVal() { + return defVal; + } + public Long getMin() { + return min; + } + public Long getMax() { + return max; + } + } + + public static class RefDef implements DefaultValued<String>{ + private String defVal; + + public RefDef(String defVal) { + super(); + this.defVal = defVal; + } + + @Override + public String getDefVal() { + return defVal; + } + } + + public static class FileDef implements DefaultValued<String>{ + private String defVal; + + public FileDef(String defVal) { + super(); + this.defVal = defVal; + } + + @Override + public String getDefVal() { + return defVal; + } + } + + public static class PathDef implements DefaultValued<String>{ + private String defVal; + + public PathDef(String defVal) { + this.defVal = defVal; + } + + @Override + public String getDefVal() { + return defVal; + } + } + + public void addEnumDef(String id, EnumDef def) { + enumDefs.put(id, def); + } + + public void addInnerArrayDef(String id) { + innerArrayDefs.put(id, new InnerArrayDef(id, version, this)); + } + + public void addLeafMapDef(String id) { + leafMapDefs.put(id, new LeafMapDef(id, version, this)); + } + + public void addEnumDef(String id, List<String> vals, String defVal) { + List<String> in = new ArrayList<String>(); + for (String ins: vals) { + in.add(ins.trim()); + } + enumDefs.put(id, new EnumDef(in, defVal)); + } + + public void addEnumDef(String id, String valsCommaSep, String defVal) { + String[] valArr = valsCommaSep.split(","); + addEnumDef(id, Arrays.asList(valArr), defVal); + } + + public void addStringDef(String id, String defVal) { + stringDefs.put(id, new StringDef(defVal)); + } + + public void addStringDef(String id) { + stringDefs.put(id, new StringDef(null)); + } + + public void addIntDef(String id, Integer defVal, Integer min, Integer max) { + intDefs.put(id, new IntDef(defVal, min, max)); + } + + public void addIntDef(String id, Integer defVal) { + addIntDef(id, defVal, INT_MIN, INT_MAX); + } + + public void addIntDef(String id) { + addIntDef(id, null); + } + + public void addLongDef(String id, Long defVal, Long min, Long max) { + longDefs.put(id, new LongDef(defVal, min, max)); + } + + public void addLongDef(String id, Long defVal) { + addLongDef(id, defVal, LONG_MIN, LONG_MAX); + } + + public void addLongDef(String id) { + addLongDef(id, null); + } + + public void addBoolDef(String id) { + boolDefs.put(id, new BoolDef(null)); + } + + public void addBoolDef(String id, Boolean defVal) { + boolDefs.put(id, new BoolDef(defVal)); + } + + public void addDoubleDef(String id, Double defVal, Double min, Double max) { + doubleDefs.put(id, new DoubleDef(defVal, min, max)); + } + + public void addDoubleDef(String id, Double defVal) { + addDoubleDef(id, defVal, DOUBLE_MIN, DOUBLE_MAX); + } + + public void addDoubleDef(String id) { + addDoubleDef(id, null); + } + + public void addReferenceDef(String refId, String defVal) { + referenceDefs.put(refId, new RefDef(defVal)); + } + + public void addReferenceDef(String refId) { + referenceDefs.put(refId, new RefDef(null)); + } + + public void addFileDef(String refId, String defVal) { + fileDefs.put(refId, new FileDef(defVal)); + } + + public void addFileDef(String refId) { + fileDefs.put(refId, new FileDef(null)); + } + + public void addPathDef(String refId, String defVal) { + pathDefs.put(refId, new PathDef(defVal)); + } + + public void addPathDef(String refId) { + pathDefs.put(refId, new PathDef(null)); + } + + public Map<String, StringDef> getStringDefs() { + return stringDefs; + } + + public Map<String, BoolDef> getBoolDefs() { + return boolDefs; + } + + public Map<String, IntDef> getIntDefs() { + return intDefs; + } + + public Map<String, LongDef> getLongDefs() { + return longDefs; + } + + public Map<String, DoubleDef> getDoubleDefs() { + return doubleDefs; + } + + public Map<String, RefDef> getReferenceDefs() { + return referenceDefs; + } + + public Map<String, FileDef> getFileDefs() { + return fileDefs; + } + + public Map<String, PathDef> getPathDefs() { + return pathDefs; + } + + public Map<String, InnerArrayDef> getInnerArrayDefs() { + return innerArrayDefs; + } + + public Map<String, LeafMapDef> getLeafMapDefs() { + return leafMapDefs; + } + + public Map<String, StructMapDef> getStructMapDefs() { + return structMapDefs; + } + + public InnerArrayDef innerArrayDef(String name) { + InnerArrayDef ret = innerArrayDefs.get(name); + if (ret!=null) { + return ret; + } + ret = new InnerArrayDef(name, version, this); + innerArrayDefs.put(name, ret); + return ret; + } + + public Map<String, StructDef> getStructDefs() { + return structDefs; + } + + public StructDef structDef(String name) { + StructDef ret = structDefs.get(name); + if (ret!=null) { + return ret; + } + ret = new StructDef(name, version, this); + structDefs.put(name, ret); + return ret; + } + + public Map<String, EnumDef> getEnumDefs() { + return enumDefs; + } + + public ArrayDef arrayDef(String name) { + ArrayDef ret = arrayDefs.get(name); + if (ret!=null) { + return ret; + } + ret = new ArrayDef(name, version, this); + arrayDefs.put(name, ret); + return ret; + } + + public Map<String, ArrayDef> getArrayDefs() { + return arrayDefs; + } + + public StructMapDef structMapDef(String name) { + StructMapDef ret = structMapDefs.get(name); + if (ret!=null) { + return ret; + } + ret = new StructMapDef(name, version, this); + structMapDefs.put(name, ret); + return ret; + } + + public LeafMapDef leafMapDef(String name) { + LeafMapDef ret = leafMapDefs.get(name); + if (ret!=null) { + return ret; + } + ret = new LeafMapDef(name, version, this); + leafMapDefs.put(name, ret); + return ret; + } + + /** + * Throws if the given value is not legal + */ + private boolean verifyDouble(String id, Double val, List<String> warnings) { + DoubleDef def = doubleDefs.get(id); + if (def==null) { + defFail("No such double in " + verifyWarning(id), warnings); + return false; + } + if (val==null) { + return true; + } + if (def.getMin()!=null && val<def.getMin()) { + failTooSmall(val, def.getMin(), toString(), getAncestorString()+id, warnings); + return false; + } + if (def.getMax()!=null && val>def.getMax()) { + failTooBig(val, def.getMax(), toString(), getAncestorString()+id, warnings); + return false; + } + return true; + } + + /** + * Throws if the given value is not legal + */ + private boolean verifyEnum(String id, String val, List<String> warnings) { + EnumDef def = enumDefs.get(id); + if (def==null) { + defFail("No such enum in " + verifyWarning(id), warnings); + return false; + } + if (!def.getVals().contains(val)) { + defFail("Invalid enum value '"+val+"' in def "+toString()+ + " enum '"+getAncestorString()+id+"'.", warnings); + return false; + } + return true; + } + + /** + * Throws if the given value is not legal + */ + private boolean verifyInt(String id, Integer val, List<String> warnings) { + IntDef def = intDefs.get(id); + if (def==null) { + defFail("No such integer in " + verifyWarning(id), warnings); + return false; + } + if (val==null) { + return true; + } + if (def.getMin()!=null && val<def.getMin()) { + failTooSmall(val, def.getMin(), name, id, warnings); + return false; + } + if (def.getMax()!=null && val>def.getMax()) { + failTooBig(val, def.getMax(), name, id, warnings); + return false; + } + return true; + } + + private boolean verifyInt(String id, String val, List<String> warnings) { + try { + return verifyInt(id, Integer.parseInt(val), warnings); + } catch (NumberFormatException e) { + ConfigDefinition.defFail(id, val, "int", e, warnings); + return false; + } + } + + private boolean verifyLong(String id, String val, List<String> warnings) { + try { + return verifyLong(id, Long.parseLong(val), warnings); + } catch (NumberFormatException e) { + ConfigDefinition.defFail(id, val, "long", e, warnings); + return false; + } + } + + /** + * Throws if the given value is not legal + */ + private boolean verifyLong(String id, Long val, List<String> warnings) { + LongDef def = longDefs.get(id); + if (def==null) { + defFail("No such long in " + verifyWarning(id), warnings); + return false; + } + if (val==null) { + return true; + } + if (def.getMin()!=null && val<def.getMin()) { + failTooSmall(val, def.getMin(), name, id, warnings); + return false; + } + if (def.getMax()!=null && val>def.getMax()) { + failTooBig(val, def.getMax(), name, id, warnings); + return false; + } + return true; + } + + static void failTooSmall(Object val, Object min, String defName, String valKey, List<String> warnings) { + defFail("Value \""+valKey+"\" outside range in definition \""+defName+"\": "+val+"<"+min, warnings); + } + + static void failTooBig(Object val, Object max, String defName, String valKey, List<String> warnings) { + defFail("Value \""+valKey+"\" outside range in definition \""+defName+"\": "+val+">"+max, warnings); + } + + static void failInvalidEnum(Object val, String defName, String defKey, List<String> warnings) { + defFail("Invalid enum value \""+val+"\" for \""+defKey+"\" in definition \""+defName, warnings); + } + + /** + * Adds the given log msg to list, and logs it + * @param msg failure message + * @param warnings list of warnings collected during model building. + * @return warnings list with msg added + */ + static List<String> defFail(String msg, List<String> warnings) { + throw new IllegalArgumentException(msg); + // Idea here is to store errors in list instead, and throw from model builder in vespamodel instead. But not so important. + /*warnings.add(msg); + log.log(LogLevel.WARNING, msg); + return warnings;*/ + } + + private boolean verifyString(String id, List<String> warnings) { + if (!stringDefs.containsKey(id)) { + defFail("No such string in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyReference(String id, List<String> warnings) { + if (!referenceDefs.containsKey(id)) { + defFail("No such reference in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyFile(String id, List<String> warnings) { + if (!fileDefs.containsKey(id)) { + defFail("No such file in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyPath(String id, List<String> warnings) { + if (!pathDefs.containsKey(id)) { + defFail("No such path in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyBool(String id, List<String> warnings) { + if (!boolDefs.containsKey(id)) { + defFail("No such bool in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyArray(String id, List<String> warnings) { + String failString = "No such array in " + verifyWarning(id); + if (!arrayDefs.containsKey(id)) { + if (innerArrayDefs.containsKey(id)) { + failString += ". However, the definition does contain an inner array with the same name."; + } + defFail(failString, warnings); + return false; + } + return true; + } + + private boolean verifyInnerArray(String id, List<String> warnings) { + String failString = "No such inner array in " + verifyWarning(id); + if (!innerArrayDefs.containsKey(id)) { + if (arrayDefs.containsKey(id)) { + failString += ". However, the definition does contain an array with the same name."; + } + defFail(failString, warnings); + return false; + } + return true; + } + + private boolean verifyStruct(String id, List<String> warnings) { + if (!structDefs.containsKey(id)) { + defFail("No such struct in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyLeafMap(String id, List<String> warnings) { + if (!leafMapDefs.containsKey(id)) { + defFail("No such leaf map in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private boolean verifyStructMap(String id, List<String> warnings) { + if (!structMapDefs.containsKey(id)) { + defFail("No such struct map in " + verifyWarning(id), warnings); + return false; + } + return true; + } + + private String verifyWarning(String id) { + return "definition '" + getRoot().toString() + "': " + getAncestorString() + id; + } + + /** + * Returns a string composed of the ancestors of this ConfigDefinition, skipping the root (which is the name + * of the .def file). For example, if this is an array called 'leafArray' and a child of 'innerArray' which + * is again a child of 'myStruct', then the returned string will be 'myStruct.innerArray.leafArray.' + * The trailing '.' is included for the caller's convenience. + * + * @return a string composed of the ancestors of this ConfigDefinition, not including the root. + */ + public String getAncestorString() { + StringBuilder ret = new StringBuilder(); + ConfigDefinition ancestor = this; + while (ancestor.getParent() != null) { + ret.insert(0, ancestor.getName() + "."); + ancestor = ancestor.getParent(); + } + return ret.toString(); + } + + @Override + public int compareTo(ConfigDefinition other) { + Objects.requireNonNull(other); + if (!getName().equals(other.getName())) { + throw new IllegalArgumentException("Different def names used to compare: "+getName()+"/"+other.getName()); + } + return new VersionComparator().compare(getVersion(),other.getVersion()); + } + + @Override + public String toString() { + return getNamespace() + "." + getName(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionBuilder.java b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionBuilder.java new file mode 100644 index 00000000000..6e9f2de5a0d --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionBuilder.java @@ -0,0 +1,209 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.codegen.LeafCNode; + +import java.util.Arrays; + +/** + * Builds a ConfigDefinition from a tree of CNodes. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class ConfigDefinitionBuilder { + + /** + * Creates a ConfigDefinition based on a tree generated from parsing a config + * definition file. + * + * @param root the root node in a tree generated from parsing a config definition file. + * @return a ConfigDefinition object + */ + public static ConfigDefinition createConfigDefinition(CNode root) { + return createConfigDefinition(root, root.getNamespace()); + } + + // TODO This method should be removed when we have full namespace support + /** + * Creates a ConfigDefinition based on a tree generated from parsing a config + * definition file. The <code>namespace</code> argument overrides the namespace + * defined in the config definition file. + * + * @param root the root node in a tree generated from parsing a config definition file. + * @param namespace Override namespace in root with this namespace + * @return a ConfigDefinition object + */ + public static ConfigDefinition createConfigDefinition(CNode root, String namespace) { + ConfigDefinition def = new ConfigDefinition(root.getName(), root.getVersion(), namespace); + + for (CNode node : root.getChildren()) { + addNode(def, node); + } + return def; + } + + /** + * + * @param def a ConfigDefinition object + * @param node the node to be added to the config definition + */ + private static void addNode(ConfigDefinition def, CNode node) { + String name = node.getName(); + if (node instanceof LeafCNode) { + if (node.isArray) { + //System.out.println("Adding array node " + name); + String enumValues = null; + String type = ((LeafCNode) node).getType(); + if ("enum".equals(type)) { + enumValues = convertToEnumValueCommaSeparated(((LeafCNode.EnumLeaf) node).getLegalValues()); + } + def.arrayDef(name).setTypeSpec( + new ConfigDefinition.TypeSpec(name, ((LeafCNode) node).getType(), null, enumValues, null, null)); + + } else if (node.isMap) { + //System.out.println("Adding leaf map node " + name); + def.leafMapDef(name).setTypeSpec(new ConfigDefinition.TypeSpec(name, ((LeafCNode) node).getType(), null, null, null, null)); + } else { + //System.out.println("Adding basic node " + name); + if (node instanceof LeafCNode.IntegerLeaf) { + addNode(def, (LeafCNode.IntegerLeaf) node); + } else if (node instanceof LeafCNode.LongLeaf) { + addNode(def, (LeafCNode.LongLeaf) node); + } else if (node instanceof LeafCNode.BooleanLeaf) { + addNode(def, (LeafCNode.BooleanLeaf) node); + } else if (node instanceof LeafCNode.DoubleLeaf) { + addNode(def, (LeafCNode.DoubleLeaf) node); + // Need to come before StringLeaf, since it is a subclass of StringLeaf + } else if (node instanceof LeafCNode.ReferenceLeaf) { + addNode(def, (LeafCNode.ReferenceLeaf) node); + } else if (node instanceof LeafCNode.FileLeaf) { + addNode(def, (LeafCNode.FileLeaf) node); + } else if (node instanceof LeafCNode.PathLeaf) { + addNode(def, (LeafCNode.PathLeaf) node); + }else if (node instanceof LeafCNode.StringLeaf) { + addNode(def, (LeafCNode.StringLeaf) node); + } else if (node instanceof LeafCNode.EnumLeaf) { + addNode(def, (LeafCNode.EnumLeaf) node); + } else { + System.err.println("Unknown node type for node with name " + name); + } + } + } else { + ConfigDefinition newDef; + if (node.isArray) { + if (node.getChildren() != null && node.getChildren().length > 0) { + //System.out.println("\tAdding inner array node " + name); + newDef = def.innerArrayDef(name); + for (CNode childNode : node.getChildren()) { + //System.out.println("\tChild node " + childNode.getName()); + addNode(newDef, childNode); + } + } + } else if (node.isMap) { + //System.out.println("Adding struct map node " + name); + newDef = def.structMapDef(name); + if (node.getChildren() != null && node.getChildren().length > 0) { + for (CNode childNode : node.getChildren()) { + //System.out.println("\tChild node " + childNode.getName()); + addNode(newDef, childNode); + } + } + + } else { + //System.out.println("Adding struct node " + name); + newDef = def.structDef(name); + if (node.getChildren() != null && node.getChildren().length > 0) { + for (CNode childNode : node.getChildren()) { + //System.out.println("\tChild node " + childNode.getName()); + addNode(newDef, childNode); + } + } + } + } + } + + + static void addNode(ConfigDefinition def, LeafCNode.IntegerLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addIntDef(leaf.getName(), new Integer(leaf.getDefaultValue().getValue())); + } else { + def.addIntDef(leaf.getName()); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.LongLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addLongDef(leaf.getName(), new Long(leaf.getDefaultValue().getValue())); + } else { + def.addLongDef(leaf.getName()); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.BooleanLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addBoolDef(leaf.getName(), Boolean.valueOf(leaf.getDefaultValue().getValue())); + } else { + def.addBoolDef(leaf.getName()); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.DoubleLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addDoubleDef(leaf.getName(), new Double(leaf.getDefaultValue().getValue())); + } else { + def.addDoubleDef(leaf.getName()); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.StringLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addStringDef(leaf.getName(), leaf.getDefaultValue().getValue()); + } else { + def.addStringDef(leaf.getName()); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.ReferenceLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addReferenceDef(leaf.getName(), leaf.getDefaultValue().getValue()); + } else { + def.addReferenceDef(leaf.getName(), null); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.FileLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addFileDef(leaf.getName(), leaf.getDefaultValue().getValue()); + } else { + def.addFileDef(leaf.getName(), null); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.PathLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addPathDef(leaf.getName(), leaf.getDefaultValue().getValue()); + } else { + def.addPathDef(leaf.getName(), null); + } + } + + static void addNode(ConfigDefinition def, LeafCNode.EnumLeaf leaf) { + if (leaf.getDefaultValue() != null) { + def.addEnumDef(leaf.getName(), Arrays.asList(leaf.getLegalValues()), leaf.getDefaultValue().getValue()); + } else { + def.addEnumDef(leaf.getName(), Arrays.asList(leaf.getLegalValues()), null); + } + } + + static String convertToEnumValueCommaSeparated(String[] enumValues) { + StringBuilder sb = new StringBuilder(); + for (String s : enumValues) { + sb.append(s); + sb.append(", "); + } + int length = sb.length(); + sb.delete(length - 2, length); + return sb.toString(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionKey.java b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionKey.java new file mode 100644 index 00000000000..29c39ef0ab0 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionKey.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * Represents one config definition key (name, namespace) + * + * @author vegardh + */ +public class ConfigDefinitionKey { + + private final String name; + private final String namespace; + + /** + * Creates a config definition key. + * @param name config definition name + * @param namespace config definition namespace + */ + public ConfigDefinitionKey(String name, String namespace) { + this.name = name; + this.namespace = namespace; + } + + public ConfigDefinitionKey(ConfigKey<?> key) { + this(key.getName(), key.getNamespace()); + } + + public String getName() { + return name; + } + + public String getNamespace() { + return namespace; + } + + @Override + public boolean equals(Object oth) { + if (!(oth instanceof ConfigDefinitionKey)) { + return false; + } + ConfigDefinitionKey other = (ConfigDefinitionKey) oth; + return name.equals(other.getName()) && + namespace.equals(other.getNamespace()); + } + + @Override + public int hashCode() { + return namespace.hashCode() + name.hashCode(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(namespace).append(".").append(name); + return sb.toString(); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionSet.java b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionSet.java new file mode 100644 index 00000000000..5e5f8db2711 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinitionSet.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.log.LogLevel; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Class to hold config definitions and resolving requests for the correct definition + * + * @author Harald Musum <musum@yahoo-inc.com> + * @since 5.1 + */ +public class ConfigDefinitionSet { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigDefinitionSet.class.getName()); + + private final Map<ConfigDefinitionKey, ConfigDefinition> defs = new ConcurrentHashMap<ConfigDefinitionKey, ConfigDefinition>(); + + public ConfigDefinitionSet() { + + } + + public void add(ConfigDefinitionKey key, ConfigDefinition def) { + log.log(LogLevel.DEBUG, "Adding to set: " + key); + defs.put(key, def); + } + + /** + * Returns a ConfigDefinition from the set matching the given <code>key</code>. If no ConfigDefinition + * is found in the set, it will try to find a ConfigDefinition with same name in the default namespace. + * @param key a {@link ConfigDefinitionKey} + * @return a ConfigDefinition if found, else null + */ + public ConfigDefinition get(ConfigDefinitionKey key) { + log.log(LogLevel.DEBUG, "Getting from set " + defs + " for key " + key); + ConfigDefinition ret = defs.get(key); + if (ret == null) { + // Return entry if we fallback to default namespace + log.log(LogLevel.DEBUG, "Found no def for key " + key + ", trying to find def with same name in default namespace"); + for (Map.Entry<ConfigDefinitionKey, ConfigDefinition> entry : defs.entrySet()) { + if (key.getName().equals(entry.getKey().getName()) && entry.getKey().getNamespace().equals(CNode.DEFAULT_NAMESPACE)) { + return entry.getValue(); + } + } + } + return ret; + } + + public int size() { + return defs.size(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (ConfigDefinitionKey key : defs.keySet()) { + sb.append(key.toString()).append("\n"); + } + return sb.toString(); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java new file mode 100644 index 00000000000..1ce81abd57a --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java @@ -0,0 +1,233 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.codegen.LeafCNode; +import com.yahoo.slime.*; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.*; +import java.util.Stack; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigFileFormat implements SlimeFormat, ObjectTraverser { + private final InnerCNode root; + private DataOutputStream out = null; + private Stack<Node> nodeStack; + + public ConfigFileFormat(InnerCNode root) { + this.root = root; + this.nodeStack = new Stack<>(); + } + + private void printPrefix() throws IOException { + for (Node node : nodeStack) { + CNode cnode = node.node; + if (cnode != root) { + encodeString(cnode.getName()); + if (cnode.isArray) { + encodeString("[" + node.arrayIndex + "]"); + if (!(cnode instanceof LeafCNode)) { + encodeString("."); + } + } else if (cnode.isMap) { + encodeString("{\"" + node.mapKey + "\"}"); + if (!(cnode instanceof LeafCNode)) { + encodeString("."); + } + } else if (cnode instanceof LeafCNode) { + encodeString(""); + } else { + encodeString("."); + } + } + } + encodeString(" "); + } + + private void encode(Inspector inspector, CNode node) throws IOException { + switch (inspector.type()) { + case BOOL: + encodeValue(String.valueOf(inspector.asBool()), (LeafCNode) node); + return; + case LONG: + encodeValue(String.valueOf(inspector.asLong()), (LeafCNode) node); + return; + case DOUBLE: + encodeValue(String.valueOf(inspector.asDouble()), (LeafCNode) node); + return; + case STRING: + encodeValue(inspector.asString(), (LeafCNode) node); + return; + case ARRAY: + encodeArray(inspector, node); + return; + case OBJECT: + if (node.isMap) { + encodeMap(inspector, node); + } else { + encodeObject(inspector, node); + } + return; + case NIX: + case DATA: + throw new IllegalArgumentException("Illegal config format supplied. Unknown type for field '" + node.getName() + "'"); + } + throw new RuntimeException("Should not be reached"); + } + + private void encodeMap(Inspector inspector, final CNode node) { + inspector.traverse(new ObjectTraverser() { + @Override + public void field(String name, Inspector inspector) { + try { + nodeStack.push(new Node(node, -1, name)); + if (inspector.type().equals(Type.OBJECT)) { + encodeObject(inspector, node); + } else { + encode(inspector, node); + } + nodeStack.pop(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } + + private void encodeArray(Inspector inspector, final CNode node) { + inspector.traverse(new ArrayTraverser() { + @Override + public void entry(int idx, Inspector inspector) { + try { + nodeStack.push(new Node(node, idx, "")); + encode(inspector, node); + nodeStack.pop(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + + } + + private void encodeObject(Inspector inspector, CNode node) { + if (!node.isArray && !node.isMap) { + nodeStack.push(new Node(node)); + inspector.traverse(this); + nodeStack.pop(); + } else { + inspector.traverse(this); + } + } + + private void encodeValue(String value, LeafCNode node) throws IOException { + printPrefix(); + try { + if (node instanceof LeafCNode.StringLeaf) { + encodeStringQuoted(value); + } else if (node instanceof LeafCNode.IntegerLeaf) { + //Integer.parseInt(value); + encodeString(value); + } else if (node instanceof LeafCNode.LongLeaf) { + //Long.parseLong(value); + encodeString(value); + } else if (node instanceof LeafCNode.DoubleLeaf) { + //Double.parseDouble(value); + encodeString(value); + } else if (node instanceof LeafCNode.BooleanLeaf) { + encodeString(String.valueOf(Boolean.parseBoolean(value))); + } else if (node instanceof LeafCNode.EnumLeaf) { + // LeafCNode.EnumLeaf enumNode = (LeafCNode.EnumLeaf) node; + // TODO: Reenable this when we can return illegal config id. + // checkLegalEnumValue(enumNode, value); + encodeString(value); + } else { + encodeStringQuoted(value); + } + encodeString("\n"); + } catch (Exception e) { + throw new IllegalArgumentException("Unable to serialize field '" + node.getFullName() + "': ", e); + } + } + + private void checkLegalEnumValue(LeafCNode.EnumLeaf enumNode, String value) { + boolean found = false; + for (String legalVal : enumNode.getLegalValues()) { + if (legalVal.equals(value)) { + found = true; + } + } + if (!found) + throw new IllegalArgumentException("Illegal enum value '" + value + "'"); + } + + private void encodeStringQuoted(String s) throws IOException { + encodeString("\"" + escapeString(s) + "\""); + } + + private String escapeString(String s) { + return ConfigUtils.escapeConfigFormatValue(s); + } + + private void encodeString(String s) throws IOException { + out.write(Utf8.toBytes(s)); + } + + @Override + public void encode(OutputStream os, Slime slime) throws IOException { + encode(os, slime.get()); + } + + public void encode(OutputStream os, Inspector inspector) throws IOException { + this.out = new DataOutputStream(os); + this.nodeStack = new Stack<>(); + nodeStack.push(new Node(root)); + encode(inspector, root); + } + + @Override + public void decode(InputStream is, Slime slime) throws IOException { + throw new UnsupportedOperationException("decode is not supported"); + } + + @Override + public void field(String fieldName, Inspector inspector) { + try { + Node parent = nodeStack.peek(); + CNode child = parent.node.getChild(fieldName); + if (child == null) { + return; // Skip this field to optimistic + } + if (!child.isArray && !child.isMap && child instanceof LeafCNode) { + nodeStack.push(new Node(child)); + encode(inspector, child); + nodeStack.pop(); + } else { + encode(inspector, child); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private class Node { + final int arrayIndex; + final String mapKey; + final CNode node; + Node(CNode node, int arrayIndex, String mapKey) { + this.node = node; + this.arrayIndex = arrayIndex; + this.mapKey = mapKey; + } + + public Node(CNode node) { + this(node, -1, ""); + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigHelper.java b/config/src/main/java/com/yahoo/vespa/config/ConfigHelper.java new file mode 100644 index 00000000000..a1469b746dc --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigHelper.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.subscription.ConfigSourceSet; + +/** + * Helper class for config applications (currently ConfigManager and ConfigProxy). + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public class ConfigHelper { + private final JRTConnectionPool jrtConnectionPool; + private TimingValues timingValues; + + /** + * @param configSourceSet The set of config sources for this helper. + */ + public ConfigHelper(ConfigSourceSet configSourceSet) { + this(configSourceSet, new TimingValues()); + } + + /** + * @param configSourceSet The set of config sources for this helper. + * @param timingValues values for timeouts and delays, see {@link TimingValues} + */ + public ConfigHelper(ConfigSourceSet configSourceSet, TimingValues timingValues) { + jrtConnectionPool = new JRTConnectionPool(configSourceSet); + this.timingValues = timingValues; + } + + /** + * @return the config sources (remote servers and/or proxies) in this helper's connection pool. + */ + public ConfigSourceSet getConfigSourceSet() { + return jrtConnectionPool.getSourceSet(); + } + + /** + * @return the connection pool for this config helper. + */ + public JRTConnectionPool getConnectionPool() { + return jrtConnectionPool; + } + + /** + * @return the timing values for this config helper. + */ + public TimingValues getTimingValues() { + return timingValues; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java new file mode 100755 index 00000000000..1e8ba43d649 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.codegen.CNode; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; + +/** + * Class for holding the key when doing cache look-ups and other management of config instances. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable<ConfigKey<?>> { + + @NonNull + private final String name; + @NonNull + private final String configId; + @NonNull + private final String namespace; + + // The two fields below are only set when ConfigKey is constructed from a config class. Can be null + private final Class<CONFIGCLASS> configClass; + private final String md5; // config definition md5 + + /** + * Constructs new key + * + * @param name config definition name + * @param configIdString Can be null. + * @param namespace namespace for this config definition + */ + public ConfigKey(String name, String configIdString, String namespace) { + this(name, configIdString, namespace, null, null); + } + + /** + * Creates a new instance from the given class and configId + * + * @param clazz Config class + * @param configIdString config id, can be null. + */ + public ConfigKey(Class<CONFIGCLASS> clazz, String configIdString) { + this(getFieldFromClass(clazz, "CONFIG_DEF_NAME"), + configIdString, getFieldFromClass(clazz, "CONFIG_DEF_NAMESPACE"), getFieldFromClass(clazz, "CONFIG_DEF_MD5"), clazz); + } + + public ConfigKey(String name, String configIdString, String namespace, String defMd5, Class<CONFIGCLASS> clazz) { + if (name == null) + throw new ConfigurationRuntimeException("Config name must be non-null!"); + this.name = name; + this.configId = (configIdString == null) ? "" : configIdString; + this.namespace = (namespace == null) ? CNode.DEFAULT_NAMESPACE : namespace; + this.md5 = (defMd5 == null) ? "" : defMd5; + this.configClass = clazz; + } + + /** + * Comparison sort order: namespace, name, configId. + */ + @Override + public int compareTo(ConfigKey<?> o) { + if (!o.getNamespace().equals(getNamespace())) return getNamespace().compareTo(o.getNamespace()); + if (!o.getName().equals(getName())) return getName().compareTo(o.getName()); + return getConfigId().compareTo(o.getConfigId()); + } + + private static String getFieldFromClass(Class<?> clazz, String fieldName) { + try { + return (String) clazz.getField(fieldName).get(null); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new ConfigurationRuntimeException("No such field '" + fieldName + "' in class " + clazz + ", or could not access field.", e); + } + } + + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof ConfigKey)) { + return false; + } + ConfigKey<?> key = (ConfigKey) o; + return (name.equals(key.name) && + configId.equals(key.configId) && + namespace.equals(key.namespace)); + } + + public int hashCode() { + int hash = 17; + hash = 37 * hash + name.hashCode(); + hash = 37 * hash + configId.hashCode(); + hash = 37 * hash + namespace.hashCode(); + return hash; + } + + @NonNull + public String getName() { + return name; + } + + @NonNull + public String getConfigId() { + return configId; + } + + @NonNull + public String getNamespace() { + return namespace; + } + + @Nullable + public Class<CONFIGCLASS> getConfigClass() { + return configClass; + } + + @Nullable + public String getMd5() { + return md5; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("name="); + sb.append(name); + sb.append(",namespace="); + sb.append(namespace); + sb.append(",configId="); + sb.append(configId); + return sb.toString(); + } + + public static ConfigKey<?> createFull(String name, String configId, String namespace, String md5) { + return new ConfigKey<>(name, configId, namespace, md5, null); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java new file mode 100644 index 00000000000..a4ac34e65aa --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.subscription.ConfigInstanceSerializer; +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeFormat; +import com.yahoo.text.Utf8Array; +import com.yahoo.text.Utf8String; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * A class that holds a representation of a config payload. + * + * @author lulf + * @since 5.1.6 + */ +public class ConfigPayload { + private final Slime slime; + + public ConfigPayload(Slime slime) { + this.slime = slime; + } + + public static ConfigPayload fromInstance(ConfigInstance instance) { + Slime slime = new Slime(); + ConfigInstanceSerializer serializer = new ConfigInstanceSerializer(slime); + ConfigInstance.serialize(instance, serializer); + return new ConfigPayload(slime); + } + + public static ConfigPayload fromBuilder(ConfigPayloadBuilder builder) { + Slime slime = new Slime(); + builder.resolve(slime.setObject()); + return new ConfigPayload(slime); + } + + public Slime getSlime() { + return slime; + } + + public void serialize(OutputStream os, SlimeFormat format) throws IOException { + format.encode(os, slime); + } + + @Override + public String toString() { + return toString(false); + } + + public String toString(boolean compact) { + return toUtf8Array(compact).toString(); + } + + public ConfigPayload applyDefaultsFromDef(InnerCNode clientDef) { + DefaultValueApplier defaultValueApplier = new DefaultValueApplier(); + defaultValueApplier.applyDefaults(slime, clientDef); + return this; + } + + public static ConfigPayload empty() { + Slime slime = new Slime(); + slime.setObject(); + return new ConfigPayload(slime); + } + + public static ConfigPayload fromString(String jsonString) { + return fromUtf8Array(new Utf8String(jsonString)); + } + + public boolean isEmpty() { + return !slime.get().valid() || slime.get().children() == 0; + } + + public Utf8Array toUtf8Array(boolean compact) { + ByteArrayOutputStream os = new ByteArrayOutputStream(10000); + try { + new JsonFormat(compact).encode(os, slime); + os.close(); + } catch (IOException e) { + e.printStackTrace(); + } + return new Utf8Array(os.toByteArray()); + } + + public static ConfigPayload fromUtf8Array(Utf8Array payload) { + return new ConfigPayload(new JsonDecoder().decode(new Slime(), payload.getBytes())); + } + + public <ConfigType extends ConfigInstance> ConfigType toInstance(Class<ConfigType> clazz, String configId) { + return ConfigInstanceUtil.getNewInstance(clazz, configId, this); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java new file mode 100644 index 00000000000..2b17305252f --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -0,0 +1,498 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigBuilder; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.FileReference; +import com.yahoo.log.LogLevel; +import com.yahoo.yolean.Exceptions; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Type; +import com.yahoo.text.Utf8; + +import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.nio.file.Path; +import java.util.*; +import java.util.logging.Logger; + +/** + * A utility class that can be used to apply a payload to a config builder. + * + * TODO: This can be refactored a lot, since many of the reflection methods are duplicated + * + * @author lulf, musum, tonyv + * @since 5.1.6 + */ +public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { + private final static Logger log = Logger.getLogger(ConfigPayloadApplier.class.getPackage().getName()); + + private final ConfigInstance.Builder rootBuilder; + private final ConfigTransformer.PathAcquirer pathAcquirer; + private final Stack<NamedBuilder> stack = new Stack<>(); + + public ConfigPayloadApplier(T builder) { + this(builder, new IdentityPathAcquirer()); + } + + public ConfigPayloadApplier(T builder, ConfigTransformer.PathAcquirer pathAcquirer) { + this.rootBuilder = builder; + this.pathAcquirer = pathAcquirer; + debug("rootBuilder=" + rootBuilder); + } + + public void applyPayload(ConfigPayload payload) { + stack.push(new NamedBuilder(rootBuilder)); + try { + handleValue(payload.getSlime().get()); + } catch (Exception e) { + throw new RuntimeException("Not able to create config builder for payload:" + payload.toString() + + ", " + Exceptions.toMessageString(e), e); + } + } + + private void handleValue(Inspector inspector) { + switch (inspector.type()) { + case NIX: + case BOOL: + case LONG: + case DOUBLE: + case STRING: + case DATA: + handleLeafValue(inspector); + break; + case ARRAY: + handleARRAY(inspector); + break; + case OBJECT: + handleOBJECT(inspector); + break; + default: + assert false : "Should not be reached"; + } + } + + private void handleARRAY(Inspector inspector) { + trace("Array"); + inspector.traverse(new ArrayTraverser() { + @Override + public void entry(int idx, Inspector inspector) { + handleArrayEntry(idx, inspector); + } + }); + } + + private void handleArrayEntry(int idx, Inspector inspector) { + try { + trace("entry, idx=" + idx); + trace("top of stack=" + stack.peek().toString()); + String name = stack.peek().nameStack().peek(); + if (inspector.type().equals(Type.OBJECT)) { + stack.push(createBuilder(stack.peek(), name)); + } + handleValue(inspector); + if (inspector.type().equals(Type.OBJECT)) { + stack.peek().nameStack().pop(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void handleOBJECT(Inspector inspector) { + trace("Object"); + printStack(); + + inspector.traverse(new ObjectTraverser() { + @Override + public void field(String name, Inspector inspector) { + handleObjectEntry(name, inspector); + } + }); + + trace("Should pop a builder from stack"); + NamedBuilder builder = stack.pop(); + printStack(); + + // Need to set e.g struct(Struct.Builder) here + if (!stack.empty()) { + trace("builder= " + builder); + try { + invokeSetter(stack.peek().builder, builder.peekName(), builder.builder); + } catch (Exception e) { + throw new RuntimeException("Could not set '" + builder.peekName() + + "' for value '" + builder.builder() + "'", e); + } + } + } + + private void handleObjectEntry(String name, Inspector inspector) { + try { + trace("field, name=" + name); + NamedBuilder parentBuilder = stack.peek(); + if (inspector.type().equals(Type.OBJECT)) { + if (isMapField(parentBuilder, name)) { + parentBuilder.nameStack().push(name); + handleMap(inspector); + parentBuilder.nameStack().pop(); + return; + } else { + stack.push(createBuilder(parentBuilder, name)); + } + } else if (inspector.type().equals(Type.ARRAY)) { + for (int i = 0; i < inspector.children(); i++) { + trace("Pushing " + name); + parentBuilder.nameStack().push(name); + } + } else { // leaf + parentBuilder.nameStack().push(name); + } + handleValue(inspector); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void handleMap(Inspector inspector) { + inspector.traverse(new ObjectTraverser() { + @Override + public void field(String name, Inspector inspector) { + switch (inspector.type()) { + case OBJECT: + handleInnerMap(name, inspector); + break; + case ARRAY: + throw new IllegalArgumentException("Never herd of array inside maps before"); + default: + setMapLeafValue(name, getValueFromInspector(inspector)); + break; + } + } + }); + } + + private void handleInnerMap(String name, Inspector inspector) { + NamedBuilder builder = createBuilder(stack.peek(), stack.peek().peekName()); + setMapLeafValue(name, builder.builder()); + stack.push(builder); + inspector.traverse(new ObjectTraverser() { + @Override + public void field(String name, Inspector inspector) { + handleObjectEntry(name, inspector); + } + }); + stack.pop(); + } + + private void setMapLeafValue(String key, Object value) { + NamedBuilder parent = stack.peek(); + ConfigBuilder builder = parent.builder(); + String methodName = parent.peekName(); + //trace("class to obtain method from: " + builder.getClass().getName()); + try { + // Need to convert reference into actual path if 'path' type is used + if (isPathField(builder, methodName)) { + FileReference wrappedPath = resolvePath((String)value); + invokeSetter(builder, methodName, key, wrappedPath); + } else { + invokeSetter(builder, methodName, key, value); + } + } catch (InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException("Name: " + methodName + ", value '" + value + "'", e); + } catch (NoSuchMethodException e) { + log.log(LogLevel.INFO, "Skipping unknown field " + methodName + " in " + rootBuilder); + } + } + + private boolean isMapField(NamedBuilder parentBuilder, String name) { + ConfigBuilder builder = parentBuilder.builder(); + try { + Field f = builder.getClass().getField(name); + return f.getType().getName().equals("java.util.Map"); + } catch (Exception e) { + return false; + } + } + + NamedBuilder createBuilder(NamedBuilder parentBuilder, String name) { + Object builder = parentBuilder.builder(); + Object newBuilder = getBuilderForStruct(findBuilderName(name), name, builder.getClass().getDeclaringClass()); + trace("New builder for " + name + "=" + newBuilder); + trace("Pushing builder for " + name + "=" + newBuilder + " onto stack"); + return new NamedBuilder((ConfigBuilder) newBuilder, name); + } + + private void handleLeafValue(Inspector value) { + trace("String "); + printStack(); + NamedBuilder peek = stack.peek(); + trace("popping name stack"); + String name = peek.nameStack().pop(); + printStack(); + ConfigBuilder builder = peek.builder(); + trace("name=" + name + ",builder=" + builder + ",value=" + value.toString()); + setValueForLeafNode(builder, name, value); + } + + // Sets values for leaf nodes (uses private accessors that take string as argument) + private void setValueForLeafNode(Object builder, String methodName, Inspector value) { + try { + // Need to convert reference into actual path if 'path' type is used + if (isPathField(builder, methodName)) { + FileReference wrappedPath = resolvePath(Utf8.toString(value.asUtf8())); + invokeSetter(builder, methodName, wrappedPath); + } else { + Object object = getValueFromInspector(value); + invokeSetter(builder, methodName, object); + } + } catch (InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException("Name: " + methodName + ", value '" + value + "'", e); + } catch (NoSuchMethodException e) { + log.log(LogLevel.INFO, "Skipping unknown field " + methodName + " in " + builder.getClass()); + } + } + + private FileReference resolvePath(String value) { + Path path = pathAcquirer.getPath(newFileReference(value)); + return newFileReference(path.toString()); + } + + private FileReference newFileReference(String fileReference) { + try { + Constructor<FileReference> constructor = FileReference.class.getDeclaredConstructor(String.class); + constructor.setAccessible(true); + return constructor.newInstance(fileReference); + } catch (Exception e) { + throw new RuntimeException("Failed invoking FileReference constructor.", e); + } + } + + private final Map<String, Method> methodCache = new HashMap<>(); + private static String methodCacheKey(Object builder, String methodName, Object[] params) { + StringBuilder sb = new StringBuilder(); + sb.append(builder.getClass().getName()) + .append(".") + .append(methodName); + for (Object param : params) { + sb.append(".").append(param.getClass().getName()); + } + return sb.toString(); + } + + private Method lookupSetter(Object builder, String methodName, Object ... params) throws NoSuchMethodException { + Class<?>[] parameterTypes = new Class<?>[params.length]; + for (int i = 0; i < params.length; i++) { + parameterTypes[i] = params[i].getClass(); + } + Method method = builder.getClass().getDeclaredMethod(methodName, parameterTypes); + method.setAccessible(true); + trace("method=" + method + ",params=" + params); + return method; + } + + private void invokeSetter(Object builder, String methodName, Object ... params) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + // TODO: Does not work for native types. + String key = methodCacheKey(builder, methodName, params); + Method method = methodCache.get(key); + if (method == null) { + method = lookupSetter(builder, methodName, params); + methodCache.put(key, method); + } + method.invoke(builder, params); + } + + private Object getValueFromInspector(Inspector inspector) { + switch (inspector.type()) { + case STRING: + return Utf8.toString(inspector.asUtf8()); + case LONG: + return String.valueOf(inspector.asLong()); + case DOUBLE: + return String.valueOf(inspector.asDouble()); + case NIX: + return null; + case BOOL: + return String.valueOf(inspector.asBool()); + case DATA: + return String.valueOf(inspector.asData()); + } + throw new IllegalArgumentException("Unhandled type " + inspector.type()); + } + + + /** + * Checks whether or not this field is of type 'path', in which + * case some special handling might be needed. Caches the result. + */ + private Set<String> pathFieldSet = new HashSet<>(); + private boolean isPathField(Object builder, String methodName) { + String key = pathFieldKey(builder, methodName); + if (pathFieldSet.contains(key)) { + return true; + } + boolean isPath = false; + try { + Field field = builder.getClass().getDeclaredField(methodName); + //Paths are stored as FileReference in Builder. + java.lang.reflect.Type fieldType = field.getGenericType(); + if (fieldType instanceof Class<?> && fieldType == FileReference.class) { + isPath = true; + } else if (fieldType instanceof ParameterizedType) { + isPath = isParameterizedWithPath((ParameterizedType) fieldType); + } + } catch (NoSuchFieldException e) { + } + if (isPath) { + pathFieldSet.add(key); + } + return isPath; + } + + private static String pathFieldKey(Object builder, String methodName) { + return builder.getClass().getName() + "." + methodName; + } + + private boolean isParameterizedWithPath(ParameterizedType fieldType) { + int numTypeArgs = fieldType.getActualTypeArguments().length; + if (numTypeArgs > 0) + return fieldType.getActualTypeArguments()[numTypeArgs - 1] == FileReference.class; + return false; + } + + + private String findBuilderName(String name) { + StringBuilder sb = new StringBuilder(); + sb.append(name.substring(0, 1).toUpperCase()).append(name.substring(1)); + return sb.toString(); + } + + private Constructor<?> lookupBuilderForStruct(String builderName, String name, Class<?> currentClass) { + final String currentClassName = currentClass.getName(); + trace("builderName=" + builderName + ", name=" + name + ",current class=" + currentClassName); + Class<?> structClass = findClass(currentClass, currentClassName + "$" + builderName); + Class<?> structBuilderClass = findClass(structClass, currentClassName + "$" + builderName + "$Builder"); + try { + return structBuilderClass.getDeclaredConstructor(new Class<?>[]{}); + } catch (NoSuchMethodException e) { + throw new RuntimeException("Could not create class '" + "'" + structBuilderClass.getName() + "'"); + } + } + + /** + * Finds a nested class or builder class with the given <code>name</code>name in <code>clazz</code> + * @param clazz a Class + * @param name a name + * @return class found, or throws an exception is no class is found + */ + private Class<?> findClass(Class<?> clazz, String name) { + for (Class<?> cls : clazz.getDeclaredClasses()) { + if (cls.getName().equals(name)) { + trace("Found class " + cls.getName()); + return cls; + } + } + throw new RuntimeException("could not find class representing '" + printCurrentConfigName() + "'"); + } + + private final Map<String, Constructor<?>> constructorCache = new HashMap<>(); + private static String constructorCacheKey(String builderName, String name, Class<?> currentClass) { + return builderName + "." + name + "." + currentClass.getName(); + } + + private Object getBuilderForStruct(String builderName, String name, Class<?> currentClass) { + String key = constructorCacheKey(builderName, name, currentClass); + Constructor<?> ctor = constructorCache.get(key); + if (ctor == null) { + ctor = lookupBuilderForStruct(builderName, name, currentClass); + constructorCache.put(key, ctor); + } + Object builder; + try { + builder = ctor.newInstance(); + } catch (Exception e) { + throw new RuntimeException("Could not create class '" + "'" + ctor.getDeclaringClass().getName() + "'"); + } + return builder; + } + + private String printCurrentConfigName() { + StringBuilder sb = new StringBuilder(); + ArrayList<String> stackElements = new ArrayList<>(); + Stack<String> nameStack = stack.peek().nameStack(); + while (!nameStack.empty()) { + stackElements.add(nameStack.pop()); + } + Collections.reverse(stackElements); + for (String s : stackElements) { + sb.append(s); + sb.append("."); + } + sb.deleteCharAt(sb.length() - 1); // remove last . + return sb.toString(); + } + + private void debug(String message) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, message); + } + } + + private void trace(String message) { + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, message); + } + } + + private void printStack() { + trace("stack=" + stack.toString()); + } + + /** + * A class that holds a builder and a stack of names + */ + private static class NamedBuilder { + private ConfigBuilder builder; + private final Stack<String> names = new Stack<>(); // if empty, the builder is the root builder + + NamedBuilder(ConfigBuilder builder) { + this.builder = builder; + } + + NamedBuilder(ConfigBuilder builder, String name) { + this(builder); + names.push(name); + } + + ConfigBuilder builder() { + return builder; + } + + String peekName() { + return names.peek(); + } + + Stack<String> nameStack() { + return names; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(builder() == null ? "null" : builder.toString()).append(" names=").append(names); + return sb.toString(); + } + } + + static class IdentityPathAcquirer implements ConfigTransformer.PathAcquirer { + @Override + public Path getPath(FileReference fileReference) { + return new File(fileReference.value()).toPath(); + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java new file mode 100644 index 00000000000..409bf307a34 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java @@ -0,0 +1,527 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.slime.*; + +import java.util.*; + +/** + * Helper class for building Slime config payloads, while supporting referring to payloads with their indices. The + * builder does not care about config field types. This is resolved by the actual config type consumer created + * from the Slime tree. + * + * TODO: Add toString + * @author lulf + * @since 5.1 + */ +public class ConfigPayloadBuilder { + private String value; + private final Map<String, ConfigPayloadBuilder> objectMap; + private final Map<String, Array> arrayMap; + private final Map<String, MapBuilder> mapBuilderMap; + private final ConfigDefinition configDefinition; + private List<String> warnings = new ArrayList<>(); + + /** + * Construct a payload builder that is not a leaf. + */ + public ConfigPayloadBuilder() { + this(null, null, null); + } + + public ConfigPayloadBuilder(ConfigDefinition configDefinition, List<String> warnings) { + this(configDefinition, null, warnings); + } + + /** + * Construct a payload builder with a leaf value + * + * @param value The value of this leaf. + */ + private ConfigPayloadBuilder(String value, List<String> warnings) { + this(null, value, warnings); + } + + private ConfigPayloadBuilder(ConfigDefinition configDefinition, String value, List<String> warnings) { + this.objectMap = new LinkedHashMap<>(); + this.arrayMap = new LinkedHashMap<>(); + this.mapBuilderMap = new LinkedHashMap<>(); + this.value = value; + this.configDefinition = configDefinition; + this.warnings=warnings; + } + + /** + * Set the value of a config field. + * + * @param name Name of the config field. + * @param value Value of the config field. + */ + public void setField(String name, String value) { + validateField(name, value, warnings); + objectMap.put(name, new ConfigPayloadBuilder(value, warnings)); + } + + private void validateField(String name, String value, List<String> warnings) { + if (configDefinition != null) { + configDefinition.verify(name, value, warnings); + } + } + + /** + * Get a new payload builder for a config struct, which can be used to add inner values to that struct. + * + * @param name Name of the struct to create. + * @return A payload builder corresponding to the name. + */ + public ConfigPayloadBuilder getObject(String name) { + ConfigPayloadBuilder p = objectMap.get(name); + if (p == null) { + validateObject(name, warnings); + p = new ConfigPayloadBuilder(getStructDef(name), warnings); + objectMap.put(name, p); + } + return p; + } + + private ConfigDefinition getStructDef(String name) { + return (configDefinition == null ? null : configDefinition.getStructDefs().get(name)); + } + + private void validateObject(String name, List<String> warnings) { + if (configDefinition != null) { + configDefinition.verify(name, warnings); + } + } + + /** + * Create a new array where new values may be added. + * + * @param name Name of array. + * @return Array object supporting adding elements to it. + */ + public Array getArray(String name) { + Array a = arrayMap.get(name); + if (a == null) { + validateArray(name, warnings); + a = new Array(configDefinition, name); + arrayMap.put(name, a); + } + return a; + } + + private void validateArray(String name, List<String> warnings) { + if (configDefinition != null) { + configDefinition.verify(name, warnings); + } + } + + /** + * Create slime tree from this builder. + * + * @param parent the parent Cursor for this builder + */ + public void resolve(Cursor parent) { + // TODO: Fix so that names do not clash + for (Map.Entry<String, ConfigPayloadBuilder> entry : objectMap.entrySet()) { + String name = entry.getKey(); + ConfigPayloadBuilder value = entry.getValue(); + if (value.getValue() == null) { + Cursor childCursor = parent.setObject(name); + value.resolve(childCursor); + } else { + // TODO: Support giving correct type + parent.setString(name, value.getValue()); + } + } + for (Map.Entry<String, ConfigPayloadBuilder.Array> entry : arrayMap.entrySet()) { + Cursor array = parent.setArray(entry.getKey()); + entry.getValue().resolve(array); + } + for (Map.Entry<String, MapBuilder> entry : mapBuilderMap.entrySet()) { + String name = entry.getKey(); + MapBuilder map = entry.getValue(); + Cursor cursormap = parent.setObject(name); + map.resolve(cursormap); + } + } + + public ConfigPayloadBuilder override(ConfigPayloadBuilder other) { + value = other.value; + for (Map.Entry<String, ConfigPayloadBuilder> entry : other.objectMap.entrySet()) { + String key = entry.getKey(); + ConfigPayloadBuilder value = entry.getValue(); + if (objectMap.containsKey(key)) { + objectMap.put(key, objectMap.get(key).override(value)); + } else { + objectMap.put(key, new ConfigPayloadBuilder(value)); + } + } + for (Map.Entry<String, Array> entry : other.arrayMap.entrySet()) { + String key = entry.getKey(); + Array value = entry.getValue(); + if (arrayMap.containsKey(key)) { + arrayMap.put(key, arrayMap.get(key).override(value)); + } else { + arrayMap.put(key, new Array(value)); + } + } + mapBuilderMap.putAll(other.mapBuilderMap); + return this; + } + + + /** + * Get the value of this field, if any. + * + * @return value of field, null if this is not a leaf. + */ + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + /** + * Create a new map where new values may be added. + * + * @param name Name of map. + * @return Map builder supporting adding elements to it. + */ + public MapBuilder getMap(String name) { + MapBuilder a = mapBuilderMap.get(name); + if (a == null) { + validateMap(name, warnings); + a = new MapBuilder(configDefinition, name); + mapBuilderMap.put(name, a); + } + return a; + } + + /** + * The definition warnings issued for this payload + * @return list of warnings + */ + public List<String> warnings() { + return warnings; + } + + private void validateMap(String name, List<String> warnings) { + if (configDefinition != null) { + configDefinition.verify(name, warnings); + } + } + + public ConfigDefinition getConfigDefinition() { + return configDefinition; + } + + public class MapBuilder { + private final Map<String, ConfigPayloadBuilder> elements = new LinkedHashMap<>(); + private final ConfigDefinition configDefinition; + private final String name; + public MapBuilder(ConfigDefinition configDefinition, String name) { + this.configDefinition = configDefinition; + this.name = name; + } + + public void put(String key, String value) { + elements.put(key, new ConfigPayloadBuilder(getLeafMapDef(name), value, warnings)); + } + + public ConfigPayloadBuilder put(String key) { + ConfigPayloadBuilder p = new ConfigPayloadBuilder(getStructMapDef(name), warnings); + elements.put(key, p); + return p; + } + + public ConfigPayloadBuilder get(String key) { + ConfigPayloadBuilder builder = elements.get(key); + if (builder == null) { + builder = put(key); + } + return builder; + } + + public void resolve(Cursor parent) { + for (Map.Entry<String, ConfigPayloadBuilder> entry : elements.entrySet()) { + ConfigPayloadBuilder child = entry.getValue(); + String childVal = child.getValue(); + if (childVal != null) { + parent.setString(entry.getKey(), childVal); + } else { + Cursor childCursor = parent.setObject(entry.getKey()); + child.resolve(childCursor); + } + } + } + + private ConfigDefinition.LeafMapDef getLeafMapDef(String name) { + return (configDefinition == null ? null : configDefinition.getLeafMapDefs().get(name)); + } + + private ConfigDefinition getStructMapDef(String name) { + return (configDefinition == null ? null : configDefinition.getStructMapDefs().get(name)); + } + + public Collection<ConfigPayloadBuilder> getElements() { + return elements.values(); + } + } + + /** + * Array modes. + */ + private enum ArrayMode { + INDEX, APPEND + } + + /** + * Representation of a config array, which supports both INDEX and APPEND modes. + */ + public class Array { + private final Map<Integer, ConfigPayloadBuilder> elements = new LinkedHashMap<>(); + private ArrayMode mode = ArrayMode.INDEX; + private final String name; + private final ConfigDefinition configDefinition; + + public Array(ConfigDefinition configDefinition, String name) { + this.configDefinition = configDefinition; + this.name = name; + } + + public Array(Array other) { + this.elements.putAll(other.elements); + this.mode = other.mode; + this.name = other.name; + this.configDefinition = other.configDefinition; + } + + /** + * Append a value to this array. + * + * @param value Value to append. + */ + public void append(String value) { + setAppend(); + validateArrayElement(getArrayDef(name), value, elements.size()); + ConfigPayloadBuilder p = new ConfigPayloadBuilder(getArrayDef(name), value, warnings); + elements.put(elements.size(), p); + } + + private void validateArrayElement(ConfigDefinition.ArrayDef arrayDef, String value, int index) { + if (arrayDef != null) { + arrayDef.verify(value, index, warnings); + } + } + + private ConfigDefinition.ArrayDef getArrayDef(String name) { + return (configDefinition == null ? null : configDefinition.getArrayDefs().get(name)); + } + + private ConfigDefinition getInnerArrayDef(String name) { + return (configDefinition == null ? null : configDefinition.getInnerArrayDefs().get(name)); + } + + public Collection<ConfigPayloadBuilder> getElements() { + return elements.values(); + } + + /** + * Create a new slime object and returns its payload builder. Append the element after all other elements + * in the array. + * + * @return a payload builder for the new slime object. + */ + public ConfigPayloadBuilder append() { + setAppend(); + ConfigPayloadBuilder p = new ConfigPayloadBuilder(getInnerArrayDef(name), warnings); + elements.put(elements.size(), p); + return p; + } + + /** + * Set the value of array element index to value + * + * @param index Index of array element to set. + * @param value Value that the element should point to. + */ + public void set(int index, String value) { + verifyIndex(); + ConfigPayloadBuilder p = new ConfigPayloadBuilder(value, warnings); + elements.put(index, p); + } + + /** + * Set Create a payload object for the given index and return it. Any previously stored version will be + * overwritten. + * + * @param index Index of new element. + * @return The payload builder for the newly created slime object. + */ + public ConfigPayloadBuilder set(int index) { + verifyIndex(); + ConfigPayloadBuilder p = new ConfigPayloadBuilder(getInnerArrayDef(name), warnings); + elements.put(index, p); + return p; + } + + /** + * Get payload builder in this array corresponding to index. If it does not exist, create a new one. + * + * @param index of element to get. + * @return The corresponding ConfigPayloadBuilder. + */ + public ConfigPayloadBuilder get(int index) { + ConfigPayloadBuilder builder = elements.get(index); + if (builder == null) { + if (mode == ArrayMode.APPEND) + builder = append(); + else + builder = set(index); + } + return builder; + } + + /** + * Try to set append mode, but do some checking if indexed mode has been used first. + */ + private void setAppend() { + if (mode == ArrayMode.INDEX && elements.size() > 0) { + throw new IllegalStateException("Cannot append elements to an array in index mode with more than one element"); + } + mode = ArrayMode.APPEND; + } + + /** + * Try and verify that index mode is possible. + */ + private void verifyIndex() { + if (mode == ArrayMode.APPEND) + throw new IllegalStateException("Cannot reference array elements with index once append is done"); + } + + public void resolve(Cursor parent) { + for (Map.Entry<Integer, ConfigPayloadBuilder> entry : elements.entrySet()) { + ConfigPayloadBuilder child = entry.getValue(); + String childVal = child.getValue(); + if (childVal != null) { + parent.addString(childVal); + } else { + Cursor childCursor = parent.addObject(); + child.resolve(childCursor); + } + } + } + + public Array override(Array superior) { + if (mode == ArrayMode.INDEX && superior.mode == ArrayMode.INDEX) { + elements.putAll(superior.elements); + } else { + for (ConfigPayloadBuilder builder : superior.elements.values()) { + append().override(builder); + } + } + return this; + } + } + + private ConfigPayloadBuilder(ConfigPayloadBuilder other) { + this.arrayMap = other.arrayMap; + this.mapBuilderMap = other.mapBuilderMap; + this.value = other.value; + this.objectMap = other.objectMap; + this.configDefinition = other.configDefinition; + this.warnings = other.warnings; + } + + public ConfigPayloadBuilder(ConfigPayload payload) { + this(new BuilderDecoder(payload.getSlime()).decode(payload.getSlime().get())); + } + + private static class BuilderDecoder { + + private final Slime slime; + public BuilderDecoder(Slime slime) { + this.slime = slime; + } + + ConfigPayloadBuilder decode(Inspector element) { + ConfigPayloadBuilder root = new ConfigPayloadBuilder(); + decodeObject(slime, root, element); + return root; + } + + private static void decodeObject(Slime slime, ConfigPayloadBuilder builder, Inspector element) { + BuilderObjectTraverser traverser = new BuilderObjectTraverser(slime, builder); + element.traverse(traverser); + } + + private static void decode(Slime slime, String name, Inspector inspector, ConfigPayloadBuilder builder) { + switch (inspector.type()) { + case STRING: + builder.setField(name, inspector.asString()); + break; + case LONG: + builder.setField(name, String.valueOf(inspector.asLong())); + break; + case DOUBLE: + builder.setField(name, String.valueOf(inspector.asDouble())); + break; + case BOOL: + builder.setField(name, String.valueOf(inspector.asBool())); + break; + case OBJECT: + ConfigPayloadBuilder objectBuilder = builder.getObject(name); + decodeObject(slime, objectBuilder, inspector); + break; + case ARRAY: + ConfigPayloadBuilder.Array array = builder.getArray(name); + decodeArray(slime, array, inspector); + break; + } + } + + private static void decodeArray(Slime slime, Array array, Inspector inspector) { + BuilderArrayTraverser traverser = new BuilderArrayTraverser(slime, array); + inspector.traverse(traverser); + } + + private static class BuilderObjectTraverser implements ObjectTraverser { + private final ConfigPayloadBuilder builder; + private final Slime slime; + public BuilderObjectTraverser(Slime slime, ConfigPayloadBuilder builder) { + this.slime = slime; + this.builder = builder; + } + + @Override + public void field(String name, Inspector inspector) { + decode(slime, name, inspector, builder); + } + } + + private static class BuilderArrayTraverser implements ArrayTraverser { + private final Array array; + private final Slime slime; + public BuilderArrayTraverser(Slime slime, Array array) { + this.array = array; + this.slime = slime; + } + + @Override + public void entry(int idx, Inspector inspector) { + switch (inspector.type()) { + case STRING: + array.append(inspector.asString()); + break; + case OBJECT: + decodeObject(slime, array.append(), inspector); + break; + } + } + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java new file mode 100644 index 00000000000..3d15804e14a --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.FileReference; + +import java.nio.file.Path; + +import static com.yahoo.vespa.config.ConfigPayloadApplier.IdentityPathAcquirer; + +/** + * A utility class that can be used to transform config from one format to another. + * + * @author lulf, musum, tonyv + * @since 5.1.6 + */ +public class ConfigTransformer<T extends ConfigInstance> { + /** + * Workaround since FileAcquirer is in a separate module that depends on config. + * Consider moving FileAcquirer into config instead. + */ + public interface PathAcquirer { + Path getPath(FileReference fileReference); + } + + private final Class<T> clazz; + + private static volatile PathAcquirer pathAcquirer = new IdentityPathAcquirer(); + + /** + * For internal use only * + */ + public static void setPathAcquirer(PathAcquirer pathAcquirer) { + ConfigTransformer.pathAcquirer = (pathAcquirer == null) ? + new IdentityPathAcquirer() : + pathAcquirer; + } + + /** + * Create a transformer capable of converting payloads to clazz + * + * @param clazz a Class for the config instance which this config payload should create a builder for + */ + public ConfigTransformer(Class<T> clazz) { + this.clazz = clazz; + } + + /** + * Create a ConfigBuilder from a payload, based on the <code>clazz</code> supplied. + * + * @param payload a Payload to be transformed to builder. + * @return a ConfigBuilder + */ + public ConfigInstance.Builder toConfigBuilder(ConfigPayload payload) { + ConfigInstance.Builder builder = getRootBuilder(); + ConfigPayloadApplier<?> creator = new ConfigPayloadApplier<>(builder, pathAcquirer); + creator.applyPayload(payload); + return builder; + } + + private ConfigInstance.Builder getRootBuilder() { + ConfigInstance.Builder builder = null; + Class<?>[] classes = clazz.getDeclaredClasses(); + for (Class<?> c : classes) { + if (c.getName().endsWith("Builder")) { + try { + builder = (ConfigInstance.Builder) c.getConstructor().newInstance(); + } catch (Exception e) { + throw new RuntimeException("Could not instantiate builder for " + clazz.getName(), e); + } + } + } + if (builder == null) { + throw new RuntimeException("Could not find builder for " + clazz.getName()); + } else { + return builder; + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java new file mode 100644 index 00000000000..eeda2eefdfa --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.util.*; + +/** + * Tool to verify that configs across multiple config servers are the same. + * + * @author lulf + * @since 5.12 + */ +public class ConfigVerification { + private final static int port = 19071; + private final static String prefix = "http://"; + + public static void main(String [] args) throws IOException { + List<String> configservers = new ArrayList<>(); + String tenant = "default"; + String appName = "default"; + String environment = "prod"; + String region = "default"; + String instance= "default"; + for (String arg : args) { + configservers.add(prefix + arg + ":" + port + "/config/v2/tenant/" + tenant + "/application/" + appName + "/environment/" + environment + "/region/" + region + "/instance/" + instance + "/?recursive=true"); + } + System.exit(compareConfigs(listConfigs(configservers))); + } + + private static Map<String, Stack<String>> listConfigs(List<String> urls) throws IOException { + Map<String, String> outputs = performRequests(urls); + + Map<String, Stack<String>> recurseMappings = new LinkedHashMap<>(); + for (Map.Entry<String, String> entry : outputs.entrySet()) { + Slime slime = new JsonDecoder().decode(new Slime(), Utf8.toBytes(entry.getValue())); + final List<String> list = new ArrayList<>(); + slime.get().field("configs").traverse(new ArrayTraverser() { + @Override + public void entry(int idx, Inspector inspector) { + list.add(inspector.asString()); + } + }); + Stack<String> stack = new Stack<>(); + Collections.sort(list); + stack.addAll(list); + recurseMappings.put(entry.getKey(), stack); + } + return recurseMappings; + } + + private static Map<String, String> performRequests(List<String> urls) throws IOException { + Map<String, String> outputs = new LinkedHashMap<>(); + for (String url : urls) { + outputs.put(url, performRequest(url)); + } + return outputs; + } + + private static int compareConfigs(Map<String, Stack<String>> mappings) throws IOException { + for (int n = 0; n < mappings.values().iterator().next().size(); n++) { + List<String> recurseUrls = new ArrayList<>(); + for (Map.Entry<String, Stack<String>> entry : mappings.entrySet()) { + recurseUrls.add(entry.getValue().pop()); + } + int ret = compareOutputs(performRequests(recurseUrls)); + if (ret != 0) { + return ret; + } + } + return 0; + } + + private static int compareOutputs(Map<String, String> outputs) { + Map.Entry<String, String> firstEntry = outputs.entrySet().iterator().next(); + for (Map.Entry<String, String> entry : outputs.entrySet()) { + if (!entry.getValue().equals(firstEntry.getValue())) { + System.out.println("output from '" + entry.getKey() + "' did not equal output from '" + firstEntry.getKey() + "'"); + return -1; + } + } + return 0; + } + + private static String performRequest(String url) throws IOException { + URLConnection connection = new URL(url).openConnection(); + InputStream response = connection.getInputStream(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int ch; + while ((ch = response.read()) > -1) { + baos.write(ch); + } + return Utf8.toString(baos.toByteArray()); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/Connection.java b/config/src/main/java/com/yahoo/vespa/config/Connection.java new file mode 100644 index 00000000000..5ba9f2b598b --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/Connection.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.jrt.Request; +import com.yahoo.jrt.RequestWaiter; + +/** + * @author musum + */ +public interface Connection { + + void invokeAsync(Request request, double jrtTimeout, RequestWaiter requestWaiter); + + void setError(int errorCode); + + void setSuccess(); + + String getAddress(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/ConnectionPool.java new file mode 100644 index 00000000000..db21acc3d6e --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ConnectionPool.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * @author musum + */ +public interface ConnectionPool { + + void close(); + + void setError(Connection connection, int i); + + Connection getCurrent(); + + Connection setNewCurrentConnection(); + + int getSize(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java b/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java new file mode 100644 index 00000000000..187a0f0199b --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.codegen.LeafCNode; +import com.yahoo.slime.*; + +/** + * Applies default values of a given config definition to a slime payload. + * TODO: Support giving correct type of default values + * + * @author lulf + * @since 5.1 + */ +public class DefaultValueApplier { + + public Slime applyDefaults(Slime slime, InnerCNode def) { + applyDefaultsRecursive(slime.get(), def); + return slime; + } + + private void applyDefaultsRecursive(Cursor cursor, InnerCNode def) { + if (def.isArray) { + applyDefaultsToArray(cursor, def); + } else if (def.isMap) { + applyDefaultsToMap(cursor, def); + } else { + applyDefaultsToObject(cursor, def); + } + } + + private void applyDefaultsToMap(final Cursor cursor, final InnerCNode def) { + cursor.traverse(new ObjectTraverser() { + @Override + public void field(String name, Inspector inspector) { + applyDefaultsToObject(cursor.field(name), def); + } + }); + } + + private void applyDefaultsToArray(final Cursor cursor, final InnerCNode def) { + cursor.traverse(new ArrayTraverser() { + @Override + public void entry(int idx, Inspector inspector) { + applyDefaultsToObject(cursor.entry(idx), def); + } + }); + } + + private void applyDefaultsToObject(Cursor cursor, InnerCNode def) { + for (CNode child : def.getChildren()) { + Cursor childCursor = cursor.field(child.getName()); + if (isLeafNode(child) && canApplyDefault(childCursor, child)) { + applyDefaultToLeaf(cursor, child); + } else if (isInnerNode(child)) { + if (!childCursor.valid()) { + if (child.isArray) { + childCursor = cursor.setArray(child.getName()); + } else { + childCursor = cursor.setObject(child.getName()); + } + } + applyDefaultsRecursive(childCursor, (InnerCNode) child); + } + } + } + + private boolean isInnerNode(CNode child) { + return child instanceof InnerCNode; + } + + private boolean isLeafNode(CNode child) { + return child instanceof LeafCNode; + } + + private void applyDefaultToLeaf(Cursor cursor, CNode child) { + cursor.setString(child.getName(), ((LeafCNode) child).getDefaultValue().getValue()); + } + + private boolean canApplyDefault(Cursor cursor, CNode child) { + return !cursor.valid() && ((LeafCNode) child).getDefaultValue() != null; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ErrorCode.java b/config/src/main/java/com/yahoo/vespa/config/ErrorCode.java new file mode 100644 index 00000000000..50fbe2170a2 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ErrorCode.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public final class ErrorCode { + // Cannot find a config with this name, version and config md5sum + public static final int UNKNOWN_CONFIG = 100000; + // No config def with that name or version number + public static final int UNKNOWN_DEFINITION = UNKNOWN_CONFIG + 1; + public static final int UNKNOWN_DEF_MD5 = UNKNOWN_CONFIG + 4; + public static final int UNKNOWN_VESPA_VERSION = UNKNOWN_CONFIG + 5; + + public static final int ILLEGAL_NAME = UNKNOWN_CONFIG + 100; + // Version is not a number + public static final int ILLEGAL_VERSION = UNKNOWN_CONFIG + 101; + public static final int ILLEGAL_CONFIGID = UNKNOWN_CONFIG + 102; + public static final int ILLEGAL_DEF_MD5 = UNKNOWN_CONFIG + 103; + public static final int ILLEGAL_CONFIG_MD5 = UNKNOWN_CONFIG + 104; + // I don't think this will actually happen ... + public static final int ILLEGAL_TIMEOUT = UNKNOWN_CONFIG + 105; + public static final int ILLEGAL_GENERATION = UNKNOWN_CONFIG + 106; + public static final int ILLEGAL_SUB_FLAG = UNKNOWN_CONFIG + 107; + public static final int ILLEGAL_NAME_SPACE = UNKNOWN_CONFIG + 108; + public static final int ILLEGAL_PROTOCOL_VERSION = UNKNOWN_CONFIG + 109; + public static final int ILLEGAL_CLIENT_HOSTNAME = UNKNOWN_CONFIG + 110; + + // hasUpdatedConfig() is true, but generation says the config is older than previous config. + public static final int OUTDATED_CONFIG = UNKNOWN_CONFIG + 150; + + public static final int INTERNAL_ERROR = UNKNOWN_CONFIG + 200; + + public static final int APPLICATION_NOT_LOADED = UNKNOWN_CONFIG + 300; + + public static final int INCONSISTENT_CONFIG_MD5 = UNKNOWN_CONFIG + 400; + + private ErrorCode() { + } + + public static String getName(int error) { + switch(error) { + case UNKNOWN_CONFIG: return "UNKNOWN_CONFIG"; + case UNKNOWN_DEFINITION: return "UNKNOWN_DEFINITION"; + case UNKNOWN_DEF_MD5: return "UNKNOWN_DEF_MD5"; + case ILLEGAL_NAME: return "ILLEGAL_NAME"; + case ILLEGAL_VERSION: return "ILLEGAL_VERSION"; + case ILLEGAL_CONFIGID: return "ILLEGAL_CONFIGID"; + case ILLEGAL_DEF_MD5: return "ILLEGAL_DEF_MD5"; + case ILLEGAL_CONFIG_MD5: return "ILLEGAL_CONFIG_MD5"; + case ILLEGAL_TIMEOUT: return "ILLEGAL_TIMEOUT"; + case ILLEGAL_GENERATION: return "ILLEGAL_GENERATION"; + case ILLEGAL_SUB_FLAG: return "ILLEGAL_SUBSCRIBE_FLAG"; + case ILLEGAL_NAME_SPACE: return "ILLEGAL_NAME_SPACE"; + case ILLEGAL_CLIENT_HOSTNAME: return "ILLEGAL_CLIENT_HOSTNAME"; + case OUTDATED_CONFIG: return "OUTDATED_CONFIG"; + case INTERNAL_ERROR: return "INTERNAL_ERROR"; + case APPLICATION_NOT_LOADED: return "APPLICATION_NOT_LOADED"; + case ILLEGAL_PROTOCOL_VERSION: return "ILLEGAL_PROTOCOL_VERSION"; + case INCONSISTENT_CONFIG_MD5: return "INCONSISTENT_CONFIG_MD5"; + case UNKNOWN_VESPA_VERSION: return "UNKNOWN_VESPA_VERSION"; + default: return "Unknown error"; + } + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/ErrorType.java b/config/src/main/java/com/yahoo/vespa/config/ErrorType.java new file mode 100644 index 00000000000..1371f0e93cc --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/ErrorType.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * @author musum + */ +public enum ErrorType { + TRANSIENT, FATAL; + + public static ErrorType getErrorType(int errorCode) { + switch (errorCode) { + case com.yahoo.jrt.ErrorCode.CONNECTION: + case com.yahoo.jrt.ErrorCode.TIMEOUT: + return ErrorType.TRANSIENT; + case ErrorCode.UNKNOWN_CONFIG: + case ErrorCode.UNKNOWN_DEFINITION: + case ErrorCode.UNKNOWN_DEF_MD5: + case ErrorCode.ILLEGAL_NAME: + case ErrorCode.ILLEGAL_VERSION: + case ErrorCode.ILLEGAL_CONFIGID: + case ErrorCode.ILLEGAL_DEF_MD5: + case ErrorCode.ILLEGAL_CONFIG_MD5: + case ErrorCode.ILLEGAL_TIMEOUT: + case ErrorCode.OUTDATED_CONFIG: + case ErrorCode.INTERNAL_ERROR: + case ErrorCode.APPLICATION_NOT_LOADED: + case ErrorCode.UNKNOWN_VESPA_VERSION: + case ErrorCode.ILLEGAL_PROTOCOL_VERSION: + case ErrorCode.INCONSISTENT_CONFIG_MD5: + return ErrorType.FATAL; + default: + return ErrorType.FATAL; + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java b/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java new file mode 100644 index 00000000000..904c1d29818 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * Interface for counters. + * + * @author lulf + * @since 5.9 + */ +public interface GenerationCounter { + /** + * Increment counter and return new value. + * + * @return incremented counter value. + */ + public long increment(); + + /** + * @return current counter value. + */ + public long get(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java new file mode 100644 index 00000000000..549a51383b1 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigBuilder; +import com.yahoo.config.ConfigInstance; + +/** + * + /** + * A generic config with an internal generic builder that mimics a real config builder in order to support builders + * when we don't have the schema. + * + * @author lulf + * @since 5.1 + */ +public class GenericConfig { + public static class GenericConfigBuilder implements ConfigInstance.Builder { + private final ConfigPayloadBuilder payloadBuilder; + private final ConfigDefinitionKey defKey; + public GenericConfigBuilder(ConfigDefinitionKey defKey, ConfigPayloadBuilder payloadBuilder) { + this.defKey = defKey; + this.payloadBuilder = payloadBuilder; + } + private ConfigBuilder override(GenericConfigBuilder superior) { + ConfigPayloadBuilder superiorPayload = superior.payloadBuilder; + payloadBuilder.override(superiorPayload); + return this; + } + + public ConfigPayload getPayload() { return ConfigPayload.fromBuilder(payloadBuilder); } + + @Override + public boolean dispatchGetConfig(ConfigInstance.Producer producer) { + return false; + } + + @Override + public String getDefName() { + return defKey.getName(); + } + + @Override + public String getDefNamespace() { + return defKey.getNamespace(); + } + + @Override + public String getDefMd5() { + return ""; + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java new file mode 100644 index 00000000000..2a12a659538 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/GetConfigRequest.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.vespa.config.protocol.DefContent; + +import java.util.Optional; + +/** + * Interface for getConfig requests. + * @author lulf + * @since 5.3 + */ + +public interface GetConfigRequest { + + /** + * Returns the ConfigKey for this request. + * + * @return the ConfigKey for this config request + */ + public ConfigKey<?> getConfigKey(); + + /** + * The def file contents in the request, or empty array if not sent/not supported + * @return the contents (payload) of the def schema + */ + public DefContent getDefContent(); + + /** + * Get Vespa version for this GetConfigRequest + */ + public Optional<com.yahoo.vespa.config.protocol.VespaVersion> getVespaVersion(); + + /** + * Whether or not the config can be retrieved from or stored in a cache. + * @return true if content should _not_ be cached, false if it should. + */ + public boolean noCache(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java new file mode 100644 index 00000000000..2ddb810981d --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.jrt.*; + +import java.text.SimpleDateFormat; +import java.util.TimeZone; +import java.util.logging.Logger; + +/** + * A JRT connection to a config server or config proxy. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + */ +public class JRTConnection implements Connection { + + private final String address; + private final Supervisor supervisor; + private Target target; + + private long lastConnectionAttempt = 0; // Timestamp for last connection attempt + private long lastSuccess = 0; + private long lastFailure = 0; + + private static final long delayBetweenConnectionMessage = 30000; //ms + + private static SimpleDateFormat yyyyMMddz; + static { + yyyyMMddz = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); + yyyyMMddz.setTimeZone(TimeZone.getTimeZone("GMT")); + } + + @Override + public void invokeAsync(Request request, double jrtTimeout, RequestWaiter requestWaiter) { + getTarget().invokeAsync(request, jrtTimeout, requestWaiter); + } + + public final static Logger logger = Logger.getLogger(JRTConnection.class.getPackage().getName()); + + + public JRTConnection(String address, Supervisor supervisor) { + this.address = address; + this.supervisor = supervisor; + } + + public String getAddress() { + return address; + } + + /** + * This is synchronized to avoid multiple ConfigInstances creating new targets simultaneously, if + * the existing target is null, invalid or has not yet been initialized. + * + * @return The existing target, or a new one if invalid or null. + */ + public synchronized Target getTarget() { + if (target == null || !target.isValid()) { + if ((System.currentTimeMillis() - lastConnectionAttempt) > delayBetweenConnectionMessage) { + logger.fine("Connecting to " + address); + } + lastConnectionAttempt = System.currentTimeMillis(); + target = supervisor.connect(new Spec(address)); + } + return target; + } + + @Override + public synchronized void setError(int errorCode) { + lastFailure = System.currentTimeMillis(); + } + + @Override + public synchronized void setSuccess() { + lastSuccess = System.currentTimeMillis(); + } + + public void setLastSuccess() { + lastSuccess = System.currentTimeMillis(); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Address: "); + sb.append(address); + if (lastSuccess > 0) { + sb.append("\n"); + sb.append("Last success: "); + sb.append(yyyyMMddz.format(lastSuccess)); + } + if (lastFailure > 0) { + sb.append("\n"); + sb.append("Last failure: "); + sb.append(yyyyMMddz.format(lastFailure)); + } + return sb.toString(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java new file mode 100644 index 00000000000..5e52dfc5e2d --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.log.LogLevel; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; +import java.util.logging.Logger; + +/** + * A pool of JRT connections to a config source (either a config server or a config proxy). + * The current connection is chosen randomly when calling {#link #setNewCurrentConnection} + * (since the connection is chosen randomly, it might end up using the same connection again, + * and it will always do so if there is only one source). + * The current connection is available with {@link #getCurrent()}. + * When calling {@link #setError(Connection, int)}, {#link #setNewCurrentConnection} will always be called. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + * @author musum + */ +public class JRTConnectionPool implements ConnectionPool { + private static final Logger log = Logger.getLogger(JRTConnectionPool.class.getName()); + + private final Supervisor supervisor = new Supervisor(new Transport()); + private final Map<String, JRTConnection> connections = new LinkedHashMap<>(); + + // The config sources used by this connection pool. + private ConfigSourceSet sourceSet = null; + + // The current connection used by this connection pool. + private volatile JRTConnection currentConnection; + + public JRTConnectionPool(ConfigSourceSet sourceSet) { + addSources(sourceSet); + } + + public JRTConnectionPool(List<String> addresses) { + this(new ConfigSourceSet(addresses)); + } + + public void addSources(ConfigSourceSet sourceSet) { + this.sourceSet = sourceSet; + synchronized (connections) { + for (String address : sourceSet.getSources()) { + connections.put(address, new JRTConnection(address, supervisor)); + } + } + setNewCurrentConnection(); + } + + /** + * Returns the current JRTConnection instance + * + * @return a JRTConnection + */ + public synchronized JRTConnection getCurrent() { + return currentConnection; + } + + /** + * Returns and set the current JRTConnection instance by randomly choosing + * from the available sources (this means that you might end up using + * the same connection). + * + * @return a JRTConnection + */ + public synchronized JRTConnection setNewCurrentConnection() { + List<JRTConnection> sources = getSources(); + currentConnection = sources.get(ThreadLocalRandom.current().nextInt(0, sources.size())); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Choosing new connection: " + currentConnection); + } + return currentConnection; + } + + List<JRTConnection> getSources() { + List<JRTConnection> ret = new ArrayList<>(); + synchronized (connections) { + for (JRTConnection source : connections.values()) { + ret.add(source); + } + } + return ret; + } + + ConfigSourceSet getSourceSet() { + return sourceSet; + } + + @Override + public void setError(Connection connection, int errorCode) { + connection.setError(errorCode); + setNewCurrentConnection(); + } + + public JRTConnectionPool updateSources(List<String> addresses) { + ConfigSourceSet newSources = new ConfigSourceSet(addresses); + return updateSources(newSources); + } + + public JRTConnectionPool updateSources(ConfigSourceSet sourceSet) { + synchronized (connections) { + for (JRTConnection conn : connections.values()) { + conn.getTarget().close(); + } + connections.clear(); + addSources(sourceSet); + } + return this; + } + + public String getAllSourceAddresses() { + StringBuilder sb = new StringBuilder(); + synchronized (connections) { + for (JRTConnection conn : connections.values()) { + sb.append(conn.getAddress()); + sb.append(","); + } + } + // Remove trailing "," + sb.deleteCharAt(sb.length() - 1); + return sb.toString(); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + synchronized (connections) { + for (JRTConnection conn : connections.values()) { + sb.append(conn.toString()); + sb.append("\n"); + } + } + return sb.toString(); + } + + public void close() { + supervisor.transport().shutdown().join(); + } + + @Override + public int getSize() { + synchronized (connections) { + return connections.size(); + } + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTMethods.java b/config/src/main/java/com/yahoo/vespa/config/JRTMethods.java new file mode 100644 index 00000000000..22c57413bc1 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/JRTMethods.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; + +/** + * Defines methods used for RPC config requests. + */ +public class JRTMethods { + + static final String getConfigMethodName = "getConfig"; + private static final String getConfigRequestTypes = "sssssll"; + static final String getConfigResponseTypes = "sssssilS"; + + static final String configV1getConfigMethodName = "config.v1.getConfig"; + private static final String configV1GetConfigRequestTypes = "sssssllsSi"; + static final String configV1GetConfigResponseTypes = "sssssilSs"; + + /** + * Creates a Method object for the RPC method getConfig. + * + * @param handler the object that will handle the method call + * @param handlerMethod the method belonging to the handler that will handle the method call + * @return a Method + */ + public static Method createGetConfigMethod(Object handler, String handlerMethod) { + return new Method(getConfigMethodName, getConfigRequestTypes, getConfigResponseTypes, + handler, handlerMethod) + .methodDesc("get config") + .paramDesc(0, "defName", "config class definition name") + .paramDesc(1, "defVersion", "config class definition version") + .paramDesc(2, "defMD5", "md5sum for config class definition") + .paramDesc(3, "configid", "config id") + .paramDesc(4, "configMD5", "md5sum for last got config, empty string if unknown") + .paramDesc(5, "timestamp", + "timestamp for last got config, only relevant to the server if useTimestamp != 0") + .paramDesc(6, "timeout", "timeout (milliseconds) before answering request if config is unchanged") + .returnDesc(0, "defName", "config name") + .returnDesc(1, "defVersion", "config version") + .returnDesc(2, "defMD5", "md5sum for config class definition") + .returnDesc(3, "configid", "requested config id") + .returnDesc(4, "configMD5", "md5sum for this config") + .returnDesc(5, "changed", "changed flag (1 if config changed, 0 otherwise") + .returnDesc(6, "timestamp", "timestamp when config was last changed") + .returnDesc(7, "payload", "config payload for the requested config"); + } + + /** + * Creates a Method object for the RPC method config.v1.getConfig. Use both for + * getting config and subscribing to config + * + * @param handler the object that will handle the method call + * @param handlerMethod the method belonging to the handler that will handle the method call + * @return a Method + */ + public static Method createConfigV1GetConfigMethod(Object handler, String handlerMethod) { + return new Method(configV1getConfigMethodName, configV1GetConfigRequestTypes, configV1GetConfigResponseTypes, + handler, handlerMethod) + .methodDesc("get config v1") + .paramDesc(0, "defName", "config class definition name") + .paramDesc(1, "defVersion", "config class definition version") + .paramDesc(2, "defMD5", "md5sum for config class definition") + .paramDesc(3, "configid", "config id") + .paramDesc(4, "configMD5", "md5sum for last got config, empty string if unknown") + .paramDesc(5, "generation", + "generation for last got config, only relevant to the server if generation != 0") + .paramDesc(6, "timeout", "timeout (milliseconds) before answering request if config is unchanged") + .paramDesc(7, "namespace", "namespace for defName") + .paramDesc(8, "defContent", "config definition content") + .paramDesc(9, "subscribe", "subscribe to config (1) or not (0)") + .returnDesc(0, "defName", "config name") + .returnDesc(1, "defVersion", "config version") + .returnDesc(2, "defMD5", "md5sum for config class definition") + .returnDesc(3, "configid", "requested config id") + .returnDesc(4, "configMD5", "md5sum for this config") + .returnDesc(5, "changed", "changed flag (1 if config changed, 0 otherwise") // TODO Maybe remove? + .returnDesc(6, "generation", "generation of config") + .returnDesc(7, "payload", "config payload for the requested config") + .returnDesc(8, "namespace", "namespace for defName"); + } + + public static final String configV2getConfigMethodName = "config.v2.getConfig"; + private static final String configV2GetConfigRequestTypes = "s"; + private static final String configV2GetConfigResponseTypes = "s"; + public static Method createConfigV2GetConfigMethod(Object handler, String handlerMethod) { + return new Method(configV2getConfigMethodName, configV2GetConfigRequestTypes, configV2GetConfigResponseTypes, + handler, handlerMethod) + .methodDesc("get config v2") + .paramDesc(0, "request", "config request") + .returnDesc(0, "response", "config response"); + } + + public static boolean checkV2ReturnTypes(Request request) { + return request.checkReturnTypes(JRTMethods.configV2GetConfigResponseTypes); + } + + public static final String configV3getConfigMethodName = "config.v3.getConfig"; + private static final String configV3GetConfigRequestTypes = "s"; + private static final String configV3GetConfigResponseTypes = "sx"; + public static Method createConfigV3GetConfigMethod(Object handler, String handlerMethod) { + return new Method(configV3getConfigMethodName, configV3GetConfigRequestTypes, configV3GetConfigResponseTypes, + handler, handlerMethod) + .methodDesc("get config v3") + .paramDesc(0, "request", "config request") + .returnDesc(0, "response", "config response") + .returnDesc(1, "payload", "config response payload"); + + } + + public static boolean checkV3ReturnTypes(Request request) { + return request.checkReturnTypes(JRTMethods.configV3GetConfigResponseTypes); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java new file mode 100644 index 00000000000..4c51afa3afe --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.vespa.config.util.ConfigUtils; +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; + +/** + * Wrapper for LZ4 compression that selects compression level based on properties. + * + * @author lulf + * @since 5.19 + */ +public class LZ4PayloadCompressor { + private static final LZ4Factory lz4Factory = LZ4Factory.safeInstance(); + private static final String VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL = "VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL"; + private static final int compressionLevel = getCompressionLevel(); + + private static int getCompressionLevel() { + return Integer.parseInt(ConfigUtils.getEnvValue("0", + System.getenv(VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL), + System.getenv("services__config_protocol_compression_level"), + System.getProperty(VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL))); + } + + public byte[] compress(byte[] input) { + return getCompressor().compress(input); + } + + public void decompress(byte[] input, byte[] outputbuffer) { + if (input.length > 0) { + lz4Factory.safeDecompressor().decompress(input, outputbuffer); + } + } + + private LZ4Compressor getCompressor() { + return (compressionLevel < 7) ? lz4Factory.fastCompressor() : lz4Factory.highCompressor(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/RawConfig.java b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java new file mode 100755 index 00000000000..a7c4f4bf788 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/RawConfig.java @@ -0,0 +1,224 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.config.protocol.CompressionInfo; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; +import com.yahoo.vespa.config.protocol.JRTConfigRequest; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; +import com.yahoo.vespa.config.protocol.Payload; +import com.yahoo.vespa.config.protocol.VespaVersion; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +/** + * Encapsulates config, usually associated with a {@link JRTConfigRequest}. An instance of this class can represent + * either a config that is not yet resolved, a successfully resolved config, or an error. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class RawConfig { + + private final ConfigKey<?> key; + private final String defMd5; + private final List<String> defContent; + private final Payload payload; + private final int errorCode; + private final String configMd5; + private final Optional<VespaVersion> vespaVersion; + private long generation; + + /** + * Constructor for an empty config (not yet resolved). + * @param key The ConfigKey + * @param defMd5 The md5 sum of the .def-file. + */ + public RawConfig(ConfigKey<?> key, String defMd5) { + this(key, defMd5, null, "", 0L, 0, Collections.<String>emptyList(), Optional.empty()); + } + + public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, String configMd5, long generation, List<String> defContent, Optional<VespaVersion> vespaVersion) { + this(key, defMd5, payload, configMd5, generation, 0, defContent, vespaVersion); + } + + /** Copy constructor */ + public RawConfig(RawConfig rawConfig) { + this(rawConfig.key, rawConfig.defMd5, rawConfig.payload, rawConfig.configMd5, + rawConfig.generation, rawConfig.errorCode, rawConfig.defContent, rawConfig.getVespaVersion()); + } + + public RawConfig(ConfigKey<?> key, String defMd5, Payload payload, + String configMd5, long generation, int errorCode, List<String> defContent, Optional<VespaVersion> vespaVersion) { + this.key = key; + this.defMd5 = ConfigUtils.getDefMd5FromRequest(defMd5, defContent); + this.payload = payload; + this.configMd5 = configMd5; + this.generation = generation; + this.errorCode = errorCode; + this.defContent = defContent; + this.vespaVersion = vespaVersion; + } + + /** + * Creates a new Config from the given request, with the values in the response parameters. + * @param req a {@link JRTClientConfigRequest} + */ + public static RawConfig createFromResponseParameters(JRTClientConfigRequest req) { + return new RawConfig(req.getConfigKey(), req.getConfigKey().getMd5(), req.getNewPayload(), req.getNewConfigMd5(), + req.getNewGeneration(), 0, req.getDefContent().asList(), req.getVespaVersion()); + } + + /** + * Creates a new Config from the given request, with the values in the response parameters. + * @param req a {@link JRTClientConfigRequest} + */ + public static RawConfig createFromServerRequest(JRTServerConfigRequest req) { + return new RawConfig(req.getConfigKey(), req.getConfigKey().getMd5() , Payload.from(new Utf8String(""), CompressionInfo.uncompressed()), req.getRequestConfigMd5(), + req.getRequestGeneration(), 0, req.getDefContent().asList(), req.getVespaVersion()); + } + + + public ConfigKey<?> getKey() { + return key; + } + + public String getName() { + return key.getName(); + } + + public String getNamespace() { + return key.getNamespace(); + } + + public String getConfigId() { + return key.getConfigId(); + } + + public String getConfigMd5() { + return configMd5; + } + + public String getDefMd5() { + return defMd5; + } + + public long getGeneration() { + return generation; + } + + public void setGeneration(long generation) { + this.generation = generation; + } + + public Payload getPayload() { + return payload; + } + + public int errorCode() { + return errorCode; + } + + public String getDefNamespace() { + return key.getNamespace(); + } + + public Optional<VespaVersion> getVespaVersion() { + return vespaVersion; + } + + /** + * Returns true if this config is equal to the config (same payload md5) in the given request. + * + * @param req The request for which to compare config payload with this config. + * @return true if this config is equal to the config in the given request. + */ + public boolean hasEqualConfig(JRTServerConfigRequest req) { + return (getConfigMd5().equals(req.getRequestConfigMd5())); + } + + /** + * Returns true if this config has a more recent generation than the config in the given request. + * + * @param req The request for which to compare generation with this config. + * @return true if this config has a more recent generation than the config in the given request. + */ + public boolean hasNewerGeneration(JRTServerConfigRequest req) { + return (getGeneration() > req.getRequestGeneration()); + } + + /** + * Convenience method. + * @return true if errorCode() returns 0, false otherwise. + */ + public boolean isError() { + return (errorCode() != 0); + } + + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (! (o instanceof RawConfig)) { + return false; + } + RawConfig other = (RawConfig) o; + if (! (key.equals(other.key) && + defMd5.equals(other.defMd5) && + (errorCode == other.errorCode)) ) { + return false; + } + // Need to check error codes before isError, since unequal error codes always means unequal requests, + // while non-zero and equal error codes means configs are equal. + if (isError()) + return true; + if (generation != other.generation) + return false; + if (configMd5 != null) { + return configMd5.equals(other.configMd5); + } else { + return (other.configMd5 == null); + } + } + + public int hashCode() { + int hash = 17; + if (key != null) { + hash = 31 * hash + key.hashCode(); + } + if (defMd5 != null) { + hash = 31 * hash + defMd5.hashCode(); + } + hash = 31 * hash + errorCode; + if (! isError()) { + // configMd5 and generation only matter when the RawConfig is not an error. + hash = 31 * hash + (int)(generation ^(generation >>>32)); + if (configMd5 != null) { + hash = 31 * hash + configMd5.hashCode(); + } + } + return hash; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(key.getNamespace()).append(".").append(key.getName()); + sb.append(","); + sb.append(getDefMd5()); + sb.append(","); + sb.append(key.getConfigId()); + sb.append(","); + sb.append(getConfigMd5()); + sb.append(","); + sb.append(getGeneration()); + sb.append(","); + sb.append(getPayload()); + return sb.toString(); + } + + public List<String> getDefContent() { + return defContent; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java b/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java new file mode 100644 index 00000000000..6a5052b66cf --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.slime.*; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Optional; + +/** + * Extra utilities/operations on slime trees that we would like to have as part of slime in the future, but + * which resides here until we have a better place to put it. + * + * @author lulf + * @since 5.8 + */ +public class SlimeUtils { + public static void copyObject(Inspector from, final Cursor to) { + if (from.type() != Type.OBJECT) { + throw new IllegalArgumentException("Cannot copy object: " + from); + } + from.traverse(new ObjectTraverser() { + @Override + public void field(String name, Inspector inspector) { + setObjectEntry(inspector, name, to); + } + }); + + } + + private static void setObjectEntry(Inspector from, String name, Cursor to) { + switch (from.type()) { + case NIX: + to.setNix(name); + break; + case BOOL: + to.setBool(name, from.asBool()); + break; + case LONG: + to.setLong(name, from.asLong()); + break; + case DOUBLE: + to.setDouble(name, from.asDouble()); + break; + case STRING: + to.setString(name, from.asString()); + break; + case DATA: + to.setData(name, from.asData()); + break; + case ARRAY: + Cursor array = to.setArray(name); + copyArray(from, array); + break; + case OBJECT: + Cursor object = to.setObject(name); + copyObject(from, object); + break; + } + } + + private static void copyArray(Inspector from, final Cursor to) { + from.traverse(new ArrayTraverser() { + @Override + public void entry(int i, Inspector inspector) { + addValue(inspector, to); + } + }); + + } + + private static void addValue(Inspector from, Cursor to) { + switch (from.type()) { + case NIX: + to.addNix(); + break; + case BOOL: + to.addBool(from.asBool()); + break; + case LONG: + to.addLong(from.asLong()); + break; + case DOUBLE: + to.addDouble(from.asDouble()); + break; + case STRING: + to.addString(from.asString()); + break; + case DATA: + to.addData(from.asData()); + break; + case ARRAY: + Cursor array = to.addArray(); + copyArray(from, array); + break; + case OBJECT: + Cursor object = to.addObject(); + copyObject(from, object); + break; + } + + } + + public static byte[] toJsonBytes(Slime slime) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + new JsonFormat(true).encode(baos, slime); + return baos.toByteArray(); + } + + public static Slime jsonToSlime(byte[] json) { + Slime slime = new Slime(); + new JsonDecoder().decode(slime, json); + return slime; + } + + public static Optional<String> optionalString(Inspector inspector) { + return Optional.of(inspector.asString()).filter(s -> !s.isEmpty()); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/Source.java b/config/src/main/java/com/yahoo/vespa/config/Source.java new file mode 100644 index 00000000000..3ec17038506 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/Source.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigurationRuntimeException; + +import java.util.logging.Logger; + +/** + * A general config source that retrieves config for its SourceConfig. + * + * This class and its subclasses are thread safe. + * + * Note that it is the responsibility of the user to set a source's state to OPEN and CANCELLED, and + * that the READY state can be set by the user as a mark of progress e.g. when waiting for a monitor/lock. + * All other states are set by this class or one of its subclasses. + * + * Originally designed for re-use by closing and reopening, but this caused problems related to + * synchronization between this class and ConfigInstance.subscribeLock. Currently (2008-05-08) + * a source cannot be reopened once it has been cancelled. + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public abstract class Source { + + public enum State { NEW, OPEN_PENDING, READY, OPEN, CANCEL_REQUESTED, CANCELLED } + + protected volatile SourceConfig config; + protected volatile State state = State.NEW; + protected long openTimestamp = 0; + public final static Logger logger = Logger.getLogger(Source.class.getPackage().getName()); + + public Source(SourceConfig sourceConfig) { + this.config = sourceConfig; + } + + /** + * Opens this config source. + * Typically called when the first subscriber subscribes to our ConfigInstance. + */ + public final synchronized void open() { + if ((state == State.OPEN) || (state == State.OPEN_PENDING)) { + return; + } else if ((state == State.CANCELLED) || (state == State.CANCEL_REQUESTED)) { + throw new ConfigurationRuntimeException("Subscription with config ID: " + config.getConfigId() + ": Trying to reopen a cancelled source, should not happen.", null); + } + state = State.OPEN_PENDING; + openTimestamp = System.currentTimeMillis(); + myOpen(); + getConfig(); + } + + /** + * Optional subclass hook for the open() method. + */ + protected void myOpen() { } + + /** + * Gets config from this config source. + */ + public final synchronized void getConfig() { + if ((state == State.CANCELLED) || (state == State.CANCEL_REQUESTED)) { + logger.info("Trying to retrieve config from source " + this + " in state: " + state); + return; + } + myGetConfig(); + } + + /** + * Mandatory subclass hook for the getConfig() method. + */ + protected abstract void myGetConfig(); + + /** + * Cancels this config source. Typically called when our ConfigInstance has no more subscribers. + * + * Irreversible. Reopening a cancelled source would cause problems with multiple threads accessing the source + * simultaneously. With better synchronization mechanisms it _should_ be possible to close and reopen a source. + */ + public final void cancel() { + logger.fine("Closing source " + this + " from state " + state); + if ((state == State.CANCELLED) || (state == State.CANCEL_REQUESTED)) { + return; + } + state = State.CANCEL_REQUESTED; + myCancel(); + } + + /** + * Optional subclass hook for the cancel() method. + * Should typically free all the subclass' resources, i.e. requests, threads etc.. + */ + protected void myCancel() { } + + public State getState() { + return state; + } + + public void setState(State state) { + this.state = state; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/SourceConfig.java b/config/src/main/java/com/yahoo/vespa/config/SourceConfig.java new file mode 100644 index 00000000000..49484e16ff0 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/SourceConfig.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import java.util.List; + +/** + * Interface for config instances that use a {@link Source} to retrieve config values. + * + * @author <a href="gv@yahoo-inc.com">Gj\u00F8ran Voldengen</a> + */ +public interface SourceConfig { + + /** + * Notify subscribers that this config has been initialized by the {@link Source}. + */ + public void notifyInitMonitor(); + + /** + * Sets the fields in the config object from the payload in the given request and updates subscribers + * with the new config. The given request can be an error response with an error code and no payload. + * + * @param req Config request containing return values, or an error response. + */ + public void setConfig(com.yahoo.vespa.config.protocol.JRTClientConfigRequest req); + + /** + * Sets this config's generation. + * + * @param generation The new generation (usually from the source). + */ + public void setGeneration(long generation); + + public String getDefName(); + public String getDefNamespace(); + public String getDefVersion(); + public List<String> getDefContent(); + public String getDefMd5(); + + public String getConfigId(); + public ConfigKey<?> getKey(); + + public String getConfigMd5(); + + public long getGeneration(); + + public RawConfig getConfig(); + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java new file mode 100644 index 00000000000..46f0854084c --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java @@ -0,0 +1,262 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import java.util.Random; + +/** + * Timeouts, delays and retries used in RPC config protocol. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + */ +public class TimingValues { + public static final long defaultNextConfigTimeout = 1000; + // See getters below for an explanation of how these values are used and interpreted + // All time values in milliseconds. + private long successTimeout = 600000; + private long errorTimeout = 20000; + private long initialTimeout = 15000; + private long subscribeTimeout = 55000; + private long configuredErrorTimeout = -1; // Don't ever timeout (and do not use error response) when we are already configured + private long nextConfigTimeout = defaultNextConfigTimeout; + + private long fixedDelay = 5000; + private long unconfiguredDelay = 1000; + private long configuredErrorDelay = 15000; + private int maxDelayMultiplier = 10; + private final Random rand; + + public TimingValues() { + this.rand = new Random(System.currentTimeMillis()); + } + + // TODO Should add nextConfigTimeout in all constructors + public TimingValues(long successTimeout, + long errorTimeout, + long initialTimeout, + long subscribeTimeout, + long unconfiguredDelay, + long configuredErrorDelay, + long fixedDelay, + int maxDelayMultiplier) { + + this.successTimeout = successTimeout; + this.errorTimeout = errorTimeout; + this.initialTimeout = initialTimeout; + this.subscribeTimeout = subscribeTimeout; + this.unconfiguredDelay = unconfiguredDelay; + this.configuredErrorDelay = configuredErrorDelay; + this.fixedDelay = fixedDelay; + this.maxDelayMultiplier = maxDelayMultiplier; + this.rand = new Random(System.currentTimeMillis()); + } + + private TimingValues(long successTimeout, + long errorTimeout, + long initialTimeout, + long subscribeTimeout, + long unconfiguredDelay, + long configuredErrorDelay, + long fixedDelay, + int maxDelayMultiplier, + Random rand) { + + this.successTimeout = successTimeout; + this.errorTimeout = errorTimeout; + this.initialTimeout = initialTimeout; + this.subscribeTimeout = subscribeTimeout; + this.unconfiguredDelay = unconfiguredDelay; + this.configuredErrorDelay = configuredErrorDelay; + this.fixedDelay = fixedDelay; + this.maxDelayMultiplier = maxDelayMultiplier; + this.rand = rand; + } + + public TimingValues(TimingValues tv) { + this(tv.successTimeout, + tv.errorTimeout, + tv.initialTimeout, + tv.subscribeTimeout, + tv.unconfiguredDelay, + tv.configuredErrorDelay, + tv.fixedDelay, + tv.maxDelayMultiplier, + tv.getRandom()); + } + + public TimingValues(TimingValues tv, Random random) { + this(tv.successTimeout, + tv.errorTimeout, + tv.initialTimeout, + tv.subscribeTimeout, + tv.unconfiguredDelay, + tv.configuredErrorDelay, + tv.fixedDelay, + tv.maxDelayMultiplier, + random); + } + + /** + * Returns timeout to use as server timeout when previous config request was a success. + * + * @return timeout in milliseconds. + */ + public long getSuccessTimeout() { + return successTimeout; + } + + /** + * Returns timeout to use as server timeout when we got an error with the previous config request. + * + * @return timeout in milliseconds. + */ + public long getErrorTimeout() { + return errorTimeout; + } + + /** + * Returns initial timeout to use as server timeout when a config is requested for the first time. + * + * @return timeout in milliseconds. + */ + public long getInitialTimeout() { + return initialTimeout; + } + + public TimingValues setInitialTimeout(long t) { + initialTimeout = t; + return this; + } + + /** + * Returns timeout to use as server timeout when subscribing for the first time. + * + * @return timeout in milliseconds. + */ + public long getSubscribeTimeout() { + return subscribeTimeout; + } + + public TimingValues setSubscribeTimeout(long t) { + subscribeTimeout = t; + return this; + } + + /** + * Returns the time to retry getting config from the remote sources, until the next error response will + * be set as config. Counted from the last ok request was received. A negative value means that + * we will always retry getting config and never set an error response as config. + * + * @return timeout in milliseconds. + */ + public long getConfiguredErrorTimeout() { + return configuredErrorTimeout; + } + + public TimingValues setConfiguredErrorTimeout(long t) { + configuredErrorTimeout = t; + return this; + } + + /** + * Returns timeout used when calling {@link com.yahoo.config.subscription.ConfigSubscriber#nextConfig()} or + * {@link com.yahoo.config.subscription.ConfigSubscriber#nextGeneration()} + * + * @return timeout in milliseconds. + */ + public long getNextConfigTimeout() { + return nextConfigTimeout; + } + + public TimingValues setNextConfigTimeout(long t) { + nextConfigTimeout = t; + return this; + } + + /** + * Returns time to wait until next attempt to get config after a failed request when the client has not + * gotten a successful response to a config subscription (i.e, the client has not been configured). + * A negative value means that there will never be a next attempt. If a negative value is set, the + * user must also setSubscribeTimeout(0) to prevent a deadlock while subscribing. + * + * @return delay in milliseconds, a negative value means never. + */ + public long getUnconfiguredDelay() { + return unconfiguredDelay; + } + + public TimingValues setUnconfiguredDelay(long d) { + unconfiguredDelay = d; + return this; + } + + /** + * Returns time to wait until next attempt to get config after a failed request when the client has + * previously gotten a successful response to a config subscription (i.e, the client is configured). + * A negative value means that there will never be a next attempt. + * + * @return delay in milliseconds, a negative value means never. + */ + public long getConfiguredErrorDelay() { + return configuredErrorDelay; + } + + public TimingValues setConfiguredErrorDelay(long d) { + configuredErrorDelay = d; + return this; + } + + /** + * Returns maximum multiplier to use when calculating delay (the delay is multiplied by the number of + * failed requests, unless that number is this maximum multiplier). + * + * @return timeout in milliseconds. + */ + public int getMaxDelayMultiplier() { + return maxDelayMultiplier; + } + + + public TimingValues setSuccessTimeout(long successTimeout) { + this.successTimeout = successTimeout; + return this; + } + + /** + * Returns fixed delay that is used when retrying getting config no matter if it was a success or an error + * and independent of number of retries. + * + * @return timeout in milliseconds. + */ + public long getFixedDelay() { + return fixedDelay; + } + + /** + * Returns a number +/- a random component + * + * @param val input + * @param fraction for instance 0.1 for +/- 10% + * @return a number + */ + public long getPlusMinusFractionRandom(long val, float fraction) { + return Math.round(val - (val * fraction) + (rand.nextFloat() * 2l * val * fraction)); + } + + Random getRandom() { + return rand; + } + + @Override + public String toString() { + return "TimingValues [successTimeout=" + successTimeout + + ", errorTimeout=" + errorTimeout + ", initialTimeout=" + + initialTimeout + ", subscribeTimeout=" + subscribeTimeout + + ", configuredErrorTimeout=" + configuredErrorTimeout + + ", fixedDelay=" + fixedDelay + ", unconfiguredDelay=" + + unconfiguredDelay + ", configuredErrorDelay=" + + configuredErrorDelay + ", maxDelayMultiplier=" + + maxDelayMultiplier + ", rand=" + rand + "]"; + } + + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/UnknownConfigIdException.java b/config/src/main/java/com/yahoo/vespa/config/UnknownConfigIdException.java new file mode 100644 index 00000000000..d78eebe3ed4 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/UnknownConfigIdException.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +/** + * Used when a config model does not recognize a config id + * @author vegardh + * + */ +@SuppressWarnings("serial") +public class UnknownConfigIdException extends IllegalArgumentException { + + public UnknownConfigIdException(String msg) { + super(msg); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java new file mode 100644 index 00000000000..07f2911cc69 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java @@ -0,0 +1,259 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.benchmark; + +import com.yahoo.collections.Tuple2; +import com.yahoo.io.IOUtils; +import com.yahoo.jrt.*; +import com.yahoo.system.CommandLineParser; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.protocol.*; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.*; +import java.util.*; + +/** + * A load client for a config server or proxy. + * + * Log messages from a run will have a # first in the line, the end result will not. + * + * @author vegardh + */ +public class LoadTester { + + private static boolean debug = false; + private Transport transport = new Transport(); + protected Supervisor supervisor = new Supervisor(transport); + private List<ConfigKey<?>> configs = new ArrayList<>(); + private Random random = new Random(System.currentTimeMillis()); + private Map<ConfigDefinitionKey, Tuple2<String, String[]>> defs = new HashMap<>(); + private long protocolVersion = Long.parseLong(JRTConfigRequestFactory.getProtocolVersion()); + private CompressionType compressionType = JRTConfigRequestFactory.getCompressionType(); + + /** + * @param args command-line arguments + */ + public static void main(String[] args) throws IOException, InterruptedException { + CommandLineParser parser = new CommandLineParser("LoadTester", args); + parser.addLegalUnarySwitch("-d", "debug"); + parser.addRequiredBinarySwitch("-c", "host (config proxy or server)"); + parser.addRequiredBinarySwitch("-p", "port"); + parser.addRequiredBinarySwitch("-i", "iterations per thread"); + parser.addRequiredBinarySwitch("-t", "threads"); + parser.addLegalBinarySwitch("-l", "configs file, on form name,configid. (To get list: configproxy-cmd -m cache | cut -d ',' -f1-2)"); + parser.addLegalBinarySwitch("-dd", "dir with def files, must be of form name.def"); + parser.parse(); + String host = parser.getBinarySwitches().get("-c"); + int port = Integer.parseInt(parser.getBinarySwitches().get("-p")); + int iterations = Integer.parseInt(parser.getBinarySwitches().get("-i")); + int threads = Integer.parseInt(parser.getBinarySwitches().get("-t")); + String configsList = parser.getBinarySwitches().get("-l"); + String defPath = parser.getBinarySwitches().get("-dd"); + debug = parser.getUnarySwitches().contains("-d"); + LoadTester loadTester = new LoadTester(); + loadTester.runLoad(host, port, iterations, threads, configsList, defPath); + } + + private void runLoad(String host, int port, int iterations, int threads, + String configsList, String defPath) throws IOException, InterruptedException { + configs = readConfigs(configsList); + defs = readDefs(defPath); + List<LoadThread> threadList = new ArrayList<>(); + long start = System.currentTimeMillis(); + Metrics m = new Metrics(); + + for (int i = 0; i < threads; i++) { + LoadThread lt = new LoadThread(iterations, host, port); + threadList.add(lt); + lt.start(); + } + + for (LoadThread lt : threadList) { + lt.join(); + m.merge(lt.metrics); + } + printOutput(start, threads, iterations, m); + } + + private Map<ConfigDefinitionKey, Tuple2<String, String[]>> readDefs(String defPath) throws IOException { + Map<ConfigDefinitionKey, Tuple2<String, String[]>> ret = new HashMap<>(); + if (defPath==null) return ret; + File defDir = new File(defPath); + if (!defDir.isDirectory()) { + System.out.println("# Given def file dir is not a directory: "+defDir.getPath()+" , will not send def contents in requests."); + return ret; + } + final File[] files = defDir.listFiles(); + if (files == null) { + System.out.println("# Given def file dir has no files: "+defDir.getPath()+" , will not send def contents in requests."); + return ret; + } + for (File f : files) { + String name = f.getName(); + if (!name.endsWith(".def")) continue; + String[] splitted = name.split("\\."); + if (splitted.length<2) continue; + String nam = splitted[splitted.length - 2]; + String contents = IOUtils.readFile(f); + ConfigDefinitionKey key = ConfigUtils.createConfigDefinitionKeyFromDefContent(nam, Utf8.toBytes(contents)); + ret.put(key, new Tuple2<>(ConfigUtils.getDefMd5(Arrays.asList(contents.split("\n"))), contents.split("\n"))); + } + System.out.println("# Read "+ret.size()+" def files from "+defDir.getPath()); + return ret; + } + + private void printOutput(long start, long threads, long iterations, Metrics metrics) { + long stop = System.currentTimeMillis(); + float durSec = (float) (stop - start) / 1000f; + StringBuilder sb = new StringBuilder(); + sb.append("#reqs/sec #bytes/sec #avglatency #minlatency #maxlatency #failedrequests\n"); + sb.append(((float) (iterations * threads)) / durSec).append(","); + sb.append((metrics.totBytes / durSec)).append(","); + sb.append((metrics.totLatency / threads / iterations)).append(","); + sb.append((metrics.minLatency)).append(","); + sb.append((metrics.maxLatency)).append(","); + sb.append((metrics.failedRequests)); + sb.append("\n"); + System.out.println(sb.toString()); + } + + private List<ConfigKey<?>> readConfigs(String configsList) throws IOException { + List<ConfigKey<?>> ret = new ArrayList<>(); + BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(configsList), "UTF-8")); + String str = br.readLine(); + while (str != null) { + String[] nameAndId = str.split(","); + Tuple2<String, String> nameAndNamespace = ConfigUtils.getNameAndNamespaceFromString(nameAndId[0]); + ConfigKey<?> key = new ConfigKey<>(nameAndNamespace.first, nameAndId[1], nameAndNamespace.second); + ret.add(key); + str = br.readLine(); + } + br.close(); + return ret; + } + + private class Metrics { + public long totBytes = 0; + public long totLatency = 0; + public long failedRequests = 0; + public long maxLatency = Long.MIN_VALUE; + public long minLatency = Long.MAX_VALUE; + + public void merge(Metrics m) { + this.totBytes += m.totBytes; + this.totLatency += m.totLatency; + this.failedRequests += m.failedRequests; + updateMin(m.minLatency); + updateMax(m.maxLatency); + } + + + public void update(long bytes, long latency) { + this.totBytes += bytes; + this.totLatency += latency; + updateMin(latency); + updateMax(latency); + } + + private void updateMin(long latency) { + if (latency < minLatency) + minLatency = latency; + } + + private void updateMax(long latency) { + if (latency > maxLatency) + maxLatency = latency; + } + + private void incFailedRequests() { + failedRequests++; + } + } + + private class LoadThread extends Thread { + int iterations = 0; + String host = ""; + int port = 0; + Metrics metrics = new Metrics(); + + public LoadThread(int iterations, String host, int port) { + this.iterations = iterations; + this.host = host; + this.port = port; + } + + @Override + public void run() { + Spec spec = new Spec(host, port); + Target target = connect(spec); + ConfigKey<?> reqKey; + JRTClientConfigRequest request; + int totConfs = configs.size(); + boolean reconnCycle = false; // to log reconn message only once, for instance at restart + for (int i = 0; i < iterations; i++) { + reqKey = configs.get(random.nextInt(totConfs)); + ConfigDefinitionKey dKey = new ConfigDefinitionKey(reqKey); + Tuple2<String, String[]> defContent = defs.get(dKey); + if (defContent==null && defs.size()>0) { // Only complain if we actually did run with a def dir + System.out.println("# No def found for "+dKey+", not sending in request."); + }/* else { + System.out.println("# FOUND: "+dKey+" : "+ StringUtilities.implode(defContent, "\n")); + }*/ + request = getRequest(ConfigKey.createFull(reqKey.getName(), reqKey.getConfigId(), reqKey.getNamespace(), defContent.first), defContent.second); + if (debug) System.out.println("# Requesting: " + reqKey); + long start = System.currentTimeMillis(); + target.invokeSync(request.getRequest(), 10.0); + long end = System.currentTimeMillis(); + if (request.isError()) { + if ("Connection lost".equals(request.errorMessage()) || "Connection down".equals(request.errorMessage())) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (!reconnCycle) { + System.out.println("# Connection lost, reconnecting..."); + reconnCycle = true; + } + target = connect(spec); + } else { + System.err.println(request.errorMessage()); + } + metrics.incFailedRequests(); + } else { + if (reconnCycle) { + reconnCycle = false; + System.out.println("# Connection OK"); + } + long duration = end - start; + + if (debug) { + String payload = request.getNewPayload().toString(); + metrics.update(payload.length(), duration); // assume 8 bit... + System.out.println("# Ret: " + payload); + } else { + metrics.update(0, duration); + } + } + } + } + + private JRTClientConfigRequest getRequest(ConfigKey<?> reqKey, String[] defContent) { + if (defContent==null) defContent=new String[0]; + final long serverTimeout = 1000; + if (protocolVersion == 3) { + return JRTClientConfigRequestV3.createWithParams(reqKey, DefContent.fromList(Arrays.asList(defContent)), + "unknown", "", 0, serverTimeout, Trace.createDummy(), + compressionType, Optional.empty()); + } else { + throw new RuntimeException("Unsupported protocol version" + protocolVersion); + } + } + + private Target connect(Spec spec) { + return supervisor.connectSync(spec); + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/StressTester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/StressTester.java new file mode 100644 index 00000000000..3f2cd9ae2fa --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/StressTester.java @@ -0,0 +1,277 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.benchmark; + +import com.yahoo.jrt.*; +import com.yahoo.system.CommandLineParser; + +import java.io.*; +import java.util.*; + +/** + * /** + * A class for stress-testing config server and config proxy. + * Includes an RPC server interface for communicating + * with test classes that implement the {@link Tester} interface. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 5.1.5 + */ +public class StressTester { + private static boolean debug = false; + private final String testClassName; + private final List<Thread> threadList = new ArrayList<>(); + private final List<TestRunner> testRunners = new ArrayList<>(); + + public StressTester(String testClass) { + this.testClassName = testClass; + } + + /** + * @param args command-line arguments + */ + public static void main(String[] args) { + CommandLineParser parser = new CommandLineParser("StressTester", args); + parser.addLegalUnarySwitch("-d", "debug"); + parser.addRequiredBinarySwitch("-c", "host (config proxy or server)"); + parser.addRequiredBinarySwitch("-p", "port"); + parser.addLegalBinarySwitch("-class", "Use class with this name from test bundle (must be given in class path)"); + parser.addLegalBinarySwitch("-serverport", "port for rpc server"); + parser.parse(); + // TODO Handle other hosts and ports + String host = parser.getBinarySwitches().get("-c"); + int port = Integer.parseInt(parser.getBinarySwitches().get("-p")); + debug = parser.getUnarySwitches().contains("-d"); + String classNameInBundle = parser.getBinarySwitches().get("-class"); + int serverPort = Integer.parseInt(parser.getBinarySwitches().get("-serverport")); + RpcServer rpcServer = new RpcServer(null, serverPort, new StressTester(classNameInBundle)); + new Thread(rpcServer).start(); + } + + static class TestRunner implements Runnable { + private final Tester tester; + private volatile boolean stop = false; + + TestRunner(Tester tester) { + this.tester = tester; + } + + @Override + public void run() { + tester.subscribe(); + while (!stop) { + tester.fetch(); + } + tester.close(); + } + + public void stop() { + stop = true; + } + } + + private Map<String, Map<String, String>> getVerificationMap(String verificationFile) { + // Read verification file into a map that test stubs should verify against + Map<String, Map<String, String>> verificationMap = new HashMap<>(); + if (verificationFile != null) { + BufferedReader reader = null; + try { + reader = new BufferedReader(new FileReader(verificationFile)); + String l; + while ((l = reader.readLine()) != null) { + String[] line = l.split(","); + String defFile = line[0]; + String fieldName = line[1]; + String expectedValue = line[2]; + Map<String, String> defExpected = verificationMap.get(defFile); + if (defExpected == null) + defExpected = new HashMap<>(); + defExpected.put(fieldName, expectedValue); + verificationMap.put(defFile, defExpected); + } + } catch (Exception e) { + throw new IllegalArgumentException("Unable to load verification file " + verificationFile); + } finally { + if (reader != null) try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + return verificationMap; + } + + private void startTesters(int threads) { + // Load and run actual test stub + Class<?> testClass; + try { + testClass = Class.forName(testClassName); + threadList.clear(); + testRunners.clear(); + for (int i = 0; i < threads; i++) { + Tester tester = (Tester) testClass.newInstance(); + TestRunner testRunner = new TestRunner(tester); + testRunners.add(testRunner); + Thread t = new Thread(testRunner); + threadList.add(t); + } + debug("Starting testers"); + // Now that all testers have been created, start them + for (Thread t : threadList) { + debug("Starting thread"); + t.start(); + } + } catch (Exception e) { + debug("error in startTesters"); + throw new IllegalArgumentException("Unable to load class with name " + testClassName, e); + } + debug("After starting testers"); + } + + public boolean verify(long generation, long timeout, String verificationFile) throws InterruptedException { + Map<String, Map<String, String>> verificationMap = getVerificationMap(verificationFile); + for (TestRunner testRunner : testRunners) { + long start = System.currentTimeMillis(); + boolean ok = false; + do { + if (testRunner.tester.verify(verificationMap, generation)) { + ok = true; + } + Thread.sleep(10); + } while (!ok && (System.currentTimeMillis() - start < timeout)); + if (!ok) { + return false; + } + } + return true; + } + + public void stop() { + debug("Stopping test runners"); + for (TestRunner testRunner : testRunners) { + testRunner.stop(); + } + debug("Stopping threads"); + for (Thread t : threadList) { + try { + t.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + debug("End of stop"); + } + + private static void debug(String s) { + if (debug) { + System.out.println(s); + } + } + + public static class RpcServer implements Runnable { + private Transport transport = new Transport(); + protected Supervisor supervisor = new Supervisor(transport); + private final Spec spec; + private final StressTester tester; + + RpcServer(String host, int port, StressTester tester) { + this.tester = tester; + setUp(this); + spec = new Spec(host, port); + } + + public void run() { + try { + Acceptor acceptor = supervisor.listen(spec); + supervisor.transport().join(); + acceptor.shutdown().join(); + } catch (ListenFailedException e) { + throw new RuntimeException("Could not listen to " + spec); + } + } + + public void shutdown() { + supervisor.transport().shutdown().join(); + } + + public final void start(Request request) { + debug("start: Got " + request); + int ret = 1; + int clients = request.parameters().get(0).asInt32(); + debug("start: starting testers"); + try { + tester.startTesters(clients); + ret = 0; + } catch (Exception e) { + debug("start: error: " + e.getMessage()); + e.printStackTrace(); + } + debug("start: Returning " + ret); + request.returnValues().add(new Int32Value(ret)); + } + + public final void verify(Request request) { + debug("verify: Got " + request); + long generation = request.parameters().get(0).asInt64(); + String verificationFile = request.parameters().get(1).asString(); + long timeout = request.parameters().get(2).asInt64(); + int ret = 0; + String errorMessage = ""; + try { + if (!tester.verify(generation, timeout, verificationFile)) { + ret = 1; + errorMessage = "Unable to get generation " + generation + " within timeout " + timeout; + } + } catch (Exception e) { + ret = 1; + errorMessage = e.getMessage(); + e.printStackTrace(); + } catch (AssertionError e) { + ret = 1; + errorMessage = e.getMessage(); + } + debug("verify: Returning " + ret); + request.returnValues().add(new Int32Value(ret)); + request.returnValues().add(new StringValue(errorMessage)); + } + + public final void stop(Request request) { + debug("stop: Got " + request); + int ret = 1; + try { + tester.stop(); + ret = 0; + } catch (Exception e) { + e.printStackTrace(); + } + debug("stop: Returning " + ret); + request.returnValues().add(new Int32Value(ret)); + } + + /** + * Set up RPC method handlers. + * + * @param handler a MethodHandler that will handle the RPC methods + */ + + protected void setUp(Object handler) { + supervisor.addMethod(new Method("start", "i", "i", + handler, "start") + .methodDesc("start") + .paramDesc(0, "clients", "number of clients") + .returnDesc(0, "ret code", "return code, 0 is OK")); + supervisor.addMethod(new Method("verify", "lsl", "is", + handler, "verify") + .methodDesc("verify") + .paramDesc(0, "generation", "config generation") + .paramDesc(1, "verification file", "name of verification file") + .paramDesc(2, "timeout", "timeout when verifying") + .returnDesc(0, "ret code", "return code, 0 is OK") + .returnDesc(1, "error message", "error message, if non zero return code")); + supervisor.addMethod(new Method("stop", "", "i", + handler, "stop") + .methodDesc("stop") + .returnDesc(0, "ret code", "return code, 0 is OK")); + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/Tester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/Tester.java new file mode 100644 index 00000000000..68d20f7a75f --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/Tester.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.benchmark; + +import java.util.Map; + +/** + * Tester interface for loadable test runners. + */ +public interface Tester { + public void subscribe(); + public boolean fetch(); + public boolean verify(Map<String, Map<String, String>> expected, long generation); + public void close(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java new file mode 100644 index 00000000000..04799d47494 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +import javax.tools.Diagnostic; +import javax.tools.DiagnosticCollector; +import javax.tools.JavaCompiler; +import javax.tools.JavaFileObject; + +/** + * Represents a compilation task that can be run and also collects diagnostic messages from the compilation. + * TODO: Assumes that diagnostics is the same as given to the task, not ideal. + * + * @author lulf + * @since 5.2 + */ +class CompilationTask { + private final JavaCompiler.CompilationTask task; + private final DiagnosticCollector<JavaFileObject> diagnostics; + + CompilationTask(JavaCompiler.CompilationTask task, DiagnosticCollector<JavaFileObject> diagnostics) { + this.task = task; + this.diagnostics = diagnostics; + } + + void call() { + boolean success = task.call(); + if (!success) { + throw new IllegalArgumentException("Compilation diagnostics: " + getDiagnosticMessage()); + } + } + + private String getDiagnosticMessage() { + StringBuilder diagnosticMessages = new StringBuilder(); + for (Diagnostic<?> diagnostic : diagnostics.getDiagnostics()) { + diagnosticMessages.append(diagnostic.getCode()).append("\n"); + diagnosticMessages.append(diagnostic.getKind()).append("\n"); + diagnosticMessages.append(diagnostic.getPosition()).append("\n"); + diagnosticMessages.append(diagnostic.getStartPosition()).append("\n"); + diagnosticMessages.append(diagnostic.getEndPosition()).append("\n"); + diagnosticMessages.append(diagnostic.getSource()).append("\n"); + diagnosticMessages.append(diagnostic.getMessage(null)).append("\n"); + } + return diagnosticMessages.toString(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java new file mode 100644 index 00000000000..403ac61b872 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +import com.yahoo.config.ConfigInstance; + +/** + * Represents a builder that can be instantiated. + * + * @author lulf + * @since 5.2 + */ +public interface CompiledBuilder { + <BUILDER extends ConfigInstance.Builder> BUILDER newInstance(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java new file mode 100644 index 00000000000..24b102321d2 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigCompiler.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +/** + * Interface towards compilers for compiling builders from a config class definition. + * + * @author lulf + * @since 5.2 + */ +public interface ConfigCompiler { + CompiledBuilder compile(ConfigDefinitionClass defClass); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigDefinition.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigDefinition.java new file mode 100644 index 00000000000..42e7ebe29b9 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigDefinition.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +import com.google.common.io.Files; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.codegen.JavaClassBuilder; +import com.yahoo.text.StringUtilities; + +import java.io.File; +import java.io.StringReader; + +/** + * Represents a higher level functionality on a config definition to (in the future) hide the InnerCNode class. + * @author lulf + */ +public class ConfigDefinition { + private final String name; + private final String[] defSchema; + private final InnerCNode cnode; + + public ConfigDefinition(String name, String[] defSchema) { + this.name = name; + this.defSchema = defSchema; + this.cnode = new DefParser(name, new StringReader(StringUtilities.implode(defSchema, "\n"))).getTree(); + } + + // TODO: Remove once no fat bundles are using this. + public ConfigDefinition(InnerCNode targetDef) { + this.name = null; + this.defSchema = null; + this.cnode = targetDef; + } + + public InnerCNode getCNode() { + return cnode; + } + + public ConfigDefinitionClass generateClass() { + File tempDir = Files.createTempDir(); + DefParser parser = new DefParser(name, new StringReader(StringUtilities.implode(defSchema, "\n"))); + JavaClassBuilder builder = new JavaClassBuilder(parser.getTree(), parser.getNormalizedDefinition(), tempDir); + String className = builder.className(); + return new ConfigDefinitionClass(className, builder.javaPackage(), builder.getConfigClass(className)); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigDefinitionClass.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigDefinitionClass.java new file mode 100644 index 00000000000..0820d77612e --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/ConfigDefinitionClass.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +/** + * @author lulf + */ +public class ConfigDefinitionClass { + private final String name; + private final String pkg; + private final String definition; + + ConfigDefinitionClass(String name, String pkg, String definition) { + this.name = name; + this.pkg = pkg; + this.definition = definition; + } + + String getDefinition() { + return definition; + } + + String getSimpleName() { + return name; + + } + + String getName() { + return pkg + "." + name; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java new file mode 100644 index 00000000000..957b4a3d3e0 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/LazyConfigCompiler.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +import com.yahoo.config.ConfigInstance; + +import javax.tools.*; +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Arrays; +import java.util.Locale; + +/** + * Represents a compiler that waits performing the compilation until the requested builder is requested from the + * {@link CompiledBuilder}. + * + * @author lulf + * @since 5.2 + */ +public class LazyConfigCompiler implements ConfigCompiler { + private final File outputDirectory; + private final ClassLoader classLoader; + private final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + + public LazyConfigCompiler(File outputDirectory) { + this.outputDirectory = outputDirectory; + try { + this.classLoader = new URLClassLoader(new URL[]{outputDirectory.toURI().toURL()}); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("Unable to create class loader for directory '" + outputDirectory.getAbsolutePath() + "'", e); + } + } + + @Override + public CompiledBuilder compile(ConfigDefinitionClass defClass) { + Iterable<? extends JavaFileObject> compilationUnits = Arrays.asList(new StringSourceObject(defClass.getName(), defClass.getDefinition())); + DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<>(); + + StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, Locale.ENGLISH, null); + Iterable<String> options = Arrays.asList("-d", outputDirectory.getAbsolutePath()); + JavaCompiler.CompilationTask task = compiler.getTask(null, fileManager, diagnostics, options, null, compilationUnits); + return new LazyCompiledBuilder(classLoader, defClass.getName(), new CompilationTask(task, diagnostics)); + } + + /** + * Lazy implementation of compiled builder that defers compilation until class is requested. + */ + private static class LazyCompiledBuilder implements CompiledBuilder { + private final ClassLoader classLoader; + private final String classUrl; + private final CompilationTask compilationTask; + private LazyCompiledBuilder(ClassLoader classLoader, String classUrl, CompilationTask compilationTask) { + this.classLoader = classLoader; + this.classUrl = classUrl; + this.compilationTask = compilationTask; + } + + @Override + public <BUILDER extends ConfigInstance.Builder> BUILDER newInstance() { + compileBuilder(); + String builderClassUrl = classUrl + "$Builder"; + return loadBuilder(builderClassUrl); + + } + + private void compileBuilder() { + try { + compilationTask.call(); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Error compiling '" + classUrl + "'", e); + } + } + + @SuppressWarnings("unchecked") + private <BUILDER extends ConfigInstance.Builder> BUILDER loadBuilder(String builderClassUrl) { + try { + Class<BUILDER> clazz = (Class<BUILDER>) classLoader.<BUILDER>loadClass(builderClassUrl); + return clazz.newInstance(); + } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) { + throw new RuntimeException("Error creating new instance of '" + builderClassUrl + "'", e); + } + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java new file mode 100644 index 00000000000..01fc0691d2e --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +import javax.tools.SimpleJavaFileObject; +import java.net.URI; + +/** + * Represents an in memory source object that can be compiled. + * + * @author lulf + * @since 5.2 + */ +class StringSourceObject extends SimpleJavaFileObject { + private final String code; + StringSourceObject(String name, String code) { + super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension),Kind.SOURCE); + this.code = code; + } + + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) { + return code; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/package-info.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/package-info.java new file mode 100644 index 00000000000..0a0e85ff47c --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.buildergen; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/vespa/config/package-info.java b/config/src/main/java/com/yahoo/vespa/config/package-info.java new file mode 100644 index 00000000000..9bb8f40a806 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/vespa/config/parser/package-info.java b/config/src/main/java/com/yahoo/vespa/config/parser/package-info.java new file mode 100644 index 00000000000..4835464c7f7 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/parser/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.parser; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java new file mode 100644 index 00000000000..22320ecfa28 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.slime.Inspector; + +import java.io.IOException; + +/** + * Contains info relevant for compression and decompression. + * + * @author lulf + * @since 5.19 + */ +public class CompressionInfo { + private static final String COMPRESSION_TYPE = "compressionType"; + private static final String UNCOMPRESSED_SIZE = "uncompressedSize"; + + public CompressionType getCompressionType() { + return compressionType; + } + public int getUncompressedSize() { + return uncompressedSize; + } + + private final CompressionType compressionType; + private final int uncompressedSize; + + private CompressionInfo(CompressionType compressionType, int uncompressedSize) { + this.compressionType = compressionType; + this.uncompressedSize = uncompressedSize; + } + + public static CompressionInfo uncompressed() { + return new CompressionInfo(CompressionType.UNCOMPRESSED, 0); + } + + public static CompressionInfo create(CompressionType type, int uncompressedSize) { + return new CompressionInfo(type, uncompressedSize); + } + + public static CompressionInfo fromSlime(Inspector field) { + CompressionType type = CompressionType.parse(field.field(COMPRESSION_TYPE).asString()); + int uncompressedSize = (int) field.field(UNCOMPRESSED_SIZE).asLong(); + return new CompressionInfo(type, uncompressedSize); + } + + public void serialize(JsonGenerator jsonGenerator) throws IOException { + jsonGenerator.writeStringField(COMPRESSION_TYPE, compressionType.name()); + jsonGenerator.writeNumberField(UNCOMPRESSED_SIZE, uncompressedSize); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CompressionInfo that = (CompressionInfo) o; + + if (uncompressedSize != that.uncompressedSize) return false; + if (compressionType != that.compressionType) return false; + + return true; + } + + @Override + public int hashCode() { + int result = compressionType.hashCode(); + result = 31 * result + uncompressedSize; + return result; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java new file mode 100644 index 00000000000..bf7e79121ea --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +/** + * @author lulf + * @since 5.18 + */ +public enum CompressionType { + UNCOMPRESSED, LZ4; + public static CompressionType parse(String value) { + for (CompressionType type : CompressionType.values()) { + if (type.name().equals(value)) { + return type; + } + } + return CompressionType.UNCOMPRESSED; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java new file mode 100644 index 00000000000..5e6918c1e88 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.text.Utf8Array; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + +/** + * A config response encapsulates the payload and some meta information. This makes it possible to + * represent the payload in different formats all up to when rendering it to the client. A subclass + * of this must be thread safe, because a response may be cached and, the methods below should be callable + * from multiple request handler threads. + * + * @author lulf + * @since 5.1.14 + */ +public interface ConfigResponse { + + Utf8Array getPayload(); + + List<String> getLegacyPayload(); + + long getGeneration(); + + String getConfigMd5(); + + void serialize(OutputStream os, CompressionType uncompressed) throws IOException; + + default boolean hasEqualConfig(JRTServerConfigRequest request) { + return (getConfigMd5().equals(request.getRequestConfigMd5())); + } + + default boolean hasNewerGeneration(JRTServerConfigRequest request) { + return (getGeneration() > request.getRequestGeneration()); + } + + CompressionInfo getCompressionInfo(); + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java new file mode 100644 index 00000000000..d22628b7b4a --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.slime.*; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** +* @author lulf +* @since 5.3 +*/ +public class DefContent { + private final List<String> data; + + private DefContent(List<String> data) { + this.data = data; + } + + public String[] asStringArray() { + return data.toArray(new String[data.size()]); + } + + public List<String> asList() { + return data; + } + + public String asString() { + return com.yahoo.text.StringUtilities.implode(asStringArray(), "\n"); + } + + static DefContent fromSlime(Inspector data) { + final List<String> lst = new ArrayList<>(); + data.traverse(new ArrayTraverser() { + @Override + public void entry(int idx, Inspector inspector) { + lst.add(inspector.asString()); + } + }); + return new DefContent(lst); + } + + public static DefContent fromClass(Class<? extends ConfigInstance> clazz) { + return fromArray(defSchema(clazz)); + } + + public static DefContent fromList(List<String> def) { + return new DefContent(def); + } + + public static DefContent fromArray(String[] schema) { + return fromList(Arrays.asList(schema)); + } + + /** + * The def file payload of the actual class of the given config + * @param configClass the class of a generated config instance + * @return a String array with the config definition (one line per element) + */ + private static String[] defSchema(Class<? extends ConfigInstance> configClass) { + if (configClass==null) return new String[0]; + try { + Field f = configClass.getField("CONFIG_DEF_SCHEMA"); + return (String[]) f.get(configClass); + } catch (NoSuchFieldException e) { + return new String[0]; + } catch (Exception e) { + throw new ConfigurationRuntimeException(e); + } + } + + public void serialize(final Cursor cursor) { + for (String line : data) { + cursor.addString(line); + } + } + + public boolean isEmpty() { + return data.isEmpty(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java new file mode 100644 index 00000000000..8997173a479 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +/** + * Interface for config requests used by clients. + * + * @author lulf + * @since 5.3 + */ +public interface JRTClientConfigRequest extends JRTConfigRequest { + /** + * Validate config response given by the server. If none is given, or an error occurred, this should return false. + * @return true if valid response, false if not. + */ + boolean validateResponse(); + + /** + * Test whether ot not the returned config has an updated generation. This should return false if no response have + * been given. + * @return true if generation is updated, false if not. + */ + boolean hasUpdatedGeneration(); + + /** + * Return the payload in the response given by the server. The payload will be empty if no response was given. + * @return the config payload. + */ + Payload getNewPayload(); + + /** + * Create a new {@link JRTClientConfigRequest} based on this request based on the same request parameters, but having + * the timeout changed. + * @param timeout server timeout of the new request. + * @return a new {@link JRTClientConfigRequest} instance. + */ + JRTClientConfigRequest nextRequest(long timeout); + + /** + * Test whether or not the returned request is an error. + * @return true if error, false if not. + */ + boolean isError(); + + /** + * Get the generation of the newly provided config. If none has been given, 0 should be returned. + * @return the new generation. + */ + long getNewGeneration(); + + /** + * Get the config md5 of the config returned by the server. Return an empty string if no response has been returned. + * @return a config md5. + */ + String getNewConfigMd5(); + + /** + * For protocols that perform an optimization when no new config has been given, this method will provide the + * payload and hasUpdatedConfig state of the previous request. + * @param payload a config payload of the previous request. + * @param hasUpdatedConfig the hasUpdatedConfig flag of the previous request. + */ + void updateRequestPayload(Payload payload, boolean hasUpdatedConfig); + + /** + * Test whether or not the payload is contained in this response or not. Should return false for error responses as well. + * @return true if empty, false if not. + */ + boolean containsPayload(); + + /** + * Test whether or not the response contains an updated config or not. False if no response has been returned. + * @return true if config is updated, false if not. + */ + boolean hasUpdatedConfig(); + + /** + * Get the {@link Trace} given in the response by the server. The {@link Trace} can be used to add further tracing + * and later printed to provide useful debug info. + * @return a {@link Trace}. + */ + Trace getResponseTrace(); + + /** + * Get config definition content. + * @return def as lines. + */ + DefContent getDefContent(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java new file mode 100644 index 00000000000..3f97f066829 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.impl.JRTConfigSubscription; +import com.yahoo.jrt.Request; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.JRTMethods; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.util.Optional; + +/** + * Represents version 3 config request for config clients. Provides methods for inspecting request and response + * values. + * + * See {@link JRTServerConfigRequestV3} for protocol details. + * + * @author lulf + * @since 5.19 + */ +public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest { + + protected JRTClientConfigRequestV3(ConfigKey<?> key, + String hostname, + DefContent defSchema, + String configMd5, + long generation, + long timeout, + Trace trace, + CompressionType compressionType, + Optional<VespaVersion> vespaVersion) { + super(key, hostname, defSchema, configMd5, generation, timeout, trace, compressionType, vespaVersion); + } + + @Override + protected String getJRTMethodName() { + return JRTMethods.configV3getConfigMethodName; + } + + @Override + protected boolean checkReturnTypes(Request request) { + return JRTMethods.checkV3ReturnTypes(request); + } + + @Override + public Payload getNewPayload() { + CompressionInfo compressionInfo = getResponseData().getCompressionInfo(); + Utf8Array payload = new Utf8Array(request.returnValues().get(1).asData()); + return Payload.from(payload, compressionInfo); + } + + @Override + public long getProtocolVersion() { + return 3; + } + + @Override + public JRTClientConfigRequest nextRequest(long timeout) { + return new JRTClientConfigRequestV3(getConfigKey(), + getClientHostName(), + getDefContent(), + isError() ? getRequestConfigMd5() : newConfMd5(), + isError() ? getRequestGeneration() : newGen(), + timeout, + Trace.createNew(), + requestData.getCompressionType(), + requestData.getVespaVersion()); + } + + public static <T extends ConfigInstance> JRTClientConfigRequest createFromSub(JRTConfigSubscription<T> sub, Trace trace, CompressionType compressionType, Optional<VespaVersion> vespaVersion) { + String hostname = ConfigUtils.getCanonicalHostName(); + ConfigKey<T> key = sub.getKey(); + T i = sub.getConfig(); + return createWithParams(key, + sub.getDefContent(), + hostname, + i != null ? i.getConfigMd5() : "", + sub.getGeneration() != null ? sub.getGeneration() : 0L, + sub.timingValues().getSubscribeTimeout(), + trace, + compressionType, + vespaVersion); + } + + + public static JRTClientConfigRequest createFromRaw(RawConfig config, long serverTimeout, Trace trace, CompressionType compressionType, Optional<VespaVersion> vespaVersion) { + String hostname = ConfigUtils.getCanonicalHostName(); + return createWithParams(config.getKey(), + DefContent.fromList(config.getDefContent()), + hostname, + config.getConfigMd5(), + config.getGeneration(), + serverTimeout, + trace, + compressionType, + vespaVersion); + } + + + public static JRTClientConfigRequest createWithParams(ConfigKey<?> reqKey, + DefContent defContent, + String hostname, + String configMd5, + long generation, + long serverTimeout, + Trace trace, + CompressionType compressionType, + Optional<VespaVersion> vespaVersion) { + return new JRTClientConfigRequestV3(reqKey, + hostname, + defContent, + configMd5, + generation, + serverTimeout, + trace, + compressionType, + vespaVersion); + } + + @Override + public Optional<VespaVersion> getVespaVersion() { + return requestData.getVespaVersion(); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java new file mode 100644 index 00000000000..98a4ddbf7f1 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.jrt.Request; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.Optional; + +/** + * Common interface for jrt config requests available both at server and client. + * @author lulf + * @since 5.3 + */ +public interface JRTConfigRequest { + /** + * Get the config key of the config request. + * @return a {@link ConfigKey}. + */ + ConfigKey<?> getConfigKey(); + + /** + * Perform request parameter validation of this config request. This method should be called before fetching + * any kind of config protocol-specific parameter. + * @return true if valid, false if not. + */ + boolean validateParameters(); + + /** + * Get the config md5 of the config request. Return an empty string if no response has been returned. + * @return a config md5. + */ + String getRequestConfigMd5(); + + /** + * Get the generation of the requested config. If none has been given, 0 should be returned. + * @return the generation in the request. + */ + long getRequestGeneration(); + + /** + * Get the JRT request object for this config request. + * TODO: This method leaks the internal jrt stuff :( + * @return a {@link Request} object. + */ + Request getRequest(); + + /** + * Get a short hand description of this request. + * @return a short description + */ + String getShortDescription(); + + /** + * Get the error code of this request + * @return the error code as defined in {@link com.yahoo.vespa.config.ErrorCode}. + */ + int errorCode(); + + /** + * Return the error message of this request, mostly corresponding to the {@link com.yahoo.vespa.config.ErrorCode}. + * @return the error message. + */ + String errorMessage(); + + /** + * Get the server timeout of this request. + * @return the timeout given to the server + */ + long getTimeout(); + + /** + * Get the config protocol version + * @return a protocol version number. + */ + long getProtocolVersion(); + + /** + * Get the wanted generation for this request. + * @return a generation that client would like. + */ + long getWantedGeneration(); + + /** + * Get the host name of the client that is requesting config. + * @return hostname of the client. + */ + String getClientHostName(); + + /** + * Get the Vespa version of the client that initiated the request + * @return Vespa version of the client + */ + Optional<VespaVersion> getVespaVersion(); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java new file mode 100644 index 00000000000..583e4ba39f5 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.impl.JRTConfigSubscription; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.util.*; +import java.util.logging.Logger; + +/** + * To hide JRT implementations. + * + * @author lulf + * @since 5.3 + */ +public class JRTConfigRequestFactory { + + public static final String VESPA_CONFIG_PROTOCOL_VERSION = "VESPA_CONFIG_PROTOCOL_VERSION"; + private final static Logger log = Logger.getLogger(JRTConfigRequestFactory.class.getName()); + private static final CompressionType compressionType = getCompressionType(); + private static final String VESPA_CONFIG_PROTOCOL_COMPRESSION = "VESPA_CONFIG_PROTOCOL_COMPRESSION"; + public static final String VESPA_VERSION = "VESPA_VERSION"; + + public static <T extends ConfigInstance> JRTClientConfigRequest createFromSub(JRTConfigSubscription<T> sub) { + // TODO: Get trace from caller + return JRTClientConfigRequestV3.createFromSub(sub, Trace.createNew(), compressionType, getVespaVersion()); + } + + public static JRTClientConfigRequest createFromRaw(RawConfig config, long serverTimeout) { + // TODO: Get trace from caller + return JRTClientConfigRequestV3.createFromRaw(config, serverTimeout, Trace.createNew(), compressionType, getVespaVersion()); + } + + public static String getProtocolVersion() { + return "3"; + } + + static String getProtocolVersion(String env, String yinstEnv, String property) { + return ConfigUtils.getEnvValue("3", env, yinstEnv, property); + } + + public static Set<Long> supportedProtocolVersions() { + return Collections.singleton(3l); + } + + public static CompressionType getCompressionType() { + return getCompressionType(System.getenv(VESPA_CONFIG_PROTOCOL_COMPRESSION), + System.getenv("services__config_protocol_compression"), + System.getProperty(VESPA_CONFIG_PROTOCOL_COMPRESSION)); + } + + static CompressionType getCompressionType(String env, String yinstEnv, String property) { + return CompressionType.valueOf(ConfigUtils.getEnvValue("LZ4", env, yinstEnv, property)); + } + + static Optional<VespaVersion> getVespaVersion() { + final String envValue = ConfigUtils.getEnvValue("", System.getenv(VESPA_VERSION), System.getProperty(VESPA_VERSION)); + if (envValue != null && !envValue.isEmpty()) { + return Optional.of(VespaVersion.fromString(envValue)); + } + return Optional.of(getCompiledVespaVersion()); + } + + static VespaVersion getCompiledVespaVersion() { + return VespaVersion.fromString(String.format("%d.%d.%d", + com.yahoo.vespa.config.VespaVersion.major, + com.yahoo.vespa.config.VespaVersion.minor, + com.yahoo.vespa.config.VespaVersion.micro)); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java new file mode 100644 index 00000000000..e201f1f08ef --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequest.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.vespa.config.GetConfigRequest; + +/** + * Interface for config requests at the server end point. + * + * @author lulf + * @since 5.3 + */ +public interface JRTServerConfigRequest extends JRTConfigRequest, GetConfigRequest { + /** + * Notify this request that its delayed due to no new config being available at this point. The value + * provided in this function should be returned when calling {@link #isDelayedResponse()}. + * @param delayedResponse true if response is delayed, false if not. + */ + void setDelayedResponse(boolean delayedResponse); + + /** + * Signal error when handling this request. The error should be reflected in the request state and propagated + * back to the client. + * @param errorCode error code, as described in {@link com.yahoo.vespa.config.ErrorCode}. + * @param message message to display for this error, typically printed by client. + */ + void addErrorResponse(int errorCode, String message); + + /** + * Signal that the request was handled and provide return values typically needed by a client. + * @param payload The config payload that the client should receive. + * @param generation The config generation of the given payload. + * @param configMd5 The md5sum of the given payload. + */ + void addOkResponse(Payload payload, long generation, String configMd5); + + /** + * Get the current config md5 of the client config. + * @return a config md5. + */ + String getRequestConfigMd5(); + + /** + * Get the current config generation of the client config. + * @return the current config generation. + */ + long getRequestGeneration(); + + /** + * Check whether or not this request is delayed. + * @return true if delayed, false if not. + */ + boolean isDelayedResponse(); + + /** + * Get the request trace for this request. The trace can be used to trace config execution to provide useful + * debug info in production environments. + * @return a {@link Trace} instance. + */ + Trace getRequestTrace(); + + /** + * Extract the appropriate payload for this request type for a given config response. + * + * @param response {@link ConfigResponse} to get payload from. + * @return A {@link Payload} that satisfies this request format. + */ + Payload payloadFromResponse(ConfigResponse response); +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java new file mode 100644 index 00000000000..54dcd649d69 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.jrt.DataValue; +import com.yahoo.jrt.Request; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * The V3 config protocol implemented on the server side. The V3 protocol uses 2 fields JRT + * + * * A metadata field containing json data describing config generation, md5 and compression info + * * A data field containing compressed or uncompressed json config payload. This field can be empty if the payload + * has not changed since last request, triggering an optimization at the client where the previous payload is used instead. + * + * The implementation of addOkResponse is optimized for doing as little copying of payload data as possible, ensuring + * that we get a lower memory footprint. + * + * @author lulf + * @since 5.19 + */ +public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest { + + protected JRTServerConfigRequestV3(Request request) { + super(request); + } + + @Override + public void addOkResponse(Payload payload, long generation, String configMd5) { + boolean changedConfig = !configMd5.equals(getRequestConfigMd5()); + boolean changedConfigAndNewGeneration = changedConfig && ConfigUtils.isGenerationNewer(generation, getRequestGeneration()); + Payload responsePayload = payload.withCompression(getCompressionType()); + ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream(4096); + try { + JsonGenerator jsonGenerator = createJsonGenerator(byteArrayOutputStream); + jsonGenerator.writeStartObject(); + addCommonReturnValues(jsonGenerator); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_MD5, configMd5); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIG_GENERATION, generation); + jsonGenerator.writeObjectFieldStart(SlimeResponseData.RESPONSE_COMPRESSION_INFO); + if (responsePayload == null) { + throw new RuntimeException("Payload is null for ' " + this + ", not able to create response"); + } + CompressionInfo compressionInfo = responsePayload.getCompressionInfo(); + // If payload is not being sent, we must adjust compression info to avoid client confusion. + if (!changedConfigAndNewGeneration) { + compressionInfo = CompressionInfo.create(compressionInfo.getCompressionType(), 0); + } + compressionInfo.serialize(jsonGenerator); + jsonGenerator.writeEndObject(); + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, getConfigKey() + ": response dataXXXXX" + payload.withCompression(CompressionType.UNCOMPRESSED) + "XXXXX"); + } + jsonGenerator.writeEndObject(); + jsonGenerator.close(); + } catch (IOException e) { + throw new IllegalArgumentException("Could not add OK response for " + this); + } + request.returnValues().add(createResponseValue(byteArrayOutputStream)); + if (changedConfigAndNewGeneration) { + request.returnValues().add(new DataValue(responsePayload.getData().getBytes())); + } else { + request.returnValues().add(new DataValue(new byte[0])); + } + } + + @Override + public long getProtocolVersion() { + return 3; + } + + public static JRTServerConfigRequestV3 createFromRequest(Request req) { + return new JRTServerConfigRequestV3(req); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java b/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java new file mode 100644 index 00000000000..d73d0e9a14c --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import java.io.ByteArrayOutputStream; + +/** + * Subclass of {@link java.io.ByteArrayOutputStream} that gives effective {@link #toByteArray()} method. + * + * @author lulf + * @since 5.19 + */ +class NoCopyByteArrayOutputStream extends ByteArrayOutputStream { + public NoCopyByteArrayOutputStream() { + super(); + } + + public NoCopyByteArrayOutputStream(int initialSize) { + super(initialSize); + } + + @Override + public byte[] toByteArray() { + return buf; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/Payload.java b/config/src/main/java/com/yahoo/vespa/config/protocol/Payload.java new file mode 100644 index 00000000000..aa0bf374363 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/Payload.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.text.Utf8Array; +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.LZ4PayloadCompressor; + +import java.util.Objects; + +/** + * An immutable config payload + * + * @author musum + * @author bratseth + */ +public class Payload { + + private final Utf8Array data; + private final CompressionInfo compressionInfo; + private final static LZ4PayloadCompressor compressor = new LZ4PayloadCompressor(); + + private Payload(ConfigPayload payload) { + this.data = payload.toUtf8Array(true); + this.compressionInfo = CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength()); + } + + private Payload(Utf8Array payload, CompressionInfo compressionInfo) { + Objects.requireNonNull(payload, "Payload"); + Objects.requireNonNull(compressionInfo, "CompressionInfo"); + this.data = payload; + this.compressionInfo = compressionInfo; + } + + public static Payload from(ConfigPayload payload) { + return new Payload(payload); + } + + /** Creates an uncompressed payload from a string */ + public static Payload from(String payload) { + return new Payload(new Utf8String(payload), CompressionInfo.uncompressed()); + } + + public static Payload from(String payload, CompressionInfo compressionInfo) { + return new Payload(new Utf8String(payload), compressionInfo); + } + + /** Creates an uncompressed payload from an Utf8Array */ + public static Payload from(Utf8Array payload) { + return new Payload(payload, CompressionInfo.uncompressed()); + } + + public static Payload from(Utf8Array payload, CompressionInfo compressionInfo) { + return new Payload(payload, compressionInfo); + } + + public Utf8Array getData() { return data; } + + /** Returns a copy of this payload where the data is compressed using the given compression */ + public Payload withCompression(CompressionType requestedCompression) { + CompressionType responseCompression = compressionInfo.getCompressionType(); + if (requestedCompression == CompressionType.UNCOMPRESSED && responseCompression == CompressionType.LZ4) { + byte[] buffer = new byte[compressionInfo.getUncompressedSize()]; + compressor.decompress(data.getBytes(), buffer); + Utf8Array data = new Utf8Array(buffer); + CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, compressionInfo.getUncompressedSize()); + return Payload.from(data, info); + } else if (requestedCompression == CompressionType.LZ4 && responseCompression == CompressionType.UNCOMPRESSED) { + Utf8Array data = new Utf8Array(compressor.compress(this.data.getBytes())); + CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, this.data.getByteLength()); + return Payload.from(data, info); + } else { + return Payload.from(data, compressionInfo); + } + } + + public CompressionInfo getCompressionInfo() { return compressionInfo; } + + @Override + public String toString() { + if (compressionInfo.getCompressionType() == CompressionType.UNCOMPRESSED) + return data.toString(); + else + return withCompression(CompressionType.UNCOMPRESSED).toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Payload other = (Payload) o; + return this.compressionInfo.equals(other.compressionInfo) && this.data.equals(other.data); + } + + @Override + public int hashCode() { + return data.hashCode() + 31 * compressionInfo.hashCode(); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java new file mode 100644 index 00000000000..d44f9190b42 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ErrorCode; + +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Static utility methods for verifying common request properties. + * + * @author lulf + * @since 5.3 + */ +public class RequestValidation { + private static final Logger log = Logger.getLogger(RequestValidation.class.getName()); + + private static final Pattern md5Pattern = Pattern.compile("[0-9a-zA-Z]+"); + + public static int validateRequest(JRTConfigRequest request) { + ConfigKey<?> key = request.getConfigKey(); + if (!RequestValidation.verifyName(key.getName())) { + log.log(LogLevel.INFO, "Illegal name '" + key.getName() + "'"); + return ErrorCode.ILLEGAL_NAME; + } + if (!RequestValidation.verifyNamespace(key.getNamespace())) { + log.log(LogLevel.INFO, "Illegal name space '" + key.getNamespace() + "'"); + return ErrorCode.ILLEGAL_NAME_SPACE; + } + if (!RequestValidation.verifyMd5(key.getMd5())) { + log.log(LogLevel.INFO, "Illegal md5 sum '" + key.getNamespace() + "'"); + return ErrorCode.ILLEGAL_DEF_MD5; + } + if (!RequestValidation.verifyMd5(request.getRequestConfigMd5())) { + log.log(LogLevel.INFO, "Illegal config md5 '" + request.getRequestConfigMd5() + "'"); + return ErrorCode.ILLEGAL_CONFIG_MD5; + } + if (!RequestValidation.verifyGeneration(request.getRequestGeneration())) { + log.log(LogLevel.INFO, "Illegal generation '" + request.getRequestGeneration() + "'"); + return ErrorCode.ILLEGAL_GENERATION; + } + if (!RequestValidation.verifyGeneration(request.getWantedGeneration())) { + log.log(LogLevel.INFO, "Illegal wanted generation '" + request.getWantedGeneration() + "'"); + return ErrorCode.ILLEGAL_GENERATION; + } + if (!RequestValidation.verifyTimeout(request.getTimeout())) { + log.log(LogLevel.INFO, "Illegal timeout '" + request.getTimeout() + "'"); + return ErrorCode.ILLEGAL_TIMEOUT; + } + if (!RequestValidation.verifyHostname(request.getClientHostName())) { + log.log(LogLevel.INFO, "Illegal client host name '" + request.getClientHostName() + "'"); + return ErrorCode.ILLEGAL_CLIENT_HOSTNAME; + } + return 0; + } + + public static boolean verifyName(String name) { + Matcher m = ConfigDefinition.namePattern.matcher(name); + return m.matches(); + } + + public static boolean verifyMd5(String md5) { + if (md5.equals("")) { + return true; // Empty md5 is ok (e.g. upon getconfig from command line tools) + } else if (md5.length() != 32) { + return false; + } + Matcher m = md5Pattern.matcher(md5); + return m.matches(); + } + + public static boolean verifyTimeout(Long timeout) { + return (timeout > 0); + } + + public static boolean verifyGeneration(Long generation) { + return (generation >= 0); + } + + private static boolean verifyNamespace(String namespace) { + Matcher m = ConfigDefinition.namespacePattern.matcher(namespace); + return m.matches(); + } + + private static boolean verifyHostname(String clientHostName) { + return !("".equals(clientHostName)); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java new file mode 100644 index 00000000000..2cb61cac427 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java @@ -0,0 +1,230 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.jrt.*; +import com.yahoo.slime.*; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ErrorCode; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Base class for new generation of config requests based on {@link Slime}. Allows for some customization of + * payload encoding and decoding, as well as adding extra request/response fields. + * + * @author lulf + * @since 5.18 + */ +public abstract class SlimeClientConfigRequest implements JRTClientConfigRequest { + + protected static final Logger log = Logger.getLogger(SlimeClientConfigRequest.class.getName()); + + protected final SlimeRequestData requestData; + private final SlimeResponseData responseData; + + protected final Request request; + + protected SlimeClientConfigRequest(ConfigKey<?> key, + String hostname, + DefContent defSchema, + String configMd5, + long generation, + long timeout, + Trace trace, + CompressionType compressionType, + Optional<VespaVersion> vespaVersion) { + Slime data = SlimeRequestData.encodeRequest(key, + hostname, + defSchema, + configMd5, + generation, + timeout, + trace, + getProtocolVersion(), + compressionType, + vespaVersion); + Request jrtReq = new Request(getJRTMethodName()); + jrtReq.parameters().add(new StringValue(encodeAsUtf8String(data, true))); + + this.requestData = new SlimeRequestData(jrtReq, data); + this.responseData = new SlimeResponseData(jrtReq); + this.request = jrtReq; + } + + protected abstract String getJRTMethodName(); + + protected static String encodeAsUtf8String(Slime data, boolean compact) { + ByteArrayOutputStream baos = new NoCopyByteArrayOutputStream(); + try { + new JsonFormat(compact).encode(baos, data); + } catch (IOException e) { + throw new RuntimeException("Unable to encode config request", e); + } + return Utf8.toString(baos.toByteArray()); + } + + public ConfigKey<?> getConfigKey() { + return requestData.getConfigKey(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("request='").append(getConfigKey()) + .append(",").append(getClientHostName()) + .append(",").append(getRequestConfigMd5()) + .append(",").append(getRequestGeneration()) + .append(",").append(getTimeout()) + .append(",").append(getVespaVersion()).append("'\n"); + sb.append("response='").append(getNewConfigMd5()) + .append(",").append(getNewGeneration()) + .append("'\n"); + return sb.toString(); + } + + @Override + public String getClientHostName() { + return requestData.getClientHostName(); + } + + @Override + public long getWantedGeneration() { + return requestData.getWantedGeneration(); + } + + @Override + public Request getRequest() { + return request; + } + + @Override + public int errorCode() { + return request.errorCode(); + } + + @Override + public String errorMessage() { + return request.errorMessage(); + } + + @Override + public String getShortDescription() { + return toString(); + } + + @Override + public boolean hasUpdatedGeneration() { + long prevGen = getRequestGeneration(); + long newGen = getNewGeneration(); + return ConfigUtils.isGenerationNewer(newGen, prevGen); + } + + @Override + public long getTimeout() { + return requestData.getTimeout(); + } + + protected String newConfMd5() { + String newMd5 = getNewConfigMd5(); + if ("".equals(newMd5)) return getRequestConfigMd5(); + return newMd5; + } + + protected long newGen() { + long newGen = getNewGeneration(); + if (newGen==0) return getRequestGeneration(); + return newGen; + } + + @Override + public DefContent getDefContent() { + return requestData.getSchema(); + } + + @Override + public boolean isError() { + return request.isError(); + } + + @Override + public void updateRequestPayload(Payload payload, boolean hasUpdatedConfig) { + // This protocol sends payload in all cases, so ignore this. + } + + @Override + public boolean containsPayload() { + return false; + } + + @Override + public boolean hasUpdatedConfig() { + String respMd5 = getNewConfigMd5(); + return !respMd5.equals("") && !getRequestConfigMd5().equals(respMd5); + } + + @Override + public Trace getResponseTrace() { + return responseData.getResponseTrace(); + } + + @Override + public String getRequestConfigMd5() { + return requestData.getRequestConfigMd5(); + } + + @Override + public boolean validateResponse() { + if (request.isError()) { + return false; + } else if (request.returnValues().size() == 0) { + return false; + } else if (!checkReturnTypes(request)) { + log.warning("Invalid return types for config response: " + errorMessage()); + return false; + } + if (hasUpdatedConfig() && ! hasUpdatedGeneration()) { + request.setError(ErrorCode.OUTDATED_CONFIG, "Config payload has changed (old config md5:" + + getRequestConfigMd5() + ", new config md5: " + getNewConfigMd5() +"), but new generation " + + getNewGeneration() + " is not newer than current generation " + getRequestGeneration() + "."); + return false; + } + return true; + } + + @Override + public boolean validateParameters() { + int errorCode = RequestValidation.validateRequest(this); + return (errorCode == 0); + } + + protected abstract boolean checkReturnTypes(Request request); + + @Override + public String getNewConfigMd5() { + return responseData.getResponseConfigMd5(); + } + + @Override + public long getNewGeneration() { + return responseData.getResponseConfigGeneration(); + } + + @Override + public long getRequestGeneration() { + return requestData.getRequestGeneration(); + } + + protected SlimeResponseData getResponseData() { + return responseData; + } + + public Optional<VespaVersion> getVespaVersion() { + return requestData.getVespaVersion(); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java new file mode 100644 index 00000000000..d5c4fc638ee --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.ConfigFileFormat; +import com.yahoo.vespa.config.ConfigPayload; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.List; + +/** + * Class for serializing config responses based on {@link com.yahoo.slime.Slime} implementing the {@link ConfigResponse} interface. + * + * @author lulf + * @since 5.1 + */ +public class SlimeConfigResponse implements ConfigResponse { + + private final Utf8Array payload; + private final CompressionInfo compressionInfo; + private final InnerCNode targetDef; + private final long generation; + private final String configMd5; + + public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, InnerCNode targetDef, long generation, String configMd5) { + Utf8Array data = payload.toUtf8Array(true); + return new SlimeConfigResponse(data, targetDef, generation, configMd5, CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength())); + } + + public SlimeConfigResponse(Utf8Array payload, InnerCNode targetDef, long generation, String configMd5, CompressionInfo compressionInfo) { + this.payload = payload; + this.targetDef = targetDef; + this.generation = generation; + this.configMd5 = configMd5; + this.compressionInfo = compressionInfo; + } + + @Override + public Utf8Array getPayload() { + return payload; + } + + @Override + public List<String> getLegacyPayload() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ConfigFileFormat format = new ConfigFileFormat(targetDef); + Payload v1payload = Payload.from(payload, compressionInfo).withCompression(CompressionType.UNCOMPRESSED); + try { + ConfigPayload.fromUtf8Array(v1payload.getData()).serialize(baos, format); + return Arrays.asList(baos.toString("UTF-8").split("\\n")); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public long getGeneration() { + return generation; + } + + @Override + public String getConfigMd5() { + return configMd5; + } + + @Override + public void serialize(OutputStream os, CompressionType type) throws IOException { + os.write(Payload.from(payload, compressionInfo).withCompression(type).getData().getBytes()); + } + + @Override + public String toString() { + return "generation=" + generation + "\n" + + "configmd5=" + configMd5 + "\n" + + Payload.from(payload, compressionInfo).withCompression(CompressionType.UNCOMPRESSED); + } + + @Override + public CompressionInfo getCompressionInfo() { return compressionInfo; } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java new file mode 100644 index 00000000000..80c050b9b61 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.jrt.Request; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.Optional; + +/** + * Contains slime request data objects. Provides methods for reading various fields from slime request data. All + * data is read lazily. + * +* @author lulf +* @since 5.18 +*/ +class SlimeRequestData { + + private static final String REQUEST_VERSION = "version"; + private static final String REQUEST_DEF_NAME = "defName"; + private static final String REQUEST_DEF_NAMESPACE = "defNamespace"; + private static final String REQUEST_DEF_CONTENT = "defContent"; + private static final String REQUEST_CLIENT_CONFIGID = "configId"; + private static final String REQUEST_CLIENT_HOSTNAME = "clientHostname"; + private static final String REQUEST_CURRENT_GENERATION = "currentGeneration"; + private static final String REQUEST_WANTED_GENERATION = "wantedGeneration"; + private static final String REQUEST_CONFIG_MD5 = "configMD5"; + private static final String REQUEST_TRACE = "trace"; + private static final String REQUEST_TIMEOUT = "timeout"; + private static final String REQUEST_DEF_MD5 = "defMD5"; + private static final String REQUEST_COMPRESSION_TYPE = "compressionType"; + private static final String REQUEST_VESPA_VERSION = "vespaVersion"; + + private final Request request; + private Slime data = null; + + SlimeRequestData(Request request) { + this.request = request; + } + + SlimeRequestData(Request request, Slime data) { + this.request = request; + this.data = data; + } + + private Slime getData() { + if (data == null) { + data = new JsonDecoder().decode(new Slime(), Utf8.toBytes(request.parameters().get(0).asString())); + } + return data; + } + + Inspector getRequestField(String requestField) { + return getData().get().field(requestField); + } + + ConfigKey<?> getConfigKey() { + return ConfigKey.createFull(getRequestField(REQUEST_DEF_NAME).asString(), + getRequestField(REQUEST_CLIENT_CONFIGID).asString(), + getRequestField(REQUEST_DEF_NAMESPACE).asString(), + getRequestField(REQUEST_DEF_MD5).asString()); + } + + DefContent getSchema() { + Inspector content = getRequestField(REQUEST_DEF_CONTENT); + return DefContent.fromSlime(content); + } + + String getClientHostName() { + return getRequestField(REQUEST_CLIENT_HOSTNAME).asString(); + } + + long getWantedGeneration() { + return getRequestField(REQUEST_WANTED_GENERATION).asLong(); + } + + long getTimeout() { + return getRequestField(REQUEST_TIMEOUT).asLong(); + } + + String getRequestConfigMd5() { + return getRequestField(REQUEST_CONFIG_MD5).asString(); + } + + long getRequestGeneration() { + return getRequestField(REQUEST_CURRENT_GENERATION).asLong(); + } + + static Slime encodeRequest(ConfigKey<?> key, + String hostname, + DefContent defSchema, + String configMd5, + long generation, + long timeout, + Trace trace, + long protocolVersion, + CompressionType compressionType, + Optional<VespaVersion> vespaVersion) { + Slime data = new Slime(); + Cursor request = data.setObject(); + request.setLong(REQUEST_VERSION, protocolVersion); + request.setString(REQUEST_DEF_NAME, key.getName()); + request.setString(REQUEST_DEF_NAMESPACE, key.getNamespace()); + request.setString(REQUEST_DEF_MD5, key.getMd5()); + request.setString(REQUEST_CLIENT_CONFIGID, key.getConfigId()); + request.setString(REQUEST_CLIENT_HOSTNAME, hostname); + defSchema.serialize(request.setArray(REQUEST_DEF_CONTENT)); + request.setString(REQUEST_CONFIG_MD5, configMd5); + request.setLong(REQUEST_CURRENT_GENERATION, generation); + request.setLong(REQUEST_WANTED_GENERATION, 0l); + request.setLong(REQUEST_TIMEOUT, timeout); + request.setString(REQUEST_COMPRESSION_TYPE, compressionType.name()); + if (vespaVersion.isPresent()) { + request.setString(REQUEST_VESPA_VERSION, vespaVersion.get().toString()); + } + trace.serialize(request.setObject(REQUEST_TRACE)); + return data; + } + + Trace getRequestTrace() { + return Trace.fromSlime(getRequestField(REQUEST_TRACE)); + } + + public CompressionType getCompressionType() { + Inspector field = getRequestField(REQUEST_COMPRESSION_TYPE); + return field.valid() ? CompressionType.parse(field.asString()) : CompressionType.UNCOMPRESSED; + } + + public Optional<VespaVersion> getVespaVersion() { + String versionString = getRequestField(REQUEST_VESPA_VERSION).asString(); // will be "" if not set, never null + return versionString.isEmpty() ? Optional.<VespaVersion>empty() : Optional.of(VespaVersion.fromString(versionString)); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java new file mode 100644 index 00000000000..8899658730c --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeResponseData.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.jrt.Request; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; + +/** + * Contains response data for a slime response and methods for decoding the response data that + * are common to all {@link Slime} based config requests. + * + * @author lulf + * @since 5.18 + */ +class SlimeResponseData { + static final String RESPONSE_VERSION = "version"; + static final String RESPONSE_DEF_NAME = "defName"; + static final String RESPONSE_DEF_NAMESPACE = "defNamespace"; + static final String RESPONSE_DEF_MD5 = "defMD5"; + static final String RESPONSE_CONFIGID = "configId"; + static final String RESPONSE_CLIENT_HOSTNAME = "clientHostname"; + static final String RESPONSE_TRACE = "trace"; + static final String RESPONSE_CONFIG_MD5 = "configMD5"; + static final String RESPONSE_CONFIG_GENERATION = "generation"; + static final String RESPONSE_COMPRESSION_INFO = "compressionInfo"; + + private final Request request; + private Slime data = null; + + SlimeResponseData(Request request) { + this.request = request; + } + + private Slime getData() { + if (request.returnValues().size() > 0) { + if (data == null) { + data = new JsonDecoder().decode(new Slime(), Utf8.toBytes(request.returnValues().get(0).asString())); + } + return data; + } else { + return new Slime(); + } + } + + Inspector getResponseField(String responseTrace) { + return getData().get().field(responseTrace); + } + + long getResponseConfigGeneration() { + Inspector inspector = getResponseField(RESPONSE_CONFIG_GENERATION); + return inspector.valid() ? inspector.asLong() : -1; + } + + Trace getResponseTrace() { + Inspector trace = getResponseField(RESPONSE_TRACE); + return trace.valid() ? Trace.fromSlime(trace) : Trace.createDummy(); + } + + String getResponseConfigMd5() { + Inspector inspector = getResponseField(RESPONSE_CONFIG_MD5); + return inspector.valid() ? inspector.asString() : ""; + } + + CompressionInfo getCompressionInfo() { + return CompressionInfo.fromSlime(getResponseField(RESPONSE_COMPRESSION_INFO)); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java new file mode 100644 index 00000000000..8eabcd7eb26 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java @@ -0,0 +1,206 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.jrt.*; +import com.yahoo.slime.*; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.ErrorCode; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Base class for new generation of config requests based on {@link Slime}. Allows for some customization of + * payload encoding and decoding, as well as adding extra request/response fields. Used by both V2 and V3 + * config protocol. + * + * @author lulf + * @since 5.18 + */ +abstract class SlimeServerConfigRequest implements JRTServerConfigRequest { + + protected static final Logger log = Logger.getLogger(SlimeServerConfigRequest.class.getName()); + + private static final JsonFactory jsonFactory = new JsonFactory(); + + private final SlimeRequestData requestData; + + // Response values + private boolean isDelayed = false; + private Trace requestTrace = null; + protected final Request request; + + protected SlimeServerConfigRequest(Request request) { + this.requestData = new SlimeRequestData(request); + this.request = request; + } + + protected static JsonGenerator createJsonGenerator(ByteArrayOutputStream byteArrayOutputStream) throws IOException { + return jsonFactory.createGenerator(byteArrayOutputStream); + } + + @Override + public ConfigKey<?> getConfigKey() { + return requestData.getConfigKey(); + } + + @Override + public DefContent getDefContent() { + return getSchema(); + } + + @Override + public boolean noCache() { + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("request='").append(getConfigKey()) + .append(",").append(getClientHostName()) + .append(",").append(getRequestConfigMd5()) + .append(",").append(getRequestGeneration()) + .append(",").append(getTimeout()).append("'\n"); + return sb.toString(); + } + + @Override + public Payload payloadFromResponse(ConfigResponse response) { + return Payload.from(response.getPayload(), response.getCompressionInfo()); + } + + private DefContent getSchema() { + return requestData.getSchema(); + } + + @Override + public long getWantedGeneration() { + return requestData.getWantedGeneration(); + } + + @Override + public String getClientHostName() { + return requestData.getClientHostName(); + } + + public Trace getRequestTrace() { + if (requestTrace == null) { + requestTrace = requestData.getRequestTrace(); + } + return requestTrace; + } + + @Override + public Request getRequest() { + return request; + } + + @Override + public boolean validateParameters() { + int errorCode = RequestValidation.validateRequest(this); + if (errorCode != 0) { + addErrorResponse(errorCode); + } + return (errorCode == 0); + } + + @Override + public String getRequestConfigMd5() { + return requestData.getRequestConfigMd5(); + } + + private void addErrorResponse(int errorCode) { + addErrorResponse(errorCode, ErrorCode.getName(errorCode)); + } + + @Override + public void setDelayedResponse(boolean delayedResponse) { + this.isDelayed = delayedResponse; + } + + @Override + public void addErrorResponse(int errorCode, String name) { + ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream(); + try { + JsonGenerator jsonWriter = jsonFactory.createGenerator(byteArrayOutputStream); + jsonWriter.writeStartObject(); + addCommonReturnValues(jsonWriter); + jsonWriter.writeEndObject(); + jsonWriter.close(); + } catch (IOException e) { + throw new IllegalArgumentException("Could not add error response for " + this); + } + request.setError(errorCode, name); + request.returnValues().add(createResponseValue(byteArrayOutputStream)); + } + + protected static Value createResponseValue(ByteArrayOutputStream byteArrayOutputStream) { + return new StringValue(new Utf8Array(byteArrayOutputStream.toByteArray())); + } + + protected void addCommonReturnValues(JsonGenerator jsonGenerator) throws IOException { + ConfigKey<?> key = requestData.getConfigKey(); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_VERSION, getProtocolVersion()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAME, key.getName()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAMESPACE, key.getNamespace()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_MD5, key.getMd5()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIGID, key.getConfigId()); + setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CLIENT_HOSTNAME, requestData.getClientHostName()); + jsonGenerator.writeFieldName(SlimeResponseData.RESPONSE_TRACE); + jsonGenerator.writeRawValue(getRequestTrace().toString(true)); + } + + protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, String value) throws IOException { + jsonGenerator.writeStringField(fieldName, value); + } + + protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, long value) throws IOException { + jsonGenerator.writeNumberField(fieldName, value); + } + + @Override + public long getRequestGeneration() { + return requestData.getRequestGeneration(); + } + + @Override + public boolean isDelayedResponse() { + return isDelayed; + } + + @Override + public int errorCode() { + return request.errorCode(); + } + + @Override + public String errorMessage() { + return request.errorMessage(); + } + + @Override + public String getShortDescription() { + return toString(); + } + + protected CompressionType getCompressionType() { + return requestData.getCompressionType(); + } + + @Override + public long getTimeout() { + return requestData.getTimeout(); + } + + @Override + public Optional<VespaVersion> getVespaVersion() { + return requestData.getVespaVersion(); + } + +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java new file mode 100644 index 00000000000..a4074a43a81 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.yolean.trace.TraceNode; + +/** + * Deserializing from a {@link Inspector} (slime) representation to a {@link TraceNode} + * + * @author lulf + * @since 5.5 + */ +public class SlimeTraceDeserializer { + private final Inspector entry; + public SlimeTraceDeserializer(Inspector inspector) { + this.entry = inspector; + } + + public TraceNode deserialize() { + return deserialize(entry); + } + + private static TraceNode deserialize(Inspector entry) { + Object payload = decodePayload(entry.field(SlimeTraceSerializer.PAYLOAD)); + long timestamp = decodeTimestamp(entry.field(SlimeTraceSerializer.TIMESTAMP)); + final TraceNode node = new TraceNode(payload, timestamp); + Inspector children = entry.field(SlimeTraceSerializer.CHILDREN); + children.traverse(new ArrayTraverser() { + @Override + public void entry(int idx, Inspector inspector) { + node.add(deserialize(inspector)); + } + }); + return node; + } + + private static long decodeTimestamp(Inspector entry) { + return entry.asLong(); + } + + private static Object decodePayload(Inspector entry) { + switch (entry.type()) { + case STRING: + return entry.asString(); + case LONG: + return entry.asLong(); + case BOOL: + return entry.asBool(); + case DOUBLE: + return entry.asDouble(); + case DATA: + return entry.asData(); + default: + return null; + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java new file mode 100644 index 00000000000..ea4151e9e36 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.slime.Cursor; +import com.yahoo.yolean.trace.TraceNode; +import com.yahoo.yolean.trace.TraceVisitor; + +import java.util.Iterator; +import java.util.Stack; + +/** + * Serialize a {@link TraceNode} to {@link com.yahoo.slime.Slime}. + * + * @author lulf + * @since 5.5 + */ +public class SlimeTraceSerializer extends TraceVisitor { + static final String TIMESTAMP = "timestamp"; + static final String PAYLOAD = "payload"; + static final String CHILDREN = "children"; + final Stack<Cursor> cursors = new Stack<>(); + + public SlimeTraceSerializer(Cursor cursor) { + cursors.push(cursor); + } + + @Override + public void visit(TraceNode node) { + Cursor current = cursors.pop(); + current.setLong(TIMESTAMP, node.timestamp()); + encodePayload(current, node.payload()); + addChildrenCursors(current, node); + + } + + private void encodePayload(Cursor current, Object payload) { + if (payload instanceof String) { + current.setString(PAYLOAD, (String)payload); + } else if (payload instanceof Long) { + current.setLong(PAYLOAD, (Long) payload); + } else if (payload instanceof Boolean) { + current.setBool(PAYLOAD, (Boolean) payload); + } else if (payload instanceof Double) { + current.setDouble(PAYLOAD, (Double) payload); + } else if (payload instanceof byte[]) { + current.setData(PAYLOAD, (byte[]) payload); + } + } + + private void addChildrenCursors(Cursor current, TraceNode node) { + Iterator<TraceNode> it = node.children().iterator(); + if (it.hasNext()) { + Cursor childrenArray = current.setArray(CHILDREN); + while (it.hasNext()) { + cursors.push(childrenArray.addObject()); + it.next(); + } + } + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java b/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java new file mode 100644 index 00000000000..58ec5024cbb --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.slime.*; +import com.yahoo.text.Utf8; +import com.yahoo.yolean.trace.TraceNode; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.time.Clock; + +/** + * A trace utility that can serialize/deserialize to/from {@link Slime} + * + * @author lulf + * @since 5.3 + */ +public class Trace { + private static final String TRACE_TRACELOG = "traceLog"; + private static final String TRACE_TRACELEVEL = "traceLevel"; + private final int traceLevel; + private final TraceNode traceNode; + private final Clock clock; + + private Trace(int traceLevel, TraceNode traceNode, Clock clock) { + this.traceLevel = traceLevel; + this.traceNode = traceNode; + this.clock = clock; + } + + + public void trace(int level, String message) { + if (shouldTrace(level)) { + addTrace(message); + } + } + + private void addTrace(String message) { + traceNode.add(new TraceNode(message, clock.millis())); + } + + public static Trace createNew(int traceLevel, Clock clock) { + return new Trace(traceLevel, new TraceNode(null, clock.millis()), clock); + } + + public static Trace createNew(int traceLevel) { + return createNew(traceLevel, Clock.systemUTC()); + } + + public static Trace fromSlime(Inspector inspector) { + int traceLevel = deserializeTraceLevel(inspector); + Clock clock = Clock.systemUTC(); + SlimeTraceDeserializer deserializer = new SlimeTraceDeserializer(inspector.field(TRACE_TRACELOG)); + return new Trace(traceLevel, deserializer.deserialize(), clock); + } + + private static int deserializeTraceLevel(Inspector inspector) { + return (int) inspector.field(TRACE_TRACELEVEL).asLong(); + } + + public void serialize(Cursor cursor) { + cursor.setLong(TRACE_TRACELEVEL, traceLevel); + SlimeTraceSerializer serializer = new SlimeTraceSerializer(cursor.setObject(TRACE_TRACELOG)); + traceNode.accept(serializer); + } + + public static Trace createDummy() { + return Trace.createNew(0); + } + + public int getTraceLevel() { + return traceLevel; + } + + public boolean shouldTrace(int level) { + return level <= traceLevel; + } + + public String toString(boolean compact) { + Slime slime = new Slime(); + serialize(slime.setObject()); + JsonFormat format = new JsonFormat(compact); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + format.encode(baos, slime); + } catch (IOException e) { + throw new IllegalArgumentException("Unable to encode trace as JSON", e); + } + return Utf8.toString(baos.toByteArray()); + } + + + @Override + public String toString() { + return toString(false); + } + + private final static int systemTraceLevel = Integer.getInteger("config.protocol.traceLevel", 0); + public static Trace createNew() { + return createNew(systemTraceLevel); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/Utf8SerializedString.java b/config/src/main/java/com/yahoo/vespa/config/protocol/Utf8SerializedString.java new file mode 100644 index 00000000000..b715792f05e --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/Utf8SerializedString.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.fasterxml.jackson.core.SerializableString; +import com.yahoo.text.Utf8Array; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +/** + * Wraps utf8array as a {@link com.fasterxml.jackson.core.SerializableString} to avoid extra copy. + * + * @author lulf + * @since 5.17 + */ +public class Utf8SerializedString implements SerializableString { + private final Utf8Array value; + public Utf8SerializedString(Utf8Array value) { + this.value = value; + } + + @Override + public String getValue() { + return value.toString(); + } + + @Override + public int charLength() { + return value.getByteLength(); + } + + @Override + public char[] asQuotedChars() { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] asUnquotedUTF8() { + return value.getBytes(); + } + + @Override + public byte[] asQuotedUTF8() { + throw new UnsupportedOperationException(); + } + + @Override + public int appendQuotedUTF8(byte[] buffer, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int appendQuoted(char[] buffer, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int appendUnquotedUTF8(byte[] buffer, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int appendUnquoted(char[] buffer, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int writeQuotedUTF8(OutputStream out) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int writeUnquotedUTF8(OutputStream out) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int putQuotedUTF8(ByteBuffer buffer) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int putUnquotedUTF8(ByteBuffer out) throws IOException { + throw new UnsupportedOperationException(); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/VespaVersion.java b/config/src/main/java/com/yahoo/vespa/config/protocol/VespaVersion.java new file mode 100644 index 00000000000..748cdce4e25 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/VespaVersion.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +/** + * A wrapper class for Vespa version + * + * @author musum + * @since 5.39 + */ +public class VespaVersion { + private final String version; + + public static VespaVersion fromString(String version) { + return new VespaVersion(version); + } + + private VespaVersion(String version) { + this.version = version; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + VespaVersion that = (VespaVersion) o; + + if (!version.equals(that.version)) return false; + + return true; + } + + @Override + public int hashCode() { + return version.hashCode(); + } + + @Override + public String toString() { + return version; + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/package-info.java b/config/src/main/java/com/yahoo/vespa/config/protocol/package-info.java new file mode 100644 index 00000000000..5a6398eda96 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.protocol; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java new file mode 100644 index 00000000000..b79ff278e57 --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java @@ -0,0 +1,454 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.util; + +import com.yahoo.collections.Tuple2; +import com.yahoo.config.codegen.CNode; +import com.yahoo.io.HexDump; +import com.yahoo.io.IOUtils; +import com.yahoo.slime.JsonFormat; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.*; + +import java.io.*; +import java.net.UnknownHostException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.DecimalFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utilities for mangling config text, finding md5sums, version numbers in .def files etc. + */ +public class ConfigUtils { + /* Patterns used for finding ranges in config definitions */ + private static final Pattern intPattern = Pattern.compile(".*int.*range.*"); + private static final Pattern doublePattern = Pattern.compile(".*double.*range.*"); + private static final Pattern spaceBeforeCommaPatter = Pattern.compile("\\s,"); + public static final String intFormattedMax = new DecimalFormat("#.#").format(0x7fffffff); + public static final String intFormattedMin = new DecimalFormat("#.#").format(-0x80000000); + public static final String doubleFormattedMax = new DecimalFormat("#.#").format(1e308); + public static final String doubleFormattedMin = new DecimalFormat("#.#").format(-1e308); + + /** + * Computes Md5 hash of a list of strings. The only change to input lines before + * computing md5 is to skip empty lines. + * + * @param payload a config payload + * @return the Md5 hash of the list, with lowercase letters + */ + public static String getMd5(ConfigPayload payload) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + payload.serialize(baos, new JsonFormat(true)); + } catch (IOException e) { + throw new RuntimeException(e); + } + MessageDigest md5 = getMd5Instance(); + md5.update(baos.toByteArray()); + return HexDump.toHexString(md5.digest()).toLowerCase(); + } + + /** + * Computes Md5 hash of a list of strings. The only change to input lines before + * computing md5 is to skip empty lines. + * + * @param lines A list of lines + * @return the Md5 hash of the list, with lowercase letters + */ + public static String getMd5(List<String> lines) { + StringBuilder sb = new StringBuilder(); + for (String line : lines) { + // Remove empty lines + line = line.trim(); + if (line.length() > 0) { + sb.append(line).append("\n"); + } + } + MessageDigest md5 = getMd5Instance(); + md5.update(Utf8.toBytes(sb.toString())); + return HexDump.toHexString(md5.digest()).toLowerCase(); + } + + /** + * Computes Md5 hash of a string. + * + * @param input the input String + * @return the Md5 hash of the input, with lowercase letters + */ + public static String getMd5(String input) { + MessageDigest md5 = getMd5Instance(); + md5.update(IOUtils.utf8ByteBuffer(input)); + return HexDump.toHexString(md5.digest()).toLowerCase(); + } + + public static String getMd5(Utf8Array input) { + MessageDigest md5 = getMd5Instance(); + md5.update(input.getBytes()); + return HexDump.toHexString(md5.digest()).toLowerCase(); + } + + private static MessageDigest getMd5Instance() { + try { + return MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + return null; + } + } + + /** + * Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing; + * @param str String to strip spaces from + * @return String with spaces stripped + */ + public static String stripSpaces(String str) { + StringBuilder ret = new StringBuilder(""); + boolean inQuotes = false; + boolean inSpaceSequence = false; + for (char c : str.toCharArray()) { + if (Character.isWhitespace(c)) { + if (inQuotes) { + ret.append(c); + continue; + } + if (!inSpaceSequence) { + // start of space sequence + inSpaceSequence=true; + ret.append(" "); + } + } else { + if (inSpaceSequence) { + inSpaceSequence=false; + } + if (c=='\"') { + inQuotes=!inQuotes; + } + ret.append(c); + } + } + return ret.toString(); + } + + /** + * Computes Md5 hash of a list of strings with the contents of a def-file. + * + * Each string is normalized according to the + * rules of Vespa config definition files before they are used: + * <ol> + * <li>Remove trailing space.<li> + * <li>Remove comment lines.</li> + * <li>Remove trailing comments, and spaces before trailing comments.</li> + * <li>Remove empty lines</li> + * <li>Remove 'version=<version-number>'</li> + * </ol> + * + * @param lines A list of lines constituting a def-file + * @return the Md5 hash of the list, with lowercase letters + */ + public static String getDefMd5(List<String> lines) { + List<String> linesCopy = new ArrayList<>(lines); + for (Iterator<String> it=linesCopy.iterator(); it.hasNext(); ) { + String line = it.next().trim(); + if (! line.startsWith("#") && ! line.equals("")) { + if (line.startsWith("version")) { + it.remove(); + } + // Quit upon 'version', or first line with real content since 'version' cannot occur after that + break; + } + } + + MessageDigest md5; + try { + md5 = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + return null; + } + StringBuilder sb = new StringBuilder(); + for (String line : linesCopy) { + // Normalize line, like it's done in make-config-preproc.pl + line = line.trim(); + // The perl script does stuff like this: + Matcher m = intPattern.matcher(line); + if (m.matches()) { + line = line.replaceFirst("\\[,", "[" + intFormattedMin + ","); + line = line.replaceFirst(",\\]", "," + intFormattedMax + "]"); + } + m = doublePattern.matcher(line); + if (m.matches()) { + line = line.replaceFirst("\\[,", "[" + doubleFormattedMin + ","); + line = line.replaceFirst(",\\]", "," + doubleFormattedMax + "]"); + } + if (line.contains("#")) { + line = line.substring(0, line.indexOf("#")); + line = line.trim(); // Remove space between "real" end of line and a trailing comment + } + if (line.length() > 0) { + line = stripSpaces(line); + m = spaceBeforeCommaPatter.matcher(line); + line = m.replaceAll(","); // Remove space before comma (for enums) + sb.append(line).append("\n"); + } + } + md5.update(Utf8.toBytes(sb.toString())); + return HexDump.toHexString(md5.digest()).toLowerCase(); + } + + /** + * Finds the def version from a reader for a def-file. Returns "" (empty string) + * if no version was found. + * + * @param in A reader to a def-file + * @return version of the def-file, or "" (empty string) if no version was found + */ + public static String getDefVersion(Reader in) { + return getDefKeyword(in, "version"); + } + + /** + * Finds the def namespace from a reader for a def-file. Returns "" (empty string) + * if no namespace was found. + * + * @param in A reader to a def-file + * @return namespace of the def-file, or "" (empty string) if no namespace was found + */ + public static String getDefNamespace(Reader in) { + return getDefKeyword(in, "namespace"); + } + + /** + * Finds the value of the keyword in <code>keyword</code> from a reader for a def-file. + * Returns "" (empty string) if no value for keyword was found. + * + * @param in A reader to a def-file + * @return value of keyword, or "" (empty string) if no line matching keyword was found + */ + public static String getDefKeyword(Reader in, String keyword) { + if (null == in) { + throw new IllegalArgumentException("Null reader."); + } + LineNumberReader reader; + try { + if (in instanceof LineNumberReader) { + reader = (LineNumberReader) in; + } else { + reader = new LineNumberReader(in); + } + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (!line.startsWith("#") && !line.equals("")) { + if (line.startsWith(keyword)) { + String[] v = line.split("="); + return v[1].trim(); + } + } + } + reader.close(); + } catch (IOException e) { + throw new RuntimeException("IOException", e); + } + return ""; + } + + /** + * Finds the name and version from a string with "name,version". + * If no name is given, the first part of the tuple will be the empty string + * If no version is given, the second part of the tuple will be the empty string + * + * @param nameCommaVersion A string consisting of "name,version" + * @return a Tuple2 with first item being name and second item being version + */ + public static Tuple2<String, String> getNameAndVersionFromString(String nameCommaVersion) { + String[] av = nameCommaVersion.split(","); + return new Tuple2<>(av[0], av.length >= 2 ? av[1] : ""); + } + + /** + * Finds the name and namespace from a string with "namespace.name". + * namespace may contain dots. + * If no namespace is given (".name" or just "name"), the second part of the tuple will be the empty string + * If no name is given, the first part of the tuple will be the empty string + * + * @param nameDotNamespace A string consisting of "namespace.name" + * @return a Tuple2 with first item being name and second item being namespace + */ + public static Tuple2<String, String> getNameAndNamespaceFromString(String nameDotNamespace) { + if (!nameDotNamespace.contains(".")) { + return new Tuple2<>(nameDotNamespace, ""); + } + String name = nameDotNamespace.substring(nameDotNamespace.lastIndexOf(".") + 1); + String namespace = nameDotNamespace.substring(0, nameDotNamespace.lastIndexOf(".")); + return new Tuple2<>(name, namespace); + } + + /** + * Creates a ConfigDefinitionKey based on a string with namespace, name and version + * (e.g. Vespa's own config definitions in $VESPA_HOME/var/db/vespa/configserver/serverdb/classes) + * + * @param input A string consisting of "namespace.name.version" + * @return a ConfigDefinitionKey + */ + @SuppressWarnings("deprecation") + public static ConfigDefinitionKey getConfigDefinitionKeyFromString(String input) { + final String name; + final String namespace; + if (!input.contains(".")) { + name = input; + namespace = ""; + } else if (input.lastIndexOf(".") == input.indexOf(".")) { + Tuple2<String, String> tuple = ConfigUtils.getNameAndNamespaceFromString(input); + boolean containsVersion = false; + for (int i=0; i < tuple.first.length(); i++) { + if (Character.isDigit(tuple.first.charAt(i))) { + containsVersion = true; + break; + } + } + if (containsVersion) { + name = tuple.second; + namespace = ""; + } else { + name = tuple.first; + namespace = tuple.second; + } + } else { + Tuple2<String, String> tuple = ConfigUtils.getNameAndNamespaceFromString(input); + + String tempName = tuple.second; + tuple = ConfigUtils.getNameAndNamespaceFromString(tempName); + name = tuple.first; + namespace = tuple.second; + } + return new ConfigDefinitionKey(name, namespace); + } + + /** + * Creates a ConfigDefinitionKey from a string for the name of a node in ZooKeeper + * that holds a config definition + * + * @param nodeName name of a node in ZooKeeper that holds a config definition + * @return a ConfigDefinitionKey + */ + @SuppressWarnings("deprecation") + public static ConfigDefinitionKey createConfigDefinitionKeyFromZKString(String nodeName) { + final String name; + final String namespace; + if (nodeName.contains(".")) { + Tuple2<String, String> tuple = ConfigUtils.getNameAndVersionFromString(nodeName); + String tempName = tuple.first; // includes namespace + tuple = ConfigUtils.getNameAndNamespaceFromString(tempName); + name = tuple.first; + namespace = tuple.second; + } else { + Tuple2<String, String> tuple = ConfigUtils.getNameAndVersionFromString(nodeName); + name = tuple.first; + namespace = ""; + } + return new ConfigDefinitionKey(name, namespace); + } + + + /** + * Creates a ConfigDefinitionKey from a file by reading the file and parsing + * contents for namespace. Name and from filename, but the filename may be prefixed + * with the namespace (if two def files has the same name for instance). + * + * @param file a config definition file + * @return a ConfigDefinitionKey + */ + public static ConfigDefinitionKey createConfigDefinitionKeyFromDefFile(File file) throws IOException { + String[] fileName = file.getName().split("\\."); + assert(fileName.length >= 2); + String name = fileName[fileName.length - 2]; + byte[] content = IOUtils.readFileBytes(file); + + return createConfigDefinitionKeyFromDefContent(name, content); + } + + /** + * Creates a ConfigDefinitionKey from a name and the content of a config definition + * + * @param name the name of the config definition + * @param content content of a config definition + * @return a ConfigDefinitionKey + */ + @SuppressWarnings("deprecation") + public static ConfigDefinitionKey createConfigDefinitionKeyFromDefContent(String name, byte[] content) { + String namespace = ConfigUtils.getDefNamespace(new StringReader(Utf8.toString(content))); + if (namespace.isEmpty()) { + namespace = CNode.DEFAULT_NAMESPACE; + } + return new ConfigDefinitionKey(name, namespace); + } + + + /** + * Escapes a config value according to the cfg format. + * @param input the string to escape + * @return the escaped string + */ + public static String escapeConfigFormatValue(String input) { + if (input == null) { + return "null"; + } + StringBuilder outputBuf = new StringBuilder(input.length()); + for (int i = 0; i < input.length(); i++) { + if (input.charAt(i) == '\\') { + outputBuf.append("\\\\"); // backslash is escaped as: \\ + } else if (input.charAt(i) == '"') { + outputBuf.append("\\\""); // double quote is escaped as: \" + } else if (input.charAt(i) == '\n') { + outputBuf.append("\\n"); // newline is escaped as: \n + } else if (input.charAt(i) == 0) { + // XXX null byte is probably not a good idea anyway + System.err.println("WARNING: null byte in config value"); + outputBuf.append("\\x00"); + } else { + // all other characters are output as-is + outputBuf.append(input.charAt(i)); + } + } + return outputBuf.toString(); + } + + + public static String getDefMd5FromRequest(String defMd5, List<String> defContent) { + if ((defMd5 == null || defMd5.isEmpty()) && defContent != null) { + return ConfigUtils.getDefMd5(defContent); + } else { + return defMd5; + } + } + + public static String getCanonicalHostName() { + try { + return com.yahoo.net.LinuxInetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } + + /** + * Loop through values and return the first one that is set and non-empty. + * + * @param defaultValue The default value to use if no environment variables are set. + * @param envVars one or more environment variable strings + * @return a String with the value of the environment variable + */ + public static String getEnvValue(String defaultValue, String ... envVars) { + String value = null; + for (String envVar : envVars) { + if (value == null || value.isEmpty()) { + value = envVar; + } + } + return (value == null || value.isEmpty()) ? defaultValue : value; + } + + public static boolean isGenerationNewer(long newGen, long oldGen) { + return (oldGen < newGen) || (newGen == 0); + } +} diff --git a/config/src/main/java/com/yahoo/vespa/config/util/package-info.java b/config/src/main/java/com/yahoo/vespa/config/util/package-info.java new file mode 100644 index 00000000000..30f76c1ecbf --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/util/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.util; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/vespa/config/xml/.gitignore b/config/src/main/java/com/yahoo/vespa/config/xml/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/config/xml/.gitignore diff --git a/config/src/main/java/com/yahoo/vespa/zookeeper/.gitignore b/config/src/main/java/com/yahoo/vespa/zookeeper/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/main/java/com/yahoo/vespa/zookeeper/.gitignore diff --git a/config/src/test/java/com/yahoo/config/subscription/AppService.java b/config/src/test/java/com/yahoo/config/subscription/AppService.java new file mode 100644 index 00000000000..1f0fc43ed4e --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/AppService.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.foo.AppConfig; +import com.yahoo.vespa.config.TimingValues; + +/** + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + * + * Application that subscribes to config defined in app.def and + * generated code in AppConfig.java. + */ +public class AppService { + protected int timesConfigured = 0; + + protected AppConfig config = null; + private final ConfigSubscriber subscriber; + protected final String configId; + + final Thread configThread; + boolean stopThread = false; + + public AppService(String configId, ConfigSourceSet csource) { + this(configId, csource, null); + } + + public int timesConfigured() { return timesConfigured; } + + public AppService(String configId, ConfigSourceSet csource, TimingValues timingValues) { + if (csource == null) throw new IllegalArgumentException("Config source cannot be null"); + this.configId = configId; + subscriber = new ConfigSubscriber(csource); + ConfigHandle<AppConfig> temp; + if (timingValues == null) { + temp = subscriber.subscribe(AppConfig.class, configId); + } else { + temp = subscriber.subscribe(AppConfig.class, configId, csource, timingValues); + } + final ConfigHandle<AppConfig> handle = temp; + configThread = new Thread(new Runnable() { + @Override + public void run() { + while (!stopThread) { + boolean changed = subscriber.nextConfig(500); + if (changed) { + configure(handle.getConfig()); + timesConfigured++; + } + } + } + }); + subscriber.nextConfig(5000); + timesConfigured++; + configure(handle.getConfig()); + configThread.setDaemon(true); + configThread.start(); + } + + public void configure(AppConfig config) { + this.config = config; + } + + public void cancelSubscription() { + subscriber.close(); + stopThread = true; + } + + public AppConfig getConfig() { + return config; + } + + public boolean isConfigured() { + return (timesConfigured > 0); + } + +} diff --git a/config/src/test/java/com/yahoo/config/subscription/BasicTest.java b/config/src/test/java/com/yahoo/config/subscription/BasicTest.java new file mode 100644 index 00000000000..0d59e9e7ee7 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/BasicTest.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.is; + +import com.yahoo.foo.AppConfig; + +import org.junit.Test; + + +public class BasicTest { + + @Test + public void testSubBasic() { + ConfigSubscriber s = new ConfigSubscriber(); + ConfigHandle<AppConfig> h = s.subscribe(AppConfig.class, "raw:times 0"); + s.nextConfig(0); + AppConfig c = h.getConfig(); + assertThat(c.times(), is(0)); + } + + @Test + public void testSubBasicGeneration() { + ConfigSubscriber s = new ConfigSubscriber(); + ConfigHandle<AppConfig> h = s.subscribe(AppConfig.class, "raw:times 2"); + s.nextGeneration(0); + AppConfig c = h.getConfig(); + assertThat(c.times(), is(2)); + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java b/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java new file mode 100644 index 00000000000..55d1e5774a5 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java @@ -0,0 +1,315 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.foo.FunctionTestConfig; +import com.yahoo.config.InnerNode; +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.vespa.config.ConfigPayload; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static com.yahoo.config.subscription.util.JsonHelper.assertJsonEquals; +import static com.yahoo.config.subscription.util.JsonHelper.inputJson; +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @author Vegard Sjonfjell + * @since 5.1 + */ +public class CfgConfigPayloadBuilderTest { + + @Test + public void createConfigPayload() { + final FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); + + final String expectedJson = inputJson( + "{", + " 'double_val': '41.23',", + " 'refarr': [", + " ':parent:',", + " ':parent',", + " 'parent:'", + " ],", + " 'pathVal': 'src/test/resources/configs/def-files/function-test.def',", + " 'string_val': 'foo',", + " 'myStructMap': {", + " 'one': {", + " 'myString': 'bull',", + " 'anotherMap': {", + " 'anotherOne': {", + " 'anInt': '3',", + " 'anIntDef': '4'", + " }", + " },", + " 'myInt': '1',", + " 'myStringDef': 'bear',", + " 'myIntDef': '2'", + " }", + " },", + " 'boolarr': [", + " 'false'", + " ],", + " 'intMap': {", + " 'dotted.key': '3',", + " 'spaced key': '4',", + " 'two': '2',", + " 'one': '1'", + " },", + " 'int_val': '5',", + " 'stringarr': [", + " 'bar'", + " ],", + " 'enum_val': 'FOOBAR',", + " 'myarray': [", + " {", + " 'anotherarray': [", + " {", + " 'foo': '7'", + " }", + " ],", + " 'intval': '-5',", + " 'fileVal': 'file0',", + " 'refval': ':parent:',", + " 'myStruct': {", + " 'a': '1',", + " 'b': '2'", + " },", + " 'stringval': [", + " 'baah',", + " 'yikes'", + " ],", + " 'enumval': 'INNER'", + " },", + " {", + " 'anotherarray': [", + " {", + " 'foo': '2'", + " }", + " ],", + " 'intval': '5',", + " 'fileVal': 'file1',", + " 'refval': ':parent:',", + " 'myStruct': {", + " 'a': '-1',", + " 'b': '-2'", + " },", + " 'enumval': 'INNER'", + " }", + " ],", + " 'fileArr': [", + " 'bin'", + " ],", + " 'enumwithdef': 'BAR2',", + " 'bool_with_def': 'true',", + " 'enumarr': [", + " 'VALUES'", + " ],", + " 'pathMap': {", + " 'one': 'pom.xml'", + " },", + " 'long_with_def': '-9876543210',", + " 'double_with_def': '-12.0',", + " 'stringMap': {", + " 'double spaced key': 'third',", + " 'double.dotted.key': 'second',", + " 'one': 'first'", + " },", + " 'refwithdef': ':parent:',", + " 'stringwithdef': 'bar and foo',", + " 'doublearr': [", + " '2344.0',", + " '123.0'", + " ],", + " 'int_with_def': '-14',", + " 'pathArr': [", + " 'pom.xml'", + " ],", + " 'rootStruct': {", + " 'innerArr': [", + " {", + " 'stringVal': 'deep',", + " 'boolVal': 'true'", + " },", + " {", + " 'stringVal': 'blue a=\\\'escaped\\\'',", + " 'boolVal': 'false'", + " }", + " ],", + " 'inner0': {", + " 'index': '11',", + " 'name': 'inner0'", + " },", + " 'inner1': {", + " 'index': '12',", + " 'name': 'inner1'", + " }", + " },", + " 'fileVal': 'etc',", + " 'refval': ':parent:',", + " 'onechoice': 'ONLYFOO',", + " 'bool_val': 'false',", + " 'longarr': [", + " '9223372036854775807',", + " '-9223372036854775808'", + " ],", + " 'basicStruct': {", + " 'intArr': [", + " '310',", + " '311'", + " ],", + " 'foo': 'basicFoo',", + " 'bar': '3'", + " },", + " 'long_val': '12345678901'", + "}" + ); + + CfgConfigPayloadBuilderTest.assertDeserializedConfigEqualsJson(config, expectedJson); + } + + // FIXME: We need to define the behavior here. + @Test + public void test_empty_struct_arrays() { + assertDeserializedConfigEqualsJson("myArray[1]", inputJson("{}")); + } + + @Test + public void test_escaped_string() { + assertDeserializedConfigEqualsJson("a b=\"escaped\"", + inputJson( + "{", + " 'a': 'b=\\\'escaped\\\''", + "}" + ) + ); + } + + @Test + public void test_empty_payload() { + assertDeserializedConfigEqualsJson("", inputJson("{}")); + } + + @Test + public void test_leading_whitespace() { + assertDeserializedConfigEqualsJson(" a 0", + inputJson( + "{", + " 'a': '0'", + "}") + ); + } + + @Test + public void test_leading_and_trailing_whitespace_string() { + assertDeserializedConfigEqualsJson( + "a \" foo \"", + inputJson( + "{", + " 'a': ' foo '", + "}")); + } + + @Test + public void test_config_with_comments() { + CfgConfigPayloadBuilderTest.assertDeserializedConfigEqualsJson( + Arrays.asList( + "fielda b\n", + "#fielda c\n", + "#fieldb c\n", + "# just a comment"), + inputJson( + "{", + " 'fielda': 'b'", + "}") + ); + } + + @Test + public void testConvertingMaps() { + List<String> payload = Arrays.asList( + "intmap{\"foo\"} 1337", + "complexmap{\"key\"}.foo 1337", + "nestedmap{\"key1\"}.foo{\"key2\"}.bar 1337" + ); + + final String expectedJson = inputJson( + "{", + " 'nestedmap': {", + " 'key1': {", + " 'foo': {", + " 'key2': {", + " 'bar': '1337'", + " }", + " }", + " }", + " },", + " 'intmap': {", + " 'foo': '1337'", + " },", + " 'complexmap': {", + " 'key': {", + " 'foo': '1337'", + " }", + " }", + "}" + ); + + CfgConfigPayloadBuilderTest.assertDeserializedConfigEqualsJson(payload, expectedJson); + } + + @Test + public void createConfigPayloadUtf8() { + SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder().stringval("Hei \u00E6\u00F8\u00E5 \uBC14\uB451 \u00C6\u00D8\u00C5 hallo"); + SimpletypesConfig config = new SimpletypesConfig(builder); + + String expectedJson = inputJson( + "{", + " 'longval': '0',", + " 'intval': '0',", + " 'stringval': 'Hei \u00e6\u00f8\u00e5 \ubc14\ub451 \u00c6\u00d8\u00c5 hallo',", + " 'boolval': 'false',", + " 'doubleval': '0.0',", + " 'enumval': 'VAL1'", + "}" + ); + + CfgConfigPayloadBuilderTest.assertDeserializedConfigEqualsJson(config, expectedJson); + } + + @Test + public void testLineParsing() { + CfgConfigPayloadBuilder builder = new CfgConfigPayloadBuilder(); + assertEquals(builder.parseFieldAndValue("foo bar").getFirst(), "foo"); + assertEquals(builder.parseFieldAndValue("foo bar").getSecond(), "bar"); + assertEquals(builder.parseFieldAndValue("foo.bar.baz{my key} baR").getFirst(), "foo.bar.baz{my key}"); + assertEquals(builder.parseFieldAndValue("foo.bar.baz{my key} baR").getSecond(), "baR"); + assertEquals(builder.parseFieldAndValue("foo.bar.baz{my.key} baRR").getFirst(), "foo.bar.baz{my.key}"); + assertEquals(builder.parseFieldAndValue("foo.bar.baz{my.key} baRR").getSecond(), "baRR"); + assertEquals(builder.parseFieldAndValue("foo.bar.baz{my key.dotted}.biz baRO").getFirst(), "foo.bar.baz{my key.dotted}.biz"); + assertEquals(builder.parseFieldAndValue("foo.bar.baz{my key.dotted}.biz baRO").getSecond(), "baRO"); + + assertEquals(builder.parseFieldList("foo.bar.baz").get(0), "foo"); + assertEquals(builder.parseFieldList("foo.bar.baz").get(1), "bar"); + assertEquals(builder.parseFieldList("foo.bar.baz").get(2), "baz"); + assertEquals(builder.parseFieldList("foo.bar{f.b}.baz{f.h h}").get(0), "foo"); + assertEquals(builder.parseFieldList("foo.bar{f.b}.baz{f.h h}").get(1), "bar{f.b}"); + assertEquals(builder.parseFieldList("foo.bar{f.b}.baz{f.h h}").get(2), "baz{f.h h}"); + } + + private static void assertDeserializedConfigEqualsJson(InnerNode config, String expectedJson) { + assertDeserializedConfigEqualsJson(ConfigInstance.serialize(config), expectedJson); + } + + private static void assertDeserializedConfigEqualsJson(String serializedConfig, String expectedJson) { + assertDeserializedConfigEqualsJson(Arrays.asList(serializedConfig), expectedJson); + } + + private static void assertDeserializedConfigEqualsJson(List<String> inputConfig, String expectedJson) { + ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(inputConfig); + assertJsonEquals(payload.toString(), expectedJson); + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigApiTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigApiTest.java new file mode 100755 index 00000000000..608bf2e12b0 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigApiTest.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.foo.*; + +import org.junit.Test; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.is; + +/** + * Tests ConfigSubscriber API, and the ConfigHandle class. + * + * @author <a href="gv@yahoo-inc.com">Harald Musum</a> + * @since 5.1 + */ +public class ConfigApiTest { + private static final String CONFIG_ID = "raw:" + "times 1\n"; + + @Test + public void testConfigSubscriber() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + ConfigHandle<AppConfig> h = subscriber.subscribe(AppConfig.class, CONFIG_ID); + assertNotNull(h); + subscriber.nextConfig(); + assertNotNull(h.getConfig()); + assertEquals(AppConfig.CONFIG_DEF_NAME, ConfigInstance.getDefName(h.getConfig().getClass())); + assertThat(h.isChanged(), is(true)); + assertTrue(h.toString().startsWith("Handle changed: true\nSub:\n")); + subscriber.close(); + assertThat(subscriber.state(), is(ConfigSubscriber.State.CLOSED)); + } + + /** + * Verifies that we get an exception when trying to subscribe after close() has been called + * for a ConfigSubscriber + */ + @Test(expected = IllegalStateException.class) + public void testSubscribeAfterClose() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + subscriber.subscribe(AppConfig.class, CONFIG_ID); + subscriber.nextConfig(); + subscriber.close(); + subscriber.subscribe(AppConfig.class, CONFIG_ID); + } + + /** + * Verifies that it is not possible to to subscribe again after calling nextConfig() + */ + @Test(expected = IllegalStateException.class) + public void testSubscribeAfterNextConfig() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + subscriber.subscribe(AppConfig.class, CONFIG_ID); + subscriber.nextConfig(); + subscriber.subscribe(AppConfig.class, CONFIG_ID); + subscriber.close(); + } + +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java new file mode 100644 index 00000000000..bb5e712fe3a --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.foo.AppConfig; + +import org.junit.Test; + +import java.io.File; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * Unit test for the {@link ConfigGetter}. + * + * @author gjoranv + */ +public class ConfigGetterTest { + private ConfigSourceSet sourceSet = new ConfigSourceSet("config-getter-test"); + + @Test + public void testGetConfig() { + int times = 11; + String message = "testGetConfig"; + String a0 = "a0"; + String configId = "raw:times " + times + "\nmessage " + message + "\na[1]\na[0].name " + a0; + + ConfigGetter<AppConfig> getter = new ConfigGetter<>(AppConfig.class); + AppConfig config = getter.getConfig(configId); + assertThat(config.times(), is(times)); + assertThat(config.message(), is(message)); + assertThat(config.a().size(), is(1)); + assertThat(config.a(0).name(), is(a0)); + + AppService service = new AppService(configId, sourceSet); + AppConfig serviceConfig = service.getConfig(); + assertTrue(service.isConfigured()); + assertThat(config, is(serviceConfig)); + } + +@Test + public void testGetFromRawSource() { + ConfigGetter<AppConfig> getter = new ConfigGetter<>(new RawSource("message \"one\""), AppConfig.class); + AppConfig config = getter.getConfig("test"); + assertThat(config.message(), is("one")); + } + + @Test + public void testGetTwice() { + ConfigGetter<AppConfig> getter = new ConfigGetter<>(AppConfig.class); + AppConfig config = getter.getConfig("raw:message \"one\""); + assertThat(config.message(), is("one")); + config = getter.getConfig("raw:message \"two\""); + assertThat(config.message(), is("two")); + } + + @Test + public void testGetFromFile() { + ConfigGetter<AppConfig> getter = new ConfigGetter<>(AppConfig.class); + AppConfig config = getter.getConfig("file:src/test/resources/configs/foo/app.cfg"); + verifyFooValues(config); + } + + @Test + public void testGetFromFileSource() { + ConfigGetter<AppConfig> getter = new ConfigGetter<>(new FileSource(new File("src/test/resources/configs/foo/app.cfg")), AppConfig.class); + AppConfig config = getter.getConfig("test"); + verifyFooValues(config); + } + + @Test + public void testGetFromDir() { + ConfigGetter<AppConfig> getter = new ConfigGetter<>(AppConfig.class); + AppConfig config = getter.getConfig("dir:src/test/resources/configs/foo/"); + verifyFooValues(config); + } + + @Test + public void testGetFromDirSource() { + AppConfig config = ConfigGetter.getConfig(AppConfig.class, "test", new DirSource(new File("src/test/resources/configs/foo/"))); + verifyFooValues(config); + } + + private void verifyFooValues(AppConfig config) { + assertThat(config.message(), is("msg1")); + assertThat(config.times(), is(3)); + assertThat(config.a(0).name(), is("a0")); + assertThat(config.a(1).name(), is("a1")); + assertThat(config.a(2).name(), is("a2")); + } + + @Test + public void testsStaticGetConfig() { + int times = 11; + String message = "testGetConfig"; + String a0 = "a0"; + String configId = "raw:times " + times + "\nmessage " + message + "\na[1]\na[0].name " + a0; + + AppConfig config = ConfigGetter.getConfig(AppConfig.class, configId); + assertThat(config.times(), is(times)); + assertThat(config.message(), is(message)); + assertThat(config.a().size(), is(1)); + assertThat(config.a(0).name(), is(a0)); + + AppService service = new AppService(configId, sourceSet); + AppConfig serviceConfig = service.getConfig(); + assertTrue(service.isConfigured()); + assertThat(config, is(serviceConfig)); + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java new file mode 100644 index 00000000000..d7a88d591e4 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java @@ -0,0 +1,188 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.FileReference; +import com.yahoo.foo.FunctionTestConfig; +import com.yahoo.foo.MaptypesConfig; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ConfigTransformer; +import org.junit.Test; + +import java.io.File; +import java.util.Arrays; +import java.util.List; + +import static com.yahoo.foo.FunctionTestConfig.*; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * @author gjoranv + * @since 5.1.6 + */ +public class ConfigInstancePayloadTest { + + static FunctionTestConfig createVariableAccessConfigWithBuilder() { + return new FunctionTestConfig(createVariableAccessBuilder()); + } + + static FunctionTestConfig.Builder createVariableAccessBuilder() { + return new FunctionTestConfig.Builder(). + bool_val(false). + bool_with_def(true). + int_val(5). + int_with_def(-14). + long_val(12345678901L). + long_with_def(-9876543210L). + double_val(41.23). + double_with_def(-12). + string_val("foo"). + stringwithdef("bar and foo"). + enum_val(Enum_val.FOOBAR). + enumwithdef(Enumwithdef.BAR2). + refval(":parent:"). + refwithdef(":parent:"). + fileVal("etc"). + pathVal(FileReference.mockFileReferenceForUnitTesting(new File("src/test/resources/configs/def-files/function-test.def"))). + boolarr(false). + longarr(9223372036854775807L). + longarr(-9223372036854775808L). + doublearr(2344.0). + doublearr(123.0). + stringarr("bar"). + enumarr(Enumarr.VALUES). + refarr(Arrays.asList(":parent:", ":parent", "parent:")). // test collection based setter + fileArr("bin"). + pathArr(FileReference.mockFileReferenceForUnitTesting(new File("pom.xml"))). + intMap("one", 1). + intMap("two", 2). + intMap("dotted.key", 3). + intMap("spaced key", 4). + stringMap("one", "first"). + stringMap("double.dotted.key", "second"). + stringMap("double spaced key", "third"). + pathMap("one", FileReference.mockFileReferenceForUnitTesting(new File("pom.xml"))). + basicStruct(new BasicStruct.Builder(). + foo("basicFoo"). + bar(3). + intArr(310).intArr(311)). + + rootStruct(new RootStruct.Builder(). + inner0(new RootStruct.Inner0.Builder(). + index(11)). + inner1(new RootStruct.Inner1.Builder(). + index(12)). + innerArr(new RootStruct.InnerArr.Builder(). + boolVal(true). + stringVal("deep")). + innerArr(new RootStruct.InnerArr.Builder(). + boolVal(false). + stringVal("blue a=\"escaped\""))). + + myarray(new Myarray.Builder(). + intval(-5). + stringval("baah"). + stringval("yikes"). + enumval(Myarray.Enumval.INNER). + refval(":parent:"). + fileVal("file0"). + anotherarray(new Myarray.Anotherarray.Builder(). + foo(7)). + myStruct(new Myarray.MyStruct.Builder(). + a(1). + b(2))). + + myarray(new Myarray.Builder(). + intval(5). + enumval(Myarray.Enumval.INNER). + refval(":parent:"). + fileVal("file1"). + anotherarray(new Myarray.Anotherarray.Builder(). + foo(1). + foo(2)). + myStruct(new Myarray.MyStruct.Builder(). + a(-1). + b(-2))). + + myStructMap("one", new MyStructMap.Builder(). + myInt(1). + myString("bull"). + myIntDef(2). + myStringDef("bear"). + anotherMap("anotherOne", new MyStructMap.AnotherMap.Builder(). + anInt(3). + anIntDef(4))); + } + + @Test + public void config_builder_can_be_created_from_generic_payload() { + FunctionTestConfig config = createVariableAccessConfigWithBuilder(); + ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(ConfigInstance.serialize(config)); + assertFunctionTestPayload(config, payload); + } + + @Test + public void config_builder_can_be_created_from_typed_payload() { + FunctionTestConfig config = createVariableAccessConfigWithBuilder(); + Slime slime = new Slime(); + ConfigInstanceSerializer serializer = new ConfigInstanceSerializer(slime); + ConfigInstance.serialize(config, serializer); + assertFunctionTestPayload(config, new ConfigPayload(slime)); + } + + private void assertFunctionTestPayload(FunctionTestConfig expected, ConfigPayload payload) { + try { + System.out.println(payload.toString(false)); + FunctionTestConfig config2 = new FunctionTestConfig((FunctionTestConfig.Builder)new ConfigTransformer<>(FunctionTestConfig.class).toConfigBuilder(payload)); + assertThat(config2, is(expected)); + assertThat(ConfigInstance.serialize(config2), is(ConfigInstance.serialize(expected))); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + + @Test + public void function_test_payload_is_correctly_deserialized() { + FunctionTestConfig orig = createVariableAccessConfigWithBuilder(); + List<String> lines = ConfigInstance.serialize(orig); + ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(lines); + FunctionTestConfig config = ConfigInstanceUtil.getNewInstance(FunctionTestConfig.class, "foo", payload); + FunctionTest.assertVariableAccessValues(config, "foo"); + } + + @Test + public void map_types_are_correctly_deserialized() { + MaptypesConfig orig = createMapTypesConfig(); + List<String> lines = ConfigInstance.serialize(orig); + ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(lines); + System.out.println(payload.toString()); + MaptypesConfig config = ConfigInstanceUtil.getNewInstance(MaptypesConfig.class, "foo", payload); + System.out.println(config); + assertThat(config.intmap().size(), is(1)); + assertThat(config.intmap("foo"), is(1337)); + assertNotNull(config.innermap("bar")); + assertThat(config.innermap("bar").foo(), is(93)); + assertThat(config.nestedmap().size(), is(1)); + assertNotNull(config.nestedmap("baz")); + assertThat(config.nestedmap("baz").inner("foo"), is(1)); + assertThat(config.nestedmap("baz").inner("bar"), is(2)); + } + + private MaptypesConfig createMapTypesConfig() { + MaptypesConfig.Builder builder = new MaptypesConfig.Builder(); + builder.intmap("foo", 1337); + MaptypesConfig.Innermap.Builder inner = new MaptypesConfig.Innermap.Builder(); + inner.foo(93); + builder.innermap("bar", inner); + MaptypesConfig.Nestedmap.Builder n1 = new MaptypesConfig.Nestedmap.Builder(); + n1.inner("foo", 1); + n1.inner("bar", 2); + builder.nestedmap("baz", n1); + return new MaptypesConfig(builder); + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java new file mode 100644 index 00000000000..97e954b95fb --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java @@ -0,0 +1,257 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.foo.FunctionTestConfig; +import com.yahoo.config.codegen.DefLine; +import com.yahoo.vespa.config.ConfigPayload; +import org.junit.Test; + +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author gjoranv + * @author vegardh + */ +public class ConfigInstanceSerializationTest { + + private DefLine.Type stringType = new DefLine.Type("string"); + private DefLine.Type intType = new DefLine.Type("int"); + private DefLine.Type longType = new DefLine.Type("long"); + private DefLine.Type boolType = new DefLine.Type("bool"); + private DefLine.Type doubleType = new DefLine.Type("double"); + private DefLine.Type fileType = new DefLine.Type("file"); + private DefLine.Type refType = new DefLine.Type("reference"); + + @Test + public void require_symmetrical_serialization_and_deserialization_with_builder() { + FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); + + // NOTE: configId must be ':parent:' because the library replaces ReferenceNodes with that value with + // the instance's configId. (And the config used here contains such nodes.) + List<String> lines = ConfigInstance.serialize(config); + ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(lines); + + FunctionTestConfig config2 = ConfigInstanceUtil.getNewInstance(FunctionTestConfig.class, ":parent:", payload); + assertThat(config, is(config2)); + assertThat(ConfigInstance.serialize(config), is(ConfigInstance.serialize(config2))); + } + +/** Looks like everything in the commented block tests unused api's.. remove? + + @Test + public void testSerializeAgainstConfigDefinitionAllowNothing() { + FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); + InnerCNode def = new InnerCNode("function-test"); + List<String> payload = Validator.serialize(config, def); + Assert.assertEquals(payload.size(), 0); + } + + @Test + public void testSerializeAgainstConfigDefinitionAllLeaves() { + FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); + InnerCNode def = new InnerCNode("function-test"); + def.children().put("bool_val", LeafCNode.newInstance(boolType, def, "bool_val")); + def.children().put("bool_with_def", LeafCNode.newInstance(boolType, def, "bool_with_def")); + def.children().put("int_val", LeafCNode.newInstance(intType, def, "int_val")); + def.children().put("int_with_def", LeafCNode.newInstance(intType, def, "int_with_def")); + def.children().put("long_val", LeafCNode.newInstance(longType, def, "long_val")); + def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def")); + def.children().put("double_val", LeafCNode.newInstance(doubleType, def, "double_val")); + def.children().put("double_with_def", LeafCNode.newInstance(doubleType, def, "double_with_def")); + def.children().put("string_val", LeafCNode.newInstance(stringType, def, "string_val")); + def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef")); + def.children().put("enum_val", LeafCNode.newInstance(enumType(new String[]{"FOO", "BAR", "FOOBAR"}), def, "enum_val")); + def.children().put("enumwithdef", LeafCNode.newInstance(enumType(new String[]{"FOO2", "BAR2", "FOOBAR2"}), def, "enumwithdef", "BAR2")); + def.children().put("refval", LeafCNode.newInstance(refType, def, "refval")); + def.children().put("refwithdef", LeafCNode.newInstance(refType, def, "refwithdef")); + def.children().put("fileVal", LeafCNode.newInstance(fileType, def, "fileVal")); + + List<String> payload = Validator.serialize(config, def); + String plString = payload.toString(); + Assert.assertTrue(plString.matches(".*bool_val false.*")); + Assert.assertTrue(plString.matches(".*bool_with_def true.*")); + Assert.assertTrue(plString.matches(".*int_val 5.*")); + Assert.assertTrue(plString.matches(".*int_with_def -14.*")); + Assert.assertTrue(plString.matches(".*long_val 12345678901.*")); + Assert.assertTrue(plString.matches(".*long_with_def -9876543210.*")); + Assert.assertTrue(plString.matches(".*double_val 41\\.23.*")); + Assert.assertTrue(plString.matches(".*double_with_def -12.*")); + Assert.assertTrue(plString.matches(".*string_val \"foo\".*")); + Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*")); + Assert.assertTrue(plString.matches(".*enum_val FOOBAR.*")); + Assert.assertTrue(plString.matches(".*enumwithdef BAR2.*")); + Assert.assertTrue(plString.matches(".*refval \\:parent\\:.*")); + Assert.assertTrue(plString.matches(".*refwithdef \\:parent\\:.*")); + Assert.assertTrue(plString.matches(".*fileVal \"etc\".*")); + } + + @Test + public void testSerializeAgainstConfigDefinitionSomeLeaves() { + FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); + InnerCNode def = new InnerCNode("function-test"); + def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef")); + def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def")); + // But not double_with_def etc, and no structs/arrays + List<String> payload = Validator.serialize(config, def); + String plString = payload.toString(); + Assert.assertTrue(plString.matches(".*long_with_def \\-9876543210.*")); + Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*")); + Assert.assertFalse(plString.matches(".*double_with_def.*")); + Assert.assertFalse(plString.matches(".*fileVal \"etc\".*")); + Assert.assertFalse(plString.matches(".*basicStruct.*")); + } + + @Test + public void testSerializationAgainstConfigDefinitionAddedValsInDef() { + FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); + InnerCNode def = new InnerCNode("function-test"); + def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef")); + def.children().put("someotherstring", LeafCNode.newInstance(stringType, def, "someotherstring", "some other")); + def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def")); + def.children().put("some_other_long", LeafCNode.newInstance(longType, def, "some_other_long", "88")); + def.children().put("some_other_enum", LeafCNode.newInstance(enumType(new String[]{"hey", "ho", "lets", "go"}), def, "some_other_enum", "lets")); + def.children().put("int_val_nofdef", LeafCNode.newInstance(intType, def, "int_val_nodef", null)); + + // But not double_with_def etc, and no structs/arrays + List<String> payload = Validator.serialize(config, def); + String plString = payload.toString(); + Assert.assertTrue(plString.matches(".*long_with_def \\-9876543210.*")); + Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*")); + Assert.assertTrue(plString.matches(".*.someotherstring \"some other\".*")); + Assert.assertTrue(plString.matches(".*some_other_long 88.*")); + Assert.assertTrue(plString.matches(".*some_other_enum lets.*")); + Assert.assertFalse(plString.matches(".*double_with_def.*")); + Assert.assertFalse(plString.matches(".*fileVal \"etc\".*")); + Assert.assertFalse(plString.matches(".*basicStruct.*")); + Assert.assertFalse(plString.matches(".*int_val_nodef.*")); + } + + @Test + public void testSerializeAgainstConfigDefinitionMismatchAllWays() { + FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); + + // Create all sorts of mismatches in the def schema used to serialize + InnerCNode def = new InnerCNode("function-test"); + def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def")); + def.children().put("stringwithdef", LeafCNode.newInstance(intType, def, "stringwithdef")); + def.children().put("basicStruct", LeafCNode.newInstance(intType, def, "basicStruct")); + InnerCNode doubleValWrong = new InnerCNode("double_val"); + doubleValWrong.children().put("foo", LeafCNode.newInstance(intType, def, "foo")); + def.children().put("double_val", doubleValWrong); + InnerCNode myArray = new InnerCNode("myarray"); + myArray.children().put("intval", LeafCNode.newInstance(stringType, myArray, "foo")); + InnerCNode myStruct = new InnerCNode("myStruct"); + myStruct.children().put("a", LeafCNode.newInstance(stringType, myStruct, "foo")); + myArray.children().put("myStruct", myStruct); + def.children().put("myarray", myArray); + + List<String> payload = Validator.serialize(config, def); + String plString = payload.toString(); + Assert.assertTrue(plString.matches(".*long_with_def.*")); + Assert.assertFalse(plString.matches(".*stringwithdef.*")); + Assert.assertFalse(plString.matches(".*basicStruct.*")); + Assert.assertFalse(plString.matches(".*double_val.*")); + Assert.assertFalse(plString.matches(".*intval.*")); + Assert.assertFalse(plString.matches(".*\\.a.*")); + } + + @Test + public void testSerializeAgainstConfigDefinitionComplex() { + FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder(); + + // Build a pretty complex def programatically + InnerCNode def = new InnerCNode("function-test"); + def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef")); + def.children().put("someUnknownStringNoDefault", LeafCNode.newInstance(stringType, def, "someUnknownStringNoDefault")); + InnerCNode basicStruct = new InnerCNode("basicStruct"); + basicStruct.children().put("foo", LeafCNode.newInstance(stringType, def, "foo")); // but not bar + InnerCNode rootStruct = new InnerCNode("rootStruct"); + InnerCNode inner1 = new InnerCNode("inner1"); + InnerCNode someUnknwonStruct = new InnerCNode("someUnknownStruct"); + InnerCNode someUnknownInner = new InnerCNode("someUnknownInner"); + InnerCNode innerArr = new InnerCNode("innerArr"); + rootStruct.children().put("inner1", inner1); + rootStruct.children().put("someUnknownStruct", someUnknwonStruct); + rootStruct.children().put("someUnknownInner", someUnknownInner); + rootStruct.children().put("innerArr", innerArr); + InnerCNode myarray = new InnerCNode("myarray"); + InnerCNode unknownInner = new InnerCNode("unknownInner"); + def.children().put("basicStruct", basicStruct); + def.children().put("rootStruct", rootStruct); + def.children().put("myarray", myarray); + def.children().put("unknownInner", unknownInner); + inner1.children().put("index", LeafCNode.newInstance(intType, inner1, "index")); + inner1.children().put("someUnknownInt", LeafCNode.newInstance(intType, inner1, "someUnknownInt", "-98")); + inner1.children().put("someUnknownIntNoDefault", LeafCNode.newInstance(intType, inner1, "someUnknownIntNoDefault")); + inner1.children().put("someUnknownEnum", LeafCNode.newInstance(enumType(new String[]{"goo", "go", "gorilla"}), inner1, "someUnknownEnum", "go")); + inner1.children().put("someUnknownEnumNoDefault", LeafCNode.newInstance(enumType(new String[]{"foo", "bar", "baz"}), inner1, "someUnknownEnumNoDefault")); + someUnknwonStruct.children().put("anint", LeafCNode.newInstance(intType, someUnknwonStruct, "anint", "3"));// But no instances of this in config + someUnknownInner.children().put("along", LeafCNode.newInstance(longType, someUnknownInner, "along", "234"));// No instance in config + innerArr.children().put("boolVal", LeafCNode.newInstance(boolType, innerArr, "boolVal")); + innerArr.children().put("someUnknownDouble", LeafCNode.newInstance(doubleType, innerArr, "someUnknownDouble", "-675.789")); + innerArr.children().put("someUnknownDoubleNoDefault", LeafCNode.newInstance(doubleType, innerArr, "someUnknownDoubleNoDefault")); + myarray.children().put("fileVal", LeafCNode.newInstance(fileType, myarray, "fileVal")); + myarray.children().put("stringval", new InnerCNode("stringval[]")); + // TODO make sure default for file is not allowed + //myarray.children().put("someUnknownFile", LeafCNode.newInstance(fileType, myarray, "someUnknownFile", "opt/")); + unknownInner.children().put("aDouble", LeafCNode.newInstance(doubleType, unknownInner, "aDouble", "1234")); + def.children().put("longarr", new InnerCNode("longarr[]")); + def.children().put("boolarr", new InnerCNode("boolarr[]")); + def.children().put("doublearr", new InnerCNode("doublearr[]")); + def.children().put("stringarr", new InnerCNode("stringarr[]")); + def.children().put("fileArr", new InnerCNode("fileArr[]")); + def.children().put("refarr", new InnerCNode("refarr[]")); + def.children().put("enumarr", new InnerCNode("enumarr[]")); + List<String> payload = Validator.serialize(config, def); + String plString = payload.toString(); + Assert.assertFalse(plString.matches(".*long_with_def \\-9876543210.*")); + Assert.assertFalse(plString.matches(".*someUnknownStringNoDefault.*")); + Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*")); + Assert.assertFalse(plString.matches(".*double_with_def.*")); + Assert.assertFalse(plString.matches(".*fileVal etc.*")); + Assert.assertTrue(plString.matches(".*basicStruct\\.foo \"basicFoo\".*")); + Assert.assertFalse(plString.matches(".*basicStruct\\.bar.*")); + Assert.assertFalse(plString.matches(".*rootStruct\\.inner0.*")); + Assert.assertFalse(plString.matches(".*unknownInner.*")); + Assert.assertFalse(plString.matches(".*rootStruct\\.someUnknownStruct.*")); + Assert.assertFalse(plString.matches(".*rootStruct\\.someUnknownInner.*")); + Assert.assertFalse(plString.matches(".*rootStruct\\.inner1\\.name.*")); + Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.index 12.*")); + Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.someUnknownInt -98.*")); + Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.someUnknownEnum go.*")); + Assert.assertTrue(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.boolVal true.*")); + Assert.assertFalse(plString.matches(".*someUnknownEnumNoDefault.*")); + Assert.assertFalse(plString.matches(".*someUnknownDoubleNoDefault.*")); + Assert.assertFalse(plString.matches(".*someUnknownIntNoDefault.*")); + Assert.assertTrue(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.someUnknownDouble -675.789.*")); + Assert.assertFalse(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.stringVal*")); + Assert.assertFalse(plString.matches(".*myarray\\[0\\].intval.*")); + Assert.assertTrue(plString.matches(".*myarray\\[0\\].fileVal \"file0\".*")); + //assertTrue(plString.matches(".*myarray\\[0\\].someUnknownFile \"opt/\".*")); + Assert.assertTrue(plString.matches(".*myarray\\[0\\].stringval\\[0\\] \"baah\".*")); + Assert.assertTrue(plString.matches(".*myarray\\[0\\].stringval\\[1\\] \"yikes\".*")); + Assert.assertTrue(plString.matches(".*myarray\\[1\\].fileVal \"file1\".*")); + Assert.assertFalse(plString.matches(".*myarray\\[1\\].enumVal.*")); + Assert.assertFalse(plString.matches(".*myarray\\[1\\].refVal.*")); + Assert.assertTrue(plString.matches(".*boolarr\\[0\\] false.*")); + Assert.assertTrue(plString.matches(".*longarr\\[0\\] 9223372036854775807.*")); + Assert.assertTrue(plString.matches(".*longarr\\[1\\] -9223372036854775808.*")); + Assert.assertTrue(plString.matches(".*doublearr\\[0\\] 2344\\.0.*")); + Assert.assertTrue(plString.matches(".*doublearr\\[1\\] 123\\.0.*")); + Assert.assertTrue(plString.matches(".*stringarr\\[0\\] \"bar\".*")); + Assert.assertTrue(plString.matches(".*enumarr\\[0\\] VALUES.*")); + Assert.assertTrue(plString.matches(".*refarr\\[0\\] \\:parent\\:.*")); + Assert.assertTrue(plString.matches(".*refarr\\[1\\] \\:parent.*")); + Assert.assertTrue(plString.matches(".*refarr\\[2\\] parent\\:.*")); + Assert.assertTrue(plString.matches(".*fileArr\\[0\\] \"bin\".*")); + } + + private DefLine.Type enumType(String[] strings) { + return new DefLine.Type("enum").setEnumArray(strings); + } +**/ +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java new file mode 100644 index 00000000000..d3713eaa401 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java @@ -0,0 +1,229 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.foo.ArraytypesConfig; +import com.yahoo.config.ConfigInstance; +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.foo.SpecialtypesConfig; +import com.yahoo.foo.StructtypesConfig; +import com.yahoo.foo.MaptypesConfig; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static com.yahoo.config.subscription.util.JsonHelper.assertJsonEquals; +import static com.yahoo.config.subscription.util.JsonHelper.inputJson; +import static org.junit.Assert.fail; + +/** + * @author lulf + * @author Vegard Sjonfjell + * @since 5.1 + */ +public class ConfigInstanceSerializerTest { + @Test + public void test_that_leaf_types_are_serialized_to_json_types() throws IOException { + SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); + builder.boolval(false); + builder.stringval("foo"); + builder.intval(13); + builder.longval(14); + builder.doubleval(3.14); + builder.enumval(SimpletypesConfig.Enumval.Enum.VAL2); + + final SimpletypesConfig config = new SimpletypesConfig(builder); + final String expectedJson = inputJson( + "{", + " 'boolval': false,", + " 'doubleval': 3.14,", + " 'enumval': 'VAL2',", + " 'intval': 13,", + " 'longval': 14,", + " 'stringval': 'foo'", + "}" + ); + + assertConfigEquals(expectedJson, config); + } + + @Test + public void test_that_nested_structs_are_formatted_to_json() throws IOException { + StructtypesConfig.Builder builder = new StructtypesConfig.Builder(); + StructtypesConfig.Nested.Builder nestedBuilder = new StructtypesConfig.Nested.Builder(); + StructtypesConfig.Nested.Inner.Builder innerBuilder = new StructtypesConfig.Nested.Inner.Builder(); + innerBuilder.name("foo"); + innerBuilder.emails("lulf@foo"); + innerBuilder.emails("lulf@bar"); + innerBuilder.gender(StructtypesConfig.Nested.Inner.Gender.Enum.MALE); + nestedBuilder.inner(innerBuilder); + builder.nested(nestedBuilder); + StructtypesConfig.Nestedarr.Builder nestedArrBuilder = new StructtypesConfig.Nestedarr.Builder(); + StructtypesConfig.Nestedarr.Inner.Builder innerNestedArrBuilder = new StructtypesConfig.Nestedarr.Inner.Builder(); + innerNestedArrBuilder.emails("foo@bar"); + innerNestedArrBuilder.name("bar"); + innerNestedArrBuilder.gender(StructtypesConfig.Nestedarr.Inner.Gender.Enum.FEMALE); + nestedArrBuilder.inner(innerNestedArrBuilder); + builder.nestedarr(nestedArrBuilder); + + final StructtypesConfig config = new StructtypesConfig(builder); + final String expectedJson = inputJson( + "{", + " 'complexarr': [],", + " 'nested': {", + " 'inner': {", + " 'emails': [", + " 'lulf@foo',", + " 'lulf@bar'", + " ],", + " 'gender': 'MALE',", + " 'name': 'foo'", + " }", + " },", + " 'nestedarr': [", + " {", + " 'inner': {", + " 'emails': [", + " 'foo@bar'", + " ],", + " 'gender': 'FEMALE',", + " 'name': 'bar'", + " }", + " }", + " ],", + " 'simple': {", + " 'emails': [],", + " 'gender': 'MALE',", + " 'name': '_default_'", + " },", + " 'simplearr': []", + "}" + ); + + assertConfigEquals(expectedJson, config); + } + + @Test + public void test_that_arrays_are_formatted_to_json() throws IOException { + ArraytypesConfig.Builder builder = new ArraytypesConfig.Builder(); + builder.boolarr(true); + builder.boolarr(false); + builder.doublearr(1.2); + builder.doublearr(1.1); + builder.enumarr(ArraytypesConfig.Enumarr.Enum.VAL1); + builder.enumarr(ArraytypesConfig.Enumarr.Enum.VAL2); + builder.intarr(3); + builder.longarr(4l); + builder.stringarr("foo"); + + final ArraytypesConfig config = new ArraytypesConfig(builder); + final String expectedJson = inputJson( + "{", + " 'boolarr': [", + " true,", + " false", + " ],", + " 'doublearr': [", + " 1.2,", + " 1.1", + " ],", + " 'enumarr': [", + " 'VAL1',", + " 'VAL2'", + " ],", + " 'intarr': [", + " 3", + " ],", + " 'longarr': [", + " 4", + " ],", + " 'stringarr': [", + " 'foo'", + " ]", + "}" + ); + + assertConfigEquals(expectedJson, config); + } + + @Test + public void test_that_maps_are_formatted_to_json() throws IOException { + MaptypesConfig.Builder builder = new MaptypesConfig.Builder(); + builder.boolmap("foo", true); + builder.intmap("bar", 3); + builder.stringmap("hei", "hallo"); + MaptypesConfig.Innermap.Builder inner = new MaptypesConfig.Innermap.Builder(); + inner.foo(133); + builder.innermap("baz", inner); + MaptypesConfig.Nestedmap.Builder nested = new MaptypesConfig.Nestedmap.Builder(); + nested.inner("foo", 33); + builder.nestedmap("bim", nested); + + final MaptypesConfig config = new MaptypesConfig(builder); + final String expectedJson = inputJson( + "{", + " 'boolmap': {", + " 'foo': true", + " },", + " 'doublemap': {},", + " 'filemap': {},", + " 'innermap': {", + " 'baz': {", + " 'foo': 133", + " }", + " },", + " 'intmap': {", + " 'bar': 3", + " },", + " 'longmap': {},", + " 'nestedmap': {", + " 'bim': {", + " 'inner': {", + " 'foo': 33", + " }", + " }", + " },", + " 'stringmap': {", + " 'hei': 'hallo'", + " }", + "}" + ); + + assertConfigEquals(expectedJson, config); + } + + @Test + public void test_that_non_standard_types_are_formatted_as_json_strings() throws IOException { + SpecialtypesConfig.Builder builder = new SpecialtypesConfig.Builder(); + builder.myfile("thefilename"); + builder.myref("thereference"); + + final SpecialtypesConfig config = new SpecialtypesConfig(builder); + final String expectedJson = inputJson( + "{", + " 'myfile': 'thefilename',", + " 'myref': 'thereference'", + "}" + ); + + assertConfigEquals(expectedJson, config); + } + + static void assertConfigEquals(String expectedJson, ConfigInstance config) { + Slime slime = new Slime(); + ConfigInstance.serialize(config, new ConfigInstanceSerializer(slime)); + JsonFormat jsonFormat = new JsonFormat(true); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + try { + jsonFormat.encode(baos, slime); + } catch (IOException e) { + fail(); + } + + assertJsonEquals(baos.toString(), expectedJson); + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java new file mode 100644 index 00000000000..ea7312ec7ef --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.foo.AppConfig; +import com.yahoo.foo.TestNonstringConfig; +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * Tests different aspects of the ConfigInstance class and its underlying Nodes. + * + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public class ConfigInstanceTest { + private ConfigSourceSet sourceSet = new ConfigSourceSet("config-instance-test"); + + /** + * Verifies that the subscriber's configure() method is only + * called once upon subscribe, even if there are more than one + * subscribers to the same ConfigInstance. This has previously + * been a problem, since ConfigInstance.subscribe() called + * configureSubscriber(), which configures all subscribers to the + * instance. Now, the new method configureSubscriber(Subscriber) + * is called instead. + */ + @Test + public void testConfigureOnlyOnceUponSubscribe() { + final String configId = "raw:times 1\n"; + AppService service1 = new AppService(configId, sourceSet); + AppService service2 = new AppService(configId, sourceSet); + + assertEquals(1, service1.timesConfigured()); + assertEquals(1, service2.timesConfigured()); + } + + /** + * Verifies that values set in previous setConfig() calls are + * retained when the payload in a new setConfig() call does not + * overwrite them. + */ + @Test + @Ignore + public void testRetainOldValuesOnConfigUpdates() { + AppConfig config = new AppConfig(new AppConfig.Builder()); + //config.setConfig(Arrays.asList("message \"one\"", "times 333"), "", 0L); + assertEquals("one", config.message()); + assertEquals(333, config.times()); + + //config.setConfig(Arrays.asList("message \"two\""), "", 0L); + assertEquals("two", config.message()); + assertEquals("config.times retains previously set value", 333, config.times()); + + //config.setConfig(Arrays.asList("times 666"), "", 0L); + assertEquals("config.message retains previously set value", "two", config.message()); + assertEquals(666, config.times()); + } + + /** + * Verifies that an exception is thrown when one attempts to set an + * illegal config value for parameters that have default values. + */ + @Test + public void testFailUponIllegalValue() { + verifyIllegalValue("i notAnInt"); + verifyIllegalValue("i 3.0"); + + verifyIllegalValue("d noDouble"); + verifyIllegalValue("d 3.0."); + + // verifyIllegalValue("b notTrueOrFalse"); + //verifyIllegalValue("b 1"); + //verifyIllegalValue("b 0"); + + verifyIllegalValue("e undeclaredEnumValue"); + verifyIllegalValue("e 0"); + } + + private void verifyIllegalValue(String line) { + String configId = "raw:" + line + "\n"; + try { + new TestNonstring(configId); + fail("Expected ConfigurationRuntimeException when setting a parameter value of wrong type."); + } catch (RuntimeException expected) { + verifyException(expected, "Not able to create config builder for payload", "Got ConfigurationRuntimeException for the wrong reason. " + + "Expected to fail when setting a parameter value of wrong type."); + } + + } + + private void verifyException(Throwable throwable, String expected, String failMessage) { + Throwable t = throwable; + boolean ok = false; + while (t != null) { + if (t.getMessage() != null && t.getMessage().contains(expected)) { + ok = true; + break; + } + t = t.getCause(); + } + if (!ok) { + throwable.printStackTrace(); + fail(failMessage); + } + } + + private class TestNonstring { + private final ConfigSubscriber subscriber; + private final ConfigHandle<TestNonstringConfig> handle; + public TestNonstring(String configId) { + subscriber = new ConfigSubscriber(); + handle = subscriber.subscribe(TestNonstringConfig.class, configId); + subscriber.nextConfig(); + handle.getConfig(); + } + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java new file mode 100644 index 00000000000..36ea09db1dc --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.config.ConfigBuilder; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.FileReference; +import com.yahoo.foo.FunctionTestConfig; +import com.yahoo.foo.TestNodefsConfig; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ConfigPayloadBuilder; +import org.junit.Test; + +import java.io.File; +import java.util.Arrays; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.containsString; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static com.yahoo.foo.FunctionTestConfig.*; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigInstanceUtilTest { + + @Test + public void require_that_builder_values_can_be_overridden_by_another_builder() { + FunctionTestConfig.Builder destination = createVariableAccessBuilder(); + + FunctionTestConfig.Builder source = new FunctionTestConfig.Builder() + .int_val(-1) + .intarr(0) + .doublearr(0.0) + .basicStruct(new FunctionTestConfig.BasicStruct.Builder() + .bar(-1) + .intArr(0)) + .myarray(new FunctionTestConfig.Myarray.Builder() + .intval(-1) + .refval("") + .fileVal("") + .myStruct(new FunctionTestConfig.Myarray.MyStruct.Builder() + .a(0) + )); + + ConfigInstanceUtil.setValues(destination, source); + + FunctionTestConfig result = new FunctionTestConfig(destination); + assertThat(result.int_val(), is(-1)); + assertThat(result.string_val(), is("foo")); + assertThat(result.intarr().size(), is(1)); + assertThat(result.intarr(0), is(0)); + assertThat(result.longarr().size(), is(2)); + assertThat(result.doublearr().size(), is(3)); + assertEquals(2344.0, result.doublearr(0), 0.01); + assertEquals(123.0, result.doublearr(1), 0.01); + assertEquals(0.0, result.doublearr(2), 0.01); + assertThat(result.basicStruct().bar(), is(-1)); + assertThat(result.basicStruct().foo(), is("basicFoo")); + assertThat(result.basicStruct().intArr().size(), is(3)); + assertThat(result.basicStruct().intArr(0), is(310)); + assertThat(result.basicStruct().intArr(1), is(311)); + assertThat(result.basicStruct().intArr(2), is(0)); + assertThat(result.myarray().size(), is(3)); + assertThat(result.myarray(2).intval(), is(-1)); + assertThat(result.myarray(2).refval(), is("")); + assertThat(result.myarray(2).fileVal().value(), is("")); + assertThat(result.myarray(2).myStruct().a(), is(0)); + + } + + @Test(expected = ConfigurationRuntimeException.class) + public void require_that_invalid_builders_fail() { + ConfigInstanceUtil.setValues(new FakeBuilder(), new FakeBuilder()); + } + + private static class FakeBuilder implements ConfigBuilder { + } + + static FunctionTestConfig.Builder createVariableAccessBuilder() { + return new FunctionTestConfig.Builder(). + bool_val(false). + bool_with_def(true). + int_val(5). + int_with_def(-14). + long_val(12345678901L). + long_with_def(-9876543210L). + double_val(41.23). + double_with_def(-12). + string_val("foo"). + stringwithdef("bar and foo"). + enum_val(Enum_val.FOOBAR). + enumwithdef(Enumwithdef.BAR2). + refval(":parent:"). + refwithdef(":parent:"). + fileVal("etc"). + pathVal(FileReference.mockFileReferenceForUnitTesting(new File("src/test/resources/configs/def-files/function-test.def"))). + boolarr(false). + longarr(9223372036854775807L). + longarr(-9223372036854775808L). + doublearr(2344.0). + doublearr(123.0). + stringarr("bar"). + enumarr(Enumarr.VALUES). + refarr(Arrays.asList(":parent:", ":parent", "parent:")). // test collection based setter + fileArr("bin"). + + basicStruct(new FunctionTestConfig.BasicStruct.Builder(). + foo("basicFoo"). + bar(3). + intArr(310).intArr(311)). + + rootStruct(new FunctionTestConfig.RootStruct.Builder(). + inner0(new FunctionTestConfig.RootStruct.Inner0.Builder(). + index(11)). + inner1(new FunctionTestConfig.RootStruct.Inner1.Builder(). + index(12)). + innerArr(new FunctionTestConfig.RootStruct.InnerArr.Builder(). + boolVal(true). + stringVal("deep")). + innerArr(new FunctionTestConfig.RootStruct.InnerArr.Builder(). + boolVal(false). + stringVal("blue a=\"escaped\""))). + + myarray(new FunctionTestConfig.Myarray.Builder(). + intval(-5). + stringval("baah"). + stringval("yikes"). + enumval(Myarray.Enumval.INNER). + refval(":parent:"). + fileVal("file0"). + anotherarray(new FunctionTestConfig.Myarray.Anotherarray.Builder(). + foo(7)). + myStruct(new FunctionTestConfig.Myarray.MyStruct.Builder(). + a(1). + b(2))). + + myarray(new FunctionTestConfig.Myarray.Builder(). + intval(5). + enumval(Myarray.Enumval.INNER). + refval(":parent:"). + fileVal("file1"). + anotherarray(new FunctionTestConfig.Myarray.Anotherarray.Builder(). + foo(1). + foo(2)). + myStruct(new FunctionTestConfig.Myarray.MyStruct.Builder(). + a(-1). + b(-2))); + + } + + @Test + public void test_require_private_getter() { + FunctionTestConfig.Builder builder = createVariableAccessBuilder(); + assertEquals(ConfigInstanceUtil.getField(builder, "int_val"), 5); + FunctionTestConfig conf = new FunctionTestConfig(builder); + assertEquals(conf.int_val(), 5); + } + + @Test + public void testGetNewInstanceErrorMessage() { + ConfigPayloadBuilder payloadBuilder = new ConfigPayloadBuilder(); + try { + ConfigInstanceUtil.getNewInstance(TestNodefsConfig.class, "id0", ConfigPayload.fromBuilder(payloadBuilder)); + assert(false); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("Failed creating new instance of 'com.yahoo.foo.TestNodefsConfig' for config id 'id0':")); + } + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java new file mode 100644 index 00000000000..d760a1c1b72 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigInterruptedExceptionTest { + @Test + public void require_that_throwable_is_preserved() { + ConfigInterruptedException e = new ConfigInterruptedException(new RuntimeException("foo")); + assertThat(e.getCause().getMessage(), is("foo")); + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java new file mode 100644 index 00000000000..adbc11b7ca0 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java @@ -0,0 +1,133 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import static org.junit.Assert.*; + +import com.yahoo.foo.AppConfig; +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.foo.StringConfig; +import com.yahoo.config.subscription.impl.ConfigSubscription; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.TimingValues; +import org.junit.Test; + +public class ConfigSetSubscriptionTest { + + @Test + public void testConfigSubscription() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + ConfigSet configSet = new ConfigSet(); + AppConfig.Builder a0builder = new AppConfig.Builder().message("A message, 0").times(88); + configSet.addBuilder("app/0", a0builder); + AppConfig.Builder a1builder = new AppConfig.Builder().message("A message, 1").times(89); + configSet.addBuilder("app/1", a1builder); + + ConfigSubscription<AppConfig> c1 = ConfigSubscription.get( + new ConfigKey<>(AppConfig.class, "app/0"), + subscriber, + configSet, + new TimingValues()); + ConfigSubscription<AppConfig> c2 = ConfigSubscription.get( + new ConfigKey<>(AppConfig.class, "app/1"), + subscriber, + configSet, + new TimingValues()); + + assertTrue(c1.equals(c1)); + assertFalse(c1.equals(c2)); + } + + @Test(expected = IllegalArgumentException.class) + public void testUnknownKey() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + ConfigSet configSet = new ConfigSet(); + AppConfig.Builder a0builder = new AppConfig.Builder().message("A message, 0").times(88); + configSet.addBuilder("app/0", a0builder); + + ConfigSubscription.get( + new ConfigKey<>(SimpletypesConfig.class, "simpletypes/1"), + subscriber, + configSet, + new TimingValues()); + } + + @Test + public void testConfigSetBasic() { + ConfigSet myConfigs = new ConfigSet(); + AppConfig.Builder a0builder = new AppConfig.Builder().message("A message, 0").times(88); + AppConfig.Builder a1builder = new AppConfig.Builder().message("A message, 1").times(89); + StringConfig.Builder barBuilder = new StringConfig.Builder().stringVal("StringVal"); + myConfigs.addBuilder("app/0", a0builder); + myConfigs.addBuilder("app/1", a1builder); + myConfigs.addBuilder("bar", barBuilder); + ConfigSubscriber subscriber = new ConfigSubscriber(myConfigs); + ConfigHandle<AppConfig> hA0 = subscriber.subscribe(AppConfig.class, "app/0"); + ConfigHandle<AppConfig> hA1 = subscriber.subscribe(AppConfig.class, "app/1"); + ConfigHandle<StringConfig> hS = subscriber.subscribe(StringConfig.class, "bar"); + + assertTrue(subscriber.nextConfig(0)); + assertTrue(hA0.isChanged()); + assertTrue(hA1.isChanged()); + assertTrue(hS.isChanged()); + + assertEquals(hA0.getConfig().message(), "A message, 0"); + assertEquals(hA1.getConfig().message(), "A message, 1"); + assertEquals(hA0.getConfig().times(), 88); + assertEquals(hA1.getConfig().times(), 89); + + assertFalse(subscriber.nextConfig(10)); + assertFalse(hA0.isChanged()); + assertFalse(hA1.isChanged()); + assertFalse(hS.isChanged()); + assertEquals(hA0.getConfig().message(), "A message, 0"); + assertEquals(hA1.getConfig().message(), "A message, 1"); + assertEquals(hA0.getConfig().times(), 88); + assertEquals(hA1.getConfig().times(), 89); + assertEquals(hS.getConfig().stringVal(), "StringVal"); + + //Reconfigure all configs, generation should change + a0builder.message("A new message, 0").times(880); + a1builder.message("A new message, 1").times(890); + barBuilder.stringVal("new StringVal"); + subscriber.reload(1); + assertTrue(subscriber.nextConfig(0)); + assertTrue(hA0.isChanged()); + assertTrue(hA1.isChanged()); + assertTrue(hS.isChanged()); + + assertEquals(hA0.getConfig().message(), "A new message, 0"); + assertEquals(hA1.getConfig().message(), "A new message, 1"); + assertEquals(hA0.getConfig().times(), 880); + assertEquals(hA1.getConfig().times(), 890); + assertEquals(hS.getConfig().stringVal(), "new StringVal"); + + // Reconfigure only one + a0builder.message("Another new message, 0").times(8800); + subscriber.reload(2); + assertTrue(subscriber.nextConfig(0)); + assertTrue(hA0.isChanged()); + assertFalse(hA1.isChanged()); + assertFalse(hS.isChanged()); + + assertEquals(hA0.getConfig().message(), "Another new message, 0"); + assertEquals(hA1.getConfig().message(), "A new message, 1"); + assertEquals(hA0.getConfig().times(), 8800); + assertEquals(hA1.getConfig().times(), 890); + assertEquals(hS.getConfig().stringVal(), "new StringVal"); + + //Reconfigure only one, and only one field on the builder + a1builder.message("Yet another message, 1"); + subscriber.reload(3); + assertTrue(subscriber.nextConfig(0)); + assertFalse(hA0.isChanged()); + assertTrue(hA1.isChanged()); + assertFalse(hS.isChanged()); + + assertEquals(hA0.getConfig().message(), "Another new message, 0"); + assertEquals(hA1.getConfig().message(), "Yet another message, 1"); + assertEquals(hA0.getConfig().times(), 8800); + assertEquals(hA1.getConfig().times(), 890); + assertEquals(hS.getConfig().stringVal(), "new StringVal"); + } + +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java new file mode 100644 index 00000000000..668fa68d264 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.foo.SimpletypesConfig; +import org.junit.Test; + +import java.util.regex.Pattern; + +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigSetTest { + @Test + public void testToString() { + ConfigSet set = new ConfigSet(); + SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); + set.addBuilder("foo", builder); + assertTrue(Pattern.matches("name=simpletypes,namespace=foo,configId=foo=>com.yahoo.foo.SimpletypesConfig.*", + set.toString())); + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java new file mode 100755 index 00000000000..23633a62cca --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import org.junit.Test; +import org.junit.After; + +import java.util.Set; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> + */ +public class ConfigSourceSetTest { + @Test + public void testEquals() { + assertTrue(new ConfigSourceSet().equals(new ConfigSourceSet())); + assertFalse(new ConfigSourceSet().equals(new ConfigSourceSet(new String[]{"a"}))); + + assertTrue(new ConfigSourceSet(new String[]{"a"}).equals(new ConfigSourceSet(new String[]{"a"}))); + assertTrue(new ConfigSourceSet(new String[]{"a"}).equals(new ConfigSourceSet(new String[]{" A "}))); + assertTrue(new ConfigSourceSet(new String[]{"a"}).equals(new ConfigSourceSet(new String[]{"A", "a"}))); + assertTrue(new ConfigSourceSet(new String[]{"A"}).equals(new ConfigSourceSet(new String[]{"a", " a "}))); + + assertFalse(new ConfigSourceSet(new String[]{"a"}).equals(new ConfigSourceSet(new String[]{"b"}))); + assertFalse(new ConfigSourceSet(new String[]{"a"}).equals(new ConfigSourceSet(new String[]{"a", "b"}))); + + assertTrue(new ConfigSourceSet(new String[]{"a", "b"}).equals(new ConfigSourceSet(new String[]{"a", "b"}))); + assertTrue(new ConfigSourceSet(new String[]{"b", "a"}).equals(new ConfigSourceSet(new String[]{"a", "b"}))); + assertTrue(new ConfigSourceSet(new String[]{"A", " b"}).equals(new ConfigSourceSet(new String[]{"a ", "B"}))); + assertTrue(new ConfigSourceSet(new String[]{"b", "a", "c"}) + .equals(new ConfigSourceSet(new String[]{"a", "b", "c"}))); + + assertFalse(new ConfigSourceSet(new String[]{"a", "b"}).equals(new ConfigSourceSet(new String[]{"b", "c"}))); + assertFalse(new ConfigSourceSet().equals("foo")); + } + + @Test + public void testIterationOrder() { + String[] hosts = new String[]{"primary", "fallback", "last-resort"}; + ConfigSourceSet css = new ConfigSourceSet(hosts); + + Set<String> sources = css.getSources(); + assertEquals(hosts.length, sources.size()); + int i = 0; + for (String s : sources) { + assertEquals(hosts[i++], s); + } + } + + @Test + public void testDefaultSourceFromProperty() { + // TODO: Unable to set environment, so only able to test property usage for now + System.setProperty("configsources", "foo:123,bar:345,tcp/baz:333,quux"); + ConfigSourceSet set = ConfigSourceSet.createDefault(); + assertThat(set.getSources().size(), is(4)); + assertTrue(set.getSources().contains("tcp/foo:123")); + assertTrue(set.getSources().contains("tcp/bar:345")); + assertTrue(set.getSources().contains("tcp/baz:333")); + assertTrue(set.getSources().contains("tcp/quux")); + } + + @After + public void cleanup() { + System.clearProperty("configsources"); + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceTest.java new file mode 100644 index 00000000000..d928592ebe9 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceTest.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.foo.SimpletypesConfig; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; + +import static junit.framework.TestCase.fail; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigSourceTest { + @Test(expected = IllegalArgumentException.class) + public void require_that_FileSource_throws_exception_on_invalid_file() { + new FileSource(new File("invalid")); + } + + @Test(expected = IllegalArgumentException.class) + public void require_that_DirSource_throws_exception_on_invalid_dir() { + new DirSource(new File("invalid")); + } + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + @Test + public void require_that_DirSource_throws_exception_on_missing_file() throws IOException { + File folder = tmpDir.newFolder(); + DirSource dirSource = new DirSource(folder); + try { + ConfigGetter.getConfig(SimpletypesConfig.class, "dir:" + tmpDir, dirSource); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Could not find a config file for '" + SimpletypesConfig.getDefName() + "' in '" + folder + "/'")); + } + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java new file mode 100644 index 00000000000..9fa299aef83 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.foo.AppConfig; +import com.yahoo.config.subscription.impl.ConfigSubscription; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.TimingValues; + +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; + +import static junit.framework.TestCase.fail; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author musum, lulf + * @since 5.1 + */ +public class ConfigSubscriptionTest { + @Test + public void testEquals() { + ConfigSubscriber sub = new ConfigSubscriber(); + final String payload = "boolval true"; + ConfigSubscription<SimpletypesConfig> a = ConfigSubscription.get(new ConfigKey<>(SimpletypesConfig.class, "test"), + sub, new RawSource(payload), new TimingValues()); + ConfigSubscription<SimpletypesConfig> b = ConfigSubscription.get(new ConfigKey<>(SimpletypesConfig.class, "test"), + sub, new RawSource(payload), new TimingValues()); + ConfigSubscription<SimpletypesConfig> c = ConfigSubscription.get(new ConfigKey<>(SimpletypesConfig.class, "test2"), + sub, new RawSource(payload), new TimingValues()); + assertThat(a, is(b)); + assertThat(a, is(a)); + assertThat(b, is(b)); + assertThat(c, is(c)); + assertThat(a, is(not(c))); + assertThat(b, is(not(c))); + + ConfigSubscriber subscriber = new ConfigSubscriber(); + ConfigSet configSet = new ConfigSet(); + AppConfig.Builder a0builder = new AppConfig.Builder().message("A message, 0").times(88); + configSet.addBuilder("app/0", a0builder); + AppConfig.Builder a1builder = new AppConfig.Builder().message("A message, 1").times(89); + configSet.addBuilder("app/1", a1builder); + + ConfigSubscription<AppConfig> c1 = ConfigSubscription.get( + new ConfigKey<>(AppConfig.class, "app/0"), + subscriber, + configSet, + new TimingValues()); + ConfigSubscription<AppConfig> c2 = ConfigSubscription.get( + new ConfigKey<>(AppConfig.class, "app/1"), + subscriber, + configSet, + new TimingValues()); + + assertTrue(c1.equals(c1)); + assertFalse(c1.equals(c2)); + } + + @Test + public void testSubscribeInterface() { + ConfigSubscriber sub = new ConfigSubscriber(); + ConfigHandle<SimpletypesConfig> handle = sub.subscribe(SimpletypesConfig.class, "raw:boolval true", 10000); + assertNotNull(handle); + sub.nextConfig(); + assertThat(handle.getConfig().boolval(), is(true)); + //assertTrue(sub.getSource() instanceof RawSource); + } + + // Test that subscription is closed and subscriptionHandles is empty if we get an exception (only the last is possible to + // test right now). + @Test + @Ignore + public void testSubscribeWithException() { + TestConfigSubscriber sub = new TestConfigSubscriber(); + ConfigSourceSet configSourceSet = new ConfigSourceSet(Collections.singletonList("tcp/localhost:99999")); + try { + sub.subscribe(SimpletypesConfig.class, "configid", configSourceSet, new TimingValues().setSubscribeTimeout(100)); + fail(); + } catch (ConfigurationRuntimeException e) { + assertThat(sub.getSubscriptionHandles().size(), is(0)); + } + } + + private static class TestConfigSubscriber extends ConfigSubscriber { + List<ConfigHandle<? extends ConfigInstance>> getSubscriptionHandles() { + return subscriptionHandles; + } + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java new file mode 100644 index 00000000000..15cfff0448a --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigURITest { + @Test + public void testDefaultUri() { + ConfigURI uri = ConfigURI.createFromId("foo"); + assertThat(uri.getConfigId(), is("foo")); + assertTrue(uri.getSource() instanceof ConfigSourceSet); + } + + @Test + public void testFileUri() throws IOException { + File file = File.createTempFile("foo", ".cfg"); + ConfigURI uri = ConfigURI.createFromId("file:" + file.getAbsolutePath()); + assertThat(uri.getConfigId(), is("")); + assertTrue(uri.getSource() instanceof FileSource); + } + + @Test + public void testDirUri() throws IOException { + ConfigURI uri = ConfigURI.createFromId("dir:."); + assertThat(uri.getConfigId(), is("")); + assertTrue(uri.getSource() instanceof DirSource); + } + + @Test + public void testCustomUri() { + ConfigURI uri = ConfigURI.createFromIdAndSource("foo", new ConfigSet()); + assertThat(uri.getConfigId(), is("foo")); + assertTrue(uri.getSource() instanceof ConfigSet); + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java b/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java new file mode 100644 index 00000000000..55b1eb97bee --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.foo.*; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests reading of a config containing + * <ul> + * <li>Missing values + * <li>Default values + * </ul> + * <p/> + * for + * <p/> + * <ul> + * <li>String and + * <li>Reference + * </ul> + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class DefaultConfigTest { + static final String CONFIG_ID = "raw:" + + "nondefaultstring ####-------missing--------\n" + + "defaultstring \"thedefault\"\n" + + "nondefaultreference ####-------missing--------\n" + + "defaultreference \"thedefault\"\n"; + + @Test(expected = IllegalArgumentException.class) + public void testFailUponUnitializedValue() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + subscriber.subscribe(DefaulttestConfig.class, "raw:" + + "defaultstring \"new value\""); + subscriber.nextConfig(); + subscriber.close(); + } + + /** + * Reads a config from a string which is exactly like one returned from + * the config server given only default values for this config. + * The parsing code is the same whether the reading happens from string + * or from a server connection, so this tests that this config can be + * received correctly from the server + */ + @Test + public void testDefaultConfig() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + ConfigHandle<DefaulttestConfig> h = subscriber.subscribe(DefaulttestConfig.class, CONFIG_ID); + assertTrue(subscriber.nextConfig()); + DefaulttestConfig config = h.getConfig(); + verifyConfigValues(config); + subscriber.close(); + } + + private static void verifyConfigValues(DefaulttestConfig config) { + assertEquals("####-------missing--------", config.nondefaultstring()); + assertEquals("thedefault", config.defaultstring()); + assertEquals("####-------missing--------", config.nondefaultreference()); + assertEquals("thedefault", config.defaultreference()); + } + +} diff --git a/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java b/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java new file mode 100644 index 00000000000..8936c62784b --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/FunctionTest.java @@ -0,0 +1,259 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.foo.FunctionTestConfig; + +import org.junit.Before; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + + +/** + * Test most data types and features of the config client API, + * e.g. parameter access, missing default values, and more. + * <p/> + * NOTE: this test does NOT test null default values. + * + * @author gjoranv + */ +public class FunctionTest { + + public static final String PATH = "src/test/resources/configs/function-test/"; + + private FunctionTestConfig config; + private ConfigSourceSet sourceSet = new ConfigSourceSet("function-test"); + + public void configure(FunctionTestConfig config, ConfigSourceSet sourceSet) { + this.config = config; + assertEquals(this.sourceSet, sourceSet); + } + + @Before + public void resetConfig() { + config = null; + } + + @Test + public void testVariableAccess() { + assertNull(config); + String configId = "file:" + PATH + "variableaccess.txt"; + ConfigGetter<FunctionTestConfig> getter = new ConfigGetter<>(FunctionTestConfig.class); + assertVariableAccessValues(getter.getConfig(configId), configId); + } + + @Test + public void testDefaultValues() { + assertNull(config); + String configId = "file:" + PATH + "defaultvalues.txt"; + ConfigGetter<FunctionTestConfig> getter = new ConfigGetter<>(FunctionTestConfig.class); + FunctionTestConfig config = getter.getConfig(configId); + + assertFalse(config.bool_val()); + assertFalse(config.bool_with_def()); + assertEquals(5, config.int_val()); + assertEquals(-545, config.int_with_def()); + assertEquals(1234567890123L, config.long_val()); + assertEquals(-50000000000L, config.long_with_def()); + assertEquals(41.23, config.double_val(), 0.000001); + assertEquals(-6.43, config.double_with_def(), 0.000001); + assertEquals("foo", config.string_val()); + assertEquals("foobar", config.stringwithdef()); + assertEquals(FunctionTestConfig.Enum_val.FOOBAR, config.enum_val()); + assertEquals(FunctionTestConfig.Enumwithdef.BAR2, config.enumwithdef()); + assertEquals(configId, config.refval()); + assertEquals(configId, config.refwithdef()); + assertEquals("vespa.log", config.fileVal().value()); + assertEquals(1, config.boolarr().size()); + assertEquals(0, config.intarr().size()); + assertEquals(0, config.longarr().size()); + assertEquals(2, config.doublearr().size()); + assertEquals(1, config.stringarr().size()); + assertEquals(1, config.enumarr().size()); + assertEquals(0, config.refarr().size()); + assertEquals(0, config.fileArr().size()); + + assertEquals(3, config.basicStruct().bar()); + assertEquals(1, config.basicStruct().intArr().size()); + assertEquals(10, config.basicStruct().intArr(0)); + assertEquals(11, config.rootStruct().inner0().index()); + assertEquals(12, config.rootStruct().inner1().index()); + + assertEquals(2, config.myarray().size()); + assertEquals(1, config.myarray(0).myStruct().a()); + assertEquals(-1, config.myarray(1).myStruct().a()); + assertEquals("command.com", config.myarray(0).fileVal().value()); + assertEquals("display.sys", config.myarray(1).fileVal().value()); + } + + + @Test + public void testRandomOrder() { + assertNull(config); + String configId = "file:" + PATH + "randomorder.txt"; + ConfigGetter<FunctionTestConfig> getter = new ConfigGetter<>(FunctionTestConfig.class); + FunctionTestConfig config = getter.getConfig(configId); + assertFalse(config.bool_val()); + assertTrue(config.bool_with_def()); + assertEquals(5, config.int_val()); + assertEquals(-14, config.int_with_def()); + assertEquals(666000666000L, config.long_val()); + assertEquals(-333000333000L, config.long_with_def()); + assertEquals(41.23, config.double_val(), 0.000001); + assertEquals(-12, config.double_with_def(), 0.000001); + assertEquals("foo", config.string_val()); + assertEquals("bar and foo", config.stringwithdef()); + assertEquals(FunctionTestConfig.Enum_val.FOOBAR, config.enum_val()); + assertEquals(FunctionTestConfig.Enumwithdef.BAR2, config.enumwithdef()); + assertEquals(configId, config.refval()); + assertEquals(configId, config.refwithdef()); + assertEquals("autoexec.bat", config.fileVal().value()); + assertEquals(1, config.boolarr().size()); + assertEquals(0, config.intarr().size()); + assertEquals(0, config.longarr().size()); + assertEquals(2, config.doublearr().size()); + assertEquals(1, config.stringarr().size()); + assertEquals(1, config.enumarr().size()); + assertEquals(0, config.refarr().size()); + assertEquals(0, config.fileArr().size()); + assertEquals(2, config.myarray().size()); + } + + @Test + public void testLackingDefaults() throws IOException { + attemptLacking("bool_val", false); + attemptLacking("int_val", false); + attemptLacking("long_val", false); + attemptLacking("double_val", false); + attemptLacking("string_val", false); + attemptLacking("enum_val", false); + attemptLacking("refval", false); + attemptLacking("fileVal", false); + + attemptLacking("boolarr", true); + attemptLacking("intarr", true); + attemptLacking("longarr", true); + attemptLacking("doublearr", true); + attemptLacking("enumarr", true); + attemptLacking("stringarr", true); + attemptLacking("refarr", true); + attemptLacking("fileArr", true); + attemptLacking("myarray", true); + + attemptLacking("basicStruct.bar", false); + attemptLacking("rootStruct.inner0.index", false); + attemptLacking("rootStruct.inner1.index", false); +// attemptLacking("rootStruct.innerArr[0].stringVal", false); + + attemptLacking("myarray[0].stringval", true); + attemptLacking("myarray[0].refval", false); + attemptLacking("myarray[0].anotherarray", true); + attemptLacking("myarray[0].anotherarray", true); + attemptLacking("myarray[0].myStruct.a", false); + } + + private void attemptLacking(String param, boolean isArray) throws IOException { + BufferedReader in = new BufferedReader(new FileReader(new File(PATH + "defaultvalues.txt"))); + StringBuilder config = new StringBuilder(); + String line; + while ((line = in.readLine()) != null) { + if ((line.length() > param.length()) && + param.equals(line.substring(0, param.length())) && + (line.charAt(param.length()) == ' ' || line.charAt(param.length()) == '[')) { + // Ignore values matched + } else { + config.append(line).append("\n"); + } + } + //System.out.println("Config lacking " + param + "-> " + config + "\n"); + try { + ConfigGetter<FunctionTestConfig> getter = new ConfigGetter<FunctionTestConfig>(FunctionTestConfig.class); + getter.getConfig("raw:\n" + config); + if (isArray) { + // Arrays are empty by default + return; + } + fail("Expected to fail when not specifying value " + param + " without default"); + } catch (IllegalArgumentException expected) { + if (isArray) { + fail("Arrays should be empty by default."); + } + } + } + + public static void assertVariableAccessValues(FunctionTestConfig config, String configId) { + assertTrue(config.bool_with_def()); + assertEquals(5, config.int_val()); + assertEquals(-14, config.int_with_def()); + assertEquals(12345678901L, config.long_val()); + assertEquals(-9876543210L, config.long_with_def()); + assertEquals(41.23, config.double_val(), 0.000001); + assertEquals(-12, config.double_with_def(), 0.000001); + assertEquals("foo", config.string_val()); + assertEquals("bar and foo", config.stringwithdef()); + assertEquals(FunctionTestConfig.Enum_val.FOOBAR, config.enum_val()); + assertEquals(FunctionTestConfig.Enumwithdef.BAR2, config.enumwithdef()); + assertEquals(configId, config.refval()); + assertEquals(configId, config.refwithdef()); + assertEquals("etc", config.fileVal().value()); + assertEquals(1, config.boolarr().size()); + assertEquals(1, config.boolarr().size()); // new api with accessor for a List of the original Java type + assertEquals(false, config.boolarr().get(0)); // new List api + assertEquals(false, config.boolarr(0)); // short-hand + assertEquals(0, config.intarr().size()); + assertEquals(2, config.longarr().size()); + assertEquals(Long.MAX_VALUE, config.longarr(0)); + assertThat(config.longarr().get(1), is(Long.MIN_VALUE)); + assertEquals(2, config.doublearr().size()); + assertEquals(1, config.stringarr().size()); + assertEquals(1, config.enumarr().size()); + assertEquals(FunctionTestConfig.Enumarr.VALUES, config.enumarr().get(0)); // new List api, don't have to call value() + assertEquals(3, config.refarr().size()); + assertEquals(1, config.fileArr().size()); + assertEquals(configId, config.refarr(0)); + assertEquals(":parent", config.refarr(1)); + assertEquals("parent:", config.refarr(2)); + assertEquals("bin", config.fileArr(0).value()); + assertEquals("pom.xml", config.pathArr(0).toString()); + assertEquals("pom.xml", config.pathMap("one").toString()); + + assertEquals("basicFoo", config.basicStruct().foo()); + assertEquals(3, config.basicStruct().bar()); // new List api + assertEquals(2, config.basicStruct().intArr().size()); + assertThat(config.basicStruct().intArr().get(0), is(310)); // new List api + assertThat(config.basicStruct().intArr().get(1), is(311)); // new List api + assertEquals(310, config.basicStruct().intArr(0)); // short-hand + assertEquals("inner0", config.rootStruct().inner0().name()); // new List api + assertEquals(11, config.rootStruct().inner0().index()); + assertEquals("inner1", config.rootStruct().inner1().name()); + assertEquals(12, config.rootStruct().inner1().index()); + assertEquals(2, config.rootStruct().innerArr().size()); + assertEquals(true, config.rootStruct().innerArr(0).boolVal()); + assertEquals("deep", config.rootStruct().innerArr(0).stringVal()); + assertEquals(false, config.rootStruct().innerArr(1).boolVal()); + assertEquals("blue a=\"escaped\"", config.rootStruct().innerArr(1).stringVal()); + + assertEquals(2, config.myarray().size()); // new List api + assertEquals(configId, config.myarray().get(0).refval()); // new List api + assertEquals(configId, config.myarray(0).refval()); // short-hand + assertEquals("file0", config.myarray(0).fileVal().value()); + assertEquals(1, config.myarray(0).myStruct().a()); + assertEquals(2, config.myarray(0).myStruct().b()); + assertEquals(configId, config.myarray(1).refval()); + assertEquals("file1", config.myarray(1).fileVal().value()); + assertEquals(-1, config.myarray(1).myStruct().a()); + assertEquals(-2, config.myarray(1).myStruct().b()); + } + +} diff --git a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java new file mode 100644 index 00000000000..26b47f7621c --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import java.util.*; + +import com.yahoo.config.subscription.impl.GenericConfigHandle; +import com.yahoo.config.subscription.impl.GenericConfigSubscriber; +import com.yahoo.config.subscription.impl.JRTConfigRequester; +import com.yahoo.config.subscription.impl.JRTConfigRequesterTest; +import com.yahoo.config.subscription.impl.MockConnection; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.JRTConnectionPool; +import com.yahoo.vespa.config.protocol.CompressionType; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * + * Test cases for the "generic" (class-less) subscription mechanism. + * + * @author lulf + * @since 5.1 + */ +public class GenericConfigSubscriberTest { + + @Test + public void testSubscribeGeneric() { + Map<ConfigSourceSet, JRTConfigRequester> requesters = new HashMap<>(); + ConfigSourceSet sourceSet = new ConfigSourceSet("blabla"); + requesters.put(sourceSet, JRTConfigRequester.get(new MockConnection(), JRTConfigRequesterTest.getTestTimingValues())); + GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters); + final List<String> defContent = Arrays.asList("myVal int"); + GenericConfigHandle handle = sub.subscribe(new ConfigKey<>("simpletypes", "id", "config"), defContent, sourceSet, JRTConfigRequesterTest.getTestTimingValues()); + assertTrue(sub.nextConfig()); + assertTrue(handle.isChanged()); + assertThat(handle.getRawConfig().getPayload().withCompression(CompressionType.UNCOMPRESSED).toString(), is("{}")); // MockConnection returns empty string + assertFalse(sub.nextConfig()); + assertFalse(handle.isChanged()); + } + + @Test + public void testGenericRequesterPooling() { + ConfigSourceSet source1 = new ConfigSourceSet("tcp/foo:78"); + ConfigSourceSet source2 = new ConfigSourceSet("tcp/bar:79"); + JRTConfigRequester req1 = JRTConfigRequester.get(new JRTConnectionPool(source1), JRTConfigRequesterTest.getTestTimingValues()); + JRTConfigRequester req2 = JRTConfigRequester.get(new JRTConnectionPool(source2), JRTConfigRequesterTest.getTestTimingValues()); + Map<ConfigSourceSet, JRTConfigRequester> requesters = new LinkedHashMap<>(); + requesters.put(source1, req1); + requesters.put(source2, req2); + GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters); + assertEquals(sub.requesters().get(source1).getConnectionPool().getCurrent().getAddress(), "tcp/foo:78"); + assertEquals(sub.requesters().get(source2).getConnectionPool().getCurrent().getAddress(), "tcp/bar:79"); + + } + + @Test(expected=UnsupportedOperationException.class) + public void testOverriddenSubscribeInvalid1() { + GenericConfigSubscriber sub = new GenericConfigSubscriber(); + sub.subscribe(null, null); + } + + @Test(expected=UnsupportedOperationException.class) + public void testOverriddenSubscribeInvalid2() { + GenericConfigSubscriber sub = new GenericConfigSubscriber(); + sub.subscribe(null, null, 0l); + } + + @Test(expected=UnsupportedOperationException.class) + public void testOverriddenSubscribeInvalid3() { + GenericConfigSubscriber sub = new GenericConfigSubscriber(); + sub.subscribe(null, null, ""); + } +}
\ No newline at end of file diff --git a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java new file mode 100644 index 00000000000..90be475bb94 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.myproject.config.NamespaceConfig; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author gjoranv + */ +public class NamespaceTest { + @Test + public void verifyConfigClassWithExplicitNamespace() { + NamespaceConfig config = new ConfigGetter<>(NamespaceConfig.class).getConfig("raw: a 0\n"); + assertThat(config.a(), is(0)); + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java new file mode 100644 index 00000000000..355c94e03b6 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription; + +import com.yahoo.foo.UnicodeConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Tests reading of a config containing unicode characters in UTF-8 + * + * @author Vidar Larsen + * @author Harald Musum + */ +public class UnicodeTest { + /** + * Reads a config from a file which is exactly like one returned from + * the config server given only default values for this config. + * The parsing code is the same whether the reading happens from file + * or from a server connection, so this tests that this config can be + * received correctly from the server + */ + @Test + public void testUnicodeConfigReading() { + ConfigGetter<UnicodeConfig> getter = new ConfigGetter<>(UnicodeConfig.class); + UnicodeConfig config = getter.getConfig("file:src/test/resources/configs/unicode/unicode.cfg"); + + assertEquals("Hei \u00E6\u00F8\u00E5 \uBC14\uB451 \u00C6\u00D8\u00C5 hallo", config.unicodestring1()); + assertEquals("abc \u00E6\u00F8\u00E5 \u56F2\u7881 \u00C6\u00D8\u00C5 ABC", config.unicodestring2()); + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java new file mode 100644 index 00000000000..ce9323f0ec7 --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java @@ -0,0 +1,105 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.foo.TestReferenceConfig; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.config.subscription.DirSource; +import com.yahoo.config.subscription.FileSource; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.TimingValues; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1.7 + */ +public class FileConfigSubscriptionTest { + private File TEST_TYPES_FILE; + + @Before + public void setUp() throws IOException { + TEST_TYPES_FILE = File.createTempFile("fooconfig", ".cfg"); + } + + private void writeConfig(String field, String value) throws IOException { + FileWriter writer = new FileWriter(TEST_TYPES_FILE); + writer.write(field + " " + value); + writer.close(); + } + + @Test + public void require_that_new_config_is_detected_in_time() throws IOException, InterruptedException { + writeConfig("intval", "23"); + ConfigSubscriber subscriber = new ConfigSubscriber(new FileSource(TEST_TYPES_FILE)); + ConfigSubscription<SimpletypesConfig> sub = new FileConfigSubscription<>( + new ConfigKey<>(SimpletypesConfig.class, ""), + subscriber, + TEST_TYPES_FILE); + assertTrue(sub.nextConfig(1000)); + assertThat(sub.config.intval(), is(23)); + Thread.sleep(1000); + writeConfig("intval", "33"); + assertTrue(sub.nextConfig(1000)); + assertThat(sub.config.intval(), is(33)); + } + + @Test + public void require_that_new_config_is_detected_on_reload() throws IOException { + writeConfig("intval", "23"); + ConfigSubscriber subscriber = new ConfigSubscriber(new FileSource(TEST_TYPES_FILE)); + ConfigSubscription<SimpletypesConfig> sub = new FileConfigSubscription<>( + new ConfigKey<>(SimpletypesConfig.class, ""), + subscriber, + TEST_TYPES_FILE); + assertTrue(sub.nextConfig(1000)); + assertThat(sub.config.intval(), is(23)); + writeConfig("intval", "33"); + sub.reload(1); + assertTrue(sub.nextConfig(1000)); + assertThat(sub.config.intval(), is(33)); + assertTrue(sub.isConfigChanged()); + assertTrue(sub.isGenerationChanged()); + sub.reload(2); + assertTrue(sub.nextConfig(1000)); + assertThat(sub.config.intval(), is(33)); + assertFalse(sub.isConfigChanged()); + assertTrue(sub.isGenerationChanged()); + } + + @Test + public void require_that_dir_config_id_reference_is_not_changed() { + final String cfgDir = "src/test/resources/configs/foo"; + final String cfgId = "dir:" + cfgDir; + final ConfigKey<TestReferenceConfig> key = new ConfigKey<>(TestReferenceConfig.class, cfgId); + ConfigSubscriber subscriber = new ConfigSubscriber(); + ConfigSubscription<TestReferenceConfig> sub = ConfigSubscription.get(key, subscriber, new DirSource(new File(cfgDir)), new TimingValues()); + assertTrue(sub.nextConfig(1000)); + assertThat(sub.config.configId(), is(cfgId)); + } + + @Test(expected = ConfigurationRuntimeException.class) + public void require_that_bad_file_throws_exception() throws IOException { + // A little trick to ensure that we can create the subscriber, but that we get an error when reading. + writeConfig("intval", "23"); + ConfigSubscriber subscriber = new ConfigSubscriber(new FileSource(TEST_TYPES_FILE)); + ConfigSubscription<SimpletypesConfig> sub = new FileConfigSubscription<>( + new ConfigKey<>(SimpletypesConfig.class, ""), + subscriber, + TEST_TYPES_FILE); + sub.reload(1); + assertTrue(TEST_TYPES_FILE.setReadable(false)); + sub.nextConfig(0); + } +} diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java new file mode 100644 index 00000000000..7d3b86db2fe --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java @@ -0,0 +1,356 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.impl; + +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.jrt.Request; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ErrorCode; +import com.yahoo.vespa.config.ErrorType; +import com.yahoo.vespa.config.TimingValues; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 5.1.11 + */ +public class JRTConfigRequesterTest { + + @Test + public void testDelayCalculation() { + TimingValues defaultTimingValues = new TimingValues(); + Random random = new Random(0); // Use seed to make tests predictable + TimingValues timingValues = new TimingValues(defaultTimingValues, random); + + // transientFailures and fatalFailures are not set until after delay has been calculated, + // so 0 is the case for the first failure + int transientFailures = 0; + int fatalFailures = 0; + boolean configured = false; + + // First time failure, not configured + long delay = JRTConfigRequester.calculateFailedRequestDelay(ErrorType.TRANSIENT, + transientFailures, fatalFailures, timingValues, configured); + assertThat(delay, is(timingValues.getUnconfiguredDelay())); + transientFailures = 5; + delay = JRTConfigRequester.calculateFailedRequestDelay(ErrorType.TRANSIENT, + transientFailures, fatalFailures, timingValues, configured); + assertThat(delay, is((transientFailures + 1) * timingValues.getUnconfiguredDelay())); + transientFailures = 0; + + + delay = JRTConfigRequester.calculateFailedRequestDelay(ErrorType.FATAL, + transientFailures, fatalFailures, timingValues, configured); + assertTrue(delay > (1 - JRTConfigRequester.randomFraction) * timingValues.getFixedDelay()); + assertTrue(delay < (1 + JRTConfigRequester.randomFraction) * timingValues.getFixedDelay()); + assertThat(delay, is(5462L)); + + // First time failure, configured + configured = true; + + delay = JRTConfigRequester.calculateFailedRequestDelay(ErrorType.TRANSIENT, + transientFailures, fatalFailures, timingValues, configured); + assertThat(delay, is(timingValues.getConfiguredErrorDelay())); + + delay = JRTConfigRequester.calculateFailedRequestDelay(ErrorType.FATAL, + transientFailures, fatalFailures, timingValues, configured); + assertTrue(delay > (1 - JRTConfigRequester.randomFraction) * timingValues.getFixedDelay()); + assertTrue(delay < (1 + JRTConfigRequester.randomFraction) * timingValues.getFixedDelay()); + assertThat(delay, is(5663L)); + + + // nth time failure, not configured + fatalFailures = 1; + configured = false; + delay = JRTConfigRequester.calculateFailedRequestDelay(ErrorType.TRANSIENT, + transientFailures, fatalFailures, timingValues, configured); + assertThat(delay, is(timingValues.getUnconfiguredDelay())); + delay = JRTConfigRequester.calculateFailedRequestDelay(ErrorType.FATAL, + transientFailures, fatalFailures, timingValues, configured); + final long l = timingValues.getFixedDelay() + timingValues.getUnconfiguredDelay(); + assertTrue(delay > (1 - JRTConfigRequester.randomFraction) * l); + assertTrue(delay < (1 + JRTConfigRequester.randomFraction) * l); + assertThat(delay, is(5377L)); + + + // nth time failure, configured + fatalFailures = 1; + configured = true; + delay = JRTConfigRequester.calculateFailedRequestDelay(ErrorType.TRANSIENT, + transientFailures, fatalFailures, timingValues, configured); + assertThat(delay, is(timingValues.getConfiguredErrorDelay())); + delay = JRTConfigRequester.calculateFailedRequestDelay(ErrorType.FATAL, + transientFailures, fatalFailures, timingValues, configured); + final long l1 = timingValues.getFixedDelay() + timingValues.getConfiguredErrorDelay(); + assertTrue(delay > (1 - JRTConfigRequester.randomFraction) * l1); + assertTrue(delay < (1 + JRTConfigRequester.randomFraction) * l1); + assertThat(delay, is(20851L)); + + + // 1 more than max delay multiplier time failure, configured + fatalFailures = timingValues.getMaxDelayMultiplier() + 1; + configured = true; + delay = JRTConfigRequester.calculateFailedRequestDelay(ErrorType.TRANSIENT, + transientFailures, fatalFailures, timingValues, configured); + assertThat(delay, is(timingValues.getConfiguredErrorDelay())); + assertTrue(delay < timingValues.getMaxDelayMultiplier() * timingValues.getConfiguredErrorDelay()); + delay = JRTConfigRequester.calculateFailedRequestDelay(ErrorType.FATAL, + transientFailures, fatalFailures, timingValues, configured); + final long l2 = timingValues.getFixedDelay() + timingValues.getMaxDelayMultiplier() * timingValues.getConfiguredErrorDelay(); + assertTrue(delay > (1 - JRTConfigRequester.randomFraction) * l2); + assertTrue(delay < (1 + JRTConfigRequester.randomFraction) * l2); + assertThat(delay, is(163520L)); + } + + @Test + public void testDelay() { + TimingValues timingValues = new TimingValues(); + + // transientFailures and fatalFailures are not set until after delay has been calculated, + // so 0 is the case for the first failure + int transientFailures = 0; + int fatalFailures = 0; + + // First time failure, configured + long delay = JRTConfigRequester.calculateFailedRequestDelay(ErrorType.TRANSIENT, + transientFailures, fatalFailures, timingValues, true); + assertThat(delay, is(timingValues.getConfiguredErrorDelay())); + assertThat(delay, is((transientFailures + 1) * timingValues.getConfiguredErrorDelay())); + } + + @Test + public void testErrorTypes() { + List<Integer> transientErrors = Arrays.asList(com.yahoo.jrt.ErrorCode.CONNECTION, com.yahoo.jrt.ErrorCode.TIMEOUT); + List<Integer> fatalErrors = Arrays.asList(ErrorCode.UNKNOWN_CONFIG, ErrorCode.UNKNOWN_DEFINITION, ErrorCode.OUTDATED_CONFIG, + ErrorCode.UNKNOWN_DEF_MD5, ErrorCode.ILLEGAL_NAME, ErrorCode.ILLEGAL_VERSION, ErrorCode.ILLEGAL_CONFIGID, + ErrorCode.ILLEGAL_DEF_MD5, ErrorCode.ILLEGAL_CONFIG_MD5, ErrorCode.ILLEGAL_TIMEOUT, ErrorCode.INTERNAL_ERROR, + 9999); // unknown should also be fatal + for (Integer i : transientErrors) { + assertThat(ErrorType.getErrorType(i), is(ErrorType.TRANSIENT)); + } + for (Integer i : fatalErrors) { + assertThat(ErrorType.getErrorType(i), is(ErrorType.FATAL)); + } + } + + @Test + public void testFirstRequestAfterSubscribing() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + final TimingValues timingValues = getTestTimingValues(); + JRTConfigSubscription<SimpletypesConfig> sub = createSubscription(subscriber, timingValues); + + final MockConnection connection = new MockConnection(); + JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues); + assertThat(requester.getConnectionPool(), is(connection)); + requester.request(sub); + final Request request = connection.getRequest(); + assertNotNull(request); + assertThat(connection.getNumberOfRequests(), is(1)); + JRTServerConfigRequestV3 receivedRequest = JRTServerConfigRequestV3.createFromRequest(request); + assertTrue(receivedRequest.validateParameters()); + assertThat(receivedRequest.getTimeout(), is(timingValues.getSubscribeTimeout())); + assertThat(requester.getFatalFailures(), is(0)); + assertThat(requester.getTransientFailures(), is(0)); + } + + @Test + public void testFatalError() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + final TimingValues timingValues = getTestTimingValues(); + + final MockConnection connection = new MockConnection(new ErrorResponseHandler()); + JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues); + requester.request(createSubscription(subscriber, timingValues)); + waitUntilResponse(connection); + assertThat(requester.getFatalFailures(), is(1)); + assertThat(requester.getTransientFailures(), is(0)); + } + + @Test + public void testFatalErrorSubscribed() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + final TimingValues timingValues = getTestTimingValues(); + JRTConfigSubscription<SimpletypesConfig> sub = createSubscription(subscriber, timingValues); + sub.setConfig(config()); + sub.setGeneration(1L); + + final MockConnection connection = new MockConnection(new ErrorResponseHandler()); + JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues); + requester.request(sub); + waitUntilResponse(connection); + assertThat(requester.getFatalFailures(), is(1)); + assertThat(requester.getTransientFailures(), is(0)); + } + + @Test + public void testTransientError() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + final TimingValues timingValues = getTestTimingValues(); + + final MockConnection connection = new MockConnection(new ErrorResponseHandler(com.yahoo.jrt.ErrorCode.TIMEOUT)); + JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues); + requester.request(createSubscription(subscriber, timingValues)); + waitUntilResponse(connection); + assertThat(requester.getFatalFailures(), is(0)); + assertThat(requester.getTransientFailures(), is(1)); + } + + @Test + public void testTransientErrorSubscribed() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + final TimingValues timingValues = getTestTimingValues(); + JRTConfigSubscription<SimpletypesConfig> sub = createSubscription(subscriber, timingValues); + sub.setConfig(config()); + sub.setGeneration(1L); + + final MockConnection connection = new MockConnection(new ErrorResponseHandler(com.yahoo.jrt.ErrorCode.TIMEOUT)); + JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues); + requester.request(sub); + waitUntilResponse(connection); + assertThat(requester.getFatalFailures(), is(0)); + assertThat(requester.getTransientFailures(), is(1)); + } + + @Test + public void testUnknownConfigDefinitionError() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + final TimingValues timingValues = getTestTimingValues(); + JRTConfigSubscription<SimpletypesConfig> sub = createSubscription(subscriber, timingValues); + sub.setConfig(config()); + sub.setGeneration(1L); + + final MockConnection connection = new MockConnection(new ErrorResponseHandler(ErrorCode.UNKNOWN_DEFINITION)); + JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues); + assertThat(requester.getConnectionPool(), is(connection)); + requester.request(sub); + waitUntilResponse(connection); + assertThat(requester.getFatalFailures(), is(1)); + assertThat(requester.getTransientFailures(), is(0)); + // TODO Check that no further request was sent? + } + + @Test + public void testClosedSubscription() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + final TimingValues timingValues = getTestTimingValues(); + JRTConfigSubscription<SimpletypesConfig> sub = createSubscription(subscriber, timingValues); + sub.close(); + + final MockConnection connection = new MockConnection(new MockConnection.OKResponseHandler()); + JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues); + requester.request(sub); + assertThat(connection.getNumberOfRequests(), is(1)); + // Check that no further request was sent? + try { + Thread.sleep(timingValues.getFixedDelay()*2); + } catch (InterruptedException e) { + e.printStackTrace(); + } + assertThat(connection.getNumberOfRequests(), is(1)); + } + + @Test + public void testTimeout() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + final TimingValues timingValues = getTestTimingValues(); + JRTConfigSubscription<SimpletypesConfig> sub = createSubscription(subscriber, timingValues); + sub.close(); + + final MockConnection connection = new MockConnection( + new DelayedResponseHandler(timingValues.getSubscribeTimeout()), + 2); // fake that we have more than one source + JRTConfigRequester requester = new JRTConfigRequester(connection, timingValues); + requester.request(createSubscription(subscriber, timingValues)); + // Check that no further request was sent? + try { + Thread.sleep(timingValues.getFixedDelay()*2); + } catch (InterruptedException e) { + e.printStackTrace(); + } + assertTrue(connection.getNumberOfFailovers() >= 1); + } + + private JRTConfigSubscription<SimpletypesConfig> createSubscription(ConfigSubscriber subscriber, TimingValues timingValues) { + return new JRTConfigSubscription<>( + new ConfigKey<>(SimpletypesConfig.class, "testid"), subscriber, null, timingValues); + } + + private SimpletypesConfig config() { + SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); + return new SimpletypesConfig(builder); + } + + private void waitUntilResponse(MockConnection connection) { + int i = 0; + while (i < 1000 && connection.getRequest() == null) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + e.printStackTrace(); + } + i++; + } + } + + public static TimingValues getTestTimingValues() { return new TimingValues( + 1000, // successTimeout + 500, // errorTimeout + 500, // initialTimeout + 2000, // subscribeTimeout + 250, // unconfiguredDelay + 500, // configuredErrorDelay + 250, // fixedDelay + 5); // maxDelayMultiplier + } + + private static class ErrorResponseHandler extends MockConnection.OKResponseHandler { + private final int errorCode; + + public ErrorResponseHandler() { + this(ErrorCode.INTERNAL_ERROR); + } + + public ErrorResponseHandler(int errorCode) { + this.errorCode = errorCode; + } + + @Override + public void run() { + System.out.println("Running error response handler"); + request().setError(errorCode, "error"); + requestWaiter().handleRequestDone(request()); + } + } + + private static class DelayedResponseHandler extends MockConnection.OKResponseHandler { + private final long waitTimeMilliSeconds; + + public DelayedResponseHandler(long waitTimeMilliSeconds) { + this.waitTimeMilliSeconds = waitTimeMilliSeconds; + } + + @Override + public void run() { + System.out.println("Running delayed response handler (waiting " + waitTimeMilliSeconds + + ") before responding"); + try { + Thread.sleep(waitTimeMilliSeconds); + } catch (InterruptedException e) { + e.printStackTrace(); + } + request().setError(com.yahoo.jrt.ErrorCode.TIMEOUT, "error"); + requestWaiter().handleRequestDone(request()); + } + } + +} diff --git a/config/src/test/java/com/yahoo/config/subscription/util/JsonHelper.java b/config/src/test/java/com/yahoo/config/subscription/util/JsonHelper.java new file mode 100644 index 00000000000..27ac1a1278c --- /dev/null +++ b/config/src/test/java/com/yahoo/config/subscription/util/JsonHelper.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.subscription.util; + +import com.google.common.base.Joiner; + +import static org.hamcrest.MatcherAssert.assertThat; +import static uk.co.datumedge.hamcrest.json.SameJSONAs.sameJSONAs; + +/** + * @author Vegard Sjonfjell + */ +public class JsonHelper { + /** + * Convenience method to input JSON without escaping double quotes and newlines + * Each parameter represents a line of JSON encoded data + * The lines are joined with newline and single quotes are replaced with double quotes + */ + public static String inputJson(String... lines) { + return Joiner.on("\n").join(lines).replaceAll("'", "\""); + } + + /** + * Structurally compare two JSON encoded strings + */ + public static void assertJsonEquals(String inputJson, String expectedJson) { + assertThat(inputJson, sameJSONAs(expectedJson)); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java new file mode 100644 index 00000000000..95ed049de6d --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.foo.ArraytypesConfig; +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.foo.StructtypesConfig; +import com.yahoo.foo.MaptypesConfig; +import org.junit.Test; + +import java.util.Arrays; + +import static com.yahoo.foo.MaptypesConfig.Innermap; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * SEO keywords: test override() on builders. overrideTest, testOverride + * + * @author lulf + * @since 5.1 + */ +public class ConfigBuilderMergeTest { + + private SimpletypesConfig.Builder createSimpleBuilder(String s, int i, long l, double d, boolean b) { + SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); + builder.stringval(s); + builder.intval(i); + builder.longval(l); + builder.doubleval(d); + builder.boolval(b); + return builder; + } + + private ArraytypesConfig.Builder createArrayBuilder(String [] strings) { + ArraytypesConfig.Builder builder = new ArraytypesConfig.Builder(); + for (String str : strings) { + builder.stringarr(str); + } + return builder; + } + + private StructtypesConfig.Builder createSimpleStructBuilder(String name, String gender, String [] emails) { + StructtypesConfig.Builder builder = new StructtypesConfig.Builder(); + StructtypesConfig.Simple.Builder simpleBuilder = new StructtypesConfig.Simple.Builder(); + simpleBuilder.name(name); + simpleBuilder.gender(StructtypesConfig.Simple.Gender.Enum.valueOf(gender)); + simpleBuilder.emails(Arrays.asList(emails)); + builder.simple(simpleBuilder); + return builder; + } + + @Test + public void require_that_simple_fields_are_overwritten_on_merge() { + SimpletypesConfig.Builder b1 = createSimpleBuilder("foo", 2, 5, 4.3, false); + SimpletypesConfig.Builder b2 = createSimpleBuilder("bar", 3, 6, 3.3, true); + ConfigInstanceUtil.setValues(b1, b2); + SimpletypesConfig c1 = new SimpletypesConfig(b1); + SimpletypesConfig c2 = new SimpletypesConfig(b2); + assertThat(c1, is(c2)); + } + + @Test + public void require_that_arrays_are_appended_on_merge() { + ArraytypesConfig.Builder b1 = createArrayBuilder(new String[] { "foo", "bar" }); + ArraytypesConfig.Builder b2 = createArrayBuilder(new String[] { "baz", "bim" }); + + ConfigInstanceUtil.setValues(b1, b2); + ArraytypesConfig c1 = new ArraytypesConfig(b1); + assertThat(c1.stringarr().size(), is(4)); + assertThat(c1.stringarr(0), is("foo")); + assertThat(c1.stringarr(1), is("bar")); + assertThat(c1.stringarr(2), is("baz")); + assertThat(c1.stringarr(3), is("bim")); + + ArraytypesConfig c2 = new ArraytypesConfig(b2); + assertThat(c2.stringarr(0), is("baz")); + assertThat(c2.stringarr(1), is("bim")); + } + + @Test + public void require_that_struct_fields_are_overwritten() { + String name1 = "foo"; + String gender1 = "MALE"; + String emails1[] = { "foo@bar", "bar@foo" }; + String name2 = "bar"; + String gender2 = "FEMALE"; + String emails2[] = { "foo@bar", "bar@foo" }; + StructtypesConfig.Builder b1 = createSimpleStructBuilder(name1, gender1, emails1); + StructtypesConfig.Builder b2 = createSimpleStructBuilder(name2, gender2, emails2); + ConfigInstanceUtil.setValues(b1, b2); + StructtypesConfig c1 = new StructtypesConfig(b1); + assertThat(c1.simple().name(), is(name2)); + assertThat(c1.simple().gender().toString(), is(gender2)); + assertThat(c1.simple().emails(0), is(emails2[0])); + assertThat(c1.simple().emails(1), is(emails2[1])); + } + + @Test + public void source_map_is_copied_into_destination_map_on_merge() { + MaptypesConfig.Builder destination = new MaptypesConfig.Builder() + .intmap("one", 1) + .innermap("first", new Innermap.Builder() + .foo(1)); + + MaptypesConfig.Builder source = new MaptypesConfig.Builder() + .intmap("two", 2) + .innermap("second", new Innermap.Builder() + .foo(2)); + + ConfigInstanceUtil.setValues(destination, source); + + MaptypesConfig config = new MaptypesConfig(destination); + assertThat(config.intmap("one"), is(1)); + assertThat(config.intmap("two"), is(2)); + assertThat(config.innermap("first").foo(), is(1)); + assertThat(config.innermap("second").foo(), is(2)); + } + +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java new file mode 100755 index 00000000000..5356acdfade --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.*; + +/** + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class ConfigCacheKeyTest { + @Test + public void testConfigCacheKey() { + final String defMd5 = "md5"; + final String defMd5_2 = "md5_2"; + + ConfigCacheKey k1 = new ConfigCacheKey("foo", "id", "ns", defMd5); + ConfigCacheKey k2 = new ConfigCacheKey("foo", "id", "ns", defMd5); + ConfigCacheKey k3 = new ConfigCacheKey("foo", "id", "ns", defMd5_2); + ConfigCacheKey k4 = new ConfigCacheKey("foo", "id", "ns_1", defMd5); + ConfigCacheKey k5 = new ConfigCacheKey("foo", "id", "ns_1", null); // test with null defMd5 + final ConfigKey<?> configKey = new ConfigKey<>("foo", "id", "ns"); + ConfigCacheKey k1_2 = new ConfigCacheKey(configKey, defMd5); + assertTrue(k1.equals(k1)); + assertTrue(k1.equals(k1_2)); + assertTrue(k1.equals(k2)); + assertFalse(k3.equals(k2)); + assertFalse(k4.equals(k1)); + assertThat(k1.hashCode(), is(k2.hashCode())); + assertThat(k1.getDefMd5(), is(defMd5)); + assertThat(k1.toString(), is(configKey.toString() + "," + defMd5)); + assertThat(k5.hashCode(), is(not(k1.hashCode()))); + } + +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java new file mode 100644 index 00000000000..cb1cccc0139 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.codegen.DefParser; +import org.junit.Test; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + + +/** + * Unit tests for ConfigDefinitionBuilder. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class ConfigDefinitionBuilderTest { + + private static final String TEST_DIR = "src/test/resources/configs/def-files"; + private static final String DEF_NAME = TEST_DIR + "/function-test.def"; + + + @Test + // TODO Test ranges + public void testCreateConfigDefinition() throws IOException, InterruptedException { + File defFile = new File(DEF_NAME); + DefParser defParser = new DefParser(defFile.getName(), new FileReader(defFile)); + CNode root = defParser.getTree(); + + ConfigDefinition def = ConfigDefinitionBuilder.createConfigDefinition(root); + + assertNotNull(def); + assertThat(def.getBoolDefs().size(), is(2)); + assertNull(def.getBoolDefs().get("bool_val").getDefVal()); + assertThat(def.getBoolDefs().get("bool_with_def").getDefVal(), is(false)); + + assertThat(def.getIntDefs().size(), is(2)); + assertNull(def.getIntDefs().get("int_val").getDefVal()); + assertThat(def.getIntDefs().get("int_with_def").getDefVal(), is(-545)); + + assertThat(def.getLongDefs().size(), is(2)); + assertNull(def.getLongDefs().get("long_val").getDefVal()); + assertThat(def.getLongDefs().get("long_with_def").getDefVal(), is(-50000000000L)); + + assertThat(def.getDoubleDefs().size(), is(2)); + assertNull(def.getDoubleDefs().get("double_val").getDefVal()); + assertThat(def.getDoubleDefs().get("double_with_def").getDefVal(), is(-6.43)); + + assertThat(def.getEnumDefs().size(), is(3)); + assertTrue(def.getEnumDefs().containsKey("enum_val")); + assertThat(def.getEnumDefs().get("enum_val").getVals().size(), is(3)); + assertThat(def.getEnumDefs().get("enum_val").getVals().get(0), is("FOO")); + assertThat(def.getEnumDefs().get("enum_val").getVals().get(1), is("BAR")); + assertThat(def.getEnumDefs().get("enum_val").getVals().get(2), is("FOOBAR")); + + assertTrue(def.getEnumDefs().containsKey("enumwithdef")); + assertThat(def.getEnumDefs().get("enumwithdef").getDefVal(), is("BAR2")); + + assertTrue(def.getEnumDefs().containsKey("onechoice")); + assertThat(def.getEnumDefs().get("onechoice").getDefVal(), is("ONLYFOO")); + + assertThat(def.getStringDefs().size(), is(2)); + assertNull(def.getStringDefs().get("string_val").getDefVal()); // The return value is a String, so null if no default value + assertThat(def.getStringDefs().get("stringwithdef").getDefVal(), is("foobar")); + + assertThat(def.getReferenceDefs().size(), is(2)); + assertNotNull(def.getReferenceDefs().get("refval")); + assertThat(def.getReferenceDefs().get("refwithdef").getDefVal(), is(":parent:")); + + assertThat(def.getFileDefs().size(), is(1)); + assertNotNull(def.getFileDefs().get("fileVal")); + + assertThat(def.getArrayDefs().size(), is(9)); + assertNotNull(def.getArrayDefs().get("boolarr")); + assertThat(def.getArrayDefs().get("boolarr").getTypeSpec().getType(), is("bool")); + + assertNotNull(def.getArrayDefs().get("enumarr")); + assertThat(def.getArrayDefs().get("enumarr").getTypeSpec().getType(), is("enum")); + assertThat(def.getArrayDefs().get("enumarr").getTypeSpec().getEnumVals().toString(), is("[ARRAY, VALUES]")); + + assertNotNull(def.getArrayDefs().get("refarr")); + assertThat(def.getArrayDefs().get("refarr").getTypeSpec().getType(), is("reference")); + + assertNotNull(def.getArrayDefs().get("fileArr")); + assertThat(def.getArrayDefs().get("fileArr").getTypeSpec().getType(), is("file")); + + assertThat(def.getStructDefs().size(), is(2)); + assertNotNull(def.getStructDefs().get("basicStruct")); + assertThat(def.getStructDefs().get("basicStruct").getStringDefs().size(), is(1)); + assertThat(def.getStructDefs().get("basicStruct").getStringDefs().get("foo").getDefVal(), is("basic")); + assertThat(def.getStructDefs().get("basicStruct").getIntDefs().size(), is(1)); + assertNull(def.getStructDefs().get("basicStruct").getIntDefs().get("bar").getDefVal()); + assertThat(def.getStructDefs().get("basicStruct").getArrayDefs().size(), is(1)); + assertThat(def.getStructDefs().get("basicStruct").getArrayDefs().get("intArr").getTypeSpec().getType(), is("int")); + + assertNotNull(def.getStructDefs().get("rootStruct")); + assertNotNull(def.getStructDefs().get("rootStruct").getStructDefs().get("inner0")); + assertNotNull(def.getStructDefs().get("rootStruct").getStructDefs().get("inner1")); + assertThat(def.getStructDefs().get("rootStruct").getInnerArrayDefs().size(), is(1)); + assertNotNull(def.getStructDefs().get("rootStruct").getInnerArrayDefs().get("innerArr")); + assertThat(def.getStructDefs().get("rootStruct").getInnerArrayDefs().get("innerArr").getStringDefs().size(), is(1)); + + assertThat(def.getInnerArrayDefs().size(), is(1)); + assertNotNull(def.getInnerArrayDefs().get("myarray")); + assertThat(def.getInnerArrayDefs().get("myarray").getIntDefs().get("intval").getDefVal(), is(14)); + assertThat(def.getInnerArrayDefs().get("myarray").getArrayDefs().size(), is(1)); + assertNotNull(def.getInnerArrayDefs().get("myarray").getArrayDefs().get("stringval")); + assertThat(def.getInnerArrayDefs().get("myarray").getArrayDefs().get("stringval").getTypeSpec().getType(), is("string")); + assertThat(def.getInnerArrayDefs().get("myarray").getEnumDefs().get("enumval").getDefVal(), is("TYPE")); + assertNull(def.getInnerArrayDefs().get("myarray").getReferenceDefs().get("refval").getDefVal()); + assertThat(def.getInnerArrayDefs().get("myarray").getInnerArrayDefs().size(), is(1)); + assertThat(def.getInnerArrayDefs().get("myarray").getInnerArrayDefs().get("anotherarray").getIntDefs().get("foo").getDefVal(), is(-4)); + assertNull(def.getInnerArrayDefs().get("myarray").getStructDefs().get("myStruct").getIntDefs().get("a").getDefVal()); + assertThat(def.getInnerArrayDefs().get("myarray").getStructDefs().get("myStruct").getIntDefs().get("b").getDefVal(), is(2)); + + // Maps + assertEquals(def.getLeafMapDefs().size(), 4); + assertEquals(def.getLeafMapDefs().get("intMap").getTypeSpec().getType(), "int"); + assertEquals(def.getLeafMapDefs().get("stringMap").getTypeSpec().getType(), "string"); + assertEquals(def.getStructMapDefs().size(), 1); + assertEquals(def.getStructMapDefs().get("myStructMap").getIntDefs().get("myInt").getDefVal(), null); + assertEquals(def.getStructMapDefs().get("myStructMap").getStringDefs().get("myString").getDefVal(), null); + assertEquals(def.getStructMapDefs().get("myStructMap").getIntDefs().get("myIntDef").getDefVal(), (Integer)56); + assertEquals(def.getStructMapDefs().get("myStructMap").getStringDefs().get("myStringDef").getDefVal(), "g"); + + } + + @Test + public void testCreateConfigDefinitionOverrideNamespace() throws IOException, InterruptedException { + File defFile = new File(DEF_NAME); + DefParser defParser = new DefParser(defFile.getName(), new FileReader(defFile)); + CNode root = defParser.getTree(); + + ConfigDefinition def = ConfigDefinitionBuilder.createConfigDefinition(root, "foo"); + assertThat(def.getNamespace(), is("foo")); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionKeyTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionKeyTest.java new file mode 100644 index 00000000000..fbdca4af6df --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionKeyTest.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import org.junit.Test; + +import static junit.framework.TestCase.assertFalse; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + + +/** + * Tests ConfigDefinitionKey + * + * @author musum + */ +public class ConfigDefinitionKeyTest { + + @Test + public void testBasic() { + ConfigDefinitionKey def1 = new ConfigDefinitionKey("foo", ""); + ConfigDefinitionKey def2 = new ConfigDefinitionKey("foo", "bar"); + + assertThat(def1.getName(), is("foo")); + assertThat(def1.getNamespace(), is("")); + + assertTrue(def1.equals(def1)); + assertFalse(def1.equals(def2)); + assertFalse(def1.equals(new Object())); + assertTrue(def2.equals(def2)); + } + + @Test + public void testCreationFromConfigKey() { + ConfigKey<?> key1 = new ConfigKey<>("foo", "id", "bar"); + ConfigDefinitionKey def1 = new ConfigDefinitionKey(key1); + + assertThat(def1.getName(), is(key1.getName())); + assertThat(def1.getNamespace(), is(key1.getNamespace())); + } + +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionSetTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionSetTest.java new file mode 100644 index 00000000000..a00c013109e --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionSetTest.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import org.junit.Test; + +import static junit.framework.TestCase.assertNull; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + + +/** + * Class to hold config definitions and resolving requests for the correct definition + * + * @author Harald Musum <musum@yahoo-inc.com> + * @since 2011-11-18 + */ +public class ConfigDefinitionSetTest { + + @Test + public void testBasic() { + ConfigDefinitionSet configDefinitionSet = new ConfigDefinitionSet(); + ConfigDefinition def1 = new ConfigDefinition("foo", "1"); + ConfigDefinition def2 = new ConfigDefinition("foo", "1", "namespace1"); + ConfigDefinition def3 = new ConfigDefinition("foo", "1", "namespace2"); + final ConfigDefinitionKey key1 = new ConfigDefinitionKey(def1.getName(), def1.getNamespace()); + configDefinitionSet.add(key1, def1); + ConfigDefinitionKey key2 = new ConfigDefinitionKey(def2.getName(), def2.getNamespace()); + configDefinitionSet.add(key2, def2); + ConfigDefinitionKey key3 = new ConfigDefinitionKey(def3.getName(), def3.getNamespace()); + configDefinitionSet.add(key3, def3); + assertThat(configDefinitionSet.size(), is(3)); + assertThat(configDefinitionSet.get(key1), is(def1)); + assertThat(configDefinitionSet.get(key2), is(def2)); + assertThat(configDefinitionSet.get(key3), is(def3)); + + String str = configDefinitionSet.toString(); + assertTrue(str.contains("namespace1.foo")); + assertTrue(str.contains("namespace2.foo")); + assertTrue(str.contains("config.foo")); + } + + @Test + public void testFallbackToDefaultNamespace() { + ConfigDefinitionSet configDefinitionSet = new ConfigDefinitionSet(); + ConfigDefinition def1 = new ConfigDefinition("foo", "1"); + ConfigDefinition def2 = new ConfigDefinition("bar", "1", "namespace"); + + configDefinitionSet.add(new ConfigDefinitionKey(def1.getName(), def1.getNamespace()), def1); + configDefinitionSet.add(new ConfigDefinitionKey(def2.getName(), def2.getNamespace()), def2); + + // fallback to default namespace + assertThat(configDefinitionSet.get(new ConfigDefinitionKey("foo", "namespace")), is(def1)); + // Should not fallback to some other config with same name, but different namespace (not default namespace) + assertNull(configDefinitionSet.get(new ConfigDefinitionKey("bar", "someothernamespace"))); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionTest.java new file mode 100755 index 00000000000..ce03d5f6823 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionTest.java @@ -0,0 +1,234 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import org.junit.Test; + +import com.yahoo.vespa.config.ConfigDefinition.EnumDef; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * Unit tests for ConfigDefinition. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class ConfigDefinitionTest { + + @Test + public void testVersionComparator() { + Comparator<String> c = new ConfigDefinition.VersionComparator(); + + assertEquals(0, c.compare("1", "1")); + assertEquals(0, c.compare("1-0", "1")); + assertEquals(0, c.compare("1-0-0", "1")); + assertEquals(0, c.compare("1-0-0", "1-0")); + assertEquals(0, c.compare("0-1-1", "0-1-1")); + assertEquals(0, c.compare("0-1-0", "0-1")); + assertEquals(-1, c.compare("0", "1")); + assertEquals(-1, c.compare("0-1-0", "0-1-1")); + assertEquals(-1, c.compare("0-1-0", "1-1-1")); + assertEquals(-1, c.compare("0-0-1", "0-1")); + assertEquals(1, c.compare("0-1-1", "0-1-0")); + assertEquals(1, c.compare("1-1-1", "0-1-0")); + assertEquals(1, c.compare("0-1", "0-0-1")); + assertEquals(1, c.compare("1-1", "1")); + + List<String> versions = Arrays.asList("25", "5", "1-1", "0-2-3", "1", "1-0"); + Collections.sort(versions, new ConfigDefinition.VersionComparator()); + List<String> solution = Arrays.asList("0-2-3", "1", "1-0", "1-1", "5", "25"); + assertEquals(solution, versions); + } + + @Test + public void testDefNumberCompare() { + ConfigDefinition df1 = new ConfigDefinition("d", "25"); + ConfigDefinition df2 = new ConfigDefinition("d", "5"); + ConfigDefinition df3 = new ConfigDefinition("d", "1-1"); + ConfigDefinition df4 = new ConfigDefinition("d", "0-2-3"); + ConfigDefinition df5 = new ConfigDefinition("d", "1"); + ConfigDefinition df6 = new ConfigDefinition("d", "1-0"); + assertTrue(df1.compareTo(df2) > 0); + assertTrue(df2.compareTo(df4) > 0); + assertEquals(1, df3.compareTo(df4)); + assertEquals(-1, df4.compareTo(df5)); + assertEquals(0, df5.compareTo(df6)); + } + + @Test + public void testIntDefaultValues() { + ConfigDefinition def = new ConfigDefinition("foo", "1"); + + def.addIntDef("foo"); + def.addIntDef("bar", 0); + def.addIntDef("baz", 1); + def.addIntDef("xyzzy", 2, 0, null); + + assertNull(def.getIntDefs().get("foo").getDefVal()); + assertThat(def.getIntDefs().get("foo").getMin(), is(ConfigDefinition.INT_MIN)); + assertThat(def.getIntDefs().get("foo").getMax(), is(ConfigDefinition.INT_MAX)); + assertThat(def.getIntDefs().get("bar").getDefVal(), is(0)); + assertThat(def.getIntDefs().get("baz").getDefVal(), is(1)); + + assertThat(def.getIntDefs().get("xyzzy").getDefVal(), is(2)); + assertThat(def.getIntDefs().get("xyzzy").getMin(), is(0)); + assertThat(def.getIntDefs().get("xyzzy").getMax(), is(ConfigDefinition.INT_MAX)); + } + + @Test + public void testLongDefaultValues() { + ConfigDefinition def = new ConfigDefinition("foo", "1"); + + def.addLongDef("foo"); + def.addLongDef("bar", 1234567890123L); + def.addLongDef("xyzzy", 2L, 0L, null); + + assertNull(def.getLongDefs().get("foo").getDefVal()); + assertThat(def.getLongDefs().get("foo").getMin(), is(ConfigDefinition.LONG_MIN)); + assertThat(def.getLongDefs().get("foo").getMax(), is(ConfigDefinition.LONG_MAX)); + assertThat(def.getLongDefs().get("bar").getDefVal(), is(1234567890123L)); + + assertThat(def.getLongDefs().get("xyzzy").getDefVal(), is(2L)); + assertThat(def.getLongDefs().get("xyzzy").getMin(), is(0L)); + assertThat(def.getLongDefs().get("xyzzy").getMax(), is(ConfigDefinition.LONG_MAX)); + } + + @Test + @SuppressWarnings("serial") + public void testDefaultsPayloadMap() { + ConfigDefinition def = new ConfigDefinition("foo", "1"); + def.addStringDef("mystring"); + def.addStringDef("mystringdef", "foo"); + def.addBoolDef("mybool"); + def.addBoolDef("mybooldef", true); + def.addIntDef("myint"); + def.addIntDef("myintdef", 1); + def.addLongDef("mylong"); + def.addLongDef("mylongdef", 11l); + def.addDoubleDef("mydouble"); + def.addDoubleDef("mydoubledef", 2d); + EnumDef ed = new EnumDef(new ArrayList<String>(){{add("a1"); add("a2");}}, null); + EnumDef eddef = new EnumDef(new ArrayList<String>(){{add("a11"); add("a22");}}, "a22"); + def.addEnumDef("myenum", ed); + def.addEnumDef("myenumdef", eddef); + def.addReferenceDef("myref"); + def.addReferenceDef("myrefdef", "reff"); + def.addFileDef("myfile"); + def.addFileDef("myfiledef", "etc"); + } + + @Test + public void testVerification() { + ConfigDefinition def = new ConfigDefinition("foo", "1", "bar"); + def.addBoolDef("boolval"); + def.addStringDef("stringval"); + def.addIntDef("intval"); + def.addLongDef("longval"); + def.addDoubleDef("doubleval"); + def.addEnumDef("enumval", new EnumDef(Arrays.asList("FOO"), "FOO")); + def.addReferenceDef("refval"); + def.addFileDef("fileval"); + def.addInnerArrayDef("innerarr"); + def.addLeafMapDef("leafmap"); + ConfigDefinition.ArrayDef intArray = def.arrayDef("intArray"); + intArray.setTypeSpec(new ConfigDefinition.TypeSpec("intArray", "int", null, null, Integer.MIN_VALUE, Integer.MAX_VALUE)); + + ConfigDefinition.ArrayDef longArray = def.arrayDef("longArray"); + longArray.setTypeSpec(new ConfigDefinition.TypeSpec("longArray", "long", null, null, Long.MIN_VALUE, Long.MAX_VALUE)); + + ConfigDefinition.ArrayDef doubleArray = def.arrayDef("doubleArray"); + doubleArray.setTypeSpec(new ConfigDefinition.TypeSpec("doubleArray", "double", null, null, Double.MIN_VALUE, Double.MAX_VALUE)); + + ConfigDefinition.ArrayDef enumArray = def.arrayDef("enumArray"); + enumArray.setTypeSpec(new ConfigDefinition.TypeSpec("enumArray", "enum", null, "VALID", null, null)); + + ConfigDefinition.ArrayDef stringArray = def.arrayDef("stringArray"); + stringArray.setTypeSpec(new ConfigDefinition.TypeSpec("stringArray", "string", null, null, null, null)); + + def.structDef("struct"); + + assertVerify(def, "boolval", "true", 0); + assertVerify(def, "boolval", "false", 0); + assertVerify(def, "boolval", "invalid", IllegalArgumentException.class); + + + assertVerify(def, "stringval", "foobar", 0); + assertVerify(def, "stringval", "foobar", 0); + assertVerify(def, "intval", "123", 0); + assertVerify(def, "intval", "foobar", IllegalArgumentException.class); + assertVerify(def, "longval", "1234", 0); + assertVerify(def, "longval", "foobar", IllegalArgumentException.class); + assertVerify(def, "doubleval", "foobar", IllegalArgumentException.class); + assertVerify(def, "doubleval", "3", 0); + assertVerify(def, "doubleval", "3.14", 0); + assertVerify(def, "enumval", "foobar", IllegalArgumentException.class); + assertVerify(def, "enumval", "foo", IllegalArgumentException.class); + assertVerify(def, "enumval", "FOO", 0); + assertVerify(def, "refval", "foobar", 0); + assertVerify(def, "fileval", "foobar", 0); + + assertVerifyComplex(def, "innerarr", 0); + assertVerifyComplex(def, "leafmap", 0); + assertVerifyComplex(def, "intArray", 0); + assertVerifyComplex(def, "longArray", 0); + assertVerifyComplex(def, "doubleArray", 0); + assertVerifyComplex(def, "enumArray", 0); + assertVerifyComplex(def, "stringArray", 0); + assertVerifyArray(intArray, "1345", 0, 0); + assertVerifyArray(intArray, "invalid", 0, IllegalArgumentException.class); + assertVerifyArray(longArray, "1345", 0, 0); + assertVerifyArray(longArray, "invalid", 0, IllegalArgumentException.class); + assertVerifyArray(doubleArray, "1345", 0, 0); + assertVerifyArray(doubleArray, "1345.3", 0, 0); + assertVerifyArray(doubleArray, "invalid", 0, IllegalArgumentException.class); + assertVerifyArray(enumArray, "valid", 0, IllegalArgumentException.class); + assertVerifyArray(enumArray, "VALID", 0, 0); + assertVerifyArray(enumArray, "inVALID", 0, IllegalArgumentException.class); + assertVerifyArray(stringArray, "VALID", 0, 0); + assertVerifyComplex(def, "struct", 0); + } + + private void assertVerifyArray(ConfigDefinition.ArrayDef def, String val, int index, int expectedNumWarnings) { + List<String> issuedWarnings = new ArrayList<>(); + def.verify(val, index, issuedWarnings); + assertThat(issuedWarnings.size(), is(expectedNumWarnings)); + } + + private void assertVerifyArray(ConfigDefinition.ArrayDef def, String val, int index, Class<?> expectedException) { + try { + def.verify(val, index, new ArrayList<String>()); + } catch (Exception e) { + if (!(e.getClass().isAssignableFrom(expectedException))) { + throw e; + } + } + } + + private void assertVerify(ConfigDefinition def, String id, String val, int expectedNumWarnings) { + List<String> issuedWarnings = new ArrayList<>(); + def.verify(id, val, issuedWarnings); + assertThat(issuedWarnings.size(), is(expectedNumWarnings)); + } + + private void assertVerify(ConfigDefinition def, String id, String val, Class<?> expectedException) { + try { + def.verify(id, val, new ArrayList<String>()); + } catch (Exception e) { + if (!(e.getClass().isAssignableFrom(expectedException))) { + throw e; + } + } + } + + private void assertVerifyComplex(ConfigDefinition def, String id, int expectedNumWarnings) { + List<String> issuedWarnings = new ArrayList<>(); + def.verify(id, issuedWarnings); + assertThat(issuedWarnings.size(), is(expectedNumWarnings)); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java new file mode 100644 index 00000000000..0e55714ca53 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java @@ -0,0 +1,336 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.foo.ArraytypesConfig; +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.foo.StructtypesConfig; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.foo.MaptypesConfig; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.text.StringUtilities; +import com.yahoo.text.Utf8; +import org.junit.Test; +import org.junit.Ignore; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringReader; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigFileFormatterTest { + + private String expected_simpletypes = "stringval \"foo\"\n" + + "intval 324234\n" + + "longval 324\n" + + "doubleval 3.455\n" + + "enumval VAL2\n" + + "boolval true\n"; + + @Test + public void require_that_basic_formatting_is_correct() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("stringval", "foo"); + root.setString("intval", "324234"); + root.setString("longval", "324"); + root.setString("doubleval", "3.455"); + root.setString("enumval", "VAL2"); + root.setString("boolval", "true"); + + assertConfigFormat(slime, expected_simpletypes); + } + + @Test + public void require_that_basic_formatting_is_correct_with_types() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("stringval", "foo"); + root.setLong("intval", 324234); + root.setLong("longval", 324); + root.setDouble("doubleval", 3.455); + root.setString("enumval", "VAL2"); + root.setBool("boolval", true); + + assertConfigFormat(slime, expected_simpletypes); + } + + private void assertConfigFormat(Slime slime, String expected_simpletypes) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + new ConfigFileFormat(def).encode(baos, slime); + assertThat(baos.toString(), is(expected_simpletypes)); + } + + @Test + public void require_that_field_not_found_is_ignored() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("nosuchfield", "bar"); + InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + new ConfigFileFormat(def).encode(baos, slime); + assertThat(baos.toString(), is("")); + } + + // TODO: Reenable this when we can reenable typechecking. + @Ignore + @Test(expected = IllegalArgumentException.class) + public void require_that_illegal_int_throws_exception() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("intval", "invalid"); + InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + new ConfigFileFormat(def).encode(new ByteArrayOutputStream(), slime); + } + + // TODO: Reenable this when we can reenable typechecking. + @Ignore + @Test(expected = IllegalArgumentException.class) + public void require_that_illegal_long_throws_exception() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("longval", "invalid"); + InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + new ConfigFileFormat(def).encode(new ByteArrayOutputStream(), slime); + } + + // TODO: Reenable this when we can reenable typechecking. + @Ignore + @Test(expected = IllegalArgumentException.class) + public void require_that_illegal_double_throws_exception() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("doubleval", "invalid"); + InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + new ConfigFileFormat(def).encode(new ByteArrayOutputStream(), slime); + } + + @Test + public void require_that_illegal_boolean_becomes_false() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("boolval", "invalid"); + InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + new ConfigFileFormat(def).encode(baos, slime); + assertThat(baos.toString(), is("boolval false\n")); + } + + // TODO: Remove this when we can reenable typechecking. + @Test + public void require_that_types_are_not_checked() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("enumval", "null"); + root.setString("intval", "null"); + root.setString("longval", "null"); + root.setString("boolval", "null"); + root.setString("doubleval", "null"); + InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + new ConfigFileFormat(def).encode(baos, slime); + assertThat(baos.toString("UTF-8"), is("enumval null\nintval null\nlongval null\nboolval false\ndoubleval null\n")); + } + + // TODO: Reenable this when we can reenable typechecking. + @Ignore + @Test(expected = IllegalArgumentException.class) + public void require_that_illegal_enum_throws_exception() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("enumval", "invalid"); + InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + new ConfigFileFormat(def).encode(new ByteArrayOutputStream(), slime); + } + + @Test + public void require_that_strings_are_encoded() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + String value = "\u7d22"; + root.setString("stringval", value); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + new ConfigFileFormat(def).encode(baos, slime); + assertThat(baos.toString("UTF-8"), is("stringval \"" + value + "\"\n")); + } + + @Test + public void require_that_array_formatting_is_correct() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor boolarr = root.setArray("boolarr"); + boolarr.addString("true"); + boolarr.addString("false"); + Cursor doublearr = root.setArray("doublearr"); + doublearr.addString("3.14"); + doublearr.addString("1.414"); + Cursor enumarr = root.setArray("enumarr"); + enumarr.addString("VAL1"); + enumarr.addString("VAL2"); + Cursor intarr = root.setArray("intarr"); + intarr.addString("3"); + intarr.addString("5"); + Cursor longarr = root.setArray("longarr"); + longarr.addString("55"); + longarr.addString("66"); + Cursor stringarr = root.setArray("stringarr"); + stringarr.addString("foo"); + stringarr.addString("bar"); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + InnerCNode def = new DefParser("arraytypes", new StringReader(StringUtilities.implode(ArraytypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + new ConfigFileFormat(def).encode(baos, slime); + assertThat(baos.toString(), is( + "boolarr[0] true\n" + + "boolarr[1] false\n" + + "doublearr[0] 3.14\n" + + "doublearr[1] 1.414\n" + + "enumarr[0] VAL1\n" + + "enumarr[1] VAL2\n" + + "intarr[0] 3\n" + + "intarr[1] 5\n" + + "longarr[0] 55\n" + + "longarr[1] 66\n" + + "stringarr[0] \"foo\"\n" + + "stringarr[1] \"bar\"\n")); + } + + @Test + public void require_that_map_formatting_is_correct() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor boolval = root.setObject("boolmap"); + boolval.setString("foo", "true"); + boolval.setString("bar", "false"); + root.setObject("intmap").setString("foo", "1234"); + root.setObject("longmap").setString("foo", "12345"); + root.setObject("doublemap").setString("foo", "3.14"); + root.setObject("stringmap").setString("foo", "bar"); + root.setObject("innermap").setObject("bar").setString("foo", "1234"); + root.setObject("nestedmap").setObject("baz").setObject("inner").setString("foo", "1234"); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + InnerCNode def = new DefParser("maptypes", new StringReader(StringUtilities.implode(MaptypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + new ConfigFileFormat(def).encode(baos, slime); + assertThat(baos.toString(), is( + "boolmap{\"foo\"} true\n" + + "boolmap{\"bar\"} false\n" + + "intmap{\"foo\"} 1234\n" + + "longmap{\"foo\"} 12345\n" + + "doublemap{\"foo\"} 3.14\n" + + "stringmap{\"foo\"} \"bar\"\n" + + "innermap{\"bar\"}.foo 1234\n" + + "nestedmap{\"baz\"}.inner{\"foo\"} 1234\n")); + } + + @Test + public void require_that_struct_formatting_is_correct() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor simple = root.setObject("simple"); + simple.setString("name", "myname"); + simple.setString("gender", "FEMALE"); + Cursor array = simple.setArray("emails"); + array.addString("foo@bar.com"); + array.addString("bar@baz.net"); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + InnerCNode def = new DefParser("structtypes", new StringReader(StringUtilities.implode(StructtypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + new ConfigFileFormat(def).encode(baos, slime); + assertThat(baos.toString(), is( + "simple.name \"myname\"\n" + + "simple.gender FEMALE\n" + + "simple.emails[0] \"foo@bar.com\"\n" + + "simple.emails[1] \"bar@baz.net\"\n" + )); + } + + @Test + public void require_that_complex_struct_formatting_is_correct() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + Cursor nested = root.setObject("nested"); + Cursor nested_inner = nested.setObject("inner"); + nested_inner.setString("name", "baz"); + nested_inner.setString("gender", "FEMALE"); + Cursor nested_inner_arr = nested_inner.setArray("emails"); + nested_inner_arr.addString("foo"); + nested_inner_arr.addString("bar"); + + Cursor nestedarr = root.setArray("nestedarr"); + Cursor nestedarr1 = nestedarr.addObject(); + Cursor inner1 = nestedarr1.setObject("inner"); + inner1.setString("name", "foo"); + inner1.setString("gender", "FEMALE"); + Cursor inner1arr = inner1.setArray("emails"); + inner1arr.addString("foo@bar"); + inner1arr.addString("bar@foo"); + + Cursor complexarr = root.setArray("complexarr"); + Cursor complexarr1 = complexarr.addObject(); + Cursor innerarr1 = complexarr1.setArray("innerarr"); + Cursor innerarr11 = innerarr1.addObject(); + innerarr11.setString("name", "bar"); + innerarr11.setString("gender", "MALE"); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + InnerCNode def = new DefParser("structtypes", new StringReader(StringUtilities.implode(StructtypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + new ConfigFileFormat(def).encode(baos, slime); + assertThat(baos.toString(), is( + "nested.inner.name \"baz\"\n" + + "nested.inner.gender FEMALE\n" + + "nested.inner.emails[0] \"foo\"\n" + + "nested.inner.emails[1] \"bar\"\n" + + "nestedarr[0].inner.name \"foo\"\n" + + "nestedarr[0].inner.gender FEMALE\n" + + "nestedarr[0].inner.emails[0] \"foo@bar\"\n" + + "nestedarr[0].inner.emails[1] \"bar@foo\"\n" + + "complexarr[0].innerarr[0].name \"bar\"\n" + + "complexarr[0].innerarr[0].gender MALE\n" + )); + } + + @Test + public void require_that_strings_are_properly_escaped() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("stringval", "some\"quotes\\\"instring"); + InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + new ConfigFileFormat(def).encode(baos, slime); + assertThat(baos.toString(), is("stringval \"some\\\"quotes\\\\\\\"instring\"\n")); + } + + @Test + @Ignore + public void require_that_utf8_works() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + final String input = "Hei \u00E6\u00F8\u00E5 \n \uBC14\uB451 \u00C6\u00D8\u00C5 hallo"; + root.setString("stringval", input); + System.out.println(bytesToHexString(Utf8.toBytes(input))); + InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + new ConfigFileFormat(def).encode(baos, slime); + System.out.println(bytesToHexString(baos.toByteArray())); + assertThat(Utf8.toString(baos.toByteArray()), is("stringval \"" + input + "\"\n")); + } + + public static String bytesToHexString(byte[] bytes){ + StringBuilder sb = new StringBuilder(); + for(byte b : bytes){ + sb.append(String.format("%02x", b&0xff)); + } + return sb.toString(); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigHelperTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigHelperTest.java new file mode 100644 index 00000000000..ce625727491 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigHelperTest.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.subscription.ConfigSourceSet; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 5.1.9 + */ +public class ConfigHelperTest { + + @Test + public void basic() { + ConfigSourceSet configSourceSet = new ConfigSourceSet("host.com"); + ConfigHelper helper = new ConfigHelper(configSourceSet); + assertThat(helper.getConfigSourceSet(), is(configSourceSet)); + assertThat(helper.getConnectionPool().getAllSourceAddresses(), is("host.com")); + assertThat(helper.getTimingValues().getSubscribeTimeout(), is(new TimingValues().getSubscribeTimeout())); + + // Specify timing values + TimingValues tv = new TimingValues(); + tv.setSubscribeTimeout(11L); + helper = new ConfigHelper(configSourceSet, tv); + assertThat(helper.getTimingValues().getSubscribeTimeout(), is(tv.getSubscribeTimeout())); + } + +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigKeyTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigKeyTest.java new file mode 100644 index 00000000000..e824571d9d1 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigKeyTest.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import com.yahoo.foo.AppConfig; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.codegen.CNode; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class ConfigKeyTest { + + @Test + public void testConfigId() { + String namespace = "bar"; + ConfigKey<?> key1 = new ConfigKey<>("foo", "a/b/c", namespace); + ConfigKey<?> key2 = new ConfigKey<>("foo", "a/b/c", namespace); + + assertEquals(key1, key2); + + ConfigKey<?> key3 = new ConfigKey<>("foo", "a/b/c/d", namespace); + assertTrue(!key1.equals(key3)); + assertFalse(key1.equals(key3)); + + assertEquals("a/b/c", new ConfigKey<>("foo", "a/b/c", namespace).getConfigId()); + assertEquals("a", new ConfigKey<>("foo", "a", namespace).getConfigId()); + assertEquals("", new ConfigKey<>("foo", "", namespace).getConfigId()); + + assertTrue(key1.equals(key1)); + assertFalse(key1.equals(key3)); + assertFalse(key1.equals(new Object())); + + ConfigKey<?> key4 = new ConfigKey<>("myConfig", null, namespace); + assertEquals("", key4.getConfigId()); + } + + @Test + public void testConfigKey() { + String name = AppConfig.CONFIG_DEF_NAME; + String namespace = AppConfig.CONFIG_DEF_NAMESPACE; + String md5 = AppConfig.CONFIG_DEF_MD5; + String configId = "myId"; + + ConfigKey<AppConfig> classKey = new ConfigKey<>(AppConfig.class, configId); + assertEquals("Name is set correctly from class", name, classKey.getName()); + assertEquals("Namespace is set correctly from class", namespace, classKey.getNamespace()); + assertEquals(configId, classKey.getConfigId()); + assertEquals("Md5 is set correctly from class", md5, classKey.getMd5()); + + ConfigKey<?> stringKey = new ConfigKey<>(name, configId, namespace); + assertEquals("Key created from class equals key created from strings", stringKey, classKey); + } + + @Test(expected = ConfigurationRuntimeException.class) + public void testNoName() { + new ConfigKey<>(null, "", ""); + } + + // Tests namespace and equals with combinations of namespace. + @Test + public void testNamespace() { + ConfigKey<?> noNamespace = new ConfigKey<>("name", "id", null); + ConfigKey<?> namespaceFoo = new ConfigKey<>("name", "id", "foo"); + ConfigKey<?> namespaceBar = new ConfigKey<>("name", "id", "bar"); + assertTrue(noNamespace.equals(noNamespace)); + assertTrue(namespaceFoo.equals(namespaceFoo)); + assertFalse(noNamespace.equals(namespaceFoo)); + assertFalse(namespaceFoo.equals(noNamespace)); + assertFalse(namespaceFoo.equals(namespaceBar)); + assertEquals(noNamespace.getNamespace(), CNode.DEFAULT_NAMESPACE); + assertEquals(namespaceBar.getNamespace(), "bar"); + } + + @Test + public void testSorting() { + ConfigKey<?> k1 = new ConfigKey<>("name3", "id2", "nsc"); + ConfigKey<?> k2 = new ConfigKey<>("name2", "id2", "nsb"); + ConfigKey<?> k3 = new ConfigKey<>("name1", "id2", "nsa"); + List<ConfigKey<?>> keys = new ArrayList<>(); + keys.add(k2); + keys.add(k1); + keys.add(k3); + Collections.sort(keys); + assertEquals(keys.get(0), k3); + assertEquals(keys.get(1), k2); + assertEquals(keys.get(2), k1); + + k1 = new ConfigKey<>("name2", "id2", "nsa"); + k2 = new ConfigKey<>("name3", "id3", "nsa"); + k3 = new ConfigKey<>("name1", "id4", "nsa"); + keys = new ArrayList<>(); + keys.add(k3); + keys.add(k2); + keys.add(k1); + Collections.sort(keys); + assertEquals(keys.get(0), k3); + assertEquals(keys.get(1), k1); + assertEquals(keys.get(2), k2); + + k1 = new ConfigKey<>("name", "idC", "nsa"); + k2 = new ConfigKey<>("name", "idA", "nsa"); + k3 = new ConfigKey<>("name", "idB", "nsa"); + keys = new ArrayList<>(); + keys.add(k1); + keys.add(k2); + keys.add(k3); + Collections.sort(keys); + assertEquals(keys.get(0), k2); + assertEquals(keys.get(1), k3); + assertEquals(keys.get(2), k1); + } + +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java new file mode 100644 index 00000000000..24ade0f83c2 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java @@ -0,0 +1,363 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigPayloadBuilderTest { + + private ConfigPayloadBuilder builderWithDef; + + private Cursor createSlime(ConfigPayloadBuilder builder) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + builder.resolve(root); + return root; + } + + @Before + public void setupBuilder() { + ConfigDefinition def = new ConfigDefinition("foo", "1", "bar"); + def.addBoolDef("boolval"); + ConfigDefinition mystruct = def.structDef("mystruct"); + mystruct.addIntDef("foofield"); + def.arrayDef("myarray").setTypeSpec(new ConfigDefinition.TypeSpec("myarray", "int", null, null, null, null)); + ConfigDefinition myinnerarray = def.innerArrayDef("myinnerarray"); + myinnerarray.addIntDef("foo"); + builderWithDef = new ConfigPayloadBuilder(def, new ArrayList<String>()); + } + + @Test + public void require_that_simple_fields_are_set() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + builder.setField("foo", "bar"); + builder.setField("bar", "barz"); + builder.getObject("bar").setValue("baz"); + Cursor root = createSlime(builder); + assertEquals("bar", root.field("foo").asString()); + assertEquals("baz", root.field("bar").asString()); + } + + @Test + public void require_that_simple_fields_can_be_overwritten() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + builder.setField("foo", "bar"); + builder.setField("foo", "baz"); + Cursor root = createSlime(builder); + // XXX: Not sure if this is the _right_ behavior. + assertEquals("baz", root.field("foo").asString()); + } + + @Test + public void require_that_struct_values_are_created() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + ConfigPayloadBuilder struct = builder.getObject("foo"); + struct.setField("bar", "baz"); + + Cursor root = createSlime(builder); + Cursor s = root.field("foo"); + assertEquals("baz", s.field("bar").asString()); + } + + + @Test + public void require_that_maps_are_created() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + ConfigPayloadBuilder.MapBuilder map = builder.getMap("foo"); + assertNotNull(map); + } + + @Test + public void require_that_maps_support_simple_values() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + ConfigPayloadBuilder.MapBuilder map = builder.getMap("foo"); + map.put("fookey", "foovalue"); + map.put("barkey", "barvalue"); + map.put("bazkey", "bazvalue"); + map.put("fookey", "lolvalue"); + assertThat(map.getElements().size(), is(3)); + Cursor root = createSlime(builder); + Cursor a = root.field("foo"); + assertThat(a.field("barkey").asString(), is("barvalue")); + assertThat(a.field("bazkey").asString(), is("bazvalue")); + assertThat(a.field("fookey").asString(), is("lolvalue")); + } + + @Test + public void require_that_arrays_are_created() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + ConfigPayloadBuilder.Array array = builder.getArray("foo"); + assertNotNull(array); + } + + @Test + public void require_that_arrays_can_be_appended_simple_values() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + ConfigPayloadBuilder.Array array = builder.getArray("foo"); + array.append("bar"); + array.append("baz"); + array.append("bim"); + assertThat(array.getElements().size(), is(3)); + Cursor root = createSlime(builder); + Cursor a = root.field("foo"); + assertEquals("bar", a.entry(0).asString()); + assertEquals("baz", a.entry(1).asString()); + assertEquals("bim", a.entry(2).asString()); + } + + @Test + public void require_that_arrays_can_be_indexed_simple_values() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + ConfigPayloadBuilder.Array array = builder.getArray("foo"); + array.set(3, "bar"); + array.set(2, "baz"); + array.set(6, "bim"); + array.set(4, "bum"); + + Cursor root = createSlime(builder); + Cursor a = root.field("foo"); + assertEquals("bar", a.entry(0).asString()); + assertEquals("baz", a.entry(1).asString()); + assertEquals("bim", a.entry(2).asString()); + assertEquals("bum", a.entry(3).asString()); + } + + @Test + public void require_that_arrays_can_be_appended_structs() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + ConfigPayloadBuilder.Array array = builder.getArray("foo"); + ConfigPayloadBuilder elem1 = array.append(); + elem1.setField("bar", "baz"); + ConfigPayloadBuilder elem2 = array.append(); + elem2.setField("foo", "bar"); + Cursor root = createSlime(builder); + Cursor a = root.field("foo"); + assertEquals("baz", a.entry(0).field("bar").asString()); + assertEquals("bar", a.entry(1).field("foo").asString()); + } + + @Test + public void require_that_arrays_can_be_indexed_structs() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + ConfigPayloadBuilder.Array array = builder.getArray("foo"); + ConfigPayloadBuilder elem1 = array.set(4); + elem1.setField("bar", "baz"); + ConfigPayloadBuilder elem2 = array.set(2); + elem2.setField("foo", "bar"); + + Cursor root = createSlime(builder); + Cursor a = root.field("foo"); + assertEquals("baz", a.entry(0).field("bar").asString()); + assertEquals("bar", a.entry(1).field("foo").asString()); + } + + @Test + public void require_that_get_can_be_used_instead() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + ConfigPayloadBuilder.Array array = builder.getArray("foo"); + // Causes append to be used + ConfigPayloadBuilder b1 = array.get(0); + ConfigPayloadBuilder b2 = array.get(1); + ConfigPayloadBuilder b3 = array.get(0); + ConfigPayloadBuilder b4 = array.get(1); + assertThat(b1, is(b3)); + assertThat(b2, is(b4)); + + ConfigPayloadBuilder.Array array_indexed = builder.getArray("bar"); + ConfigPayloadBuilder bi3 = array_indexed.set(3); + ConfigPayloadBuilder bi1 = array_indexed.set(1); + ConfigPayloadBuilder bi32 = array_indexed.get(3); + ConfigPayloadBuilder bi12 = array_indexed.get(1); + assertThat(bi12, is(bi1)); + assertThat(bi32, is(bi3)); + } + + @Test + public void require_that_builders_can_be_merged() { + ConfigPayloadBuilder b1 = new ConfigPayloadBuilder(); + ConfigPayloadBuilder b2 = new ConfigPayloadBuilder(); + ConfigPayloadBuilder b3 = new ConfigPayloadBuilder(); + b1.setField("aaa", "a"); + b1.getObject("bbb").setField("ccc", "ddd"); + + b2.setField("aaa", "b"); + b2.getObject("bbb").setField("ccc", "eee"); + b2.getArray("eee").append("kkk"); + b2.setField("uuu", "ttt"); + + b3.setField("aaa", "c"); + b3.getObject("bbb").setField("ccc", "fff"); + b3.getArray("eee").append("lll"); + b3.setField("uuu", "vvv"); + + assertThat(b1.override(b2), is(b1)); + assertThat(b1.override(b3), is(b1)); + + Cursor b1root = createSlime(b1); + Cursor b2root = createSlime(b2); + Cursor b3root = createSlime(b3); + assertThat(b3root.field("aaa").asString(), is("c")); + assertThat(b3root.field("bbb").field("ccc").asString(), is("fff")); + assertThat(b3root.field("eee").children(), is(1)); + assertThat(b3root.field("eee").entry(0).asString(), is("lll")); + assertThat(b3root.field("uuu").asString(), is("vvv")); + + assertThat(b2root.field("aaa").asString(), is("b")); + assertThat(b2root.field("bbb").field("ccc").asString(), is("eee")); + assertThat(b2root.field("eee").children(), is(1)); + assertThat(b2root.field("eee").entry(0).asString(), is("kkk")); + assertThat(b2root.field("uuu").asString(), is("ttt")); + + assertThat(b1root.field("aaa").asString(), is("c")); + assertThat(b1root.field("bbb").field("ccc").asString(), is("fff")); + assertThat(b1root.field("eee").children(), is(2)); + assertThat(b1root.field("eee").entry(0).asString(), is("kkk")); + assertThat(b1root.field("eee").entry(1).asString(), is("lll")); + assertThat(b1root.field("uuu").asString(), is("vvv")); + } + + @Test(expected=IllegalStateException.class) + public void require_that_append_conflicts_with_index() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + ConfigPayloadBuilder.Array array = builder.getArray("foo"); + array.set(0, "bar"); + array.append("baz"); + } + + @Test(expected=IllegalStateException.class) + public void require_that_index_conflicts_with_append() { + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(); + ConfigPayloadBuilder.Array array = builder.getArray("foo"); + array.append("baz"); + array.set(0, "bar"); + } + + @Test + public void require_that_builder_can_be_created_from_payload() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("foo", "bar"); + Cursor obj = root.setObject("foorio"); + obj.setString("bar", "bam"); + Cursor obj2 = obj.setObject("bario"); + obj2.setString("bim", "bul"); + Cursor a2 = obj.setArray("blim"); + Cursor arrayobj = a2.addObject(); + arrayobj.setString("fim", "fam"); + Cursor arrayobj2 = a2.addObject(); + arrayobj2.setString("blim", "blam"); + Cursor a1 = root.setArray("arrio"); + a1.addString("himbio"); + + ConfigPayloadBuilder builder = new ConfigPayloadBuilder(new ConfigPayload(slime)); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ConfigPayload.fromBuilder(builder).serialize(baos, new JsonFormat(true)); + assertThat(baos.toString(), is("{\"foo\":\"bar\",\"foorio\":{\"bar\":\"bam\",\"bario\":{\"bim\":\"bul\"},\"blim\":[{\"fim\":\"fam\"},{\"blim\":\"blam\"}]},\"arrio\":[\"himbio\"]}")); + } + + @Test(expected=IllegalArgumentException.class) + public void require_that_values_are_verified_against_def() { + builderWithDef.setField("boolval", "true"); + assertThat(builderWithDef.warnings().size(), is(0)); + builderWithDef.setField("boolval", "invalid"); + //assertThat(builderWithDef.warnings().size(), is(1)); + } + + @Test(expected=IllegalArgumentException.class) + public void require_that_arrays_must_exist() { + builderWithDef.getArray("arraydoesnotexist"); + //assertThat(builderWithDef.warnings().size(), is(1)); + } + + @Test(expected=IllegalArgumentException.class) + public void require_that_structs_must_exist() { + builderWithDef.getObject("structdoesnotexist"); + //assertThat(builderWithDef.warnings().size(), is(1)); + } + + @Test(expected=IllegalArgumentException.class) + public void require_that_definition_is_passed_to_childstruct() { + ConfigPayloadBuilder nestedStruct = builderWithDef.getObject("mystruct"); + assertThat(builderWithDef.warnings().size(), is(0)); + nestedStruct.setField("doesnotexit", "foo"); + //assertThat(builderWithDef.warnings().size(), is(1)); + } + + @Test(expected=IllegalArgumentException.class) + public void require_that_definition_is_passed_to_childstruct_but_invalid_field_will_throw() { + ConfigPayloadBuilder nestedStruct = builderWithDef.getObject("mystruct"); + nestedStruct.setField("foofield", "invalid"); + //assertThat(builderWithDef.warnings().size(), is(2)); + } + + @Test + public void require_that_definition_is_passed_to_childarray() { + ConfigPayloadBuilder.Array nestedArray = builderWithDef.getArray("myarray"); + assertThat(builderWithDef.warnings().size(), is(0)); + nestedArray.append("1337"); + assertThat(builderWithDef.warnings().size(), is(0)); + } + + @Test(expected=IllegalArgumentException.class) + public void require_that_definition_is_passed_to_childarray_but_invalid_field_will_throw() { + ConfigPayloadBuilder.Array nestedArray = builderWithDef.getArray("myarray"); + nestedArray.append("invalid"); + //assertThat(builderWithDef.warnings().size(), is(1)); + } + + @Test + public void require_that_definition_is_passed_to_inner_array_with_append() { + ConfigPayloadBuilder.Array innerArray = builderWithDef.getArray("myinnerarray"); + assertThat(builderWithDef.warnings().size(), is(0)); + ConfigPayloadBuilder innerStruct = innerArray.append(); + assertNotNull(innerStruct.getConfigDefinition()); + assertThat(builderWithDef.warnings().size(), is(0)); + innerStruct.setField("foo", "1337"); + assertThat(builderWithDef.warnings().size(), is(0)); + } + + @Test(expected=IllegalArgumentException.class) + public void require_that_definition_is_passed_to_inner_array_with_append_but_invalid_field_will_throw() { + ConfigPayloadBuilder.Array innerArray = builderWithDef.getArray("myinnerarray"); + assertThat(builderWithDef.warnings().size(), is(0)); + ConfigPayloadBuilder innerStruct = innerArray.append(); + innerStruct.setField("foo", "invalid"); + //assertThat(builderWithDef.warnings().size(), is(1)); + } + + @Test + public void require_that_definition_is_passed_to_inner_array_with_index() { + ConfigPayloadBuilder.Array innerArray = builderWithDef.getArray("myinnerarray"); + assertThat(builderWithDef.warnings().size(), is(0)); + ConfigPayloadBuilder innerStruct = innerArray.set(1); + assertNotNull(innerStruct.getConfigDefinition()); + assertThat(builderWithDef.warnings().size(), is(0)); + innerStruct.setField("foo", "1337"); + assertThat(builderWithDef.warnings().size(), is(0)); + } + + @Test(expected=IllegalArgumentException.class) + public void require_that_definition_is_passed_to_inner_array_with_index_but_invalid_field_will_throw() { + ConfigPayloadBuilder.Array innerArray = builderWithDef.getArray("myinnerarray"); + assertThat(builderWithDef.warnings().size(), is(0)); + ConfigPayloadBuilder innerStruct = innerArray.set(1); + innerStruct.setField("foo", "invalid"); + //assertThat(builderWithDef.warnings().size(), is(1)); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java new file mode 100644 index 00000000000..1e7b66fac38 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java @@ -0,0 +1,461 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.foo.*; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.text.StringUtilities; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * @author lulf 3 + * @since 5.1 + */ +public class ConfigPayloadTest { + + @Test + public void test_simple_builder() throws Exception { + SimpletypesConfig config = createSimpletypesConfig("stringval", "abcde"); + assertThat(config.stringval(), is("abcde")); + } + + @Test + public void require_that_arrays_are_built() throws Exception { + AppConfig config = createAppConfig("foo", "4", new String[] { "bar", "baz", "bim" }); + assertThat(config.message(), is("foo")); + assertThat(config.times(), is(4)); + assertThat(config.a(0).name(), is("bar")); + assertThat(config.a(1).name(), is("baz")); + assertThat(config.a(2).name(), is("bim")); + } + + @Test + public void test_int_leaf_legal() throws Exception { + SimpletypesConfig config = createSimpletypesConfig("intval", "0"); + assertThat(config.intval(), is(0)); + config = createSimpletypesConfig("intval", String.valueOf(Integer.MIN_VALUE)); + assertThat(config.intval(), is(Integer.MIN_VALUE)); + config = createSimpletypesConfig("intval", String.valueOf(Integer.MAX_VALUE)); + assertThat(config.intval(), is(Integer.MAX_VALUE)); + config = createSimpletypesConfig("intval", String.valueOf(10)); + assertThat(config.intval(), is(10)); + config = createSimpletypesConfig("intval", String.valueOf(-10)); + assertThat(config.intval(), is(-10)); + } + + @Test (expected = RuntimeException.class) + public void test_int_leaf_too_large() throws Exception { + createSimpletypesConfig("intval", String.valueOf(Integer.MAX_VALUE) + "00"); + } + + @Test (expected = RuntimeException.class) + public void test_int_leaf_too_large_neg() throws Exception { + createSimpletypesConfig("intval", String.valueOf(Integer.MIN_VALUE) + "00"); + } + + @Test(expected=RuntimeException.class) + public void test_int_leaf_illegal_string() throws Exception { + createSimpletypesConfig("intval", "illegal"); + } + + @Test(expected=RuntimeException.class) + public void test_int_leaf_illegal_string_suffix() throws Exception { + createSimpletypesConfig("intval", "123illegal"); + } + + @Test(expected=RuntimeException.class) + public void test_int_leaf_illegal_string_prefix() throws Exception { + createSimpletypesConfig("intval", "illegal123"); + } + + @Test + public void test_that_empty_is_empty() { + ConfigPayload payload = ConfigPayload.empty(); + assertTrue(payload.isEmpty()); + payload = ConfigPayload.fromString("{\"foo\":4}"); + assertFalse(payload.isEmpty()); + } + + + @Test + public void test_long_leaf() throws Exception { + SimpletypesConfig config = createSimpletypesConfig("longval", "0"); + assertThat(config.longval(), is(0l)); + config = createSimpletypesConfig("longval", String.valueOf(Long.MIN_VALUE)); + assertThat(config.longval(), is(Long.MIN_VALUE)); + config = createSimpletypesConfig("longval", String.valueOf(Long.MAX_VALUE)); + assertThat(config.longval(), is(Long.MAX_VALUE)); + config = createSimpletypesConfig("longval", String.valueOf(10)); + assertThat(config.longval(), is(10l)); + config = createSimpletypesConfig("longval", String.valueOf(-10)); + assertThat(config.longval(), is(-10l)); + } + + @Test(expected = RuntimeException.class) + public void test_long_leaf_illegal_string() throws Exception { + createSimpletypesConfig("longval", "illegal"); + } + + @Test (expected = RuntimeException.class) + public void test_long_leaf_too_large() throws Exception { + createSimpletypesConfig("longval", String.valueOf(Long.MAX_VALUE) + "00"); + } + + @Test (expected = RuntimeException.class) + public void test_long_leaf_too_large_neg() throws Exception { + createSimpletypesConfig("longval", String.valueOf(Long.MIN_VALUE) + "00"); + } + + @Test + public void test_double_leaf() throws Exception { + SimpletypesConfig config = createSimpletypesConfig("doubleval", "0"); + assertEquals(0.0, config.doubleval(), 0.01); + assertEquals(133.3, createSimpletypesConfig("doubleval", "133.3").doubleval(), 0.001); + config = createSimpletypesConfig("doubleval", String.valueOf(Double.MIN_VALUE)); + assertEquals(Double.MIN_VALUE, config.doubleval(), 0.0000001); + config = createSimpletypesConfig("doubleval", String.valueOf(Double.MAX_VALUE)); + assertEquals(Double.MAX_VALUE, config.doubleval(), 0.0000001); + } + + @Test + public void test_serializer() throws IOException { + ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); + assertThat(payload.toString(true), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); + } + + @Test(expected=RuntimeException.class) + public void test_double_leaf_illegal_string() throws Exception { + createSimpletypesConfig("doubleval", "illegal"); + } + + @Test + public void test_double_leaf_negative_infinity() throws Exception { + assertThat(createSimpletypesConfig("doubleval", "-Infinity").doubleval(), is(Double.NEGATIVE_INFINITY)); + assertThat(createSimpletypesConfig("doubleval", "Infinity").doubleval(), is(Double.POSITIVE_INFINITY)); + } + + @Test + public void test_enum_leaf() throws Exception { + assertThat(createSimpletypesConfig("enumval", "VAL1").enumval(), is(SimpletypesConfig.Enumval.Enum.VAL1)); + assertThat(createSimpletypesConfig("enumval", "VAL2").enumval(), is(SimpletypesConfig.Enumval.Enum.VAL2)); + } + + @Test(expected=RuntimeException.class) + public void test_enum_leaf_illegal_string() throws Exception { + createSimpletypesConfig("enumval", "ILLEGAL"); + } + + @Test + public void test_bool_leaf() throws Exception { + SimpletypesConfig config = createSimpletypesConfig("boolval", "true"); + assertThat(config.boolval(), is(true)); + config = createSimpletypesConfig("boolval", "false"); + assertThat(config.boolval(), is(false)); + config = createSimpletypesConfig("boolval", "TRUE"); + assertThat(config.boolval(), is(true)); + config = createSimpletypesConfig("boolval", "FALSE"); + assertThat(config.boolval(), is(false)); + } + + @Test// FIXME: (expected = RuntimeException.class) + public void test_bool_leaf_illegal() throws Exception { + createSimpletypesConfig("boolval", "illegal"); + } + + @Test + public void test_string_illegal_value() throws Exception { + // TODO: What do we consider illegal string values? + createSimpletypesConfig("stringval", "insert_illegal_value_please"); + } + + @Test + public void test_int_array() throws Exception { + // Normal behavior + ArraytypesConfig config = createArraytypesConfig("intarr", new String[] { "2", "3", "1", "-2", "5"}); + assertThat(config.intarr().size(), is(5)); + assertThat(config.intarr(0), is(2)); + assertThat(config.intarr(1), is(3)); + assertThat(config.intarr(2), is(1)); + assertThat(config.intarr(3), is(-2)); + assertThat(config.intarr(4), is(5)); + + final int size = 100; + String [] largeArray = new String[size]; + for (int i = 0; i < size; i++) { + int value = (int)(Math.random() * Integer.MAX_VALUE); + largeArray[i] = String.valueOf(value); + } + config = createArraytypesConfig("intarr", largeArray); + assertThat(config.intarr().size(), is(largeArray.length)); + for (int i = 0; i < size; i++) { + assertThat(config.intarr(i), is(Integer.valueOf(largeArray[i]))); + } + } + + @Test(expected = RuntimeException.class) + public void test_int_array_illegal() throws Exception { + createArraytypesConfig("intarr", new String[] { "2", "3", "illegal", "-2", "5"}); + } + + @Test + public void test_long_array() throws Exception { + // Normal behavior + ArraytypesConfig config = createArraytypesConfig("longarr", new String[] { "2", "3", "1", "-2", "5"}); + assertThat(config.longarr().size(), is(5)); + assertThat(config.longarr(0), is(2l)); + assertThat(config.longarr(1), is(3l)); + assertThat(config.longarr(2), is(1l)); + assertThat(config.longarr(3), is(-2l)); + assertThat(config.longarr(4), is(5l)); + + final int size = 100; + String [] largeArray = new String[size]; + for (int i = 0; i < size; i++) { + long value = (long) (Math.random() * Long.MAX_VALUE); + largeArray[i] = String.valueOf(value); + } + config = createArraytypesConfig("longarr", largeArray); + assertThat(config.longarr().size(), is(largeArray.length)); + for (int i = 0; i < size; i++) { + assertThat(config.longarr(i), is(Long.valueOf(largeArray[i]))); + } + } + + @Test + public void test_double_array() throws Exception { + // Normal behavior + ArraytypesConfig config = createArraytypesConfig("doublearr", new String[] { "2.1", "3.3", "1.5", "-2.1", "Infinity"}); + assertThat(config.doublearr().size(), is(5)); + assertEquals(2.1, config.doublearr(0), 0.01); + assertEquals(3.3, config.doublearr(1), 0.01); + assertEquals(1.5, config.doublearr(2), 0.01); + assertEquals(-2.1, config.doublearr(3), 0.01); + assertEquals(Double.POSITIVE_INFINITY, config.doublearr(4), 0.01); + } + + @Test + public void test_enum_array() throws Exception { + // Normal behavior + ArraytypesConfig config = createArraytypesConfig("enumarr", new String[] { "VAL1", "VAL2", "VAL1" }); + assertThat(config.enumarr().size(), is(3)); + assertThat(config.enumarr(0), is(ArraytypesConfig.Enumarr.Enum.VAL1)); + assertThat(config.enumarr(1), is(ArraytypesConfig.Enumarr.Enum.VAL2)); + assertThat(config.enumarr(2), is(ArraytypesConfig.Enumarr.Enum.VAL1)); + } + + @Test + public void test_simple_struct() throws Exception { + StructtypesConfig config = createStructtypesConfigSimple("foobar", "MALE", new String[] { "foo@bar", "bar@foo" }); + assertThat(config.simple().name(), is("foobar")); + assertThat(config.simple().gender(), is(StructtypesConfig.Simple.Gender.Enum.MALE)); + assertThat(config.simple().emails(0), is("foo@bar")); + assertThat(config.simple().emails(1), is("bar@foo")); + } + + + @Test + public void test_simple_struct_arrays() throws Exception { + StructtypesConfig config = createStructtypesConfigArray(new String[] { "foo", "bar" }, + new String[] { "MALE", "FEMALE" }); + assertThat(config.simplearr(0).name(), is("foo")); + assertThat(config.simplearr(0).gender(), is(StructtypesConfig.Simplearr.Gender.MALE)); + assertThat(config.simplearr(1).name(), is("bar")); + assertThat(config.simplearr(1).gender(), is(StructtypesConfig.Simplearr.Gender.FEMALE)); + } + + + @Test + public void test_nested_struct() throws Exception { + StructtypesConfig config = createStructtypesConfigNested("foo", "FEMALE"); + assertThat(config.nested().inner().name(), is("foo")); + assertThat(config.nested().inner().gender(), is(StructtypesConfig.Nested.Inner.Gender.Enum.FEMALE)); + } + + + + @Test + public void test_nested_struct_array() throws Exception { + String [] names = { "foo" ,"bar" }; + String [] genders = { "FEMALE", "MALE" }; + String [][] emails = { + { "foo@bar" , "bar@foo" }, + { "bim@bam", "bam@bim" } + }; + StructtypesConfig config = createStructtypesConfigNestedArray(names, genders, emails); + assertThat(config.nestedarr(0).inner().name(), is("foo")); + assertThat(config.nestedarr(0).inner().gender(), is(StructtypesConfig.Nestedarr.Inner.Gender.FEMALE)); + assertThat(config.nestedarr(0).inner().emails(0), is("foo@bar")); + assertThat(config.nestedarr(0).inner().emails(1), is("bar@foo")); + + assertThat(config.nestedarr(1).inner().name(), is("bar")); + assertThat(config.nestedarr(1).inner().gender(), is(StructtypesConfig.Nestedarr.Inner.Gender.MALE)); + assertThat(config.nestedarr(1).inner().emails(0), is("bim@bam")); + assertThat(config.nestedarr(1).inner().emails(1), is("bam@bim")); + } + + + @Test + public void test_complex_struct_array() throws Exception { + String [][] names = { + { "foo", "bar" }, + { "baz", "bim" } + }; + String [][] genders = { + { "FEMALE", "MALE" }, + { "MALE", "FEMALE" } + }; + StructtypesConfig config = createStructtypesConfigComplexArray(names, genders); + assertThat(config.complexarr(0).innerarr(0).name(), is("foo")); + assertThat(config.complexarr(0).innerarr(0).gender(), is(StructtypesConfig.Complexarr.Innerarr.Gender.Enum.FEMALE)); + assertThat(config.complexarr(0).innerarr(1).name(), is("bar")); + assertThat(config.complexarr(0).innerarr(1).gender(), is(StructtypesConfig.Complexarr.Innerarr.Gender.Enum.MALE)); + + assertThat(config.complexarr(1).innerarr(0).name(), is("baz")); + assertThat(config.complexarr(1).innerarr(0).gender(), is(StructtypesConfig.Complexarr.Innerarr.Gender.Enum.MALE)); + assertThat(config.complexarr(1).innerarr(1).name(), is("bim")); + assertThat(config.complexarr(1).innerarr(1).gender(), is(StructtypesConfig.Complexarr.Innerarr.Gender.Enum.FEMALE)); + } + + @Test + public void test_function_test() { + // TODO: Test function test config as a complete config example + } + + @Test + public void test_set_nonexistent_field() throws Exception { + createSimpletypesConfig("doesnotexist", "blabla"); + } + + @Test + public void test_escaped_string() throws Exception { + SimpletypesConfig config = createSimpletypesConfig("stringval", "b=\"escaped\""); + assertThat(config.stringval(), is("b=\"escaped\"")); + } + + @Test + public void test_unicode() throws Exception { + SimpletypesConfig config = createSimpletypesConfig("stringval", "Hei \u00E6\u00F8\u00E5 \uBC14\uB451 \u00C6\u00D8\u00C5 hallo"); + assertThat(config.stringval(), is("Hei \u00E6\u00F8\u00E5 \uBC14\uB451 \u00C6\u00D8\u00C5 hallo")); + } + + @Test + public void test_empty_payload() throws Exception { + Slime slime = new Slime(); + slime.setObject(); + IntConfig config = new ConfigPayload(slime).toInstance(IntConfig.class, ""); + assertThat(config.intVal(), is(1)); + } + + @Test + public void test_applying_extra_default_values() { + InnerCNode clientDef = new DefParser(SimpletypesConfig.CONFIG_DEF_NAME, new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n") + "\nnewfield int default=3\n")).getTree(); + ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); + payload = payload.applyDefaultsFromDef(clientDef); + assertThat(payload.toString(true), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\",\"newfield\":\"3\"}")); + } + + /** + * TODO: Test invalid slime trees? + * TODO: Test sending in wrong class + */ + + /********************************************************************************************** + * Helper methods. consider moving out to another class for reuse by merge tester. * + **********************************************************************************************/ + private AppConfig createAppConfig(String message, String times, String [] names) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("message", message); + root.setString("times", times); + Cursor arr = root.setArray("a"); + for (String name : names) { + Cursor obj = arr.addObject(); + obj.setString("name", name); + } + return new ConfigPayload(slime).toInstance(AppConfig.class, ""); + } + + private SimpletypesConfig createSimpletypesConfig(String field, String value) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString(field, value); + return new ConfigPayload(slime).toInstance(SimpletypesConfig.class, ""); + } + + private ArraytypesConfig createArraytypesConfig(String field, String [] values) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor array = root.setArray(field); + for (String value : values) { + array.addString(value); + } + return new ConfigPayload(slime).toInstance(ArraytypesConfig.class, ""); + } + + + private void addStructFields(Cursor struct, String name, String gender, String [] emails) { + struct.setString("name", name); + struct.setString("gender", gender); + if (emails != null) { + Cursor array = struct.setArray("emails"); + for (String email : emails) { + array.addString(email); + } + } + } + + private StructtypesConfig createStructtypesConfigSimple(String name, String gender, String [] emails) { + Slime slime = new Slime(); + addStructFields(slime.setObject().setObject("simple"), name, gender, emails); + return new ConfigPayload(slime).toInstance(StructtypesConfig.class, ""); + } + + private StructtypesConfig createStructtypesConfigArray(String[] names, String[] genders) { + Slime slime = new Slime(); + Cursor array = slime.setObject().setArray("simplearr"); + assertEquals(names.length, genders.length); + for (int i = 0; i < names.length; i++) { + addStructFields(array.addObject(), names[i], genders[i], null); + } + return new ConfigPayload(slime).toInstance(StructtypesConfig.class, ""); + } + + private StructtypesConfig createStructtypesConfigNested(String name, String gender) { + Slime slime = new Slime(); + addStructFields(slime.setObject().setObject("nested").setObject("inner"), name, gender, null); + return new ConfigPayload(slime).toInstance(StructtypesConfig.class, ""); + } + + private StructtypesConfig createStructtypesConfigNestedArray(String[] names, String [] genders, String [][] emails) { + Slime slime = new Slime(); + Cursor array = slime.setObject().setArray("nestedarr"); + assertEquals(names.length, genders.length); + for (int i = 0; i < names.length; i++) { + addStructFields(array.addObject().setObject("inner"), names[i], genders[i], emails[i]); + } + return new ConfigPayload(slime).toInstance(StructtypesConfig.class, ""); + } + + private StructtypesConfig createStructtypesConfigComplexArray(String [][] names, String [][] genders) { + Slime slime = new Slime(); + Cursor array = slime.setObject().setArray("complexarr"); + assertEquals(names.length, genders.length); + for (int i = 0; i < names.length; i++) { + assertEquals(names[i].length, genders[i].length); + + Cursor innerarr = array.addObject().setArray("innerarr"); + for (int k = 0; k < names[i].length; k++) { + addStructFields(innerarr.addObject(), names[i][k], genders[i][k], null); + } + } + return new ConfigPayload(slime).toInstance(StructtypesConfig.class, ""); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java b/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java new file mode 100644 index 00000000000..623d81993d9 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.slime.Type; +import org.junit.Test; + +import java.io.StringReader; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class DefaultValueApplierTest { + public Slime apply(Slime slime, String ... extraFields) { + StringBuilder defBuilder = new StringBuilder(); + defBuilder.append("namespace=test").append("\n"); + defBuilder.append("str string").append("\n"); + for (String field : extraFields) { + defBuilder.append(field).append("\n"); + } + Cursor cursor = slime.get(); + cursor.setString("str", "myvalue"); + InnerCNode def = new DefParser("simpletypes", new StringReader(defBuilder.toString())).getTree(); + DefaultValueApplier applier = new DefaultValueApplier(); + return applier.applyDefaults(slime, def); + } + + public Slime apply(String ... extraFields) { + Slime slime = new Slime(); + slime.setObject(); + return apply(slime, extraFields); + } + + @Test + public void require_that_simple_defaults_are_applied() { + Slime slime = apply("strdef string default=\"foo\""); + assertTrue(slime.get().field("str").valid()); + assertThat(slime.get().field("str").asString(), is("myvalue")); + assertTrue(slime.get().field("strdef").valid()); + assertThat(slime.get().field("strdef").asString(), is("foo")); + + } + + @Test + public void require_that_struct_fields_defaults_are_applied() { + Slime slime = apply("nested.str string default=\"bar\""); + assertTrue(slime.get().field("nested").valid()); + assertTrue(slime.get().field("nested").field("str").valid()); + assertThat(slime.get().field("nested").field("str").asString(), is("bar")); + } + + @Test + public void require_that_arrays_of_struct_fields_defaults_are_applied() { + Slime payload = new Slime(); + Cursor cursor = payload.setObject(); + cursor.setArray("nestedarr").addObject().setString("foo", "myfoo"); + Slime slime = apply(payload, "nestedarr[].foo string", "nestedarr[].bar string default=\"bim\""); + + assertTrue(slime.get().field("nestedarr").valid()); + assertThat(slime.get().field("nestedarr").entries(), is(1)); + assertTrue(slime.get().field("nestedarr").entry(0).field("foo").valid()); + assertThat(slime.get().field("nestedarr").entry(0).field("foo").asString(), is("myfoo")); + assertTrue(slime.get().field("nestedarr").entry(0).field("bar").valid()); + assertThat(slime.get().field("nestedarr").entry(0).field("bar").asString(), is("bim")); + } + + @Test + public void require_that_arrays_of_struct_fields_defaults_when_empty() { + Slime payload = new Slime(); + payload.setObject(); + Slime slime = apply(payload, "nestedarr[].foo string", "nestedarr[].bar string default=\"bim\""); + + assertTrue(slime.get().field("nestedarr").valid()); + assertThat(slime.get().field("nestedarr").entries(), is(0)); + assertThat(slime.get().field("nestedarr").type(), is(Type.ARRAY)); + } + + @Test + public void require_that_maps_of_struct_fields_defaults_are_applied() { + Slime payload = new Slime(); + Cursor cursor = payload.setObject(); + cursor.setObject("nestedmap").setObject("mykey").setString("foo", "myfoo"); + Slime slime = apply(payload, "nestedmap{}.foo string", "nestedmap{}.bar string default=\"bim\""); + + assertTrue(slime.get().field("nestedmap").valid()); + assertThat(slime.get().field("nestedmap").fields(), is(1)); + assertTrue(slime.get().field("nestedmap").field("mykey").field("foo").valid()); + assertThat(slime.get().field("nestedmap").field("mykey").field("foo").asString(), is("myfoo")); + assertTrue(slime.get().field("nestedmap").field("mykey").field("bar").valid()); + assertThat(slime.get().field("nestedmap").field("mykey").field("bar").asString(), is("bim")); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java b/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java new file mode 100644 index 00000000000..249a5ab6123 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 5.1.9 + */ +public class ErrorCodeTest { + @Test + public void basic() { + assertThat(ErrorCode.getName(ErrorCode.INTERNAL_ERROR), is("INTERNAL_ERROR")); + assertThat(ErrorCode.getName(ErrorCode.ILLEGAL_CONFIG_MD5), is("ILLEGAL_CONFIG_MD5")); + assertThat(ErrorCode.getName(ErrorCode.ILLEGAL_CONFIGID), is("ILLEGAL_CONFIGID")); + assertThat(ErrorCode.getName(ErrorCode.ILLEGAL_DEF_MD5), is("ILLEGAL_DEF_MD5")); + assertThat(ErrorCode.getName(ErrorCode.ILLEGAL_GENERATION), is("ILLEGAL_GENERATION")); + assertThat(ErrorCode.getName(ErrorCode.ILLEGAL_NAME), is("ILLEGAL_NAME")); + assertThat(ErrorCode.getName(ErrorCode.ILLEGAL_SUB_FLAG), is("ILLEGAL_SUBSCRIBE_FLAG")); + assertThat(ErrorCode.getName(ErrorCode.ILLEGAL_TIMEOUT), is("ILLEGAL_TIMEOUT")); + assertThat(ErrorCode.getName(ErrorCode.ILLEGAL_VERSION), is("ILLEGAL_VERSION")); + assertThat(ErrorCode.getName(ErrorCode.OUTDATED_CONFIG), is("OUTDATED_CONFIG")); + assertThat(ErrorCode.getName(ErrorCode.UNKNOWN_CONFIG), is("UNKNOWN_CONFIG")); + assertThat(ErrorCode.getName(ErrorCode.UNKNOWN_DEF_MD5), is("UNKNOWN_DEF_MD5")); + assertThat(ErrorCode.getName(ErrorCode.UNKNOWN_DEFINITION), is("UNKNOWN_DEFINITION")); + assertThat(ErrorCode.getName(ErrorCode.UNKNOWN_VESPA_VERSION), is("UNKNOWN_VESPA_VERSION")); + assertThat(ErrorCode.getName(ErrorCode.INCONSISTENT_CONFIG_MD5), is("INCONSISTENT_CONFIG_MD5")); + assertThat(ErrorCode.getName(ErrorCode.ILLEGAL_CLIENT_HOSTNAME), is("ILLEGAL_CLIENT_HOSTNAME")); + + assertThat(ErrorCode.getName(12345), is("Unknown error")); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java b/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java new file mode 100644 index 00000000000..d8af6584eca --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class ErrorTypeTest { + + @Test + public void testErrorType() { + assertThat(ErrorType.getErrorType(com.yahoo.jrt.ErrorCode.CONNECTION), is(ErrorType.TRANSIENT)); + assertThat(ErrorType.getErrorType(com.yahoo.jrt.ErrorCode.TIMEOUT), is(ErrorType.TRANSIENT)); + assertThat(ErrorType.getErrorType(ErrorCode.UNKNOWN_CONFIG), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.UNKNOWN_DEFINITION), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.UNKNOWN_DEF_MD5), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_NAME), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_VERSION), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_CONFIGID), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_DEF_MD5), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_CONFIG_MD5), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_TIMEOUT), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_GENERATION), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_SUB_FLAG), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.OUTDATED_CONFIG), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.INTERNAL_ERROR), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_SUB_FLAG), is(ErrorType.FATAL)); + assertThat(ErrorType.getErrorType(0xdeadc0de), is(ErrorType.FATAL)); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java new file mode 100644 index 00000000000..f745c70fe1b --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.slime.JsonFormat; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class GenericConfigBuilderTest { + @Test + public void require_that_builder_can_be_overridden() throws IOException { + ConfigPayloadBuilder ba = new ConfigPayloadBuilder(); + ba.setField("foo", "bar"); + ConfigPayloadBuilder bb = new ConfigPayloadBuilder(); + bb.setField("foo", "baz"); + ConfigPayloadBuilder bc = new ConfigPayloadBuilder(); + bc.setField("foo", "bim"); + GenericConfig.GenericConfigBuilder a = new GenericConfig.GenericConfigBuilder(null, ba); + GenericConfig.GenericConfigBuilder b = new GenericConfig.GenericConfigBuilder(null, bb); + GenericConfig.GenericConfigBuilder c = new GenericConfig.GenericConfigBuilder(null, bc); + assertThat(getString(a), is("{\"foo\":\"bar\"}")); + assertThat(getString(b), is("{\"foo\":\"baz\"}")); + assertThat(getString(c), is("{\"foo\":\"bim\"}")); + ConfigInstanceUtil.setValues(a, b); + assertThat(getString(a), is("{\"foo\":\"baz\"}")); + assertThat(getString(b), is("{\"foo\":\"baz\"}")); + assertThat(getString(c), is("{\"foo\":\"bim\"}")); + ConfigInstanceUtil.setValues(c, a); + assertThat(getString(a), is("{\"foo\":\"baz\"}")); + assertThat(getString(b), is("{\"foo\":\"baz\"}")); + assertThat(getString(c), is("{\"foo\":\"baz\"}")); + } + + private String getString(GenericConfig.GenericConfigBuilder builder) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + builder.getPayload().serialize(baos, new JsonFormat(true)); + return baos.toString(); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java b/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java new file mode 100644 index 00000000000..438d8edb430 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.subscription.ConfigSourceSet; +import org.junit.Test; + +import java.util.*; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.*; + +/** + * Tests for the JRTConnectionPool class. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + * @author musum + */ +public class JRTConnectionPoolTest { + private static final List<String> sources = new ArrayList<>((Arrays.asList("host0", "host1", "host2"))); + + /** + * Tests that hash-based selection through the list works. + */ + @Test + public void test_random_selection_of_sourceBasicHashBasedSelection() { + JRTConnectionPool sourcePool = new JRTConnectionPool(sources); + assertThat(sourcePool.toString(), is("Address: host0\nAddress: host1\nAddress: host2\n")); + + Map<String, Integer> sourceOccurrences = new HashMap<>(); + for (int i = 0; i < 1000; i++) { + final String address = sourcePool.setNewCurrentConnection().getAddress(); + if (sourceOccurrences.containsKey(address)) { + sourceOccurrences.put(address, sourceOccurrences.get(address) + 1); + } else { + sourceOccurrences.put(address, 1); + } + } + for (int i = 0; i < sourcePool.getSize(); i++) { + assertTrue(sourceOccurrences.get(sourcePool.getSources().get(i).getAddress()) > 200); + } + } + + /** + * Tests that when there are two sources and several clients + * the sources will be chosen with about the same probability. + */ + @Test + public void testManySources() { + Map<String, Integer> timesUsed = new LinkedHashMap<>(); + + List<String> twoSources = new ArrayList<>(); + + twoSources.add("host0"); + twoSources.add("host1"); + JRTConnectionPool sourcePool = new JRTConnectionPool(twoSources); + + int count = 1000; + for (int i = 0; i < count; i++) { + String address = sourcePool.setNewCurrentConnection().getAddress(); + if (timesUsed.containsKey(address)) { + int times = timesUsed.get(address); + timesUsed.put(address, times + 1); + } else { + timesUsed.put(address, 1); + } + } + assertConnectionDistributionIsFair(timesUsed); + } + + // Tests that the number of times each connection is used is close to equal + private void assertConnectionDistributionIsFair(Map<String, Integer> connectionsUsedPerHost) { + double devianceDueToRandomSourceSelection = 0.13; + final int size = 1000; + int minHostCount = (int) (size/2 * (1 - devianceDueToRandomSourceSelection)); + int maxHostCount = (int) (size/2 * (1 + devianceDueToRandomSourceSelection)); + + for (Map.Entry<String, Integer> entry : connectionsUsedPerHost.entrySet()) { + Integer timesUsed = entry.getValue(); + assertTrue("Host 0 used " + timesUsed + " times, expected to be < " + maxHostCount, timesUsed < maxHostCount); + assertTrue("Host 0 used " + timesUsed + " times, expected to be > " + minHostCount, timesUsed > minHostCount); + } + } + + /** + * Tests that updating config sources works. + */ + @Test + public void updateSources() { + List<String> twoSources = new ArrayList<>(); + + twoSources.add("host0"); + twoSources.add("host1"); + JRTConnectionPool sourcePool = new JRTConnectionPool(twoSources); + + ConfigSourceSet sourcesBefore = sourcePool.getSourceSet(); + + // Update to the same set, should be equal + sourcePool.updateSources(twoSources); + assertThat(sourcesBefore, is(sourcePool.getSourceSet())); + + // Update to new set + List<String> newSources = new ArrayList<>(); + newSources.add("host2"); + newSources.add("host3"); + sourcePool.updateSources(newSources); + ConfigSourceSet newSourceSet = sourcePool.getSourceSet(); + assertNotNull(newSourceSet); + assertThat(newSourceSet.getSources().size(), is(2)); + assertThat(newSourceSet, is(not(sourcesBefore))); + assertTrue(newSourceSet.getSources().contains("host2")); + assertTrue(newSourceSet.getSources().contains("host3")); + + // Update to new set with just one host + List<String> newSources2 = new ArrayList<>(); + newSources2.add("host4"); + sourcePool.updateSources(newSources2); + ConfigSourceSet newSourceSet2 = sourcePool.getSourceSet(); + assertNotNull(newSourceSet2); + assertThat(newSourceSet2.getSources().size(), is(1)); + assertThat(newSourceSet2, is(not(newSourceSet))); + assertTrue(newSourceSet2.getSources().contains("host4")); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java new file mode 100644 index 00000000000..5fb7c7a9a97 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4SafeDecompressor; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Files; + +/** + * To run this test, place a payload in src/test/ca.json. The file is not checked in because it is huge. + * + * @author lulf + * @since 5.12 + */ +public class LZ4CompressionTest { + private static LZ4Factory factory = LZ4Factory.safeInstance(); + + @Test + @Ignore + public void testCompression() throws IOException { + byte[] data = getInput(); + System.out.println("High compressor"); + for (int i = 0; i < 10; i++) { + timeCompressor(factory.highCompressor(), data); + } + System.out.println("Fast compressor"); + for (int i = 0; i < 10; i++) { + timeCompressor(factory.fastCompressor(), data); + } + } + + private byte[] getInput() throws IOException { + byte[] data = Files.readAllBytes(FileSystems.getDefault().getPath("src/test/ca.json")); + System.out.println("Input size: " + data.length); + return data; + } + + private void timeCompressor(LZ4Compressor lz4Compressor, byte[] data) { + long start = System.currentTimeMillis(); + byte[] compressed = lz4Compressor.compress(data); + long end = System.currentTimeMillis(); + System.out.println("Compression took " + (end - start) + " millis, and size of data is " + compressed.length + " bytes"); + } + + @Test + @Ignore + public void testDecompression() throws IOException { + byte[] data = getInput(); + byte[] outputbuffer = new byte[data.length]; + byte[] hcCompressedData = factory.highCompressor().compress(data); + System.out.println("High compressor"); + for (int i = 0; i < 10; i++) { + timeDecompressor(hcCompressedData, factory.safeDecompressor(), outputbuffer); + } + byte[] fastCompressedData = factory.fastCompressor().compress(data); + System.out.println("Fast compressor"); + for (int i = 0; i < 10; i++) { + timeDecompressor(fastCompressedData, factory.safeDecompressor(), outputbuffer); + } + } + + private void timeDecompressor(byte[] compressedData, LZ4SafeDecompressor decompressor, byte[] outputbuffer) { + long start = System.currentTimeMillis(); + decompressor.decompress(compressedData, outputbuffer); + long end = System.currentTimeMillis(); + System.out.println("Decompression took " + (end - start) + " millis"); + } + +} diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java new file mode 100644 index 00000000000..4b5e5ae07fe --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.text.Utf8; +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.19 + */ +public class LZ4PayloadCompressorTest { + @Test + public void testCompression() { + assertCompression("hei hallo der"); + assertCompression(""); + assertCompression("{}"); + } + + private void assertCompression(String input) { + LZ4PayloadCompressor compressor = new LZ4PayloadCompressor(); + byte[] data = Utf8.toBytes(input); + byte[] compressed = compressor.compress(data); + byte[] output = new byte[data.length]; + compressor.decompress(compressed, output); + assertThat(data, is(output)); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java new file mode 100644 index 00000000000..7a316838334 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.config.protocol.*; +import com.yahoo.vespa.config.protocol.VespaVersion; +import com.yahoo.vespa.config.util.ConfigUtils; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertNull; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertFalse; + +/** + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 5.1.9 + */ +public class RawConfigTest { + + private static final ConfigKey<?> key = new ConfigKey<>("foo", "id", "bar"); + private static List<String> defContent = Arrays.asList("version=1", "anInt int"); + private static final String defMd5 = ConfigUtils.getDefMd5FromRequest("", defContent); + private static final String configMd5 = "012345"; + private static Payload payload = Payload.from(new Utf8String("anInt 1"), CompressionInfo.uncompressed()); + private static long generation = 1L; + + @Test + public void basic() { + RawConfig config = new RawConfig(key, defMd5); + assertEquals(config.getKey(), key); + assertThat(config.getDefMd5(), is(defMd5)); + assertThat(config.getName(), is("foo")); + assertThat(config.getDefNamespace(), is("bar")); + assertThat(config.getConfigId(), is("id")); + + assertThat(config.isError(), is(false)); + + // Copy constructor + RawConfig copiedConfig = new RawConfig(config); + assertThat(copiedConfig, is(config)); + + assertThat(config.toString(), is("bar.foo," + defMd5 + ",id,,0,null")); + assertThat(config.getVespaVersion(), is(Optional.empty())); + } + + @Test + public void testEquals() { + RawConfig config = new RawConfig(key, defMd5); + + assertThat(config, is(new RawConfig(key, defMd5))); + assertThat(config, is(not(new RawConfig(key, "a")))); // different def md5 + assertThat(config.hashCode(), is(new RawConfig(key, defMd5).hashCode())); + assertThat(config.hashCode(), is(not(new RawConfig(key, "a").hashCode()))); // different def md5 + + // different generation + config = new RawConfig(key, defMd5, payload, configMd5, generation, defContent, Optional.empty()); + RawConfig config2 = new RawConfig(key, defMd5, payload, configMd5, 2L, defContent, Optional.empty()); + assertThat(config, is(not(config2))); + assertThat(config.hashCode(), is(not(config2.hashCode()))); + + // different config md5 and with vespa version + final VespaVersion vespaVersion = VespaVersion.fromString("5.37.38"); + RawConfig config3 = new RawConfig(key, defMd5, payload, "9999", generation, defContent, Optional.of(vespaVersion)); + assertThat(config, is(not(config3))); + assertThat(config.hashCode(), is(not(config3.hashCode()))); + // Check that vespa version is set correctly + assertThat(config3.getVespaVersion().get().toString(), is(vespaVersion.toString())); + assertThat(config.getVespaVersion(), is(not(config3.getVespaVersion()))); + + // null config + assertFalse(config.equals(null)); + + // different type of object + assertFalse(config.equals(key)); + + // errors + RawConfig errorConfig1 = new RawConfig(key, defMd5, payload, configMd5, generation, 1, defContent, Optional.empty()); + assertThat(errorConfig1, is(errorConfig1)); + assertThat(config, is(not(errorConfig1))); + assertThat(config.hashCode(), is(not(errorConfig1.hashCode()))); + assertThat(errorConfig1, is(errorConfig1)); + RawConfig errorConfig2 = new RawConfig(key, defMd5, payload, configMd5, generation, 2, defContent, Optional.empty()); + assertThat(errorConfig1, is(not(errorConfig2))); + assertThat(errorConfig1.hashCode(), is(not(errorConfig2.hashCode()))); + } + + @Test + public void payload() { + RawConfig config = new RawConfig(key, defMd5, payload, configMd5, generation, defContent, Optional.empty()); + assertThat(config.getPayload(), is(payload)); + assertThat(config.getConfigMd5(), is(configMd5)); + assertThat(config.getGeneration(), is(generation)); + assertThat(config.getDefContent(), is(defContent)); + } + + @Test + public void require_correct_defmd5() { + final String defMd5ForEmptyDefContent = "d41d8cd98f00b204e9800998ecf8427e"; + + RawConfig config = new RawConfig(key, null, payload, configMd5, generation, defContent, Optional.empty()); + assertThat(config.getDefMd5(), is(defMd5)); + config = new RawConfig(key, "", payload, configMd5, generation, defContent, Optional.empty()); + assertThat(config.getDefMd5(), is(defMd5)); + config = new RawConfig(key, defMd5, payload, configMd5, generation, defContent, Optional.empty()); + assertThat(config.getDefMd5(), is(defMd5)); + config = new RawConfig(key, null, payload, configMd5, generation, null, Optional.empty()); + assertNull(config.getDefMd5()); + config = new RawConfig(key, null, payload, configMd5, generation, Arrays.asList(""), Optional.empty()); + assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent)); + config = new RawConfig(key, "", payload, configMd5, generation, null, Optional.empty()); + assertThat(config.getDefMd5(), is("")); + config = new RawConfig(key, "", payload, configMd5, generation, Arrays.asList(""), Optional.empty()); + assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent)); + } + +} diff --git a/config/src/test/java/com/yahoo/vespa/config/RequestValidationTest.java b/config/src/test/java/com/yahoo/vespa/config/RequestValidationTest.java new file mode 100644 index 00000000000..1e237145b02 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/RequestValidationTest.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.vespa.config.protocol.RequestValidation; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class RequestValidationTest { + + @Test + public void testVerifyName() { + assertTrue(RequestValidation.verifyName("foo")); + assertTrue(RequestValidation.verifyName("Foo")); + assertTrue(RequestValidation.verifyName("foo-bar")); + assertFalse(RequestValidation.verifyName("1foo")); + assertTrue(RequestValidation.verifyName("foo_bar")); + } + + @Test + public void testVerifyDefMd5() { + assertTrue(RequestValidation.verifyMd5("")); + assertTrue(RequestValidation.verifyMd5("e8f0c01c7c3dcb8d3f62d7ff777fce6b")); + assertTrue(RequestValidation.verifyMd5("e8f0c01c7c3dcb8d3f62d7ff777fce6B")); + assertFalse(RequestValidation.verifyMd5("aaaaaaaaaaaaaaaaaa")); + assertFalse(RequestValidation.verifyMd5("-8f0c01c7c3dcb8d3f62d7ff777fce6b")); + } + + @Test + public void testVerifyTimeout() { + assertTrue(RequestValidation.verifyTimeout(1000L)); + assertFalse(RequestValidation.verifyTimeout(-1000L)); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java b/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java new file mode 100644 index 00000000000..742fed77c36 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; +import org.junit.Test; + +import java.io.IOException; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.8 + */ +public class SlimeUtilsTest { + @Test + public void test_copying_slime_types_into_cursor() { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("foo", "foobie"); + Cursor subobj = root.setObject("bar"); + + Slime slime2 = new Slime(); + Cursor root2 = slime2.setObject(); + root2.setString("a", "a"); + root2.setLong("b", 2); + root2.setBool("c", true); + root2.setDouble("d", 3.14); + root2.setData("e", new byte[]{0x64}); + root2.setNix("f"); + + SlimeUtils.copyObject(slime2.get(), subobj); + + assertThat(root.toString(), is("{\"foo\":\"foobie\",\"bar\":{\"a\":\"a\",\"b\":2,\"c\":true,\"d\":3.14,\"e\":\"0x64\",\"f\":null}}")); + } + + @Test + public void test_copying_slime_arrays_into_cursor() { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("foo", "foobie"); + Cursor subobj = root.setObject("bar"); + + Slime slime2 = new Slime(); + Cursor root2 = slime2.setObject(); + Cursor array = root2.setArray("a"); + array.addString("foo"); + array.addLong(4); + array.addBool(true); + array.addDouble(3.14); + array.addNix(); + array.addData(new byte[]{0x64}); + Cursor objinner = array.addObject(); + objinner.setString("inner", "binner"); + + SlimeUtils.copyObject(slime2.get(), subobj); + + assertThat(root.toString(), is("{\"foo\":\"foobie\",\"bar\":{\"a\":[\"foo\",4,true,3.14,null,\"0x64\",{\"inner\":\"binner\"}]}}")); + } + + @Test + public void test_slime_to_json() throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("foo", "foobie"); + root.setObject("bar"); + String json = Utf8.toString(SlimeUtils.toJsonBytes(slime)); + assertThat(json, is("{\"foo\":\"foobie\",\"bar\":{}}")); + } + + @Test + public void test_json_to_slime() { + byte[] json = Utf8.toBytes("{\"foo\":\"foobie\",\"bar\":{}}"); + Slime slime = SlimeUtils.jsonToSlime(json); + assertThat(slime.get().field("foo").asString(), is("foobie")); + assertTrue(slime.get().field("bar").valid()); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/SourceTest.java b/config/src/test/java/com/yahoo/vespa/config/SourceTest.java new file mode 100644 index 00000000000..989131c8256 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/SourceTest.java @@ -0,0 +1,166 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.vespa.config.protocol.CompressionType; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; +import com.yahoo.vespa.config.protocol.Payload; +import com.yahoo.vespa.config.protocol.Trace; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.1 + */ +public class SourceTest { + + @Test + public void testSourceInterface() { + MockSourceConfig config = new MockSourceConfig(new ConfigKey<>(SimpletypesConfig.class, "foobio")); + assertThat(config.getKey().getConfigId(), is("foobio")); + config.setConfig(JRTClientConfigRequestV3.createWithParams(config.getKey(), DefContent.fromList(config.getDefContent()), "host", config.getDefMd5(), config.getGeneration(), 1000, Trace.createNew(), CompressionType.LZ4, Optional.empty())); + MockSource src = new MockSource(config); + assertThat(src.getState(), is(Source.State.NEW)); + src.open(); + assertTrue(src.opened); + assertTrue(src.getconfigged); + assertThat(src.getState(), is(Source.State.OPEN_PENDING)); + src.open(); + assertThat(src.getState(), is(Source.State.OPEN_PENDING)); + src.getconfigged = false; + src.getConfig(); + assertTrue(src.getconfigged); + assertThat(src.getState(), is(Source.State.OPEN_PENDING)); + assertTrue(config.setConfigCalled); + assertTrue(src.openTimestamp > 0); + config.notifyInitMonitor(); + config.setGeneration(4); + src.cancel(); + assertTrue(src.canceled); + assertThat(src.getState(), is(Source.State.CANCEL_REQUESTED)); + src.setState(Source.State.CANCELLED); + try { + src.open(); + fail("Expected exception"); + } catch (ConfigurationRuntimeException e) { + } + src.getconfigged = false; + src.getConfig(); + assertFalse(src.getconfigged); + src.canceled = false; + src.cancel(); + assertFalse(src.canceled); + } + + public static class MockSource extends Source { + boolean opened, getconfigged, canceled = false; + + public MockSource(SourceConfig sourceConfig) { + super(sourceConfig); + } + + @Override + public void myOpen() { + opened = true; + } + + @Override + protected void myGetConfig() { + getconfigged = true; + } + + @Override + public void myCancel() { + canceled = true; + } + } + + private static class MockSourceConfig implements SourceConfig { + + boolean notifyCalled = false; + ConfigKey<?> key = null; + boolean setConfigCalled = false; + long generation = -1; + Payload payload; + + public MockSourceConfig(ConfigKey<?> key) { + this.key = key; + } + + @Override + public void notifyInitMonitor() { + notifyCalled = true; + } + + @Override + public void setConfig(com.yahoo.vespa.config.protocol.JRTClientConfigRequest req) { + key = req.getConfigKey(); + setConfigCalled = true; + } + + @Override + public void setGeneration(long generation) { + this.generation = generation; + } + + @Override + public String getDefName() { + return key.getName(); + } + + @Override + public String getDefNamespace() { + return key.getNamespace(); + } + + @Override + public String getDefVersion() { + return ""; + } + + @Override + public List<String> getDefContent() { + return Arrays.asList("foo"); + } + + @Override + public String getDefMd5() { + return key.getMd5(); + } + + @Override + public String getConfigId() { + return key.getConfigId(); + } + + @Override + public ConfigKey<?> getKey() { + return key; + } + + @Override + public String getConfigMd5() { + return "bar"; + } + + @Override + public long getGeneration() { + return 0; + } + + @Override + public RawConfig getConfig() { + return new RawConfig(getKey(), getDefMd5(), payload, getConfigMd5(), generation, getDefContent(), Optional.empty()); + } + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java b/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java new file mode 100644 index 00000000000..1bae6c79ccb --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertThat; + +/** + * Note: Most of the functionality is tested implicitly by other tests + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class TimingValuesTest { + @Test + public void basic() { + TimingValues tv = new TimingValues(); + TimingValues tv2 = new TimingValues(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1); + assertThat(tv.getRandom(), is(not(tv2.getRandom()))); + TimingValues copy = new TimingValues(tv2); + assertThat(copy.toString(), is(tv2.toString())); // No equals method, just using toString to compare + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/buildergen/ConfigBuilderGeneratorTest.java b/config/src/test/java/com/yahoo/vespa/config/buildergen/ConfigBuilderGeneratorTest.java new file mode 100644 index 00000000000..3d24372c40c --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/buildergen/ConfigBuilderGeneratorTest.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.buildergen; + +import com.google.common.io.Files; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.codegen.ConfigGenerator; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ConfigPayloadApplier; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; + +import static junit.framework.TestCase.assertNotNull; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigBuilderGeneratorTest { + @Test + public void require_that_custom_classes_can_be_generated() throws URISyntaxException, IOException, ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException { + String[] schema = new String[] { + "namespace=foo.bar", + "intval int", + "stringval string" + }; + File tempDir = Files.createTempDir(); + ConfigDefinitionKey key = new ConfigDefinitionKey("quux", "foo.bar"); + ConfigCompiler compiler = new LazyConfigCompiler(tempDir); + ConfigInstance.Builder builder = compiler.compile(new ConfigDefinition(key.getName(), schema).generateClass()).newInstance(); + assertNotNull(builder); + ConfigPayloadApplier<?> payloadApplier = new ConfigPayloadApplier<>(builder); + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("intval", "3"); + root.setString("stringval", "Hello, world"); + payloadApplier.applyPayload(new ConfigPayload(slime)); + String className = ConfigGenerator.createClassName(key.getName()); + ConfigInstance instance = (ConfigInstance) builder.getClass().getClassLoader().loadClass("com.yahoo." + key.getNamespace() + "." + className).getConstructor(new Class<?>[]{builder.getClass()}).newInstance(builder); + assertNotNull(instance); + assertThat(instance.toString(), is("intval 3\nstringval \"Hello, world\"")); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/classes/app.1.def b/config/src/test/java/com/yahoo/vespa/config/classes/app.1.def new file mode 100644 index 00000000000..63e986b58ab --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/classes/app.1.def @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 + +message string default="Hello!" + +times int default=1 + +a[].name string diff --git a/config/src/test/java/com/yahoo/vespa/config/classes/qr-logging.def b/config/src/test/java/com/yahoo/vespa/config/classes/qr-logging.def new file mode 100644 index 00000000000..57ad0050a26 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/classes/qr-logging.def @@ -0,0 +1,34 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=3 +logger string default="com.yahoo" +# Either QueryAccessLog for a regular Vespa access log, or YApacheAccessLog for a log on yApache format +speciallog[].name string + +# Leave as "" +speciallog[].type string + +speciallog[].filehandler.name string default="" + +# File name patterns supporting the expected time variables +speciallog[].filehandler.pattern string default=".%Y%m%d%H%M%S" + +speciallog[].filehandler.rotation string default="0 60 ..." + +# Defines how file rotation is done. There are two options: +# +# "date" : +# The active log file is given the name resulting from pattern (but in this case "pattern" must yield a +# time-dependent name. In addition, a symlink is created pointing to the newest file. +# The symlink is given the name of the symlink parameter (or the name of this service +# if no parameter is given. +# +# "sequence" : +# The active log file is given the name +# defined by "pattern" (which in this case will likely just be a constant string). +# At rotation, this file is given the name pattern.N where N is 1 + the largest integer found by +# extracting the integers from all files ending by .Integer in the same directory +# +speciallog[].filehandler.rotatescheme string default="date" + +speciallog[].cachehandler.name string default="" +speciallog[].cachehandler.size int default=1000 diff --git a/config/src/test/java/com/yahoo/vespa/config/classes/qr-templates.3.def b/config/src/test/java/com/yahoo/vespa/config/classes/qr-templates.3.def new file mode 100644 index 00000000000..8483bf03c44 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/classes/qr-templates.3.def @@ -0,0 +1,142 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=3 + +## Directory for temporary files +directory string default="tmp/templates" + +hey[].ho[].lets[].go string default="ramones" +hey[].ho[].lets[].fishing int default=-4 range=[-8,0] +hey[].ho[].lets[].gone int default=200 range=[0,1000000] +hey[].ho[].lets[].ref reference +hey[].ho[].gone int default=2000 range=[-10,2000000] +hey[].ho[].going bool default=false +hey[].ho[].wash double default=345.3 +hey[].ho[].me double default=-45.5 range=[-234.43,0] +hey[].ho[].now double default=-34 range=[-234,0] +hi[].there[].e enum { BATCH, REALTIME, INCREMENTAL} default=BATCH +hi[].ther[].f enum { BATCH, REALTIME } default=BATCH + +#hey[] int +mode enum { BATCH, REALTIME, INCREMENTAL} default=BATCH +bar.arrline[] string +az[] double +bar.arline[] int range=[0,999] +#bar[].version int +b1[].b2[].b3[].b4[] bool +## Capacities for all storage nodes +capacity[] double range=[0,100] + +longVal long +longWithDefault long default=9876543210 +longWithRange long range=[-9000000000,0] +longArr[] long +longArrWithRange[] long range=[0,9000000000] + +fileVal file +fileWithDefault file +fileArr[] file + +washing double default=5 range=[-1.4,34.324432] +washer double default=46 range=[-1.6,54] + +urlprefix string + +## Prefix to use in queries to choose a given template +templateset[].urlprefix string + +## The MIME type of a given template +templateset[].mimetype string default="text/html" + +## The character set of a given template +templateset[].encoding string default="iso-8859-1" + +## Not used +templateset[].rankprofile int default=0 + + +## Not used in 1.0 +templateset[].keepalive bool default=false + +## Header template. Always rendered. +templateset[].headertemplate string + +## Footer template. Always rendered. +templateset[].footertemplate string + +## Nohits template. Rendered if there are no hits in the result. +templateset[].nohitstemplate string + +## Hit template. Rendered if there are hits in the result. +templateset[].hittemplate string + +## Error template. Rendered if there is an error condition. This is +## not mutually exclusive with the (no)hit templates as such. +templateset[].errortemplate string + +groupsheadertemplate string default="[DEFAULT]" + +## Aggregated groups header template. +## Default rendering is used if missing +templateset[].groupsheadertemplate string default="[DEFAULT]" + +## Aggregated range group template. +## Default rendering is used if missing +templateset[].rangegrouptemplate string default="[DEFAULT]" + +## Aggregated exact group template +## Default rendering is used if missing +templateset[].exactgrouptemplate string default="[DEFAULT]" + +## Aggregated groups footer template. +## Default rendering is used if missing +templateset[].groupsfootertemplate string default="[DEFAULT]" + +## Tags used to highlight results, starting a bolded section. +## An empty string means the template should no override what +## was inserted by the search chain. +templateset[].highlightstarttag string default="" +## Tags used to highlight results, ending a bolded section +## An empty string means the template should no override what +## was inserted by the search chain. +templateset[].highlightendtag string default="" +## Tags used to highlight results, separating dynamic snippets +## An empty string means the template should no override what +## was inserted by the search chain. +templateset[].highlightseptag string default="" + +## The summary class to use for this template if there is none +## defined in the query. +ilscript[].name string +ilscript[].doctype string +ilscript[].content[] string +config[].id reference +config[].autostart string default="no" +musum string + +auran string + +route[].name string +route[].selector string +route[].feed string + +languages[] string +languages2[] string +foolang[].lang[] string + +# Maps +myIntMap{} int +myStringMap{} string +myStructMap{}.myInt int +myStructMap{}.myString string +myStructMap{}.myIntDef int default=56 +myStructMap{}.myStringDef string default="g" + +myStructMap{}.myNestedLeafMap{} long +myStructMap{}.myNestedArray[] long + +myStructMap{}.myNestedMap{}.myLong long +myStructMap{}.myNestedMap{}.myLongDef long default=-100 + +myStructMap{}.myStruct.a string +myStructMap{}.myStruct.b string default="pizza" +myStructMap{}.myStruct.c file diff --git a/config/src/test/java/com/yahoo/vespa/config/classes/ranges.1.def b/config/src/test/java/com/yahoo/vespa/config/classes/ranges.1.def new file mode 100644 index 00000000000..bd054750db5 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/classes/ranges.1.def @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +quux int default=5 range=[,] +xyzzy double default=5 range=[,] +longVal long default=5 range=[,] diff --git a/config/src/test/java/com/yahoo/vespa/config/classes/testfoobar.12.def b/config/src/test/java/com/yahoo/vespa/config/classes/testfoobar.12.def new file mode 100644 index 00000000000..11ac4e0b247 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/classes/testfoobar.12.def @@ -0,0 +1,925 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=12-1-2 + +longVal long +longWithDefault long default=8589934592 + +fileVal file +fileWithDefault file + +vh[] double range=[-300,300] +bg[] int default=0 range=[-10,10] +gee[] string +storage[].feeder[] string +storage[].distributor[] string + +ju[].hu[].tu[] double default=-45 range=[0,1000.1] +ju[].hu[].wu[] enum { HEY, HO} default=HO +ju[].hu[].tang[] bool default=false +ju[].hu[].clan[] int default=45 range=[-90,90] +ju[].hu[].sann reference + +foo string +headertemplate string +## If true, the bitvector part of results with number of hits within +## max bin size will be merged into array form before applying static +## rank. NB: Setting this to true may reduce performance. +applystaticranktosmallbitvectors bool default=true + +## Size of bitvector cache (in bytes). +bitvectorcachesize int default=50000000 + +## Size of boolean occurrence cache (in bytes). +boolocccachesize int default=100000000 + +## Size of dictionary cache (in bytes). +dictcachesize int default=40000000 + +onedimstruct[].val string + +## Size of document info cache (in bytes). +documentinfocachesize int default=25000000 + +## Size of filter occurrence cache (in bytes). +## This cache is used to optimizse exclusion handling. +## A too small size limit will cause serious performance degradation +## if the staticfilterindexes keyword is used. +filterocccachesize int default=30000000 + +## Size of integer range bitvector cache (in bytes). +intrangebitvectorcachesize int default=50000000 + +## Size of integer occurrence cache (in bytes). +intocccachesize int default=100000000 + +## Size of phrase occurrence cache (in bytes). +phrasecachesize int default=150000000 + +## Size of phrase occurrence index cache (in bytes). +phraseidxcachesize int default=20000000 + +## Size of position occurrence cache (in bytes). +posocccachesize int default=100000000 + +## The filename of a file specifying which hosts that should have +## access to the internal web-server of the fsearch process. +accesslist string default="" + +## The filename of the log file for HTTP access. +accesslog string default="" + +## The number of threads to perform async occurrence fetch, i.e. +## read from posocc and boolocc files and possibly generation of +## posocc/boolocc arrays for phrases. Async occurrence fetches +## will use more system CPU but can reduce latency on lightly +## loaded systems. +asyncfetchocc int default=0 + +## specifies the high limit of the ranked result bin. +## If the percentage of the resultset specified in binsize is higher +## than this limit, this will be the max size. +binhigh int default=2147483647 + +## specifies the lowest possible size of a ranked result. This is +## the lower ramp of the percentage specified in the binsize variable. +binlow int default=10000 + +## specifies the size of the ranked results as a percentage of the +## total result set size. The percentage can be ramped off with the +## binlow and binhigh variables. NB: Setting this to 100.0 may lead to +## seriously reduced performance. +binsize double default=100.0 + +## Check cache lines beyond offset + maxhits for blacklisting +## in order to provide correct "totalhits" for queries with many hits. +checktrailingcachelines bool default=false + +## specifies the number of result entries that are processed when +## doing site collapsing. Site collapsing is performed by reordering +## the first collapseentries hits in the result set, listing +## the best hit from each site first. +collapseentries int default=200 range=[0,1000000] + +## Specifies what is the default field used for collapse +defaultcollapse string default="" + +## specifies the rank penalty for additional hits from a site found +## during site collapsing. When additional hits from a site are +## found, the rank values of those hits are reduced by shifting +## them collapserankshift places to the right. +collapserankshift int default=0 + +## The maximum number of active and queued requests. Exceeding +## requests will be discarded. +cutofftransportconns int default=1024 + +## specifies the directory where the dataset resides. Only the +## directory needs to be specified. fsearch will then read the config +## files (index.cf) in that directory to discover the structure. +datasetdir string default="" + +## If set to "yes", then fallback to the default index if a query +## term specifies a non-existing index. If set to "no", then always +## return 0 hits for a query term that specifies a non-existing index. +## Note that the result set for the entire query might still contain +## hits from other query terms unless the invalid query term was an +## and-term. Default value is "no". +defaultindexfallback string default="no" + +# ??? +docattrhashsize int default=8171 +# ??? +docidhashsize int default=8171 +# ??? +docport int default=0 + +## If present, the result cache is not flushed due to reload operation +## unless the document summaries have changed. +dontflushresultcacheonreload bool default=false + +## Colon-delimeted list of catalogs for which negative dictionary +## entries should not be cached. Default is unset. Most useful for +## dictionary files that are memorymaped or memorylocked in +## indextune.cf. Note that the list must start and end with a colon (:). +dropnegativedictionarycacheentriescatalogs string default="" + +## specifies a term (used in an ANDNOT) to be used for filtering ALL queries. +excludefilter string default="" + +## If set to "yes", use of firstoccproximity is enabled. If set to +## "no", use of firstoccproximity is disabled. +firstoccproximity string default="" + +## If present, the filter occurrence cache is flushed due to reload +## operation when all queries using the old configuration has completed. +flushfilteroccscacheonreload bool default=false + +## Do not use position occurrence information, even though it might +## be present in the index. +forceemptyposoccs bool default=false + +## The port to run Fnet Remote Tools (RPC) service on. +## If set to 0, no FRT service is provided. +frtport int default=0 + +## The directory where gid based blacklist files are found. +## Used for "realtime" indexing setups. +gidblacklistdir string default="../gidblacklist" + +## A semicolon-separated list of index names and index name prefixes +## defining the set of indexes that are relevant when highlighting +## query keywords. The terms and phrases from the query targeting any +## of these indexes will be highlighted when dynamic teasers are +## generated. In order to separate index names and index name prefixes +## in the list, index name prefixes have a trailing '*'. Note that +## index aliases are treated like actual index names. This means that +## if you have an index relevant for highlighting and an index alias +## pointing to that index, you need to configure both as relevant for +## highlighting if you want to highlight keywords targeting both the +## actual index and the alias. Example config value: "normal*;title". +## Default config value: "*" (highlight all keywords). +highlightindexes string default="*" + +# ??? +hostname string default="" + +## provide a HTTP server at the given port number. +hport int default=8002 + +## If true, the TCP_NODELAY option is set on the http connections. +## This causes non-full packets to be sent even though previously sent +## data hasn't yet been acknowledged (e.g. due to the delayed ack +## feature present on various tcp stacks). +httpdnodelay bool default=false + +# ??? +intoccpoolsize int default=32768 +# ??? +intoccpoolstep int default=32768 +# ??? +jobqueuethreads int default=5 + +## Juniper configuration property map. +## currently known keys with defaults: +## [juniper.dynsum.highlight_on] = "<b>" +## [juniper.dynsum.highlight_off] = "</b>" +## [juniper.dynsum.continuation] = "..." +## [juniper.dynsum.length] = "256" +## [juniper.dynsum.min_length] = "128" +## [juniper.stem.min_length] = "5" +## [juniper.stem.max_extend] = "3" +## [juniper.dynsum.surround_max] = "128" +## [juniper.dynsum.max_matches] = "3" +## [juniper.dynsum.escape_markup] = "auto" +## [juniper.matcher.winsize] = "200" +## [juniper.dynsum.separators] = "\0x1F\0x1D" +## [juniper.dynsum.connectors] = "\0x1F\0x1D" +## [juniper.proximity.factor] = "0.25" +## [juniper.debug_mask] = "0" +## +#junipersetup properties + +## The maximum number of HTTP connections. +maxhttpconns int default=1024 + +## The maximum interval between a successful read from a socket +## before timeout, in seconds. +maxsocksilent double default=5.0 + +## The maximum number of threads to use. +maxthreads int default=100 + +## The maximum number of active requests at any time. Exceeding +## requests will be queued. +maxtransportconns int default=15 + +## If present the index to the document summary file (docsum.idx) +## is accessed on disk on each access instead of being cached in +## memory. For experimental use on systems with very many docu- +## ments but very few actual docsum requests. +nodocsumidxinmemory bool default=false + +## If set then locks on result cache are held only for very +## short intervals and only a single cache element is locked at a +## time. This simplifies the mutex locking order, but cause extra +## load due to queries that would previously first block then use +## cached value now being fully evaluted. +nonblockingresultcache bool default=false + +## A boolean value controlling removal of several common accented +## uses of characters, used when matching for highlighting. +normalize.accentremoval bool default=true + +## A boolean value controlling normalizing of LATIN CAPITAL/SMALL +## LIGATURE OE (U+0152 U+0153) to the string "oe", used when matching +## for highlighting. +normalize.ligaturesubstitution bool default=true + +## A boolean value controlling normalizing of various accented letters +## to two chars, for linguistics compatibility. +normalize.multicharexpansion bool default=true + +## A boolean value controlling normalizing of LATIN SMALL LETTER SHARP S +## (U+00DF) to the string "ss", used when matching for highlighting. +normalize.sharpssubstitution bool default=true + +## If true, queries are still handled during the reload operation +## (even when the document summaries have changed). If false then +## queries are stalled until reload has completed. +overlappedreload bool default=true + +## The partition number to report to the connecting fdispatch process +## if the dataset label didn't specify the partition number. +partition int default=0 + +## If set to "yes", use of proximity (cf. proximity and firstoccproximity) +## will affect phrases in addition to single words. If set to "no", +## use of proximity is never used for phrases. +phraseproximity string default="" + +## The file name to write the PID of the fsearch process to. +pidfile string default="" + +## Minimum value for maximum value of number of 'posocc' entries +## for a word. If set to 0, computed as 2 * binlow. +posbinhigh int default=0 + +## Maximum value for maximum value of number of 'posocc' entries +## for a word. If set to 0, computed as min(4 * binhigh, 0x7fffffff) +posbinlow int default=0 + +## The maximum value for number of 'posocc' entries for a word, +## specified as a percentage of the number of documents in the index. +## If more entries are needed for evaluation, posocc entries are not +## used for that word and evaluation will be performed without full +## proximity support. The percentage can be ramped off with the posbinlow +## and posbinhigh variables. If set to 0, computed as 2.0 * binsize. +posbinsize double default=0 + +## If set to "yes", use of posocc files is enabled, except when +## "forceemptyposoccs" is set or posocc files doesn't exist. If +## set to "no", use of posocc files is disabled. +proximity string default="" + +## Selects behavior when proximity can be used for two words but +## not three words while firstoccproximity can be used for three +## words. If set to "yes", then use proximity for two words. If +## set to "no", then use firstoccproximity for three words. +proximitypairbeforefirstoccproximitytriple string default="" + +## Selects behavior when proximity can be used for three words but +## not four words while firstoccproximity can be used for four +## words. If set to "yes", then use proximity for three words. If +## set to "no", then use firstoccproximity for four words. The +## default is "yes". +proximitytriplebeforefirstoccproximityquad string default="" + +## specifies the port number for the persistent internal transport +## protocol provided for a multi-level dispatch system. +ptport int default=8003 + +## a reference to a rank-profiles.def type configuration +## describing how to rank results. If empty, fsearch loads +## rank.cf from the dataset directory, see rank.cf(5). +rankcf reference + +## specifies a lower limit for the rankvalue of the results +## returned from the search node. +rankcutoff int default=0 + +## if set, an internal calculation is used for determining a +## rank cutoff value as above. +rankcutoffadvanced bool default=false + +## Specifies the constant value used in the internal advanced rank +## cutoff calculations done when the rankcutoffadvanced parameter is set. +## This roughly reflects the expected rank contribution of +## one good term. +rankcutoffadvval int default=0 + +# ??? +rankinginfopoolsize int default=1048576 +# ??? +rankinginfopoolstep int default=262144 +## Grace period (in seconds) after index reload where old index is +## still available. +reloadgraceperiod int default=64 + +## Maximum number of entries in the resultattributescache. +resultattributescachequeries int default=0 +## Maximum number of entries in the resultattributescache. +## 0 means no limitation. +resultattributescachesize int default=5000000 + +## The maximum lifetime of a resultcache element in seconds. +resultcachemaximumlifetime int default=7200 +## The minimum lifetime of a resultcache element in seconds. +resultcacheminimumlifetime int default=3600 + +## Maximum number of entries in the resultcache. +## 0 means no limitation. +resultcachequeries int default=0 +## Maximum size (in bytes) in the resultcache. +resultcachesize int default=50000000 + +# ??? +rewriter.indexes string default="" +# ??? +rewriter.langfield string default="bsumlanguage" +# ??? +rewriter.rootdir string default="" +# ??? +#rewritersetup properties + +## Set the number of samples a disk is marked as slow before +## starting selftest when no progress occurred during the last +## slowdisksamples samples. +slowdisklatch int default=1 + +## Set the number of milliseconds to sleep between each sample of +## disk state. +slowdisksamplemillisleep int default=100 + +## Set the number of nanoseconds to sleep between each sample of +## disk state. If 0, use slowdisksamplemillisleep instead. +## Restriction: This option only has effect on FreeBSD +## using the LinuxThreads port. +slowdisksamplenanosleep int default=0 + +## Set the number of contigous disk samples without progress and +## outstanding requests before a disk is detected as slow. +## If zero, automatic slowdisk detection is turned off. +## Recommended value is '20' (when using default values for the +## other slowdisk detection parameters). +slowdisksamples int default=0 + +# ??? +staticfilter string default="" + +## specifies a set of indexes for which to optimize exclusion +## handling. Colon is used to separate index names. Default is unset. +## The optimization use entries in the filter occurrence cache, thus +## a too small size limit of the cache will cause serious performance +## degradation. +staticfilterindexes string default="" + +# ??? +strictbind bool default=false + +## Specifies a file containing url coded queries to run as part of self +## test initiated when a slow disk has been detected. +testloadfile string default="" + +## the type of transport to use. Currently only "fnet" is available. +transport string default="" + +## Specifies the transport access log file used by the http server. +## The real log file name is created by using the strftime function +## with the given argument as template. +transportaccesslog string default="" + +## Specifies the interval between transport access log rotation. +## This is the number of minutes between log rotation, e.g a value +## of 1440 indicates that the log should be rotated every 24 hour. +transportaccesslogcycle int default=1440 + +## Specifies the offset from the start of each cycle when the +## transport access log should be cycled automatically. The unit +## is seconds, e.g. a value of 1020 indicates that the log should +## be cycled at 17 minutes past each cycle. +transportaccesslogcycleoffset int default=0 + +## Reference to VSM (Vespa Stream Matcher) configuration. If this is +## set, fsearch will run in VSM mode. +vsmconfig reference + +## If true, the TCP_NODELAY option is set on the persistent transport +## connections. This causes non-full packets to be sent even though +## previously sent data hasn't yet been acknowledged (e.g. due to the +## delayed ack feature present on various tcp stacks). +transportnodelay bool default=true + +# ??? +wordfolder bool default=false +# ??? +wordhashsize int default=524269 +# ??? +wordoccpoolsize int default=2097152 +# ??? +wordoccpoolstep int default=524288 +# ??? +wordpoolsize int default=262144 +# ??? +wordpoolstep int default=65536 + +## Connect spec for transactionlog server. +tlsspec string default="" + +## Document manager config +documentmanagerconfigid reference + +functionmodules[] string restart + +specialchars string + +tokenlist[].name string +tokenlist[].tokens[].token string +tokenlist[].tokens[].replace string default="" + +afloat double default=34 range=[0,1002] + +## qr-searchers: +tag.bold.open string default="<hi>" +tag.bold.close string default="</hi>" +tag.separator string default="<sep />" + +## This array contains the built-in searchers that should +## normally always run in a Vespa-S system. The actual list is +## in the global/ directory on the configserver. +builtin[].searcher string + +## If for some reason you need to disable one of the built-in +## searchers you can set this flag to "false". Handle with great +## care. You need to match the array index from the global/ +## directory, and this may change depending on versions. +builtin[].enabled bool default=true + +# some searcher specific configuration parameters: + +com.yahoo.prelude.searcher.FieldCollapsingSearcher.collapsesize int default=1 +com.yahoo.prelude.searcher.FieldCollapsingSearcher.extrafactor double default=2.0 +com.yahoo.prelude.searcher.FieldCollapsingSearcher.collapsefield string default="mid" + +com.yahoo.prelude.searcher.BlendingSearcher.numthreads int default=200 +com.yahoo.prelude.searcher.BlendingSearcher.docid string default="" + +com.yahoo.prelude.searcher.BoldingSearcher.source string default="" + +com.yahoo.prelude.searcher.JuniperSearcher.source string default="" +com.yahoo.prelude.searcher.JuniperSearcher.defaultdoctype string default="" + +## Query cache that can be placed anywhere in the search chain. Query/Result +## pairs (ie. entries) bigger than maxentrysizebytes will not be cached. +com.yahoo.prelude.searcher.CachingSearcher.cachesizemegabytes int default=100 +com.yahoo.prelude.searcher.CachingSearcher.timetoliveseconds int default=3600 +com.yahoo.prelude.searcher.CachingSearcher.maxentrysizebytes int default=10000 + +com.yahoo.prelude.searcher.XMLStringSearcher.source string default="" + +## relevancy as measured from the backend will usually be +## normalized into the [0,1000] range to make blending between +## several backends with different relevancy models possible; you +## can elect to skip this if you only have backends using +## relevancy scores that are directly comparable. +com.yahoo.prelude.fastsearch.FastSearcher.skipnormalizing bool default=true + +## how many aggregation groups to fetch from the backend +com.yahoo.prelude.grouping.AggregatingSearcher.maxgroups int default=100 + +com.yahoo.prelude.querytransform.PhrasingSearcher.automatonfile string default="" +com.yahoo.prelude.querytransform.NonPhrasingSearcher.automatonfile string default="" +com.yahoo.prelude.querytransform.TermReplacingSearcher.termlist[] string +com.yahoo.prelude.querytransform.CompleteBoostSearcher.source string default="" + +com.yahoo.prelude.querytransform.ExactStringSearcher.source string default="" +com.yahoo.prelude.querytransform.LiteralBoostSearcher.source string default="" +com.yahoo.prelude.querytransform.TermBoostSearcher.source string default="" +com.yahoo.prelude.querytransform.NormalizingSearcher.source string default="" +com.yahoo.prelude.querytransform.StemmingSearcher.source string default="" + +com.yahoo.prelude.statistics.StatisticsSearcher.latencybucketsize int default=30 + + +# here users may add their custom searchers +# (all strings should be class names) +customizedsearchers.rawquery[] string +customizedsearchers.transformedquery[] string +customizedsearchers.blendedresult[] string +customizedsearchers.unblendedresult[] string +customizedsearchers.backend[] string +customizedsearchers.argument[].key string +customizedsearchers.argument[].value string + +## This is for adding searchers which should be below BlendingSearcher, +## but not be linked to any Vespa cluster (directly). +external[].name string +external[].searcher[] string + +# Search cluster specific information. +## Name of search cluster. +searchcluster[].name string default="" + +## Names of search definitions served by search cluster. +searchcluster[].searchdef[] string + +## configid that may be used to get rank-profiles config for the cluster. +searchcluster[].rankprofiles.configid reference default="" + +## Indexing mode of search cluster. +searchcluster[].indexingmode enum { REALTIME, STREAMING } default=REALTIME + +## Storage cluster to use for search cluster if indexingmode is streaming. +searchcluster[].storagecluster string default="" + +# The available dispatchers on each search cluster +searchcluster[].dispatcher[].host string +searchcluster[].dispatcher[].port int + +## The number of least significant bits of the part id used to specify the +## row number (the rest of the bits specifies the column). Don't touch +## this unless you know why you are doing it. +searchcluster[].rowbits int default=0 + + +# 4 search-cluster-specific overrides of global cache parameters: + +## Internal searcher cache. Size is measured in megabytes of raw packet +## size. Hits larger than 1% of total cache size will not be cached. +searchcluster[].cache.size int default=1 + +## Timeout for internal searcher cache. Entries older than this number +## of seconds will be removed from cache. 0 means no cache timeout. +## If cachetimeoutseconds is used, the cache is not purged when reindexing. +searchcluster[].cache.timeout int default=-1 + +## If timeoutwithpurging is set, the index will be purged (possibly +## gradually) when index switching occurs, even if cache timeout > 0. +## Not used if cache timeout is 0. If searchcluster[].cache.timeout is +## not explicitly set, the global value will be used instead of the one +## set locally. +searchcluster[].cache.timeoutwithpurging bool default=false + +## Gradual cache switching causes the index to only be gradually purged +## when cache switching occurs. cacheswitchseconds is the length of the period +## when a given cache entry may be from either the previous or current +## index. Setting it to 0 makes QRS purges the cache entirely when a new +## index becomes available. At the end of the cacheswitchseconds period, +## the cache will be cleaned of any remaining entries from the previous +## index. +searchcluster[].cache.switchseconds int default=-1 + +# Per dispatcher config-id might be nice to have, remove it until needed. +# searchcluster[].dispatcher[].configid reference + +# rank-profiles +## name of this rank profile. maps to table index for internal use. +rankprofile[].name string + +## the name of a generic property available to the feature execution framework and feature plugins +rankprofile[].fef.property[].name string + +## the value of a generic property available to feature plugins +rankprofile[].fef.property[].value string + +## the catalog name overrides apply to +rankprofile[].catalog[].name string + +## Boost value for AND queries in this catalog. +rankprofile[].catalog[].andboost int default=0 + +## Boost value for OR queries in this catalog. +rankprofile[].catalog[].orboost int default=0 + +## Boost value for ANY queries in this catalog. +rankprofile[].catalog[].anyboost int default=0 + +## Boost value for NEAR queries in catalog. +rankprofile[].catalog[].nearboost int default=0 + +## Boost value for ORDEREDNEAR queries in this catalog. +rankprofile[].catalog[].orderednearboost int default=0 + +## Boost value for phrase queries in this catalog. +rankprofile[].catalog[].phraseboost int default=0 + +## Boost value for all queries in catalog. +rankprofile[].catalog[].rankboost int default=0 + +## If true, the context boost is the max value of +## the individual contextboosts. +## When false, the context boost when a term is in +## several contexts is the sum of the individual contextboosts. +rankprofile[].catalog[].bestcontextboostonly bool default=false + + +## If true, then use extnumoccboost only when calculating rank values. +## Also, do not normalize the extnumoccboost value with +## global term frequency. Default value is false. +rankprofile[].catalog[].extnumoccboostonly bool default=false + +## If yes, then use extnumoccboost only when calculating rank values. +## Also, do not normalize the extnumoccboost value with +## global term frequency. Default value is no. +rankprofile[].catalog[].numoccandextnumoccboostonly bool default=false + +## If yes, then use bitvectors when possible. +## Default value is false. +rankprofile[].catalog[].preferbitvector bool default=false + +## Load extnumoccboost for this catalog from the named file. +## extnumoccboost specifies boost values due to the number of +## occurences of a term that are external to the document. If +## "NULL" is given as file name, then all extnumoccboost values +## will be set to 0. +rankprofile[].catalog[].extnumoccboost.table string default="/home/vespa/conf/vespa/search/ranktables/constant-0000" + +## Load numoccboost for this catalog from the named file. +## numoccboost specifies boost values due to the number of occurences in +## a document. If "NULL" is given as file name, then all numoccboost +## values will be set to 0. +rankprofile[].catalog[].numoccboost.table string default="/home/vespa/conf/vespa/search/ranktables/constant-0000" + +## Load firstoccboost for catalog from the file named. +## firstoccboost specifies boost values due to the position of the +## first occurence in a document. If "NULL" is given as file name, +## then all firstoccboost values will be set to 0. +rankprofile[].catalog[].firstoccboost.table string default="/home/vespa/conf/vespa/search/ranktables/constant-0000" + + +## Load firstoccproximityboost for this catalog from the file named. +## firstoccproximity boost specifies boost values due to the correlation between +## positions of the first occurence in a document for two and two words. +## +## If "NULL" is given as file name, then all +## firstoccproximityboost values will be set to 0. If otherwise set, +## should be the name of a file to load into the table. The file +## should have 256 lines each containing a single integer. +## +## There are 256 elements in the table, handling forward distances from 1. +## The corresponding firstoccrevproximityboost table is used +## to handle closeness in reverse order. +## +## The last array index specifies the proximity table set. During +## evaluation, the bigram proximity weight supplied by the query segmenter +## specifies which proximity table set to use, with a fallback to set 0 +## when no information is available. +rankprofile[].catalog[].firstoccproximityboost[].table string default="/home/vespa/conf/vespa/search/ranktables/constant-0000" + +## Load firstoccrevproximityboost table for this catalog from the named file. +## Specifies boost values due to the correlation between positions +## of the first occurence in a document for two and two words when +## the second word in the query comes first in the document. +## See also firstoccproximityboost above. +rankprofile[].catalog[].firstoccrevproximityboost[].table string default="/home/vespa/conf/vespa/search/ranktables/constant-0000" + +## Load proximityboost for this catalog from the named file. +## proximity boost specifies boost values due to the correlation between +## positions of the occurences in a document for two and two words. +## See also firstoccproximityboost above. +rankprofile[].catalog[].proximityboost[].table string default="/home/vespa/conf/vespa/search/ranktables/constant-0000" + +## Load revproximityboost for this catalog from the named file. +## revproximity boost specifies boost values due to the correlation between +## positions of the occurences in a document for two and two words. +## See also firstoccproximityboost above. +rankprofile[].catalog[].revproximityboost[].table string default="/home/vespa/conf/vespa/search/ranktables/constant-0000" + +## Load divtable for this catalog from the named file. +## Rank values for a query term are divided by the entry +## in divtable indexed by log2 of term frequence. +## The file should contain ?? lines each with a single integer. +rankprofile[].catalog[].divtable string default="" + +## The name of a context in this catalog to specify boosts for. +rankprofile[].catalog[].context[].name string + +## Boost occurrences in this context with the given value. +## XXX -1 uses default (???) from somewhere(TM). +rankprofile[].catalog[].context[].contextboost int default=0 + +## Boost pair of occurrences in this context with +## the given value when evaluating 2 words from same catalog in +## parallell. +## XXX -1 uses default (???) from somewhere(TM). +rankprofile[].catalog[].context[].commoncontextboost.pair int default=0 + +## Boost triple of occurrences in this context with +## the given value when evaluating 3 words from same catalog in +## parallell. +## XXX -1 uses default (???) from somewhere(TM). +rankprofile[].catalog[].context[].commoncontextboost.triple int default=0 + +## Boost quad of occurrences in this context with +## the given value when evaluating 4 words from same catalog in +## parallell. +## XXX -1 uses default (???) from somewhere(TM). +rankprofile[].catalog[].context[].commoncontextboost.quad int default=0 + + +## The name of the attribute +rankprofile[].attribute[].name string + +## Boost value for queries that hit in this attribute +rankprofile[].attribute[].attributecontextboost int default=0 + +## Load weightboost for this attribute from the named file. +## weightboost specifies boost values due to the weight (weighted set) +## or number of occurences (single, array) in an attribute. +## If "NULL" is given as file name, then all weightboost values will be set to 0. +rankprofile[].attribute[].weightboost.table string default="/home/vespa/conf/vespa/search/ranktables/constant-0000" + + +## Load static rank values from the given staticrank docattr vector. +## Must be specified in index.cf as a staticrankfile. +rankprofile[].staticrankfile string default="" + +## Multiply static rank values with given value when calculating total +## rank value. +rankprofile[].staticcoefficient int default=1 + +## If false then use only static ranking when sorting result hits. +## Default is true. +rankprofile[].dynamicranking bool default=true + +## If dynamic ranking is turned off, then ascending will sort the +## result hits with lowest static rank values first, while +## descending will sort with highest static rank values +## first. Default is descending. This keyword has no effect if +## dynamic ranking is on. +rankprofile[].staticranksortorder string default="descending" + +## Load static rank mapping from the file named table. The static +## rank mapping maps each 8-bit static rank value into a 32-bit static +## rank value. This option may only be used with 8-bit static rank files. +rankprofile[].staticrankmap string default="" + +## If set to "true", total rank will be reduced when dynamic rank is less than +## 25% of static rank, to suppress irrelevant hits from popular sites. +## If set to "false", total rank is not reduced. +rankprofile[].clampstaticrank bool default=false + +## Load document datetime values used for freshness boost calculation from +## this file. The values must be coded as minutes since +## 1900-01-01T00:00Z. The value 0 has the special meaning +## "no datetime value exists". +rankprofile[].freshnessboost.file string default="" + +## Load freshnessboost lookup-table values from the file named +## table instead of using built-in default values. The file must +## contain 32 white-space separated non-negative integers. +rankprofile[].freshnessboost.table string default="/home/vespa/conf/vespa/search/ranktables/constant-0000" + +## When calculating the freshness boost value multiply difference between +## current datetime and document datetime with timeoffset before taking +## the base-2 logarithm. Default value is 1. Max value is 31. +rankprofile[].freshnessboost.timeoffset int default=1 + +## If a document has datetime value 0, then use defaultboostvalue +## as freshness boost value instead of doing table lookup. The default +## default value is 0 (no boost). +rankprofile[].freshnessboost.defaultboostvalue int default=0 + +## Multiply freshness boost value with coefficient when calculating +## total freshness boost value. If coefficient 0 is used, no freshness +## boost value will be computed or added. Default value is 0. +rankprofile[].freshnessboost.coefficient int default=0 + +## boost table files for distance ranking, 1 dimension. +## The tables have 465 elements each, where slots 0..15 represents +## distances 0..15 while the remaining slots represents distance +## (16 + (slot & 15)) << ((slot >> 4) - 1). Linear interpolation is +## used for distances "between" table slots. +## +## If "NULL" is given as the file name then all 1D distance boost values +## for that table will be set to 0. +rankprofile[].distance1dboosttable[].table string + +## boost table files for distance ranking, 2 dimensions. +## The tables have 977 elements each, where slots 0..15 represents +## square of distance being 0..15 while the remaining slots represents +## square of distance distance being +## (16 + (slot & 15)) << ((slot >> 4) - 1). Linear interpolation is +## used for distances "between" table slots. +## +## If "NULL" is given as the file name then all 2D distance boost values +## for that table will be set to 0. +rankprofile[].distance2dboosttable[].table string + +## The lowest possible size of a ranked result. This is the lower ramp +## of the percentage specified in the binsize variable. The default is +## specified in fsearchrc. +rankprofile[].binlow int default=-1 + +## The high limit of the ranked result bin. If the percentage of the +## resultset specified in binsize is higher than this limit, this will be +## the max size. The default is specified in fsearchrc. +rankprofile[].binhigh int default=-1 + +## The size of the ranked results as a percentage of the total result +## set size. The percentage can be ramped off with the binlow and binhigh +## variables. The default is specified in fsearchrc. +rankprofile[].binsize double default=-1 + +## Minimum value for maximum value of number of 'posocc' entries for a word. +## The default is specified in fsearchrc. +rankprofile[].posbinlow int default=-1 + +## Maximum value for maximum value of number of 'posocc' entries for a word. +## The default is specified in fsearchrc. +rankprofile[].posbinhigh int default=-1 + +## The maximum value for number of 'posocc' entries for a word, specified +## as a percentage of the number of documents in the index. If more +## entries are needed for evaluation, posocc entries are not used for that +## word and evaluation will be performed without full proximity support. +## The percentage can be ramped off with the posbinlow and posbinhigh +## variables. The default is specified in fsearchrc. +rankprofile[].posbinsize int default=-1 + +## After all other rank calculations, the rank value is tuned according +## to the tunefactor and tunebias values. The rank value is modified +## as follows: new_rank = old_rank * tunefactor + tunebias. +rankprofile[].tunefactor double default=1.0 + +## After all other rank calculations, the rank value is tuned according +## to the tunefactor and tunebias values. The rank value is modified +## as follows: new_rank = old_rank * tunefactor + tunebias. +rankprofile[].tunebias int default=0 + +## A lower limit for the rankvalue of the results returned from the +## search node. If rankcutoff.advanced is set to "true", determines +## the constant value used in the internal advanced rank cutoff +## calculations. This roughly reflects the expected rank contribution +## of one good term. +## The rankcutoff.val value and the rankcutoff.advanced parameter +## may be used if you only want hits with a minimum relevancy to show +## up in the resultset. +## A value below zero means no rankcutoff is done. +rankprofile[].rankcutoff.val int default=-1 + +## When rankcutoff.val is in use, this flag controls whether to use +## an internal calculation is used for determining the rank cutoff +## value. If "false", use rankcutoff.val as a direct lower limit. +rankprofile[].rankcutoff.advanced bool default=false + +## If set to "ON", use of posocc files is enabled, except when +## "forceemptyposoccs" is set in fsearchrc or posocc files doesn't exist. +## If set to "OFF", use of posocc files is disabled. +## If "NOTSET" the fsearchrc "proximity" parameter is used instead. +rankprofile[].proximity.full.enable enum { OFF, ON, NOTSET } default=NOTSET + +## If set to "ON", use of firstoccproximity is enabled. +## If set to "OFF", use of firstoccproximity is disabled. +## When NOTSET use the firstoccproximity value in fsearchrc configuration. +rankprofile[].proximity.firstocc.enable enum { OFF, ON, NOTSET } default=NOTSET + +## If set to "ON", use of proximity (cf. proximity and firstoccproximity) +## will affect phrases in addition to single words. +## If set to "OFF", proximity is never used for phrases. +## When NOTSET use the phraseproximity value in fsearchrc configuration. +rankprofile[].proximity.phrase.enable enum { OFF, ON, NOTSET } default=NOTSET + +## Selects behavior when proximity can be used for two words but not three +## words while firstoccproximity can be used for three words. +## If set to "ON", then use proximity for two words. +## If set to "OFF", then use firstoccproximity for three words. +## When NOTSET use the proximitypairbeforefirstoccproximitytriple value +## in fsearchrc configuration. +rankprofile[].proximity.pairbeforefirstocctriple.enable enum { OFF, ON, NOTSET } default=NOTSET + +## Selects behavior when proximity can be used for three words but not four +## words while firstoccproximity can be used for four words. +## If set to "ON", then use proximity for three words. +## If set to "OFF", then use firstoccproximity for four words. +## When NOTSET use the proximitytriplebeforefirstoccproximityquad value +rankprofile[].proximity.triplebeforefirstoccquad.enable enum { OFF, ON, NOTSET } default=NOTSET diff --git a/config/src/test/java/com/yahoo/vespa/config/configsglobal/qr-logging.cfg b/config/src/test/java/com/yahoo/vespa/config/configsglobal/qr-logging.cfg new file mode 100755 index 00000000000..cc640619e3a --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/configsglobal/qr-logging.cfg @@ -0,0 +1,44 @@ +logger com.yahoo +speciallog[6] +speciallog[0].name QueryAccessLog +speciallog[0].type file +speciallog[0].filehandler.name QueryAccessLog +speciallog[0].filehandler.pattern logs/vespa/qrs/QueryAccessLog.%Y%m%d%H%M%S +speciallog[0].filehandler.rotation "0 60 ..." +speciallog[0].cachehandler.name QueryAccessLog +speciallog[0].cachehandler.size 1000 +speciallog[1].name QueryResultLog +speciallog[1].type cache +speciallog[1].filehandler.name QueryResultLog +speciallog[1].filehandler.pattern logs/vespa/qrs/QueryResultLog.%Y%m%d%H%M%S +speciallog[1].filehandler.rotation "0 60 ..." +speciallog[1].cachehandler.name QueryResultLog +speciallog[1].cachehandler.size 1000 +speciallog[2].name ResultImpressionLog +speciallog[2].type file +speciallog[2].filehandler.name ResultImpressionLog +speciallog[2].filehandler.pattern logs/vespa/qrs/ResultImpressionLog.%Y%m%d%H%M%S +speciallog[2].filehandler.rotation "0 60 ..." +speciallog[2].cachehandler.name ResultImpressionLog +speciallog[2].cachehandler.size 1000 +speciallog[3].name ServiceEventLog +speciallog[3].type cache +speciallog[3].filehandler.name ServiceEventLog +speciallog[3].filehandler.pattern logs/vespa/qrs/ServiceEventLog.%Y%m%d%H%M%S +speciallog[3].filehandler.rotation "0 60 ..." +speciallog[3].cachehandler.name ServiceEventLog +speciallog[3].cachehandler.size 1000 +speciallog[4].name ServiceStatusLog +speciallog[4].type off +speciallog[4].filehandler.name ServiceStatusLog +speciallog[4].filehandler.pattern logs/vespa/qrs/ServiceStatusLog.%Y%m%d%H%M%S +speciallog[4].filehandler.rotation "0 60 ..." +speciallog[4].cachehandler.name ServiceStatusLog +speciallog[4].cachehandler.size 1000 +speciallog[5].name ServiceTraceLog +speciallog[5].type parent +speciallog[5].filehandler.name ServiceTraceLog +speciallog[5].filehandler.pattern logs/vespa/qrs/ServiceTraceLog.%Y%m%d%H%M%S +speciallog[5].filehandler.rotation "0 60 ..." +speciallog[5].cachehandler.name ServiceTraceLog +speciallog[5].cachehandler.size 1000 diff --git a/config/src/test/java/com/yahoo/vespa/config/configsglobal/qr-templates.3.cfg b/config/src/test/java/com/yahoo/vespa/config/configsglobal/qr-templates.3.cfg new file mode 100644 index 00000000000..345c20157f9 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/configsglobal/qr-templates.3.cfg @@ -0,0 +1,111 @@ +washing -0.45 +washer 0 +hey[2] +hey[0].ho[1] +hey[0].ho[0].lets[3] +hey[0].ho[0].lets[0].go "slayer" +hey[0].ho[0].lets[0].fishing -1 +hey[0].ho[0].lets[0].ref JA +hey[0].ho[0].lets[1].go "gate" +hey[0].ho[0].lets[1].ref :parent: +hey[0].ho[0].lets[str].go "strung" +hey[0].ho[0].me 0.0 +hey[fooo].ho[0] + +hey[0] 78 + +longVal 500 +longWithRange -9000000000 +longArr[2] +longArr[0] 0 +longArr[1] 1 +longArrWithRange[1] +longArrWithRange[0] 9000000000 + +bar.arrline[0] "foo" +bar.arrline[1] "bar" + +fileVal "/nodefault/file" +fileArr[2] +fileArr[0] "keyb.com" +fileArr[1] "2b4a64d0cb36d44ba8a52506d9fe480bf3511be6" + +urlprefix "foo" +templateset[2] +templateset[0].urlprefix "/basic" +#templateset[0].mimetype "text/xml" +templateset[0].encoding "ut\"f-8\n' <hit relevancy=\"$relevancy\">\n#foreach" +templateset[0].headertemplate "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n<resultset totalhits=\"$result.hitCount\">\n" +templateset[0].footertemplate "</resultset>\n" +templateset[0].nohitstemplate "<empty/>\n" +templateset[0].hittemplate "<hit relevancy=\"$relevancy\">\n#foreach( $key in $hit.getPropertyKeySet() )\n <field name='$key'>$hit.getPropertyXML($key)</field>\n#end\n</hit>\n" +templateset[0].errortemplate "<ERROR CODE=\"$result.error.code\">$result.error.message</ERROR>\n" +templateset[1].urlprefix "/xsearch" +templateset[1].mimetype "text/xml" +templateset[1].encoding "utf-8" +templateset[1].rankprofile 0 +templateset[1].headertemplate "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n<RESULTSET TOTALHITS=\"$result.hitCount\">\n" +templateset[1].footertemplate "</RESULTSET>\n" +templateset[1].nohitstemplate "0.56" +templateset[1].hittemplate "<HIT RELEVANCY=\"$relevancy\" TYPE=\"$hit.typeString\">\n<FIELD NAME=\"uri\">$uri</FIELD>\n<FIELD NAME=\"category\">$category</FIELD>\n<FIELD NAME=\"bsumtitle\">$bsumtitle</FIELD>\n</HIT>\n" +templateset[1].errortemplate "45" +config[0].id :parent: +config[0].autostart +hi[0].there[0].e BATCH +ilscript[music].name music +ilscript[music].doctype music +ilscript[music].content[1] +ilscript[music].content[0] "\"music\" | summary sddocname | lowercase | index sddocname;" + +musum * + +auran "value=\"Confirm\"/></form>\"\"Tuna - step three." + +route[1] +route[0].name "search/cluster.books2" +route[0].selector "books.isbn=\"none\"" +route[0].feed "books" + +languages[3] +languages[2] "swahili" + +languages2[2] +languages2[0] "swedish" + +foolang[5] +foolang[0].lang[3] +foolang[0].lang[1] "Swahili" +foolang[1].lang[3] +foolang[2].lang[3] +foolang[2].lang[0] "Setswana" +foolang[3].lang[4] +foolang[3].lang[3] "Norwegian" +foolang[4].lang[2] + +myIntMap{"foo"} 67 +myIntMap{"bar"} 68 +myStringMap{"fo"} "gh" +myStringMap{"ba"} "ij" +myStructMap{"FOO"}.myInt 78 +myStructMap{"FOO"}.myString "myFoo" +myStructMap{"FOOO"}.myInt 89 +myStructMap{"FOOO"}.myString "myFooS" +myStructMap{"FOOO"}.myIntDef -99 +myStructMap{"FOOO"}.myStringDef "myFooSBall" + +myStructMap{"FOO"}.myStruct.a "guitar" +myStructMap{"FOO"}.myStruct.c /tmp + +myStructMap{"FOOO"}.myStruct.a "bass" +myStructMap{"FOOO"}.myStruct.b "drums" +myStructMap{"FOOO"}.myStruct.c /var/log + +myStructMap{"FOO"}.myStructNestedArray[0] -9 +myStructMap{"FOO"}.myStructNestedArray[1] -10 + +myStructMap{"FOO"}.myNestedLeafMap{"Nested1"} 90 +myStructMap{"FOO"}.myNestedLeafMap{"Nested2"} 9000 + +myStructMap{"FOO"}.myNestedMap{"Nested3"}.myLong 809 +myStructMap{"FOO"}.myNestedMap{"Nested3"}.myLongDef 810 +myStructMap{"FOO"}.myNestedMap{"Nested4"}.myLong 811 diff --git a/config/src/test/java/com/yahoo/vespa/config/configsglobal/testfoobar.12.cfg b/config/src/test/java/com/yahoo/vespa/config/configsglobal/testfoobar.12.cfg new file mode 100644 index 00000000000..3d23f7fd29a --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/configsglobal/testfoobar.12.cfg @@ -0,0 +1,105 @@ +vh[5] +vh[0] 0.3345 +vh[1] -0.3 +vh[2] 134.3 +vh[3] 234.34 +vh[4] -24.3 +bg[5] +bg[0] 5 +bg[1] +bg[2] 10 +bg[3] 4 +bg[4] -2 +gee[2] +gee[0] "jabbaTuna" +gee[1] jabba:500/dusteri +storage[2] +storage[0].distributor[2] +storage[0].distributor[0] "tra" +storage[0].distributor[1] "de" +storage[0].feeder[1] +storage[0].feeder[0] "li" +#storage[0].feeder[1] "dum" +storage[1].distributor[3] +storage[1].distributor[0] "Etra" +storage[1].distributor[1] "Ede" +#storage[1].feeder[3] +storage[1].feeder[0] "Eli" +storage[1].feeder[1] "Edum" +storage[1].feeder[2] "TEdum" + +#ju[].hu[].tu[] double default=45 range=[0,100.1] +#ju[].hu[].wu[] enum { HEY, HO} default=HO +#ju[].hu[].tang[] bool default=false +#ju[].hu[].clan[] int default=45 range=[-90,90] + +ju[2] +ju[0].hu[2] +ju[0].hu[0].tu[2] +ju[0].hu[0].tu[0] 45.0 +ju[0].hu[0].tu[1] 0.0 +ju[0].hu[0].tang[2] +ju[0].hu[0].tang[0] true +#ju[0].hu[0].tang[1] 0.0 + +#ju[0].hu[1].tu[2] +ju[0].hu[1].tu[0] 667.865 +ju[0].hu[0].wu[2] +ju[0].hu[0].wu[0] HEY +ju[0].hu[0].wu[1] HEY +ju[0].hu[1].wu[1] +ju[0].hu[1].wu[0] HO + +ju[1].hu[2] +ju[1].hu[0].tu[2] +ju[1].hu[0].tu[0] 78 +ju[1].hu[0].tu[1] 78.9 +ju[1].hu[1].tu[1] +ju[1].hu[1].tu[0] 88.9 + +ju[1].hu[0].wu[3] +ju[1].hu[0].wu[0] HEY +ju[1].hu[0].wu[1] HEY +ju[1].hu[0].wu[2] HO +ju[1].hu[1].wu[1] +ju[1].hu[1].wu[0] HO +ju[1].hu[1].clan[4] +ju[1].hu[1].clan[0] 5 +ju[1].hu[1].clan[1] 6 +ju[1].hu[1].clan[2] 7 +ju[1].hu[1].clan[3] 8 +foo "123aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa gh:sann\" da" +headertemplate "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 Transitional//EN\">\n<html>\n<head>\n<title>ranqualizer:$hostname:$profile:</title>\n<link type=\"text/css\" rel=\"stylesheet\" href=\"/layout.css\"/>\n<meta name=\"ROBOTS\" content=\"NOINDEX,NOFOLLOW\"/><meta http-equiv=\"Cache-control\" content=\"no-cache\"/><meta http-equiv=\"Cache-control\" content=\"must-revalidate\"/><meta http-equiv=\"Cache-control\" content=\"max-age=0\"/><meta http-equiv=\"Content-type\" content=\"text/html; charset=utf-8\"/><link rel=\"icon\" href=\"/favicon.ico\"/><script type=\"text/javascript\">function show(foo,f){document.getElementById(foo).style.display=\"block\";} function hide(foo,f){document.getElementById(foo).style.display = \"none\";}</script></head>\n<body>\n" +ptport 10108 +rankcf :parent: +hport 10109 +frtport 10107 +transportnodelay true +partition 0 +datasetdir none +proximity "yes" +vsmconfig "" +documentmanagerconfigid :parent: +include: search/cluster.music +include: search/cluster.music2 + +rankprofile[extra].fef.property[4] +rankprofile[ignore].fef.property[4].name "vespa.dump.ignoredefaultfeatures" +rankprofile[ignore].fef.property[4].value true + +specialchars "索 索尼 DVD±R" + +tokenlist[1] +tokenlist[0].name "default" +tokenlist[0].tokens[8] +tokenlist[0].tokens[0].token "c++" +tokenlist[0].tokens[1].token "wal-mart" +tokenlist[0].tokens[1].replace "walmart" +tokenlist[0].tokens[2].token ".net" +tokenlist[0].tokens[3].token "-索" +tokenlist[0].tokens[4].token "sony" +tokenlist[0].tokens[4].replace "索尼" +tokenlist[0].tokens[5].token "dvd+-r" +tokenlist[0].tokens[6].token "DVD±R" +tokenlist[0].tokens[7].token "dvdplusminusr" +tokenlist[0].tokens[7].replace "dvd+-r" diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java new file mode 100644 index 00000000000..e39c0a6f623 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.text.StringUtilities; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.LZ4PayloadCompressor; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringReader; +import java.util.List; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigResponseTest { + + @Test + public void require_that_slime_response_is_initialized() throws IOException { + ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); + DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); + InnerCNode targetDef = dParser.getTree(); + ConfigResponse response = SlimeConfigResponse.fromConfigPayload(configPayload, targetDef, 3, "mymd5"); + List<String> payload = response.getLegacyPayload(); + assertNotNull(payload); + assertThat(payload.size(), is(6)); + assertThat(payload.get(0), is("boolval false")); + assertThat(response.getGeneration(), is(3l)); + assertThat(response.getConfigMd5(), is("mymd5")); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.serialize(baos, CompressionType.UNCOMPRESSED); + assertThat(baos.toString(), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); + } + + @Test + public void require_that_slime_response_decompresses_on_serialize() throws IOException { + + ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); + DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); + InnerCNode targetDef = dParser.getTree(); + Utf8Array data = configPayload.toUtf8Array(true); + Utf8Array bytes = new Utf8Array(new LZ4PayloadCompressor().compress(data.getBytes())); + ConfigResponse response = new SlimeConfigResponse(bytes, targetDef, 3, "mymd5", CompressionInfo.create(CompressionType.LZ4, data.getByteLength())); + List<String> payload = response.getLegacyPayload(); + assertNotNull(payload); + assertThat(payload.size(), is(6)); + assertThat(payload.get(0), is("boolval false")); + assertThat(response.getGeneration(), is(3l)); + assertThat(response.getConfigMd5(), is("mymd5")); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.serialize(baos, CompressionType.UNCOMPRESSED); + assertThat(baos.toString(), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java new file mode 100644 index 00000000000..0cf6491432d --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java @@ -0,0 +1,286 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.config.subscription.ConfigSet; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.config.subscription.impl.GenericConfigSubscriber; +import com.yahoo.config.subscription.impl.JRTConfigRequester; +import com.yahoo.config.subscription.impl.JRTConfigSubscription; +import com.yahoo.config.subscription.impl.MockConnection; +import com.yahoo.jrt.Request; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import com.yahoo.test.ManualClock; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ErrorCode; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.TimingValues; +import com.yahoo.vespa.config.util.ConfigUtils; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.Optional; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.3 + */ +public abstract class JRTConfigRequestBase { + + protected String defName = "mydef"; + protected String defNamespace = "my.name.space"; + protected String hostname = "myhost"; + protected String configId = "config/id"; + protected String defMd5 = "595f44fec1e92a71d3e9e77456ba80d1"; + protected long currentGeneration = 3; + protected final Optional<VespaVersion> vespaVersion = Optional.of(VespaVersion.fromString("5.38.24")); + protected long timeout = 5000; + protected Trace trace ; + protected String configMd5 = ConfigUtils.getMd5(createPayload().getData()); + protected JRTClientConfigRequest clientReq; + protected JRTServerConfigRequest serverReq; + + @Before + public void setupRequest() throws IOException { + clientReq = createReq(); + serverReq = createReq(clientReq.getRequest()); + assertTrue(serverReq.validateParameters()); + } + + private JRTClientConfigRequest createReq() throws IOException { + trace = Trace.createNew(3, new ManualClock()); + trace.trace(1, "hei"); + return createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace); + } + + private JRTClientConfigRequest createReq(Payload payload) throws IOException { + trace = Trace.createNew(3, new ManualClock()); + trace.trace(1, "hei"); + return createReq(defName, defNamespace, defMd5, hostname, configId, ConfigUtils.getMd5(payload.getData()), currentGeneration, timeout, trace); + } + + protected abstract JRTClientConfigRequest createReq(String defName, String defNamespace, String defMd5, String hostname, String configId, String configMd5, long currentGeneration, long timeout, Trace trace) throws IOException; + protected abstract JRTServerConfigRequest createReq(Request request); + protected abstract JRTClientConfigRequest createReq(JRTConfigSubscription<SimpletypesConfig> sub, Trace aNew); + protected abstract JRTClientConfigRequest createFromRaw(RawConfig rawConfig, long serverTimeout, Trace aNew); + + protected void request_is_parsed_base() { + String [] expectedContent = new String[]{ + "namespace=my.name.space", + "myfield string" + }; + System.out.println(serverReq.toString()); + assertThat(serverReq.getConfigKey().getName(), is(defName)); + assertThat(serverReq.getConfigKey().getNamespace(), is(defNamespace)); + assertThat(serverReq.getConfigKey().getMd5(), is(defMd5)); + assertThat(serverReq.getConfigKey().getConfigId(), is(configId)); + assertThat(serverReq.getDefContent().asStringArray(), is(expectedContent)); + assertFalse(serverReq.noCache()); + assertTrue(serverReq.getRequestTrace().toString().contains("hi")); + assertThat(serverReq.getRequestConfigMd5(), is(configMd5)); + assertThat(serverReq.getRequestGeneration(), is(currentGeneration)); + } + + @Test + public void delay_mechanisms_functions() { + assertFalse(serverReq.isDelayedResponse()); + serverReq.setDelayedResponse(true); + assertTrue(serverReq.isDelayedResponse()); + serverReq.setDelayedResponse(false); + assertFalse(serverReq.isDelayedResponse()); + } + + public JRTServerConfigRequest next_request_is_correct_base() { + String [] expectedContent = new String[]{ + "namespace=my.name.space", + "myfield string" + }; + JRTServerConfigRequest next = createReq(clientReq.nextRequest(6).getRequest()); + assertThat(next.getConfigKey().getName(), is(defName)); + assertThat(next.getConfigKey().getNamespace(), is(defNamespace)); + assertThat(next.getConfigKey().getMd5(), is(defMd5)); + assertThat(next.getConfigKey().getConfigId(), is(configId)); + assertThat(next.getDefContent().asStringArray(), is(expectedContent)); + assertFalse(next.noCache()); + assertThat(next.getTimeout(), is(6l)); + assertThat(next.getTimeout(), is(6l)); + return next; + } + + + @Test + public void next_request_when_error_is_correct() { + serverReq.addOkResponse(createPayload(), 999999, "newmd5"); + serverReq.addErrorResponse(ErrorCode.OUTDATED_CONFIG, "error message"); + System.out.println(serverReq); + JRTClientConfigRequest next = clientReq.nextRequest(6); + System.out.println(next); + // Should use config md5 and generation from the request, not the response + // when there are errors + assertThat(next.getRequestConfigMd5(), is(clientReq.getRequestConfigMd5())); + assertThat(next.getRequestGeneration(), is(clientReq.getRequestGeneration())); + } + + @Test + public void ok_response_is_added() { + Payload payload = createPayload("vale"); + String md5 = ConfigUtils.getMd5(payload.getData()); + long generation = 4l; + serverReq.addOkResponse(payload, generation, md5); + assertTrue(clientReq.validateResponse()); + assertThat(clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString(), is(payload.getData().toString())); + assertThat(clientReq.getNewGeneration(), is(4l)); + assertThat(clientReq.getNewConfigMd5(), is(md5)); + assertTrue(clientReq.hasUpdatedConfig()); + assertTrue(clientReq.hasUpdatedGeneration()); + } + + @Test + public void error_response_adds_common_elements() { + serverReq.addErrorResponse(ErrorCode.APPLICATION_NOT_LOADED, ErrorCode.getName(ErrorCode.APPLICATION_NOT_LOADED)); + assertThat(serverReq.getRequest().returnValues().size(), is(1)); + Slime data = new JsonDecoder().decode(new Slime(), Utf8.toBytes(serverReq.getRequest().returnValues().get(0).asString())); + Inspector response = data.get(); + assertThat(response.field(SlimeResponseData.RESPONSE_DEF_NAME).asString(), is(defName)); + assertThat(response.field(SlimeResponseData.RESPONSE_DEF_NAMESPACE).asString(), is(defNamespace)); + assertThat(response.field(SlimeResponseData.RESPONSE_DEF_MD5).asString(), is(defMd5)); + assertThat(response.field(SlimeResponseData.RESPONSE_CONFIGID).asString(), is(configId)); + assertThat(response.field(SlimeResponseData.RESPONSE_CLIENT_HOSTNAME).asString(), is(hostname)); + Trace t = Trace.fromSlime(response.field(SlimeResponseData.RESPONSE_TRACE)); + assertThat(t.toString(), is(trace.toString())); + } + + @Test + public void generation_only_is_updated() { + Payload payload = createPayload(); + serverReq.addOkResponse(payload, 4l, ConfigUtils.getMd5(payload.getData())); + boolean value = clientReq.validateResponse(); + assertTrue(clientReq.errorMessage(), value); + assertFalse(clientReq.hasUpdatedConfig()); + assertTrue(clientReq.hasUpdatedGeneration()); + } + + protected static Payload createPayload() { + return createPayload("bar"); + } + + private static Payload createPayload(String value) { + Slime slime = new Slime(); + slime.setObject().setString("myfield", value); + return Payload.from(new ConfigPayload(slime)); + } + + @Test + public void nothing_is_updated() { + Payload payload = createPayload(); + serverReq.addOkResponse(payload, currentGeneration, configMd5); + assertTrue(clientReq.validateResponse()); + assertFalse(clientReq.hasUpdatedConfig()); + assertFalse(clientReq.hasUpdatedGeneration()); + } + + @Test + public void payload_is_empty() throws IOException { + Payload payload = Payload.from(ConfigPayload.empty()); + clientReq = createReq(payload); + serverReq = createReq(clientReq.getRequest()); + serverReq.addOkResponse(payload, currentGeneration, ConfigUtils.getMd5(payload.getData())); + boolean val = clientReq.validateResponse(); + assertTrue(clientReq.errorMessage(), val); + assertFalse(clientReq.hasUpdatedConfig()); + assertFalse(clientReq.hasUpdatedGeneration()); + } + + @Test + public void request_interface_is_implemented() { + JRTClientConfigRequest request = clientReq; + assertFalse(request.containsPayload()); + assertFalse(request.isError()); + assertThat(request.errorCode(), is(clientReq.getRequest().errorCode())); + assertThat(request.errorMessage(), is(clientReq.getRequest().errorMessage())); + assertNotNull(request.getRequest()); + assertFalse(request.validateResponse()); + //assertNull(request.getNewPayload().getData()); + assertThat(request.getTimeout(), is(timeout)); + assertFalse(request.hasUpdatedConfig()); + assertFalse(request.hasUpdatedGeneration()); + } + + @Test + public void created_from_subscription() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + JRTConfigSubscription<SimpletypesConfig> sub = new JRTConfigSubscription<>(new ConfigKey<>(SimpletypesConfig.class, configId), subscriber, new ConfigSet(), new TimingValues()); + JRTClientConfigRequest request = createReq(sub, Trace.createNew(9)); + assertThat(request.getConfigKey().getName(), is(SimpletypesConfig.CONFIG_DEF_NAME)); + JRTServerConfigRequest serverRequest = createReq(request.getRequest()); + assertTrue(serverRequest.validateParameters()); + } + + @Test + public void created_from_existing_subscription() { + System.setProperty("VESPA_CONFIG_PROTOCOL_VERSION", getProtocolVersion()); + MockConnection connection = new MockConnection(new MockConnection.AbstractResponseHandler() { + @Override + protected void createResponse() { + JRTServerConfigRequest serverRequest = createReq(request); + serverRequest.addOkResponse(createPayload(), currentGeneration, configMd5); + } + }); + + ConfigSourceSet src = new ConfigSourceSet(); + ConfigSubscriber subscriber = new GenericConfigSubscriber(Collections.singletonMap(src, JRTConfigRequester.get(connection, new TimingValues()))); + JRTConfigSubscription<SimpletypesConfig> sub = new JRTConfigSubscription<>(new ConfigKey<>(SimpletypesConfig.class, configId), subscriber, src, new TimingValues()); + sub.subscribe(120_0000); + assertTrue(sub.nextConfig(120_0000)); + sub.close(); + JRTClientConfigRequest nextReq = createReq(sub, Trace.createNew()); + SimpletypesConfig config = sub.getConfig(); + assertThat(nextReq.getRequestConfigMd5(), is(config.getConfigMd5())); + assertThat(nextReq.getRequestGeneration(), is(currentGeneration)); + System.setProperty("VESPA_CONFIG_PROTOCOL_VERSION", ""); + } + + protected abstract String getProtocolVersion(); + + @Test + public void created_from_raw() throws IOException { + RawConfig rawConfig = new RawConfig(new ConfigKey<>(defName, configId, defNamespace), defMd5); + long serverTimeout = 100000L; + JRTClientConfigRequest request = createFromRaw(rawConfig, serverTimeout, Trace.createNew(9)); + assertThat(request.getConfigKey().getName(), is(defName)); + JRTServerConfigRequest serverRequest = createReq(request.getRequest()); + assertTrue(serverRequest.validateParameters()); + assertThat(serverRequest.getTimeout(), is(serverTimeout)); + assertThat(serverRequest.getDefContent().asList(), is(rawConfig.getDefContent())); + } + + + @Test + public void parameters_are_validated() throws IOException { + assertTrue(serverReq.validateParameters()); + assertValidationFail(createReq("35#$#!$@#", defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, "abcd.o#$*(!&$", defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, "34", hostname, configId, "34", currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, "34", currentGeneration, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, -34, timeout, trace)); + assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, -23, trace)); + assertValidationFail(createReq(defName, defNamespace, defMd5, "", configId, configMd5, currentGeneration, timeout, trace)); + } + + private void assertValidationFail(JRTClientConfigRequest req) { + assertFalse(createReq(req.getRequest()).validateParameters()); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactoryTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactoryTest.java new file mode 100644 index 00000000000..ac3a7f16505 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactoryTest.java @@ -0,0 +1,143 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.foo.FunctionTestConfig; +import com.yahoo.config.subscription.ConfigSet; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.config.subscription.impl.JRTConfigSubscription; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.TimingValues; +import org.junit.Test; + +import java.util.Optional; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * @author musum + */ +public class JRTConfigRequestFactoryTest { + private static VespaVersion defaultVespaVersion = JRTConfigRequestFactory.getCompiledVespaVersion(); + + @Test + public void testGetProtocolVersion() { + assertThat(JRTConfigRequestFactory.getProtocolVersion("", "", ""), is("3")); + + assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "", ""), is("1")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("", "1", ""), is("1")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("", "", "1"), is("1")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "1", ""), is("1")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "", "1"), is("1")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("", "1", "1"), is("1")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "1", "1"), is("1")); + + assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "", ""), is("2")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("", "2", ""), is("2")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("", "", "2"), is("2")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "2", ""), is("2")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "", "2"), is("2")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("", "2", "2"), is("2")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "2", "2"), is("2")); + + assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "2", ""), is("1")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "", "2"), is("1")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("", "1", "2"), is("1")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "1", ""), is("2")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "", "1"), is("2")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("", "2", "1"), is("2")); + + assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "2", "2"), is("1")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "1", "2"), is("1")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "2", "1"), is("1")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "1", "1"), is("2")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "1", "2"), is("2")); + assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "2", "1"), is("2")); + } + + @Test + public void testCompressionType() { + assertThat(JRTConfigRequestFactory.getCompressionType("", "", ""), is(CompressionType.LZ4)); + + assertThat(JRTConfigRequestFactory.getCompressionType("UNCOMPRESSED", "", ""), is(CompressionType.UNCOMPRESSED)); + assertThat(JRTConfigRequestFactory.getCompressionType("", "UNCOMPRESSED", ""), is(CompressionType.UNCOMPRESSED)); + assertThat(JRTConfigRequestFactory.getCompressionType("", "", "UNCOMPRESSED"), is(CompressionType.UNCOMPRESSED)); + assertThat(JRTConfigRequestFactory.getCompressionType("UNCOMPRESSED", "UNCOMPRESSED", ""), is(CompressionType.UNCOMPRESSED)); + assertThat(JRTConfigRequestFactory.getCompressionType("UNCOMPRESSED", "", "UNCOMPRESSED"), is(CompressionType.UNCOMPRESSED)); + assertThat(JRTConfigRequestFactory.getCompressionType("", "UNCOMPRESSED", "UNCOMPRESSED"), is(CompressionType.UNCOMPRESSED)); + assertThat(JRTConfigRequestFactory.getCompressionType("UNCOMPRESSED", "UNCOMPRESSED", "UNCOMPRESSED"), is(CompressionType.UNCOMPRESSED)); + + assertThat(JRTConfigRequestFactory.getCompressionType("LZ4", "", ""), is(CompressionType.LZ4)); + assertThat(JRTConfigRequestFactory.getCompressionType("", "LZ4", ""), is(CompressionType.LZ4)); + assertThat(JRTConfigRequestFactory.getCompressionType("", "", "LZ4"), is(CompressionType.LZ4)); + assertThat(JRTConfigRequestFactory.getCompressionType("LZ4", "LZ4", ""), is(CompressionType.LZ4)); + assertThat(JRTConfigRequestFactory.getCompressionType("LZ4", "", "LZ4"), is(CompressionType.LZ4)); + assertThat(JRTConfigRequestFactory.getCompressionType("", "LZ4", "LZ4"), is(CompressionType.LZ4)); + assertThat(JRTConfigRequestFactory.getCompressionType("LZ4", "LZ4", "LZ4"), is(CompressionType.LZ4)); + + assertThat(JRTConfigRequestFactory.getCompressionType("UNCOMPRESSED", "LZ4", ""), is(CompressionType.UNCOMPRESSED)); + assertThat(JRTConfigRequestFactory.getCompressionType("UNCOMPRESSED", "", "LZ4"), is(CompressionType.UNCOMPRESSED)); + assertThat(JRTConfigRequestFactory.getCompressionType("", "UNCOMPRESSED", "LZ4"), is(CompressionType.UNCOMPRESSED)); + assertThat(JRTConfigRequestFactory.getCompressionType("LZ4", "UNCOMPRESSED", ""), is(CompressionType.LZ4)); + assertThat(JRTConfigRequestFactory.getCompressionType("LZ4", "", "UNCOMPRESSED"), is(CompressionType.LZ4)); + assertThat(JRTConfigRequestFactory.getCompressionType("", "LZ4", "UNCOMPRESSED"), is(CompressionType.LZ4)); + + assertThat(JRTConfigRequestFactory.getCompressionType("UNCOMPRESSED", "LZ4", "LZ4"), is(CompressionType.UNCOMPRESSED)); + assertThat(JRTConfigRequestFactory.getCompressionType("UNCOMPRESSED", "UNCOMPRESSED", "LZ4"), is(CompressionType.UNCOMPRESSED)); + assertThat(JRTConfigRequestFactory.getCompressionType("UNCOMPRESSED", "LZ4", "UNCOMPRESSED"), is(CompressionType.UNCOMPRESSED)); + assertThat(JRTConfigRequestFactory.getCompressionType("LZ4", "UNCOMPRESSED", "UNCOMPRESSED"), is(CompressionType.LZ4)); + assertThat(JRTConfigRequestFactory.getCompressionType("LZ4", "UNCOMPRESSED", "LZ4"), is(CompressionType.LZ4)); + assertThat(JRTConfigRequestFactory.getCompressionType("LZ4", "LZ4", "UNCOMPRESSED"), is(CompressionType.LZ4)); + } + + @Test + public void testVespaVersion() { + assertThat(JRTConfigRequestFactory.getVespaVersion().get(), is(defaultVespaVersion)); + } + + @Test + public void testCreateFromSub() { + ConfigSubscriber subscriber = new ConfigSubscriber(); + Class<FunctionTestConfig> clazz = FunctionTestConfig.class; + final String configId = "foo"; + JRTConfigSubscription<FunctionTestConfig> sub = new JRTConfigSubscription<>( + new ConfigKey<>(clazz, configId), subscriber, new ConfigSet(), new TimingValues()); + + // Default vespa version + JRTClientConfigRequest request = JRTConfigRequestFactory.createFromSub(sub); + assertThat(request.getProtocolVersion(), is(3L)); + assertThat(request.getVespaVersion().get(), is(defaultVespaVersion)); + + // Create with vespa version set + String version = "5.37.38"; + System.setProperty(JRTConfigRequestFactory.VESPA_VERSION, version); + request = JRTConfigRequestFactory.createFromSub(sub); + assertThat(request.getProtocolVersion(), is(3L)); + assertThat(request.getVespaVersion().get(), is(VespaVersion.fromString(version))); + + System.clearProperty(JRTConfigRequestFactory.VESPA_VERSION); + } + + @Test + public void testCreateFromRaw() { + Class<FunctionTestConfig> clazz = FunctionTestConfig.class; + final String configId = "foo"; + RawConfig config = new RawConfig(new ConfigKey<>(clazz, configId), "595f44fec1e92a71d3e9e77456ba80d1"); + + // Default vespa version + JRTClientConfigRequest request = JRTConfigRequestFactory.createFromRaw(config, 1000); + assertThat(request.getProtocolVersion(), is(3L)); + assertThat(request.getVespaVersion().get(), is(defaultVespaVersion)); + + // Create with vespa version set + String version = "5.37.38"; + System.setProperty(JRTConfigRequestFactory.VESPA_VERSION, version); + request = JRTConfigRequestFactory.createFromRaw(config, 1000); + assertThat(request.getProtocolVersion(), is(3L)); + assertThat(request.getVespaVersion().get(), is(VespaVersion.fromString(version))); + + System.clearProperty(JRTConfigRequestFactory.VESPA_VERSION); + } + +} diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java new file mode 100644 index 00000000000..bcb88b2ff5e --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.config.subscription.impl.JRTConfigSubscription; +import com.yahoo.jrt.Request; +import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.util.ConfigUtils; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.19 + */ +public class JRTConfigRequestV3Test extends JRTConfigRequestBase { + + @Override + protected JRTClientConfigRequest createReq(String defName, String defNamespace, String defMd5, String hostname, String configId, String configMd5, long currentGeneration, long timeout, Trace trace) throws IOException { + return JRTClientConfigRequestV3.createWithParams(ConfigKey.createFull(defName, configId, defNamespace, defMd5), + DefContent.fromList(Arrays.asList("namespace=my.name.space", "myfield string")), + hostname, + configMd5, + currentGeneration, + timeout, + trace, + CompressionType.LZ4, + vespaVersion); + } + + @Override + protected JRTServerConfigRequest createReq(Request request) { + return JRTServerConfigRequestV3.createFromRequest(request); + } + + @Override + protected JRTClientConfigRequest createReq(JRTConfigSubscription<SimpletypesConfig> sub, Trace aNew) { + return JRTClientConfigRequestV3.createFromSub(sub, aNew, CompressionType.LZ4, vespaVersion); + } + + @Override + protected JRTClientConfigRequest createFromRaw(RawConfig rawConfig, long serverTimeout, Trace aNew) { + return JRTClientConfigRequestV3.createFromRaw(rawConfig, serverTimeout, aNew, CompressionType.LZ4, vespaVersion); + } + + @Override + protected String getProtocolVersion() { + return "3"; + } + + @Test + public void request_is_parsed() { + request_is_parsed_base(); + assertThat(serverReq.getVespaVersion().toString(), is(vespaVersion.toString())); + } + + @Test + public void next_request_is_correct() { + JRTServerConfigRequest next = next_request_is_correct_base(); + assertThat(next.getVespaVersion().toString(), is(vespaVersion.toString())); + } + + @Test + public void emptypayload() { + ConfigPayload payload = ConfigPayload.empty(); + SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload, null, 0, ConfigUtils.getMd5(payload)); + serverReq.addOkResponse(serverReq.payloadFromResponse(response), response.getGeneration(), response.getConfigMd5()); + assertTrue(clientReq.validateResponse()); + assertTrue(clientReq.hasUpdatedGeneration()); + assertThat(clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString(), is("{}")); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java new file mode 100644 index 00000000000..c3cbfecf1f1 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.google.common.testing.EqualsTester; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.LZ4PayloadCompressor; +import org.junit.Test; + +import java.io.UnsupportedEncodingException; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.fail; + +/** + * @author lulf + * @since 5.21 + */ +public class PayloadTest { + + @Test + public void testUncompressedToCompressedWithoutCompressionInfo() { + String json = "{\"foo\":13}"; + ConfigPayload configPayload = ConfigPayload.fromString(json); + Payload payload = Payload.from(configPayload); + assertThat(payload.getData().toString(), is(json)); + payload = Payload.from(payload.getData(), CompressionInfo.create(CompressionType.UNCOMPRESSED, 0)); + Payload compressed = payload.withCompression(CompressionType.LZ4); + Payload uncompressed = compressed.withCompression(CompressionType.UNCOMPRESSED); + assertThat(uncompressed.getData().toString(), is(json)); + assertThat(compressed.toString(), is(json)); + assertThat(uncompressed.toString(), is(json)); + } + + @Test + public void testEquals() { + final String foo1 = "foo 1"; + final String foo2 = "foo 2"; + + Payload a = Payload.from(foo1); + Payload b = Payload.from(foo1); + + Payload c = Payload.from(foo2); + + Slime slime = new Slime(); + slime.setString(foo1); + Payload d = Payload.from(new ConfigPayload(slime)); + + slime.setString(foo1); + Payload e = Payload.from(new ConfigPayload(slime)); + + slime.setString("foo 2"); + Payload f = Payload.from(new ConfigPayload(slime)); + + Payload g = null; + Payload h = null; + Payload i = null; + Payload j = null; + try { + g = Payload.from(new Utf8Array(foo1.getBytes("UTF-8")), CompressionInfo.uncompressed()); + h = Payload.from(new Utf8Array(foo1.getBytes("UTF-8")), CompressionInfo.uncompressed()); + + LZ4PayloadCompressor compressor = new LZ4PayloadCompressor(); + CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, foo2.length()); + Utf8Array compressed = new Utf8Array(compressor.compress(foo2.getBytes())); + + i = Payload.from(compressed, info); + j = Payload.from(compressed, info); + } catch (UnsupportedEncodingException e1) { + fail(); + } + + new EqualsTester() + .addEqualityGroup(a, b, g, h) + .addEqualityGroup(c) + .addEqualityGroup(d, e) + .addEqualityGroup(f) + .addEqualityGroup(i, j). + testEquals(); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java new file mode 100644 index 00000000000..4c69fa3e395 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; +import com.yahoo.yolean.trace.TraceNode; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.5 + */ +public class SlimeTraceSerializerTest { + @Test + public void test_serializer() throws IOException { + TraceNode root = new TraceNode(null, 1); + root.add(new TraceNode("foo", 4)); + root.add(new TraceNode("bar", 5).add(new TraceNode("baz", 2).add(new TraceNode("quux", 10)))); + assertThat(toJson(root), is("{\"timestamp\":1,\"children\":[{\"timestamp\":5,\"payload\":\"bar\",\"children\":[{\"timestamp\":2,\"payload\":\"baz\",\"children\":[{\"timestamp\":10,\"payload\":\"quux\"}]}]},{\"timestamp\":4,\"payload\":\"foo\"}]}")); + assertSerialize(root); + } + + private String toJson(TraceNode root) throws IOException { + Slime slime = new Slime(); + SlimeTraceSerializer serializer = new SlimeTraceSerializer(slime.setObject()); + root.accept(serializer); + JsonFormat format = new JsonFormat(true); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + format.encode(baos, slime); + return Utf8.toString(baos.toByteArray()); + } + + private void assertSerialize(TraceNode root) { + Slime slime = new Slime(); + SlimeTraceSerializer serializer = new SlimeTraceSerializer(slime.setObject()); + root.accept(serializer); + SlimeTraceDeserializer deserializer = new SlimeTraceDeserializer(slime.get()); + TraceNode deser = deserializer.deserialize(); + assertTraceEqual(deser, root); + } + + private void assertTraceEqual(TraceNode deser, TraceNode root) { + assertThat(deser.timestamp(), is(root.timestamp())); + assertThat(deser.payload(), is(root.payload())); + Iterator<TraceNode> actualIt = deser.children().iterator(); + Iterator<TraceNode> expectedIt = root.children().iterator(); + Map<Long, TraceNode> expectedMapping = new HashMap<>(); + Map<Long, TraceNode> actualMapping = new HashMap<>(); + while (expectedIt.hasNext()) { + assertTrue(actualIt.hasNext()); + TraceNode actualNode = actualIt.next(); + TraceNode expectedNode = expectedIt.next(); + expectedMapping.put(expectedNode.timestamp(), expectedNode); + actualMapping.put(actualNode.timestamp(), actualNode); + } + assertFalse(expectedIt.hasNext()); + assertFalse(actualIt.hasNext()); + for (long timestamp : expectedMapping.keySet()) { + assertTraceEqual(actualMapping.get(timestamp), expectedMapping.get(timestamp)); + } + } + + @Test + public void test_long() throws IOException { + TraceNode root = new TraceNode(14l, 5); + assertThat(toJson(root), is("{\"timestamp\":5,\"payload\":14}")); + assertSerialize(root); + } + + @Test + public void test_double() throws IOException { + TraceNode root = new TraceNode(3.5, 5); + assertThat(toJson(root), is("{\"timestamp\":5,\"payload\":3.5}")); + assertSerialize(root); + } + + @Test + public void test_bool() throws IOException { + TraceNode root = new TraceNode(true, 5); + assertThat(toJson(root), is("{\"timestamp\":5,\"payload\":true}")); + assertSerialize(root); + } + + @Test + public void test_string() throws IOException { + TraceNode root = new TraceNode("bar", 5); + assertThat(toJson(root), is("{\"timestamp\":5,\"payload\":\"bar\"}")); + assertSerialize(root); + } + + @Test + public void test_unknown() throws IOException { + TraceNode root = new TraceNode(new ArrayList<String>(), 5); + assertThat(toJson(root), is("{\"timestamp\":5}")); + } + + @Test + public void test_null() throws IOException { + TraceNode root = new TraceNode(null, 5); + assertThat(toJson(root), is("{\"timestamp\":5}")); + assertSerialize(root); + } + + @Test + public void test_bytearray() throws IOException { + TraceNode root = new TraceNode(new byte[] { 3, 5 }, 5); + assertThat(toJson(root), is("{\"timestamp\":5,\"payload\":\"0x0305\"}")); + assertSerialize(root); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java new file mode 100644 index 00000000000..3e9d0650ae5 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.protocol; + +import com.yahoo.slime.Slime; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.3 + */ +public class TraceTest { + + @Test + public void trace_level_limits_tracing() { + Trace trace = Trace.createNew(3); + assertTrue(trace.shouldTrace(0)); + assertTrue(trace.shouldTrace(1)); + assertTrue(trace.shouldTrace(2)); + assertTrue(trace.shouldTrace(3)); + assertFalse(trace.shouldTrace(4)); + assertFalse(trace.shouldTrace(5)); + trace.trace(1, "foo"); + trace.trace(1, "foo2"); + trace.trace(2, "bar"); + trace.trace(3, "baz"); + trace.trace(3, "baz2"); + trace.trace(4, "quux"); + trace.trace(5, "quux2"); + String str = trace.toString(); + assertTrue(str.contains("foo")); + assertTrue(str.contains("foo2")); + assertTrue(str.contains("bar")); + assertTrue(str.contains("baz")); + assertTrue(str.contains("baz2")); + assertFalse(str.contains("quux")); + assertFalse(str.contains("quux2")); + } + + @Test + public void trace_serialization() { + Trace trace = Trace.createNew(1); + trace.trace(0, "foobar"); + trace.trace(1, "barbaz"); + Slime slime = new Slime(); + trace.serialize(slime.setObject()); + Trace trace2 = Trace.fromSlime(slime.get()); + trace2.trace(1, "quux"); + String trace1Str = trace.toString(); + String trace2Str = trace2.toString(); + assertTrue(trace1Str.contains("foobar")); + assertTrue(trace1Str.contains("barbaz")); + assertFalse(trace1Str.contains("quux")); + + assertTrue(trace2Str.contains("foobar")); + assertTrue(trace2Str.contains("barbaz")); + assertTrue(trace2Str.contains("quux")); + } +} diff --git a/config/src/test/java/com/yahoo/vespa/config/util/ConfigUtilsTest.java b/config/src/test/java/com/yahoo/vespa/config/util/ConfigUtilsTest.java new file mode 100644 index 00000000000..d07c007c13f --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/util/ConfigUtilsTest.java @@ -0,0 +1,288 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.util; + +import com.yahoo.collections.Tuple2; +import com.yahoo.foo.SimpletypesConfig; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigPayload; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * Tests ConfigUtils. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class ConfigUtilsTest { + + @Test + public void testGetDefMd5() { + String expectedMd5 = "a29038b17c727dabc572a967a508dc1f"; + List<String> lines = new ArrayList<>(); + + // Create normalized lines + lines.add("a"); + lines.add("foo=1 # a comment"); + lines.add("int a default=1 range = [,]"); + lines.add(""); //empty line should not affect md5sum + lines.add("double b default=1.0 range = [,]"); + lines.add("collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE"); + + assertThat(ConfigUtils.getDefMd5(lines), is(expectedMd5)); + + lines.clear(); + + // Test various normalizing features implemented by getMd5 + + // Check that lines are trimmed + lines.add("a "); + // Check that trailing comments are trimmed + lines.add("foo=1"); + // Check that upper and lower bounds for int and double ranges are set correctly + lines.add("int a default=1 range = [-2147483648,2147483647]"); + lines.add("double b default=1.0 range = [-100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000," + + "100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000]"); + // check that space before commas are treated correctly + lines.add("collectiontype enum { SINGLE , ARRAY , WEIGHTEDSET } default=SINGLE"); + assertThat(ConfigUtils.getDefMd5(lines), is(expectedMd5)); + } + + @Test + public void testGetMd5WithComment() { + String expectedMd5 = "4395db1dfbd977c4d74190d2d23396e2"; + List<String> lines = new ArrayList<>(); + + // Create normalized lines + lines.add("foo=\"1#hello\""); + lines.add(""); //empty line should not affect md5sum + + assertThat(ConfigUtils.getMd5(lines), is(expectedMd5)); + + lines.clear(); + + // Check that comment character in string leads to a different md5 than the original + lines.add("foo=\"1#hello and some more\""); + String md5 = ConfigUtils.getMd5(lines); + assertThat(md5, is(not(expectedMd5))); + + // Check that added characters aft comment character in string leads to a different md5 than above + lines.add("foo=\"1#hello and some more and even more\""); + assertThat(ConfigUtils.getMd5(lines), is(not(md5))); + } + + @Test + public void testGetMd5OfPayload() { + String expectedMd5 = "c9246ed8c8ab55b1c463c501c84075e6"; + String expectedChangedMd5 = "f6f81062ef5f024f1912798490ba7dfc"; + ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); + System.out.println(payload); + assertThat(ConfigUtils.getMd5(payload), is(expectedMd5)); + payload.getSlime().get().setString("fabio", "bar"); + System.out.println(payload); + assertThat(ConfigUtils.getMd5(payload), is(expectedChangedMd5)); + } + + @Test + public void testGetMd5OfString() { + String expectedMd5 = "c9246ed8c8ab55b1c463c501c84075e6"; + String expectedChangedMd5 = "f6f81062ef5f024f1912798490ba7dfc"; + ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); + System.out.println(payload); + assertThat(ConfigUtils.getMd5(payload.toString(true)), is(expectedMd5)); + payload.getSlime().get().setString("fabio", "bar"); + System.out.println(payload); + assertThat(ConfigUtils.getMd5(payload.toString(true)), is(expectedChangedMd5)); + } + + @Test + public void testStripSpaces() { + assertThat(ConfigUtils.stripSpaces("a b"), is("a b")); + assertThat(ConfigUtils.stripSpaces("\"a b\""), is("\"a b\"")); + assertThat(ConfigUtils.stripSpaces("a b \"a b\""), is("a b \"a b\"")); + assertThat(ConfigUtils.stripSpaces("a b"), is("a b")); + } + + @Test + public void testGetVersion() { + StringReader reader = new StringReader("version=1\nint a default=0"); + assertThat(ConfigUtils.getDefVersion(reader), is("1")); + + // no version + reader = new StringReader("int a default=0"); + assertThat(ConfigUtils.getDefVersion(reader), is("")); + + // namespace and version + reader = new StringReader("version=1\nnamespace=foo\nint a default=0"); + assertThat(ConfigUtils.getDefVersion(reader), is("1")); + reader = new StringReader("namespace=foo\nversion=1\nint a default=0"); + assertThat(ConfigUtils.getDefVersion(reader), is("1")); + } + + @Test + public void testGetNamespace() { + StringReader reader = new StringReader("version=1\nnamespace=a\nint a default=0"); + assertThat(ConfigUtils.getDefNamespace(reader), is("a")); + // namespace first + reader = new StringReader("namespace=a\nversion=1\nint a default=0"); + assertThat(ConfigUtils.getDefNamespace(reader), is("a")); + + // No namespace + reader = new StringReader("version=1\nint a default=0"); + assertThat(ConfigUtils.getDefNamespace(reader), is("")); + + // comment lines + reader = new StringReader("#comment\nversion=1\n#comment2\nint a default=0"); + assertThat(ConfigUtils.getDefNamespace(reader), is("")); + + try { + ConfigUtils.getDefNamespace(null); + fail(); + } catch (IllegalArgumentException e) { + // + } + } + + @Test + public void testGetNameCommaVersion() { + String nameCommaversion = "foo,1"; + Tuple2<String, String> tuple = ConfigUtils.getNameAndVersionFromString(nameCommaversion); + assertThat(tuple.first, is("foo")); + assertThat(tuple.second, is("1")); + + // no version + nameCommaversion = "foo"; + tuple = ConfigUtils.getNameAndVersionFromString(nameCommaversion); + assertThat(tuple.first, is("foo")); + assertThat(tuple.second, is("")); + + // no name + nameCommaversion = ",1"; + tuple = ConfigUtils.getNameAndVersionFromString(nameCommaversion); + assertThat(tuple.first, is("")); + assertThat(tuple.second, is("1")); + } + + @Test + public void testNamespaceDotNames() { + String namespaceDotName = "foo.bar"; + Tuple2<String, String> tuple = ConfigUtils.getNameAndNamespaceFromString(namespaceDotName); + assertThat(tuple.first, is("bar")); + assertThat(tuple.second, is("foo")); + + namespaceDotName = "foo.baz.bar"; + tuple = ConfigUtils.getNameAndNamespaceFromString(namespaceDotName); + assertThat(tuple.first, is("bar")); + assertThat(tuple.second, is("foo.baz")); + + // no namespace + namespaceDotName = "bar"; + tuple = ConfigUtils.getNameAndNamespaceFromString(namespaceDotName); + assertThat(tuple.first, is("bar")); + assertThat(tuple.second, is("")); + + // no name + namespaceDotName = "foo."; + tuple = ConfigUtils.getNameAndNamespaceFromString(namespaceDotName); + assertThat(tuple.first, is("")); + assertThat(tuple.second, is("foo")); + + // no namespace + namespaceDotName = ".bar"; + tuple = ConfigUtils.getNameAndNamespaceFromString(namespaceDotName); + assertThat(tuple.first, is("bar")); + assertThat(tuple.second, is("")); + } + + @Test + public void testGetConfigDefinitionKey() { + String input = "foo"; + ConfigDefinitionKey def = ConfigUtils.getConfigDefinitionKeyFromString(input); + assertThat(def.getName(), is("foo")); + assertThat(def.getNamespace(), is("")); + + input = "foo.bar"; + def = ConfigUtils.getConfigDefinitionKeyFromString(input); + assertThat(def.getName(), is("bar")); + assertThat(def.getNamespace(), is("foo")); + + input = "foo.bar.1"; + def = ConfigUtils.getConfigDefinitionKeyFromString(input); + assertThat(def.getName(), is("bar")); + assertThat(def.getNamespace(), is("foo")); + + input = "foo.bar.qux.2"; + def = ConfigUtils.getConfigDefinitionKeyFromString(input); + assertThat(def.getName(), is("qux")); + assertThat(def.getNamespace(), is("foo.bar")); + + input = "foo.2"; + def = ConfigUtils.getConfigDefinitionKeyFromString(input); + assertThat(def.getName(), is("foo")); + assertThat(def.getNamespace(), is("")); + } + + @Test + public void testCreateConfigDefinitionKeyFromZKString() { + String input = "foo,1"; + ConfigDefinitionKey def = ConfigUtils.createConfigDefinitionKeyFromZKString(input); + assertThat(def.getName(), is("foo")); + assertThat(def.getNamespace(), is("")); + + input = "bar.foo,1"; + def = ConfigUtils.createConfigDefinitionKeyFromZKString(input); + assertThat(def.getName(), is("foo")); + assertThat(def.getNamespace(), is("bar")); + } + + @Test + public void testCreateConfigDefinitionKeyFromDefFile() { + ConfigDefinitionKey def = null; + try { + def = ConfigUtils.createConfigDefinitionKeyFromDefFile(new File("src/test/resources/configs/def-files/app.def")); + } catch (IOException e) { + e.printStackTrace(); + fail(); + } + assertThat(def.getName(), is("app")); + assertThat(def.getNamespace(), is("foo")); + + try { + def = ConfigUtils.createConfigDefinitionKeyFromDefFile(new File("src/test/resources/configs/def-files/testnamespace.def")); + } catch (IOException e) { + e.printStackTrace(); + fail(); + } + assertThat(def.getName(), is("testnamespace")); + assertThat(def.getNamespace(), is("foo")); + + try { + byte[] content = IOUtils.readFileBytes(new File("src/test/resources/configs/def-files/app.def")); + def = ConfigUtils.createConfigDefinitionKeyFromDefContent("app", content); + } catch (IOException e) { + fail(); + } + assertThat(def.getName(), is("app")); + assertThat(def.getNamespace(), is("foo")); + + try { + byte[] content = IOUtils.readFileBytes(new File("src/test/resources/configs/def-files-nogen/foo.bar.app.def")); + def = ConfigUtils.createConfigDefinitionKeyFromDefContent("app", content); + } catch (IOException e) { + fail(); + } + assertThat(def.getName(), is("app")); + assertThat(def.getNamespace(), is("mynamespace")); + } + +} diff --git a/config/src/test/java/com/yahoo/vespa/config/xml/whitespace-test.xml b/config/src/test/java/com/yahoo/vespa/config/xml/whitespace-test.xml new file mode 100644 index 00000000000..6f63e405df8 --- /dev/null +++ b/config/src/test/java/com/yahoo/vespa/config/xml/whitespace-test.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<config name="string"> + <stringVal> This is a string + that contains different kinds of whitespace </stringVal> +</config> diff --git a/config/src/test/resources/configdefinitions/bar.def b/config/src/test/resources/configdefinitions/bar.def new file mode 100644 index 00000000000..9293df5afd5 --- /dev/null +++ b/config/src/test/resources/configdefinitions/bar.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=config + +barValue string default="defaultBar" diff --git a/config/src/test/resources/configdefinitions/baz.def b/config/src/test/resources/configdefinitions/baz.def new file mode 100644 index 00000000000..a505df39830 --- /dev/null +++ b/config/src/test/resources/configdefinitions/baz.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=config + +bazValue string default="defaultBaz" diff --git a/config/src/test/resources/configdefinitions/bootstrap.def b/config/src/test/resources/configdefinitions/bootstrap.def new file mode 100644 index 00000000000..8cd61352a09 --- /dev/null +++ b/config/src/test/resources/configdefinitions/bootstrap.def @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=config + +component[].name string +component[].configid string + diff --git a/config/src/test/resources/configdefinitions/foo.def b/config/src/test/resources/configdefinitions/foo.def new file mode 100644 index 00000000000..ace1ed3676d --- /dev/null +++ b/config/src/test/resources/configdefinitions/foo.def @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=config + +fooValue string +fooArray[] int +fooStruct[].innerStruct[].bar int +fooMap{} int diff --git a/config/src/test/resources/configdefinitions/foobar.def b/config/src/test/resources/configdefinitions/foobar.def new file mode 100644 index 00000000000..427abfa8d5d --- /dev/null +++ b/config/src/test/resources/configdefinitions/foobar.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=config + +fooBarValue string default="defaultFooBar" diff --git a/config/src/test/resources/configdefinitions/foodefault.def b/config/src/test/resources/configdefinitions/foodefault.def new file mode 100644 index 00000000000..06d84a092cf --- /dev/null +++ b/config/src/test/resources/configdefinitions/foodefault.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=config + +fooValue string default = "per" diff --git a/config/src/test/resources/configdefinitions/function-test.def b/config/src/test/resources/configdefinitions/function-test.def new file mode 100644 index 00000000000..40047418eb0 --- /dev/null +++ b/config/src/test/resources/configdefinitions/function-test.def @@ -0,0 +1,74 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# This def file should test most aspects of def files that makes a difference +# for the autogenerated config classes. The goal is to trigger all blocks of +# code in the code generators. This includes: +# +# - Use all legal special characters in the def file name, to ensure that those +# that needs to be replaced in type names are actually replaced. +# - Use the same enum type twice to verify that we dont declare or define it +# twice. +# - Use the same struct type twice for the same reason. +# - Include arrays of primitives and structs. +# - Include enum primitives and array of enums. Arrays of enums must be handled +# specially by the C++ code. +# - Include enums both with and without default values. +# - Include primitive string, numbers & doubles both with and without default +# values. +# - Have an array within a struct, to verify that we correctly recurse. +# - Reuse type name further within to ensure that this works. + +namespace=config + +# Some random bool without a default value. These comments exist to check + # that comment parsing works. +bool_val bool + ## A bool with a default value set. +bool_with_def bool default=false +int_val int +int_with_def int default=-545 +long_val long +long_with_def long default=-50000000000 +double_val double +double_with_def double default=-6.43 +# Another comment +string_val string +stringwithdef string default="foobar" +enum_val enum { FOO, BAR, FOOBAR } +enumwithdef enum { FOO2, BAR2, FOOBAR2 } default=BAR2 +onechoice enum { ONLYFOO } default=ONLYFOO +refval reference +refwithdef reference default=":parent:" +fileVal file + +boolarr[] bool +intarr[] int +longarr[] long +doublearr[] double +stringarr[] string +enumarr[] enum { ARRAY, VALUES } +refarr[] reference +fileArr[] file + +# A basic struct +basicStruct.foo string default="basic" +basicStruct.bar int +basicStruct.intArr[] int + +# A struct of struct +rootStruct.inner0.name string default="inner0" +rootStruct.inner0.index int +rootStruct.inner1.name string default="inner1" +rootStruct.inner1.index int +rootStruct.inner2.array[] int +rootStruct.innerArr[].boolVal bool default=false +rootStruct.innerArr[].stringVal string + +myarray[].intval int default=14 +myarray[].stringval[] string +myarray[].enumval enum { INNER, ENUM, TYPE } default=TYPE +myarray[].refval reference # Value in array without default +myarray[].fileVal file +myarray[].anotherarray[].foo int default=-4 +myarray[].myStruct.a int +myarray[].myStruct.b int default=2 diff --git a/config/src/test/resources/configdefinitions/motd.def b/config/src/test/resources/configdefinitions/motd.def new file mode 100644 index 00000000000..554ffa8fb28 --- /dev/null +++ b/config/src/test/resources/configdefinitions/motd.def @@ -0,0 +1,84 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# This def file should test most aspects of def files that makes a difference +# for the autogenerated config classes. The goal is to trigger all blocks of +# code in the code generators. This includes: +# +# - Use all legal special characters in the def file name, to ensure that those +# that needs to be replaced in type names are actually replaced. +# - Use the same enum type twice to verify that we dont declare or define it +# twice. +# - Use the same struct type twice for the same reason. +# - Include arrays of primitives and structs. +# - Include enum primitives and array of enums. Arrays of enums must be handled +# specially by the C++ code. +# - Include enums both with and without default values. +# - Include primitive string, numbers & doubles both with and without default +# values. +# - Have an array within a struct, to verify that we correctly recurse. +# - Reuse type name further within to ensure that this works. + +namespace=config + +# Some random bool without a default value. These comments exist to check + # that comment parsing works.e +boolVal bool + ## A bool with a default value set. +bool_with_def bool default=false +intVal int +intWithDef int default=-545 +longVal long +longWithDef long default=1234567890123 +doubleVal double +double_with_def double default=-6.43 +# Another comment +stringVal string +stringwithdef string default="foobar" +stringnulldef string default="null" +enumVal enum { FOO, BAR, FOOBAR } +enumwithdef enum { FOO2, BAR2, FOOBAR2 } default=BAR2 +refVal reference +refwithdef reference default=":parent:" +fileVal file + +boolarr[] bool +intarr[] int +longarr[] long +doublearr[] double +stringarr[] string +enumarr[] enum { ARRAY, VALUES } +refarr[] reference +filearr[] file + +boolmap{} bool +intmap{} int +longmap{} long +doublemap{} double +stringmap{} string +enummap{} enum { LOL1, LOL2 } +refmap{} reference +filemap{} file + +structmap{}.foo int +mapmap{}.map{}.bar int + +# A basic struct +basic_struct.foo string default="foo" +basic_struct.bar int default=0 + +# A struct of struct +struct_of_struct.inner0.name string default="inner0" +struct_of_struct.inner0.index int default=0 +struct_of_struct.inner1.name string default="inner1" +struct_of_struct.inner1.index int default=1 + +myArray[].intVal int default=14 +myArray[].stringVal[] string +myArray[].enumVal enum { INNER, ENUM, TYPE } default=TYPE +myArray[].refVal reference # Value in array without default +myArray[].anotherArray[].foo int default=-4 + +value string default="value" +buffer int default=-1 +rhs string default="rhs" +lines string default="lines" diff --git a/config/src/test/resources/configdefinitions/my.def b/config/src/test/resources/configdefinitions/my.def new file mode 100644 index 00000000000..8c6728fc63c --- /dev/null +++ b/config/src/test/resources/configdefinitions/my.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=config + +myField string diff --git a/config/src/test/resources/configs/bar/app.cfg b/config/src/test/resources/configs/bar/app.cfg new file mode 100644 index 00000000000..8e4096805ea --- /dev/null +++ b/config/src/test/resources/configs/bar/app.cfg @@ -0,0 +1,7 @@ +message "msg2" +times 2 +a[4] +a[0].name "a0" +a[1].name "a1" +a[2].name "a2" +a[3].name "a3" diff --git a/config/src/test/resources/configs/bar/string.cfg b/config/src/test/resources/configs/bar/string.cfg new file mode 100644 index 00000000000..5cb1a496e0e --- /dev/null +++ b/config/src/test/resources/configs/bar/string.cfg @@ -0,0 +1 @@ +stringVal "My mess" diff --git a/config/src/test/resources/configs/baz/app.1.cfg.new b/config/src/test/resources/configs/baz/app.1.cfg.new new file mode 100644 index 00000000000..b734dd17297 --- /dev/null +++ b/config/src/test/resources/configs/baz/app.1.cfg.new @@ -0,0 +1,85 @@ +times 3 +a[0]=1 +a[2]=1 +a[3]=1 +a[4]=1 +a[5]=1 +a[6]=1 +a[7]=1 +a[8]=1 +a[9]=1 +a[10]=1 +a[11]=1 +a[12]=1 +a[13]=1 +a[14]=1 +a[15]=1 +a[16]=1 +a[17]=1 +a[18]=1 +a[19]=1 +a[20]=1 +a[21]=1 +a[22]=1 +a[23]=1 +a[24]=1 +a[25]=1 +a[26]=1 +a[27]=1 +a[28]=1 +a[29]=1 +a[30]=1 +a[31]=1 +a[32]=1 +a[33]=1 +a[34]=1 +a[35]=1 +a[36]=1 +a[37]=1 +a[38]=1 +a[39]=1 +a[40]=1 +a[41]=1 +a[42]=1 +a[43]=1 +a[44]=1 +a[45]=1 +a[46]=1 +a[47]=1 +a[48]=1 +a[49]=1 +a[50]=1 +a[51]=1 +a[52]=1 +a[53]=1 +a[54]=1 +a[55]=1 +a[56]=1 +a[57]=1 +a[58]=1 +a[59]=1 +a[60]=1 +a[61]=1 +a[62]=1 +a[63]=1 +a[64]=1 +a[65]=1 +a[66]=1 +a[67]=1 +a[68]=1 +a[69]=1 +version 1 +version 1 +version 1 +version 1 +version 1 +version 1 +version 1 +version 1 +version 1 +version 1 +version 1 +version 1 +version 1 +version 1 +version 2 diff --git a/config/src/test/resources/configs/baz/app.cfg b/config/src/test/resources/configs/baz/app.cfg new file mode 100644 index 00000000000..b04f16fe7c9 --- /dev/null +++ b/config/src/test/resources/configs/baz/app.cfg @@ -0,0 +1,5 @@ +times 3 +a[3] +a[0].name "a0" +a[1].name "a1" +a[2].name "a2" diff --git a/config/src/test/resources/configs/datastructures/config1.txt b/config/src/test/resources/configs/datastructures/config1.txt new file mode 100644 index 00000000000..f6121cfa486 --- /dev/null +++ b/config/src/test/resources/configs/datastructures/config1.txt @@ -0,0 +1,17 @@ +date[3] +date[0] 14-Apr-08 +date[1] 15-Apr-08 +date[2] 16-Apr-08 +stock[2] +stock[0].ticker YHOO +stock[0].type COMMON +stock[0].volume[3] +stock[0].volume[0] 1333000 +stock[0].volume[1] 996000 +stock[0].volume[2] 2400000 +stock[1].ticker EWJ +stock[1].type ETF +stock[1].volume[3] +stock[1].volume[0] 4600000 +stock[1].volume[1] 3002000 +stock[1].volume[2] 10987000 diff --git a/config/src/test/resources/configs/datastructures/config1_1.txt b/config/src/test/resources/configs/datastructures/config1_1.txt new file mode 100644 index 00000000000..33a6d6c22aa --- /dev/null +++ b/config/src/test/resources/configs/datastructures/config1_1.txt @@ -0,0 +1,14 @@ +date[2] +date[0] 16-Apr-08 +date[1] 17-Apr-08 +stock[2] +stock[0].ticker YHOO +stock[0].type COMMON +stock[0].volume[2] +stock[0].volume[0] 1333000 +stock[0].volume[1] 800800 +stock[1].ticker JNJ +stock[1].type COMMON +stock[1].volume[2] +stock[1].volume[0] 10987000 +stock[1].volume[1] 1300000 diff --git a/config/src/test/resources/configs/datastructures/config2.txt b/config/src/test/resources/configs/datastructures/config2.txt new file mode 100644 index 00000000000..6cc31add025 --- /dev/null +++ b/config/src/test/resources/configs/datastructures/config2.txt @@ -0,0 +1,19 @@ +date[3] +date[0] 14-Apr-08 +date[1] 15-Apr-08 +date[2] 16-Apr-08 +stock[2] +stock[0].ticker YHOO +stock[0].type COMMON +stock[0].volume[3] +stock[0].volume[0] 1333000 +stock[0].volume[1] 996000 +stock[0].volume[2] 2400000 +stock[1].ticker EWJ +stock[1].type ETF +stock[1].volume[3] +stock[1].volume[0] 4600000 +stock[1].volume[1] 3002000 +stock[1].volume[2] 10987000 +basicstruct.foo "bar" +basicstruct.bar 42 diff --git a/config/src/test/resources/configs/def-files-nogen/foo.bar.app.def b/config/src/test/resources/configs/def-files-nogen/foo.bar.app.def new file mode 100644 index 00000000000..d2a1df70e7a --- /dev/null +++ b/config/src/test/resources/configs/def-files-nogen/foo.bar.app.def @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=mynamespace + +message string default="Hello!" + +times int default=1 + +a[].name string diff --git a/config/src/test/resources/configs/def-files/app.def b/config/src/test/resources/configs/def-files/app.def new file mode 100644 index 00000000000..30919c866e7 --- /dev/null +++ b/config/src/test/resources/configs/def-files/app.def @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=foo +version=1 + +message string default="Hello!" + +times int default=1 + +a[].name string diff --git a/config/src/test/resources/configs/def-files/arraytypes.def b/config/src/test/resources/configs/def-files/arraytypes.def new file mode 100644 index 00000000000..b6e206ff925 --- /dev/null +++ b/config/src/test/resources/configs/def-files/arraytypes.def @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Config containing only simple array types that can be used for testing +# individual types in detail. +namespace=foo +version=1 + +boolarr[] bool +doublearr[] double +enumarr[] enum { VAL1, VAL2 } +intarr[] int +longarr[] long +stringarr[] string diff --git a/config/src/test/resources/configs/def-files/chains-test.def b/config/src/test/resources/configs/def-files/chains-test.def new file mode 100644 index 00000000000..26a23ff17da --- /dev/null +++ b/config/src/test/resources/configs/def-files/chains-test.def @@ -0,0 +1,43 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Chains configuration +namespace=foo +version=12 + +component[].id string + +# Configured functionality provided by this - comes in addition to those set in the code +component[].dependencies.provides[] string + +# Configured "before" dependencies provided by this - comes in addition to those set in the code +component[].dependencies.before[] string + +# Configured "after" dependencies provided by this - comes in addition to those set in the code +component[].dependencies.after[] string + +# The id of this chain. The id has the form name(:version)? +# where the version has the form 1(.2(.3(.identifier)?)?)?. +# The default chain must be called "default". +chain[].id string + +#The type of this chain +chain[].type enum {DOCPROC, SEARCH} default=SEARCH + +# The id of a component to include in this chain. +# The id has the form fullclassname(:version)? +# where the version has the form 1(.2(.3(.identifier)?)?)?. +chain[].component[] string + +# The optional list of chain ids this inherits. +# The ids has the form name(:version)? +# where the version has the form 1(.2(.3(.identifier)?)?)?. +# If the version is not specified the newest version is used. +chain[].inherit[] string + +# The optional list of component ids to exclude from this chain even if they exists in inherited chains +# If versions are specified in these ids, they are ignored. +chain[].exclude[] string + +# The phases for a chain +chain[].phase[].id string +chain[].phase[].before[] string +chain[].phase[].after[] string diff --git a/config/src/test/resources/configs/def-files/datastructures.def b/config/src/test/resources/configs/def-files/datastructures.def new file mode 100644 index 00000000000..803931c91bc --- /dev/null +++ b/config/src/test/resources/configs/def-files/datastructures.def @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=foo +version=4 + +date[] string + +stock[].ticker string +stock[].type enum { COMMON, ETF, ETC } default=COMMON +stock[].volume[] int + +basicstruct.foo string default="foo" +basicstruct.bar int default=0 diff --git a/config/src/test/resources/configs/def-files/defaulttest.def b/config/src/test/resources/configs/def-files/defaulttest.def new file mode 100644 index 00000000000..3934a0e4d12 --- /dev/null +++ b/config/src/test/resources/configs/def-files/defaulttest.def @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=foo +version=3 + +nondefaultstring string +defaultstring string default="thedefault" + +nondefaultreference reference +defaultreference reference default="thedefault" diff --git a/config/src/test/resources/configs/def-files/function-test.def b/config/src/test/resources/configs/def-files/function-test.def new file mode 100644 index 00000000000..4529630c6b1 --- /dev/null +++ b/config/src/test/resources/configs/def-files/function-test.def @@ -0,0 +1,89 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# This def file should test most aspects of def files that makes a difference +# for the autogenerated config classes. The goal is to trigger all blocks of +# code in the code generators. This includes: +# +# - Use all legal special characters in the def file name, to ensure that those +# that needs to be replaced in type names are actually replaced. +# - Use the same enum type twice to verify that we dont declare or define it +# twice. +# - Use the same struct type twice for the same reason. +# - Include arrays of primitives and structs. +# - Include enum primitives and array of enums. Arrays of enums must be handled +# specially by the C++ code. +# - Include enums both with and without default values. +# - Include primitive string, numbers & doubles both with and without default +# values. +# - Have an array within a struct, to verify that we correctly recurse. +# - Reuse type name further within to ensure that this works. + +version=8 + +namespace=foo + +# Some random bool without a default value. These comments exist to check + # that comment parsing works. +bool_val bool + ## A bool with a default value set. +bool_with_def bool default=false +int_val int +int_with_def int default=-545 +long_val long +long_with_def long default=-50000000000 +double_val double +double_with_def double default=-6.43 +# Another comment +string_val string +stringwithdef string default="foobar" +enum_val enum { FOO, BAR, FOOBAR } +enumwithdef enum { FOO2, BAR2, FOOBAR2 } default=BAR2 +onechoice enum { ONLYFOO } default=ONLYFOO +refval reference +refwithdef reference default=":parent:" +fileVal file +pathVal path + +boolarr[] bool +intarr[] int +longarr[] long +doublearr[] double +stringarr[] string +enumarr[] enum { ARRAY, VALUES } +refarr[] reference +fileArr[] file +pathArr[] path + +intMap{} int +stringMap{} string +filemap{} file +pathMap{} path + +# A basic struct +basicStruct.foo string default="basic" +basicStruct.bar int +basicStruct.intArr[] int + +# A struct of struct +rootStruct.inner0.name string default="inner0" +rootStruct.inner0.index int +rootStruct.inner1.name string default="inner1" +rootStruct.inner1.index int +rootStruct.innerArr[].boolVal bool default=false +rootStruct.innerArr[].stringVal string + +myarray[].intval int default=14 +myarray[].stringval[] string +myarray[].enumval enum { INNER, ENUM, TYPE } default=TYPE +myarray[].refval reference # Value in array without default +myarray[].fileVal file +myarray[].anotherarray[].foo int default=-4 +myarray[].myStruct.a int +myarray[].myStruct.b int default=2 + +myStructMap{}.myInt int +myStructMap{}.myString string +myStructMap{}.myIntDef int default=56 +myStructMap{}.myStringDef string default="g" +myStructMap{}.anotherMap{}.anInt int +myStructMap{}.anotherMap{}.anIntDef int default=11 diff --git a/config/src/test/resources/configs/def-files/int.def b/config/src/test/resources/configs/def-files/int.def new file mode 100755 index 00000000000..fd07c90dfd5 --- /dev/null +++ b/config/src/test/resources/configs/def-files/int.def @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=foo +version=1 + +intVal int default=1 diff --git a/config/src/test/resources/configs/def-files/maptypes.def b/config/src/test/resources/configs/def-files/maptypes.def new file mode 100644 index 00000000000..389a9b71012 --- /dev/null +++ b/config/src/test/resources/configs/def-files/maptypes.def @@ -0,0 +1,13 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Config containing only structs in various forms +namespace=foo + +boolmap{} bool +intmap{} int +longmap{} long +doublemap{} double +stringmap{} string +filemap{} file + +innermap{}.foo int +nestedmap{}.inner{} int diff --git a/config/src/test/resources/configs/def-files/md5test.def b/config/src/test/resources/configs/def-files/md5test.def new file mode 100644 index 00000000000..f9fd7a645d2 --- /dev/null +++ b/config/src/test/resources/configs/def-files/md5test.def @@ -0,0 +1,27 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=foo +# version=4 , version in comment does not count. + +# Added empty line to see if we can confuse +# the server's md5 calculation +version=3 + +#even adding a variable name starting with 'version' +versiontag int default=3 + +blabla string default="" +tabs string default=" " +test int + +# test multiple spaces/tabs +spaces int +singletab string +multitabs double + +# test enum +normal enum { VAL1, VAL2 } default=VAL1 +spacevalues enum { V1 , V2 , V3 , V4 } default=V3 + +# Comments and empty lines at the end + + diff --git a/config/src/test/resources/configs/def-files/namespace.def b/config/src/test/resources/configs/def-files/namespace.def new file mode 100644 index 00000000000..429b835a6aa --- /dev/null +++ b/config/src/test/resources/configs/def-files/namespace.def @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 + +namespace=myproject.config + +a int diff --git a/config/src/test/resources/configs/def-files/simpletypes.def b/config/src/test/resources/configs/def-files/simpletypes.def new file mode 100644 index 00000000000..e3954a92edc --- /dev/null +++ b/config/src/test/resources/configs/def-files/simpletypes.def @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=foo +# Config containing only simple leaf types with default values, that can be used +# for testing individual types in detail. +version=1 + +boolval bool default=false +doubleval double default=0.0 +enumval enum { VAL1, VAL2 } default=VAL1 +intval int default=0 +longval long default=0 +stringval string default="s" diff --git a/config/src/test/resources/configs/def-files/specialtypes.def b/config/src/test/resources/configs/def-files/specialtypes.def new file mode 100644 index 00000000000..7b9b68d38a6 --- /dev/null +++ b/config/src/test/resources/configs/def-files/specialtypes.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=foo +myfile file +myref reference diff --git a/config/src/test/resources/configs/def-files/standard.def b/config/src/test/resources/configs/def-files/standard.def new file mode 100644 index 00000000000..94e2e3133f4 --- /dev/null +++ b/config/src/test/resources/configs/def-files/standard.def @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Config containing only simple leaf types with default values, that can be used +# for testing individual types in detail. +namespace=foo +version=1 + +basicStruct.intVal int default=0 +basicStruct.stringVal string default="s" +stringArr[] string diff --git a/config/src/test/resources/configs/def-files/string.def b/config/src/test/resources/configs/def-files/string.def new file mode 100755 index 00000000000..0605dc6e10c --- /dev/null +++ b/config/src/test/resources/configs/def-files/string.def @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=foo +version=1 + +stringVal string default="_default_" diff --git a/config/src/test/resources/configs/def-files/structtypes.def b/config/src/test/resources/configs/def-files/structtypes.def new file mode 100644 index 00000000000..dfcec942799 --- /dev/null +++ b/config/src/test/resources/configs/def-files/structtypes.def @@ -0,0 +1,22 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Config containing only structs in various forms +namespace=foo +version=2 + +simple.name string default="_default_" +simple.gender enum { MALE, FEMALE } default=MALE +simple.emails[] string + +nested.inner.name string default="_default_" +nested.inner.gender enum { MALE, FEMALE } default=MALE +nested.inner.emails[] string + +simplearr[].name string +simplearr[].gender enum { MALE, FEMALE } + +nestedarr[].inner.name string +nestedarr[].inner.gender enum { MALE, FEMALE } +nestedarr[].inner.emails[] string + +complexarr[].innerarr[].name string +complexarr[].innerarr[].gender enum { MALE, FEMALE } diff --git a/config/src/test/resources/configs/def-files/test-nodefs.def b/config/src/test/resources/configs/def-files/test-nodefs.def new file mode 100644 index 00000000000..e35438a26ff --- /dev/null +++ b/config/src/test/resources/configs/def-files/test-nodefs.def @@ -0,0 +1,17 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=2 +namespace=foo + +# test config vars with no defaults + +s string +j int +b bool +f double +e enum { AA, BB, CC } + +basicstruct.foo string +basicstruct.bar int + +arr[].s string +arr[].i int diff --git a/config/src/test/resources/configs/def-files/test-nonstring.def b/config/src/test/resources/configs/def-files/test-nonstring.def new file mode 100644 index 00000000000..4819b32e1ba --- /dev/null +++ b/config/src/test/resources/configs/def-files/test-nonstring.def @@ -0,0 +1,10 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +namespace=foo + +# Test non-string config vars with defaults + +i int default=0 +b bool default=false +d double default=0.0 +e enum { AA, BB, CC } default=AA diff --git a/config/src/test/resources/configs/def-files/test-reference.def b/config/src/test/resources/configs/def-files/test-reference.def new file mode 100644 index 00000000000..dacb9a18544 --- /dev/null +++ b/config/src/test/resources/configs/def-files/test-reference.def @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +namespace=foo + +configId reference default=":parent:" diff --git a/config/src/test/resources/configs/def-files/testnamespace.def b/config/src/test/resources/configs/def-files/testnamespace.def new file mode 100644 index 00000000000..56710807954 --- /dev/null +++ b/config/src/test/resources/configs/def-files/testnamespace.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +namespace=foo +basicStruct.stringVal string diff --git a/config/src/test/resources/configs/def-files/unicode.def b/config/src/test/resources/configs/def-files/unicode.def new file mode 100644 index 00000000000..310564f4cfd --- /dev/null +++ b/config/src/test/resources/configs/def-files/unicode.def @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=2 +namespace=foo + +unicodestring1 string +unicodestring2 string default="abc æøå 囲碁 ÆØÅ ABC" diff --git a/config/src/test/resources/configs/foo/app.cfg b/config/src/test/resources/configs/foo/app.cfg new file mode 100644 index 00000000000..30426957db6 --- /dev/null +++ b/config/src/test/resources/configs/foo/app.cfg @@ -0,0 +1,6 @@ +message "msg1" +times 3 +a[3] +a[0].name "a0" +a[1].name "a1" +a[2].name "a2" diff --git a/config/src/test/resources/configs/foo/test-reference.cfg b/config/src/test/resources/configs/foo/test-reference.cfg new file mode 100644 index 00000000000..127d63f27dc --- /dev/null +++ b/config/src/test/resources/configs/foo/test-reference.cfg @@ -0,0 +1 @@ +configId ":parent:" diff --git a/config/src/test/resources/configs/function-test/defaultvalues.txt b/config/src/test/resources/configs/function-test/defaultvalues.txt new file mode 100644 index 00000000000..14a7de7a9ed --- /dev/null +++ b/config/src/test/resources/configs/function-test/defaultvalues.txt @@ -0,0 +1,47 @@ +bool_val false +int_val 5 +long_val 1234567890123 +double_val 41.23 +string_val "foo" +enum_val FOOBAR +refval :parent: +fileVal "vespa.log" +pathVal "pom.xml" +boolarr[1] +boolarr[0] false +intarr[0] +longarr[0] +doublearr[2] +doublearr[0] 2344 +doublearr[1] 123 +stringarr[1] +stringarr[0] "bar" +enumarr[1] +enumarr[0] VALUES +refarr[0] +fileArr[0] + +basicStruct.bar 3 +basicStruct.intArr[1] +basicStruct.intArr[0] 10 +rootStruct.inner0.index 11 +rootStruct.inner1.index 12 +rootStruct.innerArr[1] +rootStruct.innerArr[0].stringVal "deep" + +myarray[2] +myarray[0].stringval[2] +myarray[0].stringval[0] "baah" +myarray[0].stringval[1] "yikes" +myarray[0].refval ":parent:" +myarray[0].fileVal "command.com" +myarray[0].anotherarray[1] +myarray[0].anotherarray[0].foo 7 +myarray[0].myStruct.a 1 +myarray[1].stringval[0] +myarray[1].refval ":parent:" +myarray[1].fileVal "display.sys" +myarray[1].anotherarray[2] +myarray[1].anotherarray[0].foo 1 +myarray[1].anotherarray[1].foo 2 +myarray[1].myStruct.a -1 diff --git a/config/src/test/resources/configs/function-test/defaultvalues.xml b/config/src/test/resources/configs/function-test/defaultvalues.xml new file mode 100644 index 00000000000..80644c2fdb1 --- /dev/null +++ b/config/src/test/resources/configs/function-test/defaultvalues.xml @@ -0,0 +1,62 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<config name="function-test"> + <bool_val>false</bool_val> + <int_val>5</int_val> + <long_val>1234567890123</long_val> + <double_val>41.23</double_val> + <string_val>foo</string_val> + <enum_val>FOOBAR</enum_val> + <refval>:parent:</refval> + <fileVal>vespa.log</fileVal> + + <boolarr index="0">false</boolarr> + <doublearr index="0">2344</doublearr> + <doublearr index="1">123</doublearr> + <stringarr index="0">bar</stringarr> + <enumarr index="0">VALUES</enumarr> + + <basicStruct> + <bar>3</bar> + <intArr index="0">10</intArr> + </basicStruct> + + <rootStruct> + <inner0> + <index>11</index> + </inner0> + <inner1> + <index>12</index> + </inner1> + <innerArr index="0"> + <stringVal>deep</stringVal> + </innerArr> + </rootStruct> + + <myarray index="0"> + <stringval index="0">baah</stringval> + <stringval index="1">yikes</stringval> + <refval>:parent:</refval> + <fileVal>command.com</fileVal> + <anotherarray index="0"> + <foo>7</foo> + </anotherarray> + <myStruct> + <a>1</a> + </myStruct> + </myarray> + <myarray index="1"> + <refval>:parent:</refval> + <fileVal>display.sys</fileVal> + <anotherarray index="0"> + <foo>1</foo> + </anotherarray> + <anotherarray index="1"> + <foo>2</foo> + </anotherarray> + <myStruct> + <a>-1</a> + </myStruct> + </myarray> + +</config> diff --git a/config/src/test/resources/configs/function-test/missingvalue.txt b/config/src/test/resources/configs/function-test/missingvalue.txt new file mode 100644 index 00000000000..9616ac3d27b --- /dev/null +++ b/config/src/test/resources/configs/function-test/missingvalue.txt @@ -0,0 +1,39 @@ +bool_val false +long_val 123 +double_val 41.23 +string_val "foo" +enum_val FOOBAR +refval ":parent:" +fileVal "msdos.sys" +boolarr[1] +boolarr[0] false +intarr[0] +longarr[0] +doublearr[2] +doublearr[0] 2344 +doublearr[1] 123 +stringarr[1] +stringarr[0] "bar" +enumarr[1] +enumarr[0] VALUES +refarr[0] +basicStruct.bar 3 +rootStruct.inner0.index 11 +rootStruct.inner1.index 12 +rootStruct.innerArr[0] +myarray[2] +myarray[0].stringval[2] +myarray[0].stringval[0] "baah" +myarray[0].stringval[1] "yikes" +myarray[0].refval ":parent:" +myarray[0].fileVal "command.com" +myarray[0].anotherarray[1] +myarray[0].anotherarray[0].foo 7 +myarray[0].myStruct.a 1 +myarray[1].stringval[0] +myarray[1].refval ":parent:" +myarray[1].fileVal "display.sys" +myarray[1].anotherarray[2] +myarray[1].anotherarray[0].foo 1 +myarray[1].anotherarray[1].foo 2 +myarray[1].myStruct.a 1 diff --git a/config/src/test/resources/configs/function-test/randomorder.txt b/config/src/test/resources/configs/function-test/randomorder.txt new file mode 100644 index 00000000000..bd91c24f3fb --- /dev/null +++ b/config/src/test/resources/configs/function-test/randomorder.txt @@ -0,0 +1,56 @@ +boolarr[1] +boolarr[0] false +int_with_def -14 +double_val 41.23 +double_with_def -12 +enumwithdef BAR2 +refval ":parent:" +refwithdef ":parent:" +intarr[0] +basicStruct.intArr[1] +basicStruct.intArr[0] 10 +doublearr[2] +doublearr[0] 2344 +doublearr[1] 123 +string_val "foo" +stringwithdef "bar and foo" +enum_val FOOBAR +stringarr[1] +stringarr[0] "bar" +basicStruct.bar 3 +long_with_def -333000333000 +enumarr[1] +enumarr[0] VALUES +refarr[0] +rootStruct.innerArr[1] +rootStruct.innerArr[0].stringVal "deep" +fileVal "autoexec.bat" +myarray[2] +myarray[0].intval -5 +myarray[0].myStruct.a 1 +myarray[0].enumval INNER +myarray[0].refval ":parent:" +myarray[0].anotherarray[1] +myarray[0].anotherarray[0].foo 7 +myarray[0].stringval[2] +myarray[0].stringval[0] "baah" +myarray[0].stringval[1] "yikes" +myarray[0].fileVal "file0" +myarray[1].stringval[0] +myarray[1].enumval INNER +myarray[1].anotherarray[2] +myarray[1].anotherarray[0].foo 1 +myarray[1].anotherarray[1].foo 2 +myarray[1].myStruct.a -1 +myarray[1].refval ":parent:" +myarray[1].intval 5 +myarray[1].fileVal "file1" +bool_val false +bool_with_def true +longarr[0] +rootStruct.inner1.index 12 +int_val 5 +rootStruct.inner0.index 11 +long_val 666000666000 +fileArr[0] +pathVal "pom.xml"
\ No newline at end of file diff --git a/config/src/test/resources/configs/function-test/variableaccess.txt b/config/src/test/resources/configs/function-test/variableaccess.txt new file mode 100644 index 00000000000..997de21750d --- /dev/null +++ b/config/src/test/resources/configs/function-test/variableaccess.txt @@ -0,0 +1,85 @@ +pathMap{"one"} "pom.xml" +bool_val false +bool_with_def true +int_val 5 +int_with_def -14 +long_val 12345678901 +long_with_def -9876543210 +double_val 41.23 +double_with_def -12 +string_val "foo" +stringwithdef "bar and foo" +enum_val FOOBAR +enumwithdef BAR2 +refval :parent: +refwithdef ":parent:" +fileVal "etc" +pathVal "pom.xml" +boolarr[1] +boolarr[0] false +intarr[0] +longarr[2] +longarr[0] 9223372036854775807 +longarr[1] -9223372036854775808 +doublearr[2] +doublearr[0] 2344 +doublearr[1] 123 +stringarr[1] +stringarr[0] "bar" +enumarr[1] +enumarr[0] VALUES +refarr[3] +refarr[0] ":parent:" +refarr[1] ":parent" +refarr[2] "parent:" +fileArr[1] +fileArr[0] "bin" +pathArr[1] +pathArr[0] "pom.xml" + +intMap{"one"} 1 +intMap{"two"} 2 +stringMap{"one"} "first" + +basicStruct.foo "basicFoo" +basicStruct.bar 3 +basicStruct.intArr[2] +basicStruct.intArr[0] 310 +basicStruct.intArr[1] 311 +rootStruct.inner0.index 11 +rootStruct.inner1.index 12 +rootStruct.innerArr[2] +rootStruct.innerArr[0].boolVal true +rootStruct.innerArr[0].stringVal "deep" +rootStruct.innerArr[1].boolVal false +rootStruct.innerArr[1].stringVal "blue a=\"escaped\"" + +myarray[2] +myarray[0].intval -5 +myarray[0].stringval[2] +myarray[0].stringval[0] "baah" +myarray[0].stringval[1] "yikes" +myarray[0].enumval INNER +myarray[0].refval :parent: +myarray[0].fileVal "file0" +myarray[0].anotherarray[1] +myarray[0].anotherarray[0].foo 7 +myarray[0].myStruct.a 1 +myarray[0].myStruct.b 2 +myarray[1].intval 5 +myarray[1].stringval[0] +myarray[1].enumval INNER +myarray[1].refval ":parent:" +myarray[1].fileVal "file1" +myarray[1].anotherarray[2] +myarray[1].anotherarray[0].foo 1 +myarray[1].anotherarray[1].foo 2 +myarray[1].myStruct.a -1 +myarray[1].myStruct.b -2 + +myStructMap{"one"}.myInt 1 +myStructMap{"one"}.myString "bull" +myStructMap{"one"}.myIntDef 2 +myStructMap{"one"}.myStringDef "bear" +myStructMap{"one"}.anotherMap{"anotherOne"}.anInt 3 +myStructMap{"one"}.anotherMap{"anotherOne"}.anIntDef 4 diff --git a/config/src/test/resources/configs/illegal/app.cfg b/config/src/test/resources/configs/illegal/app.cfg new file mode 100644 index 00000000000..d22d3d46c90 --- /dev/null +++ b/config/src/test/resources/configs/illegal/app.cfg @@ -0,0 +1,6 @@ +message "msg1" +times thisisnotanint +a[3] +a[0].name "a0" +a[1].name "a1" +a[2].name "a2" diff --git a/config/src/test/resources/configs/unicode/unicode.cfg b/config/src/test/resources/configs/unicode/unicode.cfg new file mode 100644 index 00000000000..d2c94abea86 --- /dev/null +++ b/config/src/test/resources/configs/unicode/unicode.cfg @@ -0,0 +1,2 @@ +unicodestring1 "Hei æøå 바둑 ÆØÅ hallo" +unicodestring2 "abc æøå 囲碁 ÆØÅ ABC" diff --git a/config/src/testlist.txt b/config/src/testlist.txt new file mode 100644 index 00000000000..b002769217e --- /dev/null +++ b/config/src/testlist.txt @@ -0,0 +1,25 @@ +tests/frtconnectionpool +tests/raw_subscription +tests/file_subscription +tests/subscription +tests/getconfig +tests/configagent +tests/misc +tests/configholder +tests/configfetcher +tests/configmanager +tests/subscriber +tests/functiontest +tests/legacysubscriber +tests/print +tests/configformat +tests/configgen +tests/configparser +tests/configretriever +tests/frt +tests/unittest +tests/api +tests/failover +tests/configuri +tests/trace +tests/payload_converter diff --git a/config/src/testrun/.gitignore b/config/src/testrun/.gitignore new file mode 100644 index 00000000000..faed45bc94a --- /dev/null +++ b/config/src/testrun/.gitignore @@ -0,0 +1,10 @@ +test-report.html +test-report.html.* +test.*.*.desc +test.*.*.file.* +test.*.*.files.html +test.*.*.log +tmp.* +xsync.log +/test.*.*.result +Makefile diff --git a/config/src/tests/.gitignore b/config/src/tests/.gitignore new file mode 100644 index 00000000000..a3e9c375723 --- /dev/null +++ b/config/src/tests/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +*_test diff --git a/config/src/tests/api/.gitignore b/config/src/tests/api/.gitignore new file mode 100644 index 00000000000..67666db8b50 --- /dev/null +++ b/config/src/tests/api/.gitignore @@ -0,0 +1,3 @@ +/config-my.cpp +/config-my.h +config_api_test_app diff --git a/config/src/tests/api/CMakeLists.txt b/config/src/tests/api/CMakeLists.txt new file mode 100644 index 00000000000..93d12a5dc66 --- /dev/null +++ b/config/src/tests/api/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_api_test_app + SOURCES + api.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_api_test_app COMMAND config_api_test_app) +vespa_generate_config(config_api_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/api/api.cpp b/config/src/tests/api/api.cpp new file mode 100644 index 00000000000..54042fcef1f --- /dev/null +++ b/config/src/tests/api/api.cpp @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/config.h> +#include <config-my.h> + +using namespace config; + +TEST("require that can subscribe with empty config id") { + ConfigSet set; + ConfigContext::SP ctx(new ConfigContext(set)); + MyConfigBuilder builder; + builder.myField = "myfoo"; + set.addBuilder("", &builder); + ConfigSubscriber subscriber(ctx); + ConfigHandle<MyConfig>::UP handle = subscriber.subscribe<MyConfig>(""); + ASSERT_TRUE(subscriber.nextConfig(0)); + std::unique_ptr<MyConfig> cfg(handle->getConfig()); + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL("myfoo", cfg->myField); +} + +/* + * TODO: Convert to frt test. +TEST_MT_FFF("require that source may be unable to serve config temporarily", 2, ConfigContext::SP(new ConfigContext()), + ConfigSet(), + MyConfigBuilder()) { + if (thread_id == 0) { + ConfigSubscriber subscriber(f1, f2); + ConfigHandle<MyConfig>::UP handle = subscriber.subscribe<MyConfig>("myid", 10000); + ASSERT_TRUE(subscriber.nextConfig(10000)); + std::unique_ptr<MyConfig> cfg(handle->getConfig()); + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL("myfoo", cfg->myField); + } else { + FastOS_Thread::Sleep(1000); + f3.myField = "myfoo"; + f2.addBuilder("myid", &f3); + f1->reload(); + + } +} +*/ + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configagent/.gitignore b/config/src/tests/configagent/.gitignore new file mode 100644 index 00000000000..59a4f806257 --- /dev/null +++ b/config/src/tests/configagent/.gitignore @@ -0,0 +1,3 @@ +/config-my.cpp +/config-my.h +config_configagent_test_app diff --git a/config/src/tests/configagent/CMakeLists.txt b/config/src/tests/configagent/CMakeLists.txt new file mode 100644 index 00000000000..715e5260e30 --- /dev/null +++ b/config/src/tests/configagent/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_configagent_test_app + SOURCES + configagent.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_configagent_test_app COMMAND config_configagent_test_app) +vespa_generate_config(config_configagent_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/configagent/configagent.cpp b/config/src/tests/configagent/configagent.cpp new file mode 100644 index 00000000000..73394d443a4 --- /dev/null +++ b/config/src/tests/configagent/configagent.cpp @@ -0,0 +1,294 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/config.h> +#include <vespa/config/raw/rawsource.h> +#include <vespa/config/common/misc.h> +#include <vespa/config/common/configrequest.h> +#include <vespa/config/common/timingvalues.h> +#include <vespa/config/common/trace.h> +#include <vespa/config/frt/frtconfigagent.h> +#include <config-my.h> + +using namespace config; + +class MyConfigRequest : public ConfigRequest +{ +public: + MyConfigRequest(const ConfigKey & key) + : _key(key) + { } + + const ConfigKey & getKey() const + { + return _key; + } + + bool abort() + { + return false; + } + + bool isAborted() const + { + return false; + } + + void setError(int errorCode) + { + (void) errorCode; + } + const ConfigKey _key; +}; + +class MyConfigResponse : public ConfigResponse +{ +public: + MyConfigResponse(const ConfigKey & key, const ConfigValue & value, bool isUpdated, bool valid, + int64_t timestamp, const vespalib::string & md5, const std::string & errorMsg, int errorC0de, bool iserror) + : _key(key), + _value(value), + _isUpdated(isUpdated), + _fillCalled(false), + _valid(valid), + _state(md5, timestamp), + _errorMessage(errorMsg), + _errorCode(errorC0de), + _isError(iserror) + { } + + const ConfigKey& getKey() const + { + return _key; + } + + const ConfigValue & getValue() const + { + return _value; + } + + const ConfigState & getConfigState() const + { + return _state; + } + + bool hasValidResponse() const + { + return _valid; + } + + bool validateResponse() + { + return _valid; + } + + void fill() + { + _fillCalled = true; + } + + vespalib::string errorMessage() const + { + return _errorMessage; + } + + int errorCode() const + { + return _errorCode; + } + + bool isError() const + { + return _isError; + } + + const Trace & getTrace() const { return _trace; } + + const ConfigKey _key; + const ConfigValue _value; + bool _isUpdated; + bool _fillCalled; + bool _valid; + const ConfigState _state; + vespalib::string _errorMessage; + int _errorCode; + bool _isError; + Trace _trace; + + +/** + MyConfigResponse(const ConfigKey & key, const ConfigValue & value, bool isUpdated, bool valid, + int64_t timestamp, const vespalib::string & md5, int64_t prevTimestamp, const vespalib::string &prevMd5, + const std::string & errorMsg, int errorC0de, bool iserror) +*/ + static ConfigResponse::UP createOKResponse(const ConfigKey & key, const ConfigValue & value) + { + return ConfigResponse::UP(new MyConfigResponse(key, value, true, true, 10, "a", "", 0, false)); + } + + static ConfigResponse::UP createServerErrorResponse(const ConfigKey & key, const ConfigValue & value) + { + return ConfigResponse::UP(new MyConfigResponse(key, value, false, true, 10, "a", "whinewhine", 2, true)); + } + + static ConfigResponse::UP createConfigErrorResponse(const ConfigKey & key, const ConfigValue & value) + { + return ConfigResponse::UP(new MyConfigResponse(key, value, false, false, 10, "a", "", 0, false)); + } +}; + +class MyHolder : public IConfigHolder +{ +public: + MyHolder() + : _update() + { + } + + std::unique_ptr<ConfigUpdate> provide() + { + return std::move(_update); + } + + bool wait(uint64_t timeout) + { + (void) timeout; + return true; + } + + void handle(std::unique_ptr<ConfigUpdate> update) + { + _update = std::move(update); + } + + bool poll() { return true; } + void interrupt() { } +private: + std::unique_ptr<ConfigUpdate> _update; +}; + + +ConfigValue createValue(const std::string & myField, const std::string & md5) +{ + std::vector< vespalib::string > lines; + lines.push_back("myField \"" + myField + "\""); + return ConfigValue(lines, md5); +} + +static TimingValues testTimingValues( + 2000, // successTimeout + 500, // errorTimeout + 500, // initialTimeout + 4000, // subscribeTimeout + 0, // fixedDelay + 250, // successDelay + 250, // unconfiguredDelay + 500, // configuredErrorDelay + 5, + 1000, + 2000); // maxDelayMultiplier + +TEST("require that agent returns correct values") { + FRTConfigAgent handler(IConfigHolder::SP(new MyHolder()), testTimingValues); + ASSERT_EQUAL(500u, handler.getTimeout()); + ASSERT_EQUAL(0u, handler.getWaitTime()); + ConfigState cs; + ASSERT_EQUAL(cs.md5, handler.getConfigState().md5); + ASSERT_EQUAL(cs.generation, handler.getConfigState().generation); +} + +TEST("require that successful request is delivered to holder") { + const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); + const ConfigValue testValue(createValue("l33t", "a")); + IConfigHolder::SP latch(new MyHolder()); + + FRTConfigAgent handler(latch, testTimingValues); + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createOKResponse(testKey, testValue)); + ASSERT_TRUE(latch->poll()); + ConfigUpdate::UP update(latch->provide()); + ASSERT_TRUE(update.get() != NULL); + ASSERT_TRUE(update->hasChanged()); + MyConfig cfg(update->getValue()); + ASSERT_EQUAL("l33t", cfg.myField); +} + +TEST("require that successful request sets correct wait time") { + const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); + const ConfigValue testValue(createValue("l33t", "a")); + IConfigHolder::SP latch(new MyHolder()); + FRTConfigAgent handler(latch, testTimingValues); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createOKResponse(testKey, testValue)); + ASSERT_EQUAL(250u, handler.getWaitTime()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createOKResponse(testKey, testValue)); + ASSERT_EQUAL(250u, handler.getWaitTime()); +} + +TEST("require that bad config response returns false") { + const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); + const ConfigValue testValue(createValue("myval", "a")); + IConfigHolder::SP latch(new MyHolder()); + FRTConfigAgent handler(latch, testTimingValues); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createConfigErrorResponse(testKey, testValue)); + ASSERT_EQUAL(250u, handler.getWaitTime()); + ASSERT_EQUAL(500u, handler.getTimeout()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createConfigErrorResponse(testKey, testValue)); + ASSERT_EQUAL(500u, handler.getWaitTime()); + ASSERT_EQUAL(500u, handler.getTimeout()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createConfigErrorResponse(testKey, testValue)); + ASSERT_EQUAL(750u, handler.getWaitTime()); + ASSERT_EQUAL(500u, handler.getTimeout()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createConfigErrorResponse(testKey, testValue)); + ASSERT_EQUAL(1000u, handler.getWaitTime()); + ASSERT_EQUAL(500u, handler.getTimeout()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createConfigErrorResponse(testKey, testValue)); + ASSERT_EQUAL(1250u, handler.getWaitTime()); + ASSERT_EQUAL(500u, handler.getTimeout()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createConfigErrorResponse(testKey, testValue)); + ASSERT_EQUAL(1250u, handler.getWaitTime()); + ASSERT_EQUAL(500u, handler.getTimeout()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createOKResponse(testKey, testValue)); + ASSERT_EQUAL(250u, handler.getWaitTime()); + ASSERT_EQUAL(2000u, handler.getTimeout()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createConfigErrorResponse(testKey, testValue)); + ASSERT_EQUAL(500u, handler.getWaitTime()); + ASSERT_EQUAL(500u, handler.getTimeout()); +} + +TEST("require that bad response returns false") { + const ConfigKey testKey(ConfigKey::create<MyConfig>("mykey")); + std::vector<vespalib::string> lines; + const ConfigValue testValue(lines, "a"); + + IConfigHolder::SP latch(new MyHolder()); + FRTConfigAgent handler(latch, testTimingValues); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createServerErrorResponse(testKey, testValue)); + ASSERT_EQUAL(250u, handler.getWaitTime()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createServerErrorResponse(testKey, testValue)); + ASSERT_EQUAL(500u, handler.getWaitTime()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createServerErrorResponse(testKey, testValue)); + ASSERT_EQUAL(750u, handler.getWaitTime()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createServerErrorResponse(testKey, testValue)); + ASSERT_EQUAL(1000u, handler.getWaitTime()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createServerErrorResponse(testKey, testValue)); + ASSERT_EQUAL(1250u, handler.getWaitTime()); + + handler.handleResponse(MyConfigRequest(testKey), MyConfigResponse::createServerErrorResponse(testKey, testValue)); + ASSERT_EQUAL(1250u, handler.getWaitTime()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configeventqueue/.gitignore b/config/src/tests/configeventqueue/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/configeventqueue/.gitignore diff --git a/config/src/tests/configfetcher/.gitignore b/config/src/tests/configfetcher/.gitignore new file mode 100644 index 00000000000..459ea6bbcfa --- /dev/null +++ b/config/src/tests/configfetcher/.gitignore @@ -0,0 +1,3 @@ +/config-my.cpp +/config-my.h +config_configfetcher_test_app diff --git a/config/src/tests/configfetcher/CMakeLists.txt b/config/src/tests/configfetcher/CMakeLists.txt new file mode 100644 index 00000000000..58598815f7d --- /dev/null +++ b/config/src/tests/configfetcher/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_configfetcher_test_app + SOURCES + configfetcher.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_configfetcher_test_app COMMAND config_configfetcher_test_app) +vespa_generate_config(config_configfetcher_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/configfetcher/configfetcher.cpp b/config/src/tests/configfetcher/configfetcher.cpp new file mode 100644 index 00000000000..6d319fdff67 --- /dev/null +++ b/config/src/tests/configfetcher/configfetcher.cpp @@ -0,0 +1,158 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/helper/configfetcher.h> +#include <fstream> +#include "config-my.h" +#include <atomic> + +using namespace config; + + +class MyCallback : public IFetcherCallback<MyConfig> +{ +public: + MyCallback(const std::string & badConfig="") : _config(), _configured(false), _badConfig(badConfig) { } + void configure(std::unique_ptr<MyConfig> config) + { + _config = std::move(config); + _configured = true; + if (_config->myField.compare(_badConfig) == 0) { + throw vespalib::Exception("Buhu"); + } + } + std::unique_ptr<MyConfig> _config; + std::atomic<bool> _configured; + std::string _badConfig; +}; + +TEST("requireThatConfigIsAvailableOnConstruction") { + RawSpec spec("myField \"foo\"\n"); + MyCallback cb; + + { + ConfigFetcher fetcher(spec); + fetcher.subscribe<MyConfig>("myid", &cb); + fetcher.start(); + ASSERT_TRUE(cb._config.get() != NULL); + ASSERT_EQUAL("my", cb._config->defName()); + ASSERT_EQUAL("foo", cb._config->myField); + } +} + +#if 0 +TEST("requireThatConfigUpdatesArePerformed") { + writeFile("test1.cfg", "foo"); + FileSpec spec("test1.cfg"); + MyCallback cb; + cb._configured = false; + vespalib::ThreadStackExecutor executor(1, 128 * 1024); + + { + ConfigFetcher fetcher(500); + fetcher.subscribe<MyConfig>("test1", &cb, spec); + fetcher.start(); + ASSERT_TRUE(cb._configured); + ASSERT_TRUE(cb._config.get() != NULL); + ASSERT_EQUAL("my", cb._config->defName()); + ASSERT_EQUAL("foo", cb._config->myField); + + sleep(2); + writeFile("test1.cfg", "bar"); + + cb._configured = false; + FastOS_Time timer; + timer.SetNow(); + while (!cb._configured && timer.MilliSecsToNow() < 20000.0) { + if (cb._configured) + break; + FastOS_Thread::Sleep(1000); + } + ASSERT_TRUE(cb._configured); + ASSERT_TRUE(cb._config.get() != NULL); + ASSERT_EQUAL("my", cb._config->defName()); + ASSERT_EQUAL("bar", cb._config->myField); + } +} +#endif + +TEST("requireThatFetcherCanHandleMultipleConfigs") { + MyConfigBuilder b1, b2; + b1.myField = "foo"; + b2.myField = "bar"; + ConfigSet set; + set.addBuilder("test1", &b1); + set.addBuilder("test2", &b2); + MyCallback cb1; + MyCallback cb2; + + { + ConfigFetcher fetcher(set); + fetcher.subscribe<MyConfig>("test1", &cb1); + fetcher.subscribe<MyConfig>("test2", &cb2); + fetcher.start(); + + ASSERT_TRUE(cb1._configured); + ASSERT_TRUE(cb2._configured); + ASSERT_TRUE(cb1._config.get() != NULL); + ASSERT_TRUE(cb2._config.get() != NULL); + ASSERT_EQUAL("my", cb1._config->defName()); + ASSERT_EQUAL("foo", cb1._config->myField); + ASSERT_EQUAL("my", cb2._config->defName()); + ASSERT_EQUAL("bar", cb2._config->myField); + } +} + +TEST("verify that exceptions in callback is thrown on initial subscribe") { + MyConfigBuilder b1; + b1.myField = "foo"; + ConfigSet set; + set.addBuilder("test1", &b1); + MyCallback cb("foo"); + { + ConfigFetcher fetcher(set); + fetcher.subscribe<MyConfig>("test1", &cb); + ASSERT_EXCEPTION(fetcher.start(), vespalib::Exception, "Buhu"); + } +} + +namespace { + +struct ConfigFixture { + MyConfigBuilder builder; + ConfigSet set; + ConfigContext::SP context; + ConfigFixture() : builder(), set(), context() { + set.addBuilder("cfgid", &builder); + context.reset(new ConfigContext(set)); + } +}; + +} // namespace <unnamed> + +TEST_F("verify that config generation can be obtained from config fetcher", ConfigFixture) { + f1.builder.myField = "foo"; + MyCallback cb; + { + ConfigFetcher fetcher(f1.context); + fetcher.subscribe<MyConfig>("cfgid", &cb); + fetcher.start(); + EXPECT_EQUAL("foo", cb._config.get()->myField); + EXPECT_EQUAL(1, fetcher.getGeneration()); + f1.builder.myField = "bar"; + cb._configured = false; + f1.context->reload(); + FastOS_Time timer; + timer.SetNow(); + while (timer.MilliSecsToNow() < 120000) { + if (cb._configured) { + break; + } + FastOS_Thread::Sleep(10); + } + EXPECT_EQUAL(2, fetcher.getGeneration()); + EXPECT_EQUAL("bar", cb._config.get()->myField); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configformat/.gitignore b/config/src/tests/configformat/.gitignore new file mode 100644 index 00000000000..fe015963e1c --- /dev/null +++ b/config/src/tests/configformat/.gitignore @@ -0,0 +1,3 @@ +/config-my.cpp +/config-my.h +config_configformat_test_app diff --git a/config/src/tests/configformat/CMakeLists.txt b/config/src/tests/configformat/CMakeLists.txt new file mode 100644 index 00000000000..f17215bc66c --- /dev/null +++ b/config/src/tests/configformat/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_configformat_test_app + SOURCES + configformat.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_configformat_test_app COMMAND config_configformat_test_app) +vespa_generate_config(config_configformat_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/configformat/configformat.cpp b/config/src/tests/configformat/configformat.cpp new file mode 100644 index 00000000000..5ed10b81023 --- /dev/null +++ b/config/src/tests/configformat/configformat.cpp @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/config.h> +#include <vespa/config/print/fileconfigformatter.h> +#include <vespa/vespalib/data/slime/slime.h> + +using namespace config; +using namespace vespalib::slime::convenience; + +TEST("requireThatConfigIsFormatted") { + ConfigDataBuffer buffer; + vespalib::Slime & slime(buffer.slimeObject()); + Cursor &c = slime.setObject().setObject("configPayload").setObject("myField"); + c.setString("type", "string"); + c.setString("value", "foo"); + + FileConfigFormatter formatter; + formatter.encode(buffer); + EXPECT_EQUAL(std::string("myField \"foo\"\n"), buffer.getEncodedString()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configgen/.gitignore b/config/src/tests/configgen/.gitignore new file mode 100644 index 00000000000..e021b74ec3f --- /dev/null +++ b/config/src/tests/configgen/.gitignore @@ -0,0 +1,6 @@ +/config-motd.cpp +/config-motd.h +config_configgen_test_app +config_map_inserter_test_app +config_value_converter_test_app +config_vector_inserter_test_app diff --git a/config/src/tests/configgen/CMakeLists.txt b/config/src/tests/configgen/CMakeLists.txt new file mode 100644 index 00000000000..8a71b737919 --- /dev/null +++ b/config/src/tests/configgen/CMakeLists.txt @@ -0,0 +1,31 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_configgen_test_app + SOURCES + configgen.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_configgen_test_app COMMAND config_configgen_test_app) +vespa_generate_config(config_configgen_test_app ../../test/resources/configdefinitions/motd.def) +vespa_add_executable(config_vector_inserter_test_app + SOURCES + vector_inserter.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_vector_inserter_test_app COMMAND config_vector_inserter_test_app) +vespa_add_executable(config_map_inserter_test_app + SOURCES + map_inserter.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_map_inserter_test_app COMMAND config_map_inserter_test_app) +vespa_add_executable(config_value_converter_test_app + SOURCES + value_converter.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_value_converter_test_app COMMAND config_value_converter_test_app) +vespa_generate_config(config_value_converter_test_app ../../test/resources/configdefinitions/motd.def) diff --git a/config/src/tests/configgen/configgen.cpp b/config/src/tests/configgen/configgen.cpp new file mode 100644 index 00000000000..786268e7287 --- /dev/null +++ b/config/src/tests/configgen/configgen.cpp @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("configgen"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/config.h> +#include "config-motd.h" + +using namespace config; + +TEST("require that config type can be compiled") { + std::unique_ptr<MotdConfig> cfg = ConfigGetter<MotdConfig>::getConfig("motd", FileSpec("motd.cfg")); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configgen/map_inserter.cpp b/config/src/tests/configgen/map_inserter.cpp new file mode 100644 index 00000000000..5f37a4fdb4f --- /dev/null +++ b/config/src/tests/configgen/map_inserter.cpp @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("map_inserter"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/configgen/map_inserter.h> + +using namespace config; +using namespace config::internal; +using namespace vespalib; +using namespace vespalib::slime; + +struct MyType{ + MyType() : foo(0), bar(0) {} + MyType(const ConfigPayload & payload) + { + foo = payload.get()["foo"].asLong(); + bar = payload.get()["bar"].asLong(); + } + int foo; + int bar; +}; + +TEST("require that map of ints can be inserted") { + std::map<vespalib::string, int32_t> map; + Slime slime; + Cursor & root = slime.setObject(); + root.setLong("foo", 3); + root.setLong("bar", 2); + root.setLong("baz", 6); + MapInserter<int32_t> inserter(map); + root.traverse(inserter); + ASSERT_EQUAL(3u, map.size()); + ASSERT_EQUAL(3, map["foo"]); + ASSERT_EQUAL(2, map["bar"]); + ASSERT_EQUAL(6, map["baz"]); +} + +TEST("require that map of struct can be inserted") { + std::map<vespalib::string, MyType> map; + Slime slime; + Cursor & root = slime.setObject(); + Cursor & one = root.setObject("foo"); + one.setLong("foo", 3); + one.setLong("bar", 4); + Cursor & two = root.setObject("bar"); + two.setLong("foo", 1); + two.setLong("bar", 6); + MapInserter<MyType> inserter(map); + root.traverse(inserter); + ASSERT_EQUAL(2u, map.size()); + ASSERT_EQUAL(3, map["foo"].foo); + ASSERT_EQUAL(4, map["foo"].bar); + ASSERT_EQUAL(1, map["bar"].foo); + ASSERT_EQUAL(6, map["bar"].bar); +} + +TEST("require that map of long can be inserted") { + std::map<vespalib::string, int64_t> map; + Slime slime; + Cursor & root = slime.setObject(); + root.setLong("foo", 3); + root.setLong("bar", 2); + root.setLong("baz", 6); + MapInserter<int64_t> inserter(map); + root.traverse(inserter); + ASSERT_EQUAL(3u, map.size()); + ASSERT_EQUAL(3, map["foo"]); + ASSERT_EQUAL(2, map["bar"]); + ASSERT_EQUAL(6, map["baz"]); +} + +TEST("require that map of double can be inserted") { + std::map<vespalib::string, double> map; + Slime slime; + Cursor & root = slime.setObject(); + root.setDouble("foo", 3.1); + root.setDouble("bar", 2.4); + root.setDouble("baz", 6.6); + MapInserter<double> inserter(map); + root.traverse(inserter); + ASSERT_EQUAL(3u, map.size()); + ASSERT_EQUAL(3.1, map["foo"]); + ASSERT_EQUAL(2.4, map["bar"]); + ASSERT_EQUAL(6.6, map["baz"]); +} + +TEST("require that map of bool can be inserted") { + std::map<vespalib::string, bool> map; + Slime slime; + Cursor & root = slime.setObject(); + root.setBool("foo", true); + root.setBool("bar", false); + root.setBool("baz", true); + MapInserter<bool> inserter(map); + root.traverse(inserter); + ASSERT_EQUAL(3u, map.size()); + ASSERT_TRUE(map["foo"]); + ASSERT_FALSE(map["bar"]); + ASSERT_TRUE(map["baz"]); +} + +TEST("require that map of string can be inserted") { + std::map<vespalib::string, vespalib::string> map; + Slime slime; + Cursor & root = slime.setObject(); + root.setString("foo", "baz"); + root.setString("bar", "bar"); + root.setString("baz", "foo"); + MapInserter<vespalib::string> inserter(map); + root.traverse(inserter); + ASSERT_EQUAL(3u, map.size()); + ASSERT_EQUAL("foo", map["baz"]); + ASSERT_EQUAL("bar", map["bar"]); + ASSERT_EQUAL("baz", map["foo"]); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configgen/motd.cfg b/config/src/tests/configgen/motd.cfg new file mode 100644 index 00000000000..3b511ce35b2 --- /dev/null +++ b/config/src/tests/configgen/motd.cfg @@ -0,0 +1,22 @@ +intVal 1 +longVal 1 +doubleVal 2.3 +stringVal "foo" +stringnulldef "foo" +enumVal FOOBAR +refVal "refVal" +fileVal "fileVal" +boolVal true +boolarr[2] +boolarr[0] true +boolarr[1] false +myArray[2] +myArray[0].stringVal[1] +myArray[0].stringVal[0] "bla" +myArray[0].refVal "habba" +myArray[1].stringVal[1] +myArray[1].stringVal[0] "blabla" +myArray[1].refVal "nabba" +myArray[1].anotherArray[1] +myArray[1].anotherArray[0].foo 1337 + diff --git a/config/src/tests/configgen/value_converter.cpp b/config/src/tests/configgen/value_converter.cpp new file mode 100644 index 00000000000..467075354f2 --- /dev/null +++ b/config/src/tests/configgen/value_converter.cpp @@ -0,0 +1,105 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("value_converter"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/configgen/value_converter.h> +#include <vespa/config/common/exceptions.h> + +using namespace config; +using namespace config::internal; +using namespace vespalib; +using namespace vespalib::slime; + +struct MyType{ + MyType(const ConfigPayload & payload) + { + foo = payload.get()["foo"].asLong(); + bar = payload.get()["bar"].asLong(); + } + int foo; + int bar; +}; + +TEST("that int32_ts are converted") { + Slime slime; + Cursor & root = slime.setArray(); + root.addLong(3); + root.addLong(-2); + root.addLong(INT_MAX); + root.addLong(INT_MIN); + root.addDouble(3.14); + ValueConverter<int32_t> conv; + EXPECT_EQUAL(3, conv(root[0])); + EXPECT_EQUAL(-2, conv(root[1])); + EXPECT_EQUAL(INT_MAX, conv(root[2])); + EXPECT_EQUAL(INT_MIN, conv(root[3])); + EXPECT_EQUAL(3, conv(root[4])); +} + +TEST("that int64_ts are converted") { + Slime slime; + Cursor & root = slime.setArray(); + root.addLong(3); + root.addLong(-2); + root.addLong(LONG_MAX); + root.addLong(LONG_MIN); + root.addLong(std::numeric_limits<int64_t>::max()); + root.addLong(std::numeric_limits<int64_t>::min()); + root.addDouble(3.14); + std::string ref = "{\"val\":9223372036854775807}"; + Slime slime2; + JsonFormat::decode(ref, slime2); + EXPECT_EQUAL(std::numeric_limits<int64_t>::max(), slime2.get()["val"].asLong()); + ValueConverter<int64_t> conv; + EXPECT_EQUAL(3, conv(root[0])); + EXPECT_EQUAL(-2, conv(root[1])); + EXPECT_EQUAL(LONG_MAX, conv(root[2])); + EXPECT_EQUAL(LONG_MIN, conv(root[3])); + EXPECT_EQUAL(std::numeric_limits<int64_t>::max(), conv(root[4])); + EXPECT_EQUAL(std::numeric_limits<int64_t>::min(), conv(root[5])); + EXPECT_EQUAL(3, conv(root[6])); +} + +TEST("that values can be parsed as strings") { + Slime slime; + Cursor & root = slime.setObject(); + root.setString("intval", "1234"); + root.setString("longval", "42949672969"); + root.setString("boolval", "true"); + root.setString("doubleval", "3.14"); + ValueConverter<int32_t> intConv; + ValueConverter<int64_t> longConv; + ValueConverter<bool> boolConv; + ValueConverter<double> doubleConv; + EXPECT_EQUAL(1234, intConv(root["intval"])); + EXPECT_EQUAL(42949672969, longConv(root["longval"])); + EXPECT_EQUAL(true, boolConv(root["boolval"])); + EXPECT_APPROX(3.14, doubleConv(root["doubleval"]), 0.0001); +} + +TEST("that incompatible types throws exceptions") { + Slime slime; + Cursor & root = slime.setObject(); + root.setBool("intval", true); + root.setBool("longval", true); + root.setBool("doubleval", true); + root.setLong("boolval", 3); + ValueConverter<int32_t> intConv; + ValueConverter<int64_t> longConv; + ValueConverter<bool> boolConv; + ValueConverter<double> doubleConv; + EXPECT_EXCEPTION(intConv(root["intval"]), InvalidConfigException, ""); + EXPECT_EXCEPTION(longConv(root["longval"]), InvalidConfigException, ""); + EXPECT_EXCEPTION(doubleConv(root["doubleval"]), InvalidConfigException, ""); + EXPECT_EXCEPTION(boolConv(root["boolval"]), InvalidConfigException, ""); +} + +TEST("that non-valid fields throws exception") { + Slime slime; + Cursor & root = slime.setObject(); + ValueConverter<int64_t> conv; + EXPECT_EXCEPTION(conv("longval", root["longval"]), InvalidConfigException, "Value for 'longval' required but not found"); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configgen/vector_inserter.cpp b/config/src/tests/configgen/vector_inserter.cpp new file mode 100644 index 00000000000..e9f16f804b3 --- /dev/null +++ b/config/src/tests/configgen/vector_inserter.cpp @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("vector_inserter"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/configgen/vector_inserter.h> + +using namespace config; +using namespace config::internal; +using namespace vespalib; +using namespace vespalib::slime; + +struct MyType{ + MyType() : foo(0), bar(0) {} + MyType(const ConfigPayload & payload) + { + foo = payload.get()["foo"].asLong(); + bar = payload.get()["bar"].asLong(); + } + int foo; + int bar; +}; + +TEST("require that vector of ints can be inserted") { + std::vector<int32_t> vector; + Slime slime; + Cursor & root = slime.setArray(); + root.addLong(3); + root.addLong(2); + root.addLong(6); + VectorInserter<int32_t> inserter(vector); + root.traverse(inserter); + ASSERT_EQUAL(3u, vector.size()); + ASSERT_EQUAL(3, vector[0]); + ASSERT_EQUAL(2, vector[1]); + ASSERT_EQUAL(6, vector[2]); +} + +TEST("require that vector of struct can be inserted") { + std::vector<MyType> typeVector; + Slime slime; + Cursor & root = slime.setArray(); + Cursor & one = root.addObject(); + one.setLong("foo", 3); + one.setLong("bar", 4); + Cursor & two = root.addObject(); + two.setLong("foo", 1); + two.setLong("bar", 6); + VectorInserter<MyType> inserter(typeVector); + root.traverse(inserter); + ASSERT_EQUAL(2u, typeVector.size()); + ASSERT_EQUAL(3, typeVector[0].foo); + ASSERT_EQUAL(4, typeVector[0].bar); + ASSERT_EQUAL(1, typeVector[1].foo); + ASSERT_EQUAL(6, typeVector[1].bar); +} + +TEST("require that vector of long can be inserted") { + std::vector<int64_t> vector; + Slime slime; + Cursor & root = slime.setArray(); + root.addLong(3); + root.addLong(2); + root.addLong(6); + VectorInserter<int64_t> inserter(vector); + root.traverse(inserter); + ASSERT_EQUAL(3u, vector.size()); + ASSERT_EQUAL(3, vector[0]); + ASSERT_EQUAL(2, vector[1]); + ASSERT_EQUAL(6, vector[2]); +} + +TEST("require that vector of double can be inserted") { + std::vector<double> vector; + Slime slime; + Cursor & root = slime.setArray(); + root.addDouble(3.1); + root.addDouble(2.4); + root.addDouble(6.6); + VectorInserter<double> inserter(vector); + root.traverse(inserter); + ASSERT_EQUAL(3u, vector.size()); + ASSERT_EQUAL(3.1, vector[0]); + ASSERT_EQUAL(2.4, vector[1]); + ASSERT_EQUAL(6.6, vector[2]); +} + +TEST("require that vector of bool can be inserted") { + std::vector<bool> vector; + Slime slime; + Cursor & root = slime.setArray(); + root.addBool(true); + root.addBool(false); + root.addBool(true); + VectorInserter<bool> inserter(vector); + root.traverse(inserter); + ASSERT_EQUAL(3u, vector.size()); + ASSERT_TRUE(vector[0]); + ASSERT_FALSE(vector[1]); + ASSERT_TRUE(vector[2]); +} + +TEST("require that vector of string can be inserted") { + std::vector<vespalib::string> vector; + Slime slime; + Cursor & root = slime.setArray(); + root.addString("foo"); + root.addString("bar"); + root.addString("baz"); + VectorInserter<vespalib::string> inserter(vector); + root.traverse(inserter); + ASSERT_EQUAL(3u, vector.size()); + ASSERT_EQUAL("foo", vector[0]); + ASSERT_EQUAL("bar", vector[1]); + ASSERT_EQUAL("baz", vector[2]); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configholder/.gitignore b/config/src/tests/configholder/.gitignore new file mode 100644 index 00000000000..1c0e7857e7a --- /dev/null +++ b/config/src/tests/configholder/.gitignore @@ -0,0 +1 @@ +config_configholder_test_app diff --git a/config/src/tests/configholder/CMakeLists.txt b/config/src/tests/configholder/CMakeLists.txt new file mode 100644 index 00000000000..050160d2ad3 --- /dev/null +++ b/config/src/tests/configholder/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_configholder_test_app + SOURCES + configholder.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_configholder_test_app COMMAND config_configholder_test_app) diff --git a/config/src/tests/configholder/configholder.cpp b/config/src/tests/configholder/configholder.cpp new file mode 100644 index 00000000000..b11d2009ebf --- /dev/null +++ b/config/src/tests/configholder/configholder.cpp @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/common/configholder.h> + +using namespace config; + +TEST("Require that element order is correct") +{ + ConfigValue value(std::vector<vespalib::string>(), "foo"); + ConfigValue value2(std::vector<vespalib::string>(), "bar"); + + ConfigHolder holder; + holder.handle(ConfigUpdate::UP(new ConfigUpdate(value, true, 0))); + std::unique_ptr<ConfigUpdate> update = holder.provide(); + ASSERT_TRUE(value == update->getValue()); + + holder.handle(ConfigUpdate::UP(new ConfigUpdate(value, false, 1))); + holder.handle(ConfigUpdate::UP(new ConfigUpdate(value2, false, 2))); + update = holder.provide(); + ASSERT_TRUE(value2 == update->getValue()); +} + +TEST("Require that waiting is done") +{ + ConfigValue value; + + ConfigHolder holder; + FastOS_Time timer; + timer.SetNow(); + holder.wait(1000); + ASSERT_TRUE(timer.MilliSecsToNow() >= 1000); + ASSERT_TRUE(timer.MilliSecsToNow() < 60000); + + timer.SetNow(); + holder.handle(ConfigUpdate::UP(new ConfigUpdate(value, true, 0))); + holder.wait(100); + ASSERT_TRUE(timer.MilliSecsToNow() >= 100); +} + +TEST("Require that polling for elements work") +{ + ConfigValue value; + + ConfigHolder holder; + ASSERT_FALSE(holder.poll()); + holder.handle(ConfigUpdate::UP(new ConfigUpdate(value, true, 0))); + ASSERT_TRUE(holder.poll()); + holder.provide(); + ASSERT_TRUE(holder.poll()); +} + +TEST_MT_F("Require that wait is interrupted", 2, ConfigHolder) +{ + if (thread_id == 0) { + FastOS_Time timer; + timer.SetNow(); + TEST_BARRIER(); + f.wait(1000); + EXPECT_TRUE(timer.MilliSecsToNow() < 60000.0); + EXPECT_TRUE(timer.MilliSecsToNow() > 400.0); + TEST_BARRIER(); + } else { + TEST_BARRIER(); + FastOS_Thread::Sleep(500); + f.interrupt(); + TEST_BARRIER(); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configmanager/.gitignore b/config/src/tests/configmanager/.gitignore new file mode 100644 index 00000000000..0ac5d3be182 --- /dev/null +++ b/config/src/tests/configmanager/.gitignore @@ -0,0 +1,3 @@ +/config-my.cpp +/config-my.h +config_configmanager_test_app diff --git a/config/src/tests/configmanager/CMakeLists.txt b/config/src/tests/configmanager/CMakeLists.txt new file mode 100644 index 00000000000..87c11296dea --- /dev/null +++ b/config/src/tests/configmanager/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_configmanager_test_app + SOURCES + configmanager.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_configmanager_test_app COMMAND config_configmanager_test_app) +vespa_generate_config(config_configmanager_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/configmanager/configmanager.cpp b/config/src/tests/configmanager/configmanager.cpp new file mode 100644 index 00000000000..7d116477f1d --- /dev/null +++ b/config/src/tests/configmanager/configmanager.cpp @@ -0,0 +1,186 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("configmanager"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/noncopyable.hpp> +#include <vespa/config/common/configmanager.h> +#include <vespa/config/common/exceptions.h> +#include <vespa/config/subscription/sourcespec.h> +#include <vespa/config/raw/rawsource.h> +#include "config-my.h" + +using namespace config; + +namespace { + + ConfigValue createValue(const std::string & myField, const std::string & md5) + { + std::vector< vespalib::string > lines; + lines.push_back("myField \"" + myField + "\""); + return ConfigValue(lines, md5); + } + + struct TestContext + { + int numGetConfig; + int numUpdate; + int numClose; + int64_t generation; + bool respond; + TestContext() + : numGetConfig(0), numUpdate(0), numClose(0), generation(-1), respond(true) + { } + }; + + class MySource : public Source + { + public: + MySource(TestContext * data, const IConfigHolder::SP & holder) : _holder(holder), _data(data) { } + void getConfig() + { + _data->numGetConfig++; + if (_data->respond) { + LOG(info, "put into holder"); + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), true, _data->generation))); + } + } + void reload(int64_t generation) + { + _data->numUpdate++; + _data->generation = generation; + } + void close() + { + _data->numClose++; + } + IConfigHolder::SP _holder; + TestContext * _data; + }; + + class MySourceFactory : public SourceFactory + { + public: + MySourceFactory(TestContext * d) : data(d) { } + Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const + { + (void) key; + return Source::UP(new MySource(data, holder)); + } + TestContext * data; + }; + + class MySpec : public SourceSpec + { + public: + MySpec(TestContext * data) + : _key("foo"), + _data(data) + { + } + SourceSpecKey createKey() const { return SourceSpecKey(_key); } + SourceFactory::UP createSourceFactory(const TimingValues & timingValues) const { + (void) timingValues; + return SourceFactory::UP(new MySourceFactory(_data)); + } + SourceSpec * clone() const { return new MySpec(*this); } + private: + const std::string _key; + TestContext * _data; + }; + + static TimingValues testTimingValues( + 2000, // successTimeout + 500, // errorTimeout + 500, // initialTimeout + 4000, // unsubscribeTimeout + 0, // fixedDelay + 250, // successDelay + 250, // unconfiguredDelay + 500, // configuredErrorDelay + 5, + 1000, + 2000); // maxDelayMultiplier + + class ManagerTester { + public: + ConfigKey key; + ConfigManager _mgr; + ConfigSubscription::SP sub; + + ManagerTester(const ConfigKey & k, const MySpec & s) + : key(k), + _mgr(s.createSourceFactory(testTimingValues), 1) + { + } + + void subscribe() + { + sub = _mgr.subscribe(key, 5000); + } + }; + +} + +TEST("requireThatSubscriptionTimesout") { + const ConfigKey key(ConfigKey::create<MyConfig>("myid")); + const ConfigValue testValue(createValue("l33t", "a")); + + { // No valid response + TestContext data; + data.respond = false; + + ManagerTester tester(ConfigKey::create<MyConfig>("myid"), MySpec(&data)); + bool thrown = false; + try { + tester.subscribe(); + } catch (const ConfigRuntimeException & e) { + thrown = true; + } + ASSERT_TRUE(thrown); + ASSERT_EQUAL(1, data.numGetConfig); + } +} +TEST("requireThatSourceIsAskedForRequest") { + TestContext data; + const ConfigKey key(ConfigKey::create<MyConfig>("myid")); + const ConfigValue testValue(createValue("l33t", "a")); + try { + ManagerTester tester(key, MySpec(&data)); + tester.subscribe(); + ASSERT_EQUAL(1, data.numGetConfig); + } catch (ConfigRuntimeException & e) { + ASSERT_TRUE(false); + } + ASSERT_EQUAL(1, data.numClose); +} + +TEST("require that new sources are given the correct generation") { + TestContext data; + const ConfigKey key(ConfigKey::create<MyConfig>("myid")); + const ConfigValue testValue(createValue("l33t", "a")); + try { + ManagerTester tester(key, MySpec(&data)); + tester._mgr.reload(30); + tester.subscribe(); + ASSERT_EQUAL(30, data.generation); + } catch (ConfigRuntimeException & e) { + ASSERT_TRUE(false); + } +} + +void waitForGet(TestContext & data, int preferred, double timeout) +{ + FastOS_Time timer; + timer.SetNow(); + int numGet = -1; + while (timer.MilliSecsToNow() < timeout) { + numGet = data.numGetConfig; + if (numGet == preferred) + break; + FastOS_Thread::Sleep(50); + } + ASSERT_EQUAL(preferred, numGet); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configmanagerng/.gitignore b/config/src/tests/configmanagerng/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/configmanagerng/.gitignore diff --git a/config/src/tests/configparser/.gitignore b/config/src/tests/configparser/.gitignore new file mode 100644 index 00000000000..6fca805f0df --- /dev/null +++ b/config/src/tests/configparser/.gitignore @@ -0,0 +1,4 @@ +/config-foo.cpp +/config-foo.h +/foo.cfg +config_configparser_test_app diff --git a/config/src/tests/configparser/CMakeLists.txt b/config/src/tests/configparser/CMakeLists.txt new file mode 100644 index 00000000000..d2bdaf09d40 --- /dev/null +++ b/config/src/tests/configparser/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_configparser_test_app + SOURCES + configparser.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_configparser_test_app COMMAND config_configparser_test_app) +vespa_generate_config(config_configparser_test_app ../../test/resources/configdefinitions/foo.def) diff --git a/config/src/tests/configparser/configparser.cpp b/config/src/tests/configparser/configparser.cpp new file mode 100644 index 00000000000..00e8929f64a --- /dev/null +++ b/config/src/tests/configparser/configparser.cpp @@ -0,0 +1,145 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/config.h> +#include <vespa/config/common/configparser.h> +#include "config-foo.h" +#include <fstream> +#include <vespa/vespalib/stllike/asciistream.h> + +using namespace config; +using vespalib::asciistream; + +namespace { + + void writeFile(const vespalib::string & fileName, const vespalib::string & data) + { + std::ofstream of; + of.open(fileName.c_str()); + of << data; + of.close(); + } + + ConfigValue readConfig(const vespalib::string & fileName) + { + asciistream is(asciistream::createFromFile(fileName)); + return ConfigValue(is.getlines(), ""); + } +} + +TEST("require that default value exception provides error message") +{ + writeFile("foo.cfg", "blabla foo\n"); + try { + FooConfig config(readConfig("foo.cfg")); + ASSERT_TRUE(false); + } catch (InvalidConfigException & ice) { + ASSERT_EQUAL("Error parsing config 'foo' in namespace 'config': Config parameter fooValue has no default value and is not specified in config", ice.getMessage()); + } +} + +TEST("require that unknown fields can exist in config payload") +{ + writeFile("foo.cfg", "blablabla foo\nfooValue \"hello\"\n"); + try { + FooConfig config(readConfig("foo.cfg")); + ASSERT_EQUAL("hello", config.fooValue); + } catch (InvalidConfigException & ice) { + ASSERT_FALSE(true); + } +} + +TEST("require that required fields will throw error with unknown fields") +{ + writeFile("foo.cfg", "blablabla foo\nfooValu \"hello\"\n"); + try { + FooConfig config(readConfig("foo.cfg")); + ASSERT_TRUE(false); + } catch (InvalidConfigException & ice) { + ASSERT_TRUE(true); + } +} + +TEST("require that array lengths does not have to be specified") +{ + writeFile("foo.cfg", "\nfooValue \"hello\"\nfooArray[0] 3\nfooArray[1] 9\nfooArray[2] 33\nfooStruct[0].innerStruct[0].bar 2\nfooStruct[0].innerStruct[1].bar 3\nfooStruct[1].innerStruct[0].bar 4"); + try { + FooConfig config(readConfig("foo.cfg")); + ASSERT_EQUAL("hello", config.fooValue); + ASSERT_EQUAL(3u, config.fooArray.size()); + ASSERT_EQUAL(3, config.fooArray[0]); + ASSERT_EQUAL(9, config.fooArray[1]); + ASSERT_EQUAL(33, config.fooArray[2]); + ASSERT_EQUAL(2u, config.fooStruct.size()); + ASSERT_EQUAL(2u, config.fooStruct[0].innerStruct.size()); + ASSERT_EQUAL(1u, config.fooStruct[1].innerStruct.size()); + ASSERT_EQUAL(2, config.fooStruct[0].innerStruct[0].bar); + ASSERT_EQUAL(3, config.fooStruct[0].innerStruct[1].bar); + ASSERT_EQUAL(4, config.fooStruct[1].innerStruct[0].bar); + } catch (InvalidConfigException & ice) { + ASSERT_TRUE(false); + } +} + +TEST("require that array lengths may be specified") +{ + writeFile("foo.cfg", "\nfooValue \"hello\"\nfooArray[3]\nfooArray[0] 3\nfooArray[1] 9\nfooArray[2] 33\nfooStruct[2]\nfooStruct[0].innerStruct[2]\nfooStruct[0].innerStruct[0].bar 2\nfooStruct[0].innerStruct[1].bar 3\nfooStruct[1].innerStruct[1]\nfooStruct[1].innerStruct[0].bar 4"); + try { + FooConfig config(readConfig("foo.cfg")); + ASSERT_EQUAL("hello", config.fooValue); + ASSERT_EQUAL(3u, config.fooArray.size()); + ASSERT_EQUAL(3, config.fooArray[0]); + ASSERT_EQUAL(9, config.fooArray[1]); + ASSERT_EQUAL(33, config.fooArray[2]); + ASSERT_EQUAL(2u, config.fooStruct[0].innerStruct.size()); + ASSERT_EQUAL(1u, config.fooStruct[1].innerStruct.size()); + ASSERT_EQUAL(2, config.fooStruct[0].innerStruct[0].bar); + ASSERT_EQUAL(3, config.fooStruct[0].innerStruct[1].bar); + ASSERT_EQUAL(4, config.fooStruct[1].innerStruct[0].bar); + } catch (InvalidConfigException & ice) { + ASSERT_TRUE(false); + } +} + +TEST("require that escaped values are properly unescaped") { + std::vector<vespalib::string> payload; + payload.push_back("foo \"a\\nb\\rc\\\\d\\\"e\x42g\""); + vespalib::string value(ConfigParser::parse<vespalib::string>("foo", payload)); + ASSERT_EQUAL("a\nb\rc\\d\"eBg", value); +} + +TEST("verify that locale affects double parsing") { + std::vector<vespalib::string> payload; + setlocale(LC_NUMERIC, "nb_NO.UTF-8"); + payload.push_back("foo 3.14"); + ASSERT_EXCEPTION(ConfigParser::parse<double>("foo", payload), InvalidConfigException, "Value 3.14 is not a legal double"); + setlocale(LC_NUMERIC, "C"); +} + +TEST("require that maps can be parsed") +{ + writeFile("foo.cfg", "\nfooValue \"a\"\nfooMap{\"foo\"} 1336\nfooMap{\"bar\"} 1337\n"); + FooConfig config(readConfig("foo.cfg")); + ASSERT_EQUAL("a", config.fooValue); + ASSERT_EQUAL(2u, config.fooMap.size()); + ASSERT_EQUAL(1336, config.fooMap.at("foo")); + ASSERT_EQUAL(1337, config.fooMap.at("bar")); +} + +TEST("handles quotes for bool values") { + std::vector<vespalib::string> payload; + payload.push_back("foo \"true\""); + payload.push_back("bar \"123\""); + payload.push_back("baz \"1234\""); + payload.push_back("quux \"3.2\""); + bool b(ConfigParser::parse<bool>("foo", payload)); + int32_t i(ConfigParser::parse<int32_t>("bar", payload)); + int64_t l(ConfigParser::parse<int64_t>("baz", payload)); + double d(ConfigParser::parse<double>("quux", payload)); + EXPECT_EQUAL(true, b); + EXPECT_EQUAL(123, i); + EXPECT_EQUAL(1234, l); + EXPECT_APPROX(3.2, d, 0.001); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configretriever/.gitignore b/config/src/tests/configretriever/.gitignore new file mode 100644 index 00000000000..a5e6241747d --- /dev/null +++ b/config/src/tests/configretriever/.gitignore @@ -0,0 +1,8 @@ +/config-bar.cpp +/config-bar.h +/config-bootstrap.cpp +/config-bootstrap.h +/config-foo.cpp +/config-foo.h +/testsnapshot.txt +config_configretriever_test_app diff --git a/config/src/tests/configretriever/CMakeLists.txt b/config/src/tests/configretriever/CMakeLists.txt new file mode 100644 index 00000000000..5c0d8152734 --- /dev/null +++ b/config/src/tests/configretriever/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_configretriever_test_app + SOURCES + configretriever.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_configretriever_test_app COMMAND config_configretriever_test_app) +vespa_generate_config(config_configretriever_test_app ../../test/resources/configdefinitions/bootstrap.def) +vespa_generate_config(config_configretriever_test_app ../../test/resources/configdefinitions/foo.def) +vespa_generate_config(config_configretriever_test_app ../../test/resources/configdefinitions/bar.def) diff --git a/config/src/tests/configretriever/configretriever.cpp b/config/src/tests/configretriever/configretriever.cpp new file mode 100644 index 00000000000..9b191512e0e --- /dev/null +++ b/config/src/tests/configretriever/configretriever.cpp @@ -0,0 +1,444 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("configretriever"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/config/config.h> +#include <vespa/config/print.h> +#include <vespa/config/retriever/configretriever.h> +#include <vespa/config/retriever/simpleconfigretriever.h> +#include <vespa/config/retriever/simpleconfigurer.h> +#include <vespa/config/common/configholder.h> +#include <vespa/config/subscription/configsubscription.h> +#include "config-bootstrap.h" +#include "config-foo.h" +#include "config-bar.h" +#include <atomic> + +using namespace config; +using namespace std; +using namespace vespalib::slime; +using namespace vespalib; + +struct ComponentFixture { + typedef std::shared_ptr<ComponentFixture> SP; + FooConfigBuilder fooBuilder; + BarConfigBuilder barBuilder; +}; + +struct ConfigTestFixture { + const std::string configId; + BootstrapConfigBuilder bootstrapBuilder; + map<std::string, ComponentFixture::SP> componentConfig; + ConfigSet set; + IConfigContext::SP context; + int idcounter; + + ConfigTestFixture(const std::string & id) + : configId(id), + bootstrapBuilder(), + componentConfig(), + set(), + context(new ConfigContext(set)), + idcounter(-1) + { + set.addBuilder(configId, &bootstrapBuilder); + } + + void addComponent(const std::string & name, const std::string & fooValue, const std::string & barValue) + { + BootstrapConfigBuilder::Component component; + component.name = name; + component.configid = configId + "/" + name; + bootstrapBuilder.component.push_back(component); + + ComponentFixture::SP fixture(new ComponentFixture()); + fixture->fooBuilder.fooValue = fooValue; + fixture->barBuilder.barValue = barValue; + set.addBuilder(component.configid, &fixture->fooBuilder); + set.addBuilder(component.configid, &fixture->barBuilder); + componentConfig[name] = fixture; + } + + void removeComponent(const std::string & name) + { + for (BootstrapConfigBuilder::ComponentVector::iterator it(bootstrapBuilder.component.begin()), + mt(bootstrapBuilder.component.end()); it != mt; it++) { + if ((*it).name.compare(name) == 0) { + bootstrapBuilder.component.erase(it); + break; + } + } + } + + bool configEqual(const std::string & name, const FooConfig & fooConfig) { + ComponentFixture::SP fixture(componentConfig[name]); + return (fixture->fooBuilder == fooConfig); + } + + bool configEqual(const std::string & name, const BarConfig & barConfig) { + ComponentFixture::SP fixture(componentConfig[name]); + return (fixture->barBuilder == barConfig); + } + + bool configEqual(const BootstrapConfig & bootstrapConfig) { + return (bootstrapBuilder == bootstrapConfig); + } + + void reload() { context->reload(); } +}; + +struct SimpleSetup +{ + ConfigKeySet bootstrapKeys; + ConfigKeySet componentKeys; + std::unique_ptr<ConfigRetriever> retriever; + SimpleSetup(ConfigTestFixture & f1) + : bootstrapKeys(), componentKeys(), retriever() + { + f1.addComponent("c1", "foo1", "bar1"); + bootstrapKeys.add<BootstrapConfig>(f1.configId); + retriever.reset(new ConfigRetriever(bootstrapKeys, f1.context)); + } + +}; + +struct MySource : public Source +{ + void getConfig() { } + void close() { } + void reload(int64_t gen) { (void) gen; } +}; + +struct SubscriptionFixture +{ + IConfigHolder::SP holder; + ConfigSubscription::SP sub; + SubscriptionFixture(const ConfigKey & key, const ConfigValue value) + : holder(new ConfigHolder()), + sub(new ConfigSubscription(0, key, holder, Source::UP(new MySource()))) + { + holder->handle(ConfigUpdate::UP(new ConfigUpdate(value, 3, 3))); + ASSERT_TRUE(sub->nextUpdate(0, 0)); + sub->flip(); + } +}; + + +namespace { + +class FixedPayload : public protocol::Payload { +public: + const Inspector & getSlimePayload() const + { + return _data.get(); + } + + Slime & getData() { + return _data; + } +private: + Slime _data; +}; + +} + +ConfigValue createKeyValueV2(const vespalib::string & key, const vespalib::string & value) +{ + FixedPayload * payload = new FixedPayload(); + payload->getData().setObject().setString(key, Memory(value)); + return ConfigValue(PayloadPtr(payload), ""); +} + + +TEST_F("require that basic retriever usage works", ConfigTestFixture("myid")) { + f1.addComponent("c1", "foo1", "bar1"); + f1.addComponent("c2", "foo2", "bar2"); + + ConfigKeySet keys; + keys.add<BootstrapConfig>(f1.configId); + + ConfigRetriever ret(keys, f1.context); + ConfigSnapshot configs = ret.getBootstrapConfigs(); + ASSERT_EQUAL(1u, configs.size()); + + std::unique_ptr<BootstrapConfig> bootstrapConfig = configs.getConfig<BootstrapConfig>(f1.configId); + ASSERT_TRUE(f1.configEqual(*bootstrapConfig)); + + { + ConfigKeySet componentKeys; + for (size_t i = 0; i < bootstrapConfig->component.size(); i++) { + const vespalib::string & configId(bootstrapConfig->component[i].configid); + componentKeys.add<FooConfig>(configId); + } + configs = ret.getConfigs(componentKeys); + ASSERT_EQUAL(2u, configs.size()); + ASSERT_TRUE(f1.configEqual("c1", *configs.getConfig<FooConfig>(bootstrapConfig->component[0].configid))); + ASSERT_TRUE(f1.configEqual("c2", *configs.getConfig<FooConfig>(bootstrapConfig->component[1].configid))); + } + { + ConfigKeySet componentKeys; + for (size_t i = 0; i < bootstrapConfig->component.size(); i++) { + const vespalib::string & configId(bootstrapConfig->component[i].configid); + componentKeys.add<BarConfig>(configId); + } + configs = ret.getConfigs(componentKeys); + ASSERT_EQUAL(2u, configs.size()); + ASSERT_TRUE(f1.configEqual("c1", *configs.getConfig<BarConfig>(bootstrapConfig->component[0].configid))); + ASSERT_TRUE(f1.configEqual("c2", *configs.getConfig<BarConfig>(bootstrapConfig->component[1].configid))); + } + { + ConfigKeySet componentKeys; + for (size_t i = 0; i < bootstrapConfig->component.size(); i++) { + const vespalib::string & configId(bootstrapConfig->component[i].configid); + componentKeys.add<FooConfig>(configId); + componentKeys.add<BarConfig>(configId); + } + configs = ret.getConfigs(componentKeys); + + ASSERT_EQUAL(4u, configs.size()); + ASSERT_TRUE(f1.configEqual("c1", *configs.getConfig<FooConfig>(bootstrapConfig->component[0].configid))); + ASSERT_TRUE(f1.configEqual("c1", *configs.getConfig<BarConfig>(bootstrapConfig->component[0].configid))); + ASSERT_TRUE(f1.configEqual("c2", *configs.getConfig<FooConfig>(bootstrapConfig->component[1].configid))); + ASSERT_TRUE(f1.configEqual("c2", *configs.getConfig<BarConfig>(bootstrapConfig->component[1].configid))); + } +} + +TEST("require that SimpleConfigRetriever usage works") { + ConfigSet set; + FooConfigBuilder fooBuilder; + BarConfigBuilder barBuilder; + fooBuilder.fooValue = "barz"; + barBuilder.barValue = "fooz"; + set.addBuilder("id", &fooBuilder); + set.addBuilder("id", &barBuilder); + IConfigContext::SP ctx(new ConfigContext(set)); + ConfigKeySet sub; + sub.add<FooConfig>("id"); + sub.add<BarConfig>("id"); + SimpleConfigRetriever retr(sub, ctx); + ConfigSnapshot snap = retr.getConfigs(); + ASSERT_FALSE(snap.empty()); + ASSERT_EQUAL(2u, snap.size()); + std::unique_ptr<FooConfig> foo = snap.getConfig<FooConfig>("id"); + std::unique_ptr<BarConfig> bar = snap.getConfig<BarConfig>("id"); + ASSERT_EQUAL("barz", foo->fooValue); + ASSERT_EQUAL("fooz", bar->barValue); +} + +class ConfigurableFixture : public SimpleConfigurable +{ +public: + /** + * Note that due to some bug in gcc 5.2 this file must be compiled with + * -fno-tree-vrp which turns off some optimization. Or you can reorder the menbers here + * and put 'snap' after the atomics. + */ + ConfigurableFixture() __attribute__((noinline)); + virtual ~ConfigurableFixture() __attribute__((noinline)); + void configure(const ConfigSnapshot & snapshot) override { + (void) snapshot; + if (throwException) { + throw ConfigRuntimeException("foo"); + } + snap = snapshot; + configured = true; + } + bool waitUntilConfigured(int64_t timeoutInMillis) { + FastOS_Time timer; + timer.SetNow(); + while (timer.MilliSecsToNow() < timeoutInMillis) { + if (configured) { + return true; + } + FastOS_Thread::Sleep(200); + } + return configured; + } + + ConfigSnapshot snap; + std::atomic<bool> configured; + std::atomic<bool> throwException; +}; + +ConfigurableFixture::ConfigurableFixture() : + configured(false), + throwException(false) +{ +} + +ConfigurableFixture::~ConfigurableFixture() +{ +} + +TEST_F("require that SimpleConfigurer usage works", ConfigurableFixture()) { + ConfigSet set; + FooConfigBuilder fooBuilder; + BarConfigBuilder barBuilder; + fooBuilder.fooValue = "barz"; + barBuilder.barValue = "fooz"; + set.addBuilder("id", &fooBuilder); + set.addBuilder("id", &barBuilder); + IConfigContext::SP ctx(new ConfigContext(set)); + ConfigKeySet sub; + sub.add<FooConfig>("id"); + sub.add<BarConfig>("id"); + SimpleConfigurer configurer(SimpleConfigRetriever::UP(new SimpleConfigRetriever(sub, ctx)), &f1); + configurer.start(); + ASSERT_FALSE(f1.snap.empty()); + ASSERT_EQUAL(2u, f1.snap.size()); + ConfigSnapshot snap = f1.snap; + std::unique_ptr<FooConfig> foo = snap.getConfig<FooConfig>("id"); + std::unique_ptr<BarConfig> bar = snap.getConfig<BarConfig>("id"); + ASSERT_EQUAL("barz", foo->fooValue); + ASSERT_EQUAL("fooz", bar->barValue); + + f1.configured = false; + fooBuilder.fooValue = "bimz"; + ctx->reload(); + ASSERT_TRUE(f1.waitUntilConfigured(60000)); + snap = f1.snap; + foo = snap.getConfig<FooConfig>("id"); + ASSERT_EQUAL("bimz", foo->fooValue); + configurer.close(); + fooBuilder.fooValue = "bamz"; + f1.configured = false; + ctx->reload(); + ASSERT_FALSE(f1.waitUntilConfigured(2000)); + + SimpleConfigurer configurer2(SimpleConfigRetriever::UP(new SimpleConfigRetriever(sub, ctx)), &f1); + f1.throwException = true; + ASSERT_EXCEPTION(configurer2.start(), ConfigRuntimeException, "foo"); + configurer2.close(); +} + +TEST("require that variadic templates can be used to create key sets") { + ConfigKeySet set; + set.add<FooConfig, BarConfig, BootstrapConfig>("myid"); + ASSERT_EQUAL(3u, set.size()); +} + +TEST_FF("require that getBootstrapConfigs returns empty snapshot when closed", ConfigTestFixture("myid"), SimpleSetup(f1)) { + ConfigSnapshot configs = f2.retriever->getBootstrapConfigs(); + ASSERT_TRUE(!configs.empty()); + ASSERT_FALSE(f2.retriever->isClosed()); + f2.retriever->close(); + ASSERT_TRUE(f2.retriever->isClosed()); + configs = f2.retriever->getBootstrapConfigs(); + ASSERT_TRUE(configs.empty()); +} + +TEST_FF("require that getConfigs throws exception when closed", ConfigTestFixture("myid"), SimpleSetup(f1)) { + ConfigSnapshot configs = f2.retriever->getBootstrapConfigs(); + std::unique_ptr<BootstrapConfig> bootstrapConfig = configs.getConfig<BootstrapConfig>(f1.configId); + ConfigKeySet componentKeys; + for (size_t i = 0; i < bootstrapConfig->component.size(); i++) { + const vespalib::string & configId(bootstrapConfig->component[i].configid); + componentKeys.add<FooConfig>(configId); + componentKeys.add<BarConfig>(configId); + } + ASSERT_FALSE(f2.retriever->isClosed()); + f2.retriever->close(); + ASSERT_TRUE(f2.retriever->isClosed()); + configs = f2.retriever->getConfigs(componentKeys); + ASSERT_TRUE(configs.empty()); +} + + +TEST_FF("require that snapshots throws exception if invalid key", ConfigTestFixture("myid"), SimpleSetup(f1)) { + f1.addComponent("c3", "foo3", "bar3"); + ConfigSnapshot snap1 = f2.retriever->getBootstrapConfigs(); + ASSERT_FALSE(snap1.hasConfig<BarConfig>("doesnotexist")); + ASSERT_EXCEPTION(snap1.getConfig<BarConfig>("doesnotexist"), IllegalConfigKeyException, "Unable to find config for key name=bar,namespace=config,configId=doesnotexist"); + ASSERT_EXCEPTION(snap1.isChanged<BarConfig>("doesnotexist", 0), IllegalConfigKeyException, "Unable to find config for key name=bar,namespace=config,configId=doesnotexist"); + ASSERT_TRUE(snap1.hasConfig<BootstrapConfig>("myid")); +} + +TEST_FF("require that snapshots can be ignored", ConfigTestFixture("myid"), SimpleSetup(f1)) { + f1.addComponent("c3", "foo3", "bar3"); + ConfigSnapshot snap1 = f2.retriever->getBootstrapConfigs(); + int64_t lastGen = snap1.getGeneration(); + f1.reload(); + ASSERT_EQUAL(lastGen, snap1.getGeneration()); + ConfigSnapshot snap2 = f2.retriever->getBootstrapConfigs(); + ASSERT_EQUAL(snap2.getGeneration(), 2); + ASSERT_TRUE(snap2.isChanged<BootstrapConfig>("myid", lastGen)); + ASSERT_FALSE(snap2.isChanged<BootstrapConfig>("myid", lastGen + 1)); + f1.reload(); + ConfigSnapshot snap3 = f2.retriever->getBootstrapConfigs(); + ASSERT_TRUE(snap3.isChanged<BootstrapConfig>("myid", lastGen)); + ASSERT_FALSE(snap3.isChanged<BootstrapConfig>("myid", lastGen + 1)); +} + +TEST_FFF("require that snapshots can produce subsets", SubscriptionFixture(ConfigKey::create<FooConfig>("id"), createKeyValueV2("fooValue", "bar")), + SubscriptionFixture(ConfigKey::create<BarConfig>("id"), createKeyValueV2("barValue", "foo")), + ConfigSnapshot::SubscriptionList()) { + f3.push_back(f1.sub); + f3.push_back(f2.sub); + ConfigSnapshot parent(f3, 3); + ASSERT_FALSE(parent.empty()); + ASSERT_EQUAL(3, parent.getGeneration()); + ASSERT_EQUAL(2u, parent.size()); + + ConfigSnapshot subset1(parent.subset(ConfigKeySet().add<FooConfig>("id"))); + ASSERT_FALSE(subset1.empty()); + ASSERT_EQUAL(3, subset1.getGeneration()); + ASSERT_EQUAL(1u, subset1.size()); + std::unique_ptr<FooConfig> cfg1(subset1.getConfig<FooConfig>("id")); + ASSERT_TRUE(cfg1.get() != NULL); + + ConfigSnapshot subset2(parent.subset(ConfigKeySet().add<BarConfig>("id"))); + ASSERT_FALSE(subset2.empty()); + ASSERT_EQUAL(3, subset2.getGeneration()); + ASSERT_EQUAL(1u, subset2.size()); + std::unique_ptr<BarConfig> cfg2(subset2.getConfig<BarConfig>("id")); + ASSERT_TRUE(cfg2.get() != NULL); + + ConfigSnapshot subset3(parent.subset(ConfigKeySet().add<BarConfig>("doesnotexist"))); + ASSERT_TRUE(subset3.empty()); + ASSERT_EQUAL(3, subset3.getGeneration()); + ASSERT_EQUAL(0u, subset3.size()); + + ConfigSnapshot subset4(parent.subset(ConfigKeySet().add<BarConfig>("doesnotexist").add<FooConfig>("id").add<FooConfig>("nosuchthing").add<BarConfig>("id").add<BarConfig>("nothere"))); + ASSERT_FALSE(subset4.empty()); + ASSERT_EQUAL(3, subset4.getGeneration()); + ASSERT_EQUAL(2u, subset4.size()); + cfg1 = subset4.getConfig<FooConfig>("id"); + ASSERT_TRUE(cfg1.get() != NULL); + cfg2 = subset4.getConfig<BarConfig>("id"); + ASSERT_TRUE(cfg2.get() != NULL); +} + +TEST_FFF("require that snapshots can be serialized", SubscriptionFixture(ConfigKey::create<FooConfig>("id"), createKeyValueV2("fooValue", "bar")), + SubscriptionFixture(ConfigKey::create<BarConfig>("id"), createKeyValueV2("barValue", "foo")), + ConfigSnapshot::SubscriptionList()) { + f3.push_back(f1.sub); + f3.push_back(f2.sub); + ConfigSnapshot parent(f3, 3); + + typedef std::shared_ptr<ConfigSnapshotWriter> WSP; + typedef std::shared_ptr<ConfigSnapshotReader> RSP; + typedef std::pair<WSP, RSP> SerializePair; + typedef std::vector<SerializePair> Vec; + Vec vec; + vespalib::asciistream ss; + vec.push_back(SerializePair(WSP(new FileConfigSnapshotWriter("testsnapshot.txt")), + RSP(new FileConfigSnapshotReader("testsnapshot.txt")))); + vec.push_back(SerializePair(WSP(new AsciiConfigSnapshotWriter(ss)), + RSP(new AsciiConfigSnapshotReader(ss)))); + for (Vec::iterator it(vec.begin()), mt(vec.end()); it != mt; it++) { + ASSERT_TRUE(it->first->write(parent)); + ConfigSnapshot deser(it->second->read()); + ASSERT_EQUAL(parent.getGeneration(), deser.getGeneration()); + ASSERT_EQUAL(parent.size(), deser.size()); + ASSERT_TRUE(deser.hasConfig<FooConfig>("id")); + ASSERT_TRUE(deser.hasConfig<BarConfig>("id")); + std::unique_ptr<FooConfig> foo = deser.getConfig<FooConfig>("id"); + std::unique_ptr<BarConfig> bar = deser.getConfig<BarConfig>("id"); + ASSERT_EQUAL("bar", foo->fooValue); + ASSERT_EQUAL("foo", bar->barValue); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/configuri/.gitignore b/config/src/tests/configuri/.gitignore new file mode 100644 index 00000000000..87df7752465 --- /dev/null +++ b/config/src/tests/configuri/.gitignore @@ -0,0 +1,3 @@ +/config-my.cpp +/config-my.h +config_configuri_test_app diff --git a/config/src/tests/configuri/CMakeLists.txt b/config/src/tests/configuri/CMakeLists.txt new file mode 100644 index 00000000000..10fcac86c76 --- /dev/null +++ b/config/src/tests/configuri/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_configuri_test_app + SOURCES + configuri_test.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_configuri_test_app COMMAND config_configuri_test_app) +vespa_generate_config(config_configuri_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/configuri/configuri_test.cpp b/config/src/tests/configuri/configuri_test.cpp new file mode 100644 index 00000000000..83b99542ce4 --- /dev/null +++ b/config/src/tests/configuri/configuri_test.cpp @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/config.h> +#include "config-my.h" + +using namespace config; + +namespace { + +void assertConfigId(const std::string & expected, const ConfigUri & uri) { + ASSERT_EQUAL(expected, uri.getConfigId()); +} + +} +TEST("Require that URI can be created from const char *") { + assertConfigId("foo/bar", ConfigUri("foo/bar")); + assertConfigId("myfile", ConfigUri("file:myfile.cfg")); + assertConfigId("", ConfigUri("raw:myraw")); + assertConfigId("", ConfigUri("dir:.")); +} + +TEST("Require that URI can be created from std::string") { + assertConfigId("foo/bar", ConfigUri(std::string("foo/bar"))); + assertConfigId("myfile", ConfigUri(std::string("file:myfile.cfg"))); + assertConfigId("", ConfigUri(std::string("raw:myraw"))); + assertConfigId("", ConfigUri(std::string("dir:."))); +} + +TEST("Require that URI can be created from vespalib::string") { + assertConfigId("foo/bar", ConfigUri(vespalib::string("foo/bar"))); + assertConfigId("myfile", ConfigUri(vespalib::string("file:myfile.cfg"))); + assertConfigId("", ConfigUri(vespalib::string("raw:myraw"))); + assertConfigId("", ConfigUri(vespalib::string("dir:."))); +} + +TEST("Require that URI can be created from instance") { + MyConfigBuilder b; + b.myField = "rabarbra"; + ConfigUri uri(ConfigUri::createFromInstance(b)); + ConfigSubscriber subscriber(uri.getContext()); + ConfigHandle<MyConfig>::UP handle = + subscriber.subscribe<MyConfig>(uri.getConfigId()); + ASSERT_TRUE(subscriber.nextConfig(0)); + ASSERT_TRUE(handle->isChanged()); + std::unique_ptr<MyConfig> cfg = handle->getConfig(); + ASSERT_EQUAL(b.myField, cfg->myField); + +} + +TEST_F("Require that URI can be \"forked\"", IConfigContext::SP(new ConfigContext())) { + assertConfigId("baz", ConfigUri("foo/bar").createWithNewId("baz")); + ConfigUri parent("foo", f1); + ConfigUri child = parent.createWithNewId("baz"); + ASSERT_TRUE(parent.getContext().get() == child.getContext().get()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/failover/.gitignore b/config/src/tests/failover/.gitignore new file mode 100644 index 00000000000..8e4e371539f --- /dev/null +++ b/config/src/tests/failover/.gitignore @@ -0,0 +1,3 @@ +/config-my.cpp +/config-my.h +config_failover_test_app diff --git a/config/src/tests/failover/CMakeLists.txt b/config/src/tests/failover/CMakeLists.txt new file mode 100644 index 00000000000..aa5ab803bc5 --- /dev/null +++ b/config/src/tests/failover/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_failover_test_app + SOURCES + failover.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_failover_test_app COMMAND config_failover_test_app) +vespa_generate_config(config_failover_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/failover/failover.cpp b/config/src/tests/failover/failover.cpp new file mode 100644 index 00000000000..baaae8ba19a --- /dev/null +++ b/config/src/tests/failover/failover.cpp @@ -0,0 +1,356 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("failover"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/common/misc.h> +#include <vespa/config/frt/protocol.h> +#include <vespa/config/config.h> +#include <vespa/fnet/frt/frt.h> +#include "config-my.h" +#include <vespa/vespalib/data/slime/slime.h> + +using namespace config; +using vespalib::Barrier; +using namespace config::protocol::v2; +using namespace vespalib::slime; +using namespace vespalib; + +namespace { + +int get_port(const vespalib::string &spec) { + const char *port = (spec.data() + spec.size()); + while ((port > spec.data()) && (port[-1] >= '0') && (port[-1] <= '9')) { + --port; + } + return atoi(port); +} + +const vespalib::string requestTypes = "s"; +const vespalib::string responseTypes = "sx"; + +struct RPCServer : public FRT_Invokable { + FRT_Supervisor * supervisor; + vespalib::Barrier barrier; + int64_t gen; + + RPCServer() : supervisor(NULL), barrier(2), gen(1) { } + + void init(FRT_Supervisor * s) { + FRT_ReflectionBuilder rb(s); + rb.DefineMethod("config.v3.getConfig", requestTypes.c_str(), responseTypes.c_str(), true, + FRT_METHOD(RPCServer::getConfig), this); + } + + void getConfig(FRT_RPCRequest * req) + { + Slime slime; + Cursor & root(slime.setObject()); + root.setLong(RESPONSE_VERSION, 3); + root.setString(RESPONSE_DEF_NAME, Memory(MyConfig::CONFIG_DEF_NAME)); + root.setString(RESPONSE_DEF_NAMESPACE, Memory(MyConfig::CONFIG_DEF_NAMESPACE)); + root.setString(RESPONSE_DEF_MD5, Memory(MyConfig::CONFIG_DEF_MD5)); + Cursor &info = root.setObject("compressionInfo"); + info.setString("compressionType", "UNCOMPRESSED"); + info.setString("uncompressedSize", "0"); + root.setString(RESPONSE_CONFIGID, "myId"); + root.setString(RESPONSE_CLIENT_HOSTNAME, "myhost"); + root.setString(RESPONSE_CONFIG_MD5, "md5"); + root.setLong(RESPONSE_CONFIG_GENERATION, gen); + root.setObject(RESPONSE_TRACE); + Slime payload; + payload.setObject().setString("myField", "myval"); + + FRT_Values & ret = *req->GetReturn(); + SimpleBuffer buf; + JsonFormat::encode(slime, buf, false); + ret.AddString(buf.get().make_string().c_str()); + + SimpleBuffer pbuf; + JsonFormat::encode(payload, pbuf, false); + vespalib::string d = pbuf.get().make_string(); + ret.AddData(d.c_str(), d.size()); + LOG(info, "Answering..."); + } + void wait() { + barrier.await(); + } + void reload() { gen++; } +}; + + +void verifyConfig(std::unique_ptr<MyConfig> config) +{ + ASSERT_TRUE(config.get() != NULL); + ASSERT_EQUAL("myval", config->myField); +} + +struct ServerFixture { + typedef vespalib::LinkedPtr<ServerFixture> LP; + FRT_Supervisor * supervisor; + RPCServer server; + Barrier b; + const vespalib::string listenSpec; + ServerFixture(const vespalib::string & ls) + : supervisor(NULL), + server(), + b(2), + listenSpec(ls) + { + } + + void wait() + { + b.await(); + } + + void start() + { + supervisor = new FRT_Supervisor(); + server.init(supervisor); + supervisor->Listen(get_port(listenSpec)); + wait(); // Wait until test runner signals we can start + supervisor->Main(); + wait(); // Signalling that we have shut down + wait(); // Wait for signal saying that supervisor is deleted + } + + void stop() + { + if (supervisor != NULL) { + supervisor->ShutDown(true); + wait(); // Wait for supervisor to shut down + delete supervisor; + supervisor = NULL; + wait(); // Signal that we are done and start can return. + } + } + + ~ServerFixture() { stop(); } +}; + +struct NetworkFixture { + std::vector<ServerFixture::LP> serverList; + ServerSpec spec; + bool running; + NetworkFixture(const std::vector<vespalib::string> & serverSpecs) + : spec(serverSpecs), running(true) + { + for (size_t i = 0; i < serverSpecs.size(); i++) { + serverList.push_back(ServerFixture::LP(new ServerFixture(serverSpecs[i]))); + } + } + void start(size_t i) { + serverList[i]->start(); + } + void wait(size_t i) { + serverList[i]->wait(); + } + void waitAll() { + for (size_t i = 0; i < serverList.size(); i++) { + serverList[i]->wait(); + } + } + void run(size_t i) { + while (running) { + serverList[i]->start(); + } + } + void stopAll() { + running = false; + for (size_t i = 0; i < serverList.size(); i++) { + serverList[i]->stop(); + } + } + void stop(size_t i) { + serverList[i]->stop(); + } + void reload() { + for (size_t i = 0; i < serverList.size(); i++) { + serverList[i]->server.reload(); + } + } +}; + + +TimingValues testTimingValues( + 500, // successTimeout + 500, // errorTimeout + 500, // initialTimeout + 400, // unsubscribeTimeout + 0, // fixedDelay + 250, // successDelay + 250, // unconfiguredDelay + 500, // configuredErrorDelay + 1, // maxDelayMultiplier + 600, // transientDelay + 1200); // fatalDelay + +struct ConfigCheckFixture { + IConfigContext::SP ctx; + NetworkFixture & nf; + + ConfigCheckFixture(NetworkFixture & f2) + : ctx(new ConfigContext(testTimingValues, f2.spec)), + nf(f2) + { + } + void checkSubscribe() + { + ConfigSubscriber s(ctx); + ConfigHandle<MyConfig>::UP handle = s.subscribe<MyConfig>("myId"); + ASSERT_TRUE(s.nextConfig()); + } + void verifySubscribeFailover(size_t index) + { + nf.stop(index); + checkSubscribe(); + nf.wait(index); + } + + void verifySubscribeFailover(size_t indexA, size_t indexB) + { + nf.stop(indexA); + nf.stop(indexB); + checkSubscribe(); + nf.wait(indexA); + nf.wait(indexB); + } +}; + +struct ConfigReloadFixture { + IConfigContext::SP ctx; + NetworkFixture & nf; + ConfigSubscriber s; + ConfigHandle<MyConfig>::UP handle; + + ConfigReloadFixture(NetworkFixture & f2) + : ctx(new ConfigContext(testTimingValues, f2.spec)), + nf(f2), + s(ctx), + handle(s.subscribe<MyConfig>("myId")) + { + } + + void verifyReload() + { + nf.reload(); + ASSERT_TRUE(s.nextGeneration()); + verifyConfig(handle->getConfig()); + } + + void verifyReloadFailover(size_t index) + { + nf.stop(index); + verifyReload(); + nf.wait(index); + } + + void verifyReloadFailover(size_t indexA, size_t indexB) + { + nf.stop(indexA); + nf.stop(indexB); + verifyReload(); + nf.wait(indexA); + nf.wait(indexB); + } +}; + +struct ThreeServersFixture { + std::vector<vespalib::string> specs; + ThreeServersFixture() : specs() { + specs.push_back("tcp/localhost:18590"); + specs.push_back("tcp/localhost:18592"); + specs.push_back("tcp/localhost:18594"); + } +}; + +struct OneServerFixture { + std::vector<vespalib::string> specs; + OneServerFixture() : specs() { + specs.push_back("tcp/localhost:18590"); + } +}; + +} + +TEST_MT_FF("require that any node can be down when subscribing", + 4, + ThreeServersFixture(), + NetworkFixture(f1.specs)) +{ + if (thread_id == 0) { + ConfigCheckFixture ccf(f2); + f2.waitAll(); + ccf.checkSubscribe(); + ccf.verifySubscribeFailover(0); + ccf.verifySubscribeFailover(1); + ccf.verifySubscribeFailover(2); + f2.stopAll(); + TEST_BARRIER(); + } else { + f2.run(thread_id - 1); + TEST_BARRIER(); + } +} +/* +TEST_MT_FF("require that two out of three nodes can be down when subscribing", + 4, + ThreeServersFixture(), + NetworkFixture(f1.specs)) +{ + if (thread_id == 0) { + ConfigCheckFixture ccf(f2); + f2.waitAll(); + ccf.checkSubscribe(); + ccf.verifySubscribeFailover(0, 1); + ccf.verifySubscribeFailover(1, 2); + ccf.verifySubscribeFailover(0, 2); + f2.stopAll(); + TEST_BARRIER(); + } else { + f2.run(thread_id - 1); + TEST_BARRIER(); + } +} + +TEST_MT_FF("require that any node can be down when waiting for next generation", + 4, + ThreeServersFixture(), + NetworkFixture(f1.specs)) +{ + if (thread_id == 0) { + f2.waitAll(); + ConfigReloadFixture crf(f2); + crf.verifyReload(); + crf.verifyReloadFailover(0); + crf.verifyReloadFailover(1); + crf.verifyReloadFailover(2); + f2.stopAll(); + TEST_BARRIER(); + } else { f2.run(thread_id - 1); TEST_BARRIER(); + } +} + +TEST_MT_FF("require that two out of three nodes can be down when waiting for next generation", + 4, + ThreeServersFixture(), + NetworkFixture(f1.specs)) +{ + if (thread_id == 0) { + f2.waitAll(); + ConfigReloadFixture crf(f2); + crf.verifyReload(); + crf.verifyReloadFailover(0, 1); + crf.verifyReloadFailover(1, 2); + crf.verifyReloadFailover(0, 2); + f2.stopAll(); + TEST_BARRIER(); + } else { + f2.run(thread_id - 1); + TEST_BARRIER(); + } +} +*/ + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/file_subscription/.gitignore b/config/src/tests/file_subscription/.gitignore new file mode 100644 index 00000000000..7dd29b02329 --- /dev/null +++ b/config/src/tests/file_subscription/.gitignore @@ -0,0 +1,11 @@ +/config-my.cpp +/config-my.h +/config-bar.cpp +/config-bar.h +/config-foo.cpp +/config-foo.h +/config-foobar.cpp +/config-foobar.h +/config-foodefault.cpp +/config-foodefault.h +config_file_subscription_test_app diff --git a/config/src/tests/file_subscription/CMakeLists.txt b/config/src/tests/file_subscription/CMakeLists.txt new file mode 100644 index 00000000000..f582732ffb2 --- /dev/null +++ b/config/src/tests/file_subscription/CMakeLists.txt @@ -0,0 +1,13 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_file_subscription_test_app + SOURCES + file_subscription.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_file_subscription_test_app COMMAND config_file_subscription_test_app) +vespa_generate_config(config_file_subscription_test_app ../../test/resources/configdefinitions/my.def) +vespa_generate_config(config_file_subscription_test_app ../../test/resources/configdefinitions/foo.def) +vespa_generate_config(config_file_subscription_test_app ../../test/resources/configdefinitions/bar.def) +vespa_generate_config(config_file_subscription_test_app ../../test/resources/configdefinitions/foobar.def) +vespa_generate_config(config_file_subscription_test_app ../../test/resources/configdefinitions/foodefault.def) diff --git a/config/src/tests/file_subscription/cfgdir/bar.cfg b/config/src/tests/file_subscription/cfgdir/bar.cfg new file mode 100644 index 00000000000..cd7ce3a73ec --- /dev/null +++ b/config/src/tests/file_subscription/cfgdir/bar.cfg @@ -0,0 +1 @@ +barValue "barbar" diff --git a/config/src/tests/file_subscription/cfgdir/foo.cfg b/config/src/tests/file_subscription/cfgdir/foo.cfg new file mode 100644 index 00000000000..288a1571758 --- /dev/null +++ b/config/src/tests/file_subscription/cfgdir/foo.cfg @@ -0,0 +1 @@ +fooValue "foofoo" diff --git a/config/src/tests/file_subscription/cfgdir2/bar.cfg b/config/src/tests/file_subscription/cfgdir2/bar.cfg new file mode 100644 index 00000000000..cd7ce3a73ec --- /dev/null +++ b/config/src/tests/file_subscription/cfgdir2/bar.cfg @@ -0,0 +1 @@ +barValue "barbar" diff --git a/config/src/tests/file_subscription/cfgdir2/foobar.cfg b/config/src/tests/file_subscription/cfgdir2/foobar.cfg new file mode 100644 index 00000000000..e938175a509 --- /dev/null +++ b/config/src/tests/file_subscription/cfgdir2/foobar.cfg @@ -0,0 +1 @@ +fooBarValue "foobarlol" diff --git a/config/src/tests/file_subscription/cfgdir3/bar.bar.cfg b/config/src/tests/file_subscription/cfgdir3/bar.bar.cfg new file mode 100644 index 00000000000..5246bbb392d --- /dev/null +++ b/config/src/tests/file_subscription/cfgdir3/bar.bar.cfg @@ -0,0 +1 @@ +barValue "foobarlol" diff --git a/config/src/tests/file_subscription/cfgdir3/bar.foo.cfg b/config/src/tests/file_subscription/cfgdir3/bar.foo.cfg new file mode 100644 index 00000000000..cd7ce3a73ec --- /dev/null +++ b/config/src/tests/file_subscription/cfgdir3/bar.foo.cfg @@ -0,0 +1 @@ +barValue "barbar" diff --git a/config/src/tests/file_subscription/cfgemptyfile/bar.cfg b/config/src/tests/file_subscription/cfgemptyfile/bar.cfg new file mode 100644 index 00000000000..cd7ce3a73ec --- /dev/null +++ b/config/src/tests/file_subscription/cfgemptyfile/bar.cfg @@ -0,0 +1 @@ +barValue "barbar" diff --git a/config/src/tests/file_subscription/cfgemptyfile/foodefault.cfg b/config/src/tests/file_subscription/cfgemptyfile/foodefault.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/file_subscription/cfgemptyfile/foodefault.cfg diff --git a/config/src/tests/file_subscription/cfgnonexistingfile/bar.cfg b/config/src/tests/file_subscription/cfgnonexistingfile/bar.cfg new file mode 100644 index 00000000000..cd7ce3a73ec --- /dev/null +++ b/config/src/tests/file_subscription/cfgnonexistingfile/bar.cfg @@ -0,0 +1 @@ +barValue "barbar" diff --git a/config/src/tests/file_subscription/file_subscription.cpp b/config/src/tests/file_subscription/file_subscription.cpp new file mode 100644 index 00000000000..4b2c84281e9 --- /dev/null +++ b/config/src/tests/file_subscription/file_subscription.cpp @@ -0,0 +1,205 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/config.h> +#include <vespa/config/common/configholder.h> +#include <vespa/config/file/filesource.h> +#include <vespa/vespalib/util/sync.h> +#include <fstream> +#include <config-my.h> +#include <config-foo.h> +#include <config-foodefault.h> +#include <config-bar.h> +#include <config-foobar.h> +#include <vespa/log/log.h> +LOG_SETUP(".filesubscription_test"); + +using namespace config; + +namespace { + + void writeFile(const std::string & fileName, const std::string & myFieldVal) + { + std::ofstream of; + of.open(fileName.c_str()); + of << "myField \"" << myFieldVal << "\"\n"; + of.close(); + } +} + +TEST("requireThatFileSpecGivesCorrectKey") { + std::string str("/home/my/config.cfg"); + FileSpec spec(str); + bool thrown = false; + try { + FileSpec s1("fb"); + FileSpec s2("fb.cfh"); + FileSpec s3("fb.dch"); + FileSpec s4("fbcfg"); + FileSpec s5(".cfg"); + } catch (const InvalidConfigSourceException & e) { + thrown = true; + } + ASSERT_TRUE(thrown); + + thrown = false; + try { + FileSpec s1("fb.cfg"); + FileSpec s2("a.cfg"); + FileSpec s3("fljdlfjsalf.cfg"); + } catch (const InvalidConfigSourceException & e) { + thrown = true; + } + ASSERT_FALSE(thrown); +} + +TEST("requireThatFileSpecGivesCorrectSource") { + writeFile("my.cfg", "foobar"); + FileSpec spec("my.cfg"); + + SourceFactory::UP factory(spec.createSourceFactory(TimingValues())); + ASSERT_TRUE(factory.get() != NULL); + IConfigHolder::SP holder(new ConfigHolder()); + Source::UP src = factory->createSource(holder, ConfigKey("my", "my", "bar", "foo")); + ASSERT_TRUE(src.get() != NULL); + + src->getConfig(); + ASSERT_TRUE(holder->poll()); + ConfigUpdate::UP update(holder->provide()); + ASSERT_TRUE(update.get() != NULL); + const ConfigValue & value(update->getValue()); + ASSERT_EQUAL(1u, value.numLines()); + ASSERT_EQUAL("myField \"foobar\"", value.getLine(0)); +} + +TEST("requireThatFileSubscriptionReturnsCorrectConfig") { + writeFile("my.cfg", "foobar"); + ConfigSubscriber s(FileSpec("my.cfg")); + std::unique_ptr<ConfigHandle<MyConfig> > handle = s.subscribe<MyConfig>("my"); + s.nextConfig(0); + std::unique_ptr<MyConfig> cfg = handle->getConfig(); + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL("foobar", cfg->myField); + ASSERT_EQUAL("my", cfg->defName()); + ASSERT_FALSE(s.nextConfig(100)); +} + +TEST("requireThatReconfigIsCalledWhenConfigChanges") { + writeFile("my.cfg", "foo"); + { + IConfigContext::SP context(new ConfigContext(FileSpec("my.cfg"))); + ConfigSubscriber s(context); + std::unique_ptr<ConfigHandle<MyConfig> > handle = s.subscribe<MyConfig>(""); + s.nextConfig(0); + std::unique_ptr<MyConfig> cfg = handle->getConfig(); + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL("foo", cfg->myField); + ASSERT_EQUAL("my", cfg->defName()); + ASSERT_FALSE(s.nextConfig(3000)); + writeFile("my.cfg", "bar"); + context->reload(); + bool correctValue = false; + FastOS_Time timer; + timer.SetNow(); + while (!correctValue && timer.MilliSecsToNow() < 20000.0) { + LOG(info, "Testing value..."); + if (s.nextConfig(1000)) { + break; + } + } + cfg = handle->getConfig(); + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL("bar", cfg->myField); + ASSERT_EQUAL("my", cfg->defName()); + ASSERT_FALSE(s.nextConfig(1000)); + } +} + +TEST("requireThatMultipleSubscribersCanSubscribeToSameFile") { + writeFile("my.cfg", "foobar"); + FileSpec spec("my.cfg"); + { + ConfigSubscriber s1(spec); + std::unique_ptr<ConfigHandle<MyConfig> > h1 = s1.subscribe<MyConfig>(""); + ASSERT_TRUE(s1.nextConfig(0)); + ConfigSubscriber s2(spec); + std::unique_ptr<ConfigHandle<MyConfig> > h2 = s2.subscribe<MyConfig>(""); + ASSERT_TRUE(s2.nextConfig(0)); + } +} + +TEST("requireThatCanSubscribeToDirectory") { + DirSpec spec("cfgdir"); + ConfigSubscriber s(spec); + ConfigHandle<FooConfig>::UP fooHandle = s.subscribe<FooConfig>(""); + ConfigHandle<BarConfig>::UP barHandle = s.subscribe<BarConfig>(""); + ASSERT_TRUE(s.nextConfig(0)); + ASSERT_TRUE(fooHandle->isChanged()); + ASSERT_TRUE(barHandle->isChanged()); + std::unique_ptr<FooConfig> fooCfg = fooHandle->getConfig(); + std::unique_ptr<BarConfig> barCfg = barHandle->getConfig(); + ASSERT_TRUE(fooCfg.get() != NULL); + ASSERT_TRUE(barCfg.get() != NULL); + ASSERT_EQUAL("foofoo", fooCfg->fooValue); + ASSERT_EQUAL("barbar", barCfg->barValue); +} + +TEST("requireThatCanSubscribeToDirectoryWithEmptyCfgFile") { + DirSpec spec("cfgemptyfile"); + ConfigSubscriber s(spec); + ConfigHandle<FoodefaultConfig>::UP fooHandle = s.subscribe<FoodefaultConfig>(""); + ConfigHandle<BarConfig>::UP barHandle = s.subscribe<BarConfig>(""); + ASSERT_TRUE(s.nextConfig(0)); + ASSERT_TRUE(fooHandle->isChanged()); + ASSERT_TRUE(barHandle->isChanged()); + std::unique_ptr<FoodefaultConfig> fooCfg = fooHandle->getConfig(); + std::unique_ptr<BarConfig> barCfg = barHandle->getConfig(); + ASSERT_TRUE(fooCfg.get() != NULL); + ASSERT_TRUE(barCfg.get() != NULL); + ASSERT_EQUAL("per", fooCfg->fooValue); + ASSERT_EQUAL("barbar", barCfg->barValue); +} + +TEST("requireThatCanSubscribeToDirectoryWithNonExistingCfgFile") { + DirSpec spec("cfgnonexistingfile"); + ConfigSubscriber s(spec); + ConfigHandle<FoodefaultConfig>::UP fooHandle = s.subscribe<FoodefaultConfig>(""); + ConfigHandle<BarConfig>::UP barHandle = s.subscribe<BarConfig>(""); + ASSERT_TRUE(s.nextConfig(0)); + ASSERT_TRUE(fooHandle->isChanged()); + ASSERT_TRUE(barHandle->isChanged()); + std::unique_ptr<FoodefaultConfig> fooCfg = fooHandle->getConfig(); + std::unique_ptr<BarConfig> barCfg = barHandle->getConfig(); + ASSERT_TRUE(fooCfg.get() != NULL); + ASSERT_TRUE(barCfg.get() != NULL); + ASSERT_EQUAL("per", fooCfg->fooValue); + ASSERT_EQUAL("barbar", barCfg->barValue); +} + +TEST_F("requireThatDirSpecDoesNotMixNames", DirSpec("cfgdir2")) { + ConfigSubscriber s(f); + ConfigHandle<BarConfig>::UP barHandle = s.subscribe<BarConfig>(""); + ConfigHandle<FoobarConfig>::UP foobarHandle = s.subscribe<FoobarConfig>(""); + s.nextConfig(0); + std::unique_ptr<BarConfig> bar = barHandle->getConfig(); + std::unique_ptr<FoobarConfig> foobar = foobarHandle->getConfig(); + ASSERT_TRUE(bar.get() != NULL); + ASSERT_TRUE(foobar.get() != NULL); + ASSERT_EQUAL("barbar", bar->barValue); + ASSERT_EQUAL("foobarlol", foobar->fooBarValue); +} + +TEST_F("require that can subscribe multiple config ids of same config", DirSpec("cfgdir3")) { + ConfigSubscriber s(f1); + ConfigHandle<BarConfig>::UP fooHandle = s.subscribe<BarConfig>("foo"); + ConfigHandle<BarConfig>::UP barHandle = s.subscribe<BarConfig>("bar"); + s.nextConfig(0); + std::unique_ptr<BarConfig> bar1 = fooHandle->getConfig(); + std::unique_ptr<BarConfig> bar2 = barHandle->getConfig(); + ASSERT_TRUE(bar1.get() != NULL); + ASSERT_TRUE(bar2.get() != NULL); + ASSERT_EQUAL("barbar", bar1->barValue); + ASSERT_EQUAL("foobarlol", bar2->barValue); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/file_subscription/my.cfg b/config/src/tests/file_subscription/my.cfg new file mode 100644 index 00000000000..6172609bdff --- /dev/null +++ b/config/src/tests/file_subscription/my.cfg @@ -0,0 +1 @@ +myField "foobar" diff --git a/config/src/tests/file_subscription/test1.cfg b/config/src/tests/file_subscription/test1.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/file_subscription/test1.cfg diff --git a/config/src/tests/frt/.gitignore b/config/src/tests/frt/.gitignore new file mode 100644 index 00000000000..1a2e9c8eb78 --- /dev/null +++ b/config/src/tests/frt/.gitignore @@ -0,0 +1,5 @@ +/config-my.cpp +/config-my.h +/config-bar.cpp +/config-bar.h +config_frt_test_app diff --git a/config/src/tests/frt/CMakeLists.txt b/config/src/tests/frt/CMakeLists.txt new file mode 100644 index 00000000000..96f0fbb0076 --- /dev/null +++ b/config/src/tests/frt/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_frt_test_app + SOURCES + frt.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_frt_test_app COMMAND config_frt_test_app) +vespa_generate_config(config_frt_test_app ../../test/resources/configdefinitions/my.def) +vespa_generate_config(config_frt_test_app ../../test/resources/configdefinitions/bar.def) diff --git a/config/src/tests/frt/frt.cpp b/config/src/tests/frt/frt.cpp new file mode 100644 index 00000000000..c8d367d1558 --- /dev/null +++ b/config/src/tests/frt/frt.cpp @@ -0,0 +1,590 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("frt"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/common/iconfigholder.h> +#include <vespa/config/common/trace.h> +#include <vespa/config/common/configdefinition.h> +#include <vespa/config/frt/connection.h> +#include <vespa/config/frt/frtsource.h> +#include <vespa/config/frt/frtconfigresponse.h> +#include <vespa/config/frt/frtconfigrequest.h> +#include <vespa/config/frt/frtconfigrequestv2.h> +#include <vespa/config/frt/frtconfigresponsev2.h> +#include <vespa/config/frt/frtconfigrequestv3.h> +#include <vespa/config/frt/frtconfigresponsev3.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/slime/json_format.h> +#include <vespa/fnet/fnet.h> +#include <vespa/fnet/frt/frt.h> +#include <vespa/fnet/frt/error.h> +#include <vespa/config/frt/protocol.h> +#include <lz4.h> +#include "config-my.h" +#include "config-bar.h" + + +using namespace config; +using namespace vespalib; +using namespace vespalib::slime; +using namespace config::protocol; +using namespace config::protocol::v2; +using namespace config::protocol::v3; + +namespace { + + struct UpdateFixture : public IConfigHolder { + ConfigUpdate::UP update; + bool notified; + + UpdateFixture() + : update(), + notified(false) + { } + ConfigUpdate::UP provide() { return ConfigUpdate::UP(); } + void handle(ConfigUpdate::UP u) { update = std::move(u); } + bool wait(int timeoutInMillis) { (void) timeoutInMillis; return notified; } + bool poll() { return notified; } + void interrupt() { } + + bool waitUntilResponse(int timeoutInMillis) + { + FastOS_Time timer; + timer.SetNow(); + while (timer.MilliSecsToNow() < timeoutInMillis) { + if (notified) + break; + FastOS_Thread::Sleep(100); + } + return notified; + } + }; + + struct RPCFixture + { + std::vector<FRT_RPCRequest *> requests; + FRT_RPCRequest * createEmptyRequest() { + FRT_RPCRequest * req = new FRT_RPCRequest(); + req->SetError(FRTE_NO_ERROR); + requests.push_back(req); + return req; + } + FRT_RPCRequest * createErrorRequest() { + FRT_RPCRequest * req = new FRT_RPCRequest(); + req->SetError(FRTE_RPC_ABORT); + requests.push_back(req); + return req; + } + FRT_RPCRequest * createOKResponse(const vespalib::string & defName="", + const vespalib::string & defMd5="", + const vespalib::string & configId="", + const vespalib::string & configMd5="", + int changed=0, + long generation=0, + const std::vector<vespalib::string> & payload = std::vector<vespalib::string>(), + const vespalib::string & ns = "") + { + FRT_RPCRequest * req = new FRT_RPCRequest(); + FRT_Values & ret = *req->GetReturn(); + + ret.AddString(defName.c_str()); + ret.AddString(""); + ret.AddString(defMd5.c_str()); + ret.AddString(configId.c_str()); + ret.AddString(configMd5.c_str()); + ret.AddInt32(changed); + ret.AddInt64(generation); + FRT_StringValue * payload_arr = ret.AddStringArray(payload.size()); + for (uint32_t i = 0; i < payload.size(); i++) { + ret.SetString(&payload_arr[i], payload[i].c_str()); + } + if (!ns.empty()) + ret.AddString(ns.c_str()); + req->SetError(FRTE_NO_ERROR); + requests.push_back(req); + return req; + } + + ~RPCFixture() { + for (size_t i = 0; i < requests.size(); i++) { + requests[i]->SubRef(); + } + } + }; + + + struct MyAbortHandler : public FRT_IAbortHandler + { + bool aborted; + MyAbortHandler() : aborted(false) { } + bool HandleAbort() { aborted = true; return true; } + }; + + struct ConnectionMock : public Connection { + int errorCode; + int timeout; + FRT_RPCRequest * ans; + FRT_Supervisor supervisor; + FNET_Scheduler scheduler; + vespalib::string address; + ConnectionMock(FRT_RPCRequest * answer = NULL) + : errorCode(0), + timeout(0), + ans(answer), + supervisor(), + address() + { } + FRT_RPCRequest * allocRPCRequest() { return supervisor.AllocRPCRequest(); } + void setError(int ec) { errorCode = ec; } + void invoke(FRT_RPCRequest * req, double t, FRT_IRequestWait * waiter) + { + timeout = static_cast<int>(t); + if (ans != NULL) + waiter->RequestDone(ans); + else + waiter->RequestDone(req); + } + const vespalib::string & getAddress() const { return address; } + void setTransientDelay(int64_t delay) { (void) delay; } + }; + + struct FactoryMock : public ConnectionFactory { + ConnectionMock * current; + FactoryMock(ConnectionMock * c) : current(c) { } + Connection * getCurrent() { + return current; + } + FNET_Scheduler * getScheduler() { return ¤t->scheduler; } + void syncTransport() { } + }; + + + struct AgentResultFixture + { + bool notified; + uint64_t waitTime; + uint64_t timeout; + ConfigState state; + AgentResultFixture(uint64_t w, uint64_t t) + : notified(false), + waitTime(w), + timeout(t), + state() + { } + }; + + struct AgentFixture : public ConfigAgent + { + AgentResultFixture * result; + + AgentFixture(AgentResultFixture * r) + : result(r) + { + } + + const ConfigState & getConfigState() const { return result->state; } + uint64_t getWaitTime () const { return result->waitTime; } + uint64_t getTimeout() const { return result->timeout; } + void handleResponse(const ConfigRequest & request, ConfigResponse::UP response) + { + (void) request; + (void) response; + result->notified = true; + } + void handleRequest(ConfigRequest::UP request) + { + (void) request; + } + bool abort() { return true; } + }; + + struct SourceFixture { + RPCFixture rpc; + ConnectionMock conn; + ConfigKey key; + SourceFixture() + : rpc(), + conn(rpc.createOKResponse("foo", "baz", "4", "boo")), + key("foo", "bar", "4", "boo") + { } + + }; + + struct FRTFixture + { + AgentResultFixture result; + FRTConfigRequestFactory requestFactory; + FRTSource src; + + FRTFixture(SourceFixture & f1) + : result(2000, 10000), + requestFactory(1, 3, VespaVersion::fromString("1.2.3"), CompressionType::UNCOMPRESSED), + src(ConnectionFactory::SP(new FactoryMock(&f1.conn)), + requestFactory, + ConfigAgent::UP(new AgentFixture(&result)), + f1.key) + { } + }; +} + + +TEST_F("require that empty config response does not validate", RPCFixture()) { + FRTConfigResponseV1 fail1(f1.createEmptyRequest()); + ASSERT_FALSE(fail1.validateResponse()); + ASSERT_FALSE(fail1.hasValidResponse()); + ASSERT_TRUE(fail1.isError()); +} + +TEST_F("require that response containing errors does not validate", RPCFixture()) { + FRTConfigResponseV1 fail1(f1.createErrorRequest()); + ASSERT_FALSE(fail1.validateResponse()); + ASSERT_FALSE(fail1.hasValidResponse()); + ASSERT_TRUE(fail1.isError()); + ASSERT_TRUE(fail1.errorCode() != 0); +} + +TEST_F("require that valid response validates", RPCFixture()) { + std::vector<vespalib::string> vec; + vec.push_back("bar \"foo\""); + FRTConfigResponseV1 ok(f1.createOKResponse("foo", "baz", "bim", "boo", 12, 15, vec, "mn")); + ASSERT_TRUE(ok.validateResponse()); + ASSERT_TRUE(ok.hasValidResponse()); +} + +TEST_F("require that response contains all values", RPCFixture()) { + FRTConfigResponseV1 ok(f1.createOKResponse("foo", "baz", "bim", "boo", 12, 15)); + ASSERT_FALSE(ok.validateResponse()); + ASSERT_FALSE(ok.hasValidResponse()); +} + +TEST_F("require that valid response returns values after fill", RPCFixture()) { + std::vector<vespalib::string> vec; + vec.push_back("bar \"foo\""); + FRTConfigResponseV1 ok(f1.createOKResponse("foo", "baz", "bim", "boo", 12, 15, vec, "mn")); + ASSERT_TRUE(ok.validateResponse()); + ASSERT_TRUE(ok.hasValidResponse()); + + // Should not be valid + ASSERT_TRUE(ConfigKey() == ok.getKey()); + ASSERT_TRUE(ConfigValue() == ok.getValue()); + + ok.fill(); + ConfigKey key(ok.getKey()); + ASSERT_EQUAL("foo", key.getDefName()); + ASSERT_EQUAL("baz", key.getDefMd5()); + ASSERT_EQUAL("bim", key.getConfigId()); + ASSERT_EQUAL("mn", key.getDefNamespace()); + + ConfigValue value(ok.getValue()); + ASSERT_TRUE(vec == value.getLines()); +} + +TEST("require that request parameters are correctly initialized") { + ConnectionMock conn; + std::vector<vespalib::string> schema; + schema.push_back("foo"); + schema.push_back("bar"); + ConfigKey key("foo", "bar", "bim", "boo", schema); + FRTConfigRequestV1 frtReq(key, &conn, "mymd5", 1337, 8); + + FRT_RPCRequest * req = frtReq.getRequest(); + FRT_Values & params(*req->GetParams()); + ASSERT_EQUAL("bar", std::string(params[0]._string._str)); + ASSERT_EQUAL("", std::string(params[1]._string._str)); + ASSERT_EQUAL("boo", std::string(params[2]._string._str)); + ASSERT_EQUAL("foo", std::string(params[3]._string._str)); + ASSERT_EQUAL("mymd5", std::string(params[4]._string._str)); + ASSERT_EQUAL(1337u, params[5]._intval64); + ASSERT_EQUAL(8u, params[6]._intval64); + ASSERT_EQUAL("bim", std::string(params[7]._string._str)); + ASSERT_EQUAL(2u, params[8]._string_array._len); + ASSERT_EQUAL("foo", std::string(params[8]._string_array._pt[0]._str)); + ASSERT_EQUAL("bar", std::string(params[8]._string_array._pt[1]._str)); + ASSERT_EQUAL(1u, params[9]._intval32); +} + +TEST("require that request is aborted") { + MyAbortHandler handler; + ConnectionMock conn; + ConfigKey key("foo", "bar", "bim", "boo"); + FRTConfigRequestV1 frtReq(key, &conn, "mymd5", 1337, 8); + frtReq.getRequest()->SetAbortHandler(&handler); + ASSERT_FALSE(frtReq.isAborted()); + ASSERT_TRUE(frtReq.abort()); +} + +TEST_FF("require that request is invoked", SourceFixture(), + FRTFixture(f1)) +{ + f2.result.state = ConfigState("foo", 3); + f2.src.getConfig(); + ASSERT_TRUE(f2.src.getCurrentRequest().verifyKey(f1.key)); + ASSERT_FALSE(f2.src.getCurrentRequest().verifyKey(ConfigKey("foo", "bal", "bim", "boo", std::vector<vespalib::string>()))); + ASSERT_FALSE(f2.src.getCurrentRequest().verifyState(ConfigState("foo", 0))); + ASSERT_FALSE(f2.src.getCurrentRequest().verifyState(ConfigState("foo", 1))); + ASSERT_FALSE(f2.src.getCurrentRequest().verifyState(ConfigState("bar", 1))); + ASSERT_TRUE(f2.src.getCurrentRequest().verifyState(ConfigState("foo", 3))); + ASSERT_TRUE(f2.result.notified); + f2.src.close(); +} + +TEST_FF("require that request is config task is scheduled", SourceFixture(), + FRTFixture(f1)) +{ + f2.src.getConfig(); + ASSERT_TRUE(f2.result.notified); + f2.result.notified = false; + FastOS_Time timer; + timer.SetNow(); + while (timer.MilliSecsToNow() < 10000) { + f1.conn.scheduler.CheckTasks(); + if (f2.result.notified) + break; + FastOS_Thread::Sleep(500); + } + ASSERT_TRUE(f2.result.notified); + f2.src.close(); +} + +TEST("require that v2 request is correctly initialized") { + ConnectionMock conn; + ConfigKey key = ConfigKey::create<MyConfig>("foobi"); + vespalib::string md5 = "mymd5"; + int64_t currentGeneration = 3; + int64_t wantedGeneration = 4; + vespalib::string hostName = "myhost"; + int64_t timeout = 3000; + Trace traceIn(3); + traceIn.trace(2, "Hei"); + FRTConfigRequestV2 v2req(&conn, key, md5, currentGeneration, wantedGeneration, hostName, timeout, traceIn); + ConfigDefinition origDef(MyConfig::CONFIG_DEF_SCHEMA); + + FRT_RPCRequest * req = v2req.getRequest(); + ASSERT_TRUE(req != NULL); + FRT_Values & params(*req->GetParams()); + std::string json(params[0]._string._str); + Slime slime; + JsonFormat::decode(Memory(json), slime); + Inspector & root(slime.get()); + EXPECT_EQUAL(2, root[REQUEST_VERSION].asLong()); + EXPECT_EQUAL(key.getDefName(), root[REQUEST_DEF_NAME].asString().make_string()); + EXPECT_EQUAL(key.getDefNamespace(), root[REQUEST_DEF_NAMESPACE].asString().make_string()); + EXPECT_EQUAL(key.getDefMd5(), root[REQUEST_DEF_MD5].asString().make_string()); + EXPECT_EQUAL(key.getConfigId(), root[REQUEST_CLIENT_CONFIGID].asString().make_string()); + EXPECT_EQUAL(hostName, root[REQUEST_CLIENT_HOSTNAME].asString().make_string()); + EXPECT_EQUAL(currentGeneration, root[REQUEST_CURRENT_GENERATION].asLong()); + EXPECT_EQUAL(wantedGeneration, root[REQUEST_WANTED_GENERATION].asLong()); + EXPECT_EQUAL(md5, root[REQUEST_CONFIG_MD5].asString().make_string()); + EXPECT_EQUAL(timeout, root[REQUEST_TIMEOUT].asLong()); + Trace trace; + trace.deserialize(root[REQUEST_TRACE]); + EXPECT_TRUE(trace.shouldTrace(2)); + EXPECT_TRUE(trace.shouldTrace(3)); + EXPECT_FALSE(trace.shouldTrace(4)); + EXPECT_EQUAL(timeout, root[REQUEST_TIMEOUT].asLong()); + ConfigDefinition def; + def.deserialize(root[REQUEST_DEF_CONTENT]); + EXPECT_EQUAL(origDef.asString(), def.asString()); + ConfigResponse::UP response(v2req.createResponse(req)); + req->GetReturn()->AddString("foobar"); + EXPECT_TRUE(response->validateResponse()); +} + +TEST("require that v2 reponse is correctly initialized") { + ConnectionMock conn; + Slime slime; + ConfigKey key = ConfigKey::create<MyConfig>("foobi"); + vespalib::string md5 = "mymd5"; + int64_t generation = 3; + vespalib::string hostname = "myhhost"; + Trace traceIn(3); + traceIn.trace(2, "Hei!"); + Cursor & root(slime.setObject()); + root.setLong(RESPONSE_VERSION, 2ul); + root.setString(RESPONSE_DEF_NAME, Memory(key.getDefName())); + root.setString(RESPONSE_DEF_NAMESPACE, Memory(key.getDefNamespace())); + root.setString(RESPONSE_DEF_MD5, Memory(key.getDefMd5())); + root.setString(RESPONSE_CONFIGID, Memory(key.getConfigId())); + root.setString(RESPONSE_CLIENT_HOSTNAME, Memory(hostname)); + root.setString(RESPONSE_CONFIG_MD5, Memory(md5)); + root.setLong(RESPONSE_CONFIG_GENERATION, generation); + traceIn.serialize(root.setObject(RESPONSE_TRACE)); + Cursor & payload(root.setObject(RESPONSE_PAYLOAD)); + payload.setString("myField", "foobiar"); + SimpleBuffer buf; + JsonFormat::encode(slime, buf, true); + FRT_RPCRequest * req = conn.allocRPCRequest(); + req->GetReturn()->AddString(buf.get().make_string().c_str()); + FRTConfigResponseV2 response(req); + ASSERT_TRUE(response.validateResponse()); + response.fill(); + Trace trace(response.getTrace()); + EXPECT_TRUE(trace.shouldTrace(3)); + EXPECT_FALSE(trace.shouldTrace(4)); + ConfigKey responseKey(response.getKey()); + EXPECT_EQUAL(key.getDefName(), responseKey.getDefName()); + EXPECT_EQUAL(key.getDefNamespace(), responseKey.getDefNamespace()); + EXPECT_EQUAL(key.getDefMd5(), responseKey.getDefMd5()); + EXPECT_EQUAL(key.getConfigId(), responseKey.getConfigId()); + EXPECT_EQUAL(hostname, response.getHostName()); + ConfigState state(response.getConfigState()); + EXPECT_EQUAL(md5, state.md5); + EXPECT_EQUAL(generation, state.generation); + ConfigValue value(response.getValue()); + MyConfig::UP config(value.newInstance<MyConfig>()); + EXPECT_EQUAL("foobiar", config->myField); + req->SubRef(); +} + +TEST("require that v3 request is correctly initialized") { + ConnectionMock conn; + ConfigKey key = ConfigKey::create<MyConfig>("foobi"); + vespalib::string md5 = "mymd5"; + int64_t currentGeneration = 3; + int64_t wantedGeneration = 4; + vespalib::string hostName = "myhost"; + int64_t timeout = 3000; + Trace traceIn(3); + traceIn.trace(2, "Hei"); + FRTConfigRequestV3 v3req(&conn, key, md5, currentGeneration, wantedGeneration, hostName, timeout, traceIn, VespaVersion::fromString("1.2.3"), CompressionType::LZ4); + ConfigDefinition origDef(MyConfig::CONFIG_DEF_SCHEMA); + + FRT_RPCRequest * req = v3req.getRequest(); + ASSERT_TRUE(req != NULL); + FRT_Values & params(*req->GetParams()); + std::string json(params[0]._string._str); + Slime slime; + JsonFormat::decode(Memory(json), slime); + Inspector & root(slime.get()); + EXPECT_EQUAL(3, root[REQUEST_VERSION].asLong()); + EXPECT_EQUAL(key.getDefName(), root[REQUEST_DEF_NAME].asString().make_string()); + EXPECT_EQUAL(key.getDefNamespace(), root[REQUEST_DEF_NAMESPACE].asString().make_string()); + EXPECT_EQUAL(key.getDefMd5(), root[REQUEST_DEF_MD5].asString().make_string()); + EXPECT_EQUAL(key.getConfigId(), root[REQUEST_CLIENT_CONFIGID].asString().make_string()); + EXPECT_EQUAL(hostName, root[REQUEST_CLIENT_HOSTNAME].asString().make_string()); + EXPECT_EQUAL(currentGeneration, root[REQUEST_CURRENT_GENERATION].asLong()); + EXPECT_EQUAL(wantedGeneration, root[REQUEST_WANTED_GENERATION].asLong()); + EXPECT_EQUAL(md5, root[REQUEST_CONFIG_MD5].asString().make_string()); + EXPECT_EQUAL(timeout, root[REQUEST_TIMEOUT].asLong()); + EXPECT_EQUAL("LZ4", root[REQUEST_COMPRESSION_TYPE].asString().make_string()); + EXPECT_EQUAL(root[REQUEST_VESPA_VERSION].asString().make_string(), "1.2.3"); + Trace trace; + trace.deserialize(root[REQUEST_TRACE]); + EXPECT_TRUE(trace.shouldTrace(2)); + EXPECT_TRUE(trace.shouldTrace(3)); + EXPECT_FALSE(trace.shouldTrace(4)); + EXPECT_EQUAL(timeout, root[REQUEST_TIMEOUT].asLong()); + ConfigDefinition def; + def.deserialize(root[REQUEST_DEF_CONTENT]); + EXPECT_EQUAL(origDef.asString(), def.asString()); + ConfigResponse::UP response(v3req.createResponse(req)); + req->GetReturn()->AddString("foobar"); + req->GetReturn()->AddData("foo", 3); + EXPECT_TRUE(response->validateResponse()); +} + +struct V3RequestFixture { + ConnectionMock conn; + Slime slime; + Cursor & root; + FRT_RPCRequest * req; + ConfigKey key; + vespalib::string md5; + int64_t generation; + vespalib::string hostname; + Trace traceIn; + + V3RequestFixture() + : conn(), + slime(), + root(slime.setObject()), + req(conn.allocRPCRequest()), + key(ConfigKey::create<BarConfig>("foobi")), + md5("mymd5"), + generation(3), + hostname("myhhost"), + traceIn(3) + { + traceIn.trace(2, "Hei!"); + root.setLong(RESPONSE_VERSION, 3ul); + root.setString(RESPONSE_DEF_NAME, Memory(key.getDefName())); + root.setString(RESPONSE_DEF_NAMESPACE, Memory(key.getDefNamespace())); + root.setString(RESPONSE_DEF_MD5, Memory(key.getDefMd5())); + root.setString(RESPONSE_CONFIGID, Memory(key.getConfigId())); + root.setString(RESPONSE_CLIENT_HOSTNAME, Memory(hostname)); + root.setString(RESPONSE_CONFIG_MD5, Memory(md5)); + root.setLong(RESPONSE_CONFIG_GENERATION, generation); + traceIn.serialize(root.setObject(RESPONSE_TRACE)); + } + + ~V3RequestFixture() { + req->SubRef(); + } + + void encodePayload(const char * payload, uint32_t payloadSize, uint32_t uncompressedSize, const CompressionType & compressionType) { + Cursor & compressionInfo(root.setObject(RESPONSE_COMPRESSION_INFO)); + compressionInfo.setString("compressionType", Memory(compressionTypeToString(compressionType))); + compressionInfo.setLong("uncompressedSize", uncompressedSize); + SimpleBuffer buf; + JsonFormat::encode(slime, buf, true); + req->GetReturn()->AddString(buf.get().make_string().c_str()); + req->GetReturn()->AddData(payload, payloadSize); + } + + FRTConfigResponseV3 * createResponse() { + return new FRTConfigResponseV3(req); + } + + void assertResponse(const FRTConfigResponseV3 & response, const char *expectedValue) { + Trace trace(response.getTrace()); + EXPECT_TRUE(trace.shouldTrace(3)); + EXPECT_FALSE(trace.shouldTrace(4)); + ConfigKey responseKey(response.getKey()); + EXPECT_EQUAL(key.getDefName(), responseKey.getDefName()); + EXPECT_EQUAL(key.getDefNamespace(), responseKey.getDefNamespace()); + EXPECT_EQUAL(key.getDefMd5(), responseKey.getDefMd5()); + EXPECT_EQUAL(key.getConfigId(), responseKey.getConfigId()); + EXPECT_EQUAL(hostname, response.getHostName()); + ConfigState state(response.getConfigState()); + EXPECT_EQUAL(md5, state.md5); + EXPECT_EQUAL(generation, state.generation); + ConfigValue value(response.getValue()); + BarConfig::UP config(value.newInstance<BarConfig>()); + EXPECT_EQUAL(expectedValue, config->barValue); + } +}; + +TEST_F("require that v3 uncompressed reponse is correctly initialized", V3RequestFixture()) { + const char *payload = "{\"barValue\":\"foobiar\"}"; + f1.encodePayload(payload, strlen(payload), strlen(payload), CompressionType::UNCOMPRESSED); + std::unique_ptr<FRTConfigResponseV3> response(f1.createResponse()); + ASSERT_TRUE(response->validateResponse()); + response->fill(); + f1.assertResponse(*response, "foobiar"); +} + +TEST_F("require that v3 compressed reponse is correctly initialized", V3RequestFixture()) { + const char *payload = "{\"barValue\":\"foobiar\"}"; + int maxSize = LZ4_compressBound(strlen(payload)); + char *output = (char *)malloc(maxSize); + int sz = LZ4_compress(payload, output, strlen(payload)); + + f1.encodePayload(output, sz, strlen(payload), CompressionType::LZ4); + std::unique_ptr<FRTConfigResponseV3> response(f1.createResponse()); + ASSERT_TRUE(response->validateResponse()); + response->fill(); + f1.assertResponse(*response, "foobiar"); + free(output); +} + +TEST_F("require that empty v3 reponse is correctly initialized", V3RequestFixture()) { + const char *payload = ""; + f1.encodePayload(payload, strlen(payload), strlen(payload), CompressionType::UNCOMPRESSED); + std::unique_ptr<FRTConfigResponseV3> response(f1.createResponse()); + ASSERT_TRUE(response->validateResponse()); + response->fill(); + f1.assertResponse(*response, "defaultBar"); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/frtconnectionpool/.gitignore b/config/src/tests/frtconnectionpool/.gitignore new file mode 100644 index 00000000000..8c5f46e1254 --- /dev/null +++ b/config/src/tests/frtconnectionpool/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +frtconnectionpool_test +config_frtconnectionpool_test_app diff --git a/config/src/tests/frtconnectionpool/CMakeLists.txt b/config/src/tests/frtconnectionpool/CMakeLists.txt new file mode 100644 index 00000000000..1979577f5fb --- /dev/null +++ b/config/src/tests/frtconnectionpool/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_frtconnectionpool_test_app + SOURCES + frtconnectionpool.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_frtconnectionpool_test_app COMMAND config_frtconnectionpool_test_app) diff --git a/config/src/tests/frtconnectionpool/DESC b/config/src/tests/frtconnectionpool/DESC new file mode 100644 index 00000000000..141c067238f --- /dev/null +++ b/config/src/tests/frtconnectionpool/DESC @@ -0,0 +1 @@ +frtconnectionpool test. Take a look at frtconnectionpool.cpp for details. diff --git a/config/src/tests/frtconnectionpool/FILES b/config/src/tests/frtconnectionpool/FILES new file mode 100644 index 00000000000..d0bcc084a30 --- /dev/null +++ b/config/src/tests/frtconnectionpool/FILES @@ -0,0 +1 @@ +frtconnectionpool.cpp diff --git a/config/src/tests/frtconnectionpool/frtconnectionpool.cpp b/config/src/tests/frtconnectionpool/frtconnectionpool.cpp new file mode 100644 index 00000000000..93163065eae --- /dev/null +++ b/config/src/tests/frtconnectionpool/frtconnectionpool.cpp @@ -0,0 +1,247 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("frtconnectionpool_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/config/frt/frtconnectionpool.h> +#include <vespa/fnet/frt/error.h> +#include <sstream> +#include <set> + +using namespace config; + +class Test : public vespalib::TestApp { +private: + static ServerSpec::HostSpecList _sources; + void verifyAllSourcesInRotation(FRTConnectionPool& sourcePool); +public: + int Main(); + void testBasicRoundRobin(); + void testBasicHashBasedSelection(); + void testSetErrorRoundRobin(); + void testSetErrorAllRoundRobin(); + void testSetErrorHashBased(); + void testSetErrorAllHashBased(); + void testSuspensionTimeout(); + void testManySources(); +}; + +TEST_APPHOOK(Test); + +ServerSpec::HostSpecList Test::_sources; +TimingValues timingValues; + +int Test::Main() { + TEST_INIT("frtconnectionpool_test"); + + _sources.push_back("host0"); + _sources.push_back("host1"); + _sources.push_back("host2"); + + testBasicRoundRobin(); + TEST_FLUSH(); + + testBasicHashBasedSelection(); + TEST_FLUSH(); + + testSetErrorRoundRobin(); + TEST_FLUSH(); + + testSetErrorAllRoundRobin(); + TEST_FLUSH(); + + testSetErrorHashBased(); + TEST_FLUSH(); + + testSetErrorAllHashBased(); + TEST_FLUSH(); + + testSuspensionTimeout(); + TEST_FLUSH(); + + testManySources(); + TEST_FLUSH(); + + TEST_DONE(); + return 0; +} + +void Test::verifyAllSourcesInRotation(FRTConnectionPool& sourcePool) { + std::set<std::string> completeSet(_sources.begin(), _sources.end()); + std::set<std::string> foundSet; + for (int i = 0; i < (int)_sources.size(); i++) { + foundSet.insert(sourcePool.getNextRoundRobin()->getAddress()); + } + EXPECT_EQUAL(true, completeSet == foundSet); +} + +/** + * Tests that basic round robin selection through the list works. + */ +void Test::testBasicRoundRobin() { + const ServerSpec spec(_sources); + FRTConnectionPool sourcePool(spec, timingValues); + for (int i = 0; i < 9; i++) { + int j = i % _sources.size(); + std::stringstream s; + s << "host" << j; + EXPECT_EQUAL(s.str(), sourcePool.getNextRoundRobin()->getAddress()); + } +} + +/** + * Tests that hash-based selection through the list works. + */ +void Test::testBasicHashBasedSelection() { + const ServerSpec spec(_sources); + FRTConnectionPool sourcePool(spec, timingValues); + sourcePool.setHostname("a.b.com"); + for (int i = 0; i < 9; i++) { + EXPECT_EQUAL("host1", sourcePool.getNextHashBased()->getAddress()); + } + sourcePool.setHostname("host98"); + for (int i = 0; i < 9; i++) { + EXPECT_EQUAL("host0", sourcePool.getNextHashBased()->getAddress()); + } + + ServerSpec::HostSpecList hostnames; + hostnames.push_back("sutter-01.example.yahoo.com"); + hostnames.push_back("stroustrup-02.example.yahoo.com"); + hostnames.push_back("alexandrescu-03.example.yahoo.com"); + const ServerSpec spec2(hostnames); + FRTConnectionPool sourcePool2(spec2, timingValues); + sourcePool2.setHostname("sutter-01.example.yahoo.com"); + EXPECT_EQUAL("stroustrup-02.example.yahoo.com", sourcePool2.getNextHashBased()->getAddress()); + sourcePool2.setHostname("stroustrup-02.example.yahoo.com"); + EXPECT_EQUAL("sutter-01.example.yahoo.com", sourcePool2.getNextHashBased()->getAddress()); + sourcePool2.setHostname("alexandrescu-03.example.yahoo.com"); + EXPECT_EQUAL("alexandrescu-03.example.yahoo.com", sourcePool2.getNextHashBased()->getAddress()); +} + +/** + * Tests that a source is taken out of rotation when an error is reported, + * and that it is taken back in when a success is reported. + */ +void Test::testSetErrorRoundRobin() { + const ServerSpec spec(_sources); + FRTConnectionPool sourcePool(spec, timingValues); + FRTConnection* source = sourcePool.getNextRoundRobin(); + source->setError(FRTE_RPC_CONNECTION); + for (int i = 0; i < 9; i++) { + EXPECT_NOT_EQUAL(source->getAddress(), sourcePool.getCurrent()->getAddress()); + } + source->setSuccess(); + verifyAllSourcesInRotation(sourcePool); +} + +/** + * Tests that all sources are in rotation when all sources have errors set. + */ +void Test::testSetErrorAllRoundRobin() { + const ServerSpec spec(_sources); + FRTConnectionPool sourcePool(spec, timingValues); + for (int i = 0; i < (int)_sources.size(); i++) { + FRTConnection* source = sourcePool.getNextRoundRobin(); + source->setError(FRTE_RPC_CONNECTION); + } + verifyAllSourcesInRotation(sourcePool); +} + +/** + * Tests that a source is not used when an error is reported, + * and that the same source is used when a success is reported. + */ +void Test::testSetErrorHashBased() { + const ServerSpec spec(_sources); + FRTConnectionPool sourcePool(spec, timingValues); + FRTConnection* source = sourcePool.getNextHashBased(); + source->setError(FRTE_RPC_CONNECTION); + for (int i = 0; i < (int)_sources.size(); i++) { + EXPECT_NOT_EQUAL(source->getAddress(), sourcePool.getNextHashBased()->getAddress()); + } + source->setSuccess(); + EXPECT_EQUAL(source->getAddress(), sourcePool.getNextHashBased()->getAddress()); +} + +/** + * Tests that the same source is used when all sources have errors set. + */ +void Test::testSetErrorAllHashBased() { + const ServerSpec spec(_sources); + FRTConnectionPool sourcePool(spec, timingValues); + FRTConnection* firstSource = sourcePool.getNextHashBased(); + std::vector<FRTConnection*> readySources; + sourcePool.getReadySources(readySources); + for (int i = 0; i < (int)readySources.size(); i++) { + readySources[i]->setError(FRTE_RPC_CONNECTION); + } + std::vector<FRTConnection*> tmpSources; + EXPECT_EQUAL((int)sourcePool.getReadySources(tmpSources).size(), 0); + EXPECT_EQUAL((int)sourcePool.getSuspendedSources(tmpSources).size(), 3); + + // should get the same source now, since all are suspended + EXPECT_EQUAL(firstSource->getAddress(), sourcePool.getNextHashBased()->getAddress()); + + // set all except firstSource to OK + for (int i = 0; i < (int)readySources.size(); i++) { + if (readySources[i]->getAddress() != firstSource->getAddress()) { + readySources[i]->setSuccess(); + } + } + + EXPECT_EQUAL((int)sourcePool.getReadySources(tmpSources).size(), 2); + EXPECT_EQUAL((int)sourcePool.getSuspendedSources(tmpSources).size(), 1); + + // should not get the same source now, since original source is + // suspended, while the rest are OK + EXPECT_NOT_EQUAL(firstSource->getAddress(), sourcePool.getNextHashBased()->getAddress()); +} + +/** + * Tests that the source is put back into rotation when the suspension times out. + */ +void Test::testSuspensionTimeout() { + const ServerSpec spec(_sources); + FRTConnectionPool sourcePool(spec, timingValues); + Connection* source = sourcePool.getCurrent(); + source->setTransientDelay(1000); + source->setError(FRTE_RPC_CONNECTION); + for (int i = 0; i < 9; i++) { + EXPECT_NOT_EQUAL(source->getAddress(), sourcePool.getCurrent()->getAddress()); + } + sleep(2); + verifyAllSourcesInRotation(sourcePool); +} + +/** + * Tests that when there are two sources and several clients + * the sources will be chosen with equal probability. + */ +void Test::testManySources() { + std::vector<std::string> hostnames; + for (int i = 0; i < 20; ++i) { + hostnames.push_back("host-" + std::to_string(i) + ".example.yahoo.com"); + } + + std::map<std::string, int> timesUsed; + ServerSpec::HostSpecList twoSources; + + twoSources.push_back("host0"); + twoSources.push_back("host1"); + + const ServerSpec spec(twoSources); + FRTConnectionPool sourcePool(spec, timingValues); + + for (int i = 0; i < (int)hostnames.size(); i++) { + sourcePool.setHostname(hostnames[i]); + std::string address = sourcePool.getNextHashBased()->getAddress(); + if (timesUsed.find(address) != timesUsed.end()) { + int times = timesUsed[address]; + timesUsed[address] = times + 1; + } else { + timesUsed[address] = 1; + } + } + EXPECT_EQUAL(timesUsed["host0"], (int)hostnames.size() / 2); + EXPECT_EQUAL(timesUsed["host1"], (int)hostnames.size() / 2); +} diff --git a/config/src/tests/functiontest/.gitignore b/config/src/tests/functiontest/.gitignore new file mode 100644 index 00000000000..44d74bb5ec7 --- /dev/null +++ b/config/src/tests/functiontest/.gitignore @@ -0,0 +1,6 @@ +.depend +Makefile +functiontesttest_test +/config-function-test.cpp +/config-function-test.h +config_functiontest_test_app diff --git a/config/src/tests/functiontest/CMakeLists.txt b/config/src/tests/functiontest/CMakeLists.txt new file mode 100644 index 00000000000..d798ad53de4 --- /dev/null +++ b/config/src/tests/functiontest/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_functiontest_test_app + SOURCES + functiontest.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_functiontest_test_app COMMAND config_functiontest_test_app) +vespa_generate_config(config_functiontest_test_app ../../test/resources/configdefinitions/function-test.def) diff --git a/config/src/tests/functiontest/defaultvalues.xml b/config/src/tests/functiontest/defaultvalues.xml new file mode 100644 index 00000000000..80644c2fdb1 --- /dev/null +++ b/config/src/tests/functiontest/defaultvalues.xml @@ -0,0 +1,62 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<config name="function-test"> + <bool_val>false</bool_val> + <int_val>5</int_val> + <long_val>1234567890123</long_val> + <double_val>41.23</double_val> + <string_val>foo</string_val> + <enum_val>FOOBAR</enum_val> + <refval>:parent:</refval> + <fileVal>vespa.log</fileVal> + + <boolarr index="0">false</boolarr> + <doublearr index="0">2344</doublearr> + <doublearr index="1">123</doublearr> + <stringarr index="0">bar</stringarr> + <enumarr index="0">VALUES</enumarr> + + <basicStruct> + <bar>3</bar> + <intArr index="0">10</intArr> + </basicStruct> + + <rootStruct> + <inner0> + <index>11</index> + </inner0> + <inner1> + <index>12</index> + </inner1> + <innerArr index="0"> + <stringVal>deep</stringVal> + </innerArr> + </rootStruct> + + <myarray index="0"> + <stringval index="0">baah</stringval> + <stringval index="1">yikes</stringval> + <refval>:parent:</refval> + <fileVal>command.com</fileVal> + <anotherarray index="0"> + <foo>7</foo> + </anotherarray> + <myStruct> + <a>1</a> + </myStruct> + </myarray> + <myarray index="1"> + <refval>:parent:</refval> + <fileVal>display.sys</fileVal> + <anotherarray index="0"> + <foo>1</foo> + </anotherarray> + <anotherarray index="1"> + <foo>2</foo> + </anotherarray> + <myStruct> + <a>-1</a> + </myStruct> + </myarray> + +</config> diff --git a/config/src/tests/functiontest/defaultvalues/function-test.cfg b/config/src/tests/functiontest/defaultvalues/function-test.cfg new file mode 100644 index 00000000000..854d2a5a1cb --- /dev/null +++ b/config/src/tests/functiontest/defaultvalues/function-test.cfg @@ -0,0 +1,46 @@ +bool_val false +int_val 5 +long_val 1234567890123 +double_val 41.23 +string_val "foo" +enum_val FOOBAR +refval :parent: +fileVal "vespa.log" +boolarr[1] +boolarr[0] false +intarr[0] +longarr[0] +doublearr[2] +doublearr[0] 2344 +doublearr[1] 123 +stringarr[1] +stringarr[0] "bar" +enumarr[1] +enumarr[0] VALUES +refarr[0] +fileArr[0] + +basicStruct.bar 3 +basicStruct.intArr[1] +basicStruct.intArr[0] 10 +rootStruct.inner0.index 11 +rootStruct.inner1.index 12 +rootStruct.innerArr[1] +rootStruct.innerArr[0].stringVal "deep" + +myarray[2] +myarray[0].stringval[2] +myarray[0].stringval[0] "baah" +myarray[0].stringval[1] "yikes" +myarray[0].refval ":parent:" +myarray[0].fileVal "command.com" +myarray[0].anotherarray[1] +myarray[0].anotherarray[0].foo 7 +myarray[0].myStruct.a 1 +myarray[1].stringval[0] +myarray[1].refval ":parent:" +myarray[1].fileVal "display.sys" +myarray[1].anotherarray[2] +myarray[1].anotherarray[0].foo 1 +myarray[1].anotherarray[1].foo 2 +myarray[1].myStruct.a -1 diff --git a/config/src/tests/functiontest/errorval_double/function-test.cfg b/config/src/tests/functiontest/errorval_double/function-test.cfg new file mode 100644 index 00000000000..d6fff604406 --- /dev/null +++ b/config/src/tests/functiontest/errorval_double/function-test.cfg @@ -0,0 +1,67 @@ +bool_val false +bool_with_def true +int_val 5 +int_with_def -14 +long_val 12345678901 +long_with_def -9876543210 +double_val 41.23jf +double_with_def -12 +string_val "foo" +stringwithdef "bar" +enum_val FOOBAR +enumwithdef BAR2 +refval :parent: +refwithdef ":parent:" +fileVal "etc" +boolarr[1] +boolarr[0] false +intarr[0] +longarr[2] +longarr[0] 9223372036854775807 +longarr[1] -9223372036854775808 +doublearr[2] +doublearr[0] 2344 +doublearr[1] 123 +stringarr[1] +stringarr[0] "bar" +enumarr[1] +enumarr[0] VALUES +refarr[3] +refarr[0] ":parent:" +refarr[1] ":parent" +refarr[2] "parent:" +fileArr[1] +fileArr[0] "bin" + +basicStruct.foo "basicFoo" +basicStruct.bar 3 +basicStruct.intArr[1] +basicStruct.intArr[0] 310 +rootStruct.inner0.index 11 +rootStruct.inner1.index 12 +rootStruct.innerArr[1] +rootStruct.innerArr[0].boolVal true +rootStruct.innerArr[0].stringVal "deep" + +myarray[2] +myarray[0].intval -5 +myarray[0].stringval[2] +myarray[0].stringval[0] "baah" +myarray[0].stringval[1] "yikes" +myarray[0].enumval INNER +myarray[0].refval :parent: +myarray[0].fileVal "file0" +myarray[0].anotherarray[1] +myarray[0].anotherarray[0].foo 7 +myarray[0].myStruct.a 1 +myarray[0].myStruct.b 2 +myarray[1].intval 5 +myarray[1].stringval[0] +myarray[1].enumval INNER +myarray[1].refval ":parent:" +myarray[1].fileVal "file1" +myarray[1].anotherarray[2] +myarray[1].anotherarray[0].foo 1 +myarray[1].anotherarray[1].foo 2 +myarray[1].myStruct.a -1 +myarray[1].myStruct.b -2 diff --git a/config/src/tests/functiontest/errorval_int/function-test.cfg b/config/src/tests/functiontest/errorval_int/function-test.cfg new file mode 100644 index 00000000000..7f3eba1ba55 --- /dev/null +++ b/config/src/tests/functiontest/errorval_int/function-test.cfg @@ -0,0 +1,67 @@ +bool_val false +bool_with_def true +int_val 5foo +int_with_def -14 +long_val 12345678901 +long_with_def -9876543210 +double_val 41.23 +double_with_def -12 +string_val "foo" +stringwithdef "bar" +enum_val FOOBAR +enumwithdef BAR2 +refval :parent: +refwithdef ":parent:" +fileVal "etc" +boolarr[1] +boolarr[0] false +intarr[0] +longarr[2] +longarr[0] 9223372036854775807 +longarr[1] -9223372036854775808 +doublearr[2] +doublearr[0] 2344 +doublearr[1] 123 +stringarr[1] +stringarr[0] "bar" +enumarr[1] +enumarr[0] VALUES +refarr[3] +refarr[0] ":parent:" +refarr[1] ":parent" +refarr[2] "parent:" +fileArr[1] +fileArr[0] "bin" + +basicStruct.foo "basicFoo" +basicStruct.bar 3 +basicStruct.intArr[1] +basicStruct.intArr[0] 310 +rootStruct.inner0.index 11 +rootStruct.inner1.index 12 +rootStruct.innerArr[1] +rootStruct.innerArr[0].boolVal true +rootStruct.innerArr[0].stringVal "deep" + +myarray[2] +myarray[0].intval -5 +myarray[0].stringval[2] +myarray[0].stringval[0] "baah" +myarray[0].stringval[1] "yikes" +myarray[0].enumval INNER +myarray[0].refval :parent: +myarray[0].fileVal "file0" +myarray[0].anotherarray[1] +myarray[0].anotherarray[0].foo 7 +myarray[0].myStruct.a 1 +myarray[0].myStruct.b 2 +myarray[1].intval 5 +myarray[1].stringval[0] +myarray[1].enumval INNER +myarray[1].refval ":parent:" +myarray[1].fileVal "file1" +myarray[1].anotherarray[2] +myarray[1].anotherarray[0].foo 1 +myarray[1].anotherarray[1].foo 2 +myarray[1].myStruct.a -1 +myarray[1].myStruct.b -2 diff --git a/config/src/tests/functiontest/errorval_long/function-test.cfg b/config/src/tests/functiontest/errorval_long/function-test.cfg new file mode 100644 index 00000000000..ab14a1a9e5a --- /dev/null +++ b/config/src/tests/functiontest/errorval_long/function-test.cfg @@ -0,0 +1,67 @@ +bool_val false +bool_with_def true +int_val 5 +int_with_def -14 +long_val 12345678901foo +long_with_def -9876543210 +double_val 41.23 +double_with_def -12 +string_val "foo" +stringwithdef "bar" +enum_val FOOBAR +enumwithdef BAR2 +refval :parent: +refwithdef ":parent:" +fileVal "etc" +boolarr[1] +boolarr[0] false +intarr[0] +longarr[2] +longarr[0] 9223372036854775807 +longarr[1] -9223372036854775808 +doublearr[2] +doublearr[0] 2344 +doublearr[1] 123 +stringarr[1] +stringarr[0] "bar" +enumarr[1] +enumarr[0] VALUES +refarr[3] +refarr[0] ":parent:" +refarr[1] ":parent" +refarr[2] "parent:" +fileArr[1] +fileArr[0] "bin" + +basicStruct.foo "basicFoo" +basicStruct.bar 3 +basicStruct.intArr[1] +basicStruct.intArr[0] 310 +rootStruct.inner0.index 11 +rootStruct.inner1.index 12 +rootStruct.innerArr[1] +rootStruct.innerArr[0].boolVal true +rootStruct.innerArr[0].stringVal "deep" + +myarray[2] +myarray[0].intval -5 +myarray[0].stringval[2] +myarray[0].stringval[0] "baah" +myarray[0].stringval[1] "yikes" +myarray[0].enumval INNER +myarray[0].refval :parent: +myarray[0].fileVal "file0" +myarray[0].anotherarray[1] +myarray[0].anotherarray[0].foo 7 +myarray[0].myStruct.a 1 +myarray[0].myStruct.b 2 +myarray[1].intval 5 +myarray[1].stringval[0] +myarray[1].enumval INNER +myarray[1].refval ":parent:" +myarray[1].fileVal "file1" +myarray[1].anotherarray[2] +myarray[1].anotherarray[0].foo 1 +myarray[1].anotherarray[1].foo 2 +myarray[1].myStruct.a -1 +myarray[1].myStruct.b -2 diff --git a/config/src/tests/functiontest/functiontest.cpp b/config/src/tests/functiontest/functiontest.cpp new file mode 100644 index 00000000000..6541e76e06a --- /dev/null +++ b/config/src/tests/functiontest/functiontest.cpp @@ -0,0 +1,304 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/config/config.h> +#include "config-function-test.h" + +#include <fstream> +#include <vespa/log/log.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/data/slime/slime.h> + +LOG_SETUP("functiontest_test"); + +using namespace config; + +namespace { + +void +checkVariableAccess(const FunctionTestConfig & config) +{ + EXPECT_EQUAL(false, config.boolVal); + EXPECT_EQUAL(true, config.boolWithDef); + EXPECT_EQUAL(5, config.intVal); + EXPECT_EQUAL(-14, config.intWithDef); + EXPECT_EQUAL(12345678901LL, config.longVal); + EXPECT_EQUAL(-9876543210LL, config.longWithDef); + EXPECT_APPROX(41.23, config.doubleVal, 0.000001); + EXPECT_APPROX(-12, config.doubleWithDef, 0.000001); + EXPECT_EQUAL("foo", config.stringVal); + EXPECT_EQUAL("bar", config.stringwithdef); + EXPECT_EQUAL("FOOBAR", FunctionTestConfig::getEnumValName(config.enumVal)); + EXPECT_EQUAL("BAR2", + FunctionTestConfig::getEnumwithdefName(config.enumwithdef)); + EXPECT_EQUAL(":parent:", config.refval); + EXPECT_EQUAL(":parent:", config.refwithdef); + EXPECT_EQUAL("etc", config.fileVal); + EXPECT_EQUAL(1u, config.boolarr.size()); + EXPECT_EQUAL(0u, config.intarr.size()); + EXPECT_EQUAL(2u, config.longarr.size()); + LOG(error, "0: %" PRId64, config.longarr[0]); + LOG(error, "1: %" PRId64, config.longarr[1]); + EXPECT_EQUAL(std::numeric_limits<int64_t>::max(), config.longarr[0]); + EXPECT_EQUAL(std::numeric_limits<int64_t>::min(), config.longarr[1]); + EXPECT_EQUAL(2u, config.doublearr.size()); + EXPECT_EQUAL(1u, config.stringarr.size()); + EXPECT_EQUAL(1u, config.enumarr.size()); + EXPECT_EQUAL(3u, config.refarr.size()); + EXPECT_EQUAL(1u, config.fileArr.size()); + EXPECT_EQUAL("bin", config.fileArr[0]); + + EXPECT_EQUAL("basicFoo", config.basicStruct.foo); + EXPECT_EQUAL(3, config.basicStruct.bar); + EXPECT_EQUAL(1u, config.basicStruct.intArr.size()); + EXPECT_EQUAL(310, config.basicStruct.intArr[0]); + EXPECT_EQUAL("inner0", config.rootStruct.inner0.name); + EXPECT_EQUAL(11, config.rootStruct.inner0.index); + EXPECT_EQUAL("inner1", config.rootStruct.inner1.name); + EXPECT_EQUAL(12, config.rootStruct.inner1.index); + EXPECT_EQUAL(1u, config.rootStruct.innerArr.size()); + EXPECT_EQUAL(true, config.rootStruct.innerArr[0].boolVal); + EXPECT_EQUAL("deep", config.rootStruct.innerArr[0].stringVal); + + // TODO: replace ':parent:' with 'configId' when references are handled properly also in C++. + EXPECT_EQUAL(2u, config.myarray.size()); + EXPECT_EQUAL(":parent:", config.myarray[0].refval); + EXPECT_EQUAL("file0", config.myarray[0].fileVal); + EXPECT_EQUAL(1, config.myarray[0].myStruct.a); + EXPECT_EQUAL(2, config.myarray[0].myStruct.b); + EXPECT_EQUAL(":parent:", config.myarray[1].refval); + EXPECT_EQUAL("file1", config.myarray[1].fileVal); + EXPECT_EQUAL(-1, config.myarray[1].myStruct.a); + EXPECT_EQUAL(-2, config.myarray[1].myStruct.b); +} + +std::string +readFile(const std::string & fileName) +{ + std::ifstream f(fileName.c_str()); + ASSERT_FALSE(f.fail()); + std::string content; + std::string line; + while (getline(f, line)) { + content += line; + } + return content; +} + +} + +struct LazyTestFixture +{ + const DirSpec _spec; + ConfigSubscriber _subscriber; + ConfigHandle<FunctionTestConfig>::UP _handle; + std::unique_ptr<FunctionTestConfig> _config; + + LazyTestFixture(const std::string & dirName) + : _spec(dirName), + _subscriber(_spec), + _handle(_subscriber.subscribe<FunctionTestConfig>("")) + { + } +}; + +struct TestFixture : public LazyTestFixture +{ + TestFixture(const std::string & dirName) + : LazyTestFixture(dirName) + { + ASSERT_TRUE(_subscriber.nextConfig(0)); + _config = _handle->getConfig(); + } +}; + +struct ErrorFixture +{ + LazyTestFixture & f; + ErrorFixture(LazyTestFixture & f1) : f(f1) { } + void run() { + f._subscriber.nextConfig(0); + bool thrown = false; + try { + f._handle->getConfig(); + } catch (const InvalidConfigException & e) { + thrown = true; + LOG(info, "Error: %s", e.getMessage().c_str()); + } + ASSERT_TRUE(thrown); + } +}; + +void attemptLacking(const std::string& param, bool isArray) { + std::ifstream in("defaultvalues/function-test.cfg", std::ios_base::in); + std::ostringstream config; + std::string s; + while (std::getline(in, s)) { + if (s.size() > param.size() && + s.substr(0, param.size()) == param && + (s[param.size()] == ' ' || s[param.size()] == '[')) + { + // Ignore values matched + } else { + config << s << "\n"; + + } + } + //std::cerr << "Config lacking " << param << "\n" + // << config.str() << "\n"; + try{ + RawSpec spec(config.str()); + ConfigSubscriber subscriber(spec); + ConfigHandle<FunctionTestConfig>::UP handle = subscriber.subscribe<FunctionTestConfig>("foo"); + ASSERT_TRUE(subscriber.nextConfig(0)); + std::unique_ptr<FunctionTestConfig> cfg = handle->getConfig(); + if (isArray) { + // Arrays are empty by default + return; + } + TEST_FATAL(("Expected to fail when not specifying value " + param + + " without default").c_str()); + } catch (InvalidConfigException& e) { + if (isArray) { + TEST_FATAL("Arrays should be empty by default."); + } + } +} + +TEST_F("testVariableAccess", TestFixture("variableaccess")) { + checkVariableAccess(*f._config); +} + + +TEST("test variable access from slime") { + vespalib::Slime slime; + std::string json(readFile("slime-payload.json")); + vespalib::slime::JsonFormat::decode(json, slime); + FunctionTestConfig config(config::ConfigPayload(slime.get())); + checkVariableAccess(config); +} + +TEST_F("testDefaultValues", TestFixture("defaultvalues")) { + EXPECT_EQUAL(false, f._config->boolVal); + EXPECT_EQUAL(false, f._config->boolWithDef); + EXPECT_EQUAL(5, f._config->intVal); + EXPECT_EQUAL(-545, f._config->intWithDef); + EXPECT_EQUAL(1234567890123LL, f._config->longVal); + EXPECT_EQUAL(-50000000000LL, f._config->longWithDef); + EXPECT_APPROX(41.23, f._config->doubleVal, 0.000001); + EXPECT_APPROX(-6.43, f._config->doubleWithDef, 0.000001); + EXPECT_EQUAL("foo", f._config->stringVal); + EXPECT_EQUAL("foobar", f._config->stringwithdef); + EXPECT_EQUAL("FOOBAR", FunctionTestConfig::getEnumValName(f._config->enumVal)); + EXPECT_EQUAL("BAR2", + FunctionTestConfig::getEnumwithdefName(f._config->enumwithdef)); + EXPECT_EQUAL(":parent:", f._config->refval); + EXPECT_EQUAL(":parent:", f._config->refwithdef); + EXPECT_EQUAL("vespa.log", f._config->fileVal); + EXPECT_EQUAL(1u, f._config->boolarr.size()); + EXPECT_EQUAL(0u, f._config->intarr.size()); + EXPECT_EQUAL(0u, f._config->longarr.size()); + EXPECT_EQUAL(2u, f._config->doublearr.size()); + EXPECT_EQUAL(1u, f._config->stringarr.size()); + EXPECT_EQUAL(1u, f._config->enumarr.size()); + EXPECT_EQUAL(0u, f._config->refarr.size()); + EXPECT_EQUAL(0u, f._config->fileArr.size()); + + EXPECT_EQUAL(3, f._config->basicStruct.bar); + EXPECT_EQUAL(1u, f._config->basicStruct.intArr.size()); + EXPECT_EQUAL(10, f._config->basicStruct.intArr[0]); + EXPECT_EQUAL(11, f._config->rootStruct.inner0.index); + EXPECT_EQUAL(12, f._config->rootStruct.inner1.index); + EXPECT_EQUAL(1u, f._config->rootStruct.innerArr.size()); + EXPECT_EQUAL("deep", f._config->rootStruct.innerArr[0].stringVal); + + EXPECT_EQUAL(2u, f._config->myarray.size()); + EXPECT_EQUAL(1, f._config->myarray[0].myStruct.a); + EXPECT_EQUAL(-1, f._config->myarray[1].myStruct.a); + EXPECT_EQUAL("command.com", f._config->myarray[0].fileVal); + EXPECT_EQUAL("display.sys", f._config->myarray[1].fileVal); +} + +TEST("testLackingDefaults") { + attemptLacking("bool_val", false); + attemptLacking("int_val", false); + attemptLacking("long_val", false); + attemptLacking("double_val", false); + attemptLacking("string_val", false); + attemptLacking("enum_val", false); + attemptLacking("refval", false); + attemptLacking("fileVal", false); + + attemptLacking("boolarr", true); + attemptLacking("intarr", true); + attemptLacking("longarr", true); + attemptLacking("doublearr", true); + attemptLacking("enumarr", true); + attemptLacking("stringarr", true); + attemptLacking("refarr", true); + attemptLacking("fileArr", true); + attemptLacking("myarray", true); + + attemptLacking("basicStruct.bar", false); + attemptLacking("rootStruct.inner0.index", false); + attemptLacking("rootStruct.inner1.index", false); + + // NOTE: When this line is lacking in C++, the array will be empty, and no exception is thrown. In Java, the array + // is initialized to length 1 (by the preceeding line 'rootStruct.innerArr[1]'), and an exception is thrown + // when the value is lacking. + attemptLacking("rootStruct.innerArr[0].stringVal", true); + + attemptLacking("myarray[0].stringval", true); + attemptLacking("myarray[0].refval", false); + attemptLacking("myarray[0].anotherarray", true); + attemptLacking("myarray[0].anotherarray", true); + attemptLacking("myarray[0].myStruct.a", false); +} + +TEST_F("testRandomOrder", TestFixture("randomorder")) { + EXPECT_EQUAL(false, f._config->boolVal); + EXPECT_EQUAL(true, f._config->boolWithDef); + EXPECT_EQUAL(5, f._config->intVal); + EXPECT_EQUAL(-14, f._config->intWithDef); + EXPECT_EQUAL(666000666000LL, f._config->longVal); + EXPECT_EQUAL(-333000333000LL, f._config->longWithDef); + EXPECT_APPROX(41.23, f._config->doubleVal, 0.000001); + EXPECT_APPROX(-12, f._config->doubleWithDef, 0.000001); + EXPECT_EQUAL("foo", f._config->stringVal); + EXPECT_EQUAL("bar", f._config->stringwithdef); + EXPECT_EQUAL("FOOBAR", FunctionTestConfig::getEnumValName(f._config->enumVal)); + EXPECT_EQUAL("BAR2", + FunctionTestConfig::getEnumwithdefName(f._config->enumwithdef)); + EXPECT_EQUAL(":parent:", f._config->refval); + EXPECT_EQUAL(":parent:", f._config->refwithdef); + EXPECT_EQUAL("autoexec.bat", f._config->fileVal); + EXPECT_EQUAL(1u, f._config->boolarr.size()); + EXPECT_EQUAL(0u, f._config->intarr.size()); + EXPECT_EQUAL(0u, f._config->longarr.size()); + EXPECT_EQUAL(2u, f._config->doublearr.size()); + EXPECT_EQUAL(1u, f._config->stringarr.size()); + EXPECT_EQUAL(1u, f._config->enumarr.size()); + EXPECT_EQUAL(0u, f._config->refarr.size()); + EXPECT_EQUAL(0u, f._config->fileArr.size()); + EXPECT_EQUAL(2u, f._config->myarray.size()); +} + +TEST_FF("testErrorRangeInt32", LazyTestFixture("errorval_int"), ErrorFixture(f1)) { f2.run(); } +TEST_FF("testErrorRangeInt64", LazyTestFixture("errorval_long"), ErrorFixture(f1)) { f2.run(); } +TEST_FF("testErrorRangeDouble", LazyTestFixture("errorval_double"), ErrorFixture(f1)) { f2.run(); } + +#if 0 +TEST_F("testEquality", TestFixture("variableaccess")) { + FunctionTestConfig myconfig(*f._config); + EXPECT_EQUAL(_config, myconfig); + myconfig.intVal = 2; + EXPECT_TRUE(_config != myconfig); + EXPECT_TRUE(_config.myarray == myconfig.myarray); + myconfig.myarray[1].anotherarray[1].foo = 5; + EXPECT_TRUE(_config.myarray != myconfig.myarray); + EXPECT_EQUAL(_config.myarray[0], myconfig.myarray[0]); + EXPECT_EQUAL(_config.myarray[1].refval, myconfig.myarray[1].refval); +} +#endif + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/functiontest/missingvalue/function-test.cfg b/config/src/tests/functiontest/missingvalue/function-test.cfg new file mode 100644 index 00000000000..9616ac3d27b --- /dev/null +++ b/config/src/tests/functiontest/missingvalue/function-test.cfg @@ -0,0 +1,39 @@ +bool_val false +long_val 123 +double_val 41.23 +string_val "foo" +enum_val FOOBAR +refval ":parent:" +fileVal "msdos.sys" +boolarr[1] +boolarr[0] false +intarr[0] +longarr[0] +doublearr[2] +doublearr[0] 2344 +doublearr[1] 123 +stringarr[1] +stringarr[0] "bar" +enumarr[1] +enumarr[0] VALUES +refarr[0] +basicStruct.bar 3 +rootStruct.inner0.index 11 +rootStruct.inner1.index 12 +rootStruct.innerArr[0] +myarray[2] +myarray[0].stringval[2] +myarray[0].stringval[0] "baah" +myarray[0].stringval[1] "yikes" +myarray[0].refval ":parent:" +myarray[0].fileVal "command.com" +myarray[0].anotherarray[1] +myarray[0].anotherarray[0].foo 7 +myarray[0].myStruct.a 1 +myarray[1].stringval[0] +myarray[1].refval ":parent:" +myarray[1].fileVal "display.sys" +myarray[1].anotherarray[2] +myarray[1].anotherarray[0].foo 1 +myarray[1].anotherarray[1].foo 2 +myarray[1].myStruct.a 1 diff --git a/config/src/tests/functiontest/randomorder/function-test.cfg b/config/src/tests/functiontest/randomorder/function-test.cfg new file mode 100644 index 00000000000..d3ec59491d2 --- /dev/null +++ b/config/src/tests/functiontest/randomorder/function-test.cfg @@ -0,0 +1,55 @@ +boolarr[1] +boolarr[0] false +int_with_def -14 +double_val 41.23 +double_with_def -12 +enumwithdef BAR2 +refval ":parent:" +refwithdef ":parent:" +intarr[0] +basicStruct.intArr[1] +basicStruct.intArr[0] 10 +doublearr[2] +doublearr[0] 2344 +doublearr[1] 123 +string_val "foo" +stringwithdef "bar" +enum_val FOOBAR +stringarr[1] +stringarr[0] "bar" +basicStruct.bar 3 +long_with_def -333000333000 +enumarr[1] +enumarr[0] VALUES +refarr[0] +rootStruct.innerArr[1] +rootStruct.innerArr[0].stringVal "deep" +fileVal "autoexec.bat" +myarray[2] +myarray[0].intval -5 +myarray[0].myStruct.a 1 +myarray[0].enumval INNER +myarray[0].refval ":parent:" +myarray[0].anotherarray[1] +myarray[0].anotherarray[0].foo 7 +myarray[0].stringval[2] +myarray[0].stringval[0] "baah" +myarray[0].stringval[1] "yikes" +myarray[0].fileVal "file0" +myarray[1].stringval[0] +myarray[1].enumval INNER +myarray[1].anotherarray[2] +myarray[1].anotherarray[0].foo 1 +myarray[1].anotherarray[1].foo 2 +myarray[1].myStruct.a -1 +myarray[1].refval ":parent:" +myarray[1].intval 5 +myarray[1].fileVal "file1" +bool_val false +bool_with_def true +longarr[0] +rootStruct.inner1.index 12 +int_val 5 +rootStruct.inner0.index 11 +long_val 666000666000 +fileArr[0] diff --git a/config/src/tests/functiontest/slime-payload.json b/config/src/tests/functiontest/slime-payload.json new file mode 100644 index 00000000000..7e3a16e69eb --- /dev/null +++ b/config/src/tests/functiontest/slime-payload.json @@ -0,0 +1,103 @@ +{ + "bool_val": false, + "bool_with_def": true, + "int_val": 5, + "int_with_def": -14, + "long_val": 12345678901, + "long_with_def": -9876543210, + "double_val": 41.23, + "double_with_def": -12, + "string_val": "foo", + "stringwithdef": "bar", + "enum_val": "FOOBAR", + "enumwithdef": "BAR2", + "refval": ":parent:", + "refwithdef": ":parent:", + "fileVal": "etc", + "boolarr": [ + false + ], + "intarr": [], + "longarr": [ + 9223372036854775807, + -9223372036854775808 + ], + "doublearr": [ + 2344, + 123 + ], + "stringarr": [ + "bar" + ], + "enumarr": [ + "VALUES" + ], + "refarr": [ + ":parent:", + ":parent:", + ":parent:" + ], + "fileArr": [ + "bin" + ], + "basicStruct": { + "foo": "basicFoo", + "bar": 3, + "intArr": [ + 310 + ] + }, + "rootStruct": { + "inner0": { + "index": 11 + }, + "inner1": { + "index": 12 + }, + "innerArr": [ + { + "boolVal": true, + "stringVal": "deep" + } + ] + }, + "myarray": [ + { + "intval": -5, + "stringval": [ + "baah", + "yikes" + ], + "enumval": "INNER", + "refval": ":parent:", + "fileVal": "file0", + "anotherarray": [ + { + "foo": 7 + } + ], + "myStruct": { + "a": 1, + "b": 2 + } + }, + { + "intval": 5, + "enumval": "INNER", + "refval": ":parent:", + "fileVal": "file1", + "anotherarray": [ + { + "foo": 1 + }, + { + "foo": 2 + } + ], + "myStruct": { + "a": -1, + "b": -2 + } + } + ] +} diff --git a/config/src/tests/functiontest/variableaccess/function-test.cfg b/config/src/tests/functiontest/variableaccess/function-test.cfg new file mode 100644 index 00000000000..d15bae0b764 --- /dev/null +++ b/config/src/tests/functiontest/variableaccess/function-test.cfg @@ -0,0 +1,67 @@ +bool_val false +bool_with_def true +int_val 5 +int_with_def -14 +long_val 12345678901 +long_with_def -9876543210 +double_val 41.23 +double_with_def -12 +string_val "foo" +stringwithdef "bar" +enum_val FOOBAR +enumwithdef BAR2 +refval :parent: +refwithdef ":parent:" +fileVal "etc" +boolarr[1] +boolarr[0] false +intarr[0] +longarr[2] +longarr[0] 9223372036854775807 +longarr[1] -9223372036854775808 +doublearr[2] +doublearr[0] 2344 +doublearr[1] 123 +stringarr[1] +stringarr[0] "bar" +enumarr[1] +enumarr[0] VALUES +refarr[3] +refarr[0] ":parent:" +refarr[1] ":parent" +refarr[2] "parent:" +fileArr[1] +fileArr[0] "bin" + +basicStruct.foo "basicFoo" +basicStruct.bar 3 +basicStruct.intArr[1] +basicStruct.intArr[0] 310 +rootStruct.inner0.index 11 +rootStruct.inner1.index 12 +rootStruct.innerArr[1] +rootStruct.innerArr[0].boolVal true +rootStruct.innerArr[0].stringVal "deep" + +myarray[2] +myarray[0].intval -5 +myarray[0].stringval[2] +myarray[0].stringval[0] "baah" +myarray[0].stringval[1] "yikes" +myarray[0].enumval INNER +myarray[0].refval :parent: +myarray[0].fileVal "file0" +myarray[0].anotherarray[1] +myarray[0].anotherarray[0].foo 7 +myarray[0].myStruct.a 1 +myarray[0].myStruct.b 2 +myarray[1].intval 5 +myarray[1].stringval[0] +myarray[1].enumval INNER +myarray[1].refval ":parent:" +myarray[1].fileVal "file1" +myarray[1].anotherarray[2] +myarray[1].anotherarray[0].foo 1 +myarray[1].anotherarray[1].foo 2 +myarray[1].myStruct.a -1 +myarray[1].myStruct.b -2 diff --git a/config/src/tests/functiontestng/defaultvalues/.gitignore b/config/src/tests/functiontestng/defaultvalues/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/functiontestng/defaultvalues/.gitignore diff --git a/config/src/tests/functiontestng/errorval_double/.gitignore b/config/src/tests/functiontestng/errorval_double/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/functiontestng/errorval_double/.gitignore diff --git a/config/src/tests/functiontestng/errorval_int/.gitignore b/config/src/tests/functiontestng/errorval_int/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/functiontestng/errorval_int/.gitignore diff --git a/config/src/tests/functiontestng/errorval_long/.gitignore b/config/src/tests/functiontestng/errorval_long/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/functiontestng/errorval_long/.gitignore diff --git a/config/src/tests/functiontestng/missingvalue/.gitignore b/config/src/tests/functiontestng/missingvalue/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/functiontestng/missingvalue/.gitignore diff --git a/config/src/tests/functiontestng/randomorder/.gitignore b/config/src/tests/functiontestng/randomorder/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/functiontestng/randomorder/.gitignore diff --git a/config/src/tests/functiontestng/variableaccess/.gitignore b/config/src/tests/functiontestng/variableaccess/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/functiontestng/variableaccess/.gitignore diff --git a/config/src/tests/getconfig/.gitignore b/config/src/tests/getconfig/.gitignore new file mode 100644 index 00000000000..c1760471d5d --- /dev/null +++ b/config/src/tests/getconfig/.gitignore @@ -0,0 +1,3 @@ +/config-my.cpp +/config-my.h +config_getconfig_test_app diff --git a/config/src/tests/getconfig/CMakeLists.txt b/config/src/tests/getconfig/CMakeLists.txt new file mode 100644 index 00000000000..f8ad5c2c7ed --- /dev/null +++ b/config/src/tests/getconfig/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_getconfig_test_app + SOURCES + getconfig.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_getconfig_test_app COMMAND config_getconfig_test_app) +vespa_generate_config(config_getconfig_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/getconfig/getconfig.cpp b/config/src/tests/getconfig/getconfig.cpp new file mode 100644 index 00000000000..ec350e3140b --- /dev/null +++ b/config/src/tests/getconfig/getconfig.cpp @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/config.h> +#include <vespa/config/raw/rawsource.h> +#include "config-my.h" + +using namespace config; + +namespace { + +struct ConfigFixture { + MyConfigBuilder builder; + ConfigSet set; + ConfigContext::SP context; + ConfigFixture() : builder(), set(), context() { + set.addBuilder("cfgid", &builder); + context.reset(new ConfigContext(set)); + } +}; + +} // namespace <unnamed> + +TEST("requireThatGetConfigReturnsCorrectConfig") +{ + RawSpec spec("myField \"foo\"\n"); + + std::unique_ptr<MyConfig> cfg = ConfigGetter<MyConfig>::getConfig("myid", spec); + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL("my", cfg->defName()); + ASSERT_EQUAL("foo", cfg->myField); +} + + +TEST("requireThatGetConfigReturnsCorrectConfig") +{ + FileSpec spec("my.cfg"); + std::unique_ptr<MyConfig> cfg = ConfigGetter<MyConfig>::getConfig("", spec); + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL("my", cfg->defName()); + ASSERT_EQUAL("foobar", cfg->myField); +} + +TEST_F("require that ConfigGetter can be used to obtain config generation", ConfigFixture) { + f1.builder.myField = "foo"; + { + int64_t gen1; + int64_t gen2; + std::unique_ptr<MyConfig> cfg1 = ConfigGetter<MyConfig>::getConfig(gen1, "cfgid", f1.set); + std::unique_ptr<MyConfig> cfg2 = ConfigGetter<MyConfig>::getConfig(gen2, "cfgid", f1.context); + EXPECT_EQUAL(1, gen1); + EXPECT_EQUAL(1, gen2); + EXPECT_EQUAL("foo", cfg1.get()->myField); + EXPECT_EQUAL("foo", cfg2.get()->myField); + } + f1.builder.myField = "bar"; + f1.context->reload(); + { + int64_t gen1; + int64_t gen2; + std::unique_ptr<MyConfig> cfg1 = ConfigGetter<MyConfig>::getConfig(gen1, "cfgid", f1.set); + std::unique_ptr<MyConfig> cfg2 = ConfigGetter<MyConfig>::getConfig(gen2, "cfgid", f1.context); + EXPECT_EQUAL(1, gen1); // <-- NB: generation will not increase when using the builder set directly + EXPECT_EQUAL(2, gen2); + EXPECT_EQUAL("bar", cfg1.get()->myField); + EXPECT_EQUAL("bar", cfg2.get()->myField); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/getconfig/my.cfg b/config/src/tests/getconfig/my.cfg new file mode 100644 index 00000000000..6172609bdff --- /dev/null +++ b/config/src/tests/getconfig/my.cfg @@ -0,0 +1 @@ +myField "foobar" diff --git a/config/src/tests/getconfig/test1.cfg b/config/src/tests/getconfig/test1.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/getconfig/test1.cfg diff --git a/config/src/tests/legacysubscriber/.gitignore b/config/src/tests/legacysubscriber/.gitignore new file mode 100644 index 00000000000..336ddd89dd5 --- /dev/null +++ b/config/src/tests/legacysubscriber/.gitignore @@ -0,0 +1,7 @@ +/config-bar.cpp +/config-bar.h +/config-foo.cpp +/config-foo.h +/config-my.cpp +/config-my.h +config_legacysubscriber_test_app diff --git a/config/src/tests/legacysubscriber/CMakeLists.txt b/config/src/tests/legacysubscriber/CMakeLists.txt new file mode 100644 index 00000000000..66022e31007 --- /dev/null +++ b/config/src/tests/legacysubscriber/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_legacysubscriber_test_app + SOURCES + legacysubscriber.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_legacysubscriber_test_app COMMAND config_legacysubscriber_test_app) +vespa_generate_config(config_legacysubscriber_test_app ../../test/resources/configdefinitions/foo.def) +vespa_generate_config(config_legacysubscriber_test_app ../../test/resources/configdefinitions/bar.def) +vespa_generate_config(config_legacysubscriber_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/legacysubscriber/legacysubscriber.cpp b/config/src/tests/legacysubscriber/legacysubscriber.cpp new file mode 100644 index 00000000000..966e12a0b82 --- /dev/null +++ b/config/src/tests/legacysubscriber/legacysubscriber.cpp @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/helper/legacysubscriber.h> +#include <fstream> +#include <config-my.h> +#include <config-foo.h> +#include <config-bar.h> + +using namespace config; + +template <typename ConfigType> +class MyCallback : public IFetcherCallback<ConfigType> +{ +public: + MyCallback() : _config(), _configured(false) { } + void configure(std::unique_ptr<ConfigType> config) + { + _configured = true; + _config = std::move(config); + } + std::unique_ptr<ConfigType> _config; + bool _configured; +}; + +TEST("requireThatFileLegacyWorks") { + LegacySubscriber s; + MyCallback<MyConfig> cb; + s.subscribe<MyConfig>("file:test1.cfg", &cb); + ASSERT_TRUE(cb._configured); + ASSERT_TRUE(cb._config.get() != NULL); + ASSERT_EQUAL("bar", cb._config->myField); +} + +TEST("requireThatDirLegacyWorks") { + LegacySubscriber s; + MyCallback<MyConfig> cb; + s.subscribe<MyConfig>("dir:testdir", &cb); + ASSERT_TRUE(cb._configured); + ASSERT_TRUE(cb._config.get() != NULL); + ASSERT_EQUAL("bar", cb._config->myField); +} + + +TEST("requireThatDirMultiFileLegacyWorks") { + MyCallback<FooConfig> cb1; + MyCallback<BarConfig> cb2; + + LegacySubscriber s1, s2; + s1.subscribe<FooConfig>("dir:testdir/foobar", &cb1); + s2.subscribe<BarConfig>("dir:testdir/foobar", &cb2); + + ASSERT_TRUE(cb1._configured); + ASSERT_TRUE(cb1._config.get() != NULL); + ASSERT_EQUAL("bar", cb1._config->fooValue); + + ASSERT_TRUE(cb2._configured); + ASSERT_TRUE(cb2._config.get() != NULL); + ASSERT_EQUAL("foo", cb2._config->barValue); +} + +TEST("requireThatFileLegacyWorksMultipleTimes") { + LegacySubscriber s; + MyCallback<MyConfig> cb; + s.subscribe<MyConfig>("file:test1.cfg", &cb); + ASSERT_TRUE(cb._configured); + ASSERT_TRUE(cb._config.get() != NULL); + ASSERT_EQUAL("bar", cb._config->myField); + cb._configured = false; + LegacySubscriber s2; + s2.subscribe<MyConfig>("file:test1.cfg", &cb); + ASSERT_TRUE(cb._configured); + ASSERT_TRUE(cb._config.get() != NULL); + ASSERT_EQUAL("bar", cb._config->myField); +} + +TEST("requireThatRawLegacyWorks") { + LegacySubscriber s; + MyCallback<MyConfig> cb; + s.subscribe<MyConfig>("raw:myField \"bar\"\n", &cb); + ASSERT_TRUE(cb._configured); + ASSERT_TRUE(cb._config.get() != NULL); + ASSERT_EQUAL("bar", cb._config->myField); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/legacysubscriber/test1.cfg b/config/src/tests/legacysubscriber/test1.cfg new file mode 100644 index 00000000000..4b0bbd13c83 --- /dev/null +++ b/config/src/tests/legacysubscriber/test1.cfg @@ -0,0 +1 @@ +myField "bar" diff --git a/config/src/tests/legacysubscriber/testdir/foobar/bar.cfg b/config/src/tests/legacysubscriber/testdir/foobar/bar.cfg new file mode 100644 index 00000000000..80d5b6f10b4 --- /dev/null +++ b/config/src/tests/legacysubscriber/testdir/foobar/bar.cfg @@ -0,0 +1 @@ +barValue "foo" diff --git a/config/src/tests/legacysubscriber/testdir/foobar/foo.cfg b/config/src/tests/legacysubscriber/testdir/foobar/foo.cfg new file mode 100644 index 00000000000..6ec74b85d6a --- /dev/null +++ b/config/src/tests/legacysubscriber/testdir/foobar/foo.cfg @@ -0,0 +1 @@ +fooValue "bar" diff --git a/config/src/tests/legacysubscriber/testdir/foobar/nothing.txt b/config/src/tests/legacysubscriber/testdir/foobar/nothing.txt new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/config/src/tests/legacysubscriber/testdir/foobar/nothing.txt diff --git a/config/src/tests/legacysubscriber/testdir/my.cfg b/config/src/tests/legacysubscriber/testdir/my.cfg new file mode 100644 index 00000000000..4b0bbd13c83 --- /dev/null +++ b/config/src/tests/legacysubscriber/testdir/my.cfg @@ -0,0 +1 @@ +myField "bar" diff --git a/config/src/tests/misc/.gitignore b/config/src/tests/misc/.gitignore new file mode 100644 index 00000000000..d6c1f6760a4 --- /dev/null +++ b/config/src/tests/misc/.gitignore @@ -0,0 +1 @@ +config_misc_test_app diff --git a/config/src/tests/misc/CMakeLists.txt b/config/src/tests/misc/CMakeLists.txt new file mode 100644 index 00000000000..a37f6411e0b --- /dev/null +++ b/config/src/tests/misc/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_misc_test_app + SOURCES + misc.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_misc_test_app COMMAND config_misc_test_app) diff --git a/config/src/tests/misc/misc.cpp b/config/src/tests/misc/misc.cpp new file mode 100644 index 00000000000..b2595599125 --- /dev/null +++ b/config/src/tests/misc/misc.cpp @@ -0,0 +1,194 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/common/configupdate.h> +#include <vespa/config/common/misc.h> +#include <vespa/config/common/configvalue.h> +#include <vespa/config/common/errorcode.h> +#include <vespa/config/common/vespa_version.h> +#include <vespa/config/subscription/sourcespec.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <map> + +using namespace config; + +TEST("requireThatConfigUpdateWorks") { + std::vector<vespalib::string> lines; + lines.push_back("foo"); + + ConfigUpdate up(ConfigValue(lines, "mymd5"), true, 1337); + ASSERT_EQUAL(1337, up.getGeneration()); + ASSERT_TRUE(up.hasChanged()); + + ConfigUpdate up2(ConfigValue(lines, "mymd52"), false, 1338); + ASSERT_EQUAL(1338, up2.getGeneration()); + ASSERT_FALSE(up2.hasChanged()); +} + +TEST("requireThatConfigValueWorks") { + std::vector<vespalib::string> lines; + lines.push_back("myFooField \"bar\""); + ConfigValue v1(lines, calculateContentMd5(lines)); + ConfigValue v2(lines, calculateContentMd5(lines)); + ConfigValue v3(lines, calculateContentMd5(lines)); + lines.push_back("myFooField \"bar2\""); + ConfigValue v4(lines, calculateContentMd5(lines)); + ASSERT_TRUE(v1 == v2); + ASSERT_TRUE(v1 == v3); +} + +TEST("requireThatConfigKeyWorks") { + ConfigKey key1("id1", "def1", "namespace1", "md51"); + ConfigKey key2("id1", "def1", "namespace1", "md51"); + ConfigKey key3("id2", "def1", "namespace1", "md51"); + ConfigKey key4("id1", "def2", "namespace1", "md51"); + ConfigKey key5("id1", "def1", "namespace2", "md51"); + ConfigKey key6("id1", "def1", "namespace1", "md52"); // Special case. Md5 does not matter, so should be qual to key1 and key2 + + ASSERT_TRUE(key1 == key2); + + ASSERT_TRUE(key1 == key1); + ASSERT_TRUE(key1 == key2); + ASSERT_TRUE(key1 < key3); + ASSERT_TRUE(key1 < key4); + ASSERT_TRUE(key1 < key5); + ASSERT_TRUE(key1 == key6); + + ASSERT_TRUE(key2 == key1); + ASSERT_TRUE(key2 == key2); + ASSERT_TRUE(key2 < key3); + ASSERT_TRUE(key2 < key4); + ASSERT_TRUE(key2 < key5); + ASSERT_TRUE(key2 == key6); + + ASSERT_TRUE(key3 > key1); + ASSERT_TRUE(key3 > key2); + ASSERT_TRUE(key3 == key3); + ASSERT_TRUE(key3 > key4); + ASSERT_TRUE(key3 > key5); + ASSERT_TRUE(key3 > key6); + + ASSERT_TRUE(key4 > key1); + ASSERT_TRUE(key4 > key2); + ASSERT_TRUE(key4 < key3); + ASSERT_TRUE(key4 == key4); + ASSERT_TRUE(key4 > key5); + ASSERT_TRUE(key4 > key6); + + ASSERT_TRUE(key5 > key1); + ASSERT_TRUE(key5 > key2); + ASSERT_TRUE(key5 < key3); + ASSERT_TRUE(key5 < key4); + ASSERT_TRUE(key5 == key5); + ASSERT_TRUE(key5 > key6); + + ASSERT_TRUE(key6 == key1); + ASSERT_TRUE(key6 == key2); + ASSERT_TRUE(key6 < key3); + ASSERT_TRUE(key6 < key4); + ASSERT_TRUE(key6 < key5); + ASSERT_TRUE(key6 == key6); + + std::map<ConfigKey, int> keymap; + keymap[key1] = 1; + keymap[key2] = 2; + keymap[key3] = 3; + keymap[key4] = 4; + keymap[key5] = 5; + + ASSERT_EQUAL(2, keymap[key1]); + ASSERT_EQUAL(2, keymap[key2]); + ASSERT_EQUAL(3, keymap[key3]); + ASSERT_EQUAL(4, keymap[key4]); + ASSERT_EQUAL(5, keymap[key5]); + keymap[key6] = 6; + ASSERT_EQUAL(6, keymap[key1]); + ASSERT_EQUAL(6, keymap[key2]); + ASSERT_EQUAL(6, keymap[key6]); +} + +TEST("require that config key initializes schema") +{ + std::vector<vespalib::string> schema; + schema.push_back("foo"); + schema.push_back("bar"); + ConfigKey key("id1", "def1", "namespace1", "md51", schema); + const std::vector<vespalib::string> &vref(key.getDefSchema()); + for (size_t i = 0; i < schema.size(); i++) { + ASSERT_EQUAL(schema[i], vref[i]); + } +} + +TEST("require that error codes are correctly translated to strings") { +#define ASSERT_CONFIG(name) ASSERT_EQUAL(#name, ErrorCode::getName(ErrorCode::name)) + ASSERT_CONFIG(UNKNOWN_CONFIG); + ASSERT_CONFIG(UNKNOWN_DEFINITION); + ASSERT_CONFIG(UNKNOWN_VERSION); + ASSERT_CONFIG(UNKNOWN_CONFIGID); + ASSERT_CONFIG(UNKNOWN_DEF_MD5); + ASSERT_CONFIG(UNKNOWN_VESPA_VERSION); + ASSERT_CONFIG(ILLEGAL_NAME); + ASSERT_CONFIG(ILLEGAL_VERSION); + ASSERT_CONFIG(ILLEGAL_CONFIGID); + ASSERT_CONFIG(ILLEGAL_DEF_MD5); + ASSERT_CONFIG(ILLEGAL_CONFIG_MD5); + ASSERT_CONFIG(ILLEGAL_TIMEOUT); + ASSERT_CONFIG(ILLEGAL_TIMESTAMP); + ASSERT_CONFIG(ILLEGAL_NAME_SPACE); + ASSERT_CONFIG(ILLEGAL_PROTOCOL_VERSION); + ASSERT_CONFIG(ILLEGAL_CLIENT_HOSTNAME); + ASSERT_CONFIG(OUTDATED_CONFIG); + ASSERT_CONFIG(INTERNAL_ERROR); + ASSERT_CONFIG(APPLICATION_NOT_LOADED); + ASSERT_CONFIG(INCONSISTENT_CONFIG_MD5); + ASSERT_EQUAL("Unknown error", ErrorCode::getName(13434)); +#undef ASSERT_CONFIG +} + +TEST("require that source spec parses protocol version") { + const char * envName = "VESPA_CONFIG_PROTOCOL_VERSION"; + EXPECT_EQUAL(3, ServerSpec().protocolVersion()); + setenv(envName, "2", 1); + EXPECT_EQUAL(2, ServerSpec().protocolVersion()); + setenv(envName, "3", 1); + EXPECT_EQUAL(3, ServerSpec().protocolVersion()); + setenv(envName, "4", 1); + EXPECT_EQUAL(3, ServerSpec().protocolVersion()); + setenv(envName, "illegal", 1); + EXPECT_EQUAL(3, ServerSpec().protocolVersion()); + setenv(envName, "1", 1); + EXPECT_EQUAL(1, ServerSpec().protocolVersion()); + unsetenv(envName); +} + +TEST("require that source spec parses trace level") { + const char * envName = "VESPA_CONFIG_PROTOCOL_TRACELEVEL"; + EXPECT_EQUAL(0, ServerSpec().traceLevel()); + setenv(envName, "3", 1); + EXPECT_EQUAL(3, ServerSpec().traceLevel()); + setenv(envName, "illegal", 1); + EXPECT_EQUAL(0, ServerSpec().traceLevel()); + unsetenv(envName); +} + +TEST("require that source spec parses compression type") { + const char * envName = "VESPA_CONFIG_PROTOCOL_COMPRESSION"; + EXPECT_TRUE(CompressionType::LZ4 == ServerSpec().compressionType()); + setenv(envName, "UNCOMPRESSED", 1); + EXPECT_TRUE(CompressionType::UNCOMPRESSED == ServerSpec().compressionType()); + setenv(envName, "illegal", 1); + EXPECT_TRUE(CompressionType::LZ4 == ServerSpec().compressionType()); + setenv(envName, "LZ4", 1); + EXPECT_TRUE(CompressionType::LZ4 == ServerSpec().compressionType()); + unsetenv(envName); +} + +TEST("require that vespa version is set") { + VespaVersion vespaVersion = VespaVersion::getCurrentVersion(); + vespalib::string str = vespaVersion.toString(); + + EXPECT_TRUE(str.length() > 0); +} + + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/payload_converter/.gitignore b/config/src/tests/payload_converter/.gitignore new file mode 100644 index 00000000000..94504539459 --- /dev/null +++ b/config/src/tests/payload_converter/.gitignore @@ -0,0 +1 @@ +config_payload_converter_test_app diff --git a/config/src/tests/payload_converter/CMakeLists.txt b/config/src/tests/payload_converter/CMakeLists.txt new file mode 100644 index 00000000000..743486e7164 --- /dev/null +++ b/config/src/tests/payload_converter/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_payload_converter_test_app + SOURCES + payload_converter.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_payload_converter_test_app COMMAND config_payload_converter_test_app) diff --git a/config/src/tests/payload_converter/payload_converter.cpp b/config/src/tests/payload_converter/payload_converter.cpp new file mode 100644 index 00000000000..664683b651f --- /dev/null +++ b/config/src/tests/payload_converter/payload_converter.cpp @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("payload_converter"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/common/payload_converter.h> + +using namespace config; +using namespace vespalib; +using namespace vespalib::slime; + +TEST("require that v2 payload leaf values can be converted to cfg format") { + Slime slime; + Cursor & root(slime.setObject()); + root.setString("foo", "bar"); + root.setLong("bar", 8); + root.setDouble("baz", 3.1); + root.setBool("quux", true); + PayloadConverter converter(root); + std::vector<vespalib::string> lines(converter.convert()); + std::sort(lines.begin(), lines.end()); + + ASSERT_EQUAL(4u, lines.size()); + EXPECT_EQUAL("bar 8", lines[0]); + EXPECT_EQUAL("baz 3.1", lines[1]); + EXPECT_EQUAL("foo \"bar\"", lines[2]); + EXPECT_EQUAL("quux true", lines[3]); +} + +TEST("require that v2 payload struct values can be converted to cfg format") { + Slime slime; + Cursor & root(slime.setObject()); + Cursor & inner(root.setObject("obj")); + inner.setString("foo", "bar"); + inner.setLong("bar", 8); + PayloadConverter converter(root); + std::vector<vespalib::string> lines(converter.convert()); + std::sort(lines.begin(), lines.end()); + + ASSERT_EQUAL(2u, lines.size()); + EXPECT_EQUAL("obj.bar 8", lines[0]); + EXPECT_EQUAL("obj.foo \"bar\"", lines[1]); +} + +TEST("require that v2 payload array values can be converted to cfg format") { + Slime slime; + Cursor & root(slime.setObject()); + Cursor & inner(root.setArray("arr")); + inner.addString("foo"); + inner.addLong(8); + PayloadConverter converter(root); + std::vector<vespalib::string> lines(converter.convert()); + ASSERT_EQUAL(2u, lines.size()); + EXPECT_EQUAL("arr[0] \"foo\"", lines[0]); + EXPECT_EQUAL("arr[1] 8", lines[1]); +} + + +TEST("require that v2 payload nested structures can be converted to cfg format") { + Slime slime; + Cursor & root(slime.setObject()); + Cursor & inner(root.setArray("arr")); + Cursor & obj1(inner.addObject()); + Cursor & obj2(inner.addObject()); + obj1.setString("foo", "bar"); + obj2.setLong("bar", 5); + Cursor & inner2(root.setObject("obj")); + Cursor & innerArr(inner2.setArray("arr")); + Cursor & innerobj(innerArr.addObject()); + Cursor & innerArr2(innerobj.setArray("arr2")); + innerArr2.addString("muhaha"); + PayloadConverter converter(root); + std::vector<vespalib::string> lines(converter.convert()); + std::sort(lines.begin(), lines.end()); + ASSERT_EQUAL(3u, lines.size()); + EXPECT_EQUAL("arr[0].foo \"bar\"", lines[0]); + EXPECT_EQUAL("arr[1].bar 5", lines[1]); + EXPECT_EQUAL("obj.arr[0].arr2[0] \"muhaha\"", lines[2]); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/print/.gitignore b/config/src/tests/print/.gitignore new file mode 100644 index 00000000000..662c2639f41 --- /dev/null +++ b/config/src/tests/print/.gitignore @@ -0,0 +1,12 @@ +/config-motd.cpp +/config-motd.h +/config-my.cpp +/config-my.h +/motd2.cfg +/motd2.json +/test_1.cfg +/test_2.cfg +/test_3.cfg +/test_1.json +/test_2.json +config_print_test_app diff --git a/config/src/tests/print/CMakeLists.txt b/config/src/tests/print/CMakeLists.txt new file mode 100644 index 00000000000..7af3b842c50 --- /dev/null +++ b/config/src/tests/print/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_print_test_app + SOURCES + print.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_print_test_app COMMAND config_print_test_app) +vespa_generate_config(config_print_test_app ../../test/resources/configdefinitions/my.def) +vespa_generate_config(config_print_test_app ../../test/resources/configdefinitions/motd.def) diff --git a/config/src/tests/print/motd.cfg b/config/src/tests/print/motd.cfg new file mode 100644 index 00000000000..e7c8a0db095 --- /dev/null +++ b/config/src/tests/print/motd.cfg @@ -0,0 +1,23 @@ +intVal 1 +longVal 1 +doubleVal 2.3 +stringVal "foo" +stringnulldef "foo" +enumVal FOOBAR +refVal "refVal" +fileVal "fileVal" +boolVal true +boolarr[0] true +boolarr[1] false +myArray[2] +myArray[0].stringVal[1] +myArray[0].stringVal[0] "bla" +myArray[0].refVal "habba" +myArray[1].stringVal[1] +myArray[1].stringVal[0] "blabla" +myArray[1].refVal "nabba" +myArray[1].anotherArray[1] +myArray[1].anotherArray[0].foo 1337 +stringmap{"foo"} "bar" +structmap{"foo"}.foo 3 +mapmap{"foo"}.map{"baz"}.bar 32 diff --git a/config/src/tests/print/print.cpp b/config/src/tests/print/print.cpp new file mode 100644 index 00000000000..7faae087e74 --- /dev/null +++ b/config/src/tests/print/print.cpp @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/config.h> +#include <vespa/config/print.h> +#include <vespa/config/print/fileconfigreader.h> +#include <vespa/config/print/istreamconfigreader.h> +#include "config-my.h" +#include "config-motd.h" + +using namespace config; + +template <typename T> +struct RawFixture { + RawSpec spec; + std::unique_ptr<T> cfg; + RawFixture() + : spec("myField \"foo\"\n"), + cfg(ConfigGetter<T>::getConfig("test", spec)) + { } +}; + + +TEST_F("requireThatConfigIsWrittenToFile", RawFixture<MyConfig>) { + FileConfigWriter writer("test_1.json"); + ASSERT_TRUE(writer.write(*f.cfg, JsonConfigFormatter())); + struct stat st; + int ret = stat("test_1.json", &st); + ASSERT_EQUAL(0, ret); + ASSERT_TRUE(st.st_size > 0); +} + +TEST_F("requireThatCanPrintAsJson", RawFixture<MyConfig>) { + FileConfigWriter writer("test_2.json"); + ASSERT_TRUE(writer.write(*f.cfg, JsonConfigFormatter())); + FileConfigReader<MyConfig> reader("test_2.json"); + std::unique_ptr<MyConfig> cfg2 = reader.read(JsonConfigFormatter()); + ASSERT_TRUE(*cfg2 == *f.cfg); +} + +TEST_F("requireThatCanPrintToOstream", RawFixture<MyConfig>) { + std::ostringstream ss; + OstreamConfigWriter writer(ss); + ASSERT_TRUE(writer.write(*f.cfg)); + ASSERT_EQUAL("myField \"foo\"\n", ss.str()); +} + +TEST_F("requireThatCanReadFromIstream", RawFixture<MyConfig>) { + std::stringstream ss; + ss << "myField \"foo\"\n"; + IstreamConfigReader<MyConfig> reader(ss); + std::unique_ptr<MyConfig> cfg = reader.read(); + ASSERT_EQUAL(std::string("foo"), cfg->myField); +} + +TEST_F("requireThatCanPrintToAscii", RawFixture<MyConfig>) { + vespalib::asciistream ss; + AsciiConfigWriter writer(ss); + ASSERT_TRUE(writer.write(*f.cfg)); + ASSERT_EQUAL("myField \"foo\"\n", ss.str()); +} + +TEST_F("requireThatCanPrintAsConfigFormat", RawFixture<MyConfig>) { + FileConfigWriter writer("test_3.cfg"); + ASSERT_TRUE(writer.write(*f.cfg)); + FileConfigReader<MyConfig> reader("test_3.cfg"); + std::unique_ptr<MyConfig> cfg2 = reader.read(); + ASSERT_TRUE(*cfg2 == *f.cfg); +} + +TEST_F("require that invalid file throws exception", RawFixture<MyConfig>) { + FileConfigReader<MyConfig> reader("nonexistant.cfg"); + EXPECT_EXCEPTION(reader.read(), vespalib::IllegalArgumentException, "Unable to open file"); + +} + +TEST_F("requireThatCanLoadWrittenWithConfigFormat", RawFixture<MyConfig>) { + FileConfigWriter writer("test_3.cfg"); + ASSERT_TRUE(writer.write(*f.cfg)); + std::unique_ptr<MyConfig> cfg2 = ConfigGetter<MyConfig>::getConfig("test_3", FileSpec("test_3.cfg")); + ASSERT_TRUE(*cfg2 == *f.cfg); +} + +TEST("requireThatAllFieldsArePrintedCorrectly") { + std::unique_ptr<MotdConfig> cfg = ConfigGetter<MotdConfig>::getConfig("motd", FileSpec("motd.cfg")); + FileConfigWriter writer("motd2.cfg"); + ASSERT_TRUE(writer.write(*cfg, FileConfigFormatter())); + std::unique_ptr<MotdConfig> cfg2 = ConfigGetter<MotdConfig>::getConfig("motd2", FileSpec("motd2.cfg")); + ASSERT_TRUE(*cfg2 == *cfg); +} + +TEST("require that reading cfg format throws exception") { + FileConfigReader<MyConfig> reader("test_1.json"); + EXPECT_EXCEPTION(reader.read(FileConfigFormatter()), vespalib::IllegalArgumentException, "Reading cfg format is not supported"); +} + + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/raw_subscription/.gitignore b/config/src/tests/raw_subscription/.gitignore new file mode 100644 index 00000000000..bed4d45f522 --- /dev/null +++ b/config/src/tests/raw_subscription/.gitignore @@ -0,0 +1,3 @@ +/config-my.cpp +/config-my.h +config_raw_subscription_test_app diff --git a/config/src/tests/raw_subscription/CMakeLists.txt b/config/src/tests/raw_subscription/CMakeLists.txt new file mode 100644 index 00000000000..8f1431006e0 --- /dev/null +++ b/config/src/tests/raw_subscription/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_raw_subscription_test_app + SOURCES + raw_subscription.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_raw_subscription_test_app COMMAND config_raw_subscription_test_app) +vespa_generate_config(config_raw_subscription_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/raw_subscription/raw_subscription.cpp b/config/src/tests/raw_subscription/raw_subscription.cpp new file mode 100644 index 00000000000..69cdaa3f0d3 --- /dev/null +++ b/config/src/tests/raw_subscription/raw_subscription.cpp @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/config.h> +#include <vespa/config/common/configholder.h> +#include <vespa/config/raw/rawsource.h> +#include "config-my.h" + +using namespace config; + +TEST("require that raw spec can create source factory") +{ + RawSpec spec("myField \"foo\"\n"); + SourceFactory::UP raw = spec.createSourceFactory(TimingValues()); + ASSERT_TRUE(raw.get() != NULL); + IConfigHolder::SP holder(new ConfigHolder()); + Source::UP src = raw->createSource(holder, ConfigKey("myid", "my", "bar", "foo")); + ASSERT_TRUE(src.get() != NULL); + + src->getConfig(); + ASSERT_TRUE(holder->poll()); + ConfigUpdate::UP update(holder->provide()); + ASSERT_TRUE(update.get() != NULL); + const ConfigValue & value(update->getValue()); + ASSERT_EQUAL(1u, value.numLines()); + ASSERT_EQUAL("myField \"foo\"", value.getLine(0)); +} + +TEST("requireThatRawSubscriptionReturnsCorrectConfig") +{ + RawSpec spec("myField \"foo\"\n"); + ConfigSubscriber s(spec); + std::unique_ptr<ConfigHandle<MyConfig> > handle = s.subscribe<MyConfig>("myid"); + s.nextConfig(0); + std::unique_ptr<MyConfig> cfg = handle->getConfig(); + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL("foo", cfg->myField); + ASSERT_EQUAL("my", cfg->defName()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/subscriber/.gitignore b/config/src/tests/subscriber/.gitignore new file mode 100644 index 00000000000..1b93a34021a --- /dev/null +++ b/config/src/tests/subscriber/.gitignore @@ -0,0 +1,7 @@ +/config-bar.cpp +/config-bar.h +/config-foo.cpp +/config-foo.h +/config-baz.cpp +/config-baz.h +config_subscriber_test_app diff --git a/config/src/tests/subscriber/CMakeLists.txt b/config/src/tests/subscriber/CMakeLists.txt new file mode 100644 index 00000000000..1b1673a2512 --- /dev/null +++ b/config/src/tests/subscriber/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_subscriber_test_app + SOURCES + subscriber.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_subscriber_test_app COMMAND config_subscriber_test_app) +vespa_generate_config(config_subscriber_test_app ../../test/resources/configdefinitions/foo.def) +vespa_generate_config(config_subscriber_test_app ../../test/resources/configdefinitions/bar.def) +vespa_generate_config(config_subscriber_test_app ../../test/resources/configdefinitions/baz.def) diff --git a/config/src/tests/subscriber/subscriber.cpp b/config/src/tests/subscriber/subscriber.cpp new file mode 100644 index 00000000000..0d5d2698b42 --- /dev/null +++ b/config/src/tests/subscriber/subscriber.cpp @@ -0,0 +1,525 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/config.h> +#include <vespa/config/common/misc.h> +#include <vespa/config/common/configholder.h> +#include <vespa/config/subscription/configsubscription.h> +#include <fstream> +#include "config-foo.h" +#include "config-bar.h" +#include "config-baz.h" + +using namespace config; +using namespace vespalib; + +namespace { + + ConfigValue createValue(const std::string & value) + { + std::vector< vespalib::string > lines; + lines.push_back(value); + return ConfigValue(lines, calculateContentMd5(lines)); + } + + ConfigValue createFooValue(const std::string & value) + { + return createValue("fooValue \"" + value + "\""); + } + + ConfigValue createBarValue(const std::string & value) + { + return createValue("barValue \"" + value + "\""); + } + + ConfigValue createBazValue(const std::string & value) + { + return createValue("bazValue \"" + value + "\""); + } + + void verifyConfig(const std::string & expected, std::unique_ptr<FooConfig> cfg) + { + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL(expected, cfg->fooValue); + } + + void verifyConfig(const std::string & expected, std::unique_ptr<BarConfig> cfg) + { + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL(expected, cfg->barValue); + } + + void verifyConfig(const std::string & expected, std::unique_ptr<BazConfig> cfg) + { + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL(expected, cfg->bazValue); + } + + class MySource : public Source + { + void getConfig() { } + void close() { } + void reload(int64_t gen) { (void) gen; } + }; + + class MyManager : public IConfigManager + { + public: + + void unsubscribeAll() { } + size_t numSubscribers() const { return 0; } + + + SubscriptionId idCounter; + std::vector<IConfigHolder::SP> _holders; + int numCancel; + + + MyManager() : idCounter(0), numCancel(0) { } + + ConfigSubscription::SP subscribe(const ConfigKey & key, uint64_t timeoutInMillis) { + (void) timeoutInMillis; + IConfigHolder::SP holder(new ConfigHolder()); + _holders.push_back(holder); + + ConfigSubscription::SP s(new ConfigSubscription(0, key, holder, Source::UP(new MySource()))); + return s; + } + void unsubscribe(const ConfigSubscription::SP & subscription) { + (void) subscription; + numCancel++; + } + + void updateValue(size_t index, const ConfigValue & value, int64_t generation) { + ASSERT_TRUE(index < _holders.size()); + _holders[index]->handle(ConfigUpdate::UP(new ConfigUpdate(value, true, generation))); + } + + void updateGeneration(size_t index, int64_t generation) { + ASSERT_TRUE(index < _holders.size()); + ConfigValue value; + // Give previous value just as API. + if (_holders[index]->poll()) { + value = _holders[index]->provide()->getValue(); + } + _holders[index]->handle(ConfigUpdate::UP(new ConfigUpdate(value, false, generation))); + } + + void reload(int64_t generation) + { + (void) generation; + } + + }; + + class APIFixture : public IConfigContext + { + public: + MyManager & _m; + APIFixture(MyManager & m) + : _m(m) + { + } + + APIFixture(const APIFixture & rhs) + : IConfigContext(rhs), + _m(rhs._m) + { } + + IConfigManager & getManagerInstance() { + return _m; + } + + IConfigManager & getManagerInstance(const SourceSpec & spec) { + (void) spec; + return getManagerInstance(); + } + + void reload() { } + }; + + struct StandardFixture { + MyManager & f1; + ConfigSubscriber s; + ConfigHandle<FooConfig>::UP h1; + ConfigHandle<BarConfig>::UP h2; + + StandardFixture(MyManager & F1, APIFixture & F2) : f1(F1), s(IConfigContext::SP(new APIFixture(F2))) + { + h1 = s.subscribe<FooConfig>("myid"); + h2 = s.subscribe<BarConfig>("myid"); + f1.updateValue(0, createFooValue("foo"), 1); + f1.updateValue(1, createBarValue("bar"), 1); + ASSERT_TRUE(s.nextConfig(0)); + verifyConfig("foo", h1->getConfig()); + verifyConfig("bar", h2->getConfig()); + } + }; + + struct SimpleFixture { + ConfigSet set; + FooConfigBuilder fooBuilder; + BarConfigBuilder barBuilder; + SimpleFixture() { + fooBuilder.fooValue = "bar"; + barBuilder.barValue = "foo"; + set.addBuilder("myid", &fooBuilder); + set.addBuilder("myid", &barBuilder); + } + }; +} + +TEST_F("requireThatSubscriberCanGetMultipleTypes", SimpleFixture()) { + ConfigSubscriber s(f.set); + ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); + ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid"); + ASSERT_TRUE(s.nextConfig(0)); + std::unique_ptr<FooConfig> foo = h1->getConfig(); + std::unique_ptr<BarConfig> bar = h2->getConfig(); + ASSERT_EQUAL("bar", foo->fooValue); + ASSERT_EQUAL("foo", bar->barValue); +} + +TEST_F("requireThatNextConfigMustBeCalled", SimpleFixture()) { + ConfigSubscriber s(f.set); + ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); + bool thrown = false; + try { + std::unique_ptr<FooConfig> foo = h1->getConfig(); + } catch (const ConfigRuntimeException & e) { + thrown = true; + } + ASSERT_TRUE(thrown); +} + +TEST_F("requireThatSubscriptionsCannotBeAddedWhenFrozen", SimpleFixture()) { + ConfigSubscriber s(f.set); + ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); + ASSERT_TRUE(s.nextConfig(0)); + bool thrown = false; + try { + ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid"); + } catch (const ConfigRuntimeException & e) { + thrown = true; + } + ASSERT_TRUE(thrown); +} + +TEST_FF("requireThatNextConfigReturnsFalseUntilSubscriptionHasSucceeded", MyManager, APIFixture(f1)) { + ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); + ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid"); + ASSERT_FALSE(s.nextConfig(0)); + ASSERT_FALSE(s.nextConfig(100)); + f1.updateValue(0, createFooValue("foo"), 1); + ASSERT_FALSE(s.nextConfig(100)); + f1.updateValue(1, createBarValue("bar"), 1); + ASSERT_TRUE(s.nextConfig(100)); +} + +TEST_FFF("requireThatNewGenerationIsFetchedOnReload", MyManager, APIFixture(f1), StandardFixture(f1, f2)) { + verifyConfig("foo", f3.h1->getConfig()); + verifyConfig("bar", f3.h2->getConfig()); + + ASSERT_FALSE(f3.s.nextConfig(1000)); + + verifyConfig("foo", f3.h1->getConfig()); + verifyConfig("bar", f3.h2->getConfig()); + + f1.updateValue(0, createFooValue("foo2"), 3); + f1.updateValue(1, createBarValue("bar2"), 3); + + ASSERT_TRUE(f3.s.nextConfig(1000)); + + verifyConfig("foo2", f3.h1->getConfig()); + verifyConfig("bar2", f3.h2->getConfig()); +} + +TEST_FFF("requireThatAllConfigsMustGetTimestampUpdate", MyManager, APIFixture(f1), StandardFixture(f1, f2)) { + f1.updateValue(0, createFooValue("foo2"), 2); + ASSERT_FALSE(f3.s.nextConfig(100)); + verifyConfig("foo", f3.h1->getConfig()); + verifyConfig("bar", f3.h2->getConfig()); + + f1.updateValue(0, createFooValue("foo2"), 3); + f1.updateGeneration(1, 3); + + ASSERT_TRUE(f3.s.nextConfig(0)); + verifyConfig("foo2", f3.h1->getConfig()); + verifyConfig("bar", f3.h2->getConfig()); +} + +TEST_FFF("requireThatNextConfigMaySucceedIfInTheMiddleOfConfigUpdate", MyManager, APIFixture(f1), StandardFixture(f1, f2)) { + f1.updateValue(0, createFooValue("foo2"), 2); + ASSERT_FALSE(f3.s.nextConfig(1000)); + verifyConfig("foo", f3.h1->getConfig()); + verifyConfig("bar", f3.h2->getConfig()); + + f1.updateGeneration(1, 2); + ASSERT_TRUE(f3.s.nextConfig(0)); + verifyConfig("foo2", f3.h1->getConfig()); + verifyConfig("bar", f3.h2->getConfig()); +} + +TEST_FFF("requireThatCorrectConfigIsReturnedAfterTimestampUpdate", MyManager, APIFixture(f1), StandardFixture(f1, f2)) { + f1.updateGeneration(0, 2); + f1.updateGeneration(1, 2); + ASSERT_FALSE(f3.s.nextConfig(1000)); + verifyConfig("foo", f3.h1->getConfig()); + verifyConfig("bar", f3.h2->getConfig()); + ASSERT_TRUE(f3.s.nextGeneration(0)); + verifyConfig("foo", f3.h1->getConfig()); + verifyConfig("bar", f3.h2->getConfig()); +} + +TEST_MT_FFF("requireThatConfigIsReturnedWhenUpdatedDuringNextConfig", 2, MyManager, APIFixture(f1), StandardFixture(f1, f2)) { + if (thread_id == 0) { + FastOS_Time timer; + timer.SetNow(); + ASSERT_TRUE(f3.s.nextConfig(10000)); + ASSERT_TRUE(timer.MilliSecsToNow() > 250); + ASSERT_TRUE(timer.MilliSecsToNow() <= 5000); + verifyConfig("foo2", f3.h1->getConfig()); + verifyConfig("bar", f3.h2->getConfig()); + } else { + FastOS_Thread::Sleep(300); + f1.updateValue(0, createFooValue("foo2"), 2); + FastOS_Thread::Sleep(300); + f1.updateGeneration(1, 2); + } +} + +TEST_FFF("requireThatConfigIsReturnedWhenUpdatedBeforeNextConfig", MyManager, APIFixture(f1), StandardFixture(f1, f2)) { + FastOS_Time timer; + timer.SetNow(); + ASSERT_FALSE(f3.s.nextConfig(1000)); + ASSERT_TRUE(timer.MilliSecsToNow() > 850); + f1.updateGeneration(0, 2); + f1.updateGeneration(1, 2); + timer.SetNow(); + ASSERT_TRUE(f3.s.nextGeneration(10000)); + ASSERT_TRUE(timer.MilliSecsToNow() <= 5000); + verifyConfig("foo", f3.h1->getConfig()); + verifyConfig("bar", f3.h2->getConfig()); +} + +TEST_FFF("requireThatSubscriptionsAreUnsubscribedOnClose", MyManager, APIFixture(f1), StandardFixture(f1, f2)) { + ASSERT_FALSE(f3.s.isClosed()); + f3.s.close(); + ASSERT_TRUE(f3.s.isClosed()); + ASSERT_EQUAL(2, f1.numCancel); +} + +TEST_FFF("requireThatNothingCanBeCalledAfterClose", MyManager, APIFixture(f1), StandardFixture(f1, f2)) { + ASSERT_FALSE(f3.s.isClosed()); + f3.s.close(); + ASSERT_TRUE(f3.s.isClosed()); + ASSERT_FALSE(f3.s.nextConfig(100)); + bool thrown = false; + try { + f3.h1->getConfig(); + } catch (const ConfigRuntimeException & e) { + thrown = true; + } + ASSERT_TRUE(thrown); +} + +TEST_MT_FFF("requireThatNextConfigIsInterruptedOnClose", 2, MyManager, APIFixture(f1), StandardFixture(f1, f2)) { + if (thread_id == 0) { + FastOS_Time timer; + timer.SetNow(); + ASSERT_FALSE(f3.s.nextConfig(5000)); + ASSERT_TRUE(timer.MilliSecsToNow() >= 500.0); + ASSERT_TRUE(timer.MilliSecsToNow() < 60000.0); + } else { + FastOS_Thread::Sleep(1000); + f3.s.close(); + } +} + +TEST_FF("requireThatHandlesAreMarkedAsChanged", MyManager, APIFixture(f1)) { + ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid2"); + ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid2"); + ASSERT_FALSE(s.nextConfig(0)); + + f1.updateValue(0, createFooValue("foo"), 1); + f1.updateValue(1, createFooValue("bar"), 1); + ASSERT_TRUE(s.nextConfig(100)); + ASSERT_TRUE(h1->isChanged()); + ASSERT_TRUE(h2->isChanged()); + + ASSERT_FALSE(s.nextConfig(100)); + ASSERT_FALSE(h1->isChanged()); + ASSERT_FALSE(h2->isChanged()); + f1.updateValue(0, createFooValue("bar"), 2); + f1.updateGeneration(1, 2); + ASSERT_TRUE(s.nextConfig(100)); + ASSERT_TRUE(h1->isChanged()); + ASSERT_FALSE(h2->isChanged()); +} + +TEST_FF("requireThatNextGenerationMarksChanged", MyManager, APIFixture(f1)) { + ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid2"); + ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid2"); + f1.updateValue(0, createFooValue("foo"), 1); + f1.updateValue(1, createFooValue("bar"), 1); + ASSERT_TRUE(s.nextGeneration(0)); + ASSERT_TRUE(h1->isChanged()); + ASSERT_TRUE(h2->isChanged()); + + f1.updateValue(0, createFooValue("bar"), 2); + f1.updateGeneration(1, 2); + ASSERT_TRUE(s.nextGeneration(0)); + ASSERT_TRUE(h1->isChanged()); + ASSERT_FALSE(h2->isChanged()); + + f1.updateGeneration(0, 3); + f1.updateGeneration(1, 3); + ASSERT_TRUE(s.nextGeneration(0)); + ASSERT_FALSE(h1->isChanged()); + ASSERT_FALSE(h2->isChanged()); +} + +TEST_FF("requireThatgetGenerationIsSet", MyManager, APIFixture(f1)) { + ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid2"); + ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid2"); + f1.updateValue(0, createFooValue("foo"), 1); + f1.updateValue(1, createFooValue("bar"), 1); + ASSERT_TRUE(s.nextGeneration(0)); + ASSERT_EQUAL(1, s.getGeneration()); + ASSERT_TRUE(h1->isChanged()); + ASSERT_TRUE(h2->isChanged()); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(1, 2); + ASSERT_FALSE(s.nextGeneration(0)); + ASSERT_EQUAL(1, s.getGeneration()); + f1.updateGeneration(0, 2); + ASSERT_TRUE(s.nextGeneration(0)); + ASSERT_EQUAL(2, s.getGeneration()); +} + +TEST_FFF("requireThatConfigHandleStillHasConfigOnTimestampUpdate", MyManager, APIFixture(f1), StandardFixture(f1, f2)) { + f1.updateGeneration(0, 2); + f1.updateGeneration(1, 2); + ASSERT_TRUE(f3.s.nextGeneration(0)); + verifyConfig("foo", f3.h1->getConfig()); + verifyConfig("bar", f3.h2->getConfig()); +} + +TEST_FF("requireThatTimeStamp0Works", MyManager, APIFixture(f1)) { + ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid"); + ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); + ConfigHandle<BazConfig>::UP h3 = s.subscribe<BazConfig>("myid"); + f1.updateValue(0, createBarValue("bar"), 0); + f1.updateValue(1, createFooValue("foo"), 0); + f1.updateValue(2, createBazValue("baz"), 0); + ASSERT_TRUE(s.nextConfig(0)); + verifyConfig("bar", h2->getConfig()); + verifyConfig("foo", h1->getConfig()); + verifyConfig("baz", h3->getConfig()); +} + +TEST_FF("requireThatNextGenerationWorksWithManyConfigs", MyManager, APIFixture(f1)) { + ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigHandle<BarConfig>::UP h2 = s.subscribe<BarConfig>("myid"); + ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); + ConfigHandle<BazConfig>::UP h3 = s.subscribe<BazConfig>("myid"); + f1.updateValue(0, createBarValue("bar"), 1); + f1.updateValue(1, createFooValue("foo"), 1); + f1.updateValue(2, createBazValue("baz"), 1); + ASSERT_TRUE(s.nextGeneration(100)); + verifyConfig("bar", h2->getConfig()); + verifyConfig("foo", h1->getConfig()); + verifyConfig("baz", h3->getConfig()); + int generation = 2; + + f1.updateGeneration(0, generation); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(1, generation); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(2, generation); + ASSERT_TRUE(s.nextGeneration(100)); + + generation++; + f1.updateGeneration(0, generation); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(2, generation); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(1, generation); + ASSERT_TRUE(s.nextGeneration(100)); + + generation++; + f1.updateGeneration(1, generation); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(0, generation); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(2, generation); + ASSERT_TRUE(s.nextGeneration(100)); + + generation++; + f1.updateGeneration(1, generation); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(2, generation); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(0, generation); + ASSERT_TRUE(s.nextGeneration(100)); + + generation++; + f1.updateGeneration(2, generation); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(0, generation); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(1, generation); + ASSERT_TRUE(s.nextGeneration(100)); + + generation++; + f1.updateGeneration(2, generation); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(1, generation); + ASSERT_FALSE(s.nextGeneration(0)); + f1.updateGeneration(0, generation); + ASSERT_TRUE(s.nextGeneration(100)); +} + +TEST_FF("requireThatConfigSubscriberHandlesProxyCache", MyManager, APIFixture(f1)) { + ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); + f1.updateValue(0, createFooValue("foo"), 1); + f1.updateGeneration(0, 2); + ASSERT_TRUE(s.nextConfig(0)); + ASSERT_EQUAL(2, s.getGeneration()); + ASSERT_TRUE(h1->isChanged()); + verifyConfig("foo", h1->getConfig()); + + f1.updateGeneration(0, 3); + ASSERT_TRUE(s.nextGeneration(0)); + ASSERT_EQUAL(3, s.getGeneration()); + ASSERT_FALSE(h1->isChanged()); + verifyConfig("foo", h1->getConfig()); +} + +TEST_MT_FF("requireThatConfigSubscriberWaitsUntilNextConfigSucceeds", 2, MyManager, APIFixture(f1)) { + if (thread_id == 0) { + ConfigSubscriber s(IConfigContext::SP(new APIFixture(f2))); + ConfigHandle<FooConfig>::UP h1 = s.subscribe<FooConfig>("myid"); + f1.updateValue(0, createFooValue("foo"), 1); + ASSERT_TRUE(s.nextConfig(0)); + f1.updateGeneration(0, 2); + ASSERT_FALSE(s.nextConfig(1000)); + TEST_BARRIER(); + ASSERT_TRUE(s.nextConfig(2000)); + verifyConfig("foo2", h1->getConfig()); // First update is skipped + } else { + TEST_BARRIER(); + FastOS_Thread::Sleep(1000); + f1.updateValue(0, createFooValue("foo2"), 3); + } +} + +TEST_MAIN() { + TEST_RUN_ALL(); +} diff --git a/config/src/tests/subscription/.gitignore b/config/src/tests/subscription/.gitignore new file mode 100644 index 00000000000..65556950180 --- /dev/null +++ b/config/src/tests/subscription/.gitignore @@ -0,0 +1,3 @@ +/config-my.cpp +/config-my.h +config_subscription_test_app diff --git a/config/src/tests/subscription/CMakeLists.txt b/config/src/tests/subscription/CMakeLists.txt new file mode 100644 index 00000000000..bbc3546c1da --- /dev/null +++ b/config/src/tests/subscription/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_subscription_test_app + SOURCES + subscription.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_subscription_test_app COMMAND config_subscription_test_app) +vespa_generate_config(config_subscription_test_app ../../test/resources/configdefinitions/my.def) diff --git a/config/src/tests/subscription/subscription.cpp b/config/src/tests/subscription/subscription.cpp new file mode 100644 index 00000000000..69cba9707fe --- /dev/null +++ b/config/src/tests/subscription/subscription.cpp @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/common/misc.h> +#include <vespa/config/common/configholder.h> +#include <vespa/config/subscription/configsubscription.h> +#include <config-my.h> + +using namespace config; + +namespace { + + struct SourceFixture + { + int numClose; + int numGetConfig; + int numReload; + SourceFixture() + : numClose(0), + numGetConfig(0), + numReload(0) + { } + }; + + struct MySource : public Source + { + MySource(SourceFixture * src) + : source(src) + {} + + void getConfig() { source->numGetConfig++; } + void close() { source->numClose++; } + void reload(int64_t gen) { (void) gen; source->numReload++; } + + SourceFixture * source; + }; + + struct SubscriptionFixture + { + IConfigHolder::SP holder; + ConfigSubscription sub; + SourceFixture src; + SubscriptionFixture(const ConfigKey & key) + : holder(new ConfigHolder()), + sub(0, key, holder, Source::UP(new MySource(&src))) + { + } + }; +} + +TEST_FF("requireThatKeyIsReturned", ConfigKey("foo", "bar", "bim", "boo"), SubscriptionFixture(f1)) +{ + ASSERT_TRUE(f1 == f2.sub.getKey()); +} + +TEST_F("requireThatUpdateReturns", SubscriptionFixture(ConfigKey::create<MyConfig>("myid"))) +{ + f1.holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), 1, 1))); + ASSERT_TRUE(f1.sub.nextUpdate(0, 0)); + ASSERT_TRUE(f1.sub.hasChanged()); + ASSERT_EQUAL(1, f1.sub.getGeneration()); +} + +TEST_F("requireThatNextUpdateBlocks", SubscriptionFixture(ConfigKey::create<MyConfig>("myid"))) +{ + ASSERT_FALSE(f1.sub.nextUpdate(0, 0)); + f1.holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), 1, 1))); + FastOS_Time timer; + timer.SetNow(); + ASSERT_FALSE(f1.sub.nextUpdate(1, 500)); + ASSERT_TRUE(timer.MilliSecsToNow() > 400.0); +} + +TEST_MT_F("requireThatNextUpdateReturnsWhenNotified", 2, SubscriptionFixture(ConfigKey::create<MyConfig>("myid"))) +{ + if (thread_id == 0) { + FastOS_Time timer; + timer.SetNow(); + f1.holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), 1, 1))); + ASSERT_TRUE(f1.sub.nextUpdate(2, 5000)); + ASSERT_TRUE(timer.MilliSecsToNow() > 200.0); + } else { + FastOS_Thread::Sleep(500); + f1.holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), 1, 1))); + } +} + + +TEST_MT_F("requireThatNextUpdateReturnsInterrupted", 2, SubscriptionFixture(ConfigKey::create<MyConfig>("myid"))) +{ + if (thread_id == 0) { + FastOS_Time timer; + timer.SetNow(); + f1.holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), 1, 1))); + ASSERT_TRUE(f1.sub.nextUpdate(1, 5000)); + ASSERT_TRUE(timer.MilliSecsToNow() > 300.0); + } else { + FastOS_Thread::Sleep(500); + f1.sub.close(); + } +} + +TEST_F("Require that isChanged takes generation into account", SubscriptionFixture(ConfigKey::create<MyConfig>("myid"))) +{ + f1.holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), true, 1))); + ASSERT_TRUE(f1.sub.nextUpdate(0, 0)); + f1.sub.flip(); + ASSERT_EQUAL(1, f1.sub.getLastGenerationChanged()); + f1.holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), true, 2))); + ASSERT_TRUE(f1.sub.nextUpdate(1, 0)); + f1.sub.flip(); + ASSERT_EQUAL(2, f1.sub.getLastGenerationChanged()); + f1.holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(), false, 3))); + ASSERT_TRUE(f1.sub.nextUpdate(2, 0)); + f1.sub.flip(); + ASSERT_EQUAL(2, f1.sub.getLastGenerationChanged()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/trace/.gitignore b/config/src/tests/trace/.gitignore new file mode 100644 index 00000000000..d87470776bf --- /dev/null +++ b/config/src/tests/trace/.gitignore @@ -0,0 +1 @@ +config_trace_test_app diff --git a/config/src/tests/trace/CMakeLists.txt b/config/src/tests/trace/CMakeLists.txt new file mode 100644 index 00000000000..b328f61abdc --- /dev/null +++ b/config/src/tests/trace/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_trace_test_app + SOURCES + trace.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_trace_test_app COMMAND config_trace_test_app) diff --git a/config/src/tests/trace/trace.cpp b/config/src/tests/trace/trace.cpp new file mode 100644 index 00000000000..9945acae485 --- /dev/null +++ b/config/src/tests/trace/trace.cpp @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("frt"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/common/trace.h> +#include <vespa/vespalib/trace/tracenode.h> + + +using namespace config; +using namespace vespalib; +using namespace vespalib::slime; + +struct FixedClock : public Clock +{ + FixedClock() : currentTime(0) { } + int64_t currentTime; + int64_t currentTimeMillis() const { return currentTime; } +}; + +TEST("that trace can be serialized and deserialized") { + Trace trace(4); + trace.trace(4, "foo"); + trace.trace(3, "bar"); + trace.trace(5, "baz"); + + Slime slime; + Cursor & cursor(slime.setObject()); + trace.serialize(cursor); + + Trace trace2; + trace2.deserialize(slime.get()); + + Slime slime2; + trace2.serialize(slime2.setObject()); + Trace trace3; + trace3.deserialize(slime2.get()); + + EXPECT_EQUAL(trace.toString(), trace3.toString()); +} + +TEST_F("that trace level is taken into account", FixedClock) { + f1.currentTime = 3; + Trace trace(4, f1); + trace.trace(4, "foo"); + trace.trace(5, "bar"); + EXPECT_EQUAL("[\n" +" {\n" +" \"timestamp\": 3,\n" +" \"payload\": \"foo\"\n" +" }\n" +"]\n", trace.toString()); +} + +TEST("that trace can be copied") { + Trace trace(3); + trace.trace(2, "foo"); + trace.trace(3, "bar"); + Trace trace2(trace); + EXPECT_EQUAL(trace.toString(), trace2.toString()); +} + +TEST("ensure that system clock is used by default") { + Trace trace(2); + trace.trace(1, "foo"); + TraceNode child(trace.getRoot().getChild(0)); + EXPECT_TRUE(child.getTimestamp() > 0); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/tests/unittest/.gitignore b/config/src/tests/unittest/.gitignore new file mode 100644 index 00000000000..d44770410b1 --- /dev/null +++ b/config/src/tests/unittest/.gitignore @@ -0,0 +1,7 @@ +/config-bar.cpp +/config-bar.h +/config-foo.cpp +/config-foo.h +/config-my.cpp +/config-my.h +config_unittest_test_app diff --git a/config/src/tests/unittest/CMakeLists.txt b/config/src/tests/unittest/CMakeLists.txt new file mode 100644 index 00000000000..9eeb5c761d1 --- /dev/null +++ b/config/src/tests/unittest/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(config_unittest_test_app + SOURCES + unittest.cpp + DEPENDS + config_cloudconfig +) +vespa_add_test(NAME config_unittest_test_app COMMAND config_unittest_test_app) +vespa_generate_config(config_unittest_test_app ../../test/resources/configdefinitions/my.def) +vespa_generate_config(config_unittest_test_app ../../test/resources/configdefinitions/foo.def) +vespa_generate_config(config_unittest_test_app ../../test/resources/configdefinitions/bar.def) diff --git a/config/src/tests/unittest/unittest.cpp b/config/src/tests/unittest/unittest.cpp new file mode 100644 index 00000000000..ca83dfee486 --- /dev/null +++ b/config/src/tests/unittest/unittest.cpp @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("unittest"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/config/config.h> +#include "config-my.h" +#include "config-foo.h" +#include "config-bar.h" + +using namespace config; + +namespace { + void verifyConfig(const std::string & expected, std::unique_ptr<FooConfig> cfg) + { + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL(expected, cfg->fooValue); + } + + void verifyConfig(const std::string & expected, std::unique_ptr<BarConfig> cfg) + { + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL(expected, cfg->barValue); + } +} +#if 0 +TEST("requireThatUnitTestsCanBeCreated") { + MyConfigBuilder builder; + builder.myField = "myval"; + ConfigSet set; + set.addBuilder("myid", &builder); + std::unique_ptr<MyConfig> cfg = ConfigGetter<MyConfig>::getConfig("myid", set); +} +#endif + +TEST("requireThatConfigCanBeReloaded") { + ConfigSet set; + ConfigContext::SP ctx(new ConfigContext(set)); + MyConfigBuilder builder; + builder.myField = "myfoo"; + set.addBuilder("myid", &builder); + ConfigSubscriber subscriber(ctx); + + ConfigHandle<MyConfig>::UP handle = subscriber.subscribe<MyConfig>("myid"); + ASSERT_TRUE(subscriber.nextConfig(0)); + std::unique_ptr<MyConfig> cfg(handle->getConfig()); + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL("myfoo", cfg->myField); + ctx->reload(); + ASSERT_FALSE(subscriber.nextConfig(1000)); + builder.myField = "foobar"; + ctx->reload(); + ASSERT_TRUE(subscriber.nextConfig(10000)); + cfg = handle->getConfig(); + ASSERT_TRUE(cfg.get() != NULL); + ASSERT_EQUAL("foobar", cfg->myField); +} + +TEST("requireThatCanSubscribeWithSameIdToDifferentDefs") { + ConfigSet set; + ConfigContext::SP ctx(new ConfigContext(set)); + FooConfigBuilder fooBuilder; + BarConfigBuilder barBuilder; + + fooBuilder.fooValue = "myfoo"; + barBuilder.barValue = "mybar"; + + set.addBuilder("fooid", &fooBuilder); + set.addBuilder("fooid", &barBuilder); + + ConfigSubscriber subscriber(ctx); + ConfigHandle<FooConfig>::UP h1 = subscriber.subscribe<FooConfig>("fooid"); + ConfigHandle<BarConfig>::UP h2 = subscriber.subscribe<BarConfig>("fooid"); + + ASSERT_TRUE(subscriber.nextConfig(0)); + verifyConfig("myfoo", h1->getConfig()); + verifyConfig("mybar", h2->getConfig()); + ctx->reload(); + ASSERT_FALSE(subscriber.nextConfig(100)); + + fooBuilder.fooValue = "blabla"; + ctx->reload(); + ASSERT_TRUE(subscriber.nextConfig(5000)); + verifyConfig("blabla", h1->getConfig()); + verifyConfig("mybar", h2->getConfig()); + + barBuilder.barValue = "blabar"; + ctx->reload(); + ASSERT_TRUE(subscriber.nextConfig(5000)); + verifyConfig("blabla", h1->getConfig()); + verifyConfig("blabar", h2->getConfig()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/config/src/vespa/config/.gitignore b/config/src/vespa/config/.gitignore new file mode 100644 index 00000000000..929512da880 --- /dev/null +++ b/config/src/vespa/config/.gitignore @@ -0,0 +1,4 @@ +/.depend +/Makefile +/libconfig.so.5.1 +/libcloudconfig.so.5.1 diff --git a/config/src/vespa/config/CMakeLists.txt b/config/src/vespa/config/CMakeLists.txt new file mode 100644 index 00000000000..6781d09aef1 --- /dev/null +++ b/config/src/vespa/config/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(config_cloudconfig + SOURCES + $<TARGET_OBJECTS:config_common> + $<TARGET_OBJECTS:config_subscription> + $<TARGET_OBJECTS:config_configgen> + $<TARGET_OBJECTS:config_raw> + $<TARGET_OBJECTS:config_file> + $<TARGET_OBJECTS:config_frt> + $<TARGET_OBJECTS:config_helper> + $<TARGET_OBJECTS:config_print> + $<TARGET_OBJECTS:config_set> + $<TARGET_OBJECTS:config_retriever> + INSTALL lib64 + DEPENDS +) diff --git a/config/src/vespa/config/common/.gitignore b/config/src/vespa/config/common/.gitignore new file mode 100644 index 00000000000..cd4bc99c04f --- /dev/null +++ b/config/src/vespa/config/common/.gitignore @@ -0,0 +1,2 @@ +/Makefile +/.depend diff --git a/config/src/vespa/config/common/CMakeLists.txt b/config/src/vespa/config/common/CMakeLists.txt new file mode 100644 index 00000000000..81a81456906 --- /dev/null +++ b/config/src/vespa/config/common/CMakeLists.txt @@ -0,0 +1,21 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(config_common OBJECT + SOURCES + configmanager.cpp + misc.cpp + configparser.cpp + errorcode.cpp + timingvalues.cpp + configupdate.cpp + configholder.cpp + configcontext.cpp + configkey.cpp + configvalue.cpp + trace.cpp + payload_converter.cpp + configdefinition.cpp + compressiontype.cpp + vespa_version.cpp + exceptions.cpp + DEPENDS +) diff --git a/config/src/vespa/config/common/cancelhandler.h b/config/src/vespa/config/common/cancelhandler.h new file mode 100644 index 00000000000..10bdd9d7775 --- /dev/null +++ b/config/src/vespa/config/common/cancelhandler.h @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace config { + +class ConfigSubscription; + +struct CancelHandler +{ + /** + * Cancels this subscription. Once this operation is done, the handler + * should have no knowledge of the subscription representing this id. + * + * @param subscription ConfigSubscription to cancel + */ + virtual void unsubscribe(const ConfigSubscription::SP & subscription) = 0; + + virtual ~CancelHandler() { } +}; + +} + diff --git a/config/src/vespa/config/common/compressiontype.cpp b/config/src/vespa/config/common/compressiontype.cpp new file mode 100644 index 00000000000..5ee8d75b631 --- /dev/null +++ b/config/src/vespa/config/common/compressiontype.cpp @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "compressiontype.h" + +namespace config { + +vespalib::string +compressionTypeToString(const CompressionType & compressionType) +{ + switch (compressionType) { + case CompressionType::UNCOMPRESSED: + return "UNCOMPRESSED"; + default: + return "LZ4"; + } +} + +CompressionType +stringToCompressionType(const vespalib::string & type) +{ + if (type.compare("UNCOMPRESSED") == 0) { + return CompressionType::UNCOMPRESSED; + } else { + return CompressionType::LZ4; + } +} + +} diff --git a/config/src/vespa/config/common/compressiontype.h b/config/src/vespa/config/common/compressiontype.h new file mode 100644 index 00000000000..d923c9c5e07 --- /dev/null +++ b/config/src/vespa/config/common/compressiontype.h @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace config { + +enum class CompressionType {UNCOMPRESSED, LZ4}; +vespalib::string compressionTypeToString(const CompressionType & compressionType); +CompressionType stringToCompressionType(const vespalib::string & type); + +} + diff --git a/config/src/vespa/config/common/configcontext.cpp b/config/src/vespa/config/common/configcontext.cpp new file mode 100644 index 00000000000..d770c5f1371 --- /dev/null +++ b/config/src/vespa/config/common/configcontext.cpp @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.common.configcontext"); +#include "configcontext.h" +#include "configmanager.h" +#include "exceptions.h" + +namespace config { + +ConfigContext::ConfigContext(const SourceSpec & spec) + : _timingValues(), + _generation(1), + _manager(spec.createSourceFactory(_timingValues), _generation) +{ +} + +ConfigContext::ConfigContext(const TimingValues & timingValues, const SourceSpec & spec) + : _timingValues(timingValues), + _generation(1), + _manager(spec.createSourceFactory(_timingValues), _generation) +{ +} + +IConfigManager & +ConfigContext::getManagerInstance() +{ + return _manager; +} + +void +ConfigContext::reload() +{ + _generation++; + _manager.reload(_generation); +} + +} // namespace config diff --git a/config/src/vespa/config/common/configcontext.h b/config/src/vespa/config/common/configcontext.h new file mode 100644 index 00000000000..6ada928fd5f --- /dev/null +++ b/config/src/vespa/config/common/configcontext.h @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include "timingvalues.h" +#include "configmanager.h" +#include <vespa/config/subscription/sourcespec.h> + +namespace config { + +/** + * A ConfigContext is a context object that can be used to consolidate + * multiple ConfigSubscribers to use the same resources. It also gives the + * ability to reload config for unit testing or if using file configs. + */ +class IConfigContext +{ +public: + typedef std::shared_ptr<IConfigContext> SP; + + /** + * Get an instance of the config manager. + * + * @return reference to a manager instance. + */ + virtual IConfigManager & getManagerInstance() = 0; + + /** + * Reload config for source provided by this context. + */ + virtual void reload() = 0; + + virtual ~IConfigContext() { } +}; + +class ConfigContext : public IConfigContext +{ +public: + ConfigContext(const SourceSpec & spec = ServerSpec()); + ConfigContext(const TimingValues & timingValues, const SourceSpec & spec = ServerSpec()); + IConfigManager & getManagerInstance(); + void reload(); + +private: + TimingValues _timingValues; + int64_t _generation; + ConfigManager _manager; +}; + + +} // namespace + diff --git a/config/src/vespa/config/common/configdefinition.cpp b/config/src/vespa/config/common/configdefinition.cpp new file mode 100644 index 00000000000..4784a86ede9 --- /dev/null +++ b/config/src/vespa/config/common/configdefinition.cpp @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "configdefinition.h" +#include <vespa/vespalib/stllike/asciistream.h> + +using namespace vespalib; +using namespace vespalib::slime; + +namespace config { + +ConfigDefinition::ConfigDefinition() + : _schema() +{} + +ConfigDefinition::ConfigDefinition(const std::vector<vespalib::string> & schema) + : _schema(schema) +{} + +void +ConfigDefinition::serialize(Cursor & cursor) const +{ + for (auto it(_schema.begin()), mt(_schema.end()); it != mt; it++) { + cursor.addString(Memory(*it)); + } +} + +void +ConfigDefinition::deserialize(const Inspector & inspector) +{ + for (size_t i(0); i < inspector.entries(); i++) { + _schema.push_back(inspector[i].asString().make_string()); + } +} + +vespalib::string +ConfigDefinition::asString() const +{ + vespalib::asciistream as; + for (auto it(_schema.begin()), mt(_schema.end()); it != mt; it++) { + as << *it; + } + return as.str(); +} + +} + diff --git a/config/src/vespa/config/common/configdefinition.h b/config/src/vespa/config/common/configdefinition.h new file mode 100644 index 00000000000..3c043eb6720 --- /dev/null +++ b/config/src/vespa/config/common/configdefinition.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vector> +#include <vespa/vespalib/data/slime/slime.h> + +namespace config { + +/** + * Represents a config definition. + */ +class ConfigDefinition { +public: + ConfigDefinition(); + ConfigDefinition(const std::vector<vespalib::string> & schema); + void deserialize(const vespalib::slime::Inspector & inspector); + void serialize(vespalib::slime::Cursor & cursor) const; + vespalib::string asString() const; +private: + std::vector<vespalib::string> _schema; +}; + +} //namespace config + diff --git a/config/src/vespa/config/common/configholder.cpp b/config/src/vespa/config/common/configholder.cpp new file mode 100644 index 00000000000..1b0d1e0d16b --- /dev/null +++ b/config/src/vespa/config/common/configholder.cpp @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "configholder.h" + +namespace config { + +ConfigHolder::ConfigHolder() + : _monitor(), + _current() +{ +} + +ConfigUpdate::UP +ConfigHolder::provide() +{ + vespalib::MonitorGuard guard(_monitor); + ConfigUpdate::UP ret(new ConfigUpdate(*_current)); + return ret; +} + +void +ConfigHolder::handle(ConfigUpdate::UP update) +{ + vespalib::MonitorGuard guard(_monitor); + _current = std::move(update); + guard.broadcast(); +} + +bool +ConfigHolder::wait(uint64_t timeoutInMillis) +{ + vespalib::MonitorGuard guard(_monitor); + return guard.wait(timeoutInMillis); +} + +bool +ConfigHolder::poll() +{ + vespalib::MonitorGuard guard(_monitor); + return (_current.get() != NULL); +} + +void +ConfigHolder::interrupt() +{ + vespalib::MonitorGuard guard(_monitor); + guard.broadcast(); +} + +} // namespace config diff --git a/config/src/vespa/config/common/configholder.h b/config/src/vespa/config/common/configholder.h new file mode 100644 index 00000000000..abcc9c059cd --- /dev/null +++ b/config/src/vespa/config/common/configholder.h @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "iconfigholder.h" +#include <vespa/vespalib/util/sync.h> + +namespace config { + +/** + * A config holder contains the latest config object of a subscription. + */ +class ConfigHolder : public IConfigHolder +{ +public: + ConfigHolder(); + + ConfigUpdate::UP provide(); + void handle(ConfigUpdate::UP update); + bool wait(uint64_t timeoutInMillis); + bool poll(); + void interrupt(); +public: + vespalib::Monitor _monitor; + ConfigUpdate::UP _current; +}; + +} // namespace config + diff --git a/config/src/vespa/config/common/configkey.cpp b/config/src/vespa/config/common/configkey.cpp new file mode 100644 index 00000000000..c22042418c0 --- /dev/null +++ b/config/src/vespa/config/common/configkey.cpp @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "configkey.h" + +namespace config { + +ConfigKey::ConfigKey(const vespalib::stringref & configId, + const vespalib::stringref & defName, + const vespalib::stringref & defNamespace, + const vespalib::stringref & defMd5) + : _configId(configId), + _defName(defName), + _defNamespace(defNamespace), + _defMd5(defMd5), + _defSchema(), + _key(_configId + _defName + _defNamespace) +{} + +ConfigKey::ConfigKey(const vespalib::stringref & configId, + const vespalib::stringref & defName, + const vespalib::stringref & defNamespace, + const vespalib::stringref & defMd5, + const std::vector<vespalib::string> & defSchema) + : _configId(configId), + _defName(defName), + _defNamespace(defNamespace), + _defMd5(defMd5), + _defSchema(defSchema), + _key(_configId + _defName + _defNamespace) +{ +} + +ConfigKey::ConfigKey() + : _configId(), + _defName(), + _defNamespace(), + _defMd5(), + _defSchema(), + _key() +{} + +bool +ConfigKey::operator<(const ConfigKey & rhs) const +{ + return _key < rhs._key; +} + +bool +ConfigKey::operator>(const ConfigKey & rhs) const +{ + return _key > rhs._key; +} + +bool +ConfigKey::operator==(const ConfigKey & rhs) const +{ + return _key.compare(rhs._key) == 0; +} + +const vespalib::string & ConfigKey::getDefName() const { return _defName; } +const vespalib::string & ConfigKey::getConfigId() const { return _configId; } +const vespalib::string & ConfigKey::getDefNamespace() const { return _defNamespace; } +const vespalib::string & ConfigKey::getDefMd5() const { return _defMd5; } +const std::vector<vespalib::string> & ConfigKey::getDefSchema() const { return _defSchema; } + +const vespalib::string +ConfigKey::toString() const +{ + vespalib::string s; + s.append("name="); + s.append(_defName); + s.append(",namespace="); + s.append(_defNamespace); + s.append(",configId="); + s.append(_configId); + return s; +} + +} diff --git a/config/src/vespa/config/common/configkey.h b/config/src/vespa/config/common/configkey.h new file mode 100644 index 00000000000..f6938a4fa1b --- /dev/null +++ b/config/src/vespa/config/common/configkey.h @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vector> + +namespace config { + +class ConfigKey { +public: + ConfigKey(const vespalib::stringref & configId, + const vespalib::stringref & defName, + const vespalib::stringref & defNamespace, + const vespalib::stringref & defMd5); + + ConfigKey(const vespalib::stringref & configId, + const vespalib::stringref & defName, + const vespalib::stringref & defNamespace, + const vespalib::stringref & defMd5, + const std::vector<vespalib::string> & defSchema); + + ConfigKey(); + + bool operator<(const ConfigKey & rhs) const; + bool operator>(const ConfigKey & rhs) const; + bool operator==(const ConfigKey & rhs) const; + + const vespalib::string & getDefName() const; + const vespalib::string & getConfigId() const; + const vespalib::string & getDefNamespace() const; + const vespalib::string & getDefMd5() const; + const std::vector<vespalib::string> & getDefSchema() const; + + template <typename ConfigType> + static const ConfigKey create(const vespalib::stringref & configId) + { + return ConfigKey(configId, ConfigType::CONFIG_DEF_NAME, + ConfigType::CONFIG_DEF_NAMESPACE, + ConfigType::CONFIG_DEF_MD5, + ConfigType::CONFIG_DEF_SCHEMA); + } + + const vespalib::string toString() const; +private: + vespalib::string _configId; + vespalib::string _defName; + vespalib::string _defNamespace; + vespalib::string _defMd5; + std::vector<vespalib::string> _defSchema; + vespalib::string _key; +}; + +} //namespace config + diff --git a/config/src/vespa/config/common/configmanager.cpp b/config/src/vespa/config/common/configmanager.cpp new file mode 100644 index 00000000000..e510079239f --- /dev/null +++ b/config/src/vespa/config/common/configmanager.cpp @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.common.configmanager"); +#include "configmanager.h" +#include "exceptions.h" +#include "configholder.h" +#include <vespa/vespalib/util/exception.h> +#include <vespa/vespalib/util/atomic.h> +#include <memory> + +namespace config { + +ConfigManager::ConfigManager(SourceFactory::UP sourceFactory, int64_t initialGeneration) + : _idGenerator(0), + _sourceFactory(std::move(sourceFactory)), + _generation(initialGeneration), + _subscriptionMap(), + _lock(), + _firstLock(), + _first(true) +{ +} + +ConfigSubscription::SP +ConfigManager::subscribe(const ConfigKey & key, uint64_t timeoutInMillis) +{ + LOG(debug, "subscribing on def %s, configid %s", key.getDefName().c_str(), key.getConfigId().c_str()); + + SubscriptionId id(vespalib::Atomic::postInc(&_idGenerator)); + + IConfigHolder::SP holder(new ConfigHolder()); + Source::UP source = _sourceFactory->createSource(holder, key); + source->reload(_generation); + + source->getConfig(); + ConfigSubscription::SP subscription(new ConfigSubscription(id, key, holder, std::move(source))); + + FastOS_Time timer; + timer.SetNow(); + while (timer.MilliSecsToNow() < timeoutInMillis) { + if (holder->poll()) + break; + FastOS_Thread::Sleep(10); + } + if (!holder->poll()) { + std::ostringstream oss; + oss << "Timed out while subscribing to '" << key.getDefNamespace() << "." << key.getDefName() << "', configid '" << key.getConfigId() << "'"; + throw ConfigTimeoutException(oss.str()); + } + LOG(debug, "done subscribing"); + vespalib::LockGuard guard(_lock); + _subscriptionMap[id] = subscription; + return subscription; +} + +void +ConfigManager::unsubscribe(const ConfigSubscription::SP & subscription) +{ + vespalib::LockGuard guard(_lock); + const SubscriptionId id(subscription->getSubscriptionId()); + if (_subscriptionMap.find(id) != _subscriptionMap.end()) + _subscriptionMap.erase(id); +} + +void +ConfigManager::reload(int64_t generation) +{ + _generation = generation; + vespalib::LockGuard guard(_lock); + for (SubscriptionMap::iterator it(_subscriptionMap.begin()), mt(_subscriptionMap.end()); it != mt; it++) { + it->second->reload(_generation); + } +} + +} // namespace config diff --git a/config/src/vespa/config/common/configmanager.h b/config/src/vespa/config/common/configmanager.h new file mode 100644 index 00000000000..2c3d5cbe297 --- /dev/null +++ b/config/src/vespa/config/common/configmanager.h @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/subscription/configsubscription.h> +#include "iconfigmanager.h" +#include "sourcefactory.h" +#include <vespa/vespalib/util/sync.h> +#include <map> + +namespace config { + +class SourceSpec; +struct TimingValues; + +/** + * An instance of this class represents a manager for config subscriptions, that use a common Source. + * Getting and/or creating a new instance is done by calling the factory method getInstance(Source). + * + * The manager holds a reference to each subscription. + */ +class ConfigManager : public IConfigManager +{ +public: + ConfigManager(SourceFactory::UP sourceFactory, int64_t initialGeneration); + + // Implements IConfigManager + ConfigSubscription::SP subscribe(const ConfigKey & key, uint64_t timeoutInMillis); + + // Implements IConfigManager + void unsubscribe(const ConfigSubscription::SP & subscription); + + // Implements IConfigManager + void reload(int64_t generation); + +private: + SubscriptionId _idGenerator; + SourceFactory::UP _sourceFactory; + int64_t _generation; + + typedef std::map<SubscriptionId, ConfigSubscription::SP> SubscriptionMap; + SubscriptionMap _subscriptionMap; + vespalib::Lock _lock; + vespalib::Lock _firstLock; + bool _first; +}; + +} // namespace config + diff --git a/config/src/vespa/config/common/configparser.cpp b/config/src/vespa/config/common/configparser.cpp new file mode 100644 index 00000000000..67237ef9fdc --- /dev/null +++ b/config/src/vespa/config/common/configparser.cpp @@ -0,0 +1,374 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/config/common/configparser.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <stdio.h> +#include "misc.h" + +namespace config { + +vespalib::string +ConfigParser::deQuote(const vespalib::stringref & source) +{ + const char *src = source.c_str(); + const char *s = src; + std::vector<char> dst(1+source.length()); + char *d = &dst[0]; + bool isQuoted; + + if (*s == '"') { + isQuoted = true; + ++s; + } else { + isQuoted = false; + } + + while (1) { + const char hexchars[] = "0123456789abcdefABCDEF"; + + char c = *s++; + if (isQuoted && c == '\\') { // Escape char only allowed in quotes + char escaped = *s++; + switch (escaped) { + case 'n': + *d++ = '\n'; + break; + case 'r': + *d++ = '\r'; + break; + case '\\': + *d++ = '\\'; + break; + case '"': + *d++ = '"'; + break; + case 'x': + // XXX we should have a utility routine for this + if (strchr(hexchars, s[0]) && strchr(hexchars, s[1])) { + unsigned int hex = 0; + sscanf(s, "%2x", &hex); + s += 2; + *d++ = hex; + } else { + throwInvalid("Invalid \\x escape \\x%.2s in %s", s, src); + } + break; + default: + throwInvalid("Invalid escape character in %s: \\%c", src, escaped); + break; + } + } else if (!c) { + if (isQuoted) { + throwInvalid("Unterminated quotes in (len=%u) '%s'", (uint32_t)strlen(src), src); + } + break; + } else if (c == '"') { + if (!isQuoted) { + throwInvalid("Quote character inside unquoted string in '%s'", src); + } + if (*s) throwInvalid("string must terminate after quotes: '%s'", src); + break; + } else { + *d++ = c; + } + } + *d = 0; + return vespalib::string(&dst[0], d - &dst[0]); +} + +namespace { + +bool +getValueForKey(const vespalib::stringref & key, const vespalib::stringref & line, + vespalib::string& retval) +{ + if (line.length() <= key.length()) { + return false; + } + + vespalib::stringref sub = line.substr(0, key.length()); + if (sub != key) { + return false; + } + + int pos = key.length(); + + if (line[pos] == ' ' || line[pos] == '.') { + retval = line.substr(pos + 1); + return true; + } + if (line[pos] == '[') { + retval = line.substr(pos); + // We don't need array declarations + if (retval[retval.size() - 1] == ']') + return false; + return true; + } + if (line[pos] == '{') { + retval = line.substr(pos); + // Skip empty maps + if (retval[retval.size() - 1] == '}') + return false; + return true; + } + + return false; +} + +} + +std::vector<vespalib::string> +ConfigParser::getLinesForKey(const vespalib::stringref & key, + const vsvector & lines) +{ + vsvector retval; + + for (uint32_t i = 0; i < lines.size(); i++) { + vespalib::string value; + + if (getValueForKey(key, lines[i], value)) { + retval.push_back(value); + } + } + + return retval; +} + +void +ConfigParser::stripLinesForKey(const vespalib::stringref & key, + std::set<vespalib::string>& config) +{ + vespalib::string value; + for (std::set<vespalib::string>::iterator it = config.begin(); it != config.end();) { + if (getValueForKey(key, *it, value)) { + std::set<vespalib::string>::iterator it2 = it++; + config.erase(it2); + } else { + ++it; + } + } +} + +std::map<vespalib::string, ConfigParser::vsvector> +ConfigParser::splitMap(const vsvector & config) +{ + std::map<vespalib::string, vsvector> items; + + vespalib::string lastValue; + + // First line contains item count, skip that. + for (uint32_t i = 0; i < config.size(); i++) { + size_t pos = config[i].find("}"); + + if (config[i].size() < 3 || config[i][0] != '{' + || pos == vespalib::string::npos) + { + throw InvalidConfigException( + "Value '" + config[i] + "' is not a valid map " + "specification.", VESPA_STRLOC); + } + + vespalib::string key = deQuote(config[i].substr(1, pos - 1)); + vespalib::string value = config[i].substr(pos + 1); + + if (key != lastValue) { + items[key] = vsvector(); + lastValue = key; + } + + if (value[0] == '.') { + items[key].push_back(value.substr(1)); + } else { + items[key].push_back(value); + } + } + return items; +} + +std::vector<ConfigParser::vsvector> +ConfigParser::splitArray(const vsvector & config) +{ + std::vector<vsvector> items; + + vespalib::string lastValue; + + // First line contains item count, skip that. + for (uint32_t i = 0; i < config.size(); i++) { + size_t pos = config[i].find("]"); + + if (config[i].size() < 3 || config[i][0] != '[' + || pos == vespalib::string::npos) + { + throw InvalidConfigException( + "Value '" + config[i] + "' is not a valid array " + "specification.", VESPA_STRLOC); + } + + vespalib::string key = config[i].substr(1, pos - 1); + vespalib::string value = config[i].substr(pos + 1); + + if (key != lastValue) { + items.push_back(vsvector()); + lastValue = key; + } + + if (value[0] == '.') { + items.back().push_back(value.substr(1)); + } else { + items.back().push_back(value); + } + } + return items; +} + +vespalib::string +ConfigParser::stripWhitespace(const vespalib::stringref & source) +{ + // Remove leading spaces and return. + if (source.empty()) { + return source; + } + size_t start = 0; + bool found = false; + while (!found && start < source.size()) { + switch (source[start]) { + case ' ': + case '\t': + case '\r': + case '\f': + ++start; + break; + default: + found = true; + } + } + size_t stop = source.size() - 1; + found = false; + while (!found && stop > start) { + switch (source[stop]) { + case ' ': + case '\t': + case '\r': + case '\f': + --stop; + break; + default: + found = true; + } + } + return source.substr(start, stop - start + 1); +} + +vespalib::string +ConfigParser::arrayToString(const vsvector & array) +{ + vespalib::asciistream ost; + if (array.size() == 0) { + ost << "No entries"; + } else { + for (uint32_t i=0; i<array.size(); ++i) { + ost << array[i] << "\n"; + } + } + return ost.str(); +} + +template<> +bool +ConfigParser::convert<bool>(const vsvector & config) +{ + if (config.size() != 1) { + throw InvalidConfigException("Expected single line with bool value, " + "got " + arrayToString(config), VESPA_STRLOC); + } + std::string value = stripWhitespace(deQuote(config[0])); + + if (value == "true") { + return true; + } else if (value == "false") { + return false; + } else { + throw InvalidConfigException("Expected bool value, got " + value + + "instead", VESPA_STRLOC); + } +} + +template<> +int32_t +ConfigParser::convert<int32_t>(const vsvector & config) +{ + if (config.size() != 1) { + throw InvalidConfigException("Expected single line with int32_t value, " + "got " + arrayToString(config), VESPA_STRLOC); + } + std::string value(deQuote(stripWhitespace(config[0]))); + + const char *startp = value.c_str(); + char *endp; + errno = 0; + int32_t ret = strtol(startp, &endp, 0); + int err = errno; + if (err == ERANGE || err == EINVAL || (*endp != '\0')) + throw InvalidConfigException("Value " + value + " is not a legal int32_t.", VESPA_STRLOC); + return ret; +} + +template<> +int64_t +ConfigParser::convert<int64_t>(const vsvector & config) +{ + if (config.size() != 1) { + throw InvalidConfigException("Expected single line with int64_t value, " + "got " + arrayToString(config), VESPA_STRLOC); + } + std::string value(deQuote(stripWhitespace(config[0]))); + + const char *startp = value.c_str(); + char *endp; + errno = 0; + int64_t ret = strtoll(startp, &endp, 0); + int err = errno; + if (err == ERANGE || err == EINVAL || (*endp != '\0')) + throw InvalidConfigException("Value " + value + " is not a legal int64_t.", VESPA_STRLOC); + return ret; +} + +template<> +double +ConfigParser::convert<double>(const vsvector & config) +{ + if (config.size() != 1) { + throw InvalidConfigException("Expected single line with double value, " + "got " + arrayToString(config), VESPA_STRLOC); + } + std::string value(deQuote(stripWhitespace(config[0]))); + + const char *startp = value.c_str(); + char *endp; + errno = 0; + double ret = strtod(startp, &endp); + int err = errno; + if (err == ERANGE || (*endp != '\0')) + throw InvalidConfigException("Value " + value + " is not a legal double", + VESPA_STRLOC); + return ret; +} + +template<> +vespalib::string +ConfigParser::convert<vespalib::string>(const vsvector & config) +{ + if (config.size() != 1) { + throw InvalidConfigException("Expected single line with string value, " + "got " + arrayToString(config), VESPA_STRLOC); + } + + std::string value = stripWhitespace(config[0]); + + return deQuote(value); +} + +template bool ConfigParser::convert<bool>(const vsvector & config); + +} // config diff --git a/config/src/vespa/config/common/configparser.h b/config/src/vespa/config/common/configparser.h new file mode 100644 index 00000000000..3b1185a48fa --- /dev/null +++ b/config/src/vespa/config/common/configparser.h @@ -0,0 +1,168 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/exceptions.h> +#include <map> +#include <set> +#include <vector> +#include <errno.h> +#include <stdint.h> +#include <vespa/vespalib/stllike/string.h> + +namespace config { + +/** + * To reduce the need for code in autogenerated config classes, these + * helper functions exist to help parsing. + */ +class ConfigParser { +public: + typedef std::vector<vespalib::string> vsvector; +private: + static vsvector getLinesForKey( const vespalib::stringref & key, const vsvector & config); + + static std::vector<vsvector> splitArray( const vsvector & config); + static std::map<vespalib::string, vsvector> splitMap( const vsvector & config); + + static vespalib::string deQuote(const vespalib::stringref & source); + + template<typename T> + static T convert(const vsvector &); + + static vespalib::string arrayToString(const vsvector &); + + template<typename T, typename V> + static T parseInternal(const vespalib::stringref & key, const V & config); + template<typename T, typename V> + static T parseInternal(const vespalib::stringref & key, const V & config, T defaultValue); + + template<typename T, typename V> + static std::vector<T> parseArrayInternal(const vespalib::stringref & key, const V & config); + template<typename T, typename V> + static std::map<vespalib::string, T> parseMapInternal(const vespalib::stringref & key, const V & config); + template<typename T, typename V> + static T parseStructInternal(const vespalib::stringref & key, const V & config); + +public: + static void stripLinesForKey(const vespalib::stringref & key, + std::set<vespalib::string>& config); + static vespalib::string stripWhitespace(const vespalib::stringref & source); + + template<typename T> + static T parse(const vespalib::stringref & key, const vsvector & config) { + return parseInternal<T, vsvector>(key, config); + } + template<typename T> + static T parse(const vespalib::stringref & key, const vsvector & config, T defaultValue) { + return parseInternal(key, config, defaultValue); + } + + template<typename T> + static std::vector<T> parseArray(const vespalib::stringref & key, const vsvector & config) { + return parseArrayInternal<T, vsvector>(key, config); + } + + template<typename T> + static std::map<vespalib::string, T> parseMap(const vespalib::stringref & key, const vsvector & config) { + return parseMapInternal<T, vsvector>(key, config); + } + + template<typename T> + static T parseStruct(const vespalib::stringref & key, const vsvector & config) { + return parseStructInternal<T, vsvector>(key, config); + } + +}; + +template<typename T, typename V> +T +ConfigParser::parseInternal(const vespalib::stringref & key, const V & config) +{ + V lines = getLinesForKey(key, config); + + if (lines.size() == 0) { + throw InvalidConfigException("Config parameter " + key + " has no " + "default value and is not specified in config", VESPA_STRLOC); + } + return convert<T>(lines); +} + +template<typename T, typename V> +T +ConfigParser::parseInternal(const vespalib::stringref & key, const V & config, T defaultValue) +{ + V lines = getLinesForKey(key, config); + + if (lines.size() == 0) { + return defaultValue; + } + + return convert<T>(lines); +} + +template<typename T> +T +ConfigParser::convert(const vsvector & lines) { + return T(lines); +} + +template<typename T, typename V> +std::map<vespalib::string, T> +ConfigParser::parseMapInternal(const vespalib::stringref & key, const V & config) +{ + V lines = getLinesForKey(key, config); + typedef std::map<vespalib::string, V> SplittedMap; + SplittedMap s = splitMap(lines); + std::map<vespalib::string, T> retval; + for (typename SplittedMap::iterator it(s.begin()), mt(s.end()); it != mt; it++) { + retval[it->first] = convert<T>(it->second); + } + return retval; +} + +template<typename T, typename V> +std::vector<T> +ConfigParser::parseArrayInternal(const vespalib::stringref & key, const V & config) +{ + V lines = getLinesForKey(key, config); + std::vector<V> split = splitArray(lines); + + std::vector<T> retval; + for (uint32_t i = 0; i < split.size(); i++) { + retval.push_back(convert<T>(split[i])); + } + + return retval; +} + +template<typename T, typename V> +T +ConfigParser::parseStructInternal(const vespalib::stringref & key, const V & config) +{ + V lines = getLinesForKey(key, config); + + return convert<T>(lines); +} + +template<> +bool +ConfigParser::convert<bool>(const vsvector & config); + +template<> +int32_t +ConfigParser::convert<int32_t>(const vsvector & config); + +template<> +int64_t +ConfigParser::convert<int64_t>(const vsvector & config); + +template<> +double +ConfigParser::convert<double>(const vsvector & config); + +template<> +vespalib::string +ConfigParser::convert<vespalib::string>(const vsvector & config); + +} // config + diff --git a/config/src/vespa/config/common/configrequest.h b/config/src/vespa/config/common/configrequest.h new file mode 100644 index 00000000000..94d9aa352a2 --- /dev/null +++ b/config/src/vespa/config/common/configrequest.h @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> + +namespace vespalib { + template <typename T> + class LinkedPtr; +} + +namespace config { + +class ConfigKey; +struct ConfigState; + +/** + * Baseclass for config requests. + * + * @author <a href="gv@yahoo-inc.com">Gjøran Voldengen</a> + * @version : $Id: configrequest.h 125304 2011-08-25 07:53:59Z lulf $ + */ + +class ConfigRequest { +private: + ConfigRequest& operator=(const ConfigRequest&); + +public: + typedef vespalib::LinkedPtr<ConfigRequest> LP; + typedef std::unique_ptr<ConfigRequest> UP; + + ConfigRequest() { } + virtual ~ConfigRequest() { } + + virtual const ConfigKey & getKey() const = 0; + + /** Abort a request. */ + virtual bool abort() = 0; + + virtual bool isAborted() const = 0; + + virtual void setError(int errorCode) = 0; +}; + +} + diff --git a/config/src/vespa/config/common/configresponse.h b/config/src/vespa/config/common/configresponse.h new file mode 100644 index 00000000000..20bc1464e16 --- /dev/null +++ b/config/src/vespa/config/common/configresponse.h @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace config { + +class ConfigValue; +class ConfigKey; +class ConfigState; +class Trace; + +/** + * Baseclass for config responses. + */ +class ConfigResponse { +public: + typedef std::unique_ptr<ConfigResponse> UP; + + virtual ~ConfigResponse() { } + + virtual const ConfigKey & getKey() const = 0; + virtual const ConfigValue & getValue() const = 0; + + virtual const ConfigState & getConfigState() const = 0; + virtual const Trace & getTrace() const = 0; + + virtual bool hasValidResponse() const = 0; + + /** + * Verifies that the returned response meets any criteria (decided by the implementation) to use the getters + * for return values. The result from this validation can be found without performing the validation + * again by calling {@link ConfigResponse#hasValidResponse()}. + * + * @return true if the returned response meets criteria to use the getters for return values. + */ + virtual bool validateResponse() = 0; + + /** + * Fills all data received in the response in order to be able to retrieve + * the config values. Should not be called before the response has been + * validated. + */ + virtual void fill() = 0; + + /** @return Error message if a request has failed, null otherwise. */ + virtual vespalib::string errorMessage() const = 0; + + virtual int errorCode() const = 0; + + virtual bool isError() const = 0; +}; + +} // namespace config + diff --git a/config/src/vespa/config/common/configstate.h b/config/src/vespa/config/common/configstate.h new file mode 100644 index 00000000000..39c206a8f83 --- /dev/null +++ b/config/src/vespa/config/common/configstate.h @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include "misc.h" + +namespace config { + +/** + * A config state represents the current state of a config instance + */ +struct ConfigState +{ +public: + ConfigState() + : md5(""), + generation(0) + { } + ConfigState(const vespalib::string & md5sum, int64_t gen) + : md5(md5sum), + generation(gen) + { } + + vespalib::string md5; + int64_t generation; + + bool isNewerGenerationThan(const ConfigState & other) const { + return isGenerationNewer(generation, other.generation); + } + + bool hasDifferentPayloadFrom(const ConfigState & other) const { + return (md5.compare(other.md5) != 0); + } +}; + +} // namespace config diff --git a/config/src/vespa/config/common/configupdate.cpp b/config/src/vespa/config/common/configupdate.cpp new file mode 100644 index 00000000000..87064f4e339 --- /dev/null +++ b/config/src/vespa/config/common/configupdate.cpp @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "configupdate.h" + +namespace config { + +ConfigUpdate::ConfigUpdate(const ConfigValue & value, bool changed, int64_t generation) + : _value(value), + _hasChanged(changed), + _generation(generation) +{ +} + +const ConfigValue & ConfigUpdate::getValue() const { return _value; } +bool ConfigUpdate::hasChanged() const { return _hasChanged; } +int64_t ConfigUpdate::getGeneration() const { return _generation; } + +} // namespace config diff --git a/config/src/vespa/config/common/configupdate.h b/config/src/vespa/config/common/configupdate.h new file mode 100644 index 00000000000..ecb66990e73 --- /dev/null +++ b/config/src/vespa/config/common/configupdate.h @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include "configvalue.h" + +namespace config { + +/** + * A config update contains a config value, and metadata saying if the value is + * changed or not. + */ +class ConfigUpdate +{ +public: + typedef std::unique_ptr<ConfigUpdate> UP; + ConfigUpdate(const ConfigValue & value, bool changed, int64_t generation); + + const ConfigValue & getValue() const; + bool hasChanged() const; + int64_t getGeneration() const; +private: + ConfigValue _value; + bool _hasChanged; + int64_t _generation; +}; + +} // namespace config + diff --git a/config/src/vespa/config/common/configvalue.cpp b/config/src/vespa/config/common/configvalue.cpp new file mode 100644 index 00000000000..6577982dc64 --- /dev/null +++ b/config/src/vespa/config/common/configvalue.cpp @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "configvalue.h" +#include "payload_converter.h" +#include "misc.h" + +namespace config { + +ConfigValue::ConfigValue(const std::vector<vespalib::string> & lines, const vespalib::string & md5sum) + : _payload(), + _lines(lines), + _md5sum(md5sum) +{} + +ConfigValue::ConfigValue() + : _payload(), + _lines(), + _md5sum() +{} + +ConfigValue::ConfigValue(const PayloadPtr & payload, const vespalib::string & md5) + : _payload(payload), + _lines(), + _md5sum(md5) +{ +} + +int +ConfigValue::operator==(const ConfigValue & rhs) const +{ + return (_md5sum.compare(rhs._md5sum) == 0); +} + +int +ConfigValue::operator!=(const ConfigValue & rhs) const +{ + return (!(*this == rhs)); +} + +std::vector<vespalib::string> +ConfigValue::getLegacyFormat() const +{ + std::vector<vespalib::string> lines; + if (_payload) { + const vespalib::slime::Inspector & payload(_payload->getSlimePayload()); + PayloadConverter converter(payload); + lines = converter.convert(); + } else { + lines = _lines; + } + return lines; +} + +const vespalib::string +ConfigValue::asJson() const { + const vespalib::slime::Inspector & payload(_payload->getSlimePayload()); + return payload.toString(); +} + +void +ConfigValue::serializeV1(vespalib::slime::Cursor & cursor) const +{ + // TODO: Remove v1 when we can bump disk format. + std::vector<vespalib::string> lines(getLegacyFormat()); + for (size_t i = 0; i < lines.size(); i++) { + cursor.addString(vespalib::slime::Memory(lines[i])); + } +} + +void +ConfigValue::serializeV2(vespalib::slime::Cursor & cursor) const +{ + copySlimeObject(_payload->getSlimePayload(), cursor); +} + +} + diff --git a/config/src/vespa/config/common/configvalue.h b/config/src/vespa/config/common/configvalue.h new file mode 100644 index 00000000000..b6315c51081 --- /dev/null +++ b/config/src/vespa/config/common/configvalue.h @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vector> +#include <memory> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/config/frt/protocol.h> +#include <vespa/config/configgen/configpayload.h> + +namespace config { + +typedef std::shared_ptr<const protocol::Payload> PayloadPtr; + +/** + * Internal representation of a config value. DO NOT USE THIS!!!!! Use readers + * if you want to instantiate config objects directly. + */ +class ConfigValue { +public: + typedef std::unique_ptr<ConfigValue> UP; + ConfigValue(const std::vector<vespalib::string> & lines, const vespalib::string & md5sum); + ConfigValue(const PayloadPtr & data, const vespalib::string & md5sum); + ConfigValue(); + + int operator==(const ConfigValue & rhs) const; + int operator!=(const ConfigValue & rhs) const; + + size_t numLines() const { return _lines.size(); } + const vespalib::string & getLine(int i) const { return _lines.at(i); } + const std::vector<vespalib::string> & getLines() const { return _lines; } + std::vector<vespalib::string> getLegacyFormat() const; + const vespalib::string asJson() const; + const vespalib::string getMd5() const { return _md5sum; } + + void serializeV1(::vespalib::slime::Cursor & cursor) const; + void serializeV2(::vespalib::slime::Cursor & cursor) const; + + template <typename ConfigType> + std::unique_ptr<ConfigType> newInstance() const; + +private: + PayloadPtr _payload; + std::vector<vespalib::string> _lines; + vespalib::string _md5sum; +}; + +} //namespace config + +#include "configvalue.hpp" + diff --git a/config/src/vespa/config/common/configvalue.hpp b/config/src/vespa/config/common/configvalue.hpp new file mode 100644 index 00000000000..f464b1ddee3 --- /dev/null +++ b/config/src/vespa/config/common/configvalue.hpp @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace config { + +template <typename ConfigType> +std::unique_ptr<ConfigType> +ConfigValue::newInstance() const +{ + if (_payload) { + const vespalib::slime::Inspector & payload(_payload->getSlimePayload()); + return std::unique_ptr<ConfigType>(new ConfigType(config::ConfigPayload(payload))); + } else { + return std::unique_ptr<ConfigType>(new ConfigType(*this)); + } +} + +} diff --git a/config/src/vespa/config/common/errorcode.cpp b/config/src/vespa/config/common/errorcode.cpp new file mode 100644 index 00000000000..07907650f50 --- /dev/null +++ b/config/src/vespa/config/common/errorcode.cpp @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author Gunnar Gauslaa Bergem + * @date 2008-05-22 + * @version $Id: errorcode.cpp 119439 2011-04-19 09:32:27Z arnej $ + */ + +#include <vespa/fastos/fastos.h> +#include "errorcode.h" + +namespace config { + +vespalib::string ErrorCode::getName(int error) { + switch(error) { + case UNKNOWN_CONFIG: return "UNKNOWN_CONFIG"; + case UNKNOWN_DEFINITION: return "UNKNOWN_DEFINITION"; + case UNKNOWN_VERSION: return "UNKNOWN_VERSION"; + case UNKNOWN_CONFIGID: return "UNKNOWN_CONFIGID"; + case UNKNOWN_DEF_MD5: return "UNKNOWN_DEF_MD5"; + case UNKNOWN_VESPA_VERSION: return "UNKNOWN_VESPA_VERSION"; + case ILLEGAL_NAME: return "ILLEGAL_NAME"; + case ILLEGAL_VERSION: return "ILLEGAL_VERSION"; + case ILLEGAL_CONFIGID: return "ILLEGAL_CONFIGID"; + case ILLEGAL_DEF_MD5: return "ILLEGAL_DEF_MD5"; + case ILLEGAL_CONFIG_MD5: return "ILLEGAL_CONFIG_MD5"; + case ILLEGAL_TIMEOUT: return "ILLEGAL_TIMEOUT"; + case ILLEGAL_TIMESTAMP: return "ILLEGAL_TIMESTAMP"; + case ILLEGAL_NAME_SPACE: return "ILLEGAL_NAME_SPACE"; + case ILLEGAL_PROTOCOL_VERSION: return "ILLEGAL_PROTOCOL_VERSION"; + case ILLEGAL_CLIENT_HOSTNAME: return "ILLEGAL_CLIENT_HOSTNAME"; + case OUTDATED_CONFIG: return "OUTDATED_CONFIG"; + case INTERNAL_ERROR: return "INTERNAL_ERROR"; + case APPLICATION_NOT_LOADED: return "APPLICATION_NOT_LOADED"; + case INCONSISTENT_CONFIG_MD5: return "INCONSISTENT_CONFIG_MD5"; + default: return "Unknown error"; + } +} + +} diff --git a/config/src/vespa/config/common/errorcode.h b/config/src/vespa/config/common/errorcode.h new file mode 100644 index 00000000000..401e68358a6 --- /dev/null +++ b/config/src/vespa/config/common/errorcode.h @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author Gunnar Gauslaa Bergem + * @date 2008-05-22 + * @version $Id: errorcode.h 119465 2011-04-20 15:21:46Z arnej $ + */ + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace config { + +class ErrorCode { +private: + ErrorCode(); + ErrorCode(const ErrorCode &); + +public: + static const int UNKNOWN_CONFIG = 100000; + static const int UNKNOWN_DEFINITION = UNKNOWN_CONFIG + 1; + static const int UNKNOWN_VERSION = UNKNOWN_CONFIG + 2; + static const int UNKNOWN_CONFIGID = UNKNOWN_CONFIG + 3; + static const int UNKNOWN_DEF_MD5 = UNKNOWN_CONFIG + 4; + static const int UNKNOWN_VESPA_VERSION = UNKNOWN_CONFIG + 5; + + static const int ILLEGAL_NAME = UNKNOWN_CONFIG + 100; + static const int ILLEGAL_VERSION = UNKNOWN_CONFIG + 101; + static const int ILLEGAL_CONFIGID = UNKNOWN_CONFIG + 102; + static const int ILLEGAL_DEF_MD5 = UNKNOWN_CONFIG + 103; + static const int ILLEGAL_CONFIG_MD5 = UNKNOWN_CONFIG + 104; + static const int ILLEGAL_TIMEOUT = UNKNOWN_CONFIG + 105; + static const int ILLEGAL_TIMESTAMP = UNKNOWN_CONFIG + 106; + + static const int ILLEGAL_NAME_SPACE = UNKNOWN_CONFIG + 108; + static const int ILLEGAL_PROTOCOL_VERSION = UNKNOWN_CONFIG + 109; + static const int ILLEGAL_CLIENT_HOSTNAME = UNKNOWN_CONFIG + 110; + + // hasUpdatedConfig() is true, but timestamp says the config is older than previous config + static const int OUTDATED_CONFIG = UNKNOWN_CONFIG + 150; + + static const int INTERNAL_ERROR = UNKNOWN_CONFIG + 200; + + static const int APPLICATION_NOT_LOADED = UNKNOWN_CONFIG + 300; + + static const int INCONSISTENT_CONFIG_MD5 = UNKNOWN_CONFIG + 400; + + static vespalib::string getName(int error); +}; + +} + diff --git a/config/src/vespa/config/common/exceptions.cpp b/config/src/vespa/config/common/exceptions.cpp new file mode 100644 index 00000000000..d19c27b5999 --- /dev/null +++ b/config/src/vespa/config/common/exceptions.cpp @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "exceptions.h" + +namespace config { + +VESPA_IMPLEMENT_EXCEPTION(InvalidConfigException, vespalib::Exception); + +VESPA_IMPLEMENT_EXCEPTION(IllegalConfigKeyException, vespalib::Exception); + +VESPA_IMPLEMENT_EXCEPTION(ConfigRuntimeException, vespalib::Exception); + +VESPA_IMPLEMENT_EXCEPTION(InvalidConfigSourceException, vespalib::Exception); + +VESPA_IMPLEMENT_EXCEPTION(ConfigWriteException, vespalib::Exception); + +VESPA_IMPLEMENT_EXCEPTION(ConfigReadException, vespalib::Exception); + +VESPA_IMPLEMENT_EXCEPTION(ConfigTimeoutException, ConfigRuntimeException); + +} + diff --git a/config/src/vespa/config/common/exceptions.h b/config/src/vespa/config/common/exceptions.h new file mode 100644 index 00000000000..c3fafab1287 --- /dev/null +++ b/config/src/vespa/config/common/exceptions.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <string> +#include <stdexcept> +#include <vespa/vespalib/util/exception.h> + +namespace config { + +VESPA_DEFINE_EXCEPTION(InvalidConfigException, vespalib::Exception); + +VESPA_DEFINE_EXCEPTION(IllegalConfigKeyException, vespalib::Exception); + +VESPA_DEFINE_EXCEPTION(ConfigRuntimeException, vespalib::Exception); + +VESPA_DEFINE_EXCEPTION(InvalidConfigSourceException, vespalib::Exception); + +VESPA_DEFINE_EXCEPTION(ConfigWriteException, vespalib::Exception); + +VESPA_DEFINE_EXCEPTION(ConfigReadException, vespalib::Exception); + +VESPA_DEFINE_EXCEPTION(ConfigTimeoutException, ConfigRuntimeException); + +} + diff --git a/config/src/vespa/config/common/handler.h b/config/src/vespa/config/common/handler.h new file mode 100644 index 00000000000..ecd6351b882 --- /dev/null +++ b/config/src/vespa/config/common/handler.h @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> +#include "configupdate.h" + +namespace config { + +/** + * A Handler is a component to whom you can pass an object. + **/ +template <typename T> +struct Handler +{ + virtual void handle(std::unique_ptr<T> obj) = 0; + virtual ~Handler() {} +}; + +typedef Handler<ConfigUpdate> ConfigHandler; + +} // namespace config + diff --git a/config/src/vespa/config/common/iconfigholder.h b/config/src/vespa/config/common/iconfigholder.h new file mode 100644 index 00000000000..b455dd9cd01 --- /dev/null +++ b/config/src/vespa/config/common/iconfigholder.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/handler.h> +#include <vespa/config/common/provider.h> +#include <vespa/config/common/waitable.h> +#include <vespa/config/common/pollable.h> +#include <vespa/config/common/interruptable.h> +#include <vespa/config/common/configupdate.h> + +namespace config { + +class IConfigHolder : public ConfigHandler, + public Provider<ConfigUpdate>, + public Waitable, + public Pollable, + public Interruptable +{ +public: + typedef std::shared_ptr<IConfigHolder> SP; + virtual ~IConfigHolder() { } +}; + +} // namespace config + diff --git a/config/src/vespa/config/common/iconfigmanager.h b/config/src/vespa/config/common/iconfigmanager.h new file mode 100644 index 00000000000..15ddd0f251a --- /dev/null +++ b/config/src/vespa/config/common/iconfigmanager.h @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/subscribehandler.h> +#include <vespa/config/common/cancelhandler.h> +#include <vespa/config/common/reloadhandler.h> + +namespace config { + +class IConfigManager : public SubscribeHandler, + public CancelHandler, + public ReloadHandler +{ +public: + virtual ~IConfigManager() { } +}; + +} // namespace config + diff --git a/config/src/vespa/config/common/interruptable.h b/config/src/vespa/config/common/interruptable.h new file mode 100644 index 00000000000..32af06c1d06 --- /dev/null +++ b/config/src/vespa/config/common/interruptable.h @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> + +namespace config { + +/** + * An Interruptable is a component that can be notified that it should abort its current activities. + */ +struct Interruptable +{ + virtual void interrupt() = 0; + virtual ~Interruptable() {} +}; + +} // namespace config + diff --git a/config/src/vespa/config/common/misc.cpp b/config/src/vespa/config/common/misc.cpp new file mode 100644 index 00000000000..fb226a9734d --- /dev/null +++ b/config/src/vespa/config/common/misc.cpp @@ -0,0 +1,160 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "misc.h" +#include <vespa/vespalib/util/md5.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.common.misc"); + +namespace config { + +vespalib::string +calculateContentMd5(const std::vector<vespalib::string> & fileContents) +{ + vespalib::string normalizedLines; + int compact_md5size = 16; + unsigned char md5sum[compact_md5size]; + vespalib::asciistream s; + + // remove comments, trailing spaces and empty lines + // TODO: Remove multiple spaces and space before comma, like in Java + for (int i = 0; i < (int)fileContents.size(); i++) { + std::string line = fileContents[i]; + line = line.erase(line.find_last_not_of("#") + 1); + line = line.erase(line.find_last_not_of(" ") + 1); + if (line.size() > 0) { + line += "\n"; + normalizedLines += line; + } + } + fastc_md5sum((const unsigned char*)normalizedLines.c_str(), normalizedLines.size(), md5sum); + + // convert to 32 character hex string + for (int i = 0; i < compact_md5size; i++) { + if (md5sum[i] < 16) { + s << "0"; + } + s << std::hex << (int)md5sum[i]; + } + return s.str(); +} + +bool +isGenerationNewer(int64_t newGen, int64_t oldGen) +{ + return (newGen > oldGen) || (newGen == 0); +} + +void +throwInvalid(const char *format, ...) + throw(InvalidConfigException) +{ + char buf[4000]; + va_list args; + + va_start(args, format); + vsnprintf(buf, sizeof buf, format, args); + va_end(args); + LOG(warning, "Error in configuration: %s", buf); + + throw InvalidConfigException(buf); +} + +using namespace vespalib::slime; + +void copySlimeArray(const Inspector & src, Cursor & dest); + +class CopyObjectTraverser : public ObjectTraverser +{ +private: + Cursor & _dest; +public: + CopyObjectTraverser(Cursor & dest) : _dest(dest) {} + void field(const Memory & symbol, const Inspector & inspector) { + switch(inspector.type().getId()) { + case NIX::ID: + _dest.addNix(); + break; + case BOOL::ID: + _dest.setBool(symbol, inspector.asBool()); + break; + case LONG::ID: + _dest.setLong(symbol, inspector.asLong()); + break; + case DOUBLE::ID: + _dest.setDouble(symbol, inspector.asDouble()); + break; + case STRING::ID: + _dest.setString(symbol, inspector.asString()); + break; + case DATA::ID: + _dest.setData(symbol, inspector.asData()); + break; + case ARRAY::ID: + copySlimeArray(inspector, _dest.setArray(symbol)); + break; + case OBJECT::ID: + copySlimeObject(inspector, _dest.setObject(symbol)); + break; + } + } +}; + +class CopyArrayTraverser : public ArrayTraverser +{ +private: + Cursor & _dest; +public: + CopyArrayTraverser(Cursor & dest) : _dest(dest) {} + void entry(size_t idx, const Inspector & inspector) { + (void) idx; + switch(inspector.type().getId()) { + case NIX::ID: + _dest.addNix(); + break; + case BOOL::ID: + _dest.addBool(inspector.asBool()); + break; + case LONG::ID: + _dest.addLong(inspector.asLong()); + break; + case DOUBLE::ID: + _dest.addDouble(inspector.asDouble()); + break; + case STRING::ID: + _dest.addString(inspector.asString()); + break; + case DATA::ID: + _dest.addData(inspector.asData()); + break; + case ARRAY::ID: + copySlimeArray(inspector, _dest.addArray()); + break; + case OBJECT::ID: + copySlimeObject(inspector, _dest.addObject()); + break; + } + } +}; + +void copySlimeArray(const Inspector & src, Cursor & dest) +{ + if (src.type().getId() != ARRAY::ID) { + throw vespalib::IllegalArgumentException("Source inspector is not of type array"); + } + CopyArrayTraverser traverser(dest); + src.traverse(traverser); +} + + +void copySlimeObject(const Inspector & src, Cursor & dest) +{ + if (src.type().getId() != OBJECT::ID) { + throw vespalib::IllegalArgumentException("Source inspector is not of type object"); + } + CopyObjectTraverser traverser(dest); + src.traverse(traverser); +} + +} diff --git a/config/src/vespa/config/common/misc.h b/config/src/vespa/config/common/misc.h new file mode 100644 index 00000000000..0a9d6182299 --- /dev/null +++ b/config/src/vespa/config/common/misc.h @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vector> +#include "exceptions.h" +#include "configkey.h" + +namespace config { + +/** + * Miscellaneous utility functions specific to config. + */ +vespalib::string calculateContentMd5(const std::vector<vespalib::string> & fileContents); + +bool isGenerationNewer(int64_t newGen, int64_t oldGen); + +// Helper for throwing invalid config exception +void throwInvalid(const char *fmt, ...) throw (InvalidConfigException) + __attribute__((format(printf, 1, 2))) __attribute__((noreturn)); + +typedef std::shared_ptr<const vespalib::Slime> SlimePtr; + +/** + * Copy slime objects from under src to dest, recursively. + */ +void copySlimeObject(const vespalib::slime::Inspector & src, vespalib::slime::Cursor & dest); + +} // namespace config + diff --git a/config/src/vespa/config/common/payload_converter.cpp b/config/src/vespa/config/common/payload_converter.cpp new file mode 100644 index 00000000000..2f7feb6a1b0 --- /dev/null +++ b/config/src/vespa/config/common/payload_converter.cpp @@ -0,0 +1,147 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "payload_converter.h" + +using namespace vespalib::slime; + +namespace config { + +PayloadConverter::PayloadConverter(const Inspector & inspector) + : _inspector(inspector), + _lines() +{} + +const std::vector<vespalib::string> & +PayloadConverter::convert() +{ + _lines.clear(); + ObjectTraverser & traverser(*this); + _inspector.traverse(traverser); + return _lines; +} + +void +PayloadConverter::encodeObject(const Memory & symbol, const Inspector & object) +{ + _nodeStack.push_back(Node(symbol.make_string())); + ObjectTraverser & traverser(*this); + object.traverse(traverser); + _nodeStack.pop_back(); +} + +void +PayloadConverter::encodeArray(const Memory & symbol, const Inspector & array) +{ + _nodeStack.push_back(Node(symbol.make_string())); + ArrayTraverser & traverser(*this); + array.traverse(traverser); + _nodeStack.pop_back(); +} + +void +PayloadConverter::encode(const Inspector & inspector) +{ + ObjectTraverser & traverser(*this); + switch (inspector.type().getId()) { + case OBJECT::ID: + inspector.traverse(traverser); + break; + default: + encodeValue(inspector); + break; + } +} + +void +PayloadConverter::encode(const Memory & symbol, const Inspector & inspector) +{ + switch (inspector.type().getId()) { + case OBJECT::ID: + encodeObject(symbol, inspector); + break; + case ARRAY::ID: + encodeArray(symbol, inspector); + break; + default: + _nodeStack.push_back(Node(symbol.make_string())); + encodeValue(inspector); + _nodeStack.pop_back(); + break; + } +} + +void +PayloadConverter::field(const Memory& symbol, const Inspector & inspector) +{ + encode(symbol, inspector); +} + +void +PayloadConverter::entry(size_t idx, const Inspector & inspector) +{ + _nodeStack.push_back(Node(idx)); + encode(inspector); + _nodeStack.pop_back(); +} + +void +PayloadConverter::printPrefix() +{ + for (size_t i = 0; i < _nodeStack.size(); i++) { + bool first = (i == 0); + Node & node(_nodeStack[i]); + if (node.arrayIndex >= 0) { + encodeString("["); + encodeLong(node.arrayIndex); + encodeString("]"); + } else { + if (!first) { + encodeString("."); + } + encodeString(node.name); + } + } + encodeString(" "); +} + +void +PayloadConverter::encodeValue(const Inspector & value) +{ + printPrefix(); + switch (value.type().getId()) { + case STRING::ID: + encodeQuotedString(value.asString().make_string()); + break; + case LONG::ID: + encodeLong(value.asLong()); + break; + case DOUBLE::ID: + encodeDouble(value.asDouble()); + break; + case BOOL::ID: + encodeBool(value.asBool()); + break; + } + _lines.push_back(_buf.str()); + _buf.clear(); +} + +void +PayloadConverter::encodeString(const vespalib::string & value) +{ + _buf << value; +} + +void PayloadConverter::encodeLong(long value) { _buf << value; } +void PayloadConverter::encodeDouble(double value) { _buf << value; } +void PayloadConverter::encodeBool(bool value) { _buf << (value ? "true" : "false"); } + +void +PayloadConverter::encodeQuotedString(const vespalib::string & value) +{ + encodeString("\""); + encodeString(value); + encodeString("\""); +} + + +} // namespace config diff --git a/config/src/vespa/config/common/payload_converter.h b/config/src/vespa/config/common/payload_converter.h new file mode 100644 index 00000000000..e53705fc633 --- /dev/null +++ b/config/src/vespa/config/common/payload_converter.h @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/asciistream.h> + +namespace config { + +/** + * Converts slime payload to cfg format. + * XXX: Maps are not supported by this converter. + */ +class PayloadConverter : public vespalib::slime::ObjectTraverser, public vespalib::slime::ArrayTraverser { +public: + PayloadConverter(const vespalib::slime::Inspector & inspector); + const std::vector<vespalib::string> & convert(); + void field(const vespalib::slime::Memory & symbol, const vespalib::slime::Inspector & inspector); + void entry(size_t idx, const vespalib::slime::Inspector & inspector); +private: + void printPrefix(); + void encode(const vespalib::slime::Inspector & inspector); + void encode(const vespalib::slime::Memory & symbol, const vespalib::slime::Inspector & inspector); + void encodeObject(const vespalib::slime::Memory & symbol, const vespalib::slime::Inspector & object); + void encodeArray(const vespalib::slime::Memory & symbol, const vespalib::slime::Inspector & object); + void encodeValue(const vespalib::slime::Inspector & value); + void encodeString(const vespalib::string & value); + void encodeQuotedString(const vespalib::string & value); + void encodeLong(long value); + void encodeDouble(double value); + void encodeBool(bool value); + struct Node { + vespalib::string name; + int arrayIndex; + Node(const vespalib::string & nm, int idx) : name(nm), arrayIndex(idx) {} + Node(int idx) : name(""), arrayIndex(idx) {} + Node(const vespalib::string & nm) : name(nm), arrayIndex(-1) {} + }; + const vespalib::slime::Inspector & _inspector; + std::vector<vespalib::string> _lines; + typedef std::vector<Node> NodeStack; + NodeStack _nodeStack; + vespalib::asciistream _buf; +}; + +} // namespace config + diff --git a/config/src/vespa/config/common/pollable.h b/config/src/vespa/config/common/pollable.h new file mode 100644 index 00000000000..8814a3a77ac --- /dev/null +++ b/config/src/vespa/config/common/pollable.h @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> + +namespace config { + +/** + * A Pollable is a component that can be polled, and returns either true or + * false. + */ +struct Pollable +{ + virtual bool poll() = 0; + virtual ~Pollable() {} +}; + +} // namespace config + diff --git a/config/src/vespa/config/common/provider.h b/config/src/vespa/config/common/provider.h new file mode 100644 index 00000000000..5eebcc614f4 --- /dev/null +++ b/config/src/vespa/config/common/provider.h @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> + +namespace config { + +/** + * A Provider is a component from which you can request an object. + **/ +template <typename T> +struct Provider +{ + virtual std::unique_ptr<T> provide() = 0; + virtual ~Provider() {} +}; + +} // namespace config + diff --git a/config/src/vespa/config/common/reloadhandler.h b/config/src/vespa/config/common/reloadhandler.h new file mode 100644 index 00000000000..e64264fb2c0 --- /dev/null +++ b/config/src/vespa/config/common/reloadhandler.h @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace config { + +struct ReloadHandler +{ + /** + * Reload any configs with a given generation. + */ + virtual void reload(int64_t generation) = 0; + virtual ~ReloadHandler() { } +}; + +} + diff --git a/config/src/vespa/config/common/source.h b/config/src/vespa/config/common/source.h new file mode 100644 index 00000000000..a1c0633d47b --- /dev/null +++ b/config/src/vespa/config/common/source.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/fastos.h> +#include <memory> + +namespace config { + +/* + * Class representing a source from which constructs config requests and request + * handlers. + */ +class Source { +public: + typedef std::unique_ptr<Source> UP; + + virtual void getConfig() = 0; + virtual void reload(int64_t generation) = 0; + virtual void close() = 0; + + virtual ~Source() { } +}; + +} // namespace common + diff --git a/config/src/vespa/config/common/sourcefactory.h b/config/src/vespa/config/common/sourcefactory.h new file mode 100644 index 00000000000..bfa009f287f --- /dev/null +++ b/config/src/vespa/config/common/sourcefactory.h @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include "source.h" +#include "configkey.h" +#include "iconfigholder.h" + +namespace config { + +/* + * Source factory, creating possible config sources. + */ +class SourceFactory { +public: + typedef std::unique_ptr<SourceFactory> UP; + virtual Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const = 0; + virtual ~SourceFactory() { } +}; + +} // namespace common + diff --git a/config/src/vespa/config/common/subscribehandler.h b/config/src/vespa/config/common/subscribehandler.h new file mode 100644 index 00000000000..4177ffd2b3c --- /dev/null +++ b/config/src/vespa/config/common/subscribehandler.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/subscription/configsubscription.h> +#include <vespa/config/common/configkey.h> + +namespace config { + +struct SubscribeHandler +{ + /** + * Subscribes to a spesific config given by a subscription. + * If the subscribe call is successful, the callback handler will be called + * with the new config. + * + * @param key the subscription key to subscribe to. + * @param timeoutInMillis the timeout of the subscribe call. + * @return subscription object containing data relevant to client + */ + virtual ConfigSubscription::SP subscribe(const ConfigKey & key, uint64_t timeoutInMillis) = 0; + virtual ~SubscribeHandler() { } +}; + +} + diff --git a/config/src/vespa/config/common/timingvalues.cpp b/config/src/vespa/config/common/timingvalues.cpp new file mode 100644 index 00000000000..13844bcdfe9 --- /dev/null +++ b/config/src/vespa/config/common/timingvalues.cpp @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author Gunnar Gauslaa Bergem + * @date 2008-05-22 + * @version $Id: timingvalues.cpp 119439 2011-04-19 09:32:27Z arnej $ + */ + +#include <vespa/fastos/fastos.h> +#include "timingvalues.h" + +namespace config { + +TimingValues::TimingValues() + : successTimeout(600000), + errorTimeout(25000), + initialTimeout(15000), + subscribeTimeout(DEFAULT_SUBSCRIBE_TIMEOUT), + fixedDelay(5000), + successDelay(250), + unconfiguredDelay(1000), + configuredErrorDelay(15000), + maxDelayMultiplier(10), + transientDelay(10000), + fatalDelay(60000) +{ +} + + +TimingValues::TimingValues(uint64_t initSuccessTimeout, + uint64_t initErrorTimeout, + uint64_t initInitialTimeout, + uint64_t initSubscribeTimeout, + uint64_t initFixedDelay, + uint64_t initSuccessDelay, + uint64_t initUnconfiguredDelay, + uint64_t initConfiguredErrorDelay, + unsigned int initMaxDelayMultiplier, + uint64_t initTransientDelay, + uint64_t initFatalDelay) + : successTimeout(initSuccessTimeout), + errorTimeout(initErrorTimeout), + initialTimeout(initInitialTimeout), + subscribeTimeout(initSubscribeTimeout), + fixedDelay(initFixedDelay), + successDelay(initSuccessDelay), + unconfiguredDelay(initUnconfiguredDelay), + configuredErrorDelay(initConfiguredErrorDelay), + maxDelayMultiplier(initMaxDelayMultiplier), + transientDelay(initTransientDelay), + fatalDelay(initFatalDelay) +{ +} + +} diff --git a/config/src/vespa/config/common/timingvalues.h b/config/src/vespa/config/common/timingvalues.h new file mode 100644 index 00000000000..78a64050a22 --- /dev/null +++ b/config/src/vespa/config/common/timingvalues.h @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author Gunnar Gauslaa Bergem + * @date 2008-05-22 + * @version $Id: timingvalues.h 119465 2011-04-20 15:21:46Z arnej $ + */ + +#pragma once + +#include <vespa/fastos/fastos.h> + +namespace config { + +static const uint64_t DEFAULT_NEXTCONFIG_TIMEOUT = 55000; +static const uint64_t DEFAULT_SUBSCRIBE_TIMEOUT = 55000; +static const uint64_t DEFAULT_GETCONFIGS_TIMEOUT = 55000; + + +struct TimingValues +{ + uint64_t successTimeout; // Timeout when previous config request was a success. + uint64_t errorTimeout; // Timeout when previous config request was an error. + uint64_t initialTimeout; // Timeout used when requesting config for the first time. + uint64_t subscribeTimeout; // Timeout used to find out when to give up unsubscribe. + + uint64_t fixedDelay; // Fixed delay between config requests. + uint64_t successDelay; // Delay if until next request after successful getConfig. + uint64_t unconfiguredDelay; // Delay if failed and client not yet configured. + uint64_t configuredErrorDelay; // Delay if failed but client has gotten config for the first time earlier. + unsigned int maxDelayMultiplier; // Max multiplier when trying to get config. + + uint64_t transientDelay; // Delay between connection reuse if transient error. + uint64_t fatalDelay; // Delay between connection reuse if fatal error. + + TimingValues(); + TimingValues(uint64_t initSuccessTimeout, + uint64_t initerrorTimeout, + uint64_t initInitialTimeout, + uint64_t initSubscribeTimeout, + uint64_t initFixedDelay, + uint64_t initSuccessDelay, + uint64_t initUnconfiguredDelay, + uint64_t initConfiguredErrorDelay, + unsigned int initMaxDelayMultiplier, + uint64_t initTransientDelay, + uint64_t initFatalDelay); +}; + +} + diff --git a/config/src/vespa/config/common/trace.cpp b/config/src/vespa/config/common/trace.cpp new file mode 100644 index 00000000000..70f5ddb85a2 --- /dev/null +++ b/config/src/vespa/config/common/trace.cpp @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "trace.h" +#include <vespa/vespalib/trace/slime_trace_serializer.h> +#include <vespa/vespalib/trace/slime_trace_deserializer.h> + +using namespace vespalib; +using namespace vespalib::slime; + +namespace config { + +struct SystemClock : public Clock +{ + int64_t currentTimeMillis() const { + fastos::TimeStamp ts(fastos::ClockSystem::now()); + return ts.ms(); + } +}; + +static SystemClock systemClock; + +const Memory Trace::TRACELOG("traceLog"); +const Memory Trace::TRACELEVEL("traceLevel"); + +Trace::Trace(const Trace & other) + : _root(other._root), + _traceLevel(other._traceLevel), + _clock(other._clock) +{ +} + +Trace::Trace() + : _root(), + _traceLevel(0), + _clock(systemClock) +{ +} + + +Trace::Trace(uint32_t traceLevel) + : _traceLevel(traceLevel), + _clock(systemClock) +{ +} + + +Trace::Trace(uint32_t traceLevel, const Clock & clock) + : _traceLevel(traceLevel), + _clock(clock) +{ +} + +void +Trace::deserialize(const Inspector & inspector) +{ + _traceLevel = inspector[TRACELEVEL].asLong(); + deserializeTraceLog(inspector[TRACELOG]); +} + +void +Trace::deserializeTraceLog(const Inspector & root) +{ + SlimeTraceDeserializer deserializer(root); + _root = deserializer.deserialize(); +} + +bool +Trace::shouldTrace(uint32_t level) const +{ + return (level <= _traceLevel); +} + +void +Trace::trace(uint32_t level, const vespalib::string & message) +{ + if (shouldTrace(level)) { + _root.addChild(message, _clock.currentTimeMillis()); + } +} + +void +Trace::serialize(Cursor & cursor) const +{ + cursor.setLong(TRACELEVEL, _traceLevel); + SlimeTraceSerializer serializer(cursor.setObject(TRACELOG)); + _root.accept(serializer); +} + +void +Trace::serializeTraceLog(Cursor & array) const +{ + for (uint32_t i(0); i < _root.getNumChildren(); i++) { + SlimeTraceSerializer serializer(array.addObject()); + _root.getChild(i).accept(serializer); + } +} + +vespalib::string +Trace::toString() const +{ + Slime slime; + serializeTraceLog(slime.setArray()); + SimpleBuffer buf; + JsonFormat::encode(slime.get(), buf, false); + return buf.get().make_string(); +} + + +} // namespace config diff --git a/config/src/vespa/config/common/trace.h b/config/src/vespa/config/common/trace.h new file mode 100644 index 00000000000..268d46d0de6 --- /dev/null +++ b/config/src/vespa/config/common/trace.h @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/trace/tracenode.h> +#include <vespa/vespalib/stllike/string.h> +#include <memory> + +namespace config { + +/** + * Clock interface for acquiring time. + */ +struct Clock { + virtual int64_t currentTimeMillis() const = 0; + virtual ~Clock() {} +}; + +/** + * A simple trace interface which can be used to create a serial trace log of events. Each entry is given a timestamp. The trace + * can be serialized to/constructed from slime. Is not thread safe. + */ +class Trace +{ +public: + Trace(const Trace & other); + Trace(); + Trace(uint32_t traceLevel); + Trace(uint32_t traceLevel, const Clock & clock); + + bool shouldTrace(uint32_t level) const; + void trace(uint32_t level, const vespalib::string & message); + + void serialize(vespalib::slime::Cursor & cursor) const; + void deserialize(const vespalib::slime::Inspector & inspector); + const vespalib::TraceNode & getRoot() const { return _root; } + + vespalib::string toString() const; +private: + void serializeTraceLog(vespalib::slime::Cursor & array) const; + void deserializeTraceLog(const vespalib::slime::Inspector & inspector); + static const vespalib::slime::Memory TRACELOG; + static const vespalib::slime::Memory TRACELEVEL; + vespalib::TraceNode _root; + uint32_t _traceLevel; + const Clock & _clock; +}; + +} // namespace config + diff --git a/config/src/vespa/config/common/vespa_version.cpp b/config/src/vespa/config/common/vespa_version.cpp new file mode 100644 index 00000000000..595f4dfd23b --- /dev/null +++ b/config/src/vespa/config/common/vespa_version.cpp @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "vespa_version.h" + +#ifndef V_TAG_COMPONENT +#define V_TAG_COMPONENT "1.0.0" +#endif + +namespace config { + +const VespaVersion currentVersion(VespaVersion::fromString(vespalib::string(V_TAG_COMPONENT))); + + +VespaVersion::VespaVersion(const VespaVersion & vespaVersion) + : _versionString(vespaVersion._versionString) +{ +} + +VespaVersion::VespaVersion(const vespalib::string & versionString) + : _versionString(versionString) +{ +} + +VespaVersion +VespaVersion::fromString(const vespalib::string & versionString) +{ + return VespaVersion(versionString); +} + +const VespaVersion & +VespaVersion::getCurrentVersion() { + return currentVersion; +} + +const vespalib::string & +VespaVersion::toString() const +{ + return _versionString; +} + +} diff --git a/config/src/vespa/config/common/vespa_version.h b/config/src/vespa/config/common/vespa_version.h new file mode 100644 index 00000000000..eca173793d1 --- /dev/null +++ b/config/src/vespa/config/common/vespa_version.h @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace config { + +/** + * Represents a version used in config protocol + **/ +struct VespaVersion +{ +public: + static VespaVersion fromString(const vespalib::string & versionString); + static const VespaVersion & getCurrentVersion(); + VespaVersion(const VespaVersion & version); + const vespalib::string & toString() const; +private: + VespaVersion(const vespalib::string & versionString); + vespalib::string _versionString; +}; + +} // namespace config + diff --git a/config/src/vespa/config/common/waitable.h b/config/src/vespa/config/common/waitable.h new file mode 100644 index 00000000000..0a306d572f8 --- /dev/null +++ b/config/src/vespa/config/common/waitable.h @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> + +namespace config { + +/** + * A Waitable is a component that can be used to wait for some event to happen. + */ +struct Waitable +{ + virtual bool wait(uint64_t timeoutInMillis) = 0; + virtual ~Waitable() {} +}; + +} // namespace config + diff --git a/config/src/vespa/config/config.h b/config/src/vespa/config/config.h new file mode 100644 index 00000000000..b7331a04d9a --- /dev/null +++ b/config/src/vespa/config/config.h @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/subscription/configsubscriber.h> +#include <vespa/config/subscription/confighandle.h> +#include <vespa/config/subscription/sourcespec.h> +#include <vespa/config/subscription/configuri.h> +#include <vespa/config/helper/configgetter.h> +#include <vespa/config/helper/configfetcher.h> +#include <vespa/config/common/exceptions.h> +#include <vespa/config/common/misc.h> +#include <vespa/config/retriever/configretriever.h> + +/*! \mainpage Cloud Config API for C++ + * + * /section Introduction + * + * This document is provided as an API reference to use when developing with the + * C++ config API. + */ + +/** + * @section DESCRIPTION + * + * This file contains all necessary includes as well as functions used to + * subscribe to and retrieve config. + */ + diff --git a/config/src/vespa/config/configgen/.gitignore b/config/src/vespa/config/configgen/.gitignore new file mode 100644 index 00000000000..7e7c0fe7fae --- /dev/null +++ b/config/src/vespa/config/configgen/.gitignore @@ -0,0 +1,2 @@ +/.depend +/Makefile diff --git a/config/src/vespa/config/configgen/CMakeLists.txt b/config/src/vespa/config/configgen/CMakeLists.txt new file mode 100644 index 00000000000..487c7293240 --- /dev/null +++ b/config/src/vespa/config/configgen/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(config_configgen OBJECT + SOURCES + value_converter.cpp + DEPENDS +) diff --git a/config/src/vespa/config/configgen/configinstance.h b/config/src/vespa/config/configgen/configinstance.h new file mode 100644 index 00000000000..1e0d796c005 --- /dev/null +++ b/config/src/vespa/config/configgen/configinstance.h @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include <vespa/vespalib/stllike/string.h> + +namespace config { + +class ConfigDataBuffer; + +/** + * Interface implemented by all generated config objects. + */ +class ConfigInstance { +public: + typedef std::unique_ptr<ConfigInstance> UP; + typedef std::shared_ptr<ConfigInstance> SP; + // Static for this instance's type + virtual const vespalib::string & defName() const = 0; + virtual const vespalib::string & defMd5() const = 0; + virtual const vespalib::string & defNamespace() const = 0; + + virtual void serialize(ConfigDataBuffer & buffer) const = 0; + + virtual ~ConfigInstance() { } +}; + +} // namespace config + diff --git a/config/src/vespa/config/configgen/configpayload.h b/config/src/vespa/config/configgen/configpayload.h new file mode 100644 index 00000000000..411fdc0d443 --- /dev/null +++ b/config/src/vespa/config/configgen/configpayload.h @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/data/slime/slime.h> + +namespace config { + +class ConfigPayload { +public: + ConfigPayload(const ::vespalib::slime::Inspector & inspector) + : _inspector(inspector) + {} + const ::vespalib::slime::Inspector & get() const { return _inspector; } +private: + const ::vespalib::slime::Inspector & _inspector; +}; + +} // namespace config + diff --git a/config/src/vespa/config/configgen/map_inserter.h b/config/src/vespa/config/configgen/map_inserter.h new file mode 100644 index 00000000000..d03cf4eb6cd --- /dev/null +++ b/config/src/vespa/config/configgen/map_inserter.h @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/data/slime/slime.h> +#include "value_converter.h" +#include <map> + +namespace config { + +namespace internal { + +template<typename T, typename Converter = config::internal::ValueConverter<T> > +class MapInserter : public ::vespalib::slime::ObjectTraverser { +public: + MapInserter(std::map<vespalib::string, T> & map); + void field(const ::vespalib::slime::Memory & symbol, const ::vespalib::slime::Inspector & inspector); +private: + std::map<vespalib::string, T> & _map; +}; + +} // namespace internal + +} // namespace config + +#include "map_inserter.hpp" + diff --git a/config/src/vespa/config/configgen/map_inserter.hpp b/config/src/vespa/config/configgen/map_inserter.hpp new file mode 100644 index 00000000000..cf1f7150041 --- /dev/null +++ b/config/src/vespa/config/configgen/map_inserter.hpp @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace config { + +namespace internal { + +template<typename T, typename Converter> +MapInserter<T, Converter>::MapInserter(std::map<vespalib::string, T> & map) + : _map(map) +{} + +template<typename T, typename Converter> +void +MapInserter<T, Converter>::field(const ::vespalib::slime::Memory & symbol, const ::vespalib::slime::Inspector & inspector) +{ + Converter converter; + _map[symbol.make_string()] = converter(inspector); +} + +} // namespace internal + +} diff --git a/config/src/vespa/config/configgen/value_converter.cpp b/config/src/vespa/config/configgen/value_converter.cpp new file mode 100644 index 00000000000..6b7438bba02 --- /dev/null +++ b/config/src/vespa/config/configgen/value_converter.cpp @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/config/common/exceptions.h> +#include "value_converter.h" + +using namespace vespalib; +using namespace vespalib::slime; + +namespace config { + +namespace internal { + +template<> +int32_t convertValue(const ::vespalib::slime::Inspector & __inspector) { + switch (__inspector.type().getId()) { + case LONG::ID: return static_cast<int32_t>(__inspector.asLong()); + case DOUBLE::ID: return static_cast<int32_t>(__inspector.asDouble()); + case STRING::ID: return static_cast<int32_t>(strtoll(__inspector.asString().make_string().c_str(), 0, 0)); + } + throw InvalidConfigException("Expected int32_t, but got incompatible config type " + __inspector.type().getId()); +} + +template<> +int64_t convertValue(const ::vespalib::slime::Inspector & __inspector) { + switch (__inspector.type().getId()) { + case LONG::ID: return static_cast<int64_t>(__inspector.asLong()); + case DOUBLE::ID: return static_cast<int64_t>(__inspector.asDouble()); + case STRING::ID: return static_cast<int64_t>(strtoll(__inspector.asString().make_string().c_str(), 0, 0)); + } + throw InvalidConfigException("Expected int64_t, but got incompatible config type " + __inspector.type().getId()); +} + +template<> +double convertValue(const ::vespalib::slime::Inspector & __inspector) { + switch (__inspector.type().getId()) { + case LONG::ID: return static_cast<double>(__inspector.asLong()); + case DOUBLE::ID: return static_cast<double>(__inspector.asDouble()); + case STRING::ID: return static_cast<double>(strtod(__inspector.asString().make_string().c_str(), 0)); + } + throw InvalidConfigException("Expected double, but got incompatible config type " + __inspector.type().getId()); +} + +template<> +bool convertValue(const ::vespalib::slime::Inspector & __inspector) { + switch (__inspector.type().getId()) { + case BOOL::ID: return __inspector.asBool(); + case STRING::ID: + vespalib::string s(__inspector.asString().make_string()); + return s.compare("true") == 0 ? true : false; + } + throw InvalidConfigException("Expected bool, but got incompatible config type " + __inspector.type().getId()); +} + +template<> +vespalib::string convertValue(const ::vespalib::slime::Inspector & __inspector) { return __inspector.asString().make_string(); } + +void +requireValid(const vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector) { + if (!__inspector.valid()) { + throw ::config::InvalidConfigException("Value for '" + __fieldName + "' required but not found"); + } +} + +} + +} diff --git a/config/src/vespa/config/configgen/value_converter.h b/config/src/vespa/config/configgen/value_converter.h new file mode 100644 index 00000000000..c951f558781 --- /dev/null +++ b/config/src/vespa/config/configgen/value_converter.h @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include "configpayload.h" + +namespace config { + +namespace internal { + +void requireValid(const vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector); + +template<typename T> +T convertValue(const ::vespalib::slime::Inspector & __inspector) { return T(::config::ConfigPayload(__inspector)); } + +template<> +int32_t convertValue(const ::vespalib::slime::Inspector & __inspector); + +template<> +int64_t convertValue(const ::vespalib::slime::Inspector & __inspector); + +template<> +double convertValue(const ::vespalib::slime::Inspector & __inspector); + +template<> +bool convertValue(const ::vespalib::slime::Inspector & __inspector); + +template<> +vespalib::string convertValue(const ::vespalib::slime::Inspector & __inspector); + +template<typename T> +struct ValueConverter { + T operator()(const vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector) { + requireValid(__fieldName, __inspector); + return convertValue<T>(__inspector); + } + T operator()(const ::vespalib::slime::Inspector & __inspector) { + return __inspector.valid() ? convertValue<T>(__inspector) : T(); + } + T operator()(const ::vespalib::slime::Inspector & __inspector, T __t) { + return __inspector.valid() ? convertValue<T>(__inspector) : __t; + } +}; + +} // namespace internal + +} // namespace config + diff --git a/config/src/vespa/config/configgen/vector_inserter.h b/config/src/vespa/config/configgen/vector_inserter.h new file mode 100644 index 00000000000..36d324f3555 --- /dev/null +++ b/config/src/vespa/config/configgen/vector_inserter.h @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/data/slime/slime.h> +#include "value_converter.h" + +namespace config { + +namespace internal { + +template<typename T, typename Converter = ::config::internal::ValueConverter<T> > +class VectorInserter : public ::vespalib::slime::ArrayTraverser { +public: + VectorInserter(std::vector<T> & vector); + void entry(size_t idx, const ::vespalib::slime::Inspector & inspector); +private: + std::vector<T> & _vector; +}; + +} // namespace internal + +} // namespace config + +#include "vector_inserter.hpp" + diff --git a/config/src/vespa/config/configgen/vector_inserter.hpp b/config/src/vespa/config/configgen/vector_inserter.hpp new file mode 100644 index 00000000000..94c2b86ce65 --- /dev/null +++ b/config/src/vespa/config/configgen/vector_inserter.hpp @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace config { + +namespace internal { + +template<typename T, typename Converter> +VectorInserter<T, Converter>::VectorInserter(std::vector<T> & vector) + : _vector(vector) +{} + +template<typename T, typename Converter> +void +VectorInserter<T, Converter>::entry(size_t idx, const ::vespalib::slime::Inspector & inspector) +{ + (void) idx; + Converter converter; + _vector.push_back(converter(inspector)); +} + +} // namespace internal + +} diff --git a/config/src/vespa/config/file/.gitignore b/config/src/vespa/config/file/.gitignore new file mode 100644 index 00000000000..7e7c0fe7fae --- /dev/null +++ b/config/src/vespa/config/file/.gitignore @@ -0,0 +1,2 @@ +/.depend +/Makefile diff --git a/config/src/vespa/config/file/CMakeLists.txt b/config/src/vespa/config/file/CMakeLists.txt new file mode 100644 index 00000000000..743e9ac5e74 --- /dev/null +++ b/config/src/vespa/config/file/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(config_file OBJECT + SOURCES + filesource.cpp + filesourcefactory.cpp + DEPENDS +) diff --git a/config/src/vespa/config/file/filesource.cpp b/config/src/vespa/config/file/filesource.cpp new file mode 100644 index 00000000000..e2ee7248943 --- /dev/null +++ b/config/src/vespa/config/file/filesource.cpp @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.file.filesource"); +#include "filesource.h" +#include <vespa/config/subscription/sourcespec.h> +#include <vespa/config/common/misc.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/stllike/asciistream.h> + +using vespalib::asciistream; + +namespace config { + +FileSource::FileSource(const IConfigHolder::SP & holder, const vespalib::string & fileName) + : _holder(holder), + _fileName(fileName), + _lastLoaded(-1), + _generation(1) +{ +} + +void +FileSource::getConfig() +{ + std::vector<vespalib::string> lines(readConfigFile(_fileName)); + int64_t last = getLast(_fileName); + + if (last > _lastLoaded) { + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, calculateContentMd5(lines)), true, _generation))); + _lastLoaded = last; + } else { + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, calculateContentMd5(lines)), false, _generation))); + } +} + +void +FileSource::reload(int64_t generation) +{ + _generation = generation; +} + +int64_t +FileSource::getLast(const vespalib::string & fileName) +{ + struct stat filestat; + memset(&filestat, 0, sizeof(filestat)); + stat(fileName.c_str(), &filestat); + return filestat.st_mtime; +} + +std::vector<vespalib::string> +FileSource::readConfigFile(const vespalib::string & fileName) +{ + asciistream is(asciistream::createFromFile(fileName)); + return is.getlines(); +} + +void +FileSource::close() +{ +} + +} // namespace config diff --git a/config/src/vespa/config/file/filesource.h b/config/src/vespa/config/file/filesource.h new file mode 100644 index 00000000000..141d14faf5c --- /dev/null +++ b/config/src/vespa/config/file/filesource.h @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <string> +#include <vector> +#include <map> +#include <stack> +#include <vespa/config/common/source.h> +#include <vespa/config/common/iconfigholder.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/noncopyable.hpp> + +namespace config { + +class FileSpec; +class DirSpec; + +class FileSource : public Source, + public vespalib::noncopyable +{ +private: + IConfigHolder::SP _holder; + const vespalib::string _fileName; + int64_t _lastLoaded; + int64_t _generation; + + std::vector<vespalib::string> readConfigFile(const vespalib::string & fileName); + int64_t getLast(const vespalib::string & fileName); + +public: + FileSource(const IConfigHolder::SP & holder, const vespalib::string & fileName); + void getConfig(); + void close(); + void reload(int64_t generation); +}; + +} // namespace config + diff --git a/config/src/vespa/config/file/filesourcefactory.cpp b/config/src/vespa/config/file/filesourcefactory.cpp new file mode 100644 index 00000000000..ee827617368 --- /dev/null +++ b/config/src/vespa/config/file/filesourcefactory.cpp @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "filesourcefactory.h" +#include "filesource.h" +#include <vespa/config/common/exceptions.h> +#include <vespa/config/subscription/sourcespec.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.filesourcefactory"); + +namespace config { + +DirSourceFactory::DirSourceFactory(const DirSpec & dirSpec) + : _dirName(dirSpec.getDirName()), + _fileNames() +{ + vespalib::DirectoryList files(vespalib::listDirectory(_dirName)); + for (size_t i = 0; i < files.size(); i++) { + const vespalib::DirectoryList::value_type & fname(files[i]); + if (fname.length() > 4 && fname.substr(fname.length() - 4) == ".cfg") { + _fileNames.push_back(fname); + } + } +} + +Source::UP +DirSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const +{ + vespalib::string fileId(key.getDefName()); + if (!key.getConfigId().empty()) { + fileId += "." + key.getConfigId(); + } + fileId += ".cfg"; + + bool found(false); + for (const auto & fileName : _fileNames) { + if (fileName.compare(fileId) == 0) { + found = true; + break; + } + } + if ( !found ) { + LOG(warning, "Filename '%s' was expected in the spec, but does not exist.", fileId.c_str()); + } + vespalib::string fName = _dirName; + if (!fName.empty()) fName += "/"; + fName += fileId; + return Source::UP(new FileSource(holder, fName)); +} + +FileSourceFactory::FileSourceFactory(const FileSpec & fileSpec) + : _fileName(fileSpec.getFileName()) +{ +} + +Source::UP +FileSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const +{ + (void) key; + return Source::UP(new FileSource(holder, _fileName)); +} + +} // namespace config + diff --git a/config/src/vespa/config/file/filesourcefactory.h b/config/src/vespa/config/file/filesourcefactory.h new file mode 100644 index 00000000000..c38b00aa034 --- /dev/null +++ b/config/src/vespa/config/file/filesourcefactory.h @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/sourcefactory.h> +#include <vespa/vespalib/stllike/string.h> + + +namespace config { + +class DirSpec; +class FileSpec; + +/** + * Factory creating config payload from config instances. + */ +class FileSourceFactory : public SourceFactory +{ +public: + FileSourceFactory(const FileSpec & fileSpec); + + /** + * Create source handling config described by key. + */ + Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const; +private: + vespalib::string _fileName; +}; + +/** + * Factory creating config payload from config instances. + */ +class DirSourceFactory : public SourceFactory +{ +public: + DirSourceFactory(const DirSpec & dirSpec); + + /** + * Create source handling config described by key. + */ + Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const; +private: + vespalib::string _dirName; + std::vector<vespalib::string> _fileNames; +}; + + +} // namespace config + diff --git a/config/src/vespa/config/frt/.gitignore b/config/src/vespa/config/frt/.gitignore new file mode 100644 index 00000000000..7e7c0fe7fae --- /dev/null +++ b/config/src/vespa/config/frt/.gitignore @@ -0,0 +1,2 @@ +/.depend +/Makefile diff --git a/config/src/vespa/config/frt/CMakeLists.txt b/config/src/vespa/config/frt/CMakeLists.txt new file mode 100644 index 00000000000..fcc8abfa1b8 --- /dev/null +++ b/config/src/vespa/config/frt/CMakeLists.txt @@ -0,0 +1,21 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(config_frt OBJECT + SOURCES + frtsource.cpp + frtconnectionpool.cpp + frtconnection.cpp + frtconfigrequest.cpp + frtconfigresponse.cpp + frtsourcefactory.cpp + frtconfigagent.cpp + frtconfigrequestv2.cpp + frtconfigrequestfactory.cpp + frtconfigresponsev2.cpp + protocol.cpp + slimeconfigrequest.cpp + slimeconfigresponse.cpp + frtconfigrequestv3.cpp + frtconfigresponsev3.cpp + compressioninfo.cpp + DEPENDS +) diff --git a/config/src/vespa/config/frt/compressioninfo.cpp b/config/src/vespa/config/frt/compressioninfo.cpp new file mode 100644 index 00000000000..270d335b7ae --- /dev/null +++ b/config/src/vespa/config/frt/compressioninfo.cpp @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/config/frt/protocol.h> +#include "compressioninfo.h" + +using namespace vespalib; +using namespace vespalib::slime; +using namespace config::protocol; + +namespace config { + +CompressionInfo::CompressionInfo() + : compressionType(CompressionType::UNCOMPRESSED), + uncompressedSize(0) +{ +} + +void +CompressionInfo::deserialize(const Inspector & inspector) +{ + compressionType = stringToCompressionType(inspector["compressionType"].asString().make_string()); + uncompressedSize = inspector["uncompressedSize"].asLong(); +} + +} diff --git a/config/src/vespa/config/frt/compressioninfo.h b/config/src/vespa/config/frt/compressioninfo.h new file mode 100644 index 00000000000..2ba5d5c52c7 --- /dev/null +++ b/config/src/vespa/config/frt/compressioninfo.h @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/frt/protocol.h> + +namespace config { + +struct CompressionInfo { + CompressionInfo(); + CompressionType compressionType; + uint32_t uncompressedSize; + void deserialize(const vespalib::slime::Inspector & inspector); +}; + +} + diff --git a/config/src/vespa/config/frt/connection.h b/config/src/vespa/config/frt/connection.h new file mode 100644 index 00000000000..7bd8229411f --- /dev/null +++ b/config/src/vespa/config/frt/connection.h @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +class FRT_RPCRequest; +class FRT_IRequestWait; + +namespace config { + +class Connection { +public: + virtual FRT_RPCRequest * allocRPCRequest() = 0; + virtual void setError(int errorCode) = 0; + virtual void invoke(FRT_RPCRequest * req, double timeout, FRT_IRequestWait * waiter) = 0; + virtual const vespalib::string & getAddress() const = 0; + virtual void setTransientDelay(int64_t delay) = 0; + virtual ~Connection() { } +}; + +} + diff --git a/config/src/vespa/config/frt/connectionfactory.h b/config/src/vespa/config/frt/connectionfactory.h new file mode 100644 index 00000000000..c2c5bfe0c96 --- /dev/null +++ b/config/src/vespa/config/frt/connectionfactory.h @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + + +class FNET_Scheduler; + +namespace config { + +class Connection; + +class ConnectionFactory +{ +public: + typedef std::unique_ptr <ConnectionFactory> UP; + typedef std::shared_ptr<ConnectionFactory> SP; + virtual Connection * getCurrent() = 0; + virtual void syncTransport() = 0; + virtual FNET_Scheduler * getScheduler() = 0; + virtual ~ConnectionFactory() { } +}; + +} + diff --git a/config/src/vespa/config/frt/frtconfigagent.cpp b/config/src/vespa/config/frt/frtconfigagent.cpp new file mode 100644 index 00000000000..397067b85a1 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigagent.cpp @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.frtconfigagent"); +#include "frtconfigagent.h" +#include <vespa/config/common/trace.h> +#include <vespa/config/common/configrequest.h> +#include <vespa/config/common/configkey.h> + +namespace config { + +FRTConfigAgent::FRTConfigAgent(const IConfigHolder::SP & holder, const TimingValues & timingValues) + : _holder(holder), + _timingValues(timingValues), + _configState(), + _latest(), + _waitTime(0), + _numConfigured(0), + _failedRequests(0), + _nextTimeout(_timingValues.initialTimeout) +{ +} + +void +FRTConfigAgent::handleResponse(const ConfigRequest & request, ConfigResponse::UP response) +{ + if (LOG_WOULD_LOG(spam)) { + const ConfigKey & key(request.getKey()); + LOG(spam, "current state for %s: generation %" PRId64 " md5 %s", key.toString().c_str(), _configState.generation, _configState.md5.c_str()); + } + if (response->validateResponse() && !response->isError()) { + handleOKResponse(std::move(response)); + } else { + handleErrorResponse(request, std::move(response)); + } +} + +void +FRTConfigAgent::handleOKResponse(ConfigResponse::UP response) +{ + _failedRequests = 0; + response->fill(); + if (LOG_WOULD_LOG(spam)) { + LOG(spam, "trace(%s)", response->getTrace().toString().c_str()); + } + + ConfigState newState = response->getConfigState(); + bool isNewGeneration = newState.isNewerGenerationThan(_configState); + if (isNewGeneration) { + handleUpdatedGeneration(response->getKey(), newState, response->getValue()); + } + setWaitTime(_timingValues.successDelay, 1); + _nextTimeout = _timingValues.successTimeout; +} + +void +FRTConfigAgent::handleUpdatedGeneration(const ConfigKey & key, const ConfigState & newState, const ConfigValue & configValue) +{ + if (LOG_WOULD_LOG(spam)) { + LOG(spam, "new generation %" PRId64 " for key %s", newState.generation, key.toString().c_str()); + } + _configState.generation = newState.generation; + bool hasDifferentPayload = newState.hasDifferentPayloadFrom(_configState); + if (hasDifferentPayload) { + if (LOG_WOULD_LOG(spam)) { + LOG(spam, "new payload for key %s, existing md5(%s), new md5(%s)", key.toString().c_str(), _configState.md5.c_str(), newState.md5.c_str()); + } + _configState.md5 = newState.md5; + _latest = configValue; + } + + if (LOG_WOULD_LOG(spam)) { + LOG(spam, "updating holder for key %s, payload changed: %d", key.toString().c_str(), hasDifferentPayload ? 1 : 0); + } + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(_latest, hasDifferentPayload, _configState.generation))); + _numConfigured++; +} + +void +FRTConfigAgent::handleErrorResponse(const ConfigRequest & request, ConfigResponse::UP response) +{ + _failedRequests++; + int multiplier = std::min(_failedRequests, _timingValues.maxDelayMultiplier); + setWaitTime(_numConfigured > 0 ? _timingValues.configuredErrorDelay : _timingValues.unconfiguredDelay, multiplier); + _nextTimeout = _timingValues.errorTimeout; + const ConfigKey & key(request.getKey()); + LOG(info, "Error response or no response from config server (key: %s) (errcode=%d, validresponse:%d), trying again in %" PRId64 " milliseconds", key.toString().c_str(), response->errorCode(), response->hasValidResponse() ? 1 : 0, _waitTime); +} + +void +FRTConfigAgent::setWaitTime(uint64_t delay, int multiplier) +{ + uint64_t prevWait = _waitTime; + _waitTime = _timingValues.fixedDelay + (multiplier * delay); + LOG(spam, "Adjusting waittime from %" PRId64 " to %" PRId64, prevWait, _waitTime); +} + +uint64_t FRTConfigAgent::getTimeout() const { return _nextTimeout; } +uint64_t FRTConfigAgent::getWaitTime() const { return _waitTime; } +const ConfigState & FRTConfigAgent::getConfigState() const { return _configState; } + +} diff --git a/config/src/vespa/config/frt/frtconfigagent.h b/config/src/vespa/config/frt/frtconfigagent.h new file mode 100644 index 00000000000..b058d184ea0 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigagent.h @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/configstate.h> +#include <vespa/config/common/timingvalues.h> +#include <vespa/config/common/iconfigholder.h> +#include <vespa/config/common/configresponse.h> +#include <vespa/config/common/configrequest.h> + +namespace config { + +class ConfigAgent +{ +public: + typedef std::unique_ptr<ConfigAgent> UP; + virtual void handleResponse(const ConfigRequest & request, ConfigResponse::UP response) = 0; + + virtual uint64_t getTimeout() const = 0; + virtual uint64_t getWaitTime() const = 0; + virtual const ConfigState & getConfigState() const = 0; + + virtual ~ConfigAgent() { } +}; + +class FRTConfigAgent : public ConfigAgent +{ +public: + FRTConfigAgent(const IConfigHolder::SP & holder, const TimingValues & timingValues); + void handleResponse(const ConfigRequest & request, ConfigResponse::UP response); + uint64_t getTimeout() const; + uint64_t getWaitTime() const; + const ConfigState & getConfigState() const; +private: + void handleUpdatedGeneration(const ConfigKey & key, const ConfigState & newState, const ConfigValue & configValue); + void handleOKResponse(ConfigResponse::UP response); + void handleErrorResponse(const ConfigRequest & request, ConfigResponse::UP response); + void setWaitTime(uint64_t delay, int multiplier); + + IConfigHolder::SP _holder; + const TimingValues _timingValues; + ConfigState _configState; + ConfigValue _latest; + uint64_t _waitTime; + uint64_t _numConfigured; + unsigned int _failedRequests; + uint64_t _nextTimeout; +}; + +} + diff --git a/config/src/vespa/config/frt/frtconfigrequest.cpp b/config/src/vespa/config/frt/frtconfigrequest.cpp new file mode 100644 index 00000000000..0a8088a191a --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigrequest.cpp @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.frtconfigrequest"); +#include "frtconfigrequest.h" +#include "frtconfigresponse.h" +#include "connection.h" +#include <vespa/fnet/frt/frt.h> +#include <vespa/config/common/configkey.h> +#include <vespa/config/common/configstate.h> + +namespace config { + +FRTConfigRequest::FRTConfigRequest(Connection * connection, const ConfigKey & key) + : _request(connection->allocRPCRequest()), + _parameters(*_request->GetParams()), + _connection(connection), + _key(key) +{ +} + +FRTConfigRequest::~FRTConfigRequest() +{ + _request->SubRef(); +} + +bool +FRTConfigRequest::abort() +{ + return _request->Abort(); +} + +void +FRTConfigRequest::setError(int errorCode) +{ + _connection->setError(errorCode); +} + +const ConfigKey & +FRTConfigRequest::getKey() const +{ + return _key; +} + +bool +FRTConfigRequest::isAborted() const +{ + return (_request->GetErrorCode() == FRTE_RPC_ABORT); +} + +const vespalib::string FRTConfigRequestV1::REQUEST_TYPES = "sssssllsSi"; + +FRTConfigRequestV1::FRTConfigRequestV1(const ConfigKey & key, + Connection * connection, + const vespalib::string & configMd5, + int64_t generation, + int64_t serverTimeout) + : FRTConfigRequest(connection, key) +{ + _request->SetMethodName("config.v1.getConfig"); + _parameters.AddString(key.getDefName().c_str()); + _parameters.AddString(""); + _parameters.AddString(key.getDefMd5().c_str()); + _parameters.AddString(key.getConfigId().c_str()); + _parameters.AddString(configMd5.c_str()); + _parameters.AddInt64(generation); + _parameters.AddInt64(serverTimeout); + _parameters.AddString(key.getDefNamespace().c_str()); + const std::vector<vespalib::string> & schema(key.getDefSchema()); + FRT_StringValue * schemaValue = _parameters.AddStringArray(schema.size()); + for (size_t i = 0; i < schema.size(); i++) { + _parameters.SetString(&schemaValue[i], schema[i].c_str()); + } + _parameters.AddInt32(1); +} + +bool +FRTConfigRequestV1::verifyKey(const ConfigKey & key) const +{ + return (key.getDefName().compare(_parameters[0]._string._str) == 0 && + key.getDefNamespace().compare(_parameters[7]._string._str) == 0 && + key.getConfigId().compare(_parameters[3]._string._str) == 0 && + key.getDefMd5().compare(_parameters[2]._string._str) == 0); +} + +bool +FRTConfigRequestV1::verifyState(const ConfigState & state) const +{ + return (state.md5.compare(_parameters[4]._string._str) == 0 && + state.generation == static_cast<int64_t>(_parameters[5]._intval64)); +} + +ConfigResponse::UP +FRTConfigRequestV1::createResponse(FRT_RPCRequest * request) const +{ + return ConfigResponse::UP(new FRTConfigResponseV1(request)); +} + +} // namespace config diff --git a/config/src/vespa/config/frt/frtconfigrequest.h b/config/src/vespa/config/frt/frtconfigrequest.h new file mode 100644 index 00000000000..ac7f66ffba9 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigrequest.h @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/configrequest.h> +#include <vespa/config/common/configresponse.h> +#include <vespa/config/common/configkey.h> +#include <vespa/vespalib/stllike/string.h> + +class FRT_Values; +class FRT_RPCRequest; + +namespace config { + +class ConfigKey; +class Connection; + +/** + * Class representing a FRT config request. + */ +class FRTConfigRequest : public ConfigRequest { +public: + typedef std::unique_ptr<FRTConfigRequest> UP; + FRTConfigRequest(Connection * connection, const ConfigKey & key); + virtual ~FRTConfigRequest(); + virtual bool verifyKey(const ConfigKey & key) const = 0; + virtual bool verifyState(const ConfigState & state) const = 0; + + bool abort(); + bool isAborted() const; + void setError(int errorCode); + const ConfigKey & getKey() const; + + FRT_RPCRequest* getRequest() { return _request; } + virtual ConfigResponse::UP createResponse(FRT_RPCRequest * request) const = 0; +protected: + FRT_RPCRequest *_request; + FRT_Values & _parameters; +private: + Connection * _connection; + const ConfigKey _key; +}; + +class FRTConfigRequestV1 : public FRTConfigRequest { +public: + FRTConfigRequestV1(const ConfigKey & key, + Connection * connection, + const vespalib::string & configMd5, + int64_t generation, + int64_t serverTimeout); + bool verifyKey(const ConfigKey & key) const; + bool verifyState(const ConfigState & state) const; + ConfigResponse::UP createResponse(FRT_RPCRequest * request) const; +private: + static const vespalib::string REQUEST_TYPES; +}; + +} + diff --git a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp new file mode 100644 index 00000000000..7565d99a8aa --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "frtconfigrequestfactory.h" +#include "frtconfigrequest.h" +#include "frtconfigrequestv2.h" +#include "frtconfigrequestv3.h" +#include <vespa/config/common/trace.h> +#include <vespa/config/common/compressiontype.h> +#include <vespa/vespalib/util/host_name.h> +#include <unistd.h> +#include <limits.h> + + +namespace config { + +/** + * Factory for creating config requests depending on protocol version; + */ +FRTConfigRequestFactory::FRTConfigRequestFactory(int protocolVersion, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType) + : _protocolVersion(protocolVersion), + _traceLevel(traceLevel), + _vespaVersion(vespaVersion), + _hostName(vespalib::HostName::get()), + _compressionType(compressionType) +{ +} + +FRTConfigRequest::UP +FRTConfigRequestFactory::createConfigRequest(const ConfigKey & key, Connection * connection, const ConfigState & state, int64_t serverTimeout) const +{ + if (1 == _protocolVersion) { + return FRTConfigRequest::UP(new FRTConfigRequestV1(key, connection, state.md5, state.generation, serverTimeout)); + } else if (2 == _protocolVersion) { + return FRTConfigRequest::UP(new FRTConfigRequestV2(connection, key, state.md5, state.generation, 0u, _hostName, serverTimeout, Trace(_traceLevel))); + } + return FRTConfigRequest::UP(new FRTConfigRequestV3(connection, key, state.md5, state.generation, 0u, _hostName, serverTimeout, Trace(_traceLevel), _vespaVersion, _compressionType)); +} + +} // namespace config diff --git a/config/src/vespa/config/frt/frtconfigrequestfactory.h b/config/src/vespa/config/frt/frtconfigrequestfactory.h new file mode 100644 index 00000000000..98d7175a352 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigrequestfactory.h @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/configkey.h> +#include <vespa/config/common/configstate.h> +#include <vespa/config/common/trace.h> +#include <vespa/config/common/compressiontype.h> +#include <vespa/config/common/vespa_version.h> +#include "frtconfigrequest.h" +#include "protocol.h" +#include "connection.h" + +namespace config { + +/** + * Factory for creating config requests depending on protocol version; + */ +class FRTConfigRequestFactory +{ +public: + FRTConfigRequestFactory(int protocolVersion, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType); + FRTConfigRequest::UP createConfigRequest(const ConfigKey & key, Connection * connection, const ConfigState & state, int64_t serverTimeout) const; +private: + const int _protocolVersion; + const int _traceLevel; + const VespaVersion _vespaVersion; + vespalib::string _hostName; + const CompressionType _compressionType; +}; + +} // namespace config + diff --git a/config/src/vespa/config/frt/frtconfigrequestv2.cpp b/config/src/vespa/config/frt/frtconfigrequestv2.cpp new file mode 100644 index 00000000000..97335468a97 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigrequestv2.cpp @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.frtconfigrequestv2"); +#include "frtconfigrequestv2.h" +#include "frtconfigresponsev2.h" +#include "connection.h" +#include <vespa/config/common/trace.h> +#include <vespa/config/common/vespa_version.h> + +using namespace config::protocol; + +namespace config { + +FRTConfigRequestV2::FRTConfigRequestV2(Connection * connection, + const ConfigKey & key, + const vespalib::string & configMd5, + int64_t currentGeneration, + int64_t wantedGeneration, + const vespalib::string & hostName, + int64_t serverTimeout, + const Trace & trace) + : SlimeConfigRequest(connection, key, configMd5, currentGeneration, wantedGeneration, hostName, serverTimeout, trace, VespaVersion::getCurrentVersion(), 2, CompressionType::UNCOMPRESSED, "config.v2.getConfig") +{ +} + + + +ConfigResponse::UP +FRTConfigRequestV2::createResponse(FRT_RPCRequest * request) const +{ + return ConfigResponse::UP(new FRTConfigResponseV2(request)); +} + +} diff --git a/config/src/vespa/config/frt/frtconfigrequestv2.h b/config/src/vespa/config/frt/frtconfigrequestv2.h new file mode 100644 index 00000000000..d3ed95a26c3 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigrequestv2.h @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "slimeconfigrequest.h" + +class FRT_Values; +class FRT_RPCRequest; + +namespace config { + +class ConfigKey; +class Connection; +class Trace; + +class FRTConfigRequestV2 : public SlimeConfigRequest { +public: + FRTConfigRequestV2(Connection * connection, + const ConfigKey & key, + const vespalib::string & configMd5, + int64_t currentGeneration, + int64_t wantedGeneration, + const vespalib::string & hostName, + int64_t serverTimeout, + const Trace & trace); + ConfigResponse::UP createResponse(FRT_RPCRequest * request) const; +}; + +} + diff --git a/config/src/vespa/config/frt/frtconfigrequestv3.cpp b/config/src/vespa/config/frt/frtconfigrequestv3.cpp new file mode 100644 index 00000000000..11a3496348f --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigrequestv3.cpp @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.frtconfigrequestv3"); +#include "frtconfigrequestv3.h" +#include "frtconfigresponsev3.h" +#include "connection.h" +#include <vespa/config/common/trace.h> +#include <vespa/config/common/vespa_version.h> + +using namespace config::protocol; + +namespace config { + +FRTConfigRequestV3::FRTConfigRequestV3(Connection * connection, + const ConfigKey & key, + const vespalib::string & configMd5, + int64_t currentGeneration, + int64_t wantedGeneration, + const vespalib::string & hostName, + int64_t serverTimeout, + const Trace & trace, + const VespaVersion & vespaVersion, + const CompressionType & compressionType) + : SlimeConfigRequest(connection, key, configMd5, currentGeneration, wantedGeneration, hostName, serverTimeout, trace, vespaVersion, 3, compressionType, "config.v3.getConfig") +{ +} + + + +ConfigResponse::UP +FRTConfigRequestV3::createResponse(FRT_RPCRequest * request) const +{ + return ConfigResponse::UP(new FRTConfigResponseV3(request)); +} + +} diff --git a/config/src/vespa/config/frt/frtconfigrequestv3.h b/config/src/vespa/config/frt/frtconfigrequestv3.h new file mode 100644 index 00000000000..a8f9580f232 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigrequestv3.h @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "slimeconfigrequest.h" + +class FRT_Values; +class FRT_RPCRequest; + +namespace config { + +class ConfigKey; +class Connection; +class Trace; +class VespaVersion; + +class FRTConfigRequestV3 : public SlimeConfigRequest { +public: + FRTConfigRequestV3(Connection * connection, + const ConfigKey & key, + const vespalib::string & configMd5, + int64_t currentGeneration, + int64_t wantedGeneration, + const vespalib::string & hostName, + int64_t serverTimeout, + const Trace & trace, + const VespaVersion & vespaVersion, + const CompressionType & compressionType); + ConfigResponse::UP createResponse(FRT_RPCRequest * request) const; +}; + +} + diff --git a/config/src/vespa/config/frt/frtconfigresponse.cpp b/config/src/vespa/config/frt/frtconfigresponse.cpp new file mode 100644 index 00000000000..d815525cb40 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigresponse.cpp @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.frtconfigresponse"); +#include "frtconfigresponse.h" +#include <vespa/config/common/misc.h> +#include <vespa/fnet/frt/frt.h> +#include <vespa/vespalib/stllike/string.h> + +namespace config { + +FRTConfigResponse::FRTConfigResponse(FRT_RPCRequest * request) + : _request(request), + _responseState(EMPTY), + _returnValues(_request->GetReturn()) +{ + _request->AddRef(); +} + +FRTConfigResponse::~FRTConfigResponse() +{ + _request->SubRef(); +} + +bool +FRTConfigResponse::validateResponse() +{ + if (_request->IsError()) + _responseState = ERROR; + if (_request->GetReturn()->GetNumValues() == 0) + _responseState = EMPTY; + if (_request->CheckReturnTypes(getResponseTypes().c_str())) { + _returnValues = _request->GetReturn(); + _responseState = OK; + } + return (_responseState == OK); +} + +bool +FRTConfigResponse::hasValidResponse() const +{ + return (_responseState == OK); +} + +vespalib::string FRTConfigResponse::errorMessage() const { return _request->GetErrorMessage(); } +int FRTConfigResponse::errorCode() const { return _request->GetErrorCode(); } +bool FRTConfigResponse::isError() const { return _request->IsError(); } + +// +// V1 Implementation +// +const vespalib::string FRTConfigResponseV1::RESPONSE_TYPES = "sssssilSs"; + +FRTConfigResponseV1::FRTConfigResponseV1(FRT_RPCRequest * request) + : FRTConfigResponse(request), + _key(), + _value() +{ +} + +const vespalib::string & +FRTConfigResponseV1::getResponseTypes() const +{ + return RESPONSE_TYPES; +} + +void +FRTConfigResponseV1::fill() +{ + const std::vector<vespalib::string> payload(getPayLoad()); + for (size_t i = 0; i < payload.size(); i++) { + LOG(spam, "payload(%lu): %s", i, payload[i].c_str()); + } + _value = ConfigValue(payload, calculateContentMd5(payload)); + _key = readKey(); + _state = ConfigState(vespalib::string((*_returnValues)[4]._string._str), (*_returnValues)[6]._intval64); +} + +const ConfigKey +FRTConfigResponseV1::readKey() const +{ + return ConfigKey((*_returnValues)[3]._string._str, (*_returnValues)[0]._string._str, (*_returnValues)[8]._string._str, (*_returnValues)[2]._string._str); +} + +const std::vector<vespalib::string> +FRTConfigResponseV1::getPayLoad() const +{ + uint32_t numStrings = (*_returnValues)[7]._string_array._len; + FRT_StringValue *s = (*_returnValues)[7]._string_array._pt; + std::vector<vespalib::string> payload; + payload.reserve(numStrings); + for (uint32_t i = 0; i < numStrings; i++) { + payload.push_back(vespalib::string(s[i]._str)); + } + return payload; +} + +} // namespace config diff --git a/config/src/vespa/config/frt/frtconfigresponse.h b/config/src/vespa/config/frt/frtconfigresponse.h new file mode 100644 index 00000000000..49a7bd8ba58 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigresponse.h @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/configresponse.h> +#include <vespa/config/common/configkey.h> +#include <vespa/config/common/configvalue.h> +#include <vespa/config/common/trace.h> +#include <vespa/config/common/configstate.h> + +class FRT_RPCRequest; +class FRT_Values; + +namespace config { + +/** + * Baseclass for config responses. + */ +class FRTConfigResponse : public ConfigResponse { +private: + FRTConfigResponse& operator=(const FRTConfigResponse&); +public: + typedef std::unique_ptr<FRTConfigResponse> UP; + FRTConfigResponse(FRT_RPCRequest * request); + virtual ~FRTConfigResponse(); + + bool validateResponse(); + bool hasValidResponse() const; + vespalib::string errorMessage() const; + int errorCode() const; + bool isError() const; + virtual const vespalib::string & getResponseTypes() const = 0; + +private: + enum ResponseState { EMPTY, OK, ERROR }; + + FRT_RPCRequest * _request; + ResponseState _responseState; +protected: + FRT_Values * _returnValues; +}; + +class FRTConfigResponseV1 : public FRTConfigResponse { +private: + FRTConfigResponseV1& operator=(const FRTConfigResponseV1&); +public: + FRTConfigResponseV1(FRT_RPCRequest * request); + + const ConfigKey & getKey() const { return _key; } + const ConfigValue & getValue() const { return _value; } + const Trace & getTrace() const { return _trace; } + + const ConfigState & getConfigState() const { return _state; } + + void fill(); + +private: + static const vespalib::string RESPONSE_TYPES; + + const std::vector<vespalib::string> getPayLoad() const; + const ConfigKey readKey() const; + const vespalib::string & getResponseTypes() const; + + ConfigKey _key; + ConfigValue _value; + ConfigState _state; + Trace _trace; +}; + +} // namespace config + diff --git a/config/src/vespa/config/frt/frtconfigresponsev2.cpp b/config/src/vespa/config/frt/frtconfigresponsev2.cpp new file mode 100644 index 00000000000..8de19b95145 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigresponsev2.cpp @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.frtconfigresponsev2"); +#include "frtconfigresponsev2.h" +#include <vespa/config/common/misc.h> +#include <vespa/fnet/frt/frt.h> +#include <vespa/vespalib/stllike/string.h> + +using namespace vespalib; +using namespace vespalib::slime; +using namespace vespalib::slime::convenience; +using namespace config::protocol::v2; + +namespace config { + +class V2Payload : public protocol::Payload { +public: + V2Payload(const SlimePtr & data) + : _data(data) + {} + const Inspector & getSlimePayload() const + { + return extractPayload(*_data); + } +private: + SlimePtr _data; +}; + +const vespalib::string FRTConfigResponseV2::RESPONSE_TYPES = "s"; + +FRTConfigResponseV2::FRTConfigResponseV2(FRT_RPCRequest * request) + : SlimeConfigResponse(request) +{ +} + +const vespalib::string & +FRTConfigResponseV2::getResponseTypes() const +{ + return RESPONSE_TYPES; +} + +const ConfigValue +FRTConfigResponseV2::readConfigValue() const +{ + vespalib::string md5(_data->get()[RESPONSE_CONFIG_MD5].asString().make_string()); + return ConfigValue(PayloadPtr(new V2Payload(_data)), md5); +} + +} // namespace config diff --git a/config/src/vespa/config/frt/frtconfigresponsev2.h b/config/src/vespa/config/frt/frtconfigresponsev2.h new file mode 100644 index 00000000000..ce370b3880d --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigresponsev2.h @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "slimeconfigresponse.h" +#include <vespa/config/common/configkey.h> +#include <vespa/config/common/configvalue.h> +#include <vespa/config/common/trace.h> +#include <vespa/vespalib/data/slime/slime.h> +#include "protocol.h" + +class FRT_RPCRequest; +class FRT_Values; + +namespace config { + +/** + * Baseclass for config responses. + */ +class FRTConfigResponseV2 : public SlimeConfigResponse { +private: + FRTConfigResponseV2& operator=(const FRTConfigResponseV2&); +public: + FRTConfigResponseV2(FRT_RPCRequest * request); + +private: + static const vespalib::string RESPONSE_TYPES; + const vespalib::string & getResponseTypes() const; + const ConfigValue readConfigValue() const; +}; + +} // namespace config + diff --git a/config/src/vespa/config/frt/frtconfigresponsev3.cpp b/config/src/vespa/config/frt/frtconfigresponsev3.cpp new file mode 100644 index 00000000000..5421f0cb667 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigresponsev3.cpp @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.frtconfigresponsev3"); +#include "frtconfigresponsev3.h" +#include "compressioninfo.h" +#include <vespa/config/common/misc.h> +#include <vespa/fnet/frt/frt.h> +#include <vespa/vespalib/stllike/string.h> + +using namespace vespalib; +using namespace vespalib::slime; +using namespace vespalib::slime::convenience; +using namespace config::protocol; +using namespace config::protocol::v2; +using namespace config::protocol::v3; + +namespace config { + +std::string make_json(const Slime &slime, bool compact) { + vespalib::slime::SimpleBuffer buf; + vespalib::slime::JsonFormat::encode(slime, buf, compact); + return buf.get().make_string(); +} + +class V3Payload : public Payload +{ +public: + V3Payload(const SlimePtr & data) + : _data(data) + { + } + + const Inspector & getSlimePayload() const + { + return _data->get(); + } +private: + SlimePtr _data; +}; + +const vespalib::string FRTConfigResponseV3::RESPONSE_TYPES = "sx"; + +FRTConfigResponseV3::FRTConfigResponseV3(FRT_RPCRequest * request) + : SlimeConfigResponse(request) +{ +} + +const vespalib::string & +FRTConfigResponseV3::getResponseTypes() const +{ + return RESPONSE_TYPES; +} + +const ConfigValue +FRTConfigResponseV3::readConfigValue() const +{ + vespalib::string md5(_data->get()[RESPONSE_CONFIG_MD5].asString().make_string()); + CompressionInfo info; + info.deserialize(_data->get()[RESPONSE_COMPRESSION_INFO]); + Slime * rawData = new Slime(); + SlimePtr payloadData(rawData); + DecompressedData data(decompress(((*_returnValues)[1]._data._buf), ((*_returnValues)[1]._data._len), info.compressionType, info.uncompressedSize)); + size_t consumedSize = JsonFormat::decode(data.memRef, *rawData); + if (consumedSize != data.size) { + std::string json(make_json(*payloadData, true)); + LOG(error, "Error decoding JSON. Consumed size: %lu, uncompressed size: %u, compression type: %s, assumed uncompressed size(%u), compressed size: %u, slime(%s)", consumedSize, data.size, compressionTypeToString(info.compressionType).c_str(), info.uncompressedSize, ((*_returnValues)[1]._data._len), json.c_str()); + assert(false); + } + if (LOG_WOULD_LOG(spam)) { + LOG(spam, "read config value md5(%s), payload size: %lu", md5.c_str(), data.memRef.size); + } + return ConfigValue(PayloadPtr(new V3Payload(payloadData)), md5); +} + +} // namespace config diff --git a/config/src/vespa/config/frt/frtconfigresponsev3.h b/config/src/vespa/config/frt/frtconfigresponsev3.h new file mode 100644 index 00000000000..e2b9b9e5286 --- /dev/null +++ b/config/src/vespa/config/frt/frtconfigresponsev3.h @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "slimeconfigresponse.h" +#include <vespa/config/common/configkey.h> +#include <vespa/config/common/configvalue.h> +#include <vespa/config/common/trace.h> +#include <vespa/vespalib/data/slime/slime.h> +#include "protocol.h" + +class FRT_RPCRequest; +class FRT_Values; + +namespace config { + +/** + * Baseclass for config responses. + */ +class FRTConfigResponseV3 : public SlimeConfigResponse { +private: + FRTConfigResponseV3& operator=(const FRTConfigResponseV3&); +public: + FRTConfigResponseV3(FRT_RPCRequest * request); + +private: + static const vespalib::string RESPONSE_TYPES; + const vespalib::string & getResponseTypes() const; + const ConfigValue readConfigValue() const; +}; + +} // namespace config + diff --git a/config/src/vespa/config/frt/frtconnection.cpp b/config/src/vespa/config/frt/frtconnection.cpp new file mode 100644 index 00000000000..13e37f71b42 --- /dev/null +++ b/config/src/vespa/config/frt/frtconnection.cpp @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <time.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.frtconnection"); +#include <vespa/vespalib/util/atomic.h> +#include <vespa/config/common/errorcode.h> +#include "frtconnection.h" + +using namespace vespalib; + +namespace config { + +FRTConnection::FRTConnection(const vespalib::string& address, FRT_Supervisor& supervisor, const TimingValues & timingValues) + : _address(address), + _supervisor(supervisor), + _target(0), + _suspendedUntil(0), + _suspendWarned(0), + _transientFailures(0), + _fatalFailures(0), + _transientDelay(timingValues.transientDelay), + _fatalDelay(timingValues.fatalDelay) +{ +} + +FRTConnection::~FRTConnection() +{ + if (_target != NULL) { + _target->SubRef(); + _target = NULL; + } +} + +FRT_Target * +FRTConnection::getTarget() +{ + if (_target == NULL) { + _target = _supervisor.GetTarget(_address.c_str()); + } else if ( ! _target->IsValid()) { + _target->SubRef(); + _target = _supervisor.GetTarget(_address.c_str()); + } + return _target; +} + +void +FRTConnection::invoke(FRT_RPCRequest * req, double timeout, FRT_IRequestWait * waiter) +{ + getTarget()->InvokeAsync(req, timeout, waiter); +} + +void +FRTConnection::setError(int errorCode) +{ + switch(errorCode) { + case FRTE_RPC_CONNECTION: + case FRTE_RPC_TIMEOUT: + calculateSuspension(TRANSIENT); break; + case ErrorCode::UNKNOWN_CONFIG: + case ErrorCode::UNKNOWN_DEFINITION: + case ErrorCode::UNKNOWN_VERSION: + case ErrorCode::UNKNOWN_CONFIGID: + case ErrorCode::UNKNOWN_DEF_MD5: + case ErrorCode::ILLEGAL_NAME: + case ErrorCode::ILLEGAL_VERSION: + case ErrorCode::ILLEGAL_CONFIGID: + case ErrorCode::ILLEGAL_DEF_MD5: + case ErrorCode::ILLEGAL_CONFIG_MD5: + case ErrorCode::ILLEGAL_TIMEOUT: + case ErrorCode::OUTDATED_CONFIG: + case ErrorCode::INTERNAL_ERROR: + calculateSuspension(FATAL); break; + } +} + +void FRTConnection::setSuccess() +{ + _transientFailures = 0; + _fatalFailures = 0; + _suspendedUntil = 0; +} + +void FRTConnection::calculateSuspension(ErrorType type) +{ + int64_t delay = 0; + switch(type) { + case TRANSIENT: + Atomic::postInc(&_transientFailures); + delay = _transientFailures * getTransientDelay(); + if (delay > getMaxTransientDelay()) { + delay = getMaxTransientDelay(); + } + LOG(warning, "Connection to %s failed or timed out", _address.c_str()); + break; + case FATAL: + Atomic::postInc(&_fatalFailures); + delay = _fatalFailures * getFatalDelay(); + if (delay > getMaxFatalDelay()) { + delay = getMaxFatalDelay(); + } + break; + } + int64_t now = time(0); + now *= 1000; + _suspendedUntil = now + delay; + if (_suspendWarned < (now - 5000)) { + char date[32]; + struct tm* timeinfo; + time_t suspendedSeconds = _suspendedUntil / 1000; + timeinfo = gmtime(&suspendedSeconds); + strftime(date, 32, "%Y-%m-%d %H:%M:%S %Z", timeinfo); + LOG(warning, "FRT Connection %s suspended until %s", _address.c_str(), date); + _suspendWarned = now; + } +} + +} diff --git a/config/src/vespa/config/frt/frtconnection.h b/config/src/vespa/config/frt/frtconnection.h new file mode 100644 index 00000000000..a8c0b03b288 --- /dev/null +++ b/config/src/vespa/config/frt/frtconnection.h @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <string> +#include <vespa/fnet/frt/frt.h> +#include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/config/common/timingvalues.h> +#include "connection.h" + +namespace config { + +class FRTConnection : public Connection { + +private: + FRTConnection(const FRTConnection&); + FRTConnection& operator=(const FRTConnection&); + + const vespalib::string _address; + FRT_Supervisor& _supervisor; + FRT_Target* _target; + int64_t _suspendedUntil; + int64_t _suspendWarned; + int _transientFailures; + int _fatalFailures; + int64_t _transientDelay; + int64_t _fatalDelay; + + FRT_Target * getTarget(); + +public: + typedef std::shared_ptr<FRTConnection> SP; + enum ErrorType { TRANSIENT, FATAL }; + + FRTConnection(const vespalib::string & address, FRT_Supervisor & supervisor, const TimingValues & timingValues); + ~FRTConnection(); + + FRT_RPCRequest * allocRPCRequest() + { + return _supervisor.AllocRPCRequest(); + } + + void invoke(FRT_RPCRequest * req, double timeout, FRT_IRequestWait * waiter); + const vespalib::string & getAddress() const { return _address; } + int64_t getSuspendedUntil() { return _suspendedUntil; } + void setError(int errorCode); + void setSuccess(); + void calculateSuspension(ErrorType type); + int64_t getTransientDelay() { return _transientDelay; } + int64_t getMaxTransientDelay() { return getTransientDelay() * 6; } + void setTransientDelay(int64_t delay) { _transientDelay = delay; } + int64_t getFatalDelay() { return _fatalDelay; } + int64_t getMaxFatalDelay() { return getFatalDelay() * 6; } + void setFatalDelay(int64_t delay) { _fatalDelay = delay; } +}; + +} // namespace config + diff --git a/config/src/vespa/config/frt/frtconnectionpool.cpp b/config/src/vespa/config/frt/frtconnectionpool.cpp new file mode 100644 index 00000000000..09630a6b999 --- /dev/null +++ b/config/src/vespa/config/frt/frtconnectionpool.cpp @@ -0,0 +1,152 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <cstdlib> +#include <vespa/vespalib/util/host_name.h> +#include "frtconnectionpool.h" + +namespace config { + +FRTConnectionPool::FRTConnectionKey::FRTConnectionKey(int idx, const vespalib::string& hostname) + : _idx(idx), + _hostname(hostname) +{ + +} + +int +FRTConnectionPool::FRTConnectionKey::operator<(const FRTConnectionPool::FRTConnectionKey& right) const +{ + return _idx < right._idx; +} + +int +FRTConnectionPool::FRTConnectionKey::operator==(const FRTConnectionKey& right) const +{ + return _hostname == right._hostname; +} + +FRTConnectionPool::FRTConnectionPool(const ServerSpec & spec, const TimingValues & timingValues) + : _supervisor(), + _selectIdx(0), + _hostname("") +{ + for (size_t i(0); i < spec.numHosts(); i++) { + FRTConnectionKey key(i, spec.getHost(i)); + _connections[key].reset(new FRTConnection(spec.getHost(i), _supervisor, timingValues)); + } + setHostname(); + _supervisor.Start(); +} + +FRTConnectionPool::~FRTConnectionPool() +{ + _supervisor.ShutDown(true); +} + +void +FRTConnectionPool::syncTransport() +{ + _supervisor.GetTransport()->sync(); +} + +Connection * +FRTConnectionPool::getCurrent() +{ + if (_hostname.compare("") == 0) { + return getNextRoundRobin(); + } else { + return getNextHashBased(); + } +} + +FRTConnection * +FRTConnectionPool::getNextRoundRobin() +{ + std::vector<FRTConnection *> readySources; + getReadySources(readySources); + std::vector<FRTConnection *> suspendedSources; + getSuspendedSources(suspendedSources); + FRTConnection* nextFRTConnection = NULL; + + if (!readySources.empty()) { + int sel = _selectIdx % (int)readySources.size(); + _selectIdx = sel + 1; + nextFRTConnection = readySources[sel]; + } else if (!suspendedSources.empty()) { + int sel = _selectIdx % (int)suspendedSources.size(); + _selectIdx = sel + 1; + nextFRTConnection = suspendedSources[sel]; + } + return nextFRTConnection; +} + +FRTConnection * +FRTConnectionPool::getNextHashBased() +{ + std::vector<FRTConnection*> readySources; + getReadySources(readySources); + std::vector<FRTConnection*> suspendedSources; + getSuspendedSources(suspendedSources); + FRTConnection* nextFRTConnection = NULL; + + if (!readySources.empty()) { + int sel = abs(hashCode(_hostname) % (int)readySources.size()); + nextFRTConnection = readySources[sel]; + } else { + int sel = abs(hashCode(_hostname) % (int)suspendedSources.size()); + nextFRTConnection = suspendedSources[sel]; + } + return nextFRTConnection; +} + +const std::vector<FRTConnection *> & +FRTConnectionPool::getReadySources(std::vector<FRTConnection*> & readySources) const +{ + readySources.clear(); + for (ConnectionMap::const_iterator iter = _connections.begin(); iter != _connections.end(); iter++) { + FRTConnection* source = iter->second.get(); + int64_t tnow = time(0); + tnow *= 1000; + int64_t timestamp = tnow; + if (source->getSuspendedUntil() < timestamp) { + readySources.push_back(source); + } + } + return readySources; +} + +const std::vector<FRTConnection *> & +FRTConnectionPool::getSuspendedSources(std::vector<FRTConnection*> & suspendedSources) const +{ + suspendedSources.clear(); + for (ConnectionMap::const_iterator iter = _connections.begin(); iter != _connections.end(); iter++) { + FRTConnection* source = iter->second.get(); + int64_t tnow = time(0); + tnow *= 1000; + int64_t timestamp = tnow; + if (source->getSuspendedUntil() >= timestamp) { + suspendedSources.push_back(source); + } + } + return suspendedSources; +} + +int FRTConnectionPool::hashCode(const vespalib::string & s) +{ + int hashval = 0; + + for (int i = 0; i < (int)s.length(); i++) { + hashval = 31 * hashval + s[i]; + } + return hashval; +} + +void +FRTConnectionPool::setHostname() +{ + _hostname = vespalib::HostName::get(); +} + + +} diff --git a/config/src/vespa/config/frt/frtconnectionpool.h b/config/src/vespa/config/frt/frtconnectionpool.h new file mode 100644 index 00000000000..28f2ee83557 --- /dev/null +++ b/config/src/vespa/config/frt/frtconnectionpool.h @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vector> +#include <string> +#include <map> +#include <vespa/fnet/frt/frt.h> +#include "frtconnection.h" +#include <vespa/config/subscription/sourcespec.h> +#include "connectionfactory.h" + +namespace config { + +class FRTConnectionPool : public ConnectionFactory { + +private: + FRTConnectionPool(const FRTConnectionPool&); + FRTConnectionPool& operator=(const FRTConnectionPool&); + + /** + * This class makes it possible to iterate over the entries in the + * connections map in the order they were inserted. Used to keep + * consistency with the Java version that uses LinkedHashMap. + */ + class FRTConnectionKey { + private: + int _idx; + vespalib::string _hostname; + public: + FRTConnectionKey() : FRTConnectionKey(0, "") {} + FRTConnectionKey(int idx, const vespalib::string& hostname); + int operator<(const FRTConnectionKey& right) const; + int operator==(const FRTConnectionKey& right) const; + }; + + FRT_Supervisor _supervisor; + int _selectIdx; + vespalib::string _hostname; + typedef std::map<FRTConnectionKey, FRTConnection::SP> ConnectionMap; + ConnectionMap _connections; + +public: + FRTConnectionPool(const ServerSpec & spec, const TimingValues & timingValues); + ~FRTConnectionPool(); + + void syncTransport(); + + /** + * Sets the hostname to the host where this program is running. + */ + void setHostname(); + + /** + * Sets the hostname. + * + * @param hostname the hostname + */ + void setHostname(const vespalib::string & hostname) { _hostname = hostname; } + + FNET_Scheduler * getScheduler() { return _supervisor.GetScheduler(); } + + /** + * Gets the hostname. + * + * @return the hostname + */ + vespalib::string & getHostname() { return _hostname; } + + /** + * Trim away leading and trailing spaces. + * + * @param s the string to trim away spaces from + * @return string without leading or trailing spaces + */ + vespalib::string trim(vespalib::string s); + + /** + * Returns the current FRTConnection instance, taken from the list of error-free sources. + * If no sources are error-free, an instance from the list of sources with errors + * is returned. + * + * @return The next FRTConnection instance in the list. + */ + Connection* getCurrent(); + + /** + * Returns the next FRTConnection instance from the list of error-free sources in a round robin + * fashion. If no sources are error-free, an instance from the list of sources with errors + * is returned. + * + * @return The next FRTConnection instance in the list. + */ + FRTConnection* getNextRoundRobin(); + + /** + * Returns the current FRTConnection instance from the list of error-free sources, based on the + * hostname where this program is currently running. If no sources are error-free, an instance + * from the list of sources with errors is returned. + * + * @return The next FRTConnection instance in the list. + */ + FRTConnection* getNextHashBased(); + + /** + * Gets list of sources that are not suspended. + * + * @return list of FRTConnection pointers + */ + const std::vector<FRTConnection*> & getReadySources(std::vector<FRTConnection*> & readySources) const; + + /** + * Gets list of sources that are suspended. + * + * @param suspendedSources is list of FRTConnection pointers + */ + const std::vector<FRTConnection*> & getSuspendedSources(std::vector<FRTConnection*> & suspendedSources) const; + + /** + * Implementation of the Java hashCode function for the String class. + * + * Ensures that the same hostname maps to the same configserver/proxy + * for both language implementations. + * + * @param s the string to compute the hash from + * @return the hash value + */ + static int hashCode(const vespalib::string & s); +}; + +} // namespace config + diff --git a/config/src/vespa/config/frt/frtsource.cpp b/config/src/vespa/config/frt/frtsource.cpp new file mode 100644 index 00000000000..c209dc69618 --- /dev/null +++ b/config/src/vespa/config/frt/frtsource.cpp @@ -0,0 +1,134 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.frtsource"); +#include <vespa/fnet/frt/frt.h> +#include "frtconfigrequest.h" +#include "frtconfigresponse.h" +#include "frtsource.h" +#include "connection.h" +#include <vespa/vespalib/util/closuretask.h> + +using vespalib::Closure; +using vespalib::makeClosure; + +namespace config { + +class GetConfigTask : public FNET_Task { +public: + GetConfigTask(FNET_Scheduler * scheduler, FRTSource * source) + : FNET_Task(scheduler), + _source(source) + { + } + ~GetConfigTask() + { + Kill(); + } + void PerformTask() + { + _source->getConfig(); + } +private: + FRTSource * _source; +}; + +FRTSource::FRTSource(const ConnectionFactory::SP & connectionFactory, const FRTConfigRequestFactory & requestFactory, ConfigAgent::UP agent, const ConfigKey & key) + : _connectionFactory(connectionFactory), + _requestFactory(requestFactory), + _agent(std::move(agent)), + _currentRequest(), + _key(key), + _task(new GetConfigTask(_connectionFactory->getScheduler(), this)), + _lock(), + _closed(false) +{ + LOG(spam, "New source!"); +} + +FRTSource::~FRTSource() +{ + LOG(spam, "Destructing source"); + close(); +} + +void +FRTSource::getConfig() +{ + int64_t serverTimeout = _agent->getTimeout(); + double clientTimeout = (serverTimeout / 1000.0) + 5.0; // The additional 5 seconds is the time allowed for the server to respond after serverTimeout has elapsed. + Connection * connection = _connectionFactory->getCurrent(); + const ConfigState & state(_agent->getConfigState()); + // LOG(debug, "invoking request with md5 %s, gen %" PRId64 ", servertimeout(%" PRId64 "), client(%f)", state.md5.c_str(), state.generation, serverTimeout, clientTimeout); + + + FRTConfigRequest::UP request = _requestFactory.createConfigRequest(_key, connection, state, serverTimeout); + FRT_RPCRequest * req = request->getRequest(); + + _currentRequest = std::move(request); + connection->invoke(req, clientTimeout, this); +} + + +void +FRTSource::RequestDone(FRT_RPCRequest * request) +{ + if (request->GetErrorCode() == FRTE_RPC_ABORT) { + LOG(debug, "request aborted, stopping"); + return; + } + assert(_currentRequest.get() != NULL); + // If this was error from FRT side and nothing to do with config, notify + // connection about the error. + if (request->IsError()) { + _currentRequest->setError(request->GetErrorCode()); + } + _agent->handleResponse(*_currentRequest, _currentRequest->createResponse(request)); + LOG(spam, "Calling schedule"); + scheduleNextGetConfig(); +} + +void +FRTSource::close() +{ + { + vespalib::LockGuard guard(_lock); + if (_closed) + return; + LOG(spam, "Killing task"); + _task->Kill(); + } + LOG(spam, "Aborting"); + if (_currentRequest.get() != NULL) + _currentRequest->abort(); + LOG(spam, "Syncing"); + _connectionFactory->syncTransport(); + _currentRequest.reset(0); + LOG(spam, "closed"); +} + +void +FRTSource::scheduleNextGetConfig() +{ + vespalib::LockGuard guard(_lock); + if (_closed) + return; + double sec = _agent->getWaitTime() / 1000.0; + LOG(debug, "Scheduling task in %f seconds", sec); + _task->Schedule(sec); + LOG(debug, "Done scheduling task"); +} + +void +FRTSource::reload(int64_t generation) +{ + (void) generation; +} + +const FRTConfigRequest & +FRTSource::getCurrentRequest() const +{ + return *_currentRequest; +} + +} // namespace config diff --git a/config/src/vespa/config/frt/frtsource.h b/config/src/vespa/config/frt/frtsource.h new file mode 100644 index 00000000000..2e20bbc4a1a --- /dev/null +++ b/config/src/vespa/config/frt/frtsource.h @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/source.h> +#include "connectionfactory.h" +#include "frtconfigagent.h" +#include <vespa/config/common/configkey.h> +#include <vespa/config/common/configrequest.h> +#include "frtconfigrequestfactory.h" + +#include <vespa/fnet/frt/frt.h> +#include <vespa/vespalib/util/sync.h> + +namespace config { + +/** + * Class for sending and receiving config requests via FRT. + */ +class FRTSource : public Source, + public FRT_IRequestWait +{ +public: + FRTSource(const ConnectionFactory::SP & connectionFactory, const FRTConfigRequestFactory & requestFactory, ConfigAgent::UP agent, const ConfigKey & key); + ~FRTSource(); + + void RequestDone(FRT_RPCRequest * request); + void close(); + void reload(int64_t generation); + void getConfig(); + + const FRTConfigRequest & getCurrentRequest() const; + +private: + void scheduleNextGetConfig(); + + ConnectionFactory::SP _connectionFactory; + const FRTConfigRequestFactory & _requestFactory; + ConfigAgent::UP _agent; + FRTConfigRequest::UP _currentRequest; + const ConfigKey _key; + + std::unique_ptr<FNET_Task> _task; + vespalib::Lock _lock; // Protects _task and _closed + bool _closed; +}; + +} // namespace config + diff --git a/config/src/vespa/config/frt/frtsourcefactory.cpp b/config/src/vespa/config/frt/frtsourcefactory.cpp new file mode 100644 index 00000000000..d11b5d6799c --- /dev/null +++ b/config/src/vespa/config/frt/frtsourcefactory.cpp @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.frtsourcefactory"); +#include "frtsourcefactory.h" +#include "frtsource.h" + +namespace config { + +FRTSourceFactory::FRTSourceFactory(ConnectionFactory::UP connectionFactory, const TimingValues & timingValues, int protocolVersion, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType) + : _connectionFactory(connectionFactory.release()), + _requestFactory(protocolVersion, traceLevel, vespaVersion, compressionType), + _timingValues(timingValues) +{ +} + +Source::UP +FRTSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const +{ + return Source::UP(new FRTSource(_connectionFactory, _requestFactory, ConfigAgent::UP(new FRTConfigAgent(holder, _timingValues)), key)); +} + +} // namespace config diff --git a/config/src/vespa/config/frt/frtsourcefactory.h b/config/src/vespa/config/frt/frtsourcefactory.h new file mode 100644 index 00000000000..a16a628ee1f --- /dev/null +++ b/config/src/vespa/config/frt/frtsourcefactory.h @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/sourcefactory.h> +#include <vespa/config/common/timingvalues.h> +#include "connectionfactory.h" +#include "frtconfigrequestfactory.h" + +namespace config { + +/** + * Class for sending and receiving config requests via FRT. + */ +class FRTSourceFactory : public SourceFactory +{ +public: + FRTSourceFactory(ConnectionFactory::UP connectionFactory, const TimingValues & timingValues, int protocolVersion, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType); + + /** + * Create source handling config described by key. + */ + Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const; + +private: + ConnectionFactory::SP _connectionFactory; + FRTConfigRequestFactory _requestFactory; + const TimingValues _timingValues; +}; + +} // namespace config + diff --git a/config/src/vespa/config/frt/protocol.cpp b/config/src/vespa/config/frt/protocol.cpp new file mode 100644 index 00000000000..be02bc59862 --- /dev/null +++ b/config/src/vespa/config/frt/protocol.cpp @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.protocol"); +#include "protocol.h" +#include <lz4.h> +#include <vespa/vespalib/util/stringfmt.h> + +using namespace vespalib; +using namespace vespalib::slime; + +namespace config { +namespace protocol { +namespace v2 { + +const Memory REQUEST_VERSION = "version"; +const Memory REQUEST_DEF_NAME = "defName"; +const Memory REQUEST_DEF_NAMESPACE = "defNamespace"; +const Memory REQUEST_DEF_MD5 = "defMD5"; +const Memory REQUEST_DEF_CONTENT = "defContent"; +const Memory REQUEST_CLIENT_CONFIGID = "configId"; +const Memory REQUEST_CLIENT_HOSTNAME = "clientHostname"; +const Memory REQUEST_CONFIG_MD5 = "configMD5"; +const Memory REQUEST_CURRENT_GENERATION = "currentGeneration"; +const Memory REQUEST_WANTED_GENERATION = "wantedGeneration"; +const Memory REQUEST_TIMEOUT = "timeout"; +const Memory REQUEST_TRACE = "trace"; +const Memory REQUEST_VESPA_VERSION = "vespaVersion"; + +const Memory RESPONSE_VERSION = "version"; +const Memory RESPONSE_DEF_NAME = "defName"; +const Memory RESPONSE_DEF_NAMESPACE = "defNamespace"; +const Memory RESPONSE_DEF_MD5 = "defMD5"; +const Memory RESPONSE_CONFIGID = "configId"; +const Memory RESPONSE_CLIENT_HOSTNAME = "clientHostname"; +const Memory RESPONSE_CONFIG_MD5 = "configMD5"; +const Memory RESPONSE_CONFIG_GENERATION = "generation"; +const Memory RESPONSE_PAYLOAD = "payload"; +const Memory RESPONSE_TRACE = "trace"; + +const Inspector & +extractPayload(const Slime & data) +{ + const Inspector & payload(data.get()[RESPONSE_PAYLOAD]); + if (LOG_WOULD_LOG(debug)) { + LOG(debug, "payload: %s", payload.toString().c_str()); + } + return payload; +} + +} + +namespace v3 { +const Memory REQUEST_COMPRESSION_TYPE = "compressionType"; +const Memory RESPONSE_COMPRESSION_INFO = "compressionInfo"; +const Memory RESPONSE_COMPRESSION_INFO_TYPE = "compressionType"; +const Memory RESPONSE_COMPRESSION_INFO_UNCOMPRESSED_SIZE = "uncompressedSize"; + +DecompressedData +decompress_lz4(const char * input, uint32_t inputLen, int uncompressedLength) +{ + DefaultAlloc::UP memory(new DefaultAlloc(uncompressedLength)); + int sz = LZ4_decompress_safe(input, static_cast<char *>(memory->get()), inputLen, uncompressedLength); + if (sz >= 0 && sz != uncompressedLength) { + if (LOG_WOULD_LOG(debug)) { + LOG(debug, "Returned compressed size (%d) is not the same as uncompressed size(%d)", sz, uncompressedLength); + } + DefaultAlloc * copy = new DefaultAlloc(sz); + memcpy(copy->get(), memory->get(), sz); + memory.reset(copy); + } + assert(sz >= 0); + return DecompressedData(std::move(memory), static_cast<uint32_t>(sz)); +} + +DecompressedData +decompress(const char * input, uint32_t len, const CompressionType & compressionType, uint32_t uncompressedLength) +{ + // No payload means no data + if (len == 0) { + return DecompressedData(Memory(input, len), len); + } + switch (compressionType) { + case CompressionType::LZ4: + return decompress_lz4(input, len, uncompressedLength); + break; + case CompressionType::UNCOMPRESSED: + default: + return DecompressedData(Memory(input, len), len); + break; + } +} + +} + +const int DEFAULT_PROTOCOL_VERSION = 3; +const int DEFAULT_TRACE_LEVEL = 0; + +int +verifyProtocolVersion(int protocolVersion) +{ + if (1 == protocolVersion || 2 == protocolVersion || 3 == protocolVersion) { + return protocolVersion; + } + LOG(info, "Unknown protocol version %d, using default (%d)", protocolVersion, DEFAULT_PROTOCOL_VERSION); + return DEFAULT_PROTOCOL_VERSION; +} + +int +readProtocolVersion() +{ + int protocolVersion = DEFAULT_PROTOCOL_VERSION; + char *versionStringPtr = getenv("VESPA_CONFIG_PROTOCOL_VERSION"); + if (versionStringPtr == NULL) { + versionStringPtr = getenv("services__config_protocol_version_override"); + } + if (versionStringPtr != NULL) { + std::stringstream versionString(versionStringPtr); + versionString >> protocolVersion; + } + return verifyProtocolVersion(protocolVersion); +} + +int +readTraceLevel() +{ + int traceLevel = DEFAULT_TRACE_LEVEL; + char *traceLevelStringPtr = getenv("VESPA_CONFIG_PROTOCOL_TRACELEVEL"); + if (traceLevelStringPtr == NULL) { + traceLevelStringPtr = getenv("services__config_protocol_tracelevel"); + } + if (traceLevelStringPtr != NULL) { + std::stringstream traceLevelString(traceLevelStringPtr); + traceLevelString >> traceLevel; + } + return traceLevel; +} + +CompressionType +readProtocolCompressionType() +{ + CompressionType type = CompressionType::LZ4; + char *compressionTypeStringPtr = getenv("VESPA_CONFIG_PROTOCOL_COMPRESSION"); + if (compressionTypeStringPtr == NULL) { + compressionTypeStringPtr = getenv("services__config_protocol_compression"); + } + if (compressionTypeStringPtr != NULL) { + type = stringToCompressionType(vespalib::string(compressionTypeStringPtr)); + } + return type; +} + +} +} diff --git a/config/src/vespa/config/frt/protocol.h b/config/src/vespa/config/frt/protocol.h new file mode 100644 index 00000000000..79d916bf3e7 --- /dev/null +++ b/config/src/vespa/config/frt/protocol.h @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/config/common/compressiontype.h> +#include <vespa/vespalib/util/alloc.h> + +namespace config { + +namespace protocol { + +int readProtocolVersion(); +int readTraceLevel(); +CompressionType readProtocolCompressionType(); + +struct Payload { + virtual ~Payload() {} + virtual const vespalib::slime::Inspector & getSlimePayload() const = 0; +}; + + +namespace v2 { + +extern const vespalib::slime::Memory REQUEST_VERSION; +extern const vespalib::slime::Memory REQUEST_DEF_NAME; +extern const vespalib::slime::Memory REQUEST_DEF_NAMESPACE; +extern const vespalib::slime::Memory REQUEST_DEF_MD5; +extern const vespalib::slime::Memory REQUEST_DEF_CONTENT; +extern const vespalib::slime::Memory REQUEST_CLIENT_CONFIGID; +extern const vespalib::slime::Memory REQUEST_CLIENT_HOSTNAME; +extern const vespalib::slime::Memory REQUEST_CONFIG_MD5; +extern const vespalib::slime::Memory REQUEST_CURRENT_GENERATION; +extern const vespalib::slime::Memory REQUEST_WANTED_GENERATION; +extern const vespalib::slime::Memory REQUEST_TIMEOUT; +extern const vespalib::slime::Memory REQUEST_TRACE; +extern const vespalib::slime::Memory REQUEST_VESPA_VERSION; + +extern const vespalib::slime::Memory RESPONSE_VERSION; +extern const vespalib::slime::Memory RESPONSE_DEF_NAME; +extern const vespalib::slime::Memory RESPONSE_DEF_NAMESPACE; +extern const vespalib::slime::Memory RESPONSE_DEF_MD5; +extern const vespalib::slime::Memory RESPONSE_CONFIGID; +extern const vespalib::slime::Memory RESPONSE_CLIENT_HOSTNAME; +extern const vespalib::slime::Memory RESPONSE_CONFIG_MD5; +extern const vespalib::slime::Memory RESPONSE_CONFIG_GENERATION; +extern const vespalib::slime::Memory RESPONSE_PAYLOAD; +extern const vespalib::slime::Memory RESPONSE_TRACE; + +const vespalib::slime::Inspector & extractPayload(const vespalib::Slime & data); + +} + +namespace v3 { + +extern const vespalib::slime::Memory REQUEST_COMPRESSION_TYPE; +extern const vespalib::slime::Memory RESPONSE_COMPRESSION_INFO; +extern const vespalib::slime::Memory RESPONSE_COMPRESSION_INFO_TYPE; +extern const vespalib::slime::Memory RESPONSE_COMPRESSION_INFO_UNCOMPRESSED_SIZE; + +struct DecompressedData { + DecompressedData(vespalib::DefaultAlloc::UP mem, uint32_t sz) + : memory(std::move(mem)), + memRef(static_cast<const char *>(memory->get()), sz), + size(sz) + { } + DecompressedData(const vespalib::slime::Memory & mem, uint32_t sz) + : memory(), + memRef(mem), + size(sz) + {} + + vespalib::DefaultAlloc::UP memory; + vespalib::slime::Memory memRef; + uint32_t size; +}; + +DecompressedData decompress(const char * buf, uint32_t len, const CompressionType & compressionType, uint32_t uncompressedLength); + +} + +} + +} + diff --git a/config/src/vespa/config/frt/slimeconfigrequest.cpp b/config/src/vespa/config/frt/slimeconfigrequest.cpp new file mode 100644 index 00000000000..0f6458d0d66 --- /dev/null +++ b/config/src/vespa/config/frt/slimeconfigrequest.cpp @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.slimeconfigrequest"); +#include "slimeconfigrequest.h" +#include "connection.h" +#include <vespa/fnet/frt/frt.h> +#include <vespa/config/common/configkey.h> +#include <vespa/config/common/configstate.h> +#include <vespa/config/common/configdefinition.h> +#include <vespa/config/common/trace.h> +#include <vespa/config/common/vespa_version.h> + +using namespace vespalib; +using namespace vespalib::slime; +using namespace vespalib::slime::convenience; +using namespace config::protocol; +using namespace config::protocol::v2; +using namespace config::protocol::v3; + +namespace config { + +const vespalib::string SlimeConfigRequest::REQUEST_TYPES = "s"; + +SlimeConfigRequest::SlimeConfigRequest(Connection * connection, + const ConfigKey & key, + const vespalib::string & configMd5, + int64_t currentGeneration, + int64_t wantedGeneration, + const vespalib::string & hostName, + int64_t serverTimeout, + const Trace & trace, + const VespaVersion & vespaVersion, + int64_t protocolVersion, + const CompressionType & compressionType, + const vespalib::string & methodName) + : FRTConfigRequest(connection, key), + _data() +{ + populateSlimeRequest(key, configMd5, currentGeneration, wantedGeneration, hostName, serverTimeout, trace, vespaVersion, protocolVersion, compressionType); + _request->SetMethodName(methodName.c_str()); + _parameters.AddString(createJsonFromSlime(_data).c_str()); +} + +bool +SlimeConfigRequest::verifyKey(const ConfigKey & key) const +{ + return (key.getDefName().compare(_parameters[0]._string._str) == 0 && + key.getDefNamespace().compare(_parameters[7]._string._str) == 0 && + key.getConfigId().compare(_parameters[3]._string._str) == 0 && + key.getDefMd5().compare(_parameters[2]._string._str) == 0); +} + +bool +SlimeConfigRequest::verifyState(const ConfigState & state) const +{ + return (state.md5.compare(_parameters[4]._string._str) == 0 && + state.generation == static_cast<int64_t>(_parameters[5]._intval64)); +} + +void +SlimeConfigRequest::populateSlimeRequest(const ConfigKey & key, + const vespalib::string & configMd5, + int64_t currentGeneration, + int64_t wantedGeneration, + const vespalib::string & hostName, + int64_t serverTimeout, + const Trace & trace, + const VespaVersion & vespaVersion, + int64_t protocolVersion, + const CompressionType & compressionType) +{ + Cursor & root(_data.setObject()); + root.setLong(REQUEST_VERSION, protocolVersion); + root.setString(REQUEST_DEF_NAME, Memory(key.getDefName())); + root.setString(REQUEST_DEF_NAMESPACE, Memory(key.getDefNamespace())); + root.setString(REQUEST_DEF_MD5, Memory(key.getDefMd5())); + ConfigDefinition def(key.getDefSchema()); + def.serialize(root.setArray(REQUEST_DEF_CONTENT)); + root.setString(REQUEST_CLIENT_CONFIGID, Memory(key.getConfigId())); + root.setString(REQUEST_CLIENT_HOSTNAME, Memory(hostName)); + root.setString(REQUEST_CONFIG_MD5, Memory(configMd5)); + root.setLong(REQUEST_CURRENT_GENERATION, currentGeneration); + root.setLong(REQUEST_WANTED_GENERATION, wantedGeneration); + root.setLong(REQUEST_TIMEOUT, serverTimeout); + trace.serialize(root.setObject(REQUEST_TRACE)); + root.setString(REQUEST_COMPRESSION_TYPE, Memory(compressionTypeToString(compressionType))); + root.setString(REQUEST_VESPA_VERSION, Memory(vespaVersion.toString())); +} + +vespalib::string +SlimeConfigRequest::createJsonFromSlime(const Slime & data) +{ + SimpleBuffer buf; + JsonFormat::encode(data, buf, true); + return buf.get().make_string(); +} + +} diff --git a/config/src/vespa/config/frt/slimeconfigrequest.h b/config/src/vespa/config/frt/slimeconfigrequest.h new file mode 100644 index 00000000000..af5272597be --- /dev/null +++ b/config/src/vespa/config/frt/slimeconfigrequest.h @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/data/slime/slime.h> +#include "frtconfigrequest.h" +#include "protocol.h" + +class FRT_Values; +class FRT_RPCRequest; + +namespace config { + +class ConfigKey; +class Connection; +class Trace; +class VespaVersion; + +class SlimeConfigRequest : public FRTConfigRequest { +public: + SlimeConfigRequest(Connection * connection, + const ConfigKey & key, + const vespalib::string & configMd5, + int64_t currentGeneration, + int64_t wantedGeneration, + const vespalib::string & hostName, + int64_t serverTimeout, + const Trace & trace, + const VespaVersion & vespaVersion, + int64_t protocolVersion, + const CompressionType & compressionType, + const vespalib::string & methodName); + virtual ~SlimeConfigRequest() {} + bool verifyKey(const ConfigKey & key) const; + bool verifyState(const ConfigState & state) const; + virtual ConfigResponse::UP createResponse(FRT_RPCRequest * request) const = 0; +private: + void populateSlimeRequest(const ConfigKey & key, + const vespalib::string & configMd5, + int64_t currentGeneration, + int64_t wantedGeneration, + const vespalib::string & hostName, + int64_t serverTimeout, + const Trace & trace, + const VespaVersion & vespaVersion, + int64_t protocolVersion, + const CompressionType & compressionType); + static vespalib::string createJsonFromSlime(const vespalib::Slime & data); + static const vespalib::string REQUEST_TYPES; + vespalib::Slime _data; +}; + +} + diff --git a/config/src/vespa/config/frt/slimeconfigresponse.cpp b/config/src/vespa/config/frt/slimeconfigresponse.cpp new file mode 100644 index 00000000000..ac84852b256 --- /dev/null +++ b/config/src/vespa/config/frt/slimeconfigresponse.cpp @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.frt.slimeconfigresponse"); +#include "slimeconfigresponse.h" +#include <vespa/config/common/misc.h> +#include <vespa/fnet/frt/frt.h> +#include <vespa/vespalib/stllike/string.h> + +using namespace vespalib; +using namespace vespalib::slime; +using namespace vespalib::slime::convenience; +using namespace config::protocol::v2; + +namespace config { + +SlimeConfigResponse::SlimeConfigResponse(FRT_RPCRequest * request) + : FRTConfigResponse(request), + _key(), + _value(), + _trace(), + _filled(false) +{ +} + +void +SlimeConfigResponse::fill() +{ + if (_filled) { + LOG(info, "SlimeConfigResponse::fill() called twice, probably a bug"); + return; + } + Memory json((*_returnValues)[0]._string._str); + Slime * data = new Slime(); + JsonFormat::decode(json, *data); + _data.reset(data); + _key = readKey(); + _state = readState(); + _value = readConfigValue(); + readTrace(); + _filled = true; + if (LOG_WOULD_LOG(debug)) { + LOG(debug, "trace at return(%s)", _trace.toString().c_str()); + } +} + +void +SlimeConfigResponse::readTrace() +{ + Inspector & root(_data->get()); + _trace.deserialize(root[RESPONSE_TRACE]); +} + +const ConfigKey +SlimeConfigResponse::readKey() const +{ + Inspector & root(_data->get()); + return ConfigKey(root[RESPONSE_CONFIGID].asString().make_string(), + root[RESPONSE_DEF_NAME].asString().make_string(), + root[RESPONSE_DEF_NAMESPACE].asString().make_string(), + root[RESPONSE_DEF_MD5].asString().make_string()); +} + +const ConfigState +SlimeConfigResponse::readState() const +{ + const Slime & data(*_data); + return ConfigState(data.get()[RESPONSE_CONFIG_MD5].asString().make_string(), data.get()[RESPONSE_CONFIG_GENERATION].asLong()); +} + +vespalib::string +SlimeConfigResponse::getHostName() const +{ + Inspector & root(_data->get()); + return root[RESPONSE_CLIENT_HOSTNAME].asString().make_string(); +} + +} // namespace config diff --git a/config/src/vespa/config/frt/slimeconfigresponse.h b/config/src/vespa/config/frt/slimeconfigresponse.h new file mode 100644 index 00000000000..4f22f0726bc --- /dev/null +++ b/config/src/vespa/config/frt/slimeconfigresponse.h @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "frtconfigresponse.h" +#include <vespa/config/common/configkey.h> +#include <vespa/config/common/configvalue.h> +#include <vespa/config/common/trace.h> +#include <vespa/config/common/misc.h> +#include <vespa/vespalib/data/slime/slime.h> +#include "protocol.h" + +class FRT_RPCRequest; +class FRT_Values; + +namespace config { + +/** + * Baseclass for config responses. + */ +class SlimeConfigResponse : public FRTConfigResponse { +private: + SlimeConfigResponse& operator=(const SlimeConfigResponse&); +public: + SlimeConfigResponse(FRT_RPCRequest * request); + virtual ~SlimeConfigResponse() {} + + const ConfigKey & getKey() const { return _key; } + const ConfigValue & getValue() const { return _value; } + const ConfigState & getConfigState() const { return _state; } + const Trace & getTrace() const { return _trace; } + + vespalib::string getHostName() const; + vespalib::string getConfigMd5() const; + + void fill(); + +protected: + virtual const ConfigValue readConfigValue() const = 0; + +private: + ConfigKey _key; + ConfigValue _value; + ConfigState _state; + Trace _trace; + bool _filled; + + const ConfigKey readKey() const; + const ConfigState readState() const; + void readTrace(); + +protected: + SlimePtr _data; +}; + +} // namespace config + diff --git a/config/src/vespa/config/helper/.gitignore b/config/src/vespa/config/helper/.gitignore new file mode 100644 index 00000000000..7e7c0fe7fae --- /dev/null +++ b/config/src/vespa/config/helper/.gitignore @@ -0,0 +1,2 @@ +/.depend +/Makefile diff --git a/config/src/vespa/config/helper/CMakeLists.txt b/config/src/vespa/config/helper/CMakeLists.txt new file mode 100644 index 00000000000..bc1688364bc --- /dev/null +++ b/config/src/vespa/config/helper/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(config_helper OBJECT + SOURCES + configfetcher.cpp + legacysubscriber.cpp + legacy.cpp + configpoller.cpp + DEPENDS +) diff --git a/config/src/vespa/config/helper/configfetcher.cpp b/config/src/vespa/config/helper/configfetcher.cpp new file mode 100644 index 00000000000..f6e39523900 --- /dev/null +++ b/config/src/vespa/config/helper/configfetcher.cpp @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/log/log.h> +LOG_SETUP(".config.helper.configfetcher"); +#include "configfetcher.h" + +namespace config { + +ConfigFetcher::ConfigFetcher(const IConfigContext::SP & context) + : _poller(context), + _thread(_poller), + _closed(false), + _started(false) +{ +} + +ConfigFetcher::ConfigFetcher(const SourceSpec & spec) + : _poller(IConfigContext::SP(new ConfigContext(spec))), + _thread(_poller), + _closed(false), + _started(false) +{ +} + +void +ConfigFetcher::start() +{ + if (!_closed) { + LOG(debug, "Polling for config"); + _poller.poll(); + LOG(debug, "Starting fetcher thread..."); + _thread.start(); + _started = true; + LOG(debug, "Fetcher thread started"); + } +} + +ConfigFetcher::~ConfigFetcher() +{ + close(); +} + +void +ConfigFetcher::close() +{ + if (!_closed) { + _poller.close(); + if (_started) + _thread.join(); + } +} + +} // namespace config diff --git a/config/src/vespa/config/helper/configfetcher.h b/config/src/vespa/config/helper/configfetcher.h new file mode 100644 index 00000000000..ad7c2a8b549 --- /dev/null +++ b/config/src/vespa/config/helper/configfetcher.h @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/config.h> +#include <vespa/config/common/timingvalues.h> +#include <vespa/vespalib/util/thread.h> +#include "configpoller.h" + +#include <atomic> + +namespace config { + +/** + * A config fetcher subscribes to a config and notifies a callback when done + */ +class ConfigFetcher +{ +public: + ConfigFetcher(const IConfigContext::SP & context); + ConfigFetcher(const SourceSpec & spec = ServerSpec()); + ~ConfigFetcher(); + + template <typename ConfigType> + void subscribe(const std::string & configId, IFetcherCallback<ConfigType> * callback, uint64_t subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); + + void subscribeGenerationChanges(IGenerationCallback * callback) { + _poller.subscribeGenerationChanges(callback); + } + + void start(); + void close(); + int64_t getGeneration() const { return _poller.getGeneration(); } +private: + ConfigPoller _poller; + vespalib::Thread _thread; + std::atomic<bool> _closed; + std::atomic<bool> _started; +}; + +} // namespace config + + +#include "configfetcher.hpp" + diff --git a/config/src/vespa/config/helper/configfetcher.hpp b/config/src/vespa/config/helper/configfetcher.hpp new file mode 100644 index 00000000000..9567c716029 --- /dev/null +++ b/config/src/vespa/config/helper/configfetcher.hpp @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace config { + +template <typename ConfigType> +void +ConfigFetcher::subscribe(const std::string & configId, IFetcherCallback<ConfigType> * callback, uint64_t subscribeTimeout) +{ + _poller.subscribe<ConfigType>(configId, callback, subscribeTimeout); +} + +} // namespace config diff --git a/config/src/vespa/config/helper/configgetter.h b/config/src/vespa/config/helper/configgetter.h new file mode 100644 index 00000000000..ee0cba1988c --- /dev/null +++ b/config/src/vespa/config/helper/configgetter.h @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/config.h> +#include <vespa/config/common/timingvalues.h> +#include <vespa/config/subscription/confighandle.h> +#include <vespa/vespalib/util/threadstackexecutor.h> +#include "ifetchercallback.h" + +namespace config { + +/** + * A config fetcher subscribes to a config and notifies a callback when done + */ +template <typename ConfigType> +class ConfigGetter +{ +public: + static std::unique_ptr<ConfigType> getConfig(int64_t &generation, const std::string & configId, const SourceSpec & spec = ServerSpec()); + static std::unique_ptr<ConfigType> getConfig(int64_t &generation, const std::string & configId, const IConfigContext::SP & context, uint64_t subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); + static std::unique_ptr<ConfigType> getConfig(const std::string & configId, const SourceSpec & spec = ServerSpec()); + static std::unique_ptr<ConfigType> getConfig(const std::string & configId, const IConfigContext::SP & context, uint64_t subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); +}; + +} // namespace config + + +#include "configgetter.hpp" + diff --git a/config/src/vespa/config/helper/configgetter.hpp b/config/src/vespa/config/helper/configgetter.hpp new file mode 100644 index 00000000000..5452908a144 --- /dev/null +++ b/config/src/vespa/config/helper/configgetter.hpp @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace config { + +template <typename ConfigType> +std::unique_ptr<ConfigType> +ConfigGetter<ConfigType>::getConfig(int64_t &generation, const std::string & configId, const SourceSpec & spec) +{ + ConfigSubscriber s(spec); + std::unique_ptr< ConfigHandle<ConfigType> > h = s.subscribe<ConfigType>(configId); + s.nextConfig(0); + generation = s.getGeneration(); + return h->getConfig(); +} + +template <typename ConfigType> +std::unique_ptr<ConfigType> +ConfigGetter<ConfigType>::getConfig(int64_t &generation, const std::string & configId, const IConfigContext::SP & context, uint64_t subscribeTimeout) +{ + ConfigSubscriber s(context); + std::unique_ptr< ConfigHandle<ConfigType> > h = s.subscribe<ConfigType>(configId, subscribeTimeout); + s.nextConfig(0); + generation = s.getGeneration(); + return h->getConfig(); +} + +template <typename ConfigType> +std::unique_ptr<ConfigType> +ConfigGetter<ConfigType>::getConfig(const std::string & configId, const SourceSpec & spec) +{ + int64_t ignoreGeneration; + return getConfig(ignoreGeneration, configId, spec); +} + +template <typename ConfigType> +std::unique_ptr<ConfigType> +ConfigGetter<ConfigType>::getConfig(const std::string & configId, const IConfigContext::SP & context, uint64_t subscribeTimeout) +{ + int64_t ignoreGeneration; + return getConfig(ignoreGeneration, configId, context, subscribeTimeout); +} + +} // namespace config diff --git a/config/src/vespa/config/helper/configpoller.cpp b/config/src/vespa/config/helper/configpoller.cpp new file mode 100644 index 00000000000..281fe8df140 --- /dev/null +++ b/config/src/vespa/config/helper/configpoller.cpp @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/log/log.h> +LOG_SETUP(".config.helper.configpoller"); +#include "configpoller.h" + +namespace config { + +ConfigPoller::ConfigPoller(const IConfigContext::SP & context) + : _generation(-1), + _subscriber(context), + _handleList(), + _callbackList(), + _genCallback(0) +{ +} + +void +ConfigPoller::run() +{ + while (!_subscriber.isClosed()) { + try { + poll(); + } catch (const std::exception & e) { + LOG(fatal, "Fatal error while configuring: %s", e.what()); + } + } +} + +void +ConfigPoller::poll() +{ + LOG(debug, "Checking for new config"); + if (_subscriber.nextGeneration()) { + if (_subscriber.isClosed()) + return; + LOG(debug, "Got new config, reconfiguring"); + _generation = _subscriber.getGeneration(); + for (size_t i = 0; i < _handleList.size(); i++) { + ICallback * callback(_callbackList[i]); + if (_handleList[i]->isChanged()) + callback->configure(std::move(_handleList[i]->getConfig())); + } + if (_genCallback) { + _genCallback->notifyGenerationChange(_generation); + } + } else { + LOG(debug, "No new config available"); + } +} + +void +ConfigPoller::close() +{ + _subscriber.close(); +} + +} diff --git a/config/src/vespa/config/helper/configpoller.h b/config/src/vespa/config/helper/configpoller.h new file mode 100644 index 00000000000..f614cab5175 --- /dev/null +++ b/config/src/vespa/config/helper/configpoller.h @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/subscription/configsubscriber.h> +#include <vespa/config/common/timingvalues.h> +#include <vespa/vespalib/util/runnable.h> +#include "ifetchercallback.h" +#include "ihandle.h" + +namespace config { + +/** + * A config poller runs a polling sequence on a set of configs that it has + * subscribed to. + */ +class ConfigPoller : public vespalib::Runnable { +public: + ConfigPoller(const IConfigContext::SP & context); + void run(); + template <typename ConfigType> + void subscribe(const std::string & configId, IFetcherCallback<ConfigType> * callback, uint64_t subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); + void subscribeGenerationChanges(IGenerationCallback * callback) { _genCallback = callback; } + void poll(); + void close(); + int64_t getGeneration() const { return _generation; } +private: + int64_t _generation; + ConfigSubscriber _subscriber; + std::vector<IHandle::UP> _handleList; + std::vector<ICallback *> _callbackList; + IGenerationCallback *_genCallback; +}; + + +} // namespace config + +#include "configpoller.hpp" + + diff --git a/config/src/vespa/config/helper/configpoller.hpp b/config/src/vespa/config/helper/configpoller.hpp new file mode 100644 index 00000000000..96e6ea334e8 --- /dev/null +++ b/config/src/vespa/config/helper/configpoller.hpp @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace config { + +template <typename ConfigType> +void +ConfigPoller::subscribe(const std::string & configId, IFetcherCallback<ConfigType> * callback, uint64_t subscribeTimeout) +{ + + std::unique_ptr<ConfigHandle<ConfigType> > handle(_subscriber.subscribe<ConfigType>(configId, subscribeTimeout)); + _handleList.emplace_back(std::make_unique<GenericHandle<ConfigType>>(std::move(handle))); + _callbackList.push_back(callback); +} + +} // namespace config diff --git a/config/src/vespa/config/helper/ifetchercallback.h b/config/src/vespa/config/helper/ifetchercallback.h new file mode 100644 index 00000000000..19ad4a8984b --- /dev/null +++ b/config/src/vespa/config/helper/ifetchercallback.h @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/configgen/configinstance.h> + +namespace config { + +class IGenerationCallback +{ +public: + virtual void notifyGenerationChange(int64_t generation) = 0; + virtual ~IGenerationCallback() {} +}; + +class ICallback +{ +public: + virtual void configure(std::unique_ptr<const ConfigInstance> config) = 0; + virtual ~ICallback() { } +}; + +/** + * Interface for callback methods used by ConfigFetcher, ConfigPoller and + * LegacySubscriber. + */ +template <typename ConfigType> +class IFetcherCallback : public ICallback +{ +public: + virtual ~IFetcherCallback() { } +protected: + virtual void configure(std::unique_ptr<const ConfigInstance> config) { + configure(std::unique_ptr<ConfigType>(static_cast<const ConfigType *>(config.release()))); + } + virtual void configure(std::unique_ptr<ConfigType> config) = 0; +}; + +} // namespace config + diff --git a/config/src/vespa/config/helper/ihandle.h b/config/src/vespa/config/helper/ihandle.h new file mode 100644 index 00000000000..d84f324487a --- /dev/null +++ b/config/src/vespa/config/helper/ihandle.h @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/subscription/confighandle.h> + +namespace config { + +class IHandle +{ +public: + typedef std::unique_ptr<IHandle> UP; + virtual std::unique_ptr<const ConfigInstance> getConfig() = 0; + virtual bool isChanged() = 0; + virtual ~IHandle() { } +}; + +template <typename ConfigType> +class GenericHandle : public IHandle +{ +public: + GenericHandle(std::unique_ptr<ConfigHandle<ConfigType> > handle) + : _handle(std::move(handle)) + { + } + + std::unique_ptr<const ConfigInstance> getConfig() { + return std::unique_ptr<const ConfigInstance>(_handle->getConfig().release()); + } + bool isChanged() { return _handle->isChanged(); } +private: + std::unique_ptr<ConfigHandle <ConfigType> > _handle; +}; + +} + diff --git a/config/src/vespa/config/helper/legacy.cpp b/config/src/vespa/config/helper/legacy.cpp new file mode 100644 index 00000000000..5598769f07d --- /dev/null +++ b/config/src/vespa/config/helper/legacy.cpp @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "legacy.h" +#include <vespa/vespalib/io/fileutil.h> + +namespace { + bool isFileLegacy(const std::string & configId) { + return configId.compare(0, 5, "file:") == 0; + } + bool isDirLegacy(const std::string & configId) { + return configId.compare(0, 4, "dir:") == 0; + } + const std::string dirNameFromId(const std::string & configId) { + return configId.substr(4); + } + const std::string createFileSpecFromId(const std::string & configId) { + return configId.substr(5); + } + const std::string createBaseId(const std::string & configId) { + std::string::size_type end = configId.find_last_of("."); + return configId.substr(5, end - 5); + } + bool isRawLegacy(const std::string & configId) { + return configId.compare(0, 4, "raw:") == 0; + } + const std::string createRawSpecFromId(const std::string & configId) { + return configId.substr(4); + } +} + +namespace config { + +bool +isLegacyConfigId(const std::string & configId) +{ + return (isRawLegacy(configId) || + isFileLegacy(configId) || + isDirLegacy(configId)); +} + +std::unique_ptr<SourceSpec> +legacyConfigId2Spec(const std::string & configId) +{ + std::unique_ptr<SourceSpec> spec(new ServerSpec()); + if (isFileLegacy(configId)) { + spec.reset(new FileSpec(createFileSpecFromId(configId))); + } else if (isDirLegacy(configId)) { + spec.reset(new DirSpec(dirNameFromId(configId))); + } else if (isRawLegacy(configId)) { + spec.reset(new RawSpec(createRawSpecFromId(configId))); + } + return spec; +} + +const std::string +legacyConfigId2ConfigId(const std::string & configId) +{ + std::string newId(configId); + if (isFileLegacy(configId)) { + newId = createBaseId(configId); + } else if (isRawLegacy(configId) || isDirLegacy(configId)) { + newId = ""; + } + return newId; +} + +} diff --git a/config/src/vespa/config/helper/legacy.h b/config/src/vespa/config/helper/legacy.h new file mode 100644 index 00000000000..530d613bba8 --- /dev/null +++ b/config/src/vespa/config/helper/legacy.h @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <string> +#include <vespa/config/subscription/sourcespec.h> + +namespace config { + +bool isLegacyConfigId(const std::string & configId); +std::unique_ptr<SourceSpec> legacyConfigId2Spec(const std::string & configId); +const std::string legacyConfigId2ConfigId(const std::string & configId); + +} + + diff --git a/config/src/vespa/config/helper/legacysubscriber.cpp b/config/src/vespa/config/helper/legacysubscriber.cpp new file mode 100644 index 00000000000..1adde34c4a7 --- /dev/null +++ b/config/src/vespa/config/helper/legacysubscriber.cpp @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "legacysubscriber.h" + +namespace config { + +LegacySubscriber::LegacySubscriber() + : _fetcher(), + _configId() +{ +} + +void +LegacySubscriber::close() +{ + if (_fetcher.get() != NULL) + _fetcher->close(); +} + +} // namespace config diff --git a/config/src/vespa/config/helper/legacysubscriber.h b/config/src/vespa/config/helper/legacysubscriber.h new file mode 100644 index 00000000000..714e465572d --- /dev/null +++ b/config/src/vespa/config/helper/legacysubscriber.h @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "configfetcher.h" +#include <vespa/config/subscription/sourcespec.h> +#include "legacy.h" + +namespace config { + +/** + * A LegacySubscriber subscribes for a config similar to the old config API. + */ +class LegacySubscriber +{ +public: + LegacySubscriber(); + const vespalib::string & id() const { return _configId; } + + template <typename ConfigType> + void subscribe(const std::string & configId, IFetcherCallback<ConfigType> * callback); + + void close(); +private: + std::unique_ptr<ConfigFetcher> _fetcher; + vespalib::string _configId; +}; + +} // namespace config + +#include "legacysubscriber.hpp" + diff --git a/config/src/vespa/config/helper/legacysubscriber.hpp b/config/src/vespa/config/helper/legacysubscriber.hpp new file mode 100644 index 00000000000..fb91e79c915 --- /dev/null +++ b/config/src/vespa/config/helper/legacysubscriber.hpp @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace config { + +template <typename ConfigType> +void +LegacySubscriber::subscribe(const std::string & configId, IFetcherCallback<ConfigType> * callback) +{ + if (isLegacyConfigId(configId)) { + std::string legacyId(legacyConfigId2ConfigId(configId)); + std::unique_ptr<SourceSpec> spec(legacyConfigId2Spec(configId)); + _fetcher.reset(new ConfigFetcher(IConfigContext::SP(new ConfigContext(*spec)))); + _fetcher->subscribe<ConfigType>(legacyId, callback); + } else { + _fetcher.reset(new ConfigFetcher()); + _fetcher->subscribe<ConfigType>(configId, callback); + } + _configId = configId; + _fetcher->start(); +} + +} // namespace config diff --git a/config/src/vespa/config/print.h b/config/src/vespa/config/print.h new file mode 100644 index 00000000000..0b0471c9e13 --- /dev/null +++ b/config/src/vespa/config/print.h @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/print/configwriter.h> +#include <vespa/config/print/fileconfigwriter.h> +#include <vespa/config/print/ostreamconfigwriter.h> +#include <vespa/config/print/asciiconfigwriter.h> +#include <vespa/config/print/asciiconfigwriter.h> +#include <vespa/config/print/configformatter.h> +#include <vespa/config/print/jsonconfigformatter.h> +#include <vespa/config/print/fileconfigformatter.h> +#include <vespa/config/print/fileconfigsnapshotwriter.h> +#include <vespa/config/print/fileconfigsnapshotreader.h> +#include <vespa/config/print/asciiconfigsnapshotwriter.h> +#include <vespa/config/print/asciiconfigsnapshotreader.h> + +/** + * @section DESCRIPTION + * + * This file contains all necessary includes as well as functions used to print + * config. + */ + diff --git a/config/src/vespa/config/print/.gitignore b/config/src/vespa/config/print/.gitignore new file mode 100644 index 00000000000..7e7c0fe7fae --- /dev/null +++ b/config/src/vespa/config/print/.gitignore @@ -0,0 +1,2 @@ +/.depend +/Makefile diff --git a/config/src/vespa/config/print/CMakeLists.txt b/config/src/vespa/config/print/CMakeLists.txt new file mode 100644 index 00000000000..469ec46f0a8 --- /dev/null +++ b/config/src/vespa/config/print/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(config_print OBJECT + SOURCES + fileconfigformatter.cpp + asciiconfigwriter.cpp + fileconfigwriter.cpp + ostreamconfigwriter.cpp + jsonconfigformatter.cpp + fileconfigsnapshotwriter.cpp + fileconfigsnapshotreader.cpp + asciiconfigsnapshotwriter.cpp + asciiconfigsnapshotreader.cpp + DEPENDS +) diff --git a/config/src/vespa/config/print/asciiconfigreader.h b/config/src/vespa/config/print/asciiconfigreader.h new file mode 100644 index 00000000000..135d2d4d51f --- /dev/null +++ b/config/src/vespa/config/print/asciiconfigreader.h @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/asciistream.h> +#include "configreader.h" +#include "configformatter.h" + +namespace config { + +/** + * Read a config from istream + */ +template <typename ConfigType> +class AsciiConfigReader : public ConfigReader<ConfigType> +{ +public: + AsciiConfigReader(vespalib::asciistream & is); + std::unique_ptr<ConfigType> read(); + std::unique_ptr<ConfigType> read(const ConfigFormatter & formatter); +private: + vespalib::asciistream & _is; +}; + +} // namespace config + +#include "asciiconfigreader.hpp" + diff --git a/config/src/vespa/config/print/asciiconfigreader.hpp b/config/src/vespa/config/print/asciiconfigreader.hpp new file mode 100644 index 00000000000..bc73f1285e2 --- /dev/null +++ b/config/src/vespa/config/print/asciiconfigreader.hpp @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace config { + +template <typename ConfigType> +AsciiConfigReader<ConfigType>::AsciiConfigReader(vespalib::asciistream & is) + : _is(is) +{ +} + +template <typename ConfigType> +std::unique_ptr<ConfigType> +AsciiConfigReader<ConfigType>::read(const ConfigFormatter & formatter) +{ + ConfigDataBuffer buffer; + buffer.setEncodedString(_is.str()); + formatter.decode(buffer); + return std::unique_ptr<ConfigType>(new ConfigType(buffer)); +} + +template <typename ConfigType> +std::unique_ptr<ConfigType> +AsciiConfigReader<ConfigType>::read() +{ + std::vector<vespalib::string> lines; + vespalib::string line; + while (getline(_is, line)) { + lines.push_back(line); + } + return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentMd5(lines)))); +} + +} // namespace config diff --git a/config/src/vespa/config/print/asciiconfigsnapshotreader.cpp b/config/src/vespa/config/print/asciiconfigsnapshotreader.cpp new file mode 100644 index 00000000000..83fc8f55cfd --- /dev/null +++ b/config/src/vespa/config/print/asciiconfigsnapshotreader.cpp @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "asciiconfigsnapshotreader.h" +#include "jsonconfigformatter.h" + +namespace config { + +AsciiConfigSnapshotReader::AsciiConfigSnapshotReader(const vespalib::asciistream & is) + : _is(is) +{ +} + +ConfigSnapshot +AsciiConfigSnapshotReader::read() +{ + ConfigDataBuffer buffer; + buffer.setEncodedString(_is.str()); + JsonConfigFormatter formatter(true); + formatter.decode(buffer); + ConfigSnapshot snapshot; + snapshot.deserialize(buffer); + return snapshot; +} + +} // namespace config diff --git a/config/src/vespa/config/print/asciiconfigsnapshotreader.h b/config/src/vespa/config/print/asciiconfigsnapshotreader.h new file mode 100644 index 00000000000..28120b38b93 --- /dev/null +++ b/config/src/vespa/config/print/asciiconfigsnapshotreader.h @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/asciistream.h> +#include "configsnapshotreader.h" + +namespace config { + +/** + * Read config snapshots from an ascii stream. + */ +class AsciiConfigSnapshotReader : public ConfigSnapshotReader { +public: + AsciiConfigSnapshotReader(const vespalib::asciistream & is); + + /** + * Read a config snapshot. + * + * @return Snapshot containing the configs. + */ + ConfigSnapshot read(); +private: + const vespalib::asciistream & _is; +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/asciiconfigsnapshotwriter.cpp b/config/src/vespa/config/print/asciiconfigsnapshotwriter.cpp new file mode 100644 index 00000000000..a338e88b862 --- /dev/null +++ b/config/src/vespa/config/print/asciiconfigsnapshotwriter.cpp @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "asciiconfigsnapshotwriter.h" +#include "jsonconfigformatter.h" + +namespace config { + +AsciiConfigSnapshotWriter::AsciiConfigSnapshotWriter(vespalib::asciistream & os) + : _os(os) +{ +} + +bool +AsciiConfigSnapshotWriter::write(const ConfigSnapshot & snapshot) +{ + ConfigDataBuffer buffer; + snapshot.serialize(buffer); + JsonConfigFormatter formatter(true); + formatter.encode(buffer); + _os << buffer.getEncodedString(); + return !_os.fail(); +} + +} // namespace config diff --git a/config/src/vespa/config/print/asciiconfigsnapshotwriter.h b/config/src/vespa/config/print/asciiconfigsnapshotwriter.h new file mode 100644 index 00000000000..f49da791477 --- /dev/null +++ b/config/src/vespa/config/print/asciiconfigsnapshotwriter.h @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/asciistream.h> +#include "configsnapshotwriter.h" + +namespace config { + +/** + * Write a config snapshot to an ascii stream. + */ +class AsciiConfigSnapshotWriter : public ConfigSnapshotWriter { +public: + AsciiConfigSnapshotWriter(vespalib::asciistream & os); + bool write(const ConfigSnapshot & snapshot); +private: + vespalib::asciistream & _os; +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/asciiconfigwriter.cpp b/config/src/vespa/config/print/asciiconfigwriter.cpp new file mode 100644 index 00000000000..9f738fe716a --- /dev/null +++ b/config/src/vespa/config/print/asciiconfigwriter.cpp @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "asciiconfigwriter.h" +#include "fileconfigformatter.h" + +namespace config { + +AsciiConfigWriter::AsciiConfigWriter(vespalib::asciistream & os) + : _os(os) +{ +} + +bool +AsciiConfigWriter::write(const ConfigInstance & config) +{ + return write(config, FileConfigFormatter()); +} + +bool +AsciiConfigWriter::write(const ConfigInstance & config, const ConfigFormatter & formatter) +{ + ConfigDataBuffer buffer; + config.serialize(buffer); + formatter.encode(buffer); + _os << buffer.getEncodedString(); + return !_os.fail(); +} + +} // namespace config diff --git a/config/src/vespa/config/print/asciiconfigwriter.h b/config/src/vespa/config/print/asciiconfigwriter.h new file mode 100644 index 00000000000..a22e3608855 --- /dev/null +++ b/config/src/vespa/config/print/asciiconfigwriter.h @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "configwriter.h" +#include "configformatter.h" +#include <vespa/vespalib/stllike/asciistream.h> + +namespace config { + +class AsciiConfigWriter : public ConfigWriter { +public: + AsciiConfigWriter(vespalib::asciistream & os); + bool write(const ConfigInstance & config); + bool write(const ConfigInstance & config, const ConfigFormatter & formatter); +private: + vespalib::asciistream & _os; +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/configdatabuffer.h b/config/src/vespa/config/print/configdatabuffer.h new file mode 100644 index 00000000000..ff485a19058 --- /dev/null +++ b/config/src/vespa/config/print/configdatabuffer.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/stllike/string.h> + +namespace config { + +/** + * Simple data container for slime object. + */ +class ConfigDataBuffer +{ +public: + vespalib::Slime & slimeObject() { return _slime; } + const vespalib::Slime & slimeObject() const { return _slime; } + const vespalib::string & getEncodedString() const { return _encoded; } + void setEncodedString(const vespalib::string & encoded) { _encoded = encoded; } +private: + vespalib::Slime _slime; + vespalib::string _encoded; +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/configformatter.h b/config/src/vespa/config/print/configformatter.h new file mode 100644 index 00000000000..f8fc1d8134b --- /dev/null +++ b/config/src/vespa/config/print/configformatter.h @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "configdatabuffer.h" + +namespace config { + +/** + * Interface used by all config formatters. A config formatter is capable of + * encoding and decoding into any kind of format that can be put into a string. + */ +class ConfigFormatter { +public: + /** + * Encode the slime object in a config data buffer, and put it into its + * string. + * + * @param buffer A ConfigDataBuffer containing a slime object that should be + * encoded. + */ + virtual void encode(ConfigDataBuffer & buffer) const = 0; + + /** + * Decode a string in the config data buffer and populate its slime object. + * + * @param buffer A ConfigDataBuffer containing a string of the config. + */ + virtual size_t decode(ConfigDataBuffer & buffer) const = 0; + + virtual ~ConfigFormatter() { } +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/configreader.h b/config/src/vespa/config/print/configreader.h new file mode 100644 index 00000000000..84789c1b485 --- /dev/null +++ b/config/src/vespa/config/print/configreader.h @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "configformatter.h" + +namespace config { + +/** + * Interface implemented by all classes capable of reading config of a specific + * type. + */ +template <typename ConfigType> +class ConfigReader { +public: + /** + * Read a config using a provided formatter, and return the correct type. + * + * @param formatter Something implementing ConfigFormatter interface. + * @return Instance of correct type. + */ + virtual std::unique_ptr<ConfigType> read(const ConfigFormatter & formatter) = 0; + virtual ~ConfigReader() { } +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/configsnapshotreader.h b/config/src/vespa/config/print/configsnapshotreader.h new file mode 100644 index 00000000000..c5fd416cace --- /dev/null +++ b/config/src/vespa/config/print/configsnapshotreader.h @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/retriever/configsnapshot.h> + +namespace config { + +/** + * Interface implemented by all classes capable of reading config of a specific + * type. + */ +class ConfigSnapshotReader { +public: + /** + * Read a config snapshot. + * + * @return Snapshot containing the configs. + */ + virtual ConfigSnapshot read() = 0; + virtual ~ConfigSnapshotReader() { } +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/configsnapshotwriter.h b/config/src/vespa/config/print/configsnapshotwriter.h new file mode 100644 index 00000000000..103668b82a8 --- /dev/null +++ b/config/src/vespa/config/print/configsnapshotwriter.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/retriever/configsnapshot.h> + +namespace config { + +/** + * Interface for classes capable of writing a config snapshots somewhere. + */ +class ConfigSnapshotWriter { +public: + /** + * Write this config snapshot to a place decided by the implementer of this + * class. + * + * @param config The config snapshot to write. + * @return true if successful, false if not. + */ + virtual bool write(const ConfigSnapshot & snapshot) = 0; + virtual ~ConfigSnapshotWriter() { } +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/configwriter.h b/config/src/vespa/config/print/configwriter.h new file mode 100644 index 00000000000..782a1e57dab --- /dev/null +++ b/config/src/vespa/config/print/configwriter.h @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/configgen/configinstance.h> +#include "configformatter.h" + +namespace config { + +/** + * Interface for classes capable of writing a config instance somewhere. + */ +class ConfigWriter { +public: + /** + * Write this config instance to a place decided by the implementer of this + * class. + * + * @param config The config instance to write. + */ + virtual bool write(const ConfigInstance & config) = 0; + + /** + * Write this config instance to a place decided by the implementer of this + * class. The provided formatter decides the format of the output. + * + * @param config The config instance to write. + * @param formatter The config formatter to use for formatting config. + */ + virtual bool write(const ConfigInstance & config, const ConfigFormatter & formatter) = 0; + virtual ~ConfigWriter() { } +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/fileconfigformatter.cpp b/config/src/vespa/config/print/fileconfigformatter.cpp new file mode 100644 index 00000000000..4c9c96f06c4 --- /dev/null +++ b/config/src/vespa/config/print/fileconfigformatter.cpp @@ -0,0 +1,236 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <cmath> +#include <stack> +#include <vector> +#include "fileconfigformatter.h" +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/exceptions.h> + +using namespace vespalib::slime::convenience; + +using vespalib::slime::ArrayTraverser; +using vespalib::slime::ObjectTraverser; +using vespalib::slime::BufferedOutput; +using vespalib::slime::SimpleBuffer; +using vespalib::slime::Output; + +namespace config { + void doEncode(ConfigDataBuffer & buffer, Output & output); +} + +namespace { + +struct ConfigEncoder : public ArrayTraverser, + public ObjectTraverser +{ + BufferedOutput &out; + int level; + bool head; + std::vector<std::string> prefixList; + + ConfigEncoder(BufferedOutput &out_in) + : out(out_in), level(0), head(true) {} + + void printPrefix() { + for (size_t i = 0; i < prefixList.size(); i++) { + out.printf("%s", prefixList[i].c_str()); + } + } + + void encodeBOOL(bool value) { + if (value) { + out.printf("true"); + } else { + out.printf("false"); + } + } + void encodeLONG(int64_t value) { + out.printf("%ld", value); + } + void encodeDOUBLE(double value) { + out.printf("%g", value); + } + void encodeSTRINGNOQUOTE(const Memory &memory) { + const char *hex = "0123456789ABCDEF"; + char *p = out.reserve(memory.size * 6); + size_t len = 0; + const char *pos = memory.data; + const char *end = memory.data + memory.size; + for (; pos < end; ++pos) { + uint8_t c = *pos; + switch(c) { + case '"': *p++ = '\\'; *p++ = '"'; len += 2; break; + case '\\': *p++ = '\\'; *p++ = '\\'; len += 2; break; + case '\b': *p++ = '\\'; *p++ = 'b'; len += 2; break; + case '\f': *p++ = '\\'; *p++ = 'f'; len += 2; break; + case '\n': *p++ = '\\'; *p++ = 'n'; len += 2; break; + case '\r': *p++ = '\\'; *p++ = 'r'; len += 2; break; + case '\t': *p++ = '\\'; *p++ = 't'; len += 2; break; + default: + if (c > 0x1f) { + *p++ = c; ++len; + } else { // requires escaping according to RFC 4627 + *p++ = '\\'; *p++ = 'u'; *p++ = '0'; *p++ = '0'; + *p++ = hex[(c >> 4) & 0xf]; *p++ = hex[c & 0xf]; + len += 6; + } + } + } + out.commit(len); + } + void encodeSTRING(const Memory &memory) { + out.writeByte('\"'); + encodeSTRINGNOQUOTE(memory); + out.writeByte('\"'); + } + void encodeARRAY(const Inspector &inspector) { + ArrayTraverser &array_traverser = *this; + inspector.traverse(array_traverser); + } + void encodeMAP(const Inspector & inspector) { + for (size_t i = 0; i < inspector.children(); i++) { + const Inspector & child(inspector[i]); + vespalib::asciistream ss; + ss << "{\"" << child["key"].asString().make_string() << "\"}"; + prefixList.push_back(ss.str()); + encodeMAPEntry(child); + prefixList.pop_back(); + } + } + void encodeMAPEntry(const Inspector & inspector) { + if (inspector["type"].valid()) { + std::string type(inspector["type"].asString().make_string()); + if (type.compare("struct") == 0) { + prefixList.push_back("."); + encodeOBJECT(inspector["value"]); + prefixList.pop_back(); + } else { + printPrefix(); + out.writeByte(' '); + if (type.compare("enum") == 0) encodeSTRINGNOQUOTE(inspector["value"].asString()); + else encodeValue(inspector["value"]); + out.writeByte('\n'); + } + } + } + void encodeOBJECT(const Inspector &inspector) { + ObjectTraverser &object_traverser = *this; + inspector.traverse(object_traverser); + } + void encodeValue(const Inspector &inspector) { + switch (inspector.type().getId()) { + case vespalib::slime::BOOL::ID: return encodeBOOL(inspector.asBool()); + case vespalib::slime::LONG::ID: return encodeLONG(inspector.asLong()); + case vespalib::slime::DOUBLE::ID: return encodeDOUBLE(inspector.asDouble()); + case vespalib::slime::STRING::ID: return encodeSTRING(inspector.asString()); + case vespalib::slime::ARRAY::ID: return encodeARRAY(inspector); + case vespalib::slime::OBJECT::ID: return encodeOBJECT(inspector); + case vespalib::slime::NIX::ID: return; + } + abort(); // should not be reached + } + virtual void entry(size_t idx, const Inspector &inspector); + virtual void field(const Memory &symbol_name, const Inspector &inspector); + + static void encode(Inspector & root, BufferedOutput &out) { + ConfigEncoder encoder(out); + encoder.encodeValue(root); + } +}; + +void +ConfigEncoder::entry(size_t index, const Inspector &inspector) +{ + if (inspector["type"].valid()) { + std::string type(inspector["type"].asString().make_string()); + if (type.compare("array") == 0) { + vespalib::asciistream ss; + ss << "[" << index << "]"; + prefixList.push_back(ss.str()); + encodeARRAY(inspector["value"]); + prefixList.pop_back(); + } else if (type.compare("struct") == 0) { + vespalib::asciistream ss; + ss << "[" << index << "]."; + prefixList.push_back(ss.str()); + encodeOBJECT(inspector["value"]); + prefixList.pop_back(); + } else { + printPrefix(); + out.writeByte('['); + encodeLONG(index); + out.writeByte(']'); + out.writeByte(' '); + + if (type.compare("enum") == 0) encodeSTRINGNOQUOTE(inspector["value"].asString()); + else encodeValue(inspector["value"]); + out.writeByte('\n'); + } + } +} + +void +ConfigEncoder::field(const Memory &symbol_name, const Inspector &inspector) +{ + if (inspector["type"].valid()) { + std::string type(inspector["type"].asString().make_string()); + if (type.compare("array") == 0) { + size_t len = inspector["value"].children(); + if (len > 0) { + prefixList.push_back(symbol_name.make_string()); + encodeARRAY(inspector["value"]); + prefixList.pop_back(); + } + } else if (type.compare("map") == 0) { + size_t len = inspector["value"].children(); + if (len > 0) { + prefixList.push_back(symbol_name.make_string()); + encodeMAP(inspector["value"]); + prefixList.pop_back(); + } + } else if (type.compare("struct") == 0) { + prefixList.push_back(symbol_name.make_string() + "."); + encodeOBJECT(inspector["value"]); + prefixList.pop_back(); + } else { + printPrefix(); + encodeSTRINGNOQUOTE(symbol_name); + out.writeByte(' '); + + if (type.compare("enum") == 0) encodeSTRINGNOQUOTE(inspector["value"].asString()); + else encodeValue(inspector["value"]); + out.writeByte('\n'); + } + } +} + +} + +namespace config { + +void +doEncode(ConfigDataBuffer & buffer, Output & output) +{ + BufferedOutput out(output); + ConfigEncoder::encode(buffer.slimeObject().get()["configPayload"], out); +} + +void +FileConfigFormatter::encode(ConfigDataBuffer & buffer) const +{ + SimpleBuffer buf; + doEncode(buffer, buf); + buffer.setEncodedString(buf.get().make_string()); +} + +size_t +FileConfigFormatter::decode(ConfigDataBuffer & buffer) const +{ + (void) buffer; + throw vespalib::IllegalArgumentException("Reading cfg format is not supported"); + return 0; +} + +} // namespace config diff --git a/config/src/vespa/config/print/fileconfigformatter.h b/config/src/vespa/config/print/fileconfigformatter.h new file mode 100644 index 00000000000..68476bf95b5 --- /dev/null +++ b/config/src/vespa/config/print/fileconfigformatter.h @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "configformatter.h" + +namespace config { + +/** + * Formatter capable of encoding config into old config format. Decoding is not + * supported. + */ +class FileConfigFormatter : public ConfigFormatter { +public: + // Inherits ConfigFormatter + void encode(ConfigDataBuffer & buffer) const; + // Inherits ConfigFormatter + size_t decode(ConfigDataBuffer & buffer) const; +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/fileconfigreader.h b/config/src/vespa/config/print/fileconfigreader.h new file mode 100644 index 00000000000..86ab43dcfc5 --- /dev/null +++ b/config/src/vespa/config/print/fileconfigreader.h @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <sstream> +#include <fstream> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/config/common/misc.h> +#include <vespa/config/common/configvalue.h> +#include <vespa/vespalib/util/exceptions.h> +#include "configreader.h" + +namespace config { + +template <typename ConfigType> +class FileConfigReader : public ConfigReader<ConfigType> { +public: + FileConfigReader(const vespalib::string & fileName); + + // Implements ConfigReader + std::unique_ptr<ConfigType> read(const ConfigFormatter & formatter); + + /** + * Read config from this file using old config format. + * + * @return An instance of the correct type. + */ + std::unique_ptr<ConfigType> read(); +private: + const vespalib::string _fileName; +}; + +} // namespace config + +#include "fileconfigreader.hpp" + diff --git a/config/src/vespa/config/print/fileconfigreader.hpp b/config/src/vespa/config/print/fileconfigreader.hpp new file mode 100644 index 00000000000..4731b62047a --- /dev/null +++ b/config/src/vespa/config/print/fileconfigreader.hpp @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace config { + +template <typename ConfigType> +FileConfigReader<ConfigType>::FileConfigReader(const vespalib::string & fileName) + : _fileName(fileName) +{ +} + +template <typename ConfigType> +std::unique_ptr<ConfigType> +FileConfigReader<ConfigType>::read(const ConfigFormatter & formatter) +{ + ConfigDataBuffer buffer; + std::ifstream file(_fileName.c_str()); + if (!file.is_open()) + throw ConfigReadException("error: unable to read file '%s'", _fileName.c_str()); + + std::stringstream buf; + buf << file.rdbuf(); + buffer.setEncodedString(buf.str()); + formatter.decode(buffer); + return std::unique_ptr<ConfigType>(new ConfigType(buffer)); +} + +template <typename ConfigType> +std::unique_ptr<ConfigType> +FileConfigReader<ConfigType>::read() +{ + std::vector<vespalib::string> lines; + std::ifstream f(_fileName.c_str()); + if (f.fail()) + throw vespalib::IllegalArgumentException(std::string("Unable to open file ") + _fileName); + std::string line; + while (getline(f, line)) { + lines.push_back(line); + } + return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentMd5(lines)))); +} + +} // namespace config diff --git a/config/src/vespa/config/print/fileconfigsnapshotreader.cpp b/config/src/vespa/config/print/fileconfigsnapshotreader.cpp new file mode 100644 index 00000000000..0cc7fe9fe38 --- /dev/null +++ b/config/src/vespa/config/print/fileconfigsnapshotreader.cpp @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <fstream> +#include <sstream> +#include "fileconfigsnapshotreader.h" +#include "jsonconfigformatter.h" +#include <iostream> + +namespace config { + +FileConfigSnapshotReader::FileConfigSnapshotReader(const vespalib::string & fileName) + : _fileName(fileName) +{ +} + +ConfigSnapshot +FileConfigSnapshotReader::read() +{ + std::ifstream file(_fileName.c_str()); + if (!file.is_open()) + throw ConfigReadException("error: unable to read file '%s'", _fileName.c_str()); + + std::stringstream buf; + buf << file.rdbuf(); + + ConfigDataBuffer buffer; + buffer.setEncodedString(buf.str()); + JsonConfigFormatter formatter(true); + formatter.decode(buffer); + ConfigSnapshot snapshot; + snapshot.deserialize(buffer); + return snapshot; +} + +} // namespace config diff --git a/config/src/vespa/config/print/fileconfigsnapshotreader.h b/config/src/vespa/config/print/fileconfigsnapshotreader.h new file mode 100644 index 00000000000..fdf3dbd303c --- /dev/null +++ b/config/src/vespa/config/print/fileconfigsnapshotreader.h @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include "configsnapshotreader.h" + +namespace config { + +/** + * Read config snapshots from file. + */ +class FileConfigSnapshotReader : public ConfigSnapshotReader { +public: + FileConfigSnapshotReader(const vespalib::string & fileName); + + /** + * Read a config snapshot. + * + * @return Snapshot containing the configs. + */ + ConfigSnapshot read(); +private: + const vespalib::string _fileName; +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/fileconfigsnapshotwriter.cpp b/config/src/vespa/config/print/fileconfigsnapshotwriter.cpp new file mode 100644 index 00000000000..de5c3af065d --- /dev/null +++ b/config/src/vespa/config/print/fileconfigsnapshotwriter.cpp @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <fstream> +#include "fileconfigsnapshotwriter.h" +#include "jsonconfigformatter.h" + +namespace config { + +FileConfigSnapshotWriter::FileConfigSnapshotWriter(const vespalib::string & fileName) + : _fileName(fileName) +{ +} + +bool +FileConfigSnapshotWriter::write(const ConfigSnapshot & snapshot) +{ + std::ofstream file(_fileName.c_str()); + if (!file.is_open()) + throw ConfigWriteException("error: could not open output file '%s'\n", _fileName.c_str()); + + ConfigDataBuffer buffer; + snapshot.serialize(buffer); + JsonConfigFormatter formatter(true); + formatter.encode(buffer); + file << buffer.getEncodedString(); + return !file.fail(); +} + +} // namespace config diff --git a/config/src/vespa/config/print/fileconfigsnapshotwriter.h b/config/src/vespa/config/print/fileconfigsnapshotwriter.h new file mode 100644 index 00000000000..104966827be --- /dev/null +++ b/config/src/vespa/config/print/fileconfigsnapshotwriter.h @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include "configsnapshotwriter.h" + +namespace config { + +/** + * Write a config snapshot to a file. + */ +class FileConfigSnapshotWriter : public ConfigSnapshotWriter { +public: + FileConfigSnapshotWriter(const vespalib::string & fileName); + bool write(const ConfigSnapshot & snapshot); +private: + const vespalib::string _fileName; +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/fileconfigwriter.cpp b/config/src/vespa/config/print/fileconfigwriter.cpp new file mode 100644 index 00000000000..fc817f9e290 --- /dev/null +++ b/config/src/vespa/config/print/fileconfigwriter.cpp @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <fstream> +#include "fileconfigwriter.h" +#include "fileconfigformatter.h" +#include "ostreamconfigwriter.h" +#include <vespa/config/common/exceptions.h> + +namespace config { + +FileConfigWriter::FileConfigWriter(const vespalib::string & fileName) + : _fileName(fileName) +{ +} + +bool +FileConfigWriter::write(const ConfigInstance & config) +{ + return write(config, FileConfigFormatter()); +} + +bool +FileConfigWriter::write(const ConfigInstance & config, const ConfigFormatter & formatter) +{ + std::ofstream file(_fileName.c_str()); + if (!file.is_open()) + throw ConfigWriteException("error: could not open output file: '%s'\n", _fileName.c_str()); + OstreamConfigWriter osw(file); + return osw.write(config, formatter); +} + +} // namespace config diff --git a/config/src/vespa/config/print/fileconfigwriter.h b/config/src/vespa/config/print/fileconfigwriter.h new file mode 100644 index 00000000000..b79f7456199 --- /dev/null +++ b/config/src/vespa/config/print/fileconfigwriter.h @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include "configwriter.h" +#include "configformatter.h" + +namespace config { + +/** + * Writes a config to file, optionally using a ConfigFormatter for formatting. + */ +class FileConfigWriter : public ConfigWriter { +public: + FileConfigWriter(const vespalib::string & fileName); + // Implements ConfigWriter + bool write(const ConfigInstance & config); + bool write(const ConfigInstance & config, const ConfigFormatter & formatter); +private: + const vespalib::string _fileName; +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/istreamconfigreader.h b/config/src/vespa/config/print/istreamconfigreader.h new file mode 100644 index 00000000000..b2446104419 --- /dev/null +++ b/config/src/vespa/config/print/istreamconfigreader.h @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <istream> +#include "configreader.h" +#include "configformatter.h" + +namespace config { + +/** + * Read a config from istream + */ +template <typename ConfigType> +class IstreamConfigReader : public ConfigReader<ConfigType> +{ +public: + IstreamConfigReader(std::istream & is); + std::unique_ptr<ConfigType> read(); + std::unique_ptr<ConfigType> read(const ConfigFormatter & formatter); +private: + std::istream & _is; +}; + +} // namespace config + +#include "istreamconfigreader.hpp" + diff --git a/config/src/vespa/config/print/istreamconfigreader.hpp b/config/src/vespa/config/print/istreamconfigreader.hpp new file mode 100644 index 00000000000..2e43fd229e3 --- /dev/null +++ b/config/src/vespa/config/print/istreamconfigreader.hpp @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace config { + +template <typename ConfigType> +IstreamConfigReader<ConfigType>::IstreamConfigReader(std::istream & is) + : _is(is) +{ +} + +template <typename ConfigType> +std::unique_ptr<ConfigType> +IstreamConfigReader<ConfigType>::read(const ConfigFormatter & formatter) +{ + ConfigDataBuffer buffer; + std::stringstream buf; + buf << _is.rdbuf(); + buffer.setEncodedString(buf.str()); + formatter.decode(buffer); + return std::unique_ptr<ConfigType>(new ConfigType(buffer)); +} + +template <typename ConfigType> +std::unique_ptr<ConfigType> +IstreamConfigReader<ConfigType>::read() +{ + std::vector<vespalib::string> lines; + std::string line; + while (getline(_is, line)) { + lines.push_back(line); + } + return std::unique_ptr<ConfigType>(new ConfigType(ConfigValue(lines, calculateContentMd5(lines)))); +} + +} // namespace config diff --git a/config/src/vespa/config/print/jsonconfigformatter.cpp b/config/src/vespa/config/print/jsonconfigformatter.cpp new file mode 100644 index 00000000000..998895eeac4 --- /dev/null +++ b/config/src/vespa/config/print/jsonconfigformatter.cpp @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <cmath> +#include <stack> +#include <vector> +#include "jsonconfigformatter.h" +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/data/slime/json_format.h> + +using namespace vespalib::slime::convenience; + +using vespalib::slime::SimpleBuffer; +using vespalib::slime::Output; +using vespalib::slime::JsonFormat; + +namespace config { + +JsonConfigFormatter::JsonConfigFormatter(bool compact) + : _compact(compact) +{ +} + +void +JsonConfigFormatter::encode(ConfigDataBuffer & buffer) const +{ + SimpleBuffer buf; + JsonFormat::encode(buffer.slimeObject(), buf, _compact); + buffer.setEncodedString(buf.get().make_string()); +} + +size_t +JsonConfigFormatter::decode(ConfigDataBuffer & buffer) const +{ + std::string ref(buffer.getEncodedString()); + return JsonFormat::decode(ref, buffer.slimeObject()); +} + +} // namespace config diff --git a/config/src/vespa/config/print/jsonconfigformatter.h b/config/src/vespa/config/print/jsonconfigformatter.h new file mode 100644 index 00000000000..f1b981870b2 --- /dev/null +++ b/config/src/vespa/config/print/jsonconfigformatter.h @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "configformatter.h" + +namespace config { + +/** + * Formatter capable of encoding and decoding config as json. + */ +class JsonConfigFormatter : public ConfigFormatter { +public: + JsonConfigFormatter(bool compact = false); + // Inherits ConfigFormatter + void encode(ConfigDataBuffer & buffer) const; + // Inherits ConfigFormatter + size_t decode(ConfigDataBuffer & buffer) const; +private: + const bool _compact; +}; + +} // namespace config + diff --git a/config/src/vespa/config/print/ostreamconfigwriter.cpp b/config/src/vespa/config/print/ostreamconfigwriter.cpp new file mode 100644 index 00000000000..eda15d9e2c0 --- /dev/null +++ b/config/src/vespa/config/print/ostreamconfigwriter.cpp @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "ostreamconfigwriter.h" +#include "fileconfigformatter.h" + +namespace config { + +OstreamConfigWriter::OstreamConfigWriter(std::ostream & os) + : _os(os) +{ +} + +bool +OstreamConfigWriter::write(const ConfigInstance & config, const ConfigFormatter & formatter) +{ + ConfigDataBuffer buffer; + config.serialize(buffer); + formatter.encode(buffer); + _os << buffer.getEncodedString(); + return !_os.fail(); +} + +bool +OstreamConfigWriter::write(const ConfigInstance & config) +{ + return write(config, FileConfigFormatter()); +} + +} // namespace config diff --git a/config/src/vespa/config/print/ostreamconfigwriter.h b/config/src/vespa/config/print/ostreamconfigwriter.h new file mode 100644 index 00000000000..fa211f9085f --- /dev/null +++ b/config/src/vespa/config/print/ostreamconfigwriter.h @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <ostream> +#include "configwriter.h" +#include "configformatter.h" + +namespace config { + +/** + * Write config to an ostream. + */ +class OstreamConfigWriter : public ConfigWriter +{ +public: + OstreamConfigWriter(std::ostream & os); + bool write(const ConfigInstance & config); + bool write(const ConfigInstance & config, const ConfigFormatter & formatter); +private: + std::ostream & _os; +}; + +} // namespace config + diff --git a/config/src/vespa/config/raw/.gitignore b/config/src/vespa/config/raw/.gitignore new file mode 100644 index 00000000000..7e7c0fe7fae --- /dev/null +++ b/config/src/vespa/config/raw/.gitignore @@ -0,0 +1,2 @@ +/.depend +/Makefile diff --git a/config/src/vespa/config/raw/CMakeLists.txt b/config/src/vespa/config/raw/CMakeLists.txt new file mode 100644 index 00000000000..f5d2ec9f064 --- /dev/null +++ b/config/src/vespa/config/raw/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(config_raw OBJECT + SOURCES + rawsource.cpp + rawsourcefactory.cpp + DEPENDS +) diff --git a/config/src/vespa/config/raw/rawsource.cpp b/config/src/vespa/config/raw/rawsource.cpp new file mode 100644 index 00000000000..3cb3003cb7e --- /dev/null +++ b/config/src/vespa/config/raw/rawsource.cpp @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/config/common/misc.h> +#include "rawsource.h" +#include <vespa/vespalib/stllike/asciistream.h> + +namespace config { + + +RawSource::RawSource(const IConfigHolder::SP & holder, const vespalib::string & payload) + : _holder(holder), + _payload(payload) +{ +} + +void +RawSource::getConfig() +{ + auto lines(readConfig()); + ConfigValue value(lines, calculateContentMd5(lines)); + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(value, true, 1))); +} + +void +RawSource::reload(int64_t generation) +{ + (void) generation; +} + +void +RawSource::close() +{ +} + +std::vector<vespalib::string> +RawSource::readConfig() +{ + vespalib::asciistream is(_payload); + return is.getlines(); +} + +} diff --git a/config/src/vespa/config/raw/rawsource.h b/config/src/vespa/config/raw/rawsource.h new file mode 100644 index 00000000000..b93e58c2111 --- /dev/null +++ b/config/src/vespa/config/raw/rawsource.h @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/source.h> +#include <vespa/config/common/iconfigholder.h> +#include <vespa/vespalib/stllike/string.h> + +namespace config { + +/** + * Class for sending and receiving config request from a raw string. + */ +class RawSource : public Source { +public: + RawSource(const IConfigHolder::SP & holder, const vespalib::string & payload); + + void getConfig(); + void reload(int64_t generation); + void close(); +private: + IConfigHolder::SP _holder; + std::vector<vespalib::string> readConfig(); + const vespalib::string _payload; +}; + +} + diff --git a/config/src/vespa/config/raw/rawsourcefactory.cpp b/config/src/vespa/config/raw/rawsourcefactory.cpp new file mode 100644 index 00000000000..c432c1db6dd --- /dev/null +++ b/config/src/vespa/config/raw/rawsourcefactory.cpp @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/config/common/source.h> +#include "rawsourcefactory.h" +#include "rawsource.h" + +namespace config { + +Source::UP +RawSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const +{ + (void) key; + return Source::UP(new RawSource(holder, _payload)); +} + +} diff --git a/config/src/vespa/config/raw/rawsourcefactory.h b/config/src/vespa/config/raw/rawsourcefactory.h new file mode 100644 index 00000000000..e994bb54ba8 --- /dev/null +++ b/config/src/vespa/config/raw/rawsourcefactory.h @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/sourcefactory.h> + +namespace config { + +/** + * Factory for RawSource + */ +class RawSourceFactory : public SourceFactory { +public: + RawSourceFactory(const vespalib::string & payload) + : _payload(payload) + { } + + Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const; +private: + const vespalib::string _payload; +}; + +} + diff --git a/config/src/vespa/config/retriever/.gitignore b/config/src/vespa/config/retriever/.gitignore new file mode 100644 index 00000000000..cd4bc99c04f --- /dev/null +++ b/config/src/vespa/config/retriever/.gitignore @@ -0,0 +1,2 @@ +/Makefile +/.depend diff --git a/config/src/vespa/config/retriever/CMakeLists.txt b/config/src/vespa/config/retriever/CMakeLists.txt new file mode 100644 index 00000000000..03a6d30ffa2 --- /dev/null +++ b/config/src/vespa/config/retriever/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(config_retriever OBJECT + SOURCES + configretriever.cpp + configsnapshot.cpp + genericconfigsubscriber.cpp + fixedconfigsubscriber.cpp + configkeyset.cpp + simpleconfigretriever.cpp + simpleconfigurer.cpp + DEPENDS +) diff --git a/config/src/vespa/config/retriever/configkeyset.cpp b/config/src/vespa/config/retriever/configkeyset.cpp new file mode 100644 index 00000000000..d64c1a4b9ed --- /dev/null +++ b/config/src/vespa/config/retriever/configkeyset.cpp @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "configkeyset.h" + +namespace config { + +ConfigKeySet & +ConfigKeySet::add(const ConfigKeySet & configKeySet) +{ + insert(configKeySet.begin(), configKeySet.end()); + return *this; +} + +} diff --git a/config/src/vespa/config/retriever/configkeyset.h b/config/src/vespa/config/retriever/configkeyset.h new file mode 100644 index 00000000000..52ab9962230 --- /dev/null +++ b/config/src/vespa/config/retriever/configkeyset.h @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/config/common/configkey.h> +#include <set> + +namespace config { + +/** + * A ConfigKeySet is a set of ConfigKey objects. Each ConfigKey represents a + * config by its definition name, version, md5, namespace and config id. + */ +class ConfigKeySet : public std::set<ConfigKey> +{ +public: + /** + * Add a new config type with a config id to this set. + * + * @param configId the configId of this key. + * @return *this for chaining. + */ + template <typename... ConfigTypes> + ConfigKeySet & add(const vespalib::string & configId); + + /** + * Add add another key set to this set. + * + * @param configKeySet The set to add. + * @return *this for chaining. + */ + ConfigKeySet & add(const ConfigKeySet & configKeySet); +private: + template<typename... ConfigTypes> + struct TypeTag {}; + + template<typename ConfigType> + void addImpl(const vespalib::string & configId, TypeTag<ConfigType>); + + template<typename ConfigType, typename... ConfigTypes> + void addImpl(const vespalib::string & configId, TypeTag<ConfigType, ConfigTypes...>); +}; + +} // namespace config + +#include "configkeyset.hpp" + diff --git a/config/src/vespa/config/retriever/configkeyset.hpp b/config/src/vespa/config/retriever/configkeyset.hpp new file mode 100644 index 00000000000..579fa454703 --- /dev/null +++ b/config/src/vespa/config/retriever/configkeyset.hpp @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace config { + + +template <typename... ConfigTypes> +ConfigKeySet & +ConfigKeySet::add(const vespalib::string & configId) +{ + addImpl(configId, TypeTag<ConfigTypes...>()); + return *this; +} + +template <typename ConfigType> +void +ConfigKeySet::addImpl(const vespalib::string & configId, TypeTag<ConfigType>) +{ + insert(ConfigKey::create<ConfigType>(configId)); +} + +template <typename ConfigType, typename... ConfigTypes> +void +ConfigKeySet::addImpl(const vespalib::string & configId, TypeTag<ConfigType, ConfigTypes...>) +{ + insert(ConfigKey::create<ConfigType>(configId)); + addImpl(configId, TypeTag<ConfigTypes...>()); +} + + +} diff --git a/config/src/vespa/config/retriever/configretriever.cpp b/config/src/vespa/config/retriever/configretriever.cpp new file mode 100644 index 00000000000..d09aaf0d332 --- /dev/null +++ b/config/src/vespa/config/retriever/configretriever.cpp @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/config/common/exceptions.h> +LOG_SETUP(".config.retriever.configretriever"); +#include "configretriever.h" + +namespace config { + + +ConfigRetriever::ConfigRetriever(const ConfigKeySet & bootstrapSet, + const IConfigContext::SP & context, + int64_t subscribeTimeout) + : _bootstrapSubscriber(bootstrapSet, context, subscribeTimeout), + _configSubscriber(), + _lock(), + _subscriptionList(), + _lastKeySet(), + _context(context), + _closed(false), + _generation(-1), + _subscribeTimeout(subscribeTimeout), + _bootstrapRequired(true) +{ +} + +ConfigSnapshot +ConfigRetriever::getBootstrapConfigs(int timeoutInMillis) +{ + bool ret = _bootstrapSubscriber.nextGeneration(timeoutInMillis); + if (!ret) { + return ConfigSnapshot(); + } + _bootstrapRequired = false; + return _bootstrapSubscriber.getConfigSnapshot(); +} + +ConfigSnapshot +ConfigRetriever::getConfigs(const ConfigKeySet & keySet, int timeoutInMillis) +{ + if (_closed) + return ConfigSnapshot(); + if (_bootstrapRequired) { + throw ConfigRuntimeException("Cannot change keySet until bootstrap getBootstrapConfigs() has been called"); + } + assert(!keySet.empty()); + if (keySet != _lastKeySet) { + _lastKeySet = keySet; + { + vespalib::LockGuard guard(_lock); + if (_closed) + return ConfigSnapshot(); + _configSubscriber.reset(new GenericConfigSubscriber(_context)); + } + _subscriptionList.clear(); + for (ConfigKeySet::const_iterator it(keySet.begin()), mt(keySet.end()); it != mt; it++) { + _subscriptionList.push_back(_configSubscriber->subscribe(*it, _subscribeTimeout)); + } + } + // Try update the subscribers generation if older than bootstrap + if (_configSubscriber->getGeneration() < _bootstrapSubscriber.getGeneration()) + _configSubscriber->nextGeneration(timeoutInMillis); + + // If we failed to get a new generation, the user should call us again. + if (_configSubscriber->getGeneration() < _bootstrapSubscriber.getGeneration()) { + return ConfigSnapshot(); + } + // If we are not in sync, even though we got a new generation, we should get + // another bootstrap. + _bootstrapRequired = _configSubscriber->getGeneration() > _bootstrapSubscriber.getGeneration(); + if (_bootstrapRequired) + return ConfigSnapshot(); + + _generation = _configSubscriber->getGeneration(); + return ConfigSnapshot(_subscriptionList, _generation); +} + +void +ConfigRetriever::close() +{ + vespalib::LockGuard guard(_lock); + _closed = true; + _bootstrapSubscriber.close(); + if (_configSubscriber.get() != NULL) + _configSubscriber->close(); +} + +bool +ConfigRetriever::isClosed() const +{ + return (_closed); +} + +} diff --git a/config/src/vespa/config/retriever/configretriever.h b/config/src/vespa/config/retriever/configretriever.h new file mode 100644 index 00000000000..8c09dd06ce6 --- /dev/null +++ b/config/src/vespa/config/retriever/configretriever.h @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/config/common/configkey.h> +#include <vespa/config/subscription/configsubscription.h> +#include "configkeyset.h" +#include "configsnapshot.h" +#include "genericconfigsubscriber.h" +#include "fixedconfigsubscriber.h" + +namespace config { + +/** + * A ConfigRetriever is a helper class for retrieving a set of dynamically + * changing and depending configs. You should use this class whenever you have a + * set of bootstrap configs, and want to subscribe to a dynamically changing set + * of configs based on those. + * + * The retriever should be used from one thread only, but close can be called + * from another thread. + */ +class ConfigRetriever +{ +public: + ConfigRetriever(const ConfigKeySet & bootstrapSet, + const IConfigContext::SP & context, + int64_t subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); + + /** + * Waits for the next generation of bootstrap configs to arrive, and returns + * them. If no new generation has arrived, return an empty snapshot. + * + * @param timeoutInMillis The timeout of the nextGeneration call, in + * milliseconds. Optional. + * @return a snapshot of bootstrap configs, empty if no new snapshot or + * retriever has been closed. + * @throws ConfigTimeoutException if initial subscribe timed out. + */ + ConfigSnapshot getBootstrapConfigs(int timeoutInMillis = DEFAULT_NEXTGENERATION_TIMEOUT); + + /** + * Return the configs represented by a ConfigKeySet in a snapshot, and makes + * sure that it is in sync with the bootstrap config. If it is not, an empty + * snapshot is returned. + * + * @param keySet The set of configs that should be fetched. The set may only + * change when bootstrap has been changed. + * @param timeoutInMillis The timeout, in milliseconds. Optional. + * @return a snapshot of configs corresponding to the keySet or + * an empty snapshot if + * a) retriever has been closed. The isClosed() method can be + * used to check for this condition. + * b) no new generation was found, in which case getConfigs() + * should be called again. + * c) generation is not in sync with bootstrap, in which case + * getBootstrapConfigs() must be called. The bootstrapRequired + * method can be used to check for this condition. + * @throws ConfigTimeoutException if resubscribe timed out. + */ + ConfigSnapshot getConfigs(const ConfigKeySet & keySet, int timeoutInMillis = DEFAULT_NEXTGENERATION_TIMEOUT); + + /** + * Close this retriever in order to shut down. + */ + void close(); + + /** + * Check if this retriever is closed. + * + * @return true if closed, false if not. + */ + bool isClosed() const; + + /** + * Returns if a new bootstrap call is required. + * + * @return true if required, false if not. + */ + bool bootstrapRequired() const { return _bootstrapRequired; } + + /** + * Get the current generation of the configs managed by this retriever. + * + * @return the generation + */ + int64_t getGeneration() const { return _generation; } + + static const int DEFAULT_SUBSCRIBE_TIMEOUT = 60000; + static const int DEFAULT_NEXTGENERATION_TIMEOUT = 60000; +private: + FixedConfigSubscriber _bootstrapSubscriber; + std::unique_ptr<GenericConfigSubscriber> _configSubscriber; + vespalib::Lock _lock; + std::vector<ConfigSubscription::SP> _subscriptionList; + ConfigKeySet _lastKeySet; + IConfigContext::SP _context; + std::unique_ptr<SourceSpec> _spec; + bool _closed; + int64_t _generation; + int64_t _subscribeTimeout; + bool _bootstrapRequired; +}; + +} // namespace config + diff --git a/config/src/vespa/config/retriever/configsnapshot.cpp b/config/src/vespa/config/retriever/configsnapshot.cpp new file mode 100644 index 00000000000..d7041f3c680 --- /dev/null +++ b/config/src/vespa/config/retriever/configsnapshot.cpp @@ -0,0 +1,258 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "configsnapshot.h" +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/config/common/misc.h> + +using vespalib::Slime; +using vespalib::slime::Cursor; +using vespalib::slime::Inspector; +using vespalib::slime::Memory; + +namespace config { + +const int64_t ConfigSnapshot::SNAPSHOT_FORMAT_VERSION = 1; + +ConfigSnapshot::ConfigSnapshot() + : _valueMap(), + _generation(0) +{} + +ConfigSnapshot::~ConfigSnapshot() +{ +} + +ConfigSnapshot::ConfigSnapshot(const ConfigSnapshot & rhs) : + _valueMap(rhs._valueMap), + _generation(rhs._generation) +{ +} + +ConfigSnapshot & +ConfigSnapshot::operator = (const ConfigSnapshot & rhs) +{ + if (&rhs != this) { + ConfigSnapshot tmp(rhs); + tmp.swap(*this); + } + return *this; +} + +void +ConfigSnapshot::swap(ConfigSnapshot & rhs) +{ + _valueMap.swap(rhs._valueMap); + std::swap(_generation, rhs._generation); +} + +ConfigSnapshot::ConfigSnapshot(const SubscriptionList & subscriptionList, int64_t generation) + : _valueMap(), + _generation(generation) +{ + for (SubscriptionList::const_iterator it(subscriptionList.begin()), mt(subscriptionList.end()); it != mt; it++) { + _valueMap[(*it)->getKey()] = Value((*it)->getLastGenerationChanged(), (*it)->getConfig()); + } +} + +ConfigSnapshot::ConfigSnapshot(const ValueMap & valueMap, int64_t generation) + : _valueMap(valueMap), + _generation(generation) +{ +} + +ConfigSnapshot +ConfigSnapshot::subset(const ConfigKeySet & keySet) const +{ + ValueMap subSet; + for (ConfigKeySet::const_iterator it(keySet.begin()), mt(keySet.end()); it != mt; it++) { + ValueMap::const_iterator found(_valueMap.find(*it)); + if (found != _valueMap.end()) { + subSet[*it] = found->second; + } + } + return ConfigSnapshot(subSet, _generation); +} + +int64_t ConfigSnapshot::getGeneration() const { return _generation; } +size_t ConfigSnapshot::size() const { return _valueMap.size(); } +bool ConfigSnapshot::empty() const { return _valueMap.empty(); } + +void +ConfigSnapshot::serialize(ConfigDataBuffer & buffer) const +{ + Slime & slime(buffer.slimeObject()); + Cursor & root(slime.setObject()); + root.setDouble("version", SNAPSHOT_FORMAT_VERSION); + + switch (SNAPSHOT_FORMAT_VERSION) { + case 1: + serializeV1(root); + break; + case 2: + serializeV2(root); + break; + default: + vespalib::asciistream ss; + ss << "Version '" << SNAPSHOT_FORMAT_VERSION << "' is not a valid version."; + throw ConfigWriteException(ss.str()); + } +} + +void +ConfigSnapshot::serializeV1(Cursor & root) const +{ + root.setDouble("generation", _generation); + Cursor & snapshots(root.setArray("snapshots")); + for (ValueMap::const_iterator it(_valueMap.begin()), mt(_valueMap.end()); it != mt; it++) { + Cursor & snapshot(snapshots.addObject()); + serializeKeyV1(snapshot.setObject("configKey"), it->first); + serializeValueV1(snapshot.setObject("configPayload"), it->second); + } +} + +void +ConfigSnapshot::serializeV2(Cursor & root) const +{ + root.setDouble("generation", _generation); + Cursor & snapshots(root.setArray("snapshots")); + for (ValueMap::const_iterator it(_valueMap.begin()), mt(_valueMap.end()); it != mt; it++) { + Cursor & snapshot(snapshots.addObject()); + serializeKeyV1(snapshot.setObject("configKey"), it->first); + serializeValueV2(snapshot.setObject("configPayload"), it->second); + } +} + +void +ConfigSnapshot::serializeKeyV1(Cursor & cursor, const ConfigKey & key) const +{ + typedef std::vector<vespalib::string> SchemaVector; + cursor.setString("configId", Memory(key.getConfigId())); + cursor.setString("defName", Memory(key.getDefName())); + cursor.setString("defNamespace", Memory(key.getDefNamespace())); + cursor.setString("defMd5", Memory(key.getDefMd5())); + Cursor & defSchema(cursor.setArray("defSchema")); + const SchemaVector & vec(key.getDefSchema()); + for (SchemaVector::const_iterator it(vec.begin()), mt(vec.end()); it != mt; it++) { + defSchema.addString(vespalib::slime::Memory(*it)); + } +} + +void +ConfigSnapshot::serializeValueV1(Cursor & cursor, const Value & value) const +{ + cursor.setDouble("lastChanged", value.first); + value.second.serializeV1(cursor.setArray("lines")); +} + +void +ConfigSnapshot::serializeValueV2(Cursor & cursor, const Value & value) const +{ + cursor.setDouble("lastChanged", value.first); + cursor.setString("md5", Memory(value.second.getMd5())); + value.second.serializeV2(cursor.setObject("payload")); +} + +void +ConfigSnapshot::deserialize(const ConfigDataBuffer & buffer) +{ + const Slime & slime(buffer.slimeObject()); + Inspector & inspector(slime.get()); + int64_t version = static_cast<int64_t>(inspector["version"].asDouble()); + switch (version) { + case 1: + deserializeV1(inspector); + break; + case 2: + deserializeV2(inspector); + break; + default: + vespalib::asciistream ss; + ss << "Version '" << version << "' is not a valid version."; + throw ConfigReadException(ss.str()); + } +} + +void +ConfigSnapshot::deserializeV1(Inspector & root) +{ + _generation = static_cast<int64_t>(root["generation"].asDouble()); + Inspector & snapshots(root["snapshots"]); + for (size_t i = 0; i < snapshots.children(); i++) { + Inspector & snapshot(snapshots[i]); + ConfigKey key(deserializeKeyV1(snapshot["configKey"])); + Value value(deserializeValueV1(snapshot["configPayload"])); + _valueMap[key] = value; + } +} + +void +ConfigSnapshot::deserializeV2(Inspector & root) +{ + _generation = static_cast<int64_t>(root["generation"].asDouble()); + Inspector & snapshots(root["snapshots"]); + for (size_t i = 0; i < snapshots.children(); i++) { + Inspector & snapshot(snapshots[i]); + ConfigKey key(deserializeKeyV1(snapshot["configKey"])); + Value value(deserializeValueV2(snapshot["configPayload"])); + _valueMap[key] = value; + } +} + +ConfigKey +ConfigSnapshot::deserializeKeyV1(Inspector & inspector) const +{ + std::vector<vespalib::string> schema; + Inspector & s(inspector["defSchema"]); + for (size_t i = 0; i < s.children(); i++) { + schema.push_back(s[i].asString().make_string()); + } + return ConfigKey(inspector["configId"].asString().make_string(), + inspector["defName"].asString().make_string(), + inspector["defNamespace"].asString().make_string(), + inspector["defMd5"].asString().make_string(), + schema); + +} + +std::pair<int64_t, ConfigValue> +ConfigSnapshot::deserializeValueV1(Inspector & inspector) const +{ + std::vector<vespalib::string> payload; + int64_t lastChanged = static_cast<int64_t>(inspector["lastChanged"].asDouble()); + Inspector & s(inspector["lines"]); + for (size_t i = 0; i < s.children(); i++) { + payload.push_back(s[i].asString().make_string()); + } + return Value(lastChanged, ConfigValue(payload, calculateContentMd5(payload))); +} + +namespace { + +class FixedPayload : public protocol::Payload { +public: + const Inspector & getSlimePayload() const + { + return _data.get(); + } + + Slime & getData() { + return _data; + } +private: + Slime _data; +}; + +} + +std::pair<int64_t, ConfigValue> +ConfigSnapshot::deserializeValueV2(Inspector & inspector) const +{ + int64_t lastChanged = static_cast<int64_t>(inspector["lastChanged"].asDouble()); + vespalib::string md5(inspector["md5"].asString().make_string()); + FixedPayload * payload = new FixedPayload(); + PayloadPtr data(payload); + copySlimeObject(inspector["payload"], payload->getData().setObject()); + return Value(lastChanged, ConfigValue(data, md5)); +} + +} diff --git a/config/src/vespa/config/retriever/configsnapshot.h b/config/src/vespa/config/retriever/configsnapshot.h new file mode 100644 index 00000000000..344fc86929c --- /dev/null +++ b/config/src/vespa/config/retriever/configsnapshot.h @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/subscription/configsubscription.h> +#include <vespa/config/common/exceptions.h> +#include <vespa/config/print/configdatabuffer.h> +#include <vespa/vespalib/stllike/string.h> +#include <map> +#include "configkeyset.h" + +namespace config { + +/** + * A ConfigSnapshot contains a map of config keys to config instances. You may + * request an instance of a config by calling the getConfig method. + */ +class ConfigSnapshot +{ +public: + typedef std::vector<ConfigSubscription::SP> SubscriptionList; + + /** + * Construct an empty config snapshot. + */ + ConfigSnapshot(); + ~ConfigSnapshot(); + + ConfigSnapshot(const ConfigSnapshot & rhs); + + /** + * Construct a config snapshot from a list of subscriptions and their + * current generation. + * + * @param subscriptionList A list of config subscriptions used to populate + * the snapshot. + * @param generation The latest generation of configs. + */ + ConfigSnapshot(const SubscriptionList & subscriptionList, int64_t generation); + + /** + * Instantiate one of the configs from this snapshot identified by its type + * and config id. + * + * @param configId The configId of the desired instance. + * @return an std::unqiue_ptr to an instance of this config. + * @throws InvalidConfigException if unable instantiate the given type or + * parse config. + * @throws IllegalConfigKeyException if the config does not exist. + */ + template <typename ConfigType> + std::unique_ptr<ConfigType> getConfig(const vespalib::string & configId) const; + + /** + * Query snapshot to check if a config of type ConfigType and id configId is + * changed relative to a provided generation. + * + * @param configId The configId of the instance to check. + * @param currentGeneration The generation of the current active config in + * use by the caller. + * @return true if changed, false if not. + * @throws IllegalConfigKeyException if the config does not exist. + */ + template <typename ConfigType> + bool isChanged(const vespalib::string & configId, int64_t currentGeneration) const; + + ConfigSnapshot & operator = (const ConfigSnapshot & rhs); + void swap(ConfigSnapshot & rhs); + + /** + * Query snapshot to check if a config of type ConfigType and id configId + * exists in this snapshot. + * + * @param configId The configId of the instance to check. + * @return true if exists, false if not. + */ + template <typename ConfigType> + bool hasConfig(const vespalib::string & configId) const; + + /** + * Create a new snapshot as a subset of this snapshot based on a set of keys. + * If a key does not exist in this snapshot, the new snapshot will not + * contain an entry for that key. + * + * @param keySet The keySet to use for selecting which configs to put in the + * new snapshot. + * @return a new snapshot. + */ + ConfigSnapshot subset(const ConfigKeySet & keySet) const; + + int64_t getGeneration() const; + size_t size() const; + bool empty() const; + + void serialize(ConfigDataBuffer & buffer) const; + void deserialize(const ConfigDataBuffer & buffer); +private: + typedef std::pair<int64_t, ConfigValue> Value; + typedef std::map<ConfigKey, Value> ValueMap; + const static int64_t SNAPSHOT_FORMAT_VERSION; + + ConfigSnapshot(const ValueMap & valueMap, int64_t generation); + void serializeV1(vespalib::slime::Cursor & root) const; + void serializeKeyV1(vespalib::slime::Cursor & root, const ConfigKey & key) const; + void serializeValueV1(vespalib::slime::Cursor & root, const Value & value) const; + + void deserializeV1(vespalib::slime::Inspector & root); + ConfigKey deserializeKeyV1(vespalib::slime::Inspector & inspector) const; + Value deserializeValueV1(vespalib::slime::Inspector & inspector) const; + + void serializeV2(vespalib::slime::Cursor & root) const; + void serializeValueV2(vespalib::slime::Cursor & root, const Value & value) const; + + void deserializeV2(vespalib::slime::Inspector & root); + Value deserializeValueV2(vespalib::slime::Inspector & inspector) const; + + ValueMap _valueMap; + int64_t _generation; +}; + +} // namespace config + +#include "configsnapshot.hpp" + diff --git a/config/src/vespa/config/retriever/configsnapshot.hpp b/config/src/vespa/config/retriever/configsnapshot.hpp new file mode 100644 index 00000000000..b8e14fbceca --- /dev/null +++ b/config/src/vespa/config/retriever/configsnapshot.hpp @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace config { + +template <typename ConfigType> +std::unique_ptr<ConfigType> +ConfigSnapshot::getConfig(const vespalib::string & configId) const +{ + ConfigKey key(ConfigKey::create<ConfigType>(configId)); + ValueMap::const_iterator it(_valueMap.find(key)); + if (it == _valueMap.end()) { + throw IllegalConfigKeyException("Unable to find config for key " + key.toString()); + } + return it->second.second.newInstance<ConfigType>(); +} + +template <typename ConfigType> +bool +ConfigSnapshot::isChanged(const vespalib::string & configId, int64_t currentGeneration) const +{ + ConfigKey key(ConfigKey::create<ConfigType>(configId)); + ValueMap::const_iterator it(_valueMap.find(key)); + if (it == _valueMap.end()) { + throw IllegalConfigKeyException("Unable to find config for key " + key.toString()); + } + return currentGeneration < it->second.first; +} + +template <typename ConfigType> +bool +ConfigSnapshot::hasConfig(const vespalib::string & configId) const +{ + ConfigKey key(ConfigKey::create<ConfigType>(configId)); + return (_valueMap.find(key) != _valueMap.end()); +} + +} diff --git a/config/src/vespa/config/retriever/fixedconfigsubscriber.cpp b/config/src/vespa/config/retriever/fixedconfigsubscriber.cpp new file mode 100644 index 00000000000..7fb1ec9b337 --- /dev/null +++ b/config/src/vespa/config/retriever/fixedconfigsubscriber.cpp @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "fixedconfigsubscriber.h" + +namespace config { +FixedConfigSubscriber::FixedConfigSubscriber(const ConfigKeySet & keySet, + const IConfigContext::SP & context, + int64_t subscribeTimeout) + : _set(context), + _subscriptionList() +{ + for (ConfigKeySet::const_iterator it(keySet.begin()), mt(keySet.end()); it != mt; it++) { + _subscriptionList.push_back(_set.subscribe(*it, subscribeTimeout)); + } +} + +bool +FixedConfigSubscriber::nextGeneration(int timeoutInMillis) +{ + return _set.acquireSnapshot(timeoutInMillis, true); +} + +void +FixedConfigSubscriber::close() +{ + _set.close(); +} + +int64_t +FixedConfigSubscriber::getGeneration() const +{ + return _set.getGeneration(); +} + +ConfigSnapshot +FixedConfigSubscriber::getConfigSnapshot() const +{ + return ConfigSnapshot(_subscriptionList, _set.getGeneration()); +} + +} diff --git a/config/src/vespa/config/retriever/fixedconfigsubscriber.h b/config/src/vespa/config/retriever/fixedconfigsubscriber.h new file mode 100644 index 00000000000..da570f5cf77 --- /dev/null +++ b/config/src/vespa/config/retriever/fixedconfigsubscriber.h @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/subscription/configsubscriptionset.h> +#include "configkeyset.h" +#include "configsnapshot.h" + +namespace config { + +/** + * The FixedConfigSubscriber takes an entires set of keys and subscribes to + * all of them. Once this is done, it cannot be resubscribed. + */ +class FixedConfigSubscriber +{ +public: + FixedConfigSubscriber(const ConfigKeySet & keySet, const IConfigContext::SP & context, int64_t subscribeTimeout); + bool nextGeneration(int timeoutInMillis); + void close(); + int64_t getGeneration() const; + ConfigSnapshot getConfigSnapshot() const; +private: + ConfigSubscriptionSet _set; + std::vector<ConfigSubscription::SP> _subscriptionList; +}; + +} // namespace config + diff --git a/config/src/vespa/config/retriever/genericconfigsubscriber.cpp b/config/src/vespa/config/retriever/genericconfigsubscriber.cpp new file mode 100644 index 00000000000..c2a7e05d538 --- /dev/null +++ b/config/src/vespa/config/retriever/genericconfigsubscriber.cpp @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "genericconfigsubscriber.h" + +namespace config { + +GenericConfigSubscriber::GenericConfigSubscriber(const IConfigContext::SP & context) + : _set(context) +{ } + +bool +GenericConfigSubscriber::nextGeneration(int timeoutInMillis) +{ + return _set.acquireSnapshot(timeoutInMillis, true); +} + +ConfigSubscription::SP +GenericConfigSubscriber::subscribe(const ConfigKey & key, int timeoutInMillis) +{ + return _set.subscribe(key, timeoutInMillis); +} + +void +GenericConfigSubscriber::close() +{ + _set.close(); +} + +int64_t +GenericConfigSubscriber::getGeneration() const +{ + return _set.getGeneration(); +} + +} diff --git a/config/src/vespa/config/retriever/genericconfigsubscriber.h b/config/src/vespa/config/retriever/genericconfigsubscriber.h new file mode 100644 index 00000000000..087e50a70e8 --- /dev/null +++ b/config/src/vespa/config/retriever/genericconfigsubscriber.h @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/subscription/configsubscriptionset.h> + +namespace config { + +/** + * The GenericConfigSubscriber is a generic form of a config subscriber, which + * does not require any type to be known. It also only supports generation + * changes. + */ +class GenericConfigSubscriber +{ +public: + GenericConfigSubscriber(const IConfigContext::SP & context); + bool nextGeneration(int timeoutInMillis); + ConfigSubscription::SP subscribe(const ConfigKey & key, int timeoutInMillis); + void close(); + int64_t getGeneration() const; +private: + ConfigSubscriptionSet _set; +}; + +} // namespace config + diff --git a/config/src/vespa/config/retriever/simpleconfigretriever.cpp b/config/src/vespa/config/retriever/simpleconfigretriever.cpp new file mode 100644 index 00000000000..011a9f6a519 --- /dev/null +++ b/config/src/vespa/config/retriever/simpleconfigretriever.cpp @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "simpleconfigretriever.h" + +namespace config { +SimpleConfigRetriever::SimpleConfigRetriever(const ConfigKeySet & keySet, + const IConfigContext::SP & context, + uint64_t subscribeTimeout) + : _set(context), + _subscriptionList() +{ + for (ConfigKeySet::const_iterator it(keySet.begin()), mt(keySet.end()); it != mt; it++) { + _subscriptionList.push_back(_set.subscribe(*it, subscribeTimeout)); + } +} + +ConfigSnapshot +SimpleConfigRetriever::getConfigs(uint64_t timeoutInMillis) +{ + if (_set.acquireSnapshot(timeoutInMillis, true)) { + return ConfigSnapshot(_subscriptionList, _set.getGeneration()); + } + return ConfigSnapshot(); +} + +void +SimpleConfigRetriever::close() +{ + _set.close(); +} + +bool +SimpleConfigRetriever::isClosed() const +{ + return _set.isClosed(); +} + +} diff --git a/config/src/vespa/config/retriever/simpleconfigretriever.h b/config/src/vespa/config/retriever/simpleconfigretriever.h new file mode 100644 index 00000000000..fc3f14bbeaa --- /dev/null +++ b/config/src/vespa/config/retriever/simpleconfigretriever.h @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/subscription/configsubscriptionset.h> +#include <vespa/config/common/timingvalues.h> +#include "configkeyset.h" +#include "configsnapshot.h" + +namespace config { + +/** + * The SimpleConfigRetriever takes an entires set of keys and subscribes to + * all of them. Once this is done, it cannot be resubscribed. You can poll this + * for new snapshots. + */ +class SimpleConfigRetriever +{ +public: + typedef std::unique_ptr<SimpleConfigRetriever> UP; + + SimpleConfigRetriever(const ConfigKeySet & keySet, + const IConfigContext::SP & context, + uint64_t subscribeTimeout = DEFAULT_SUBSCRIBE_TIMEOUT); + + /** + * Attempt retrieving a snapshot of configs. + * @param timeoutInMillis The amount of time to wait for a new snapshot. + * @return A new snapshot. The snapshot is empty if timeout was reached or + * if the retriever was closed. + */ + ConfigSnapshot getConfigs(uint64_t timeoutInMillis = DEFAULT_GETCONFIGS_TIMEOUT); + void close(); + bool isClosed() const; + +private: + ConfigSubscriptionSet _set; + std::vector<ConfigSubscription::SP> _subscriptionList; +}; + +} // namespace config + diff --git a/config/src/vespa/config/retriever/simpleconfigurer.cpp b/config/src/vespa/config/retriever/simpleconfigurer.cpp new file mode 100644 index 00000000000..1b0e9cf9efe --- /dev/null +++ b/config/src/vespa/config/retriever/simpleconfigurer.cpp @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/log/log.h> +LOG_SETUP(".config.retriever.simpleconfigurer"); +#include "simpleconfigurer.h" + +namespace config { + +SimpleConfigurer::SimpleConfigurer(SimpleConfigRetriever::UP retriever, SimpleConfigurable * const configurable) + : _retriever(std::move(retriever)), + _configurable(configurable), + _thread(*this), + _started(false) +{ + assert(_retriever.get() != NULL); +} + +void +SimpleConfigurer::start() +{ + if (!_retriever->isClosed()) { + LOG(debug, "Polling for config"); + runConfigure(); + _thread.start(); + _started = true; + } +} + +SimpleConfigurer::~SimpleConfigurer() +{ + close(); +} + +void +SimpleConfigurer::close() +{ + _retriever->close(); + if (_started) + _thread.join(); +} + +void +SimpleConfigurer::runConfigure() +{ + ConfigSnapshot snapshot(_retriever->getConfigs()); + if (!snapshot.empty()) { + _configurable->configure(snapshot); + } +} + +void +SimpleConfigurer::run() +{ + while (!_retriever->isClosed()) { + try { + runConfigure(); + } catch (const std::exception & e) { + LOG(fatal, "Fatal error while configuring: %s", e.what()); + } + } +} + +} // namespace config diff --git a/config/src/vespa/config/retriever/simpleconfigurer.h b/config/src/vespa/config/retriever/simpleconfigurer.h new file mode 100644 index 00000000000..69d4eea545d --- /dev/null +++ b/config/src/vespa/config/retriever/simpleconfigurer.h @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "simpleconfigretriever.h" +#include "configsnapshot.h" + +#include <vespa/vespalib/util/thread.h> +#include <vespa/vespalib/util/runnable.h> + +#include <atomic> + +namespace config { + +class SimpleConfigurable +{ +public: + virtual ~SimpleConfigurable() { } + virtual void configure(const ConfigSnapshot & snapshot) = 0; +}; + +/** + * A SimpleConfigurer runs in its own thread, uses a SimpleConfigRetriever to retrieve configs, and + * performs a callback whenever a newsnapshot is ready. + */ +class SimpleConfigurer : public vespalib::Runnable +{ +public: + SimpleConfigurer(SimpleConfigRetriever::UP retriever, SimpleConfigurable * const configurable); + ~SimpleConfigurer(); + + /** + * Start the configurer thread. configure() is guaranteed to be called + * before this method returns. + */ + void start(); + + /** + * Close the configurer. This will close the retriever as well! + */ + void close(); + + void run(); + +private: + void runConfigure(); + + SimpleConfigRetriever::UP _retriever; + SimpleConfigurable * const _configurable; + vespalib::Thread _thread; + std::atomic<bool> _started; +}; + +} // namespace config + diff --git a/config/src/vespa/config/set/.gitignore b/config/src/vespa/config/set/.gitignore new file mode 100644 index 00000000000..7e7c0fe7fae --- /dev/null +++ b/config/src/vespa/config/set/.gitignore @@ -0,0 +1,2 @@ +/.depend +/Makefile diff --git a/config/src/vespa/config/set/CMakeLists.txt b/config/src/vespa/config/set/CMakeLists.txt new file mode 100644 index 00000000000..8f94598e577 --- /dev/null +++ b/config/src/vespa/config/set/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(config_set OBJECT + SOURCES + configsetsource.cpp + configsetsourcefactory.cpp + configinstancesourcefactory.cpp + DEPENDS +) diff --git a/config/src/vespa/config/set/configinstancesourcefactory.cpp b/config/src/vespa/config/set/configinstancesourcefactory.cpp new file mode 100644 index 00000000000..fcf1b0f213b --- /dev/null +++ b/config/src/vespa/config/set/configinstancesourcefactory.cpp @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "configinstancesourcefactory.h" +#include <vespa/config/common/source.h> +#include <vespa/config/common/misc.h> + +namespace { + +class ConfigInstanceSource : public config::Source { +public: + ConfigInstanceSource(const config::IConfigHolder::SP & holder, const vespalib::asciistream & buffer) + : _holder(holder), + _buffer(buffer), + _generation(-1) + { } + virtual void close() { } + virtual void getConfig() { + std::vector<vespalib::string> lines(_buffer.getlines()); + std::string currentMd5(config::calculateContentMd5(lines)); + _holder->handle(config::ConfigUpdate::UP(new config::ConfigUpdate(config::ConfigValue(lines, currentMd5), true, _generation))); + + } + virtual void reload(int64_t generation) { _generation = generation; } +private: + config::IConfigHolder::SP _holder; + vespalib::asciistream _buffer; + int64_t _generation; +}; + +} + +namespace config { + +ConfigInstanceSourceFactory::ConfigInstanceSourceFactory(const ConfigKey & key, const vespalib::asciistream & buffer) + : _key(key), + _buffer(buffer) +{ +} + +Source::UP +ConfigInstanceSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const +{ + (void) key; + // TODO: Check key against _key + return Source::UP(new ConfigInstanceSource(holder, _buffer)); +} + +} // namespace config + diff --git a/config/src/vespa/config/set/configinstancesourcefactory.h b/config/src/vespa/config/set/configinstancesourcefactory.h new file mode 100644 index 00000000000..8b1a4c68cbd --- /dev/null +++ b/config/src/vespa/config/set/configinstancesourcefactory.h @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/sourcefactory.h> +#include "configsetsource.h" +#include <vespa/vespalib/stllike/asciistream.h> + +namespace config { + +class Source; +class IConfigHolder; +class ConfigKey; + +/** + * Factory creating config payload from a single config instance + */ +class ConfigInstanceSourceFactory : public SourceFactory +{ +public: + ConfigInstanceSourceFactory(const ConfigKey & key, const vespalib::asciistream & buffer); + + /** + * Create source handling config described by key. + */ + Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const; +private: + const ConfigKey _key; + vespalib::asciistream _buffer; +}; + +} // namespace config + diff --git a/config/src/vespa/config/set/configsetsource.cpp b/config/src/vespa/config/set/configsetsource.cpp new file mode 100644 index 00000000000..d98b28b8b6a --- /dev/null +++ b/config/src/vespa/config/set/configsetsource.cpp @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.set.configsetsource"); +#include "configsetsource.h" +#include <vespa/config/print/asciiconfigwriter.h> +#include <vespa/config/common/misc.h> +#include <vespa/vespalib/stllike/asciistream.h> + +namespace config { + +ConfigSetSource::ConfigSetSource(const IConfigHolder::SP & holder, const ConfigKey & key, const BuilderMapSP & builderMap) + : _holder(holder), + _key(key), + _generation(1), + _builderMap(builderMap) +{ + if (!validRequest(key)) + throw ConfigRuntimeException("Invalid subscribe for key " + key.toString() + ", not builder found"); +} + +void +ConfigSetSource::getConfig() +{ + BuilderMap::const_iterator it(_builderMap->find(_key)); + ConfigInstance * instance = it->second; + vespalib::asciistream ss; + AsciiConfigWriter writer(ss); + writer.write(*instance); + std::vector<vespalib::string> lines(ss.getlines()); + std::string currentMd5(calculateContentMd5(lines)); + + if (isGenerationNewer(_generation, _lastState.generation) && currentMd5.compare(_lastState.md5) != 0) { + LOG(debug, "New generation, updating"); + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, currentMd5), true, _generation))); + _lastState.md5 = currentMd5; + _lastState.generation = _generation; + } else { + LOG(debug, "Sending timestamp update"); + _holder->handle(ConfigUpdate::UP(new ConfigUpdate(ConfigValue(lines, currentMd5), false, _generation))); + _lastState.generation = _generation; + } +} + +void +ConfigSetSource::reload(int64_t generation) +{ + LOG(debug, "Running update with generation(%" PRId64 ")", generation); + _generation = generation; +} + +void +ConfigSetSource::close() +{ +} + +bool +ConfigSetSource::validRequest(const ConfigKey & key) +{ + if (_builderMap->find(key) == _builderMap->end()) + return false; + BuilderMap::const_iterator it(_builderMap->find(key)); + ConfigInstance * instance = it->second; + return (key.getDefName().compare(instance->defName()) == 0 && + key.getDefNamespace().compare(instance->defNamespace()) == 0); +} + +} // namespace config diff --git a/config/src/vespa/config/set/configsetsource.h b/config/src/vespa/config/set/configsetsource.h new file mode 100644 index 00000000000..78666ddb533 --- /dev/null +++ b/config/src/vespa/config/set/configsetsource.h @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/source.h> +#include <vespa/config/common/configkey.h> +#include <vespa/config/common/iconfigholder.h> +#include <vespa/config/common/configstate.h> +#include <map> + +namespace config { + +class ConfigInstance; + +/** + * Class for sending and receiving config request from a raw string. + */ +class ConfigSetSource : public Source { +public: + typedef std::map<ConfigKey, ConfigInstance *> BuilderMap; + typedef std::shared_ptr<BuilderMap> BuilderMapSP; + ConfigSetSource(const IConfigHolder::SP & holder, const ConfigKey & key, const BuilderMapSP & builderMap); + + void getConfig(); + void reload(int64_t generation); + void close(); +private: + IConfigHolder::SP _holder; + const ConfigKey _key; + int64_t _generation; + BuilderMapSP _builderMap; + int64_t _lastGeneration; + ConfigState _lastState; + + bool validRequest(const ConfigKey & key); +}; + +} + diff --git a/config/src/vespa/config/set/configsetsourcefactory.cpp b/config/src/vespa/config/set/configsetsourcefactory.cpp new file mode 100644 index 00000000000..778ae9ee3cd --- /dev/null +++ b/config/src/vespa/config/set/configsetsourcefactory.cpp @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "configsetsourcefactory.h" + +namespace config { + +ConfigSetSourceFactory::ConfigSetSourceFactory(const BuilderMapSP & builderMap) + : _builderMap(builderMap) +{ +} + +Source::UP +ConfigSetSourceFactory::createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const +{ + return Source::UP(new ConfigSetSource(holder, key, _builderMap)); +} + +} // namespace config + diff --git a/config/src/vespa/config/set/configsetsourcefactory.h b/config/src/vespa/config/set/configsetsourcefactory.h new file mode 100644 index 00000000000..e3bac385e07 --- /dev/null +++ b/config/src/vespa/config/set/configsetsourcefactory.h @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/sourcefactory.h> +#include "configsetsource.h" + +namespace config { + +class Source; +class IConfigHolder; +class ConfigKey; + +/** + * Factory creating config payload from config instances. + */ +class ConfigSetSourceFactory : public SourceFactory +{ +public: + typedef ConfigSetSource::BuilderMap BuilderMap; + typedef ConfigSetSource::BuilderMapSP BuilderMapSP; + ConfigSetSourceFactory(const BuilderMapSP & builderMap); + + /** + * Create source handling config described by key. + */ + Source::UP createSource(const IConfigHolder::SP & holder, const ConfigKey & key) const; +private: + BuilderMapSP _builderMap; +}; + +} // namespace config + diff --git a/config/src/vespa/config/subscription/.gitignore b/config/src/vespa/config/subscription/.gitignore new file mode 100644 index 00000000000..7e7c0fe7fae --- /dev/null +++ b/config/src/vespa/config/subscription/.gitignore @@ -0,0 +1,2 @@ +/.depend +/Makefile diff --git a/config/src/vespa/config/subscription/CMakeLists.txt b/config/src/vespa/config/subscription/CMakeLists.txt new file mode 100644 index 00000000000..cce728c878a --- /dev/null +++ b/config/src/vespa/config/subscription/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(config_subscription OBJECT + SOURCES + sourcespec.cpp + configsubscription.cpp + configsubscriber.cpp + configsubscriptionset.cpp + configuri.cpp + DEPENDS +) diff --git a/config/src/vespa/config/subscription/confighandle.h b/config/src/vespa/config/subscription/confighandle.h new file mode 100644 index 00000000000..fddb1f5ccb6 --- /dev/null +++ b/config/src/vespa/config/subscription/confighandle.h @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include <vespa/config/subscription/configsubscription.h> + +namespace config { + +/** + * A ConfigHandle is a subscription handle that is capable of looking up config + * objects of a generic type. + */ +template <typename ConfigType> +class ConfigHandle +{ +public: + typedef std::unique_ptr<ConfigHandle <ConfigType> > UP; + + ConfigHandle(const ConfigSubscription::SP & subscription); + + /** + * Return the currently available config known to the ConfigHandle. Throws + * a ConfigRuntimeException if the ConfigSubscriber has not yet been polled + * for config, and InvalidConfigException, if there are errors with the + * config payload. + * + * @return current config. + * @throws InvalidConfigException if unable instantiate the given type or + * parse config. + */ + std::unique_ptr<ConfigType> getConfig() const; + + /** + * Returns whether or not this handles config has changed since the last + * call to ConfigSubscriber.nextConfig() was made. + * + * @return true if changed, false if not. + */ + bool isChanged() const; +private: + ConfigSubscription::SP _subscription; +}; + +} // namespace config + +#include "confighandle.hpp" + diff --git a/config/src/vespa/config/subscription/confighandle.hpp b/config/src/vespa/config/subscription/confighandle.hpp new file mode 100644 index 00000000000..86afc8d4f43 --- /dev/null +++ b/config/src/vespa/config/subscription/confighandle.hpp @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +namespace config { + +template <typename ConfigType> +ConfigHandle<ConfigType>::ConfigHandle(const ConfigSubscription::SP & subscription) + : _subscription(subscription) +{ +} + +template <typename ConfigType> +std::unique_ptr<ConfigType> +ConfigHandle<ConfigType>::getConfig() const +{ + return _subscription->getConfig().newInstance<ConfigType>(); +} + +template <typename ConfigType> +bool +ConfigHandle<ConfigType>::isChanged() const +{ + return _subscription->isChanged(); +} + +} // namespace config diff --git a/config/src/vespa/config/subscription/configprovider.h b/config/src/vespa/config/subscription/configprovider.h new file mode 100644 index 00000000000..56c5f685eae --- /dev/null +++ b/config/src/vespa/config/subscription/configprovider.h @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config/common/configvalue.h> + +namespace config { + + +class ConfigProvider +{ +public: + virtual ~ConfigProvider() { } + + /** + * Fetches the appropriate ConfigValue. + * + * @return the current ConfigValue. + */ + virtual ConfigValue getConfig() const = 0; + + /** + * Checks whether or not the config has changed. + * + * @return true if changed, false if not. + */ + virtual bool isChanged() const = 0; +}; + +} // namespace config + diff --git a/config/src/vespa/config/subscription/configsubscriber.cpp b/config/src/vespa/config/subscription/configsubscriber.cpp new file mode 100644 index 00000000000..092c460ece1 --- /dev/null +++ b/config/src/vespa/config/subscription/configsubscriber.cpp @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.subscription.configsubscriber"); + +#include "configsubscriber.h" +#include <vespa/config/common/exceptions.h> + +namespace config { + + +ConfigSubscriber::ConfigSubscriber(const IConfigContext::SP & context) + : _set(context) + +{ } + +ConfigSubscriber::ConfigSubscriber(const SourceSpec & spec) + : _set(IConfigContext::SP(new ConfigContext(spec))) +{ } + +bool +ConfigSubscriber::nextConfig(uint64_t timeoutInMillis) +{ + return _set.acquireSnapshot(timeoutInMillis, false); +} + +bool +ConfigSubscriber::nextGeneration(uint64_t timeoutInMillis) +{ + return _set.acquireSnapshot(timeoutInMillis, true); +} + +void +ConfigSubscriber::close() +{ + _set.close(); +} + +bool +ConfigSubscriber::isClosed() const +{ + return _set.isClosed(); +} + +int64_t +ConfigSubscriber::getGeneration() const +{ + return _set.getGeneration(); +} + +} // namespace config diff --git a/config/src/vespa/config/subscription/configsubscriber.h b/config/src/vespa/config/subscription/configsubscriber.h new file mode 100644 index 00000000000..194aad75276 --- /dev/null +++ b/config/src/vespa/config/subscription/configsubscriber.h @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <memory> +#include <map> +#include <vespa/config/common/iconfigholder.h> +#include <vespa/config/common/configcontext.h> +#include <vespa/config/common/timingvalues.h> +#include "confighandle.h" +#include "subscriptionid.h" +#include "configsubscription.h" +#include "configsubscriptionset.h" +#include "configprovider.h" +#include "sourcespec.h" + +namespace config { + +/** + * A subscriber is a class capable of subscribing to one or more configs. The + * class should be used as follows: + * - subscribe for all configs you need. + * - run nextConfig or nextGeneration to fetch the next generation of configs. + * + * Once nextConfig/nextGeneration is called, the state of a ConfigSubscriber is + * FROZEN, which means that in order to change the set of subscriptions, you + * have to recreate the ConfigSubscriber. + * + * Note that this class is NOT thread safe and that you should design your + * application so that you only need to use it from one thread. + */ +class ConfigSubscriber +{ +public: + typedef std::unique_ptr<ConfigSubscriber> UP; + + /** + * Constructs a new ConfigSubscriber object which can be used to subscribe + * for 1 or more configs. + * + * @param spec The source spec from which to get config. + */ + ConfigSubscriber(const SourceSpec & spec = ServerSpec()); + + /** + * Constructs a new ConfigSubscriber object which can be used to subscribe + * for 1 or more configs. The provided context is used to share resources + * betweeen multiple config subscribers. + * + * @param context A ConfigContext shared between all subscribers. + */ + ConfigSubscriber(const IConfigContext::SP & context); + + /** + * Checks if one or more of the configs in the set is updated or not. + * + * @param timeoutInMillis The timeout in milliseconds. + * @return true if new configs are available, false if timeout was reached + * or subscriber has been closed. + */ + bool nextConfig(uint64_t timeoutInMillis = DEFAULT_NEXTCONFIG_TIMEOUT); + + /** + * Checks if the generation of this config set is updated. + * + * @param timeoutInMillis The timeout in milliseconds. + * @return true if a new generation are available, false if timeout was reached + * or subscriber has been closed. + */ + bool nextGeneration(uint64_t timeoutInMillis = DEFAULT_NEXTCONFIG_TIMEOUT); + + /** + * Subscribe to a config fetched from the default source specification. + * + * @param configId The configId to get config for. + * @param timeoutInMillis An optional timeout on the subscribe call, in + * milliseconds. + * @return A subscription handle which can be used to + * retrieve config. + * @throws ConfigTimeoutException if subscription timed out. + * @throws ConfigRuntimeException if subscriber has been closed. + */ + template <typename ConfigType> + std::unique_ptr<ConfigHandle<ConfigType> > + subscribe(const std::string & configId, uint64_t timeoutInMillis = DEFAULT_SUBSCRIBE_TIMEOUT); + + /** + * Return the current generation number for configs. + * + * @return generation number + */ + int64_t getGeneration() const; + + /** + * Closes the set, which will interrupt nextConfig() or nextGeneration(), and unsubscribe all + * configs currently subscribed for. + */ + void close(); + + /** + * Check if this retriever is closed. + * + * @return true if closed, false if not. + */ + bool isClosed() const; + +private: + ConfigSubscriptionSet _set; // The set of subscriptions for this set. +}; + +} // namespace config + +#include "configsubscriber.hpp" + diff --git a/config/src/vespa/config/subscription/configsubscriber.hpp b/config/src/vespa/config/subscription/configsubscriber.hpp new file mode 100644 index 00000000000..eed67897c9a --- /dev/null +++ b/config/src/vespa/config/subscription/configsubscriber.hpp @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace config { + +template <typename ConfigType> +std::unique_ptr<ConfigHandle<ConfigType> > +ConfigSubscriber::subscribe(const std::string & configId, uint64_t timeoutInMillis) +{ + const ConfigKey key(ConfigKey::create<ConfigType>(configId)); + return std::unique_ptr<ConfigHandle<ConfigType> >(new ConfigHandle<ConfigType>(_set.subscribe(key, timeoutInMillis))); +} + +} diff --git a/config/src/vespa/config/subscription/configsubscription.cpp b/config/src/vespa/config/subscription/configsubscription.cpp new file mode 100644 index 00000000000..c8d4f66091c --- /dev/null +++ b/config/src/vespa/config/subscription/configsubscription.cpp @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/config/common/exceptions.h> +#include <vespa/config/common/misc.h> +#include "configsubscription.h" + +namespace config { + +ConfigSubscription::ConfigSubscription(const SubscriptionId & id, const ConfigKey & key, const IConfigHolder::SP & holder, Source::UP source) + : _id(id), + _key(key), + _source(std::move(source)), + _holder(holder), + _next(), + _current(), + _isChanged(false), + _lastGenerationChanged(-1), + _closed(false) +{ +} + +ConfigSubscription::~ConfigSubscription() +{ + close(); +} + + +bool +ConfigSubscription::nextUpdate(int64_t generation, uint64_t timeoutInMillis) +{ + if (_closed || !_holder->poll()) + return false; + _next.reset(_holder->provide().release()); + if (isGenerationNewer(_next->getGeneration(), generation)) { + return true; + } + return (!_closed && _holder->wait(timeoutInMillis)); +} + +bool +ConfigSubscription::hasChanged() const +{ + return (!_closed && (_next->hasChanged() || _current.get() == NULL)); +} + +int64_t +ConfigSubscription::getGeneration() const +{ + return _next->getGeneration(); +} + +const ConfigKey & +ConfigSubscription::getKey() const +{ + return _key; +} + +void +ConfigSubscription::close() +{ + if (!_closed) { + _closed = true; + _holder->interrupt(); + _source->close(); + } +} + +void +ConfigSubscription::reset() +{ + _isChanged = false; +} + +bool +ConfigSubscription::isChanged() const +{ + return _isChanged; +} + +int64_t +ConfigSubscription::getLastGenerationChanged() const +{ + return _lastGenerationChanged; +} + +void +ConfigSubscription::flip() +{ + bool change = hasChanged(); + if (change) { + _current.reset(_next.release()); + _lastGenerationChanged = _current->getGeneration(); + } else { + _current.reset(new ConfigUpdate(_current->getValue(), false, _next->getGeneration())); + } + _isChanged = change; +} + +ConfigValue +ConfigSubscription::getConfig() const +{ + if (_closed) + throw ConfigRuntimeException("Subscription is closed, config no longer available"); + if (_current.get() == NULL) + throw ConfigRuntimeException("No configuration available"); + return _current->getValue(); +} + +void +ConfigSubscription::reload(int64_t generation) +{ + _source->reload(generation); + _source->getConfig(); +} + +} // namespace config diff --git a/config/src/vespa/config/subscription/configsubscription.h b/config/src/vespa/config/subscription/configsubscription.h new file mode 100644 index 00000000000..cfc33cdbb3e --- /dev/null +++ b/config/src/vespa/config/subscription/configsubscription.h @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <memory> +#include <vespa/config/common/iconfigholder.h> +#include <vespa/config/common/configkey.h> +#include <vespa/config/common/source.h> +#include "subscriptionid.h" + +#include <atomic> + +namespace config { + +/** + * A subscription can be polled for config updates, and handles interruption of + * the nextUpdate call. + */ +class ConfigSubscription +{ +public: + typedef std::unique_ptr<ConfigSubscription> UP; + typedef std::shared_ptr<ConfigSubscription> SP; + + ConfigSubscription(const SubscriptionId & id, const ConfigKey & key, const IConfigHolder::SP & holder, Source::UP source); + ~ConfigSubscription(); + + /** + * Fetches the appropriate ConfigValue. + * + * @return the current ConfigValue. + */ + ConfigValue getConfig() const; + + /** + * Checks whether or not the config has changed. + * + * @return true if changed, false if not. + */ + bool isChanged() const; + + /** + * Returns the last generation that actually changed the config. + */ + int64_t getLastGenerationChanged() const; + + /// Used by ConfigSubscriptionSet + SubscriptionId getSubscriptionId() const { return _id; } + const ConfigKey & getKey() const; + bool nextUpdate(int64_t generation, uint64_t timeoutInMillis); + int64_t getGeneration() const; + bool hasChanged() const; + void flip(); + void reset(); + void close(); + + // Used by ConfigManager + void reload(int64_t generation); + +private: + const SubscriptionId _id; + const ConfigKey _key; + Source::UP _source; + IConfigHolder::SP _holder; + ConfigUpdate::UP _next; + ConfigUpdate::UP _current; + bool _isChanged; + int64_t _lastGenerationChanged; + std::atomic<bool> _closed; +}; + +typedef std::vector<ConfigSubscription::SP> SubscriptionList; + +} // namespace config + diff --git a/config/src/vespa/config/subscription/configsubscriptionset.cpp b/config/src/vespa/config/subscription/configsubscriptionset.cpp new file mode 100644 index 00000000000..b19812a3fc6 --- /dev/null +++ b/config/src/vespa/config/subscription/configsubscriptionset.cpp @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.subscription.configsubscriptionset"); + +#include "configsubscriptionset.h" +#include <vespa/config/common/exceptions.h> + +namespace config { + +ConfigSubscriptionSet::ConfigSubscriptionSet(const IConfigContext::SP & context) + : _context(context), + _mgr(context->getManagerInstance()), + _currentGeneration(-1), + _subscriptionList(), + _state(OPEN) +{ } + +ConfigSubscriptionSet::~ConfigSubscriptionSet() +{ + close(); +} + +bool +ConfigSubscriptionSet::acquireSnapshot(uint64_t timeoutInMillis, bool ignoreChange) +{ + if (_state == CLOSED) { + return false; + } else if (_state == OPEN) + _state = FROZEN; + + FastOS_Time timer; + timer.SetNow(); + int timeLeft = timeoutInMillis; + int64_t lastGeneration = _currentGeneration; + bool inSync = false; + + for (SubscriptionList::iterator it(_subscriptionList.begin()), mt(_subscriptionList.end()); + it != mt; + it++) { + (*it)->reset(); + } + + LOG(debug, "Going into nextConfig loop, time left is %d", timeLeft); + while (_state != CLOSED && timeLeft >= 0 && !inSync) { + size_t numChanged = 0; + size_t numGenerationChanged = 0; + bool generationsInSync = true; + int64_t generation = -1; + + // Run nextUpdate on all subscribers to get them in sync. + for (SubscriptionList::iterator it(_subscriptionList.begin()), mt(_subscriptionList.end()); + it != mt; + it++) { + ConfigSubscription::SP subscription = *it; + + if (!subscription->nextUpdate(_currentGeneration, timeLeft)) + break; + + const ConfigKey & key(subscription->getKey()); + if (subscription->hasChanged()) { + LOG(spam, "Config subscription has changed id(%s), defname(%s)", key.getConfigId().c_str(), key.getDefName().c_str()); + numChanged++; + } else { + LOG(spam, "Config subscription did not change, id(%s), defname(%s)", key.getConfigId().c_str(), key.getDefName().c_str()); + } + LOG(spam, "Previous generation is %" PRId64 ", updates is %" PRId64, generation, subscription->getGeneration()); + if (isGenerationNewer(subscription->getGeneration(), _currentGeneration)) { + numGenerationChanged++; + } + if (generation < 0) + generation = subscription->getGeneration(); + if (subscription->getGeneration() != generation) + generationsInSync = false; + // Adjust timeout + timeLeft = timeoutInMillis - static_cast<uint64_t>(timer.MilliSecsToNow()); + } + inSync = generationsInSync && (_subscriptionList.size() == numGenerationChanged) && (ignoreChange || numChanged > 0); + lastGeneration = generation; + timeLeft = timeoutInMillis - static_cast<uint64_t>(timer.MilliSecsToNow()); + if (!inSync && timeLeft > 0) { + FastOS_Thread::Sleep(10); + } + } + + bool updated = inSync && isGenerationNewer(lastGeneration, _currentGeneration); + if (updated) { + LOG(spam, "Config was updated from %" PRId64 " to %" PRId64, _currentGeneration, lastGeneration); + _currentGeneration = lastGeneration; + _state = CONFIGURED; + for (SubscriptionList::iterator it(_subscriptionList.begin()), mt(_subscriptionList.end()); + it != mt; + it++) { + (*it)->flip(); + } + } + return updated; +} + +void +ConfigSubscriptionSet::close() +{ + _state = CLOSED; + for (SubscriptionList::iterator it(_subscriptionList.begin()), mt(_subscriptionList.end()); it != mt; it++) { + _mgr.unsubscribe(*it); + (*it)->close(); + } +} + +bool +ConfigSubscriptionSet::isClosed() const +{ + return (_state == CLOSED); +} + +ConfigSubscription::SP +ConfigSubscriptionSet::subscribe(const ConfigKey & key, uint64_t timeoutInMillis) +{ + if (_state != OPEN) { + throw ConfigRuntimeException("Adding subscription after calling nextConfig() is not allowed"); + } + LOG(debug, "Subscribing with config Id(%s), defName(%s)", key.getConfigId().c_str(), key.getDefName().c_str()); + + ConfigSubscription::SP s = _mgr.subscribe(key, timeoutInMillis); + _subscriptionList.push_back(s); + return s; +} + +int64_t +ConfigSubscriptionSet::getGeneration() const +{ + return _currentGeneration; +} + +} // namespace config diff --git a/config/src/vespa/config/subscription/configsubscriptionset.h b/config/src/vespa/config/subscription/configsubscriptionset.h new file mode 100644 index 00000000000..3c425baf26c --- /dev/null +++ b/config/src/vespa/config/subscription/configsubscriptionset.h @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <memory> +#include <map> +#include <vespa/config/common/iconfigholder.h> +#include <vespa/config/common/configcontext.h> +#include "confighandle.h" +#include "subscriptionid.h" +#include "configsubscription.h" +#include "configprovider.h" +#include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/util/priority_queue.h> + +namespace config { + +/** + * A ConfigSubscriptionSet is a set of configs that can be subscribed to. + */ +class ConfigSubscriptionSet +{ +public: + /** + * Constructs a new ConfigSubscriptionSet object which can be used to subscribe for 1 + * or more configs from a specific source. + * + * @param context A ConfigContext shared between all subscriptions. + */ + ConfigSubscriptionSet(const IConfigContext::SP & context); + + ~ConfigSubscriptionSet(); + + /** + * Return the current generation number for configs. + * + * @return generation number + */ + int64_t getGeneration() const; + + /** + * Closes the set, which will interrupt acquireSnapshot and unsubscribe all + * configs currently subscribed for. + */ + void close(); + + /** + * Checks if this subscription set is closed. + */ + bool isClosed() const; + + // Helpers for doing the subscription + ConfigSubscription::SP subscribe(const ConfigKey & key, uint64_t timeoutInMillis); + + // Tries to acquire a new snapshot of config within the timeout + bool acquireSnapshot(uint64_t timeoutInMillis, bool requireDifference); + +private: + // Describes the state of the subscriber. + enum SubscriberState { OPEN, FROZEN, CONFIGURED, CLOSED }; + + IConfigContext::SP _context; // Context to keep alive managers. + IConfigManager & _mgr; // The config manager that we use. + int64_t _currentGeneration; // Holds the current config generation. + SubscriptionList _subscriptionList; // List of current subscriptions. + + SubscriberState _state; // Current state of this subscriber. +}; + +} // namespace config + diff --git a/config/src/vespa/config/subscription/configuri.cpp b/config/src/vespa/config/subscription/configuri.cpp new file mode 100644 index 00000000000..68f883b70b5 --- /dev/null +++ b/config/src/vespa/config/subscription/configuri.cpp @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "configuri.h" +#include <vespa/config/helper/legacy.h> +#include <vespa/config/subscription/sourcespec.h> + +namespace { +bool checkEmpty(const vespalib::string & configId) { + return configId.empty(); +} +} +namespace config { + +ConfigUri::ConfigUri(const vespalib::string &configId) + : _configId(legacyConfigId2ConfigId(configId)), + _context(new ConfigContext(*legacyConfigId2Spec(configId))), + _empty(checkEmpty(configId)) +{ +} + +ConfigUri::ConfigUri(const vespalib::string &configId, const IConfigContext::SP & context) + : _configId(configId), + _context(context), + _empty(false) +{ +} + +ConfigUri +ConfigUri::createWithNewId(const vespalib::string & configId) const +{ + return ConfigUri(configId, _context); +} + +const vespalib::string & ConfigUri::getConfigId() const { return _configId; } +const IConfigContext::SP & ConfigUri::getContext() const { return _context; } + +ConfigUri +ConfigUri::createFromInstance(const ConfigInstance & instance) +{ + return ConfigUri("", IConfigContext::SP(new ConfigContext(ConfigInstanceSpec(instance)))); +} + +ConfigUri +ConfigUri::createEmpty() +{ + ConfigUri uri("", IConfigContext::SP(new ConfigContext(RawSpec("")))); + uri._empty = true; + return uri; +} + +ConfigUri ConfigUri::createFromSpec(const vespalib::string& configId, const SourceSpec& spec) +{ + return ConfigUri(configId, IConfigContext::SP(new ConfigContext(spec))); +} + + +} // namespace config diff --git a/config/src/vespa/config/subscription/configuri.h b/config/src/vespa/config/subscription/configuri.h new file mode 100644 index 00000000000..c88f092cf66 --- /dev/null +++ b/config/src/vespa/config/subscription/configuri.h @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <string> +#include <vespa/config/common/configcontext.h> + +namespace config { + +/** + * A ConfigUri is a single representation of a configId and its source. The + * purpose of this class is to make it more convenient to deal with config + * subscription for single-config components. The uri can be constructed from a + * config id, or a config id combined with a context, or by using the static + * factory methods to create an uri from a single instance. + */ +class ConfigUri { +public: + /** + * Construct a config URI from a given config id. + * @param configId The config id. + */ + ConfigUri(const char * configId) : ConfigUri(vespalib::string(configId)) {} + + /** + * Construct a config URI from a given config id. + * @param configId The config id. + */ + ConfigUri(const std::string &configId) : ConfigUri(vespalib::string(configId)) {} + + /** + * Construct a config URI from a given config id. + * @param configId The config id. + */ + ConfigUri(const vespalib::stringref &configId) : ConfigUri(vespalib::string(configId)) {} + + /** + * Construct a config URI from a given config id. + * @param configId The config id. + */ + ConfigUri(const vespalib::string &configId); + + /** + * Construct a config URI from a config id and a context. + * @param configId The config id. + * @param context A context object that can be shared with multiple URIs. + */ + ConfigUri(const vespalib::string &configId, const IConfigContext::SP & context); + + /** + * Create a new config Uri with a different config id, but with the same + * context as this URI. + * @param configId The config id to give the new URI. + * @return A new config URI. + */ + ConfigUri createWithNewId(const vespalib::string & configId) const; + + /** + * Create a config uri from a config instance. The instance does not need + * to be kept alive. + * @param instance The config instance to use as source. + * @return A config uri. + */ + static ConfigUri createFromInstance(const ConfigInstance & instance); + + /** + * Create uri from a config id and a source spec. + * + * @param configId The config id to subscribe to. + * @param spec The source spec pointing to the config source. + */ + static ConfigUri createFromSpec(const vespalib::string & configId, + const SourceSpec & spec); + + /** + * Create a new empty config uri as initialization convenience. + */ + static ConfigUri createEmpty(); + + /** + * Get this URIs config id. Used by subscriber. + * @return The config id of this uri. + */ + const vespalib::string & getConfigId() const; + + /** + * Get the context for this uri. Used by subscriber. + * @return The context. + */ + const IConfigContext::SP & getContext() const; + + /** + * Empty if the original id was empty or created with createEmpty + * @return true if empty. + */ + bool empty() const { return _empty; } + +private: + vespalib::string _configId; + IConfigContext::SP _context; + bool _empty; +}; + +} // namespace config + diff --git a/config/src/vespa/config/subscription/sourcespec.cpp b/config/src/vespa/config/subscription/sourcespec.cpp new file mode 100644 index 00000000000..0b5951ddec7 --- /dev/null +++ b/config/src/vespa/config/subscription/sourcespec.cpp @@ -0,0 +1,166 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".config.subscription.sourcespec"); +#include "sourcespec.h" +#include <vespa/config/common/exceptions.h> +#include <vespa/config/raw/rawsourcefactory.h> +#include <vespa/config/file/filesourcefactory.h> +#include <vespa/config/frt/frtsourcefactory.h> +#include <vespa/config/frt/frtconnectionpool.h> +#include <vespa/config/frt/protocol.h> +#include <vespa/config/frt/connectionfactory.h> +#include <vespa/config/set/configsetsourcefactory.h> +#include <vespa/config/set/configinstancesourcefactory.h> +#include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/config/print/asciiconfigwriter.h> +#include <vespa/vespalib/stllike/asciistream.h> + +namespace config { + +RawSpec::RawSpec(const vespalib::string & config) + : _config(config) +{ +} + +SourceFactory::UP +RawSpec::createSourceFactory(const TimingValues & timingValues) const +{ + (void) timingValues; + return SourceFactory::UP(new RawSourceFactory(_config)); +} + +FileSpec::FileSpec(const vespalib::string & fileName) + : _fileName(fileName) +{ + verifyName(_fileName); +} + +void +FileSpec::verifyName(const vespalib::string & fileName) +{ + if (fileName.length() > 4) { + std::string ending(fileName.substr(fileName.length() - 4, 4)); + if (ending.compare(".cfg") != 0) + throw InvalidConfigSourceException("File name '" + fileName + "' is invalid, must end with .cfg"); + } else { + throw InvalidConfigSourceException("File name '" + fileName + "' is invalid"); + } +} + +SourceFactory::UP +FileSpec::createSourceFactory(const TimingValues & timingValues) const +{ + (void) timingValues; + return SourceFactory::UP(new FileSourceFactory(*this)); +} + +DirSpec::DirSpec(const vespalib::string & dirName) + : _dirName(dirName) +{ +} + +SourceFactory::UP +DirSpec::createSourceFactory(const TimingValues & timingValues) const +{ + (void) timingValues; + return SourceFactory::UP(new DirSourceFactory(*this)); +} + +ServerSpec::ServerSpec() + : _hostList(), + _protocolVersion(protocol::readProtocolVersion()), + _traceLevel(protocol::readTraceLevel()), + _compressionType(protocol::readProtocolCompressionType()) +{ + char* cfgSourcesPtr = getenv("VESPA_CONFIG_SOURCES"); + if (cfgSourcesPtr != NULL) { + vespalib::string cfgSourcesStr(cfgSourcesPtr); + initialize(cfgSourcesStr); + } else { + initialize("localhost"); + } +} + +void +ServerSpec::initialize(const vespalib::string & hostSpec) +{ + typedef vespalib::StringTokenizer tokenizer; + tokenizer tok(hostSpec, ","); + for (tokenizer::Iterator it = tok.begin(); it != tok.end(); it++) { + std::string srcHost = *it; + vespalib::asciistream spec; + if (srcHost.find("tcp/") == std::string::npos) { + spec << "tcp/"; + } + spec << srcHost; + if (srcHost.find(":") == std::string::npos) { + spec << ":" << DEFAULT_PROXY_PORT; + } + _hostList.push_back(spec.str()); + } +} + +ServerSpec::ServerSpec(const HostSpecList & hostList) + : _hostList(hostList), + _protocolVersion(protocol::readProtocolVersion()), + _traceLevel(protocol::readTraceLevel()), + _compressionType(protocol::readProtocolCompressionType()) +{ +} + +ServerSpec::ServerSpec(const vespalib::string & hostSpec) + : _hostList(), + _protocolVersion(protocol::readProtocolVersion()), + _traceLevel(protocol::readTraceLevel()), + _compressionType(protocol::readProtocolCompressionType()) +{ + initialize(hostSpec); +} + +SourceFactory::UP +ServerSpec::createSourceFactory(const TimingValues & timingValues) const +{ + const auto vespaVersion = VespaVersion::getCurrentVersion(); + return SourceFactory::UP(new FRTSourceFactory(ConnectionFactory::UP(new FRTConnectionPool(*this, timingValues)), timingValues, _protocolVersion, _traceLevel, vespaVersion, _compressionType)); +} + + +ConfigSet::ConfigSet() + : _builderMap(new BuilderMap()) +{ +} + +SourceFactory::UP +ConfigSet::createSourceFactory(const TimingValues & timingValues) const +{ + (void) timingValues; + return SourceFactory::UP(new ConfigSetSourceFactory(_builderMap)); +} + +void +ConfigSet::addBuilder(const vespalib::string & configId, ConfigInstance * builder) +{ + assert(builder != NULL); + BuilderMap & builderMap(*_builderMap); + const ConfigKey key(configId, builder->defName(), builder->defNamespace(), builder->defMd5()); + builderMap[key] = builder; +} + +ConfigInstanceSpec::ConfigInstanceSpec(const ConfigInstance& instance) + : _key("", instance.defName(), instance.defNamespace(), instance.defMd5()), + _buffer() +{ + AsciiConfigWriter writer(_buffer); + writer.write(instance); +} + +SourceFactory::UP +ConfigInstanceSpec::createSourceFactory(const TimingValues& timingValues) const +{ + (void) timingValues; + return SourceFactory::UP(new ConfigInstanceSourceFactory(_key, _buffer)); +} + + +} diff --git a/config/src/vespa/config/subscription/sourcespec.h b/config/src/vespa/config/subscription/sourcespec.h new file mode 100644 index 00000000000..2378ee0bb06 --- /dev/null +++ b/config/src/vespa/config/subscription/sourcespec.h @@ -0,0 +1,251 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/hash_fun.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/config/common/sourcefactory.h> +#include <vespa/config/common/timingvalues.h> +#include <vespa/config/common/compressiontype.h> +#include <vespa/config/set/configsetsourcefactory.h> +#include <vespa/config/configgen/configinstance.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <map> + +namespace config { + +typedef vespalib::string SourceSpecKey; + +/** + * A source spec is a user provided specification of which sources to fetch + * config from. + */ +class SourceSpec +{ +public: + typedef std::unique_ptr<SourceSpec> UP; /// Convenience typedef + + /** + * Creates a source factory from which to create config sources for new + * subscriptions. The UpdateHandler should be + * provided to the source for it to post any update given any config + * request. + * + * @param handler A pointer to the update handler that will receive config + * updates from the source. + * @param timingValues Timing values to be used for this source. + * @return An std::unique_ptr<Source> that can be used to ask for config. + */ + virtual SourceFactory::UP createSourceFactory(const TimingValues & timingValues) const = 0; + virtual ~SourceSpec() { } +}; + + +/** + * A RawSpec gives the ability to specify config as a raw config string. + */ +class RawSpec : public SourceSpec +{ +public: + /** + * Constructs a new RawSpec that can be sent with a subscribe call. + * + * @param config The config represented as a raw string. + */ + RawSpec(const vespalib::string & config); + + // Implements SourceSpec + SourceFactory::UP createSourceFactory(const TimingValues & timingValues) const; + + /** + * Returns the string representation of this config. + * + * @return the config in a string. + */ + const vespalib::string & toString() const { return _config; } +private: + vespalib::string _config; +}; + +/** + * A FileSpec gives the ability to serve config from a file. The filenames in + * this spec must match the config definition name when subscribing. + */ +class FileSpec : public SourceSpec +{ +public: + /** + * Creates a FileSpec to serve config from a file. Multiple files may be + * added to the spec. + * + * @param fileName Path to the file to serve config from. + */ + FileSpec(const vespalib::string & fileName); + + /** + * Get the file name of this spec. + * + * @return the filename from which to serve config. + */ + const vespalib::string & getFileName() const { return _fileName; } + + // Implements SourceSpec + SourceFactory::UP createSourceFactory(const TimingValues & timingValues) const; +private: + void verifyName(const vespalib::string & fileName); + vespalib::string _fileName; +}; + +/** + * A DirSpec gives the ability to serve config from a directory. + */ +class DirSpec : public SourceSpec +{ +public: + /** + * Create a DirSpec to serve config from. The files within this directory + * must have the names of the config definition ending with a .cfg suffix. + * + * @param dirName Directory to serve config from. + */ + DirSpec(const vespalib::string & dirName); + + /** + * Get directory handled by this spec. + * + * @return the directory from which to serve config. + */ + const vespalib::string & getDirName() const { return _dirName; } + + // Implements SourceSpec + SourceFactory::UP createSourceFactory(const TimingValues & timingValues) const; +private: + vespalib::string _dirName; +}; + +/** + * A server spec is a user provided specification of one or more config servers + * that may provide config. + */ +class ServerSpec : public SourceSpec +{ +public: + /// A list of host specifications + typedef std::vector<vespalib::string> HostSpecList; + + /** + * Construct a ServerSpec that fetches the host specs from the + * VESPA_CONFIG_SOURCES environment variable. + */ + ServerSpec(); + + /** + * Construct a ServerSpec with a list host specifications on the form + * tcp/hostname:port + * + * @param list a list of host specifications. + */ + ServerSpec(const HostSpecList & list); + + /** + * Construct a ServerSpec with a host specification. + * + * @param hostSpec the host specification on the form "tcp/hostname:port" + */ + ServerSpec(const vespalib::string & hostSpec); + + // Implements SourceSpec + virtual SourceFactory::UP createSourceFactory(const TimingValues & timingValues) const; + + /** + * Add another host to this source spec, allowing failover. + * + * @param host hostname formatted as tcp/hostname:port + */ + void addHost(const vespalib::string & host) { _hostList.push_back(host); } + + /** + * Inspect how many hosts this source refers to. + * + * @return the number of hosts referred. + */ + size_t numHosts() const { return _hostList.size(); } + + /** + * Retrieve host specification element i + * + * @param i the spec element to retrieve. + */ + const vespalib::string & getHost(size_t i) const { return _hostList[i]; } + + /** + * Get the protocol version as parsed by this source spec. + */ + int protocolVersion() const { return _protocolVersion; } + + /** + * Get the trace level as parsed by this source spec. + */ + int traceLevel() const { return _traceLevel; } + + /** + * Get the compression type as parsed by this source spec. + */ + CompressionType compressionType() const { return _compressionType; } +private: + void initialize(const vespalib::string & hostSpec); + HostSpecList _hostList; + const int _protocolVersion; + const int _traceLevel; + const CompressionType _compressionType; + const static int DEFAULT_PROXY_PORT = 19090; +}; + + + +/** + * A ConfigSet gives the ability to serve config from a set of ConfigInstance + * builders. + */ +class ConfigSet : public SourceSpec +{ +public: + /// Constructs a new empty ConfigSet + ConfigSet(); + + /// Mapping of config keys to builders + typedef ConfigSetSourceFactory::BuilderMap BuilderMap; + typedef ConfigSetSourceFactory::BuilderMapSP BuilderMapSP; + /** + * Add a builder to serve config from. The builder must be of a + * 'ConfigType'Builder, and the configId must be the id to which you want to + * serve the config generated by this builder. + * + * @param configId The configId that should be used to get the config + * produced by this builder. + * @param builder A builder instance that you can use to change config later + * and then call reload on the ConfigContext object. + */ + void addBuilder(const vespalib::string & configId, ConfigInstance * builder); + + // Implements SourceSpec + SourceFactory::UP createSourceFactory(const TimingValues & timingValues) const; +private: + BuilderMapSP _builderMap; +}; + +/** + * A ConfigInstanceSpec serves a config from a config instance that does not change. + */ +class ConfigInstanceSpec : public SourceSpec +{ +public: + ConfigInstanceSpec(const ConfigInstance & instance); + SourceFactory::UP createSourceFactory(const TimingValues & timingValues) const; +private: + const ConfigKey _key; + vespalib::asciistream _buffer; +}; + +} + diff --git a/config/src/vespa/config/subscription/subscriptionid.h b/config/src/vespa/config/subscription/subscriptionid.h new file mode 100644 index 00000000000..00af2a55696 --- /dev/null +++ b/config/src/vespa/config/subscription/subscriptionid.h @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <vespa/fastos/fastos.h> + +namespace config { + +typedef uint64_t SubscriptionId; + +} // namespace config + |