diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /vespaclient |
Publish
Diffstat (limited to 'vespaclient')
74 files changed, 7395 insertions, 0 deletions
diff --git a/vespaclient/.gitignore b/vespaclient/.gitignore new file mode 100644 index 00000000000..235d7663475 --- /dev/null +++ b/vespaclient/.gitignore @@ -0,0 +1,9 @@ +*.iml +*.ipr +*.iws +target +/testng.out.log +/tmp +/pom.xml.build +Makefile +Testing diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt new file mode 100644 index 00000000000..2866bf6b577 --- /dev/null +++ b/vespaclient/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + DEPENDS + vespadefaults + fastos + configdefinitions + config_cloudconfig + vespalog + document + documentapi + vespalib + staging_vespalib + + LIBS + src/vespa/vespaclient/clusterlist + + APPS + src/vespa/vespaclient/spoolmaster + src/vespa/vespaclient/vdsstates + src/vespa/vespaclient/vespadoclocator + src/vespa/vespaclient/vesparoute +) diff --git a/vespaclient/OWNERS b/vespaclient/OWNERS new file mode 100644 index 00000000000..0e39145d8c3 --- /dev/null +++ b/vespaclient/OWNERS @@ -0,0 +1 @@ +dybdahl diff --git a/vespaclient/README b/vespaclient/README new file mode 100644 index 00000000000..356ba8de559 --- /dev/null +++ b/vespaclient/README @@ -0,0 +1 @@ +VESPA tools and applications diff --git a/vespaclient/bin/vdsgetsystemstate.sh b/vespaclient/bin/vdsgetsystemstate.sh new file mode 100755 index 00000000000..599bff9b2de --- /dev/null +++ b/vespaclient/bin/vdsgetsystemstate.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +echo "WARNING: This binary has been renamed from vdsgetsystemstate to " +echo " vdsgetclusterstate. Currently, this script calls the other. " +echo " as a convinience. This script will be removed later." + +vdsgetclusterstate $@ diff --git a/vespaclient/man/vespastat.1 b/vespaclient/man/vespastat.1 new file mode 100644 index 00000000000..755ce7b82f7 --- /dev/null +++ b/vespaclient/man/vespastat.1 @@ -0,0 +1,38 @@ +.TH VESPASTAT 1 2008-08-29 "Vespa" "Vespa Documentation" +.SH NAME +vespastat \- Shows status of a given document in Vespa +.SH SYNPOSIS +.B vespastat +[\fIOPTIONS\fR] <\fIDOCUMENT IDENTIFIER\fR] +.SH DESCRIPTION +.PP +This utility makes it possible to check internal information about all +instances of given document that still exist in Vespa. It is mostly used during +debugging. +.PP +It tells you what storage nodes currently has instances of the document, and +also informs about older instances of documents that still exist as the space +has not yet been reclaimed. +.PP +The results shows a sorted list of all instances found. The list should be +sorted by timestamps, with highest timestamp (newest in time) appearing on +top of the list. Only the entries with the freshest timestamp should be +used during normal operations. Though, if not all the nodes with copies has +the newest instance (highest timestamp), the file the document is in is not +in sync. +.PP +Mandatory arguments to long options are mandatory for short options too. +Short options can not currently be concatenated together. +.TP +\fB\-h\fR, \fB\-\-help\fR +Shows a short syntax reminder. +.TP +\fB\-r\fR, \fB\-\-route\fR +Send the stat to the following messagebus route. The default should be fine in +most cases, but in cases where you have multiple storage clusters, you might +want to send it to a specific cluster. + +.SH AUTHOR +Written by Haakon Humberset. +.SH "SEE ALSO" +Full documentation for Vespa diff --git a/vespaclient/src/.gitignore b/vespaclient/src/.gitignore new file mode 100644 index 00000000000..a39df0815b3 --- /dev/null +++ b/vespaclient/src/.gitignore @@ -0,0 +1,3 @@ +Makefile.ini +config_command.sh +project.dsw diff --git a/vespaclient/src/java/.gitignore b/vespaclient/src/java/.gitignore new file mode 100644 index 00000000000..861dbb24ab9 --- /dev/null +++ b/vespaclient/src/java/.gitignore @@ -0,0 +1,9 @@ +archive +build +classes +lib +log +log.log +staging +temp +testLogs diff --git a/vespaclient/src/java/etc/status.html b/vespaclient/src/java/etc/status.html new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespaclient/src/java/etc/status.html diff --git a/vespaclient/src/perl/PERL_BEST_PRACTISES b/vespaclient/src/perl/PERL_BEST_PRACTISES new file mode 100644 index 00000000000..29bbc3f01bf --- /dev/null +++ b/vespaclient/src/perl/PERL_BEST_PRACTISES @@ -0,0 +1,361 @@ +To try and make the perl tools good and consistent, here is a list of best +practises used within the modules. + +(Whether they are best can of course be debated, but what's listed is what is +currently used) + +1. Always use strict and warnings first thing. + +There is a lot of stuff legal in perl for backward compatability and ease of +writing one liners. However, these statements are frequent source of bugs in +real code. All modules and binaries should use strict and warnings to ensure +that these checks are enabled. (There is a unit test in the module grepping +source to ensure this). Thus, pretty much the first thing in all perl files +should be: + + use strict; + use warnings; + +2. Use perl modules. + +We want to group functionality into multiple files in perl too. A perl module is +just another perl file with a .pm extension, which minimally can look something +like this: + +Yahoo/Vespa/VespaModel.pm: + + package Yahoo::Vespa::VespaModel; + + use strict; + use warnings; + + my %CACHED_MODEL; # Prevent multiple fetches by caching results + + return 1; + + sub get { + ... + } + +Yahoo/Vespa/Bin/MyBinary.pl: + + use strict; + use warnings; + use Yahoo::Vespa::VespaModel; + + my $model = Yahoo::Vespa::VespaModel::get(); + +2a. Module install locations. + +Perl utilities are installed under $VESPA_HOME/lib/perl5/site_perl + +2b. Aliasing namespace. + +Perl doesn't have that great namespace handling. It's not like in C++, where we +can be in the storage::api namespace and thus address something in the +storage::lib namespace as lib::foo or even refer to another instance in the +same namespace. Thus, if the user of the VespaModel module above were +Yahoo::Vespa::MyLib, it still has to address VespaModel with full path by +default. + +It is possible to create aliases in Perl to help this. Using an alias the +MyBinary.pl code above could look like: + + ... + use Yahoo::Vespa::VespaModel; + + BEGIN { + *VespaModel:: = *Yahoo::Vespa::VespaModel:: ; + } + + my $model = VespaModel::get(); + +The alias declaration doesn't look very pretty, but it can be helpful to get +code looking simple. + +2b. Exporting members into users namespace. + +Another option to using long prefixed names or aliasing, is to export names +into the callers namespace. This can be done in a module doing something like +this: + +Yahoo/Vespa/VespaModel.pm: + + package Yahoo::Vespa::VespaModel; + + use strict; + use warnings; + + BEGIN { + use base 'Exporter'; + our @EXPORT = qw( getVespaModel ); + our @EXPORT_OK = qw( otherFunction ); + } + + my %CACHED_MODEL; + + return 1; + + sub getVespaModel { + ... + } + sub otherFunction { + ... + } + +Yahoo/Vespa/Bin/MyBinary.pl: + + use strict; + use warnings; + use Yahoo::Vespa::VespaModel; + + my $model = getVespaModel(); + +In this example, the getVespaModel function is imported by default, while +otherFunction is not, but can be included optionally. You can specify what to +include by adding arguments to the use statements: + +use Yahoo::Vespa::VespaMode; # Import defaults +use Yahoo::Vespa::VespaModel (); # Import nothing + # Import other function but not getVespaModel +use Yahoo::Vespa::VespaModel qw( otherFunction ); + +(The qw(...) function is just a function to generate an array from a whitespace separated string. Writing qw( foo bar ) is equivalent to writing ('foo', 'bar')) + +You can also export/import variables, but then you need to prefix the names +with the type, as in "our @EXPORT = qw( $number, @list, %hash );". + +Note that you should prefer to export as little functions as possible as they +can clash with names used in caller. Also, the tokens you do export should have +fairly descriptive names to reduce the chance of this happening. An exported +name does not have a module name tagged to it to include context. Thus, if you +don't export you can for instance use Json::encode, but if you do export you +likely need to call the function encodeJson or similar instead. + +2c. Prefer private variables (my instead of our) + +When declaring variables with 'my' they become private to the module, and you +know outsiders can't alter it. This makes it easier when debugging as there are +less possibilities for what can happen. + +2d. Prefer calling functions or exported variables rather than referencing +global variables in a module from the outside. + +Referencing non-declared variables in another module does not seem to create +compiler warnings, nor does using private (my) declared variables. Thus it's +better to refer to imported variables or call a function, such that the +compiler will tell you when this doesn't work anymore. + +2e. Put all function declarations at the bottom. + +When a perl module is loaded, the code within it run. If that doesn't return +true, that means the module fails to load. Thus, traditionally, perl modules +often end with 1; (equivalent to return 1;) to ensure this. However, this mean +you have to read through the entire module to look for module code run. + +By doing exit(...) call in main prog before function declaration and return; in +modules before function declarations, it is easier for reader to see that you +haven't hidden other code between the function declarations. (Unless you've +hacked it into a BEGIN{} block to enforce it to run before everything else) + +2f. Make it easy to reinitialize in unit tests. + +By putting initialization steps in a separate init function, rather than doing +it on load, unit tests can easily call it to reinitialize the module between +tests. Also this separates declarations of what exist from the initialization so +it is easier to see what variables are there. + +3. Confess instead of die. + +The typical perl assert is use of the 'die' function, as in: + + defined $foo or die "We expected 'foo' to be defined here"; + +The Utils package contains a confess function to be used instead (Wrapping an +external dependency), which will do the same as 'die', but will add a +stacktrace too, such that when encountered, it is much easier to find the +culprit. + +4. Do not call exit() in libraries. + +We want to be able to unit test all types of functions in unit tests, also +functionality that makes application abort and exit. The Utils defines an +exitApplication that is mocked for unit tests. Assertion types of exits with +die/confess can also be catched in unit tests. + +5. Code conventions. + + - Upper case, underscore divided, module level variables. + - Camel case function names. + - Four space indent. + +6. Naming function arguments. + +For perl, a function is just a call to a subroutine with a list containing +whatever arguments, called @_. Using this directly makes the code hard to read. +Naming variables makes this a bit easier.. + + sub getVespaModel { # (ConfigServerHost, ConfigServerPort) + return Json::parse(Http::get("http://$_[0]:$_[1]/foo"#)); + } + + sub getVespaModel { # (ConfigServerHost, ConfigServerPort) -> ObjTree + my ($host, $port) = @_; + return Json::parse(Http::get("http://$host:$port/foo"#)); + } + +In the latter example it is easier to read the code. + +The argument comment is something I usually add for function declarations to +look better with vim folding.. When I fold functions in vim, the below line will +look like + ++-- 4 lines: sub getVespaModel (ConfigServerHost, ConfigServerPort) -> ObjTree + +Using such a convention it is thus easier to read the code, as you may be able +to see all your other function declarations while working on the function you +have expanded. + +6b. Functions with many arguments. + +If you create functions with loads of parameters you can end up with a messy +function, and a hard time to adjust all the uses of it when you want to extend +it. At these times you may use hashes to name variables, such that the order +is no longer important.. + + sub getVespaModel { # (ConfigServerHost, ConfigServerPort) -> ObjTree + my $args = $_[0]; + return Json::parse(Http::get("http://$$args{'host':$$args{'port'}/foo"#)); + } + + getVespaModel({ 'host' => 'myhost', 'port' => 80 }); + +Using this trick, you can have defaults for various arguments that can be +ignored by users not caring, rather than having to pass undef at many positions +to ensure order of parameters is correct. + +Note however, that this looks a bit more messy in the function itself, and it +makes it more important to make comments of what arguments are actually handled +and which ones are not optional.. I prefer to try and have short argument +lists instead. + +7. Constants + +Sometimes you want to declare constants. Valid flag values for instance. You +can of course just declare global variables, but you have no way of ensuring +that they never change, which can be confusing. To define constants you can +do the following: + + use constant MY_FLAG => 8; + +This constant is referred to without the usual $ prefix too, so it is easy to +distinguish it from variables. These constants can also be exported, enabling +you to create function calls like: + + MyModule::foo("bar", OPTION_ARGH | OPTION_BAZ); + +Though this of course pollutes callers namespace again, so he has to +specifically not include them if he otherwise would have a name clash. + +8. Libraries not in search path + +Sometimes people install perl libraries in non-default locations. If temporary +you can fix this by add directory to PERLLIB on command line, but if permanent, +the recommended way to find the libraries is to add the directory to the search +path where you include it, like the Yahoo installation for the JSON library: + + use lib '$VESPA_HOME/lib64/perl5/site_perl/5.14/'; + use JSON; + +9. Perl references + +In perl you can create references to variables by prefixing a backslash '\'. + + my @foo ; my $listref = \@foo; + my $var ; my $scalarref = \$var; + my %bar ; my $hashref = \%bar; + +You can also create references to lists and hashes directly: + + my $listref = [ 1, 2, 4 ]; # [] instead of () to get ref instead of list. + my $hashref = { 'foo' => 3, 'bar' => 'hmm' }; # {} instead of () + +To check what a variable is you can use the ref() function: + + ref($scalarref) eq 'SCALAR' + ref($listref) eq 'ARRAY' + ref($hashref) eq 'HASH' + ref($var) == undef + +To dereference a reference you can add a deref clause around it: + my @foo = @{ $listref }; + my %bar = %{ $hashref }; + my $scalar = ${ $scalarref }; + +If the insides of the clause is easy, you also omit it. + my $scalar = $$scalarref; + my %bar = %$hashref; + my $value = $$hashref{'foo'} + +You can also dereference using the -> operator. + my $value = $hashref->{'foo'}; + my $value2 = $listref->[3]; # Element 3 in the list + +The -> operator is typically used when traversing object structures. + +10. Perl structs + +Perl object programming requires some blessing and doesn't look that awesome, +so I typically mostly program functionally. However, at the bare minimum one +needs to be able to create some structs to contain data that isn't bare +primitives. + +Perl's Class::Struct module implements a way to define structs in a simple +fashion without needing to know how bless works, module inheritation and so +forth. + +An example use case here is Yahoo::Vespa::ClusterState + + use Class::Struct; + + struct( ClusterState => { + globalState => '$', + distributor => '%', + storage => '%' + }); + + struct( Node => { + group => '$', + unit => 'State', + generated => 'State', + user => 'State' + }); + + struct( State => { + state => '$', + reason => '$', + timestamp => '$', + source => '$' + }); + +# Some file using it. + + use Yahoo::Vespa::ClusterState; + + my $clusterState = new ClusterState; + $clusterState->globalState('UP'); + my $node = new Node; + $node->group('Foo'); + $clusterState->distributor('0', $node); + + ... + + my $group = $clusterState->distributor->{'0'}->group; + my $nodetype = 'storage'; + my $group = $clusterState->$nodetype->{'0'}->group; + +Some notes: + - The names of the structs are automatically imported. Thus you don't need to + worry about prefixing or aliasing, but be aware names can collide for user. + - $, % or @ indicates if content is scalar, hash or list. A name indicates the + name of another struct that should have the content. diff --git a/vespaclient/src/perl/bin/GetClusterState.pl b/vespaclient/src/perl/bin/GetClusterState.pl new file mode 100755 index 00000000000..2352a5a0ca6 --- /dev/null +++ b/vespaclient/src/perl/bin/GetClusterState.pl @@ -0,0 +1,74 @@ +#!/usr/local/bin/perl -w +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN perl environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +use File::Basename; +use File::Path; + +sub findpath { + my $myfullname = ${0}; + my($myname, $mypath) = fileparse($myfullname); + + return $mypath if ( $mypath && -d $mypath ); + $mypath=`pwd`; + + my $pwdfullname = $mypath . "/" . $myname; + return $mypath if ( -f $pwdfullname ); + return 0; +} + +# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise +sub is_vespa_home { + my($VESPA_HOME) = shift; + my $COMMON_ENV="libexec/vespa/common-env.sh"; + if ( $VESPA_HOME && -d $VESPA_HOME ) { + my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; + return $VESPA_HOME if -f $common_env; + } + return 0; +} + +# Returns the home of Vespa, or dies if it cannot +sub findhome { + # Try the VESPA_HOME env variable + return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); + if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly + die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; + } + + # Try the ROOT env variable + $ROOT = $ENV{'ROOT'}; + return $ROOT if is_vespa_home($ROOT); + + # Try the script location or current dir + my $mypath = findpath(); + if ($mypath) { + while ( $mypath =~ s|/[^/]*$|| ) { + return $mypath if is_vespa_home($mypath); + } + } + die "FATAL: Missing VESPA_HOME environment variable\n"; +} + +BEGIN { + my $tmp = findhome(); + if ( $tmp !~ m{[/]$} ) { $tmp .= "/"; } + $ENV{'VESPA_HOME'} = $tmp; +} +my $VESPA_HOME = $ENV{'VESPA_HOME'}; + +# END perl environment bootstrap section + +use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use Yahoo::Vespa::Defaults; +readConfFile(); + +use strict; +use warnings; +use lib '$VESPA_HOME/lib/perl5/site_perl'; + +use Yahoo::Vespa::Bin::GetClusterState; + +exit(getClusterState(\@ARGV)); diff --git a/vespaclient/src/perl/bin/GetNodeState.pl b/vespaclient/src/perl/bin/GetNodeState.pl new file mode 100755 index 00000000000..d373eadb65b --- /dev/null +++ b/vespaclient/src/perl/bin/GetNodeState.pl @@ -0,0 +1,74 @@ +#!/usr/local/bin/perl -w +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN perl environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +use File::Basename; +use File::Path; + +sub findpath { + my $myfullname = ${0}; + my($myname, $mypath) = fileparse($myfullname); + + return $mypath if ( $mypath && -d $mypath ); + $mypath=`pwd`; + + my $pwdfullname = $mypath . "/" . $myname; + return $mypath if ( -f $pwdfullname ); + return 0; +} + +# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise +sub is_vespa_home { + my($VESPA_HOME) = shift; + my $COMMON_ENV="libexec/vespa/common-env.sh"; + if ( $VESPA_HOME && -d $VESPA_HOME ) { + my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; + return $VESPA_HOME if -f $common_env; + } + return 0; +} + +# Returns the home of Vespa, or dies if it cannot +sub findhome { + # Try the VESPA_HOME env variable + return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); + if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly + die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; + } + + # Try the ROOT env variable + $ROOT = $ENV{'ROOT'}; + return $ROOT if is_vespa_home($ROOT); + + # Try the script location or current dir + my $mypath = findpath(); + if ($mypath) { + while ( $mypath =~ s|/[^/]*$|| ) { + return $mypath if is_vespa_home($mypath); + } + } + die "FATAL: Missing VESPA_HOME environment variable\n"; +} + +BEGIN { + my $tmp = findhome(); + if ( $tmp !~ m{[/]$} ) { $tmp .= "/"; } + $ENV{'VESPA_HOME'} = $tmp; +} +my $VESPA_HOME = $ENV{'VESPA_HOME'}; + +# END perl environment bootstrap section + +use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use Yahoo::Vespa::Defaults; +readConfFile(); + +use strict; +use warnings; +use lib '$VESPA_HOME/lib/perl5/site_perl'; + +use Yahoo::Vespa::Bin::GetNodeState; + +exit(getNodeState(\@ARGV)); diff --git a/vespaclient/src/perl/bin/SetNodeState.pl b/vespaclient/src/perl/bin/SetNodeState.pl new file mode 100755 index 00000000000..7002ab523b5 --- /dev/null +++ b/vespaclient/src/perl/bin/SetNodeState.pl @@ -0,0 +1,71 @@ +#!/usr/local/bin/perl -w +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN perl environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +use File::Basename; +use File::Path; + +sub findpath { + my $myfullname = ${0}; + my($myname, $mypath) = fileparse($myfullname); + + return $mypath if ( $mypath && -d $mypath ); + $mypath=`pwd`; + + my $pwdfullname = $mypath . "/" . $myname; + return $mypath if ( -f $pwdfullname ); + return 0; +} + +# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise +sub is_vespa_home { + my($VESPA_HOME) = shift; + my $COMMON_ENV="libexec/vespa/common-env.sh"; + if ( $VESPA_HOME && -d $VESPA_HOME ) { + my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; + return $VESPA_HOME if -f $common_env; + } + return 0; +} + +# Returns the home of Vespa, or dies if it cannot +sub findhome { + # Try the VESPA_HOME env variable + return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); + if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly + die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; + } + + # Try the ROOT env variable + $ROOT = $ENV{'ROOT'}; + return $ROOT if is_vespa_home($ROOT); + + # Try the script location or current dir + my $mypath = findpath(); + if ($mypath) { + while ( $mypath =~ s|/[^/]*$|| ) { + return $mypath if is_vespa_home($mypath); + } + } + die "FATAL: Missing VESPA_HOME environment variable\n"; +} + +BEGIN { + my $tmp = findhome(); + if ( $tmp !~ m{[/]$} ) { $tmp .= "/"; } + $ENV{'VESPA_HOME'} = $tmp; +} +my $VESPA_HOME = $ENV{'VESPA_HOME'}; + +# END perl environment bootstrap section + +use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl'; +use Yahoo::Vespa::Defaults; +readConfFile(); + +use strict; +use warnings; +use Yahoo::Vespa::Bin::SetNodeState; +exit(setNodeState(\@ARGV)); diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/ArgParser.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/ArgParser.pm new file mode 100644 index 00000000000..c6b0fb0f157 --- /dev/null +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/ArgParser.pm @@ -0,0 +1,689 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Argument parser. +# +# Intentions: +# - Make it very easy for programs to get info from command line. +# - Allow shared libraries to register own options, such that a program can +# delegate command line options to libraries used. (For instance, verbosity +# arguments will be automatically delegated to console output module without +# program needing to care much. +# - Create a unified looking syntax page for all command line tools. +# - Be able to reuse input validation. For instance that an integer don't +# have a decimal point and that a hostname can be resolved. +# + +package Yahoo::Vespa::ArgParser; + +use strict; +use warnings; +use Yahoo::Vespa::ConsoleOutput; +use Yahoo::Vespa::Utils; + +BEGIN { # - Define exports and dependency aliases for module. + use base 'Exporter'; + our @EXPORT = qw( + addArgParserValidator + setProgramBinaryName setProgramDescription + setArgument setOptionHeader + setFlagOption setHostOption setPortOption setStringOption + setIntegerOption setFloatOption setUpCountingOption setDownCountingOption + handleCommandLineArguments + OPTION_SECRET OPTION_INVERTEDFLAG OPTION_REQUIRED + ); + # Alias so we can avoid writing the entire package name + *ConsoleOutput:: = *Yahoo::Vespa::ConsoleOutput:: +} + +my @ARGUMENTS; +my $DESCRIPTION; +my $BINARY_NAME; +my @ARG_SPEC_ARRAY; +my %OPTION_SPEC; +my @OPTION_SPEC_ARRAY; +my $SYNTAX_PAGE; +my $SHOW_HIDDEN; +my @VALIDATORS; +use constant OPTION_SECRET => 1; +use constant OPTION_INVERTEDFLAG => 2; +use constant OPTION_ADDFIRST => 4; +use constant OPTION_REQUIRED => 8; + +# These variables are properties needed by ConsoleOutput module. ArgParser +# handles that modules argument settings as it cannot possibly depend upon +# ArgParser itself. +my $VERBOSITY; # Default verbosity before parsing arguments +my $ANSI_COLORS; # Whether to use ansi colors or not. + +&initialize(); + +return 1; + +########################## Default exported functions ######################## + +sub handleCommandLineArguments { # () Parses and sets all values + my ($args, $validate_args_sub) = @_; + + ®isterInternalParameters(); + if (!&parseCommandLineArguments($args)) { + &writeSyntaxPage(); + exitApplication(1); + } + if (defined $validate_args_sub && !&$validate_args_sub()) { + &writeSyntaxPage(); + exitApplication(1); + } + if ($SYNTAX_PAGE) { + &writeSyntaxPage(); + exitApplication(0); + } +} + +sub addArgParserValidator { # (Validator) Add callback to verify parsing + # Using such callbacks you can verify more than is supported natively by + # argument parser, such that you can fail argument parsing at same step as + # internally supported checks are handled. + scalar @_ == 1 or confess "Invalid number of arguments given."; + push @VALIDATORS, $_[0]; +} +sub setProgramBinaryName { # (Name) Defaults to name used on command line + scalar @_ == 1 or confess "Invalid number of arguments given."; + ($BINARY_NAME) = @_; +} +sub setProgramDescription { # (Description) + scalar @_ == 1 or confess "Invalid number of arguments given."; + ($DESCRIPTION) = @_; +} + +sub setOptionHeader { # (Description) + my ($desc) = @_; + push @OPTION_SPEC_ARRAY, $desc; +} + +sub setFlagOption { # (ids[], Result&, Description, Flags) + scalar @_ >= 3 or confess "Invalid number of arguments given."; + my ($ids, $result, $description, $flags) = @_; + if (!defined $flags) { $flags = 0; } + my %optionspec = ( + 'result' => $result, + 'flags' => $flags, + 'ids' => $ids, + 'description' => $description, + 'arg_count' => 0, + 'initializer' => sub { + $$result = (($flags & OPTION_INVERTEDFLAG) == 0 ? 0 : 1); + return 1; + }, + 'result_evaluator' => sub { + $$result = (($flags & OPTION_INVERTEDFLAG) == 0 ? 1 : 0); + return 1; + } + ); + setGenericOption($ids, \%optionspec); +} +sub setHostOption { # (ids[], Result&, Description, Flags) + my ($ids, $result, $description, $flags) = @_; + my %optionspec = ( + 'result' => $result, + 'flags' => $flags, + 'ids' => $ids, + 'description' => $description, + 'arg_count' => 1, + 'result_evaluator' => sub { + my ($id, $args) = @_; + scalar @$args == 1 or confess "Should have one arg here."; + my $host = $$args[0]; + if (!&validHost($host)) { + printError "Invalid host '$host' given to option '$id'. " + . "Not a valid host\n"; + return 0; + } + printSpam "Set value of '$id' to $host.\n"; + $$result = $host; + return 1; + } + ); + setGenericOption($ids, \%optionspec); +} +sub setPortOption { # (ids[], Result&, Description, Flags) + my ($ids, $result, $description, $flags) = @_; + my %optionspec = ( + 'result' => $result, + 'flags' => $flags, + 'ids' => $ids, + 'description' => $description, + 'arg_count' => 1, + 'result_evaluator' => sub { + my ($id, $args) = @_; + scalar @$args == 1 or confess "Should have one arg here."; + my $val = $$args[0]; + if ($val !~ /^\d+$/ || $val < 0 || $val >= 65536) { + printError "Invalid value '$val' given to port option '$id'." + . " Must be an unsigned 16 bit integer.\n"; + return 0; + } + printSpam "Set value of '$id' to $val.\n"; + $$result = $val; + return 1; + } + ); + setGenericOption($ids, \%optionspec); +} +sub setIntegerOption { # (ids[], Result&, Description, Flags) + my ($ids, $result, $description, $flags) = @_; + my %optionspec = ( + 'result' => $result, + 'flags' => $flags, + 'ids' => $ids, + 'description' => $description, + 'arg_count' => 1, + 'result_evaluator' => sub { + my ($id, $args) = @_; + scalar @$args == 1 or confess "Should have one arg here."; + my $val = $$args[0]; + if ($val !~ /^(?:[-\+])?\d+$/) { + printError "Invalid value '$val' given to integer option " + . "'$id'.\n"; + return 0; + } + printSpam "Set value of '$id' to $val.\n"; + $$result = $val; + return 1; + } + ); + setGenericOption($ids, \%optionspec); +} +sub setFloatOption { # (ids[], Result&, Description, Flags) + my ($ids, $result, $description, $flags) = @_; + my %optionspec = ( + 'result' => $result, + 'flags' => $flags, + 'ids' => $ids, + 'description' => $description, + 'arg_count' => 1, + 'result_evaluator' => sub { + my ($id, $args) = @_; + scalar @$args == 1 or confess "Should have one arg here."; + my $val = $$args[0]; + if ($val !~ /^(?:[-\+])?\d+(?:\.\d+)?$/) { + printError "Invalid value '$val' given to float option " + . "'$id'.\n"; + return 0; + } + printSpam "Set value of '$id' to $val.\n"; + $$result = $val; + return 1; + } + ); + setGenericOption($ids, \%optionspec); +} +sub setStringOption { # (ids[], Result&, Description, Flags) + my ($ids, $result, $description, $flags) = @_; + my %optionspec = ( + 'result' => $result, + 'flags' => $flags, + 'ids' => $ids, + 'description' => $description, + 'arg_count' => 1, + 'result_evaluator' => sub { + my ($id, $args) = @_; + scalar @$args == 1 or confess "Should have one arg here."; + my $val = $$args[0]; + printSpam "Set value of '$id' to $val.\n"; + $$result = $val; + return 1; + } + ); + setGenericOption($ids, \%optionspec); +} +sub setUpCountingOption { # (ids[], Result&, Description, Flags) + my ($ids, $result, $description, $flags) = @_; + my $org = $$result; + my %optionspec = ( + 'result' => $result, + 'flags' => $flags, + 'ids' => $ids, + 'description' => $description, + 'arg_count' => 0, + 'initializer' => sub { + $$result = $org; + return 1; + }, + 'result_evaluator' => sub { + if (!defined $$result) { + $$result = 0; + } + ++$$result; + return 1; + } + ); + setGenericOption($ids, \%optionspec); +} +sub setDownCountingOption { # (ids[], Result&, Description, Flags) + my ($ids, $result, $description, $flags) = @_; + my $org = $$result; + my %optionspec = ( + 'result' => $result, + 'flags' => $flags, + 'ids' => $ids, + 'description' => $description, + 'arg_count' => 0, + 'initializer' => sub { + $$result = $org; + return 1; + }, + 'result_evaluator' => sub { + if (!defined $$result) { + $$result = 0; + } + --$$result; + return 1; + } + ); + setGenericOption($ids, \%optionspec); +} + +sub setArgument { # (Result&, Name, Description) + my ($result, $name, $description, $flags) = @_; + if (!defined $flags) { $flags = 0; } + if (scalar @ARG_SPEC_ARRAY > 0 && ($flags & OPTION_REQUIRED) != 0) { + my $last = $ARG_SPEC_ARRAY[scalar @ARG_SPEC_ARRAY - 1]; + if (($$last{'flags'} & OPTION_REQUIRED) == 0) { + confess "Cannot add required argument after optional argument"; + } + } + my %argspec = ( + 'result' => $result, + 'flags' => $flags, + 'name' => $name, + 'description' => $description, + 'result_evaluator' => sub { + my ($arg) = @_; + $$result = $arg; + return 1; + } + ); + push @ARG_SPEC_ARRAY, \%argspec; +} + +######################## Externally usable functions ####################### + +sub registerInternalParameters { # () + # Register console output parameters too, as the output module can't depend + # on this tool. + setFlagOption( + ['show-hidden'], + \$SHOW_HIDDEN, + 'Also show hidden undocumented debug options.', + OPTION_ADDFIRST); + setDownCountingOption( + ['s'], + \$VERBOSITY, + 'Create less verbose output.', + OPTION_ADDFIRST); + setUpCountingOption( + ['v'], + \$VERBOSITY, + 'Create more verbose output.', + OPTION_ADDFIRST); + setFlagOption( + ['h', 'help'], + \$SYNTAX_PAGE, + 'Show this help page.', + OPTION_ADDFIRST); + + # If color use is supported and turned on by default, give option to not use + if ($ANSI_COLORS) { + setOptionHeader(''); + setFlagOption( + ['nocolors'], + \$ANSI_COLORS, + 'Do not use ansi colors in print.', + OPTION_SECRET | OPTION_INVERTEDFLAG); + } +} +sub setShowHidden { # (Bool) + $SHOW_HIDDEN = ($_[0] ? 1 : 0); +} + +############## Utility functions - Not intended for external use ############# + +sub initialize { # () + $VERBOSITY = 3; + $ANSI_COLORS = Yahoo::Vespa::ConsoleOutput::ansiColorsSupported(); + $DESCRIPTION = undef; + $BINARY_NAME = $0; + if ($BINARY_NAME =~ /\/([^\/]+)$/) { + $BINARY_NAME = $1; + } + %OPTION_SPEC = (); + @OPTION_SPEC_ARRAY = (); + @ARG_SPEC_ARRAY = (); + @VALIDATORS = (); + $SYNTAX_PAGE = undef; + $SHOW_HIDDEN = undef; + @ARGUMENTS = undef; +} +sub parseCommandLineArguments { # (ArgumentListRef) + printDebug "Parsing command line arguments\n"; + @ARGUMENTS = @{ $_[0] }; + foreach my $spec (@OPTION_SPEC_ARRAY) { + if (ref($spec) && exists $$spec{'initializer'}) { + my $initsub = $$spec{'initializer'}; + &$initsub(); + } + } + my %eaten_args; + if (!&parseOptions(\%eaten_args)) { + printDebug "Option parsing failed\n"; + return 0; + } + if (!&parseArguments(\%eaten_args)) { + printDebug "Argument parsing failed\n"; + return 0; + } + ConsoleOutput::setVerbosity($VERBOSITY); + ConsoleOutput::setUseAnsiColors($ANSI_COLORS); + return 1; +} +sub writeSyntaxPage { # () + if (defined $DESCRIPTION) { + printResult $DESCRIPTION . "\n\n"; + } + printResult "Usage: " . $BINARY_NAME; + if (scalar keys %OPTION_SPEC > 0) { + printResult " [Options]"; + } + foreach my $arg (@ARG_SPEC_ARRAY) { + if (($$arg{'flags'} & OPTION_REQUIRED) != 0) { + printResult " <" . $$arg{'name'} . ">"; + } else { + printResult " [" . $$arg{'name'} . "]"; + } + } + printResult "\n"; + + if (scalar @ARG_SPEC_ARRAY > 0) { + &writeArgumentSyntax(); + } + if (scalar keys %OPTION_SPEC > 0) { + &writeOptionSyntax(); + } +} +sub setGenericOption { # (ids[], Optionspec) + my ($ids, $spec) = @_; + if (!defined $$spec{'flags'}) { + $$spec{'flags'} = 0; + } + foreach my $id (@$ids) { + if (length $id == 1 && $id =~ /[0-9]/) { + confess "A short option can not be a digit. Reserved so we can parse " + . "-4 as a negative number argument rather than an option 4"; + } + } + foreach my $id (@$ids) { + $OPTION_SPEC{$id} = $spec; + } + if (($$spec{'flags'} & OPTION_ADDFIRST) == 0) { + push @OPTION_SPEC_ARRAY, $spec; + } else { + unshift @OPTION_SPEC_ARRAY, $spec; + } +} +sub parseArguments { # (EatenArgs) + my ($eaten_args) = @_; + my $stopIndex = 10000000; + my $argIndex = 0; + printSpam "Parsing arguments\n"; + for (my $i=0; $i<scalar @ARGUMENTS; ++$i) { + printSpam "Processing arg '$ARGUMENTS[$i]'.\n"; + if ($i <= $stopIndex && $ARGUMENTS[$i] eq '--') { + printSpam "Found --. Further dash prefixed args will be args\n"; + $stopIndex = $i; + } elsif ($i <= $stopIndex && $ARGUMENTS[$i] =~ /^-/) { + printSpam "Option declaration. Ignoring\n"; + } elsif (exists $$eaten_args{$i}) { + printSpam "Already eaten argument. Ignoring\n"; + } elsif ($argIndex < scalar @ARG_SPEC_ARRAY) { + my $spec = $ARG_SPEC_ARRAY[$argIndex]; + my $name = $$spec{'name'}; + if (!&{$$spec{'result_evaluator'}}($ARGUMENTS[$i])) { + printDebug "Failed evaluate result of arg $name. Aborting\n"; + return 0; + } + printSpam "Successful parsing of argument '$name'.\n"; + $$eaten_args{$i} = 1; + ++$argIndex; + } else { + printError "Unhandled argument '$ARGUMENTS[$i]'.\n"; + return 0; + } + } + if ($SYNTAX_PAGE) { # Ignore required arg check if syntax page is to be shown + return 1; + } + for (my $i=$argIndex; $i<scalar @ARG_SPEC_ARRAY; ++$i) { + my $spec = $ARG_SPEC_ARRAY[$i]; + if (($$spec{'flags'} & OPTION_REQUIRED) != 0) { + my $name = $$spec{'name'}; + printError "Argument $name is required but not specified.\n"; + return 0; + } + } + return 1; +} +sub getOptionArguments { # (Count, MinIndex, EatenArgs) + my ($count, $minIndex, $eaten_args) = @_; + my $stopIndex = 10000000; + my @result; + if ($count == 0) { return \@result; } + for (my $i=0; $i<scalar @ARGUMENTS; ++$i) { + printSpam "Processing arg '$ARGUMENTS[$i]'.\n"; + if ($i <= $stopIndex && $ARGUMENTS[$i] eq '--') { + printSpam "Found --. Further dash prefixed args will be args\n"; + $stopIndex = $i; + } elsif ($i <= $stopIndex && $ARGUMENTS[$i] =~ /^-[^0-9]/) { + printSpam "Option declaration. Ignoring\n"; + } elsif (exists $$eaten_args{$i}) { + printSpam "Already eaten argument. Ignoring\n"; + } elsif ($i < $minIndex) { + printSpam "Not eaten, but too low index to be option arg.\n"; + } else { + printSpam "Using argument\n"; + push @result, $ARGUMENTS[$i]; + $$eaten_args{$i} = 1; + if (scalar @result == $count) { + return \@result; + } + } + } + printSpam "Too few option arguments found. Returning undef\n"; + return; +} +sub parseOption { # (Id, EatenArgs, Index) + my ($id, $eaten_args, $index) = @_; + if (!exists $OPTION_SPEC{$id}) { + printError "Unknown option '$id'.\n"; + return 0; + } + my $spec = $OPTION_SPEC{$id}; + my $args = getOptionArguments($$spec{'arg_count'}, $index, $eaten_args); + if (!defined $args) { + printError "Too few arguments for option '$id'.\n"; + return 0; + } + printSpam, "Found " . (scalar @$args) . " args\n"; + if (!&{$$spec{'result_evaluator'}}($id, $args)) { + printDebug "Failed evaluate result of option '$id'. Aborting\n"; + return 0; + } + printSpam "Successful parsing of option '$id'.\n"; + return 1; +} +sub parseOptions { # (EatenArgs) + my ($eaten_args) = @_; + for (my $i=0; $i<scalar @ARGUMENTS; ++$i) { + if ($ARGUMENTS[$i] =~ /^--(.+)$/) { + my $id = $1; + printSpam "Parsing long option '$id'.\n"; + if (!&parseOption($id, $eaten_args, $i)) { + return 0; + } + } elsif ($ARGUMENTS[$i] =~ /^-([^0-9].*)$/) { + my $shortids = $1; + while ($shortids =~ /^(.)(.*)$/) { + my ($id, $rest) = ($1, $2); + printSpam "Parsing short option '$id'.\n"; + if (!&parseOption($id, $eaten_args, $i)) { + return 0; + } + $shortids = $rest; + } + } + } + printSpam "Successful parsing of all options.\n"; + return 1; +} +sub writeArgumentSyntax { # () + printResult "\nArguments:\n"; + my $max_name_length = &getMaxNameLength(); + if ($max_name_length > 30) { $max_name_length = 30; } + foreach my $spec (@ARG_SPEC_ARRAY) { + &writeArgumentName($$spec{'name'}, $max_name_length); + &writeOptionDescription($spec, $max_name_length + 3); + } +} +sub getMaxNameLength { # () + my $max = 0; + foreach my $spec (@ARG_SPEC_ARRAY) { + my $len = 1 + length $$spec{'name'}; + if ($len > $max) { $max = $len; } + } + return $max; +} +sub writeArgumentName { # (Name, MaxNameLength) + my ($name, $maxnamelen) = @_; + printResult " $name"; + my $totalLength = 1 + length $name; + if ($totalLength <= $maxnamelen) { + for (my $i=$totalLength; $i<$maxnamelen; ++$i) { + printResult ' '; + } + } else { + printResult "\n"; + for (my $i=0; $i<$maxnamelen; ++$i) { + printResult ' '; + } + } + printResult " : "; +} +sub writeOptionSyntax { # () + printResult "\nOptions:\n"; + my $max_id_length = &getMaxIdLength(); + if ($max_id_length > 30) { $max_id_length = 30; } + my $cachedHeader; + foreach my $spec (@OPTION_SPEC_ARRAY) { + if (ref($spec) eq 'HASH') { + my $flags = $$spec{'flags'}; + if ($SHOW_HIDDEN || ($flags & OPTION_SECRET) == 0) { + if (defined $cachedHeader) { + printResult "\n"; + if ($cachedHeader ne '') { + &writeOptionHeader($cachedHeader); + } + $cachedHeader = undef; + } + &writeOptionId($spec, $max_id_length); + &writeOptionDescription($spec, $max_id_length + 3); + } + } else { + $cachedHeader = $spec; + } + } +} +sub getMaxIdLength { # () + my $max = 0; + foreach my $spec (@OPTION_SPEC_ARRAY) { + if (!ref($spec)) { next; } # Ignore option headers + my $size = 0; + foreach my $id (@{ $$spec{'ids'} }) { + my $len = length $id; + if ($len == 1) { + $size += 3; + } else { + $size += 3 + $len; + } + } + if ($size > $max) { $max = $size; } + } + return $max; +} +sub writeOptionId { # (Spec, MaxNameLength) + my ($spec, $maxidlen) = @_; + my $totalLength = 0; + foreach my $id (@{ $$spec{'ids'} }) { + my $len = length $id; + if ($len == 1) { + printResult " -" . $id; + $totalLength += 3; + } else { + printResult " --" . $id; + $totalLength += 3 + $len; + } + } + if ($totalLength <= $maxidlen) { + for (my $i=$totalLength; $i<$maxidlen; ++$i) { + printResult ' '; + } + } else { + printResult "\n"; + for (my $i=0; $i<$maxidlen; ++$i) { + printResult ' '; + } + } + printResult " : "; +} +sub writeOptionDescription { # (Spec, MaxNameLength) + my ($spec, $maxidlen) = @_; + my $width = ConsoleOutput::getTerminalWidth() - $maxidlen; + my $desc = $$spec{'description'}; + my $min = int ($width / 2); + while (length $desc > $width) { + if ($desc =~ /^(.{$min,$width}) (.*)$/s) { + my ($first, $rest) = ($1, $2); + printResult $first . "\n"; + for (my $i=0; $i<$maxidlen; ++$i) { + printResult ' '; + } + $desc = $rest; + } else { + last; + } + } + printResult $desc . "\n"; +} +sub writeOptionHeader { # (Description) + my ($desc) = @_; + my $width = ConsoleOutput::getTerminalWidth(); + my $min = 2 * $width / 3; + while (length $desc > $width) { + if ($desc =~ /^(.{$min,$width}) (.*)$/s) { + my ($first, $rest) = ($1, $2); + printResult $first . "\n"; + $desc = $rest; + } else { + last; + } + } + printResult $desc . "\n"; +} +sub validHost { # (Hostname) + my ($host) = @_; + if ($host !~ /^[a-zA-Z][-_a-zA-Z0-9\.]*$/) { + return 0; + } + if (system("host $host >/dev/null 2>/dev/null") != 0) { + return 0; + } + return 1; +} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/GetClusterState.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/GetClusterState.pm new file mode 100644 index 00000000000..13d645d46de --- /dev/null +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/GetClusterState.pm @@ -0,0 +1,124 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package Yahoo::Vespa::Bin::GetClusterState; + +use strict; +use warnings; +use Yahoo::Vespa::ArgParser; +use Yahoo::Vespa::ClusterController; +use Yahoo::Vespa::ConsoleOutput; +use Yahoo::Vespa::ContentNodeSelection; +use Yahoo::Vespa::Utils; +use Yahoo::Vespa::VespaModel; + +BEGIN { + use base 'Exporter'; + our @EXPORT = qw( + getClusterState + ); +} + +my %cluster_states; + +return &init(); + +sub init { + %cluster_states = (); + return 1; +} + +# Run the get node state tool +sub getClusterState { # (Command line arguments) + my ($argsref) = @_; + &handleCommandLine($argsref); + detectClusterController(); + &showSettings(); + &showNodeStates(); +} + +# Parse command line arguments +sub handleCommandLine { # (Command line arguments) + my ($args) = @_; + my $description = <<EOS; +Get the cluster state of a given cluster. + +EOS + $description =~ s/(\S)\n(\S)/$1 $2/gs; + chomp $description; + + setProgramDescription($description); + Yahoo::Vespa::ContentNodeSelection::registerCommandLineArguments( + NO_LOCALHOST_CONSTRAINT | CLUSTER_ONLY_LIMITATION); + Yahoo::Vespa::VespaModel::registerCommandLineArguments(); + handleCommandLineArguments($args); +} + +# Show what settings this tool is running with (if verbosity is high enough) +sub showSettings { # () + &Yahoo::Vespa::ClusterController::showSettings(); +} + +# Print all state we want to show for this request +sub showNodeStates { # () + + Yahoo::Vespa::ContentNodeSelection::visit(\&showNodeStateForNode); +} + +# Get the node state from cluster controller, unless already cached +sub getStateForNode { # (Type, Index, Cluster) + my ($type, $index, $cluster) = @_; + if (!exists $cluster_states{$cluster}) { + my $state = getContentClusterState($cluster); + $cluster_states{$cluster} = $state; + if ($state->globalState eq "up") { + printResult "\nCluster $cluster:\n"; + } else { + printResult "\nCluster $cluster is " . COLOR_ERR + . $state->globalState . COLOR_RESET + . ". Too few nodes available.\n"; + } + } + return $cluster_states{$cluster}->$type->{$index}; +} + +# Print all states for a given node +sub showNodeStateForNode { # (Service, Index, NodeState, Model, ClusterName) + my ($info) = @_; + my ($cluster, $type, $index) = ( + $$info{'cluster'}, $$info{'type'}, $$info{'index'}); + my $nodestate = &getStateForNode($type, $index, $cluster); + defined $nodestate or confess "No nodestate for $type $index $cluster"; + my $generated = $nodestate->generated; + my $id = $cluster . "/"; + if (defined $nodestate->group) { + $id .= $nodestate->group; + } + my $msg = "$cluster/$type/$index: "; + if ($generated->state ne 'up') { + $msg .= COLOR_ERR; + } + $msg .= $generated->state; + if ($generated->state ne 'up') { + $msg .= COLOR_RESET; + } + # TODO: Make the Cluster Controller always populate the reason for the + # generated state. Until then we'll avoid printing it to avoid confusion. + # Use vespa-get-node-state to see the reasons on generated, user, and unit. + # + # if (length $generated->reason > 0) { + # $msg .= ': ' . $generated->reason; + # } + printResult $msg . "\n"; +} + +# ClusterState(Version: 7, Cluster state: Up, Distribution bits: 1) { +# Group 0: mygroup. 1 node [0] { +# All nodes in group up and available. +# } +# } + +# ClusterState(Version: 7, Cluster state: Up, Distribution bits: 1) { +# Group 0: mygroup. 1 node [0] { +# storage.0: Retired: foo +# } +# } diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/GetNodeState.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/GetNodeState.pm new file mode 100644 index 00000000000..1e82c05db0a --- /dev/null +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/GetNodeState.pm @@ -0,0 +1,119 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package Yahoo::Vespa::Bin::GetNodeState; + +use strict; +use warnings; +use Yahoo::Vespa::ArgParser; +use Yahoo::Vespa::ClusterController; +use Yahoo::Vespa::ConsoleOutput; +use Yahoo::Vespa::ContentNodeSelection; +use Yahoo::Vespa::Utils; + +BEGIN { + use base 'Exporter'; + our @EXPORT = qw( + getNodeState + ); +} + +our $resultdesc; +our %cluster_states; + +return 1; + +# Run the get node state tool +sub getNodeState { # (Command line arguments) + my ($argsref) = @_; + &handleCommandLine($argsref); + detectClusterController(); + &showSettings(); + &showNodeStates(); +} + +# Parse command line arguments +sub handleCommandLine { # (Command line arguments) + my ($args) = @_; + $resultdesc = <<EOS; +Shows the various states of one or more nodes in a Vespa Storage cluster. +There exist three different type of node states. They are: + + Unit state - The state of the node seen from the cluster controller. + User state - The state we want the node to be in. By default up. Can be + set by administrators or by cluster controller when it + detects nodes that are behaving badly. + Generated state - The state of a given node in the current cluster state. + This is the state all the other nodes know about. This + state is a product of the other two states and cluster + controller logic to keep the cluster stable. +EOS + $resultdesc =~ s/\s*\n(\S.)/ $1/gs; + chomp $resultdesc; + my $description = <<EOS; +Retrieve the state of one or more storage services from the fleet controller. +Will list the state of the locally running services, possibly restricted to +less by options. + +$resultdesc + +EOS + $description =~ s/(\S)\n(\S)/$1 $2/gs; + chomp $description; + + setProgramDescription($description); + Yahoo::Vespa::ContentNodeSelection::registerCommandLineArguments(); + Yahoo::Vespa::VespaModel::registerCommandLineArguments(); + handleCommandLineArguments($args); +} + +# Show what settings this tool is running with (if verbosity is high enough) +sub showSettings { # () + &Yahoo::Vespa::ClusterController::showSettings(); + &Yahoo::Vespa::ContentNodeSelection::showSettings(); +} + +# Print all state we want to show for this request +sub showNodeStates { # () + printInfo $resultdesc . "\n"; + Yahoo::Vespa::ContentNodeSelection::visit(\&showNodeStateForNode); +} + +# Get the node state from cluster controller, unless already cached +sub getStateForNode { # (Type, Index, Cluster) + my ($type, $index, $cluster) = @_; + if (!exists $cluster_states{$cluster}) { + $cluster_states{$cluster} = getContentClusterState($cluster); + } + return $cluster_states{$cluster}->$type->{$index}; +} + +# Print all states for a given node +sub showNodeStateForNode { # (Service, Index, NodeState, Model, ClusterName) + my ($info) = @_; + my ($cluster, $type, $index) = ( + $$info{'cluster'}, $$info{'type'}, $$info{'index'}); + printResult "\n$cluster/$type.$index:\n"; + my $nodestate = &getStateForNode($type, $index, $cluster); + printState('Unit', $nodestate->unit); + printState('Generated', $nodestate->generated); + printState('User', $nodestate->user); +} + +# Print the value of a single state type for a node +sub printState { # (State name, State) + my ($name, $state) = @_; + if (!defined $state) { + printResult $name . ": UNKNOWN\n"; + } else { + my $msg = $name . ": "; + if ($state->state ne 'up') { + $msg .= COLOR_ERR; + } + $msg .= $state->state; + if ($state->state ne 'up') { + $msg .= COLOR_RESET; + } + $msg .= ": " . $state->reason . "\n"; + printResult $msg; + } +} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/SetNodeState.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/SetNodeState.pm new file mode 100644 index 00000000000..bdf276c3677 --- /dev/null +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/SetNodeState.pm @@ -0,0 +1,97 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package Yahoo::Vespa::Bin::SetNodeState; + +use strict; +use warnings; +use Yahoo::Vespa::ClusterController; +use Yahoo::Vespa::ConsoleOutput; +use Yahoo::Vespa::ContentNodeSelection; +use Yahoo::Vespa::ArgParser; +use Yahoo::Vespa::Utils; + +BEGIN { + use base 'Exporter'; + our @EXPORT = qw( + setNodeState + ); +} + +our $wanted_state; +our $wanted_state_description; +our $nodes_attempted_set; +our $success; + +return 1; + +# Run the set node state tool +sub setNodeState { # (Command line arguments) + my ($argsref) = @_; + &handleCommandLine($argsref); + detectClusterController(); + &showSettings(); + &execute(); +} + +# Parse command line arguments +sub handleCommandLine { # (Command line arguments) + my ($args) = @_; + my $description = <<EOS; +Set the user state of a node. This will set the generated state to the user +state if the user state is "better" than the generated state that would have +been created if the user state was up. For instance, a node that is currently +in initializing state can be forced into down state, while a node that is +currently down can not be forced into retired state, but can be forced into +maintenance state. +EOS + $description =~ s/(\S)\n(\S)/$1 $2/gs; + chomp $description; + + setProgramDescription($description); + + setArgument(\$wanted_state, "Wanted State", + "User state to set. This must be one of " + . "up, down, maintenance or retired.", + OPTION_REQUIRED); + setArgument(\$wanted_state_description, "Description", + "Give a reason for why you are altering the user state, which " + . "will show up in various admin tools. (Use double quotes to " + . "give a reason with whitespace in it)"); + + Yahoo::Vespa::ContentNodeSelection::registerCommandLineArguments(); + Yahoo::Vespa::VespaModel::registerCommandLineArguments(); + handleCommandLineArguments($args); + + if (!Yahoo::Vespa::ContentNodeSelection::validateCommandLineArguments( + $wanted_state)) { + exitApplication(1); + } +} + +# Show what settings this tool is running with (if verbosity is high enough) +sub showSettings { # () + Yahoo::Vespa::ClusterController::showSettings(); +} + +# Sets the node state +sub execute { # () + $success = 1; + $nodes_attempted_set = 0; + Yahoo::Vespa::ContentNodeSelection::visit(\&setNodeStateForNode); + if ($nodes_attempted_set == 0) { + printWarning("Attempted setting of user state for no nodes"); + exitApplication(1); + } + if (!$success) { + exitApplication(1); + } +} + +sub setNodeStateForNode { + my ($info) = @_; + my ($cluster, $type, $index) = ( + $$info{'cluster'}, $$info{'type'}, $$info{'index'}); + $success &&= setNodeUserState($cluster, $type, $index, $wanted_state, + $wanted_state_description); + ++$nodes_attempted_set; +} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/ClusterController.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/ClusterController.pm new file mode 100644 index 00000000000..cbe6deea9e4 --- /dev/null +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/ClusterController.pm @@ -0,0 +1,273 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Handles Rest API requests to State Rest API in cluster controller, making +# wanted data programmatically available. +# +package Yahoo::Vespa::ClusterController; + +use strict; +use warnings; +use Class::Struct; +use Yahoo::Vespa::ArgParser; +use Yahoo::Vespa::ClusterState; +use Yahoo::Vespa::ConsoleOutput; +use Yahoo::Vespa::Http; +use Yahoo::Vespa::Json; +use Yahoo::Vespa::Utils; +use Yahoo::Vespa::VespaModel; + +BEGIN { # - Exports and aliases for the module + use base 'Exporter'; + our $VERSION = '1.0'; + our @EXPORT = qw( + detectClusterController + getContentClusterState + setNodeUserState + ); # Exported unless specifically left out by user + # Alias namespaces + *VespaModel:: = *Yahoo::Vespa::VespaModel:: ; + *Http:: = *Yahoo::Vespa::Http:: ; + *Json:: = *Yahoo::Vespa::Json:: ; +} + +struct( ClusterController => { + index => '$', # Logical index of the cluster controller + host => '$', # Host on which cluster controller runs + port => '$' # Port where cluster controller is available +}); + +my %CACHED_CLUSTER_STATES; +my @CLUSTER_CONTROLLERS; + +return &init(); + +########################## Default exported functions ######################## + +sub init { + %CACHED_CLUSTER_STATES = (); + @CLUSTER_CONTROLLERS = (); + return 1; +} + +sub detectClusterController { # () + if (scalar @CLUSTER_CONTROLLERS == 0) { + use Yahoo::Vespa::VespaModel; + printDebug "Attempting to auto-detect cluster controller location\n"; + my $sockets = VespaModel::getSocketForService( + type => 'container-clustercontroller', tag => 'state'); + foreach my $sock (sort { $a->{'index'} <=> $b->{'index'} } @$sockets) { + my $cc = new ClusterController; + $cc->index($sock->{'index'}); + $cc->host($sock->{'host'}); + $cc->port($sock->{'port'}); + push @CLUSTER_CONTROLLERS, $cc; + } + if (scalar @$sockets == 0) { + my $oldVal = enableAutomaticLineBreaks(0); + printSpam dumpStructure(VespaModel::get()); + enableAutomaticLineBreaks($oldVal); + printError "Failed to detect cluster controller to talk to. " + . "Resolve issue that failed automatic detection or " + . "provide cluster controller socket through command " + . "line options. (See --help)\n"; + exitApplication(1); + } + &showSettings(); + printSpam "Content of vespa model inspected to find cluster " + . "controller:\n"; + my $oldVal = enableAutomaticLineBreaks(0); + printSpam dumpStructure(VespaModel::get()); + enableAutomaticLineBreaks($oldVal); + } +} +sub setNodeUserState { # (ClusterName, NodeType, Index, State, Reason) + my ($cluster, $service, $index, $state, $reason) = @_; + my @params = (); + my @headers = ( + 'Content-Type' => 'application/json' + ); + $state =~ tr/A-Z/a-z/; + $state =~ /(?:up|down|maintenance|retired)$/ + or confess "Invalid state '$state' attempted set.\n"; + if (!defined $reason) { + $reason = ""; + } + my $request = { + "state" => { + "user" => { + "state" => $state, + "reason" => $reason + } + } + }; + my $content = Json::encode($request); + + my $path = &getPathToNode($cluster, $service, $index); + my %response = &requestCC('POST', $path, \@params, $content, \@headers); + if (defined $response{'all'}) { printSpam $response{'all'}; } + printDebug $response{'code'} . " " . $response{'status'} . "\n"; + printInfo exists($response{'content'}) ? $response{'content'} : ''; + if ($response{'code'} >= 200 && $response{'code'} < 300) { + printResult "$response{'status'}\n"; + return 1 + } else { + printWarning "Failed to set node state for node " + . "$cluster/$service/$index: " + . "$response{'code'} $response{'status'}\n"; + return 0 + } +} +sub getContentClusterState { # (ClusterName) -> ClusterState + my ($cluster) = @_; + if (!exists $CACHED_CLUSTER_STATES{$cluster}) { + $CACHED_CLUSTER_STATES{$cluster} = &fetchContentClusterState($cluster); + } + return $CACHED_CLUSTER_STATES{$cluster}; +} + +######################## Externally usable functions ####################### + +sub getClusterControllers { # () + return \@CLUSTER_CONTROLLERS; +} +sub showSettings { # () + printDebug "Cluster controllers:\n"; + foreach my $cc (@CLUSTER_CONTROLLERS) { + printDebug " " . $cc->index . ": " + . $cc->host . ":" . $cc->port . "\n"; + } +} + +############## Utility functions - Not intended for external use ############# + +sub fetchContentClusterState { # (ClusterName) -> ClusterState + my ($cluster) = @_; + my @params = ( + 'recursive' => 'true' + ); + my %response = &getCC("/cluster/v2/$cluster/", \@params); + if ($response{'code'} != 200) { + printError "Failed to fetch cluster state of content cluster " + . "'$cluster':\n" . $response{'all'} . "\n"; + exitApplication(1); + } + my $json = Json::parse($response{'content'}); + my $result = new ClusterState; + &fillInGlobalState($cluster, $result, $json); + &fillInNodes($result, 'distributor', + &getJsonValue($json, ['service', 'distributor', 'node'])); + &fillInNodes($result, 'storage', + &getJsonValue($json, ['service', 'storage', 'node'])); + return $result; +} +sub fillInGlobalState { # (ClusterName, StateToFillIn, JsonToParse) + my ($cluster, $state, $json) = @_; + my $e = &getJsonValue($json, ['state', 'generated', 'state']); + if (defined $e) { + $state->globalState($e); + if (!Yahoo::Vespa::ClusterState::legalState($state->globalState())) { + printWarning "Illegal global cluster state $e found.\n"; + } + } else { + printDebug dumpStructure($json) . "\n"; + printWarning "Found no global cluster state\n"; + } +} +sub getPathToNode { # (ClusterName, NodeType, Index) + my ($cluster, $service, $index) = @_; + return "/cluster/v2/$cluster/$service/$index"; +} +sub listContentClusters { # () -> (ContentClusterName, ...) + my %result = &getCC("/cluster/v2/"); + if ($result{'code'} != 200) { + printError "Failed to fetch list of content clusters:\n" + . $result{'all'} . "\n"; + exitApplication(1); + } + my $json = Json::parse($result{'content'}); + return keys %{ $json->{'cluster'} }; +} +sub fillInNodes { # (StateToFillIn, ServiceType, json) + my ($state, $service, $json) = @_; + foreach my $index (%{ $json }) { + my $node = new Node; + &parseNode($node, $json->{$index}); + $state->$service($index, $node); + } +} +sub parseNode { # (StateToFillIn, JsonToParse) + my ($node, $json) = @_; + my $group = &getJsonValue($json, ['attributes', 'hierarchical-group']); + if (defined $group && $group =~ /^[^\.]*\.(.*)$/) { + $node->group($1); + } + parseState($node, $json, 'unit'); + parseState($node, $json, 'generated'); + parseState($node, $json, 'user'); + my $partitions = $json->{'partition'}; + if (defined $partitions) { + foreach my $index (%{ $json->{'partition'} }) { + my $partition = new Partition; + parsePartition($partition, $json->{'partition'}->{$index}); + $node->partition($index, $partition); + } + } +} +sub parsePartition { # (StateToFillIn, JsonToParse) + my ($partition, $json) = @_; + my $buckets = &getJsonValue($json, ['metrics', 'bucket-count']); + my $doccount = &getJsonValue($json, ['metrics', 'unique-document-count']); + my $size = &getJsonValue($json, ['metrics', 'unique-document-total-size']); + $partition->bucketcount($buckets); + $partition->doccount($doccount); + $partition->totaldocsize($size); +} +sub parseState { # (StateToFillIn, JsonToParse, StateType) + my ($node, $json, $type) = @_; + my $value = &getJsonValue($json, ['state', $type, 'state']); + my $reason = &getJsonValue($json, ['state', $type, 'reason']); + if (defined $value) { + my $state = new State; + $state->state($value); + $state->reason($reason); + $state->source($type); + $node->$type($state); + } +} +sub getJsonValue { # (json, [ keys ]) + my ($json, $keys) = @_; + foreach my $key (@$keys) { + if (!defined $json) { return; } + $json = $json->{$key}; + } + return $json; +} +sub getCC { # (Path, Params, Headers) -> Response + my ($path, $params, $headers) = @_; + return requestCC('GET', $path, $params, undef, $headers); +} +sub requestCC { # (Type, Path, Params, Content, Headers) -> Response + my ($type, $path, $params, $content, $headers) = @_; + my %response; + foreach my $cc (@CLUSTER_CONTROLLERS) { + %response = Http::request($type, $cc->host, $cc->port, $path, + $params, $content, $headers); + if ($response{'code'} == 200) { + return %response; + } elsif ($response{'code'} == 307) { + my %headers = $response{'headers'}; + my $masterlocation = $headers{'Location'}; + if (defined $masterlocation) { + if ($masterlocation =~ /http:\/\/([^\/:]+):(\d+)\//) { + my ($host, $port) = ($1, $2); + return Http::request($type, $host, $port, $path, + $params, $content, $headers); + } else { + printError("Unhandled relocaiton URI '$masterlocation'."); + exitApplication(1); + } + } + } + } + return %response; +} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/ClusterState.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/ClusterState.pm new file mode 100644 index 00000000000..648f158f9db --- /dev/null +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/ClusterState.pm @@ -0,0 +1,45 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Defines structs to represent a cluster state +# +package Yahoo::Vespa::ClusterState; + +use strict; +use warnings; +use Class::Struct; + +struct( ClusterState => { + globalState => '$', # A state primitive + distributor => '%', # Index to Node map + storage => '%' # Index to Node map +}); + +struct( Node => { + group => '$', # Hierarchical group node belongs to + unit => 'State', + generated => 'State', + user => 'State', + partition => '%' +}); + +struct( Partition => { + generated => 'State', + bucketcount => '$', + doccount => '$', + totaldocsize => '$' +}); + +struct( State => { + state => '$', # A state primitive + reason => '$', # Textual reason for it to be set. + timestamp => '$', # Timestamp of the time it got set. + source => '$' # What type of state is it (unit/generated/user) +}); + +return 1; + +sub legalState { # (State) -> Bool + my ($state) = @_; + return ($state =~ /^(up|down|maintenance|retired|stopping|initializing)$/); +} + diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/ConsoleOutput.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/ConsoleOutput.pm new file mode 100644 index 00000000000..73a0a016592 --- /dev/null +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/ConsoleOutput.pm @@ -0,0 +1,331 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Output handler +# +# Intentions: +# - Make it easy for unit tests to redirect output. +# - Allow programmers to add all sorts of debug information into tools usable +# for debugging, while hiding it by default for real users. +# - Allow generic functionality that can be reused by all. For instance color +# coding of very important information. +# +# Ideas for improvement: +# - Could possibly detect terminal width and do proper line breaking of long +# lines +# +# A note about colors: +# - This module will detect if terminal supports colors. If not, it will not +# print any. (Color support can be turned off by giving --nocolors argument +# through argument parser, by setting a TERM value that does not support +# colors or programmatically call setUseAnsiColors(0). +# - Currently only red and grey are used in addition to default. These colors +# should work well for both light and dark backgrounds. +# + +package Yahoo::Vespa::ConsoleOutput; + +use strict; +use warnings; +use Yahoo::Vespa::Utils; + +BEGIN { # - Define exports for modul + use base 'Exporter'; + our @EXPORT = qw( + printResult printError printWarning printInfo printDebug printSpam + enableAutomaticLineBreaks + COLOR_RESET COLOR_WARN COLOR_ERR COLOR_ANON + ); + our @EXPORT_OK = qw( + getTerminalWidth getVerbosity usingAnsiColors ansiColorsSupported + setVerbosity + ); +} + +my %TYPES = ( + 'result' => 0, # Output from a tool. Expected when app runs successfully. + 'error' => 1, # Error found, typically aborting the script with a failure. + 'warning' => 2, # An issue that may or may not cause the program to fail. + 'info' => 3, # Useful information to get from the script. + 'debug' => 4, # Debug information useful to debug script or to see + # internals of what is happening. + 'spam' => 5, # Spammy information used when large amounts of details is + # wanted. Typically to debug some failure. +); +my $VERBOSITY; # Current verbosity level +my $ANSI_COLORS_SUPPORTED; # True if terminal supports colors +my $ANSI_COLORS; # True if we want to use colors (and support it) +my %ATTRIBUTE_PREFIX; # Ansi escape prefixes for verbosity levels +my %ATTRIBUTE_POSTFIX; # Ansi escape postfixes for verbosity levels +my %OUTPUT_STREAM; # Where to write different verbosity levels (stdout|stderr) +my $TERMINAL_WIDTH; # With of terminal in columns +my $COLUMN_POSITION; # Current index of cursor in terminal +my $ENABLE_AUTO_LINE_BREAKS; + +use constant COLOR_RESET => "\e[0m"; +use constant COLOR_ERR => "\e[91m"; +use constant COLOR_WARN => "\e[93m"; +use constant COLOR_ANON => "\e[90m"; + +&initialize(*STDOUT, *STDERR); + +return 1; + +########################## Default exported functions ######################## + +sub printResult { # (Output...) + printAtLevel('result', @_); +} +sub printError { # (Output...) + printAtLevel('error', @_); +} +sub printWarning { # (Output...) + printAtLevel('warning', @_); +} +sub printInfo { # (Output...) + printAtLevel('info', @_); +} +sub printDebug { # (Output...) + printAtLevel('debug', @_); +} +sub printSpam { # (Output...) + printAtLevel('spam', @_); +} +sub enableAutomaticLineBreaks { # (Bool) -> (OldValue) + my $oldval = $ENABLE_AUTO_LINE_BREAKS; + $ENABLE_AUTO_LINE_BREAKS = ($_[0] ? 1 : 0); + return $oldval; +} + +######################## Optionally exported functions ####################### + +sub getTerminalWidth { # () -> ColumnCount + # May be undefined if someone prints before initialized + return (defined $TERMINAL_WIDTH ? $TERMINAL_WIDTH : 80); +} +sub getVerbosity { # () -> VerbosityLevel + return $VERBOSITY; +} +sub usingAnsiColors { # () -> Bool + return $ANSI_COLORS; +} +sub ansiColorsSupported { # () -> Bool + return $ANSI_COLORS_SUPPORTED; +} +sub setVerbosity { # (VerbosityLevel) + $VERBOSITY = $_[0]; +} + +################## Functions for unit tests to mock internals ################ + +sub setTerminalWidth { # (ColumnCount) + $TERMINAL_WIDTH = $_[0]; +} +sub setUseAnsiColors { # (Bool) + if ($ANSI_COLORS_SUPPORTED && $_[0]) { + $ANSI_COLORS = 1; + } else { + $ANSI_COLORS = 0; + } +} + +############## Utility functions - Not intended for external use ############# + +sub initialize { # () + my ($stdout, $stderr, $use_colors_by_default) = @_; + if (!defined $VERBOSITY) { + $VERBOSITY = &getDefaultVerbosity(); + } + $COLUMN_POSITION = 0; + $ENABLE_AUTO_LINE_BREAKS = 1; + %ATTRIBUTE_PREFIX = map { $_ => '' } keys %TYPES; + %ATTRIBUTE_POSTFIX = map { $_ => '' } keys %TYPES; + &setAttribute('error', COLOR_ERR, COLOR_RESET); + &setAttribute('warning', COLOR_WARN, COLOR_RESET); + &setAttribute('debug', COLOR_ANON, COLOR_RESET); + &setAttribute('spam', COLOR_ANON, COLOR_RESET); + %OUTPUT_STREAM = map { $_ => $stdout } keys %TYPES; + $OUTPUT_STREAM{'error'} = $stderr; + $OUTPUT_STREAM{'warning'} = $stderr; + if (defined $use_colors_by_default) { + $ANSI_COLORS_SUPPORTED = $use_colors_by_default; + $ANSI_COLORS = $ANSI_COLORS_SUPPORTED; + } else { + &detectTerminalColorSupport(); + } + if (!defined $TERMINAL_WIDTH) { + $TERMINAL_WIDTH = &detectTerminalWidth(); + } +} +sub setAttribute { # (type, prefox, postfix) + my ($type, $prefix, $postfix) = @_; + $ATTRIBUTE_PREFIX{$type} = $prefix; + $ATTRIBUTE_POSTFIX{$type} = $postfix; +} +sub stripAnsiEscapes { # (Line) -> (StrippedLine) + $_[0] =~ s/\e\[[^m]*m//g; + return $_[0]; +} +sub getDefaultVerbosity { # () -> VerbosityLevel + # We can not print at correct verbosity levels before argument parsing has + # completed. We try some simple arg parsing here assuming default options + # used to set verbosity, such that we likely guess correctly, allowing + # correct verbosity from the start. + my $default = 3; + foreach my $arg (@ARGV) { + if ($arg eq '--') { return $default; } + if ($arg =~ /^-([^-]+)/) { + my $optstring = $1; + while ($optstring =~ /^(.)(.*)$/) { + my $char = $1; + $optstring = $2; + if ($char eq 'v') { + ++$default; + } + if ($char eq 's') { + if ($default > 0) { + --$default; + } + } + } + } + } + return $default; +} +sub detectTerminalWidth { #() -> ColumnCount + my $cols = &checkConsoleFeature('cols'); + if (!defined $cols) { + printDebug "Assuming terminal width of 80.\n"; + return 80; + } + if ($cols =~ /^\d+$/ && $cols > 10 && $cols < 500) { + printDebug "Detected terminal width of $cols.\n"; + return $cols; + } else { + printDebug "Unexpected terminal width of '$cols' given. " + . "Assuming size of 80.\n"; + return 80; + } +} +sub detectTerminalColorSupport { # () -> Bool + my $colorcount = &checkConsoleFeature('colors'); + if (!defined $colorcount) { + $ANSI_COLORS_SUPPORTED = 0; + printDebug "Assuming no color support.\n"; + return 0; + } + if ($colorcount =~ /^\d+$/ && $colorcount >= 8) { + $ANSI_COLORS_SUPPORTED = 1; + if (!defined $ANSI_COLORS) { + $ANSI_COLORS = $ANSI_COLORS_SUPPORTED; + } + printDebug "Color support detected.\n"; + return 1; + } +} +sub checkConsoleFeature { # (Feature) -> Bool + my ($feature) = @_; + # Unit tests must mock. Can't depend on TERM being set. + assertNotUnitTest(); + if (!exists $ENV{'TERM'}) { + printDebug "Terminal not set. Unknown.\n"; + return; + } + if (-f '/usr/bin/tput') { + my ($fh, $result); + if (open ($fh, "tput $feature 2>/dev/null |")) { + $result = <$fh>; + close $fh; + } else { + printDebug "Failed to open tput pipe.\n"; + return; + } + if ($? != 0) { + printDebug "Failed tput call to detect feature $feature $!\n"; + return; + } + chomp $result; + #printSpam "Console feature $feature: '$result'\n"; + return $result; + } else { + printDebug "No tput binary. Dont know how to detect feature.\n"; + return; + } +} +sub printAtLevel { # (Level, Output...) + # Prints an array of data that may contain newlines + my $level = shift @_; + exists $TYPES{$level} or confess "Unknown print level '$level'."; + if ($TYPES{$level} > $VERBOSITY) { + return; + } + my $buffer = ''; + my $width = &getTerminalWidth(); + foreach my $printable (@_) { + my @lines = split(/\n/, $printable, -1); + my $current = 0; + for (my $i=0; $i < scalar @lines; ++$i) { + if ($i != 0) { + $buffer .= "\n"; + $COLUMN_POSITION = 0; + } + my $last = ($i + 1 == scalar @lines); + printLineAtLevel($level, $lines[$i], \$buffer, $last); + } + } + my $stream = $OUTPUT_STREAM{$level}; + print $stream $buffer; +} +sub printLineAtLevel { # (Level, Line, Buffer, Last) + # Prints a single line, which might still have to be broken into multiple + # lines + my ($level, $data, $buffer, $last) = @_; + if (!$ANSI_COLORS) { + $data = &stripAnsiEscapes($data); + } + my $width = &getTerminalWidth(); + while (1) { + my $remaining = $width - $COLUMN_POSITION; + if (&prefixLineWithLevel($level)) { + $remaining -= (2 + length $level); + } + if ($ENABLE_AUTO_LINE_BREAKS && $remaining < length $data) { + my $min = int (2 * $width / 3) - $COLUMN_POSITION; + if ($min < 1) { $min = 1; } + if ($data =~ /^(.{$min,$remaining}) (.*?)$/s) { + my ($first, $rest) = ($1, $2); + &printLinePartAtLevel($level, $first, $buffer); + $$buffer .= "\n"; + $data = $rest; + $COLUMN_POSITION = 0; + } else { + last; + } + } else { + last; + } + } + if (!$last || length $data > 0) { + &printLinePartAtLevel($level, $data, $buffer); + } +} +sub printLinePartAtLevel { # ($Level, Line, Buffer) + # Print a single line that should fit on one line + my ($level, $data, $buffer) = @_; + if ($ANSI_COLORS) { + $$buffer .= $ATTRIBUTE_PREFIX{$level}; + } + if (&prefixLineWithLevel($level)) { + $$buffer .= $level . ": "; + $COLUMN_POSITION = (length $level) + 2; + } + $$buffer .= $data; + $COLUMN_POSITION += length $data; + if ($ANSI_COLORS) { + $$buffer .= $ATTRIBUTE_POSTFIX{$level}; + } +} +sub prefixLineWithLevel { # (Level) -> Bool + my ($level) = @_; + return ($TYPES{$level} > 2 && $VERBOSITY >= 4 && $COLUMN_POSITION == 0); +} + diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/ContentNodeSelection.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/ContentNodeSelection.pm new file mode 100644 index 00000000000..f5507ce478e --- /dev/null +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/ContentNodeSelection.pm @@ -0,0 +1,141 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# This module implements a way to select a subset of nodes from a Vespa +# application. +# + +package Yahoo::Vespa::ContentNodeSelection; + +use strict; +use warnings; +use Yahoo::Vespa::ArgParser; +use Yahoo::Vespa::ConsoleOutput; +use Yahoo::Vespa::Utils; +use Yahoo::Vespa::VespaModel; + +BEGIN { # - Declare exports and dependency aliases for module + use base 'Exporter'; + our @EXPORT = qw( + NO_LOCALHOST_CONSTRAINT + CLUSTER_ONLY_LIMITATION + ); + # Package aliases + *VespaModel:: = *Yahoo::Vespa::VespaModel:: ; +} + +my $CLUSTER; +my $NODE_TYPE; +my $INDEX; +my $FORCE = 0; +our $LOCALHOST; + +use constant NO_LOCALHOST_CONSTRAINT => 1; +use constant CLUSTER_ONLY_LIMITATION => 2; + +return 1; + +######################## Externally usable functions ####################### + +sub registerCommandLineArguments { # (Flags) + my ($flags) = @_; + if (!defined $flags) { $flags = 0; } + if (($flags & NO_LOCALHOST_CONSTRAINT) == 0) { + $LOCALHOST = getHostname(); + } else { + $LOCALHOST = undef; + } + if (($flags & CLUSTER_ONLY_LIMITATION) == 0) { + setOptionHeader("Node selection options. By default, nodes running " + . "locally will be selected:"); + } + setStringOption( + ['c', 'cluster'], + \$CLUSTER, + 'Cluster name of cluster to query. ' + . 'If unspecified, and vespa is installed on current node, ' + . 'information will be attempted auto-extracted'); + setFlagOption( + ['f', 'force'], + \$FORCE, + 'Force the execution of a dangerous command.'); + if (($flags & CLUSTER_ONLY_LIMITATION) == 0) { + setStringOption( + ['t', 'type'], + \$NODE_TYPE, + 'Node type to query. This can either be \'storage\' or ' + . '\'distributor\'. If not specified, the operation will show ' + . 'state for all types.'); + setIntegerOption( + ['i', 'index'], + \$INDEX, + 'The node index to show state for. If not specified, all nodes ' + . 'found running on this host will be shown.'); + } +} +sub visit { # (Callback) + my ($callback) = @_; + printDebug "Visiting selected services: " + . "Cluster " . (defined $CLUSTER ? $CLUSTER : 'undef') + . " node type " . (defined $NODE_TYPE ? $NODE_TYPE : 'undef') + . " index " . (defined $INDEX ? $INDEX : 'undef') + . " localhost only ? " . ($LOCALHOST ? "true" : "false") . "\n"; + VespaModel::visitServices(sub { + my ($info) = @_; + $$info{'type'} = &convertType($$info{'type'}); + if (!&validType($$info{'type'})) { return; } + if (defined $CLUSTER && $CLUSTER ne $$info{'cluster'}) { return; } + if (defined $NODE_TYPE && $NODE_TYPE ne $$info{'type'}) { return; } + if (defined $INDEX && $INDEX ne $$info{'index'}) { return; } + if (!defined $INDEX && defined $LOCALHOST + && $LOCALHOST ne $$info{'host'}) + { + return; + } + # printResult "Ok $$info{'cluster'} $$info{'type'} $$info{'index'}\n"; + &$callback($info); + }); +} +sub showSettings { # () + printDebug "Visiting selected services: " + . "Cluster " . (defined $CLUSTER ? $CLUSTER : 'undef') + . " node type " . (defined $NODE_TYPE ? $NODE_TYPE : 'undef') + . " index " . (defined $INDEX ? $INDEX : 'undef') + . " localhost only ? " . ($LOCALHOST ? "true" : "false") . "\n"; +} + +sub validateCommandLineArguments { # (WantedState) + my ($wanted_state) = @_; + + if (defined $NODE_TYPE) { + if ($NODE_TYPE !~ /^(distributor|storage)$/) { + printWarning "Invalid value '$NODE_TYPE' given for node type.\n"; + return 0; + } + } + + if (!$FORCE && + (!defined $NODE_TYPE || $NODE_TYPE eq "distributor") && + $wanted_state eq "maintenance") { + printWarning "Setting the distributor to maintenance mode may have " + . "severe consequences for feeding!\n" + . "Please specify -t storage to only set the storage node to " + . "maintenance mode, or -f to override this error.\n"; + return 0; + } + + printDebug "Command line arguments validates ok\n"; + return 1; +} + +############## Utility functions - Not intended for external use ############# + +sub validType { # (ServiceType) -> Bool + my ($type) = @_; + return $type =~ /^(?:distributor|storage)$/; +} +sub convertType { # (ServiceType) -> Bool + my ($type) = @_; + if ($type eq 'storagenode') { return 'storage'; } + return $type; +} + diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm new file mode 100644 index 00000000000..8e25442a64d --- /dev/null +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm @@ -0,0 +1,160 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Simple HTTP wrapper library +# +# Intentions: +# - Make it very easy for programs to do HTTP requests towards Rest APIs. +# - Allow unit tests to fake returned data +# - Allow using another external dependency for HTTP without affecting apps +# +# An HTTP request returns a Response that is a hash containing: +# code - The HTTP status code +# status - The HTTP status string that comes with the code +# content - The content of the reply +# all - The entire response coming over the TCP connection +# This is here for debugging and testing. If you need specifics like +# HTTP headers, we should just add specific fields for them rather than +# to parse all content. +# +# Examples: +# +# my @headers = ( +# "X-Foo" => 'Bar' +# ); +# my @params = ( +# "verbose" => 1 +# ); +# +# $response = Http::get('localhost', 80, '/status.html'); +# $response = Http::get('localhost', 80, '/status.html', \@params, \@headers); +# $response = Http::request('POST', 'localhost', 80, '/test', \@params, +# "Some content", \@headers); +# + +package Yahoo::Vespa::Http; + +use strict; +use warnings; + +use Net::INET6Glue::INET_is_INET6; +use LWP::Simple (); +use URI (); +use URI::Escape qw( uri_escape ); +use Yahoo::Vespa::ConsoleOutput; +use Yahoo::Vespa::Utils; + +my %LEGAL_TYPES; +my $BROWSER; +my $EXECUTE; + +&initialize(); + +return 1; + +######################## Externally usable functions ####################### + +sub get { # (Host, Port, Path, Params, Headers) -> Response + my ($host, $port, $path, $params, $headers) = @_; + return &request('GET', $host, $port, $path, $params, undef, $headers); +} +sub request { # (Type, Host, Port, Path, Params, Content, Headers) -> Response + my ($type, $host, $port, $path, $params, $content, $headers) = @_; + if (!exists $LEGAL_TYPES{$type}) { + confess "Invalid HTTP type '$type' specified."; + } + if (defined $params && ref($params) ne "ARRAY") { + confess 'HTTP request attempted without array ref for params'; + } + if (defined $headers && ref($headers) ne "ARRAY") { + confess 'HTTP request attempted without array ref for headers'; + } + return &$EXECUTE( + $type, $host, $port, $path, $params, $content, $headers); +} +sub encodeForm { # (KeyValueMap) -> RawString + my $data; + for (my $i=0; $i < scalar @_; $i += 2) { + my ($key, $value) = ($_[$i], $_[$i+1]); + if ($i != 0) { + $data .= '&'; + } + $data .= uri_escape($key); + if (defined $value) { + $data .= '=' . uri_escape($value); + } + } + return $data; +} + +################## Functions for unit tests to mock internals ################ + +sub setHttpExecutor { # (Function) + $EXECUTE = $_[0] +} + +############## Utility functions - Not intended for external use ############# + +sub initialize { # () + %LEGAL_TYPES = map { $_ => 1 } ( 'GET', 'POST', 'PUT', 'DELETE'); + $BROWSER = LWP::UserAgent->new; + $BROWSER->agent('Vespa-perl-script'); + $EXECUTE = \&execute; +} +sub execute { # (Type, Host, Port, Path, Params, Content, Headers) -> Response + my ($type, $host, $port, $path, $params, $content, $headers) = @_; + if (!defined $headers) { $headers = []; } + if (!defined $params) { $params = []; } + my $url = URI->new(&buildUri($host, $port, $path)); + if (defined $params) { + $url->query_form(@$params); + } + printSpam "Performing HTTP request $type '$url'.\n"; + my $response; + if ($type eq 'GET') { + !defined $content or confess "$type requests cannot have content"; + $response = $BROWSER->get($url, @$headers); + } elsif ($type eq 'POST') { + if (defined $content) { + $response = $BROWSER->post($url, $params, @$headers, + 'Content' => $content); + } else { + $response = $BROWSER->post($url, $params, @$headers); + } + } elsif ($type eq 'PUT') { + if (defined $content) { + $response = $BROWSER->put($url, $params, @$headers, + 'Content' => $content); + } else { + $response = $BROWSER->put($url, $params, @$headers); + } + } elsif ($type eq 'DELETE') { + !defined $content or confess "$type requests cannot have content"; + $response = $BROWSER->put($url, $params, @$headers); + } else { + confess "Unknown type $type"; + } + my $autoLineBreak = enableAutomaticLineBreaks(0); + printSpam "Got HTTP result: '" . $response->as_string . "'\n"; + enableAutomaticLineBreaks($autoLineBreak); + return ( + 'code' => $response->code, + 'headers' => $response->headers(), + 'status' => $response->message, + 'content' => $response->content, + 'all' => $response->as_string + ); +} +sub buildUri { # (Host, Port, Path) -> UriString + my ($host, $port, $path) = @_; + my $uri = "http:"; + if (defined $host) { + $uri .= '//' . $host; + if (defined $port) { + $uri .= ':' . $port; + } + } + if (defined $path) { + $uri .= $path; + } + return $uri; +} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Json.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Json.pm new file mode 100644 index 00000000000..8acadbe59ae --- /dev/null +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/Json.pm @@ -0,0 +1,52 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Minimal JSON wrapper. +# +# Intentions: +# - If needed, be able to switch the implementation of the JSON parser +# without components using this class seeing it. +# - Make API as simple as possible to use. +# +# Currently uses JSON.pm from ypan/perl-JSON +# +# Example usage: +# +# my $json = <<EOS; +# { +# 'foo' : [ +# { 'key1' : 2 }, +# { 'key2' : 5 } +# ] +# } +# +# my $result = Json::parse($json); +# my $firstkey = $result->{'foo'}->[0]->{'key1'} +# my @keys = @{ $result->{'foo'} }; +# +# See JsonTest for more usage. Add tests there if unsure. +# + +package Yahoo::Vespa::Json; + +use strict; +use warnings; + # Location of JSON.pm is not in default search path on tested Yahoo nodes. +use lib ($ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl/5.14/'); +use JSON; + +return 1; + +# Parses a string with json data returning an object tree +sub parse { # (RawString) -> ObjTree + my ($raw) = @_; + my $json = decode_json($raw); + return $json; +} + +# Encodes an object tree as returned from parse back to a raw string +sub encode { # (ObjTree) -> RawString + my ($json) = @_; + my $JSON = JSON->new->allow_nonref; + my $encoded = $JSON->pretty->encode($json); + return $encoded; +} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Utils.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Utils.pm new file mode 100644 index 00000000000..63e1a3093bc --- /dev/null +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/Utils.pm @@ -0,0 +1,97 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Some simple utilities to allow unit tests to mock behavior. +# + +package Yahoo::Vespa::Utils; + +use strict; +use warnings; +use Carp (); +use Sys::Hostname qw(hostname); + +BEGIN { # - Define exports from this module + use base 'Exporter'; + our @EXPORT = qw( + exitApplication + getHostname + confess + assertNotUnitTest + dumpStructure + ); +} + +my $HOSTNAME; +my $EXIT_HANDLER; +my $IS_UNIT_TEST; + +&initialize(); + +return 1; + +########################## Default exported functions ######################## + +# Use this function to get hostname to allow unit test mocking for tests to be +# independent of computer they run on. +sub getHostname { # () + if (!defined $HOSTNAME) { + $HOSTNAME = hostname; + &assertNotUnitTest(); + $HOSTNAME = `hostname`; + chomp $HOSTNAME; + } + return $HOSTNAME; +} + +# Use instead of exit() to allow unit tests to mock the call to avoid aborting +sub exitApplication { #(ExitCode) + if ($IS_UNIT_TEST && $EXIT_HANDLER == \&defaultExitHandler) { + &confess("Exit handler not overridden in unit test"); + } + &$EXIT_HANDLER(@_); +} + +# Use instead of die to get backtrace when dieing +sub confess { # (Reason) + Carp::confess(@_); +} + +# Call for behavior that you want to ensure is not used in unit tests. +# Typically unit tests have to mock commands that for instance fetch host name +# or require that terminal is set etc. Unit tests use mocks for this. This +# command can be used in code, such that unit tests die if they reach the +# non-mocked code. +sub assertNotUnitTest { # () + if ($IS_UNIT_TEST) { + confess "Unit tests should not reach here. Mock required. " + . "Initialize mock"; + } +} + +# Use to look at content of a perl struct. +sub dumpStructure { # (ObjTree) -> ReadableString + my ($var) = @_; + use Data::Dumper; + local $Data::Dumper::Indent = 1; + local $Data::Dumper::Sortkeys = 1; + return Dumper($var); +} + +################## Functions for unit tests to mock internals ################ + +sub initializeUnitTest { # (Hostname, ExitHandler) + my ($host, $exitHandler) = @_; + $IS_UNIT_TEST = 1; + $HOSTNAME = $host; + $EXIT_HANDLER = $exitHandler; +} + +############## Utility functions - Not intended for external use ############# + +sub initialize { # () + $EXIT_HANDLER = \&defaultExitHandler; +} +sub defaultExitHandler { # () + my ($exitcode) = @_; + exit($exitcode); +} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/VespaModel.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/VespaModel.pm new file mode 100644 index 00000000000..9e1fd90eeb3 --- /dev/null +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/VespaModel.pm @@ -0,0 +1,350 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Vespa model +# +# Make vespa model information available for tools. To for instance get an +# overview of where services are running. +# +# Possible improvements: +# +# - Depending on config Rest API and config server might be better than +# depending on getvespaconfig tool and config format. +# - Support direct communication with config server if config proxy is not +# running (unless getvespaconfig does that for us) +# - Support specifying config server, to be able to run tool external from the +# vespa system to talk to. +# - Return a list of all matching sockets instead of first found. +# - Be able to specify a set of port tags needed for a match. +# + +package Yahoo::Vespa::VespaModel; + +use strict; +use warnings; +use Yahoo::Vespa::ArgParser; +use Yahoo::Vespa::ConsoleOutput; +use Yahoo::Vespa::Utils; + +my $RETRIEVE_MODEL_CONFIG; # Allow unit tests to switch source of config info +my $MODEL; +my $CONFIG_SERVER_HOST; +my $CONFIG_SERVER_PORT; +my $CONFIG_REQUEST_TIMEOUT; + +&initialize(); + +return 1; + +######################## Externally usable functions ####################### + +sub registerCommandLineArguments { # () + setOptionHeader("Config retrieval options:"); + setHostOption( + ['config-server'], + \$CONFIG_SERVER_HOST, + 'Host name of config server to query'); + setPortOption( + ['config-server-port'], + \$CONFIG_SERVER_PORT, + 'Port to connect to config server on'); + setFloatOption( + ['config-request-timeout'], + \$CONFIG_REQUEST_TIMEOUT, + 'Timeout of config request'); +} + +sub visitServices { # (Callback) + my $model = &get(); + my ($callback) = @_; + my @services = @{ &getServices($model); }; + foreach my $service (sort serviceOrder @services) { + &$callback($service); + } +} + +sub getServices { + my $model = &get(); + my @result; + foreach my $hostindex (keys %{ $$model{'hosts'} }) { + my $host = ${ $$model{'hosts'} }{ $hostindex }; + foreach my $serviceindex (keys %{ $$host{'services'} }) { + my $service = ${ $$host{'services'} }{ $serviceindex }; + my %info = ( + 'name' => $$service{'name'}, + 'type' => $$service{'type'}, + 'configid' => $$service{'configid'}, + 'cluster' => $$service{'clustername'}, + 'host' => $$host{'name'} + ); + if (exists $$service{'index'}) { + $info{'index'} = $$service{'index'}; + } + push @result, \%info; + } + } + return \@result; +} + +# Get socket for given service matching given conditions (Given as a hash) +# Legal conditions: +# type - Service type +# tag - Port tag +# index - Service index +# clustername - Name of cluster. +# Example: getSocketForService( 'type' => 'distributor', 'index' => 3, +# 'tag' => 'http', 'tag' => 'state' ); +sub getSocketForService { # (Conditions) => [{host=>$,port=>$,index=>$}...] + my $model = &get(); + my $conditions = \@_; + printDebug "Looking at model to find socket for a service.\n"; + &validateConditions($conditions); + my $hosts = $$model{'hosts'}; + if (!defined $hosts) { return; } + my @results; + foreach my $hostindex (keys %$hosts) { + my $host = $$hosts{$hostindex}; + my $services = $$host{'services'}; + if (defined $services) { + printSpam "Searching services on host $$host{'name'}\n"; + foreach my $serviceindex (keys %$services) { + my $service = $$services{$serviceindex}; + my $type = $$service{'type'}; + my $cluster = $$service{'clustername'}; + if (!&serviceTypeMatchConditions($conditions, $type)) { + printSpam "Unwanted service '$type'.\n"; + next; + } + if (!&indexMatchConditions($conditions, $$service{'index'})) { + printSpam "Unwanted index '$$service{'index'}'.\n"; + next; + } + if (!&clusterNameMatchConditions($conditions, $cluster)) { + printSpam "Unwanted index '$$service{'index'}'.\n"; + next; + } + my $ports = $$service{'ports'}; + if (defined $ports) { + my $resultcount = 0; + foreach my $portindex (keys %$ports) { + my $port = $$ports{$portindex}; + my $tags = $$port{'tags'}; + if (defined $tags) { + if (!&tagsMatchConditions($conditions, $tags)) { + next; + } + } + push @results, { 'host' => $$host{'name'}, + 'port' => $$port{'number'}, + 'index' => $$service{'index'} }; + ++$resultcount; + } + if ($resultcount == 0) { + printSpam "No ports with acceptable tags found. " + . "Ignoring $type.$$service{'index'}\n"; + } + } else { + printSpam "No ports defined. " + . "Ignoring $type.$$service{'index'}\n"; + } + } + } + } + return \@results; +} + +############## Utility functions - Not intended for external use ############# + +sub initialize { # () + $RETRIEVE_MODEL_CONFIG = \&retrieveModelConfigDefault; +} +sub setModelRetrievalFunction { # (Function) + $RETRIEVE_MODEL_CONFIG = $_[0]; +} +sub retrieveModelConfigDefault { # () + my $VESPA_HOME= $ENV{'VESPA_HOME'}; + my $cmd = ${VESPA_HOME} . '/bin/getvespaconfig -n cloud.config.model -i admin/model'; + + if (defined $CONFIG_REQUEST_TIMEOUT) { + $cmd .= " -w $CONFIG_REQUEST_TIMEOUT"; + } + + my $temp = `${VESPA_HOME}/libexec/vespa/vespa-config.pl -configsources`; + my @configSources = split(",", $temp); + my $firstConfigSource = $configSources[0]; + if (!defined $CONFIG_SERVER_HOST) { + my @temp = split('/', $firstConfigSource); + my @configHost = split(':', $temp[1]); + $CONFIG_SERVER_HOST = $configHost[0]; + } + $cmd .= " -s $CONFIG_SERVER_HOST"; + + if (!defined $CONFIG_SERVER_PORT) { + my @configPort = split(':', $firstConfigSource); + $CONFIG_SERVER_PORT = $configPort[1]; + } + $cmd .= " -p $CONFIG_SERVER_PORT"; + + printDebug "Fetching model config '$cmd'.\n"; + my @data = `$cmd 2>&1`; + if ($? != 0 || join(' ', @data) =~ /^error/) { + printError "Failed to get model config from config command line tool:\n" + . "Command: $cmd\n" + . "Exit code: $?\n" + . "Output: " . join("\n", @data) . "\n"; + exitApplication(1); + } + return @data; +} +sub fetch { # () + my @data = &$RETRIEVE_MODEL_CONFIG(); + $MODEL = &parseConfig(@data); + return $MODEL; +} +sub get { # () + if (!defined $MODEL) { + return &fetch(); + } + return $MODEL; +} +sub validateConditions { # (ConditionArrayRef) + my ($condition) = @_; + for (my $i=0, my $n=scalar @$condition; $i<$n; $i += 2) { + if ($$condition[$i] !~ /^(type|tag|index|clustername)$/) { + printError "Invalid socket for service condition " + . "'$$condition[$i]' given.\n"; + exitApplication(1); + } + } +} +sub tagsMatchConditions { # (Condition, TagList) -> Bool + my ($condition, $taglist) = @_; + my %tags = map { $_ => 1 } @$taglist; + for (my $i=0, my $n=scalar @$condition; $i<$n; $i += 2) { + if ($$condition[$i] eq 'tag' && !exists $tags{$$condition[$i + 1]}) { + return 0; + } + } + return 1; +} +sub serviceTypeMatchConditions { # (Condition, ServiceType) -> Bool + my ($condition, $type) = @_; + for (my $i=0, my $n=scalar @$condition; $i<$n; $i += 2) { + if ($$condition[$i] eq 'type' && $$condition[$i + 1] ne $type) { + return 0; + } + } + return 1; +} +sub clusterNameMatchConditions { # (Condition, ClusterName) -> Bool + my ($condition, $cluster) = @_; + for (my $i=0, my $n=scalar @$condition; $i<$n; $i += 2) { + if ($$condition[$i] eq 'clustername' && $$condition[$i + 1] ne $cluster) + { + return 0; + } + } + return 1; +} +sub indexMatchConditions { # (Condition, Index) -> Bool + my ($condition, $index) = @_; + for (my $i=0, my $n=scalar @$condition; $i<$n; $i += 2) { + if ($$condition[$i] eq 'index' && $$condition[$i + 1] ne $index) { + return 0; + } + } + return 1; +} +sub parseConfig { # () + my $model = {}; + printDebug "Parsing vespa model raw config to create object tree\n"; + my $autoLineBreak = enableAutomaticLineBreaks(0); + foreach my $line (@_) { + chomp $line; + printSpam "Parsing line '$line'\n"; + if ($line =~ /^hosts\[(\d+)\]\.(([a-z]+).*)$/) { + my ($hostindex, $tag, $rest) = ($1, $3, $2); + my $host = &getHost($hostindex, $model); + if ($tag eq 'services') { + &parseService($host, $rest); + } else { + &parseValue($host, $rest); + } + } + } + enableAutomaticLineBreaks($autoLineBreak); + return $model; +} +sub parseService { # (Host, Line) + my ($host, $line) = @_; + if ($line =~ /^services\[(\d+)\].(([a-z]+).*)$/) { + my ($serviceindex, $tag, $rest) = ($1, $3, $2); + my $service = &getService($serviceindex, $host); + if ($tag eq 'ports') { + &parsePort($service, $rest); + } else { + &parseValue($service, $rest); + } + } +} +sub parsePort { # (Service, Line) + my ($service, $line) = @_; + if ($line =~ /^ports\[(\d+)\].(([a-z]+).*)$/) { + my ($portindex, $tag, $rest) = ($1, $3, $2); + my $port = &getPort($portindex, $service); + &parseValue($port, $rest); + } +} +sub parseValue { # (Entity, Line) + my ($entity, $line) = @_; + $line =~ /^(\S+) (?:\"(.*)\"|(\d+))$/ or confess "Unexpected line '$line'."; + my ($id, $string, $number) = ($1, $2, $3); + if ($id eq 'tags' && defined $string) { + my @tags = split(/\s+/, $string); + $$entity{$id} = \@tags; + } elsif (defined $string) { + $$entity{$id} = $string; + } else { + defined $number or confess "Should not happen"; + $$entity{$id} = $number; + } +} +sub getEntity { # (Type, Index, ParentEntity) + my ($type, $index, $parent) = @_; + if (!exists $$parent{$type}) { + $$parent{$type} = {}; + } + my $list = $$parent{$type}; + if (!exists $$list{$index}) { + $$list{$index} = {}; + } + return $$list{$index}; +} +sub getHost { # (Index, Model) + return &getEntity('hosts', $_[0], $_[1]); +} +sub getService { # (Index, Host) + return &getEntity('services', $_[0], $_[1]); +} +sub getPort { # (Index, Service) + return &getEntity('ports', $_[0], $_[1]); +} +sub serviceOrder { + if ($a->{'cluster'} ne $b->{'cluster'}) { + return $a->{'cluster'} cmp $b->{'cluster'}; + } + if ($a->{'type'} ne $b->{'type'}) { + return $a->{'type'} cmp $b->{'type'}; + } + if ($a->{'index'} != $b->{'index'}) { + return $a->{'index'} <=> $b->{'index'}; + } + if ($a->{'host'} ne $b->{'host'}) { + return $a->{'host'} cmp $b->{'host'}; + } + if ($a->{'configid'} ne $b->{'configid'}) { + return $a->{'configid'} cmp $b->{'configid'}; + } + confess "Unsortable elements: " . dumpStructure($a) . "\n" + . dumpStructure($b) . "\n"; +} + diff --git a/vespaclient/src/perl/test/Generic/UseTest.pl b/vespaclient/src/perl/test/Generic/UseTest.pl new file mode 100644 index 00000000000..d2c051d395a --- /dev/null +++ b/vespaclient/src/perl/test/Generic/UseTest.pl @@ -0,0 +1,34 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# That that all perl files use strict and warnings +# + +use Test::More; +use TestUtils::VespaTest; + +use strict; +use warnings; + +my @dirs = ( + '../bin', + '../lib', + 'Yahoo/Vespa/Mocks' +); + +my $checkdirs = join(' ', @dirs); + +my @files = `find $checkdirs -name \\*.pm -or -name \\*.pl`; +chomp @files; + +printTest "Checking " . (scalar @files) . " files for includes.\n"; + +foreach my $file (@files) { + ok( system("cat $file | grep 'use strict;' >/dev/null") == 0, + "$file use strict" ); + ok( system("cat $file | grep 'use warnings;' >/dev/null") == 0, + "$file use warnings" ); +} + +done_testing(); + +exit(0); diff --git a/vespaclient/src/perl/test/TestUtils/OutputCapturer.pm b/vespaclient/src/perl/test/TestUtils/OutputCapturer.pm new file mode 100644 index 00000000000..cb36807999e --- /dev/null +++ b/vespaclient/src/perl/test/TestUtils/OutputCapturer.pm @@ -0,0 +1,112 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package TestUtils::OutputCapturer; + +use Test::More; +use Yahoo::Vespa::ConsoleOutput; + +BEGIN { + use base 'Exporter'; + our @EXPORT = qw( + getOutput + isOutput + matchesOutput + ); +} + +Yahoo::Vespa::ConsoleOutput::setTerminalWidth(79); + +our ($stdout, $stderr); +my $USE_COLORS = 1; + +&openStreams(); + +END { + &closeStreams(); +} + +return 1; + +sub useColors { + $USE_COLORS = $_[0]; + &closeStreams(); + &openStreams(); +} + +sub isOutput { # (stdout, stderr, test) + my ($expected_cout, $expected_cerr, $test) = @_; + my ($cout, $cerr) = &getOutput(); + &diff($expected_cout, $cout); + ok ($cout eq $expected_cout, $test . " - stdout"); + &diff($expected_cerr, $cerr); + ok ($cerr eq $expected_cerr, $test . " - stderr"); +} + +sub matchesOutput { # (stdout_pattern, stderr_pattern, test) + my ($cout_pat, $cerr_pat, $test) = @_; + my ($cout, $cerr) = &getOutput(); + if ($cout !~ $cout_pat) { + diag("Output did not match standard out pattern:\n/$cout_pat/:\n$cout"); + } + ok ($cout =~ $cout_pat, $test . " - stdout"); + if ($cerr !~ $cerr_pat) { + diag("Stderr output did not match standard err pattern:\n" + . "/$cerr_pat/:\n$cerr"); + } + ok ($cerr =~ $cerr_pat, $test . " - stdout"); +} + +sub getOutput { + my $cout = &getStdOut(); + my $cerr = &getStdErr(); + &closeStreams(); + &openStreams(); + return ($cout, $cerr); +} + +sub openStreams { + open ($stdout, ">/tmp/vespaclient.perltest.stdout.log") + or die "Failed to create tmp file for stdout"; + open ($stderr, ">/tmp/vespaclient.perltest.stderr.log") + or die "Failed to create tmp file for stdout"; + Yahoo::Vespa::ConsoleOutput::initialize($stdout, $stderr, $USE_COLORS); +} + +sub closeStreams { + close $stdout; + close $stderr; + system("rm /tmp/vespaclient.perltest.stdout.log"); + system("rm /tmp/vespaclient.perltest.stderr.log"); +} + +sub getStdOut { + my $data = `cat /tmp/vespaclient.perltest.stdout.log`; + if (!defined $data) { $data = ''; } + return $data; +} + +sub getStdErr { + my $data = `cat /tmp/vespaclient.perltest.stderr.log`; + if (!defined $data) { $data = ''; } + return $data; +} + +sub diff { + my ($expected, $actual) = @_; + if ($expected eq $actual) { return; } + &writeToFile("/tmp/vespaclient.perltest.expected", $expected); + &writeToFile("/tmp/vespaclient.perltest.actual", $actual); + print "Output differs. Diff:\n"; + system("diff -u /tmp/vespaclient.perltest.expected " + . "/tmp/vespaclient.perltest.actual"); + system("rm -f /tmp/vespaclient.perltest.expected"); + system("rm -f /tmp/vespaclient.perltest.actual"); +} + +sub writeToFile { + my ($file, $data) = @_; + my $fh; + open ($fh, ">$file") or die "Failed to open temp file for writing."; + print $fh $data; + close $fh; +} diff --git a/vespaclient/src/perl/test/TestUtils/VespaTest.pm b/vespaclient/src/perl/test/TestUtils/VespaTest.pm new file mode 100644 index 00000000000..5df153e5938 --- /dev/null +++ b/vespaclient/src/perl/test/TestUtils/VespaTest.pm @@ -0,0 +1,92 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package TestUtils::VespaTest; + +use Test::More; +use TestUtils::OutputCapturer; +use Yahoo::Vespa::Utils; + +BEGIN { + use base 'Exporter'; + our @EXPORT = qw( + isOutput + matchesOutput + setApplication + assertRun + assertRunMatches + printTest + useColors + setLocalHost + ); +} + +my $APPLICATION; + +&initialize(); + +return 1; + +sub initialize { + Yahoo::Vespa::Utils::initializeUnitTest( + 'testhost.yahoo.com', \&mockedExitHandler); +} + +sub setLocalHost { + my ($host) = @_; + Yahoo::Vespa::Utils::initializeUnitTest( + $host, \&mockedExitHandler); +} + +sub useColors { + TestUtils::OutputCapturer::useColors(@_); +} + +sub mockedExitHandler { + my ($exitcode) = @_; + die "Application exited with exitcode $exitcode."; +} + +sub setApplication { + my ($main_func) = @_; + $APPLICATION = $main_func; +} + +sub assertRun { + my ($testname, $argstring, + $expected_exitcode, $expected_stdout, $expected_stderr) = @_; + my $exitcode = &run($argstring); + is( $exitcode, $expected_exitcode, "$testname - exitcode" ); + # print OutputCapturer::getStdOut(); + isOutput($expected_stdout, $expected_stderr, $testname); +} + +sub assertRunMatches { + my ($testname, $argstring, + $expected_exitcode, $expected_stdout, $expected_stderr) = @_; + my $exitcode = &run($argstring); + is( $exitcode, $expected_exitcode, "$testname - exitcode" ); + # print OutputCapturer::getStdOut(); + matchesOutput($expected_stdout, $expected_stderr, $testname); +} + +sub run { + my ($argstring) = @_; + my @args = split(/\s+/, $argstring); + eval { + Yahoo::Vespa::ArgParser::initialize(); + &$APPLICATION(\@args); + }; + my $exitcode = 0; + if ($@) { + if ($@ =~ /Application exited with exitcode (\d+)\./) { + $exitcode = 1; + } else { + print "Unknown die signal '" . $@ . "'\n"; + } + } + return $exitcode; +} + +sub printTest { + print "Test: ", @_; +} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/ArgParserTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/ArgParserTest.pl new file mode 100644 index 00000000000..78924f1bdcc --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/ArgParserTest.pl @@ -0,0 +1,313 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +use Test::More; + +BEGIN { use_ok( 'Yahoo::Vespa::ArgParser' ); } +require_ok( 'Yahoo::Vespa::ArgParser' ); + +BEGIN { *ArgParser:: = *Yahoo::Vespa::ArgParser:: } + +use TestUtils::OutputCapturer; + +TestUtils::OutputCapturer::useColors(1); + +&testSyntaxPage(); + +TestUtils::OutputCapturer::useColors(0); + +&testStringOption(); +&testIntegerOption(); +&testHostOption(); +&testPortOption(); +&testFlagOption(); +&testCountOption(); +&testComplexParsing(); +&testArguments(); + +done_testing(); + +exit(0); + +sub testSyntaxPage { + # Empty + ArgParser::writeSyntaxPage(); + my $expected = <<EOS; +Usage: ArgParserTest.pl +EOS + isOutput($expected, '', 'Empty syntax page'); + + # Built in only + Yahoo::Vespa::ArgParser::registerInternalParameters(); + ArgParser::writeSyntaxPage(); + $expected = <<EOS; +Usage: ArgParserTest.pl [Options] + +Options: + -h --help : Show this help page. + -v : Create more verbose output. + -s : Create less verbose output. + --show-hidden : Also show hidden undocumented debug options. +EOS + isOutput($expected, '', 'Syntax page with default args'); + + # Actual example + ArgParser::initialize(); + + setProgramBinaryName("testprog"); + setProgramDescription( + "This is a multiline description of what the program is that " + . "should be split accordingly to look nice. For now probably hard " + . "coded, but can later be extended to detect terminal width."); + my $arg; + setArgument(\$arg, "Test Arg", "This argument is not used for anything.", + OPTION_REQUIRED); + my $optionalArg; + setArgument(\$arg, "Another Test Arg", + "This argument is not used for anything either."); + + setOptionHeader("My prog headers. Also a long line just to check that it " + . "is also split accordingly."); + my $stringval; + my $flag; + my $intval; + setStringOption(['string', 'j'], \$stringval, "A random string"); + setFlagOption(['flag', 'f'], \$flag, "A flag option with a pretty long " + . "description that might need to be split into multiple lines."); + setOptionHeader("More options"); + setIntegerOption(['integer', 'i'], \$intval, "A secret integer option.", + OPTION_SECRET); + Yahoo::Vespa::ArgParser::registerInternalParameters(); + ArgParser::writeSyntaxPage(); + $expected = <<EOS; +This is a multiline description of what the program is that should be split +accordingly to look nice. For now probably hard coded, but can later be +extended to detect terminal width. + +Usage: testprog [Options] <Test Arg> [Another Test Arg] + +Arguments: + Test Arg : This argument is not used for anything. + Another Test Arg : This argument is not used for anything either. + +Options: + -h --help : Show this help page. + -v : Create more verbose output. + -s : Create less verbose output. + --show-hidden : Also show hidden undocumented debug options. + +My prog headers. Also a long line just to check that it is also split +accordingly. + --string -j : A random string + --flag -f : A flag option with a pretty long description that might need + to be split into multiple lines. +EOS + isOutput($expected, '', 'Actual syntax page example'); + + ArgParser::setShowHidden(1); + ArgParser::writeSyntaxPage(); + $expected = <<EOS; +This is a multiline description of what the program is that should be split +accordingly to look nice. For now probably hard coded, but can later be +extended to detect terminal width. + +Usage: testprog [Options] <Test Arg> [Another Test Arg] + +Arguments: + Test Arg : This argument is not used for anything. + Another Test Arg : This argument is not used for anything either. + +Options: + -h --help : Show this help page. + -v : Create more verbose output. + -s : Create less verbose output. + --show-hidden : Also show hidden undocumented debug options. + +My prog headers. Also a long line just to check that it is also split +accordingly. + --string -j : A random string + --flag -f : A flag option with a pretty long description that might need + to be split into multiple lines. + +More options + --integer -i : A secret integer option. + + --nocolors : Do not use ansi colors in print. +EOS + isOutput($expected, '', 'Actual syntax page example with hidden'); +} + +sub setUpParseTest { + Yahoo::Vespa::ArgParser::initialize(); +} + +sub parseFail { + my ($optstring, $expectedError) = @_; + my @args = split(/\s+/, $optstring); + my $name = $expectedError; + chomp $name; + if (length $name > 40 && $name =~ /^(.{20,70}?)\./) { + $name = $1; + } elsif (length $name > 55 && $name =~ /^(.{40,55})\s/) { + $name = $1; + } + ok( !ArgParser::parseCommandLineArguments(\@args), + "Expected parse failure: $name"); + isOutput('', $expectedError, $name); +} + +sub parseSuccess { + my ($optstring, $testname) = @_; + my @args = split(/\s+/, $optstring); + ok( ArgParser::parseCommandLineArguments(\@args), + "Expected parse success: $testname"); + isOutput('', '', $testname); +} + +sub testStringOption { + &setUpParseTest(); + my $val; + setStringOption(['s'], \$val, 'foo'); + parseFail("-s", "Too few arguments for option 's'\.\n"); + ok( !defined $val, 'String value unset on failure' ); + parseSuccess("-s foo", "String option"); + ok( $val eq 'foo', "String value set" ); +} + +sub testIntegerOption { + &setUpParseTest(); + my $val; + setIntegerOption(['i'], \$val, 'foo'); + parseFail("-i", "Too few arguments for option 'i'\.\n"); + ok( !defined $val, 'Integer value unset on failure' ); + parseFail("-i foo", "Invalid value 'foo' given to integer option 'i'\.\n"); + parseFail("-i 0.5", "Invalid value '0.5' given to integer option 'i'\.\n"); + parseSuccess("-i 5", "Integer option"); + ok( $val == 5, "Integer value set" ); + # Don't allow numbers as first char in id, so this can be detected as + # argument for integer. + parseSuccess("-i -8", "Negative integer option"); + ok( $val == -8, "Integer value set" ); + # Test big numbers + parseSuccess("-i 8000000000", "Big integer option"); + ok( $val / 1000000 == 8000, "Integer value set" ); + parseSuccess("-i -8000000000", "Big negative integer option"); + ok( $val / 1000000 == -8000, "Integer value set" ); +} + +sub testHostOption { + &setUpParseTest(); + my $val; + setHostOption(['h'], \$val, 'foo'); + parseFail("-h", "Too few arguments for option 'h'\.\n"); + ok( !defined $val, 'Host value unset on failure' ); + parseFail("-h 5", "Invalid host '5' given to option 'h'\. Not a valid host\n"); + parseFail("-h non.existing.host.no", "Invalid host 'non.existing.host.no' given to option 'h'\. Not a valid host\n"); + parseSuccess("-h localhost", "Host option set"); + is( $val, 'localhost', 'Host value set' ); +} + +sub testPortOption { + &setUpParseTest(); + my $val; + setPortOption(['p'], \$val, 'foo'); + parseFail("-p", "Too few arguments for option 'p'\.\n"); + ok( !defined $val, 'Host value unset on failure' ); + parseFail("-p -1", "Invalid value '-1' given to port option 'p'\. Must be an unsigned 16 bit\ninteger\.\n"); + parseFail("-p 65536", "Invalid value '65536' given to port option 'p'\. Must be an unsigned 16 bit\ninteger\.\n"); + parseSuccess("-p 65535", "Port option set"); + is( $val, 65535, 'Port value set' ); +} + +sub testFlagOption { + &setUpParseTest(); + my $val; + setFlagOption(['f'], \$val, 'foo'); + setFlagOption(['g'], \$val2, 'foo', OPTION_INVERTEDFLAG); + parseFail("-f 3", "Unhandled argument '3'\.\n"); + parseSuccess("-f", "First flag option set"); + is( $val, 1, 'Flag value set' ); + is( $val2, 1, 'Flag value set' ); + parseSuccess("-f", "First flag option reset"); + is( $val, 1, 'Flag value set' ); + is( $val2, 1, 'Flag value set' ); + parseSuccess("-g", "Second flag option set"); + is( $val, 0, 'Flag value set' ); + is( $val2, 0, 'Flag value set' ); + parseSuccess("-fg", "Both flag options set"); + is( $val, 1, 'Flag value set' ); + is( $val2, 0, 'Flag value set' ); +} + +sub testCountOption { + &setUpParseTest(); + my $val; + setUpCountingOption(['u'], \$val, 'foo'); + setDownCountingOption(['d'], \$val, 'foo'); + parseSuccess("", "Count not set"); + ok( !defined $val, 'Count value not set if not specified' ); + parseSuccess("-u", "Counting undefined"); + is( $val, 1, 'Count value set' ); + parseSuccess("-d", "Counting undefined - down"); + is( $val, -1, 'Count value set' ); + parseSuccess("-uuuud", "Counting both ways"); + is( $val, 3, 'Count value set' ); +} + +sub testComplexParsing { + &setUpParseTest(); + my $count; + my $int; + my $string; + setUpCountingOption(['u', 'up'], \$count, 'foo'); + setIntegerOption(['i', 'integer'], \$int, 'bar'); + setStringOption(['s', 'string'], \$string, 'baz'); + parseSuccess("-uis 3 foo", "Complex parsing managed"); + is( $count, 1, 'count counted' ); + is( $int, 3, 'integer set' ); + is( $string, 'foo', 'string set' ); + parseSuccess("-uiusi 3 foo 5", "Complex parsing managed 2"); + is( $count, 2, 'count counted' ); + is( $int, 5, 'integer set' ); + is( $string, 'foo', 'string set' ); + parseSuccess("-s -i foo -u 3", "Complex parsing managed 3"); + is( $count, 1, 'count counted' ); + is( $int, 3, 'integer set' ); + is( $string, 'foo', 'string set' ); +} + +sub testArguments { + &testOptionalArgument(); + &testRequiredArgument(); + &testRequiredArgumentAfterOptional(); +} + +sub testOptionalArgument { + &setUpParseTest(); + my $val; + setArgument(\$val, "Name", "Description"); + parseSuccess("", "Unset optional argument"); + ok( !defined $val, "Argument unset if not specified" ); + parseSuccess("myval", "Optional argument set"); + is( $val, 'myval', 'Optional argument set to correct value' ); +} + +sub testRequiredArgument { + &setUpParseTest(); + my $val; + setArgument(\$val, "Name", "Description", OPTION_REQUIRED); + parseFail("", "Argument Name is required but not specified\.\n"); + ok( !defined $val, "Argument unset on failure" ); + parseSuccess("myval", "Required argument set"); + is( $val, 'myval', 'Required argument set to correct value' ); +} + +sub testRequiredArgumentAfterOptional { + &setUpParseTest(); + my ($val, $val2); + setArgument(\$val, "Name", "Description"); + eval { + setArgument(\$val2, "Name2", "Description2", OPTION_REQUIRED); + }; + like( $@, qr/Cannot add required argument after optional/, + 'Fails adding required arg after optional' ); +} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Bin/GetClusterStateTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/Bin/GetClusterStateTest.pl new file mode 100644 index 00000000000..3339d872de5 --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/Bin/GetClusterStateTest.pl @@ -0,0 +1,65 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +use Test::More; +use strict; +use warnings; + +BEGIN { use_ok( 'Yahoo::Vespa::Bin::GetClusterState' ); } +require_ok( 'Yahoo::Vespa::Bin::GetClusterState' ); + +use TestUtils::VespaTest; +use Yahoo::Vespa::Mocks::ClusterControllerMock; +use Yahoo::Vespa::Mocks::VespaModelMock; + +# Set which application is called on assertRun / assertRunMatches calls +setApplication( \&getClusterState ); + +useColors(0); + +&testSimple(); +&testSyntaxPage(); +&testClusterDown(); + +done_testing(); + +exit(0); + +sub testSimple { + my $stdout = <<EOS; + +Cluster books: +books/storage/0: down +books/storage/1: up + +Cluster music: +music/distributor/0: down +music/distributor/1: up +music/storage/0: retired +EOS + assertRun("Default - no arguments", "", 0, $stdout, ""); +} + +sub testClusterDown { + Yahoo::Vespa::Mocks::ClusterControllerMock::setClusterDown(); + Yahoo::Vespa::ClusterController::init(); + Yahoo::Vespa::Bin::GetClusterState::init(); + my $stdout = <<EOS; + +Cluster books: +books/storage/0: down +books/storage/1: up + +Cluster music is down. Too few nodes available. +music/distributor/0: down +music/distributor/1: up +music/storage/0: retired +EOS + assertRun("Music cluster down", "", 0, $stdout, ""); +} + +sub testSyntaxPage { + my $stdout = <<EOS; +EOS + my $pat = qr/^Get the cluster state of a given cluster.*Usage:.*GetClusterState.*Options.*--help.*/s; + assertRunMatches("Syntax page", "--help", 1, $pat, qr/^$/); +} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Bin/GetNodeStateTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/Bin/GetNodeStateTest.pl new file mode 100644 index 00000000000..86cff2b28b3 --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/Bin/GetNodeStateTest.pl @@ -0,0 +1,71 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +use Test::More; +use strict; +use warnings; + +BEGIN { use_ok( 'Yahoo::Vespa::Bin::GetNodeState' ); } +require_ok( 'Yahoo::Vespa::Bin::GetNodeState' ); + +use TestUtils::VespaTest; +use Yahoo::Vespa::Mocks::ClusterControllerMock; +use Yahoo::Vespa::Mocks::VespaModelMock; + +useColors(0); + +# Set which application is called on assertRun / assertRunMatches calls +setApplication( \&getNodeState ); + +&testSimple(); +&testSyntaxPage(); +&testRetired(); + +done_testing(); + +exit(0); + +sub testSimple { + my $stdout = <<EOS; +Shows the various states of one or more nodes in a Vespa Storage cluster. There +exist three different type of node states. They are: + + Unit state - The state of the node seen from the cluster controller. + User state - The state we want the node to be in. By default up. Can be + set by administrators or by cluster controller when it + detects nodes that are behaving badly. + Generated state - The state of a given node in the current cluster state. + This is the state all the other nodes know about. This + state is a product of the other two states and cluster + controller logic to keep the cluster stable. + +books/storage.0: +Unit: down: Not in slobrok +Generated: down: Not seen +User: down: default + +music/distributor.0: +Unit: up: Now reporting state U +Generated: down: Setting it down +User: down: Setting it down +EOS + assertRun("Default - no arguments", "", 0, $stdout, ""); +} + +sub testRetired { + setLocalHost("other.host.yahoo.com"); + my $stdout = <<EOS; + +music/storage.0: +Unit: up: Now reporting state U +Generated: retired: Stop using +User: retired: Stop using +EOS + assertRun("Other node", "-c music -t storage -i 0 -s", 0, $stdout, ""); +} + +sub testSyntaxPage { + my $stdout = <<EOS; +EOS + my $pat = qr/^Retrieve the state of one or more.*Usage:.*GetNodeState.*Options.*--help.*/s; + assertRunMatches("Syntax page", "--help", 1, $pat, qr/^$/); +} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Bin/SetNodeStateTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/Bin/SetNodeStateTest.pl new file mode 100644 index 00000000000..1c6f4180dab --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/Bin/SetNodeStateTest.pl @@ -0,0 +1,129 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +use Test::More; +use strict; +use warnings; + +BEGIN { use_ok( 'Yahoo::Vespa::Bin::SetNodeState' ); } +require_ok( 'Yahoo::Vespa::Bin::SetNodeState' ); + +use TestUtils::VespaTest; +use Yahoo::Vespa::Mocks::ClusterControllerMock; +use Yahoo::Vespa::Mocks::VespaModelMock; + +# Set which application is called on assertRun / assertRunMatches calls +setApplication( \&setNodeState ); + +&testSimple(); +&testSyntaxPage(); +&testHelp(); +&testDownState(); +&testDownFailure(); +&testDefaultMaintenanceFails(); +&testForcedMaintenanceSucceeds(); + +done_testing(); + +exit(0); + +sub testSimple { + my $stdout = <<EOS; +Set user state for books/storage/0 to 'up' with reason '' +Set user state for music/distributor/0 to 'up' with reason '' +EOS + assertRun("Default - Min arguments", "up", 0, $stdout, ""); +} + +sub testSyntaxPage { + my $stdout = <<EOS; +EOS + my $pat = qr/^Set the user state of a node.*Usage:.*SetNodeState.*Arguments:.*Options:.*--help.*/s; + assertRunMatches("Syntax page", "--help", 1, $pat, qr/^$/); +} + +sub testHelp { + my $stdout = <<EOS; +Set the user state of a node. This will set the generated state to the user +state if the user state is "better" than the generated state that would have +been created if the user state was up. For instance, a node that is currently +in initializing state can be forced into down state, while a node that is +currently down can not be forced into retired state, but can be forced into +maintenance state. + +Usage: SetNodeStateTest.pl [Options] <Wanted State> [Description] + +Arguments: + Wanted State : User state to set. This must be one of up, down, maintenance or + retired. + Description : Give a reason for why you are altering the user state, which + will show up in various admin tools. (Use double quotes to give + a reason with whitespace in it) + +Options: + -h --help : Show this help page. + -v : Create more verbose output. + -s : Create less verbose output. + --show-hidden : Also show hidden undocumented debug options. + +Node selection options. By default, nodes running locally will be selected: + -c --cluster : Cluster name of cluster to query. If unspecified, + and vespa is installed on current node, information + will be attempted auto-extracted + -f --force : Force the execution of a dangerous command. + -t --type : Node type to query. This can either be 'storage' or + 'distributor'. If not specified, the operation will + show state for all types. + -i --index : The node index to show state for. If not specified, + all nodes found running on this host will be shown. + +Config retrieval options: + --config-server : Host name of config server to query + --config-server-port : Port to connect to config server on + --config-request-timeout : Timeout of config request +EOS + + assertRun("Help text", "-h", 1, $stdout, ""); +} + +sub testDownState { + my $stdout = <<EOS; +Set user state for books/storage/0 to 'down' with reason 'testing' +Set user state for music/distributor/0 to 'down' with reason 'testing' +EOS + assertRun("Down state", "down testing", 0, $stdout, ""); +} + +sub testDownFailure { + $Yahoo::Vespa::Mocks::ClusterControllerMock::forceInternalServerError = 1; + + my $stderr = <<EOS; +Failed to set node state for node books/storage/0: 500 Internal Server Error +(forced) +EOS + + assertRun("Down failure", "--nocolors down testing", 1, "", $stderr); + + $Yahoo::Vespa::Mocks::ClusterControllerMock::forceInternalServerError = 0; +} + +sub testDefaultMaintenanceFails { + my $stderr = <<EOS; +Setting the distributor to maintenance mode may have severe consequences for +feeding! +Please specify -t storage to only set the storage node to maintenance mode, or +-f to override this error. +EOS + + assertRun("Default maintenance fails", "--nocolors maintenance testing", + 1, "", $stderr); +} + +sub testForcedMaintenanceSucceeds { + my $stdout = <<EOS; +Set user state for books/storage/0 to 'maintenance' with reason 'testing' +Set user state for music/distributor/0 to 'maintenance' with reason 'testing' +EOS + + assertRun("Forced maintenance succeeds", "-f maintenance testing", + 0, $stdout, ""); +} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/ClusterControllerTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/ClusterControllerTest.pl new file mode 100644 index 00000000000..c70d7287566 --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/ClusterControllerTest.pl @@ -0,0 +1,49 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +use Test::More; +use Data::Dumper; + +BEGIN { use_ok( 'Yahoo::Vespa::ClusterController' ); } +require_ok( 'Yahoo::Vespa::ClusterController' ); + +use TestUtils::OutputCapturer; +use Yahoo::Vespa::Mocks::ClusterControllerMock; +use Yahoo::Vespa::Mocks::VespaModelMock; + +Yahoo::Vespa::ConsoleOutput::setVerbosity(0); # Squelch output when running test +detectClusterController(); +Yahoo::Vespa::ConsoleOutput::setVerbosity(3); + +my $cclist = Yahoo::Vespa::ClusterController::getClusterControllers(); +is( scalar @$cclist, 1, "Cluster controllers detected" ); +is( $$cclist[0]->host, 'testhost.yahoo.com', 'Host autodetected' ); +is( $$cclist[0]->port, 19050, 'Port autodetected' ); + +is( join (' - ', Yahoo::Vespa::ClusterController::listContentClusters()), + "music - books", 'Content clusters' ); + +my $state = getContentClusterState('music'); + +$Data::Dumper::Indent = 1; +# print Dumper($state); + +is( $state->globalState, 'up', 'Generated state for music' ); + +is( $state->distributor->{'0'}->unit->state, 'up', 'Unit state for music' ); +is( $state->distributor->{'1'}->unit->state, 'up', 'Unit state for music' ); +is( $state->storage->{'0'}->unit->state, 'up', 'Unit state for music' ); +is( $state->storage->{'1'}->unit->state, 'up', 'Unit state for music' ); +is( $state->distributor->{'0'}->generated->state, 'down', 'Generated state' ); +is( $state->distributor->{'1'}->generated->state, 'up', 'Generated state' ); +is( $state->storage->{'0'}->generated->state, 'retired', 'Generated state' ); +is( $state->storage->{'1'}->generated->state, 'up', 'Generated state' ); +is( $state->distributor->{'0'}->user->state, 'down', 'User state' ); +is( $state->distributor->{'1'}->user->state, 'up', 'User state' ); +is( $state->storage->{'0'}->user->state, 'retired', 'User state' ); +is( $state->storage->{'1'}->user->state, 'up', 'User state' ); + +is( $state->storage->{'1'}->unit->reason, 'Now reporting state U', 'Reason' ); + +done_testing(); + +exit(0); diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/ConsoleOutputTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/ConsoleOutputTest.pl new file mode 100644 index 00000000000..bd398a5b9f7 --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/ConsoleOutputTest.pl @@ -0,0 +1,47 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +use Test::More; + +BEGIN { use_ok( 'Yahoo::Vespa::ConsoleOutput' ); } +require_ok( 'Yahoo::Vespa::ConsoleOutput' ); + +ok( Yahoo::Vespa::ConsoleOutput::getVerbosity() == 3, + 'Default verbosity is 3' ); +ok( Yahoo::Vespa::ConsoleOutput::usingAnsiColors(), + 'Using ansi colors by default' ); + +use TestUtils::VespaTest; + +printSpam "test\n"; +isOutput('', '', "No spam at level 3"); + +printDebug "test\n"; +isOutput('', '', "No spam at level 3"); + +printInfo "info test\n"; +isOutput("info test\n", '', "Info at level 3"); + +printWarning "foo\n"; +isOutput("", "\e[93mfoo\e[0m\n", "Stderr output for warning"); + +useColors(0); +printWarning "foo\n"; +isOutput("", "foo\n", "Stderr output without ansi colors"); + +Yahoo::Vespa::ConsoleOutput::setVerbosity(4); +printSpam "test\n"; +isOutput('', '', "No spam at level 4"); + +printDebug "test\n"; +isOutput("debug: test\n", '', "Debug at level 4"); + +Yahoo::Vespa::ConsoleOutput::setVerbosity(5); +printSpam "test\n"; +isOutput("spam: test\n", '', "Spam at level 5"); + +printInfo "info test\n"; +isOutput("info: info test\n", '', "Type prefix at high verbosity"); + +done_testing(); + +exit(0); diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/HttpTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/HttpTest.pl new file mode 100644 index 00000000000..88c2961e3a2 --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/HttpTest.pl @@ -0,0 +1,140 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Tests of the Http wrapper library.. +# +# NOTE: Test server set up does not support content not ending in newline. +# + +use strict; +use Test::More; +use Yahoo::Vespa::Mocks::HttpServerMock; + +BEGIN { + use_ok( 'Yahoo::Vespa::Http' ); + *Http:: = *Yahoo::Vespa::Http:: +} +require_ok( 'Yahoo::Vespa::Http' ); + +my $httpTestServerPort = setupTestHttpServer(); +ok(defined $httpTestServerPort, "Test server set up"); + +&testSimpleGet(); +&testAdvancedGet(); +&testFailingGet(); +&testSimplePost(); +&testJsonReturnInPost(); + +done_testing(); + +exit(0); + +sub filterRequest { + my ($request) = @_; + $request =~ s/\r//g; + $request =~ s/(Content-Length:\s*)\d+/$1##/g; + $request =~ s/(Host: localhost:)\d+/$1##/g; + $request =~ s/(?:Connection|TE|Client-[^:]+):[^\n]*\n//g; + + return $request; +} + +sub testSimpleGet { + my %r = Http::get('localhost', $httpTestServerPort, '/foo'); + is( $r{'code'}, 200, "Get request code" ); + is( $r{'status'}, 'OK', "Get request status" ); + + my $expected = <<EOS; +HTTP/1.1 200 OK +Content-Length: ## +Content-Type: text/plain; charset=utf-8 + +GET /foo HTTP/1.1 +Host: localhost:## +User-Agent: Vespa-perl-script +EOS + is( &filterRequest($r{'all'}), $expected, 'Get result' ); +} + +sub testAdvancedGet { + my @headers = ("X-Foo" => 'Bar'); + my @uri_param = ("uricrap" => 'special=?&%value', + "other" => 'hmm'); + my %r = Http::request('GET', 'localhost', $httpTestServerPort, '/foo', + \@uri_param, undef, \@headers); + is( $r{'code'}, 200, "Get request code" ); + is( $r{'status'}, 'OK', "Get request status" ); + + my $expected = <<EOS; +HTTP/1.1 200 OK +Content-Length: ## +Content-Type: text/plain; charset=utf-8 + +GET /foo?uricrap=special%3D%3F%26%25value&other=hmm HTTP/1.1 +Host: localhost:## +User-Agent: Vespa-perl-script +X-Foo: Bar +EOS + is( &filterRequest($r{'all'}), $expected, 'Get result' ); +} + +sub testFailingGet { + my @uri_param = ("code" => '501', + "status" => 'Works'); + my %r = Http::request('GET', 'localhost', $httpTestServerPort, '/foo', + \@uri_param); + is( $r{'code'}, 501, "Get request code" ); + is( $r{'status'}, 'Works', "Get request status" ); + + my $expected = <<EOS; +HTTP/1.1 501 Works +Content-Length: ## +Content-Type: text/plain; charset=utf-8 + +GET /foo?code=501&status=Works HTTP/1.1 +Host: localhost:## +User-Agent: Vespa-perl-script +EOS + is( &filterRequest($r{'all'}), $expected, 'Get result' ); +} + +sub testSimplePost { + my @uri_param = ("uricrap" => 'Rrr' ); + my %r = Http::request('POST', 'localhost', $httpTestServerPort, '/foo', + \@uri_param, "Some content\n"); + is( $r{'code'}, 200, "Get request code" ); + is( $r{'status'}, 'OK', "Get request status" ); + + my $expected = <<EOS; +HTTP/1.1 200 OK +Content-Length: ## +Content-Type: text/plain; charset=utf-8 + +POST /foo?uricrap=Rrr HTTP/1.1 +Host: localhost:## +User-Agent: Vespa-perl-script +Content-Length: ## +Content-Type: application/x-www-form-urlencoded + +Some content +EOS + is( &filterRequest($r{'all'}), $expected, 'Get result' ); +} + +sub testJsonReturnInPost +{ + my @uri_param = ("contenttype" => 'application/json' ); + my $json = "{ \"key\" : \"value\" }\n"; + my %r = Http::request('POST', 'localhost', $httpTestServerPort, '/foo', + \@uri_param, $json); + is( $r{'code'}, 200, "Get request code" ); + is( $r{'status'}, 'OK', "Get request status" ); + + my $expected = <<EOS; +HTTP/1.1 200 OK +Content-Length: ## +Content-Type: application/json + +{ "key" : "value" } +EOS + is( &filterRequest($r{'all'}), $expected, 'Get json result' ); +} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/JsonTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/JsonTest.pl new file mode 100644 index 00000000000..5da8ad0e270 --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/JsonTest.pl @@ -0,0 +1,67 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Tests of the Json wrapper library.. +# + +use Test::More; + +use strict; + +BEGIN { + use_ok( 'Yahoo::Vespa::Json' ); + *Json:: = *Yahoo::Vespa::Json:: # Alias namespace +} +require_ok( 'Yahoo::Vespa::Json' ); + +&testSimpleJson(); + +done_testing(); + +exit(0); + +sub testSimpleJson { + my $json = <<EOS; +{ + "foo" : "bar", + "map" : { + "abc" : "def", + "num" : 13.0 + }, + "array" : [ + { "val1" : 3 }, + { "val2" : 6 } + ] +} +EOS + my $parsed = Json::parse($json); + is( $parsed->{'foo'}, 'bar', 'json test 1' ); + is( $parsed->{'map'}->{'abc'}, 'def', 'json test 2' ); + is( $parsed->{'map'}->{'num'}, 13.0, 'json test 3' ); + my $prettyPrint = <<EOS; +{ + "array" : [ + { + "val1" : 3 + }, + { + "val2" : 6 + } + ], + "map" : { + "num" : 13, + "abc" : "def" + }, + "foo" : "bar" +} +EOS + is( Json::encode($parsed), $prettyPrint, 'simple json test - encode' ); + my @keys = sort keys %{$parsed->{'map'}}; + is( scalar @keys, 2, 'simple json test - map keys' ); + is( $keys[0], 'abc', 'simple json test - map key 1' ); + is( $keys[1], 'num', 'simple json test - map key 2' ); + + @keys = @{ $parsed->{'array'} }; + is( scalar @keys, 2, 'simple json test - list keys' ); + is( $keys[0]->{'val1'}, 3, 'simple json test - list key 1' ); + is( $keys[1]->{'val2'}, 6, 'simple json test - list key 2' ); +} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/ClusterControllerMock.pm b/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/ClusterControllerMock.pm new file mode 100644 index 00000000000..661d8a5e051 --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/ClusterControllerMock.pm @@ -0,0 +1,258 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package Yahoo::Vespa::Mocks::ClusterControllerMock; + +use strict; +use warnings; +use URI::Escape; +use Yahoo::Vespa::ConsoleOutput; +use Yahoo::Vespa::Mocks::HttpClientMock; +use Yahoo::Vespa::Utils; + +BEGIN { + use base 'Exporter'; + our @EXPORT = qw( + ); +} + +our $forceInternalServerError = 0; + +# Register a handler in the Http Client mock +registerHttpClientHandler(\&handleCCRequest); + +our $clusterListJson = <<EOS; +{ + "cluster" : { + "books" : { + "link" : "/cluster/v2/books" + }, + "music" : { + "link" : "/cluster/v2/music" + } + } +} +EOS +our $musicClusterJson = <<EOS; +{ + "state" : { + "generated" : { + "state" : "up", + "reason" : "" + } + }, + "service" : { + "distributor" : { + "node" : { + "0" : { + "attributes" : { "hierarchical-group" : "top" }, + "state" : { + "generated" : { "state" : "down", "reason" : "Setting it down" }, + "unit" : { "state" : "up", "reason" : "Now reporting state U" }, + "user" : { "state" : "down", "reason" : "Setting it down" } + } + }, + "1" : { + "attributes" : { "hierarchical-group" : "top" }, + "state" : { + "generated" : { "state" : "up", "reason" : "Setting it up" }, + "unit" : { "state" : "up", "reason" : "Now reporting state U" }, + "user" : { "state" : "up", "reason" : "" + } + } + } + } + }, + "storage" : { + "node" : { + "0" : { + "attributes" : { "hierarchical-group" : "top" }, + "state" : { + "generated" : { "state" : "retired", "reason" : "Stop using" }, + "unit" : { "state" : "up", "reason" : "Now reporting state U" }, + "user" : { "state" : "retired", "reason" : "Stop using" } + }, + "partition" : { + "0" : { + "metrics" : { + "bucket-count" : 5, + "unique-document-count" : 10, + "unique-document-total-size" : 1000 + } + } + } + }, + "1" : { + "attributes" : { "hierarchical-group" : "top" }, + "state" : { + "generated" : { "state" : "up", "reason" : "Setting it up" }, + "unit" : { "state" : "up", "reason" : "Now reporting state U" }, + "user" : { "state" : "up", "reason" : "" + } + }, + "partition" : { + "0" : { + "metrics" : { + "bucket-count" : 50, + "unique-document-count" : 100, + "unique-document-total-size" : 10000 + } + } + } + } + } + } + } +} +EOS +our $booksClusterJson = <<EOS; +{ + "state" : { + "generated" : { + "state" : "up", + "reason" : "" + } + }, + "service" : { + "distributor" : { + "node" : { + "0" : { + "attributes" : { "hierarchical-group" : "top.g1" }, + "state" : { + "generated" : { "state" : "down", "reason" : "Setting it down" }, + "unit" : { "state" : "up", "reason" : "Now reporting state U" }, + "user" : { "state" : "down", "reason" : "Setting it down" } + } + }, + "1" : { + "attributes" : { "hierarchical-group" : "top.g2" }, + "state" : { + "generated" : { "state" : "up", "reason" : "Setting it up" }, + "unit" : { "state" : "up", "reason" : "Now reporting state U" }, + "user" : { "state" : "up", "reason" : "" + } + } + } + } + }, + "storage" : { + "node" : { + "0" : { + "attributes" : { "hierarchical-group" : "top.g1" }, + "state" : { + "generated" : { "state" : "down", "reason" : "Not seen" }, + "unit" : { "state" : "down", "reason" : "Not in slobrok" }, + "user" : { "state" : "down", "reason" : "default" } + } + }, + "1" : { + "attributes" : { "hierarchical-group" : "top.g2" }, + "state" : { + "generated" : { "state" : "up", "reason" : "Setting it up" }, + "unit" : { "state" : "up", "reason" : "Now reporting state U" }, + "user" : { "state" : "up", "reason" : "" + } + } + } + } + } + } +} +EOS + +return &init(); + +sub init { + #print "Verifying that cluster list json is parsable.\n"; + my $json = Json::parse($clusterListJson); + #print "Verifying that music json is parsable\n"; + $json = Json::parse($musicClusterJson); + #print "Verifying that books json is parsable\n"; + $json = Json::parse($booksClusterJson); + #print "All seems parsable.\n"; + return 1; +} + +sub setClusterDown { + $musicClusterJson =~ s/"up"/"down"/; + $musicClusterJson =~ s/""/"Not enough nodes up"/; + #print "Cluster state: $musicClusterJson\n"; + #print "Verifying that music json is parsable\n"; + my $json = Json::parse($musicClusterJson); +} + +sub handleCCRequest { # (Type, Host, Port, Path, ParameterMap, Content, Headers) + my ($type, $host, $port, $path, $params, $content, $headers) = @_; + my %paramHash; + if (defined $params) { + %paramHash = @$params; + } + if ($forceInternalServerError) { + printDebug "Forcing internal server error response\n"; + return ( + 'code' => 500, + 'status' => 'Internal Server Error (forced)' + ); + } + if ($path eq "/cluster/v2/") { + printDebug "Handling cluster list request\n"; + return ( + 'code' => 200, + 'status' => 'OK', + 'content' => $clusterListJson + ); + } + if ($path eq "/cluster/v2/music/" + && (exists $paramHash{'recursive'} + && $paramHash{'recursive'} eq 'true')) + { + printDebug "Handling cluster music state request\n"; + return ( + 'code' => 200, + 'status' => 'OK', + 'content' => $musicClusterJson + ); + } + if ($path eq "/cluster/v2/books/" + && (exists $paramHash{'recursive'} + && $paramHash{'recursive'} eq 'true')) + { + printDebug "Handling cluster books state request\n"; + return ( + 'code' => 200, + 'status' => 'OK', + 'content' => $booksClusterJson + ); + } + if ($path =~ /^\/cluster\/v2\/(books|music)\/(storage|distributor)\/(\d+)$/) + { + my ($cluster, $service, $index) = ($1, $2, $3); + my $json = Json::parse($content); + my $state = $json->{'state'}->{'user'}->{'state'}; + my $description = $json->{'state'}->{'user'}->{'reason'}; + if (!defined $description && $state eq 'up') { + $description = ""; + } + if ($state !~ /^(?:up|down|maintenance|retired)$/) { + return ( + 'code' => 500, + 'status' => "Unknown state '$state' specified" + ); + } + if (!defined $state || !defined $description) { + return ( + 'code' => 500, + 'status' => "Invalid form data or failed parsing: '$content'" + ); + } + printDebug "Handling set user state request $cluster/$service/$index"; + return ( + 'code' => 200, + 'status' => "Set user state for $cluster/$service/$index to " + . "'$state' with reason '$description'" + ); + } + printDebug "Request to '$path' not matched. Params:\n"; + foreach my $key (keys %paramHash) { + printDebug " $key => '$paramHash{$key}'\n"; + } + return; +} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/HttpClientMock.pm b/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/HttpClientMock.pm new file mode 100644 index 00000000000..22a15de28b7 --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/HttpClientMock.pm @@ -0,0 +1,55 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# Switched the backend implementation of the Vespa::Http library, such that +# requests are sent here rather than onto the network. Register handlers here +# to respond to requests. +# +# Handlers are called in sequence until one of them returns a defined result. +# If none do, return a generic failure. +# + +package Yahoo::Vespa::Mocks::HttpClientMock; + +use strict; +use warnings; +use Yahoo::Vespa::ConsoleOutput; +use Yahoo::Vespa::Http; + +BEGIN { # - Define default exports for module + use base 'Exporter'; + our @EXPORT = qw( + registerHttpClientHandler + ); +} + +my @HANDLERS; + +&initialize(); + +return 1; + +#################### Default exported functions ############################# + +sub registerHttpClientHandler { # (Handler) + push @HANDLERS, $_[0]; +} + +##################### Internal utility functions ########################## + +sub initialize { # () + Yahoo::Vespa::Http::setHttpExecutor(\&clientMock); +} +sub clientMock { # (HttpRequest to forward) -> Response + foreach my $handler (@HANDLERS) { + my %result = &$handler(@_); + if (exists $result{'code'}) { + return %result; + } + } + return ( + 'code' => 500, + 'status' => 'No client handler for given request', + 'content' => '', + 'all' => '' + ); +} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/HttpServerMock.pm b/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/HttpServerMock.pm new file mode 100644 index 00000000000..267a905b67d --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/HttpServerMock.pm @@ -0,0 +1,156 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# A mock of an HTTP server, such that HTTP client library can be tested. +# +# Known limitations: +# - Does line by line reading of TCP data, so the content part of the HTML +# request has to end in a newline, otherwise, the server will block waiting +# for more data. +# +# Default connection handler: +# - If no special case, server returns request 200 OK, with the complete +# client request as text/plain utf8 content. +# - If request matches contenttype=\S+ (Typically due to setting a URI +# parameter), the response will contain the content of the request with the +# given content type set. +# - If request matches code=\d+ (Typically due to setting a URI parameter), +# the response will use that return code. +# - If request matches status=\S+ (Typically due to setting a URI parameter), +# the response will use that status line +# + +package Yahoo::Vespa::Mocks::HttpServerMock; + +use strict; +use warnings; +use IO::Socket::IP; +use URI::Escape; + +BEGIN { # - Set up exports for module + use base 'Exporter'; + our @EXPORT = qw( + setupTestHttpServer + ); +} + +my $HTTP_TEST_SERVER; +my $HTTP_TEST_SERVER_PORT; +my $HTTP_TEST_SERVER_PID; +my $CONNECTION_HANDLER = \&defaultConnectionHandler; + +END { # - Kill forked HTTP handler process on exit + if (defined $HTTP_TEST_SERVER_PID) { + kill(9, $HTTP_TEST_SERVER_PID); + } +} + +return 1; + +####################### Default exported functions ############################ + +sub setupTestHttpServer { # () -> HttpServerPort + my $portfile = "/tmp/vespaclient.$$.perl.httptestserverport"; + unlink($portfile); + my $pid = fork(); + if ($pid == 0) { + $HTTP_TEST_SERVER = IO::Socket::IP->new( + 'Proto' => 'tcp', + 'LocalPort' => 0, + 'Listen' => SOMAXCONN, + 'ReuseAddr' => 1, + ); + # print "Started server listening to port " . $HTTP_TEST_SERVER->sockport() + # . "\n"; + my $fh; + open ($fh, ">$portfile") or die "Failed to write port used to file."; + print $fh "<" . $HTTP_TEST_SERVER->sockport() . ">"; + close $fh; + defined $HTTP_TEST_SERVER or die "Failed to set up test HTTP server"; + while (1) { + &$CONNECTION_HANDLER(); + } + exit(0); + } else { + $HTTP_TEST_SERVER_PID = $pid; + while (1) { + if (-e $portfile) { + my $port = `cat $portfile`; + chomp $port; + if (defined $port && $port =~ /\<(\d+)\>/) { + #print "Client using port $1\n"; + $HTTP_TEST_SERVER_PORT = $1; + last; + } + } + sleep(0.01); + } + } + unlink($portfile); + return $HTTP_TEST_SERVER_PORT; +} + +####################### Internal utility functions ############################ + +sub defaultConnectionHandler { + my $client = $HTTP_TEST_SERVER->accept(); + defined $client or die "No connection to accept?"; + my $request; + my $line; + my $content_length = 0; + my $content_type; + while ($line = <$client>) { + if ($line =~ /^(.*?)\s$/) { + $line = $1; + } + if ($line =~ /Content-Length:\s(\d+)/) { + $content_length = $1; + } + if ($line =~ /contenttype=(\S+)/) { + $content_type = uri_unescape($1); + } + #print "Got line '$line'\n"; + if ($line eq '') { + last; + } + $request .= $line . "\n"; + } + if ($content_length > 0) { + $request .= "\n"; + if (defined $content_type) { + $request = ""; + } + my $read = 0; + while ($line = <$client>) { + $read += length $line; + if ($line =~ /^(.*?)\s$/) { + $line = $1; + } + $request .= $line; + if ($read >= $content_length) { + last; + } + } + } + # print "Got request '$request'.\n"; + $request =~ s/\n/\r\n/g; + my $code = 200; + my $status = "OK"; + if ($request =~ /code=(\d+)/) { + $code = $1; + } + if ($request =~ /status=([A-Za-z0-9]+)/) { + $status = $1; + } + my $response = "HTTP/1.1 $code $status\n"; + if (defined $content_type) { + $response .= "Content-Type: $content_type\n"; + } else { + $response .= "Content-Type: text/plain; charset=utf-8\n"; + } + $response .= "Content-Length: " . (length $request) . "\n" + . "\n"; + $response =~ s/\n/\r\n/g; + $response .= $request; + print $client $response; + close $client; +} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/VespaModelMock.pm b/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/VespaModelMock.pm new file mode 100644 index 00000000000..78bce3f1e6c --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/VespaModelMock.pm @@ -0,0 +1,96 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package Yahoo::Vespa::Mocks::VespaModelMock; + +use strict; +use warnings; +use Yahoo::Vespa::VespaModel; + +Yahoo::Vespa::VespaModel::setModelRetrievalFunction(\&getModelConfig); + +our $defaultModelConfig = <<EOS; +hosts[0].name "testhost.yahoo.com" +hosts[0].services[0].name "container-clustercontroller" +hosts[0].services[0].type "container-clustercontroller" +hosts[0].services[0].configid "admin/cluster-controllers/0" +hosts[0].services[0].clustertype "" +hosts[0].services[0].clustername "cluster-controllers" +hosts[0].services[0].index 0 +hosts[0].services[0].ports[0].number 19050 +hosts[0].services[0].ports[0].tags "state external query http" +hosts[0].services[0].ports[1].number 19100 +hosts[0].services[0].ports[1].tags "external http" +hosts[0].services[0].ports[2].number 19101 +hosts[0].services[0].ports[2].tags "messaging rpc" +hosts[0].services[0].ports[3].number 19102 +hosts[0].services[0].ports[3].tags "admin rpc" +hosts[0].services[1].name "distributor2" +hosts[0].services[1].type "distributor" +hosts[0].services[1].configid "music/distributor/0" +hosts[0].services[1].clustertype "content" +hosts[0].services[1].clustername "music" +hosts[0].services[1].index 0 +hosts[0].services[1].ports[0].number 19131 +hosts[0].services[1].ports[0].tags "messaging" +hosts[0].services[1].ports[1].number 19132 +hosts[0].services[1].ports[1].tags "status rpc" +hosts[0].services[1].ports[2].number 19133 +hosts[0].services[1].ports[2].tags "status http" +hosts[0].services[2].name "storagenode3" +hosts[0].services[2].type "storagenode" +hosts[0].services[2].configid "storage/storage/0" +hosts[0].services[2].clustertype "content" +hosts[0].services[2].clustername "books" +hosts[0].services[2].index 0 +hosts[0].services[2].ports[0].number 19134 +hosts[0].services[2].ports[0].tags "messaging" +hosts[0].services[2].ports[1].number 19135 +hosts[0].services[2].ports[1].tags "status rpc" +hosts[0].services[2].ports[2].number 19136 +hosts[0].services[2].ports[2].tags "status http" +hosts[1].name "other.host.yahoo.com" +hosts[1].services[0].name "distributor2" +hosts[1].services[0].type "distributor" +hosts[1].services[0].configid "music/distributor/1" +hosts[1].services[0].clustertype "content" +hosts[1].services[0].clustername "music" +hosts[1].services[0].index 1 +hosts[1].services[0].ports[0].number 19131 +hosts[1].services[0].ports[0].tags "messaging" +hosts[1].services[0].ports[1].number 19132 +hosts[1].services[0].ports[1].tags "status rpc" +hosts[1].services[0].ports[2].number 19133 +hosts[1].services[0].ports[2].tags "status http" +hosts[1].services[1].name "storagenode3" +hosts[1].services[1].type "storagenode" +hosts[1].services[1].configid "storage/storage/1" +hosts[1].services[1].clustertype "content" +hosts[1].services[1].clustername "books" +hosts[1].services[1].index 1 +hosts[1].services[1].ports[0].number 19134 +hosts[1].services[1].ports[0].tags "messaging" +hosts[1].services[1].ports[1].number 19135 +hosts[1].services[1].ports[1].tags "status rpc" +hosts[1].services[1].ports[2].number 19136 +hosts[1].services[1].ports[2].tags "status http" +hosts[1].services[2].name "storagenode2" +hosts[1].services[2].type "storagenode" +hosts[1].services[2].configid "storage/storage/0" +hosts[1].services[2].clustertype "content" +hosts[1].services[2].clustername "music" +hosts[1].services[2].index 0 +hosts[1].services[2].ports[0].number 19134 +hosts[1].services[2].ports[0].tags "messaging" +hosts[1].services[2].ports[1].number 19135 +hosts[1].services[2].ports[1].tags "status rpc" +hosts[1].services[2].ports[2].number 19136 +hosts[1].services[2].ports[2].tags "status http" + +EOS + +sub getModelConfig { + my @output = split(/\n/, $defaultModelConfig); + return @output; +} + +1; diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/VespaModelTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/VespaModelTest.pl new file mode 100644 index 00000000000..fdb6a85bb16 --- /dev/null +++ b/vespaclient/src/perl/test/Yahoo/Vespa/VespaModelTest.pl @@ -0,0 +1,63 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +use Test::More; +use Yahoo::Vespa::Mocks::VespaModelMock; + +BEGIN { + use_ok( 'Yahoo::Vespa::VespaModel' ); + *VespaModel:: = *Yahoo::Vespa::VespaModel:: ; +} +require_ok( 'Yahoo::Vespa::VespaModel' ); + +&testGetSocketForService(); +&testVisitServices(); + +done_testing(); + +exit(0); + +sub testGetSocketForService { + my $sockets = VespaModel::getSocketForService( + type => 'container-clustercontroller', tag => 'state'); + my ($host, $port) = ($$sockets[0]->{'host'}, $$sockets[0]->{'port'}); + is( $host, 'testhost.yahoo.com', "Host for state API" ); + is( $port, 19050, 'Port for state API' ); + $sockets = VespaModel::getSocketForService( + type => 'container-clustercontroller', tag => 'admin'); + ($host, $port) = ($$sockets[0]->{'host'}, $$sockets[0]->{'port'}); + is( $host, 'testhost.yahoo.com', "Host for state API" ); + is( $port, 19102, 'Port for state API' ); + $sockets = VespaModel::getSocketForService( + type => 'container-clustercontroller', tag => 'http'); + ($host, $port) = ($$sockets[0]->{'host'}, $$sockets[0]->{'port'}); + is( $port, 19100, 'Port for state API' ); + + $sockets = VespaModel::getSocketForService( + type => 'distributor', index => 0); + ($host, $port) = ($$sockets[0]->{'host'}, $$sockets[0]->{'port'}); + is( $host, 'testhost.yahoo.com', 'host for distributor 0' ); +} + +my @services; + +sub serviceCallback { + my ($info) = @_; + push @services, "Name($$info{'name'}) Type($$info{'type'}) " + . "Cluster($$info{'cluster'}) Host($$info{'host'}) " + . "Index($$info{'index'})"; +} + +sub testVisitServices { + @services = (); + VespaModel::visitServices(\&serviceCallback); + my $expected = <<EOS; +Name(storagenode3) Type(storagenode) Cluster(books) Host(testhost.yahoo.com) Index(0) +Name(storagenode3) Type(storagenode) Cluster(books) Host(other.host.yahoo.com) Index(1) +Name(container-clustercontroller) Type(container-clustercontroller) Cluster(cluster-controllers) Host(testhost.yahoo.com) Index(0) +Name(distributor2) Type(distributor) Cluster(music) Host(testhost.yahoo.com) Index(0) +Name(distributor2) Type(distributor) Cluster(music) Host(other.host.yahoo.com) Index(1) +Name(storagenode2) Type(storagenode) Cluster(music) Host(other.host.yahoo.com) Index(0) +EOS + chomp $expected; + is ( join("\n", @services), $expected, "Services visited correctly" ); +} diff --git a/vespaclient/src/perl/test/testrunner.pl b/vespaclient/src/perl/test/testrunner.pl new file mode 100644 index 00000000000..c5307671b1a --- /dev/null +++ b/vespaclient/src/perl/test/testrunner.pl @@ -0,0 +1,110 @@ +#!/usr/bin/perl -w +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# +# Searches around in test dir to find test binaries and run them. Sadly these +# seem to return exit code 0 on some failures for unknown reasons. To counter +# that the testrunner grabs the output of the test and triggers test to fail if +# it finds unexpected data in the output. +# +# Unit tests should mostly not write as this will clutter report, but if they +# want to write some status they have to write it so it does not trigger +# failure here. Use printTest in VespaTest suite to prefix all test output to +# something we match here. +# + +use strict; +use warnings; + +$| = 1; +my @files = `find . -name \*Test.pl`; +chomp @files; + +my $tempdir = `mktemp -d /tmp/mockup-vespahome-XXXXXX`; +chomp $tempdir; +$ENV{'VESPA_HOME'} = $tempdir . "/"; +mkdir "${tempdir}/libexec"; +mkdir "${tempdir}/libexec/vespa" or die "Cannot mkdir ${tempdir}/libexec/vespa\n"; +`touch ${tempdir}/libexec/vespa/common-env.sh`; + +my $pat; +if (exists $ENV{'TEST_SUBSET'}) { + $pat = $ENV{'TEST_SUBSET'}; +} + +my $failure_pattern = qr/(?:Tests were run but no plan was declared and done_testing\(\) was not seen)/; +my $accepted_pattern = qr/^(?:\s*|\d+\.\.\d+|ok\s+\d+\s+-\s+.*|Test: .*|.*spam: .*)$/; + +my $failures = 0; +foreach my $file (@files) { + $file =~ /^(?:\.\/)?(.*)\.pl$/ or die "Strange file name '$file'."; + my $test = $1; + if (!defined $pat || $test =~ /$pat/) { + print "\nRunning test suite $test.\n\n"; + my ($code, $result) = captureCommand("PERLLIB=../lib perl -w $file"); + my @data = split(/\n/, $result); + if ($code != 0) { + ++$failures; + print "Test binary returned with non-null exitcode. Failure.\n"; + } elsif (&matchesFailurePattern(\@data)) { + ++$failures; + } elsif (¬MatchesSuccessPattern(\@data)) { + ++$failures; + } + } else { + # print "Skipping test suite '$test' not matching '$pat'.\n"; + } +} + +if ($failures > 0) { + print "\n\n$failures test suites failed.\n"; + exit(1); +} else { + print "\n\nAll tests succeeded.\n"; +} + +`rm -rv ${tempdir}`; + +exit(0); + +sub matchesFailurePattern { # (LineArrayRef) + my ($data) = @_; + foreach my $line (@$data) { + if ($line =~ $failure_pattern) { + print "Line '$line' indicates failure. Failing test suite.\n"; + return 1; + } + } + return 0; +} + +sub notMatchesSuccessPattern { # (LineArrayRef) + my ($data) = @_; + foreach my $line (@$data) { + if ($line !~ $accepted_pattern) { + print "Suspicious line '$line'.\n"; + print "Failing test due to line suspected to indicate failure.\n" + . "(Use printTest to print debug data during test to have it " + . "not been marked suspected.\n"; + return 1; + } + } + return 0; +} + +# Run a given command, giving exitcode and output back, but let command write +# directly to stdout/stderr. (Useful for long running commands or commands that +# may stall, such that you can see where it got into trouble) +sub captureCommand { # (Cmd) -> (ExitCode, Output) + my ($cmd) = @_; + my ($fh, $line); + my $data; + open ($fh, "$cmd 2>&1 |") or die "Failed to run '$cmd'."; + while ($line = <$fh>) { + print $line; + $data .= $line; + } + close $fh; + my $exitcode = $?; + return ($exitcode >> 8, $data); +} diff --git a/vespaclient/src/php/php_client.php b/vespaclient/src/php/php_client.php new file mode 100644 index 00000000000..95732f893a2 --- /dev/null +++ b/vespaclient/src/php/php_client.php @@ -0,0 +1,112 @@ +<?php + +# Client for putting or getting vespa documents. +# +# gunnarga@yahoo-inc.com +# september 2007 +# + +if ($argc < 2) { + echo "Usage: $argv[0] <url> [feedfile]\n"; + echo "\turl\t\tHttpGateway URL, e.g. http://myhost:myport/document/?abortondocumenterror=false\n"; + echo "\tfeedfile\tXML file to feed\n"; + exit(1); +} + +$url = $argv[1]; +$filename = $argv[2]; + +# split uri into subcomponents +$parsed_url = parse_url($url); +$hostname = $parsed_url['host']; +$port = $parsed_url['port']; +$path = $parsed_url['path']; +$query = $parsed_url['query']; + +$socket = stream_socket_client("{$hostname}:{$port}", $errno, $errstr, 30); + +if (file_exists($filename)) { + echo "* Parsing file {$filename}...\n"; + $feedfile = fopen($filename, "r"); + if ($feedfile) { + $fsize = filesize($filename); + $version = phpversion(); + $header = "POST {$path}?{$query} HTTP/1.1\r\n"; + $header .= "Host: {$hostname}:{$port}\r\n"; + $header .= "User-Agent: PHP/$version\r\n"; + $header .= "Content-Length: $fsize\r\n"; + $header .= "Content-Type: application/xml\r\n"; + $header .= "\r\n"; + + fwrite($socket, $header); + while (!feof($feedfile)) { + $buf = fgets($feedfile); + fwrite($socket, $buf); + } + fclose($feedfile); + } + +} else { + $version = phpversion(); + $header = "GET {$path}?{$query} HTTP/1.1\r\n"; + $header .= "Host: {$hostname}:{$port}\r\n"; + $header .= "User-Agent: PHP/$version\r\n"; + $header .= "\r\n"; + + fwrite($socket, $header); +} + +# check HTTP response +$firstline = fgets($socket); +if (!preg_match("/HTTP\/1.1 200 OK/", $firstline)) { + echo "HTTP gateway returned error message: $firstline"; +} + +# read rest of the HTTP headers +while (!feof($socket)) { + $line = fgets($socket); + if (preg_match("/Content-Length: (\d+)/", $line, $matches)) { + $content_length = $matches[1]; + } + if ($line == "\r\n") { + break; + } +} + +# collect xml data +$xmldata = stream_get_contents($socket, $content_length); + +# parse xml data +$xml = new SimpleXMLElement($xmldata); +foreach ($xml->error as $error) { + echo "Critical error: $error\n"; +} +foreach($xml->xpath("//messages/message") as $message) { + echo $message->asXML(); + echo "\n"; +} + +if (isset($xml->report->successes)) { + echo "\nSuccessful operations:"; + $ok_puts = $xml->report->successes["put"]; + $ok_updates = $xml->report->successes["update"]; + $ok_removes = $xml->report->successes["remove"]; + + if (isset($ok_puts)) { + echo " {$ok_puts} puts"; + } + if (isset($ok_updates)) { + echo " {$ok_updates} updates"; + } + if (isset($ok_removes)) { + echo " {$ok_removes} removes"; + } + echo "\n"; +} + +foreach($xml->xpath("/result/document") as $document) { + echo $document->asXML(); + echo "\n"; +} + +?> diff --git a/vespaclient/src/ruby/ruby_client.rb b/vespaclient/src/ruby/ruby_client.rb new file mode 100755 index 00000000000..193ddbb3907 --- /dev/null +++ b/vespaclient/src/ruby/ruby_client.rb @@ -0,0 +1,130 @@ +#!/usr/bin/env ruby +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +require 'socket' +require 'uri' +require 'optparse' +require 'rexml/document' + +# Client for putting or getting vespa documents. The standard library +# routines for http requests are not used because they don't as per +# ruby 1.8.5 support streaming file transfer. +# +# gunnarga@yahoo-inc.com +# september 2007 +# +class RubyClient + include REXML + + CHUNKSIZE = 8192 + + def initialize(uri) + uri_components = URI.split(uri) + @hostname = uri_components[2] + @port = uri_components[3] + @path = uri_components[5] + @query = uri_components[7] + @socket = TCPSocket.new(@hostname, @port) + end + + def http_post(feedfile) + puts "* Parsing file #{feedfile}..." + fsize = File.stat(feedfile).size + header = "POST #{@path}?#{@query} HTTP/1.1\r\n" + header += "Host: #{@hostname}:#{@port}\r\n" + header += "User-Agent: Ruby/#{RUBY_VERSION}\r\n" + header += "Content-Length: #{fsize}\r\n" + header += "Content-Type: application/xml\r\n" + header += "\r\n" + + begin + @socket.print(header) + File.open(feedfile) do |file| + while buf = file.read(CHUNKSIZE) + @socket.print(buf) + end + end + rescue Exception => e + puts "Exception caught: #{e}" + end + + print_response + end + + def http_get + header = "GET #{@path}?#{@query} HTTP/1.1\r\n" + header += "Host: #{@hostname}:#{@port}\r\n" + header += "User-Agent: Ruby/#{RUBY_VERSION}\r\n" + header += "\r\n" + + begin + @socket.print(header) + rescue Exception => e + puts "Exception caught: #{e}" + end + + print_response + end + + def print_response + xmldata = "" + begin + firstline = @socket.gets + if not firstline =~ /HTTP\/1.1 200 OK/ + puts "HTTP gateway returned error message: #{firstline}" + end + while line = @socket.gets + if line =~ /Content-Length: (\d+)/ + content_length = $1.to_i + end + if line == "\r\n" + break + end + end + + xmldata = @socket.read(content_length) + rescue Exception => e + puts "Exception caught: #{e}" + end + + begin + xmldoc = Document.new(xmldata) + xmldoc.elements.each("/result/error") {|e| puts "Critical error: #{e.text}"} + xmldoc.elements.each("//messages/message") {|e| puts e.to_s} + successes = XPath.first(xmldoc, "//successes") + if (successes) + print "\nSuccessful operations:" + if (ok_puts = successes.attribute("put")) + puts " #{ok_puts} puts" + end + if (ok_updates = successes.attribute("update")) + puts " #{ok_updates} updates" + end + if (ok_removes = successes.attribute("remove")) + puts " #{ok_removes} removes" + end + end + xmldoc.elements.each("/result/document") {|e| puts e.to_s} + rescue Exception => e + puts "Exception caught: #{e}" + end + end + +end + +if ARGV.length < 1 + puts "Usage: #{$0} <url> [feedfile]\n"; + puts "\turl\t\tHttpGateway URL, e.g. http://myhost:myport/document/?abortondocumenterror=false"; + puts "\tfeedfile\tXML file to feed"; + exit 1 +end + +url = ARGV[0] +filename = ARGV[1] + +feeder = RubyClient.new(url) +if filename and File.exists?(filename) + feeder.http_post(filename) +else + feeder.http_get +end diff --git a/vespaclient/src/test/.gitignore b/vespaclient/src/test/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespaclient/src/test/.gitignore diff --git a/vespaclient/src/vespa/vespaclient/.gitignore b/vespaclient/src/vespa/vespaclient/.gitignore new file mode 100644 index 00000000000..a4057dba627 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/.gitignore @@ -0,0 +1 @@ +features.h diff --git a/vespaclient/src/vespa/vespaclient/clusterlist/.gitignore b/vespaclient/src/vespa/vespaclient/clusterlist/.gitignore new file mode 100644 index 00000000000..5dae353d999 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/clusterlist/.gitignore @@ -0,0 +1,2 @@ +.depend +Makefile diff --git a/vespaclient/src/vespa/vespaclient/clusterlist/CMakeLists.txt b/vespaclient/src/vespa/vespaclient/clusterlist/CMakeLists.txt new file mode 100644 index 00000000000..137919cafd3 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/clusterlist/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vespaclient_clusterlist STATIC + SOURCES + clusterlist.cpp + DEPENDS +) diff --git a/vespaclient/src/vespa/vespaclient/clusterlist/clusterlist.cpp b/vespaclient/src/vespa/vespaclient/clusterlist/clusterlist.cpp new file mode 100644 index 00000000000..3816fc765d2 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/clusterlist/clusterlist.cpp @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespaclient/clusterlist/clusterlist.h> +#include <vespa/config/config.h> +#include <sstream> + +using namespace vespaclient; + +VESPA_IMPLEMENT_EXCEPTION(VCClusterNotFoundException, vespalib::IllegalArgumentException); + +ClusterList::ClusterList() +{ + configure(*config::ConfigGetter<cloud::config::ClusterListConfig>::getConfig("client")); +} + +void +ClusterList::configure(const cloud::config::ClusterListConfig& config) +{ + _contentClusters.clear(); + for (uint32_t i = 0; i < config.storage.size(); i++) { + _contentClusters.push_back( + Cluster(config.storage[i].name, config.storage[i].configid)); + } +} + +std::string +ClusterList::getContentClusterList() const +{ + std::ostringstream ost; + + for (uint32_t j = 0; j < _contentClusters.size(); j++) { + if (j != 0) { + ost << ","; + } + ost << _contentClusters[j].getName(); + } + + return ost.str(); +} + +const ClusterList::Cluster& +ClusterList::verifyContentCluster(const std::string& cluster) const +{ + if (cluster.length()) { + for (uint32_t j = 0; j < _contentClusters.size(); j++) { + if (_contentClusters[j].getName() == cluster) { + return _contentClusters[j]; + } + } + + std::ostringstream ost; + ost << "Cluster " << cluster + << " has not been configured in the vespa cluster. Legal clusters are [" + << getContentClusterList() << "]"; + throw ClusterNotFoundException(ost.str()); + } else if (_contentClusters.size() == 1) { + return _contentClusters[0]; + } else { + std::ostringstream ost; + ost << "No content cluster specified. Legal clusters are [" << getContentClusterList() << "]"; + throw ClusterNotFoundException(ost.str()); + } +} + diff --git a/vespaclient/src/vespa/vespaclient/clusterlist/clusterlist.h b/vespaclient/src/vespa/vespaclient/clusterlist/clusterlist.h new file mode 100644 index 00000000000..7469194233e --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/clusterlist/clusterlist.h @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/config-cluster-list.h> +#include <vespa/vespalib/util/exceptions.h> + +namespace vespaclient { + +VESPA_DEFINE_EXCEPTION(VCClusterNotFoundException, vespalib::IllegalArgumentException); + +/** + Contains a list of all the different clusters in the + vespa application. Currently supports only content clusters. +*/ +class ClusterList +{ +public: + typedef VCClusterNotFoundException ClusterNotFoundException; + class Cluster { + public: + Cluster(const std::string& name, const std::string& configId) + : _name(name), + _configId(configId) {}; + + const std::string& getName() const { return _name; } + const std::string& getConfigId() const { return _configId; } + private: + std::string _name; + std::string _configId; + }; + + ClusterList(); + + const std::vector<Cluster>& getContentClusters() const { return _contentClusters; } + + + /** + If the given cluster exists, or if it is empty and there exists only one content cluster, + return the cluster. Otherwise, throws a ClusterErrorException. + */ + const Cluster& verifyContentCluster(const std::string& contentCluster) const; + +private: + void configure(const cloud::config::ClusterListConfig& cfg); + + std::vector<Cluster> _contentClusters; + + std::string getContentClusterList() const; +}; + +} + diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/.gitignore b/vespaclient/src/vespa/vespaclient/spoolmaster/.gitignore new file mode 100644 index 00000000000..1a890e8015f --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/spoolmaster/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +spoolmaster diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/CMakeLists.txt b/vespaclient/src/vespa/vespaclient/spoolmaster/CMakeLists.txt new file mode 100644 index 00000000000..7ed480e1fbe --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/spoolmaster/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespaclient_spoolmaster_app + SOURCES + main.cpp + application.cpp + OUTPUT_NAME spoolmaster + INSTALL bin + DEPENDS +) diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp b/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp new file mode 100644 index 00000000000..84d271ab69e --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp @@ -0,0 +1,204 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/defaults.h> +#include <vector> +#include <string> +#include <iostream> +#include <algorithm> +#include <dirent.h> +#include <stdio.h> +#include <unistd.h> + +#include "application.h" + +namespace { + +std::string masterInbox() { + std::string dir = vespa::Defaults::vespaHome(); + dir.append("var/spool/master/inbox"); + return dir; +} + +std::string outboxParent() { + std::string dir = vespa::Defaults::vespaHome(); + dir.append("var/spool/vespa"); + return dir; +} + +} + +namespace spoolmaster { + +Application::Application() + : _masterInbox(masterInbox()), + _inboxFiles(), + _outboxParentDir(outboxParent()), + _outboxes() +{ + // empty +} + +Application::~Application() +{ + // empty +} + +bool +Application::scanInbox() +{ + std::vector<std::string> rv; + DIR *d = opendir(_masterInbox.c_str()); + if (d == NULL) { + perror(_masterInbox.c_str()); + mkdir(_masterInbox.c_str(), 0775); + return false; + } + + struct dirent *entry; + while ((entry = readdir(d)) != NULL) { + if (strcmp(entry->d_name, ".") == 0) continue; + if (strcmp(entry->d_name, "..") == 0) continue; + + std::string fn = _masterInbox; + fn.append("/"); + fn.append(entry->d_name); + + struct stat sb; + if (stat(fn.c_str(), &sb) == 0) { + if (S_ISREG(sb.st_mode)) { + rv.push_back(fn); + } + } else { + perror(fn.c_str()); + } + } + closedir(d); + + if (access(_masterInbox.c_str(), W_OK) < 0) { + perror(_masterInbox.c_str()); + return false; + } + + _inboxFiles = rv; + return (rv.size() > 0); +} + +bool +Application::findOutboxes() +{ + std::vector<std::string> rv; + DIR *d = opendir(_outboxParentDir.c_str()); + if (d == NULL) { + perror(_outboxParentDir.c_str()); + return false; + } + struct dirent *entry; + while ((entry = readdir(d)) != NULL) { + if (strcmp(entry->d_name, ".") == 0) continue; + if (strcmp(entry->d_name, "..") == 0) continue; + + /* XXX: should check if d_name starts with "colo." ? */ + + std::string fn = _outboxParentDir; + fn.append("/"); + fn.append(entry->d_name); + fn.append("/inbox"); + + if (fn == _masterInbox) continue; + + struct stat sb; + if (stat(fn.c_str(), &sb) == 0) { + if (S_ISDIR(sb.st_mode)) { + if (access(fn.c_str(), W_OK) < 0) { + std::cerr << "Cannot write to directory "; + perror(fn.c_str()); + continue; + } + rv.push_back(fn); + } + } else { + perror(fn.c_str()); + } + } + closedir(d); + if (rv.size() > 0) { + std::sort(rv.begin(), rv.end()); + sviter_t ni = rv.begin(); + sviter_t oi = _outboxes.begin(); + + while (ni != rv.end()) { + const std::string &newval = *ni; + if (oi == _outboxes.end()) { + std::cerr << "Found new slave inbox: " << newval << std::endl; + ++ni; + continue; + } + const std::string &oldval = *oi; + if (newval == oldval) { + ++ni; + ++oi; + } else if (newval < oldval) { + std::cerr << "Found new slave inbox: " << newval << std::endl; + ++ni; + } else /* oldval < newval */ { + std::cerr << "Slave inbox removed: " << oldval << std::endl; + ++oi; + } + } + _outboxes = rv; + return true; + } + std::cerr << "Did not find any slave inboxes in: " << _outboxParentDir << std::endl; + return false; +} + +void +Application::moveLinks() +{ + for (sviter_t fni = _inboxFiles.begin(); fni != _inboxFiles.end(); ++fni) { + const std::string& filename = *fni; + size_t ldp = filename.rfind("/"); + std::string basename = filename.substr(ldp+1); + for (sviter_t obi = _outboxes.begin(); obi != _outboxes.end(); ++obi) { + std::string newFn = *obi; + newFn.append("/"); + newFn.append(basename); + + std::cout << "linking " << filename << " -> " << newFn << std::endl; + if (link(filename.c_str(), newFn.c_str()) < 0) { + std::cerr << "linking " << filename << " -> " << newFn; + perror("failed"); + return; + } + } + if (unlink(filename.c_str()) < 0) { + std::cerr << "cannot remove " << filename; + perror(", error"); + } + } +} + + +int +Application::Main() +{ + bool aborted = false; + findOutboxes(); + + try { + while (!aborted) { + if (scanInbox() && findOutboxes()) { + moveLinks(); + } else { + FastOS_Thread::Sleep(200); + } + } + } + catch(std::exception &e) { + fprintf(stderr, "ERROR: %s\n", e.what()); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +} // namespace spoolmaster diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/application.h b/vespaclient/src/vespa/vespaclient/spoolmaster/application.h new file mode 100644 index 00000000000..7697169d9b4 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/spoolmaster/application.h @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <string> +#include <vector> +#include <vespa/fastos/fastos.h> + +namespace spoolmaster { +/** + * main spoolmaster application class + */ +class Application : public FastOS_Application { +private: + std::string _masterInbox; + std::vector<std::string> _inboxFiles; + + std::string _outboxParentDir; + std::vector<std::string> _outboxes; + + typedef std::vector<std::string>::iterator sviter_t; + + bool scanInbox(); + bool findOutboxes(); + void moveLinks(); +public: + /** + * Constructs a new spoolmaster object. + */ + Application(); + + /** + * Destructor. Frees any allocated resources. + */ + virtual ~Application(); + + // Implements FastOS_Application. + int Main(); +}; + +} + diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/main.cpp b/vespaclient/src/vespa/vespaclient/spoolmaster/main.cpp new file mode 100644 index 00000000000..1c28b03589f --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/spoolmaster/main.cpp @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "application.h" + +int +main(int argc, char** argv) +{ + spoolmaster::Application *app = new spoolmaster::Application(); + int retVal = app->Entry(argc, argv); + delete app; + return retVal; +} + diff --git a/vespaclient/src/vespa/vespaclient/vdsstates/.gitignore b/vespaclient/src/vespa/vespaclient/vdsstates/.gitignore new file mode 100644 index 00000000000..30187c17166 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vdsstates/.gitignore @@ -0,0 +1,5 @@ +.*.swp +.depend +Makefile +vdsgetnodestate +vdsgetnodestate-bin diff --git a/vespaclient/src/vespa/vespaclient/vdsstates/CMakeLists.txt b/vespaclient/src/vespa/vespaclient/vdsstates/CMakeLists.txt new file mode 100644 index 00000000000..d362a1e9ad0 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vdsstates/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespaclient_vdsgetnodestate_app + SOURCES + statesapp.cpp + OUTPUT_NAME vdsgetnodestate-bin + INSTALL bin + DEPENDS + vespaclient_clusterlist +) diff --git a/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp b/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp new file mode 100644 index 00000000000..ac863e30478 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vdsstates/statesapp.cpp @@ -0,0 +1,464 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> +#include <vespa/document/util/stringutil.h> +#include <vespa/fnet/frt/frt.h> +#include <vespa/slobrok/sbmirror.h> +#include <iostream> +#include <vespa/log/log.h> +#include <vespa/vdslib/distribution/distribution.h> +#include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vespalib/util/programoptions.h> +#include <vespa/vespaclient/clusterlist/clusterlist.h> +#include <vespa/vespalib/text/lowercase.h> + +LOG_SETUP("vdsstatetool"); + +namespace storage { + +enum Mode { SETNODESTATE, GETNODESTATE, GETCLUSTERSTATE }; + +namespace { + Mode getMode(std::string calledAs) { + std::string::size_type pos = calledAs.rfind('/'); + if (pos != std::string::npos) { + calledAs = calledAs.substr(pos + 1); + } + if (calledAs == "vdssetnodestate-bin") return SETNODESTATE; + if (calledAs == "vdsgetclusterstate-bin") return GETCLUSTERSTATE; + if (calledAs == "vdsgetsystemstate-bin") return GETCLUSTERSTATE; + if (calledAs == "vdsgetnodestate-bin") return GETNODESTATE; + std::cerr << "Tool called through unknown name '" << calledAs << "'. Assuming you want to " + << "get node state.\n"; + return GETNODESTATE; + } + + uint64_t getTimeInMillis() { + struct timeval t; + gettimeofday(&t, 0); + return (t.tv_sec * uint64_t(1000)) + (t.tv_usec / uint64_t(1000)); + } + + struct Sorter { + bool operator()(const std::pair<std::string, std::string>& first, + const std::pair<std::string, std::string>& second) + { return (first.first < second.first); } + }; + + const lib::State* getState(const std::string& s) { + vespalib::string lower = vespalib::LowerCase::convert(s); + if (lower == "up") return &lib::State::UP; + if (lower == "down") return &lib::State::DOWN; + if (lower == "retired") return &lib::State::RETIRED; + if (lower == "maintenance") return &lib::State::MAINTENANCE; + return 0; + } + + template<typename T> + struct ConfigReader : public T::Subscriber + { + T config; + + ConfigReader(const std::string& configId) { + T::subscribe(configId, *this); + } + void configure(const T& c) { config = c; } + }; +} + +struct Options : public vespalib::ProgramOptions { + Mode _mode; + bool _showSyntax; + std::string _clusterName; + vespaclient::ClusterList::Cluster _cluster; + uint32_t _nodeIndex; + std::string _slobrokConfigId; + std::string _slobrokConnectionSpec; + std::string _nodeType; + bool _nonfriendlyOutput; + std::string _state; + std::string _message; + std::string _doc; + uint32_t _slobrokTimeout; + + Options(Mode mode) : _mode(mode), _cluster("", ""), _nodeIndex(0xffffffff), _nonfriendlyOutput(false), _slobrokTimeout(0) { + _doc = "https://yahoo.github.io/vespa/"; + if (_mode == SETNODESTATE) { + setSyntaxMessage( + "Set the wanted node state of a storage node. This will " + "override the state the node is in in the cluster state, if " + "the current state is \"better\" than the wanted state. " + "For instance, a node that is currently in initializing state " + "can be forced into down state, while a node that is currently" + " down can not be forced into retired state, but can be forced" + " into maintenance state.\n\n" + "For more info on states refer to\n" + _doc + ); + } else if (_mode == GETCLUSTERSTATE) { + setSyntaxMessage( + "Get the cluster state of a given cluster.\n\n" + "For more info on states refer to\n" + _doc + ); + } else { + setSyntaxMessage( + "Retrieve the state of a one or more storage services from the " + "fleet controller. Will list the state of the locally running " + "services, possibly restricted to less by options.\n\n" + "The result will show the slobrok address of the service, and " + "three states. The first state will show how the state of that " + "given service looks in the current cluster state. This state " + "is the state the fleetcontroller is reporting to all nodes " + "in the cluster this service is in. The second state is the " + "reported state, which is the state the given node is reporting" + " to be in itself. The third state is the wanted state, which " + "is the state we want the node to be in. In most cases this " + "should be the up state, but in some cases the fleet controller" + " or an administrator may have set the wanted state otherwise, " + "in order to get problem nodes out of the cluster.\n\n" + "For more info on states refer to\n" + _doc + ); + } + addOption("h help", _showSyntax, false, + "Show this help page."); + + addOption("c cluster", _clusterName, std::string("storage"), + "Which cluster to connect to. By default it will attempt to " + "connect to cluster named 'storage'."); + if (_mode != GETCLUSTERSTATE) { + addOption("t type", _nodeType, std::string(""), + "Node type to query. This can either be 'storage' or " + "'distributor'. If not specified, the operation will " + "affect both types."); + addOption("i index", _nodeIndex, uint32_t(0xffffffff), + "The node index of the distributor or storage node to " + "contact. If not specified, all indexes running locally " + "on this node will be queried"); + } + if (_mode != SETNODESTATE) { + addOption("r raw", _nonfriendlyOutput, false, + "Show the serialized state formats directly instead of " + "reformatting them to look more user friendly."); + } + if (_mode == SETNODESTATE) { + addArgument("Wanted state", _state, "Wanted state to set node in. " + "This must be one of up, down or maintenance. Or if " + "it's not a distributor it can also be retired."); + addArgument("Reason", _message, std::string(""), + "Give a reason for why you're altering the wanted " + "state, which will show up in various admin tools. " + "(Use double quotes to give a reason with whitespace " + "in it)"); + } + addOptionHeader("Advanced options. Not needed for most usecases"); + addOption("l slobrokconfig", _slobrokConfigId, + std::string("admin/slobrok.0"), + "Config id of slobrok. Will use the default config id of " + "admin/slobrok.0 if not specified."); + addOption("p slobrokspec", _slobrokConnectionSpec, std::string(""), + "Slobrok connection spec. By setting this, this application " + "will not need config at all, but will use the given " + "connection spec to talk with slobrok."); + addOption("s slobroktimeout", _slobrokTimeout, uint32_t(5 * 60), + "Seconds to wait for slobrok client to connect to a slobrok " + "server before failing."); + } + + bool validate() { + if (_nodeType != "" + && _nodeType != "storage" && _nodeType != "distributor") + { + std::cerr << "Illegal nodetype '" << _nodeType << "'.\n"; + return false; + } + if (_mode == SETNODESTATE) { + const lib::State* state = getState(_state); + if (state == 0) { + std::cerr << "Illegal state '" << _state << "'.\n"; + return false; + } + if (*state == lib::State::RETIRED || + *state == lib::State::MAINTENANCE) + { + if (_nodeType != "storage") { + std::cerr << "Given state is only valid for storage nodes. " + << "Thus you need to specify only to\n" + << "set state of storage nodes.\n"; + return false; + } + } + if (*state != lib::State::UP && *state != lib::State::RETIRED + && _message == "") + { + std::cerr << "You should always have a reason for setting the " + "node in a non-available state.\n"; + return false; + } + } + + vespaclient::ClusterList clusterList; + try { + _cluster = clusterList.verifyContentCluster(_clusterName); + _clusterName = _cluster.getName(); + } catch (const vespaclient::ClusterList::ClusterNotFoundException& e) { + std::cerr << e.getMessage() << "\n"; + exit(1); + } + return true; + } +}; + +struct StateApp : public FastOS_Application { + Options _options; + + StateApp(std::string calledAs) : _options(getMode(calledAs)) {} + + int Main() { + _options.setCommandLineArguments(_argc, _argv); + try{ + _options.parse(); + } catch (vespalib::InvalidCommandLineArgumentsException& e) { + if (!_options._showSyntax) { + std::cerr << e.getMessage() << "\n"; + _options.writeSyntaxPage(std::cerr, false); + std::cerr << "\n"; + return 1; + } + } + if (_options._showSyntax) { + _options.writeSyntaxPage(std::cerr, false); + std::cerr << "\n"; + return 0; + } + if (!_options.validate()) { + _options.writeSyntaxPage(std::cerr, false); + return 1; + } + return run(); + } + + int run() { + FRT_Supervisor supervisor; + supervisor.Start(); + + std::unique_ptr<slobrok::api::MirrorAPI> slobrok; + if (_options._slobrokConnectionSpec == "") { + config::ConfigUri config(_options._slobrokConfigId); + slobrok.reset(new slobrok::api::MirrorAPI(supervisor, config)); + } else { + std::vector<std::string> specList; + specList.push_back(_options._slobrokConnectionSpec); + slobrok.reset(new slobrok::api::MirrorAPI(supervisor, specList)); + } + LOG(debug, "Waiting for slobrok data to be available."); + uint64_t startTime = getTimeInMillis(); + uint64_t warnTime = 5 * 1000; + uint64_t timeout = _options._slobrokTimeout * 1000; + while (true) { + uint64_t currentTime = getTimeInMillis(); + if (currentTime >= startTime + timeout) break; + if (slobrok->ready()) break; + if (currentTime >= startTime + warnTime) { + if (warnTime > 5000) { + std::cerr << "Still waiting for slobrok to respond. Have " + << "gotten no response in " + << ((currentTime - startTime) / 1000) + << " seconds.\n"; + } else { + std::cerr << "Waiting for slobrok server to respond. Have " + << "gotten no response in " + << ((currentTime - startTime) / 1000) << "\n" + << "seconds. Likely cause being one or more " + << "slobrok server nodes being down.\n(Thus not " + << "replying that socket is closed)\n"; + } + warnTime *= 4; + } + FastOS_Thread::Sleep(10); + } + if (!slobrok->ready()) { + std::cerr << "Slobrok not ready.\n"; + supervisor.ShutDown(true); + return 1; + } + + config::ConfigUri uri(_options._cluster.getConfigId()); + lib::Distribution distribution(*config::ConfigGetter<vespa::config::content::StorDistributionConfig>::getConfig(uri.getConfigId(), uri.getContext())); + + LOG(debug, "Got slobrok data"); + std::string mask = "storage/cluster." + _options._cluster.getName() + "/fleetcontroller/*"; + slobrok::api::MirrorAPI::SpecList specs = slobrok->lookup(mask); + if (specs.size() == 0) { + std::cerr << "No fleet controller could be found for '" + << mask << ".\n"; + supervisor.ShutDown(true); + return 1; + } + std::sort(specs.begin(), specs.end(), Sorter()); + LOG(debug, "Found fleet controller %s - %s\n", + specs.front().first.c_str(), specs.front().second.c_str()); + FRT_Target *target = supervisor.GetTarget(specs.front().second.c_str()); + if (!_options._nonfriendlyOutput && _options._mode == GETNODESTATE) + { + std::cerr << +"Shows the various states of one or more nodes in a Vespa Storage cluster.\n" +"There exist three different type of node states. They are:\n" +"\n" +" Reported state - The state reported to the fleet controller by the node.\n" +" Wanted state - The state administrators want the node to be in.\n" +" Current state - The state of a given node in the current cluster state.\n" +" This is the state all the other nodes know about. This\n" +" state is a product of the other two states and fleet\n" +" controller logic to keep the cluster stable.\n" +"\n" +"For more information about states of Vespa storage nodes, refer to\n" + << _options._doc << "\n\n"; + } + bool failed = false; + for (int i=0; i<2; ++i) { + std::string nodeType(_options._nodeType); + if ((_options._nodeType != "" || _options._mode == GETCLUSTERSTATE) + && i > 0) + { + break; + } + if (_options._nodeType == "") { + nodeType = (i == 0 ? "storage" : "distributor"); + } + std::vector<uint32_t> indexes; + if (_options._nodeIndex != 0xffffffff + || _options._mode == GETCLUSTERSTATE) + { + indexes.push_back(_options._nodeIndex); + } else { + std::vector<char> host(HOST_NAME_MAX); + int result = gethostname(&host[0], host.size()); + if (result != 0) { + std::cerr << "Failed to retrieve hostname of this node, " + "thus we cannot figure out what services run " + "on this node.\n"; + break; + } + std::string hostname(&host[0]); + FRT_RPCRequest* req = supervisor.AllocRPCRequest(); + req->SetMethodName("getNodeList"); + target->InvokeSync(req, 10.0); + std::string prefix = _options._cluster.getConfigId() + "/" + nodeType + "/"; + failed = (req->GetErrorCode() != FRTE_NO_ERROR); + if (failed) { + std::cerr << "Failed RPC call against " + << specs.front().second << ".\nError " + << req->GetErrorCode() << " : " + << req->GetErrorMessage() << "\n"; + break; + } + uint32_t arraySize( + req->GetReturn()->GetValue(0)._string_array._len); + for (uint32_t j=0; j<arraySize; ++j) { + std::string slobrokAddress(req->GetReturn()->GetValue(0) + ._string_array._pt[j]._str); + std::string rpcAddress(req->GetReturn()->GetValue(1) + ._string_array._pt[j]._str); + std::string::size_type pos = slobrokAddress.find(prefix); + std::string::size_type match = rpcAddress.find(hostname); + //std::cerr << "1. '" << slobrokAddress << "'.\n"; + //std::cerr << "2. '" << rpcAddress << "'.\n"; + if (pos != std::string::npos && match != std::string::npos) + { + uint32_t index = atoi(slobrokAddress.substr( + pos + prefix.size()).c_str()); + indexes.push_back(index); + } + } + } + if (indexes.size() == 0) { + std::cerr << "Could not find any storage or distributor " + << "services on this node.\n" + << "Specify node index with --index parameter.\n"; + failed = true; + break; + } + for (uint32_t j=0; j<indexes.size(); ++j) { + FRT_RPCRequest* req = supervisor.AllocRPCRequest(); + if (_options._mode == GETNODESTATE) { + req->SetMethodName("getNodeState"); + req->GetParams()->AddString(nodeType.c_str()); + req->GetParams()->AddInt32(indexes[j]); + } else if (_options._mode == SETNODESTATE) { + req->SetMethodName("setNodeState"); + std::ostringstream address; + address << _options._cluster.getConfigId() << "/" + << nodeType << "/" << indexes[j]; + lib::NodeState ns(lib::NodeType::get(nodeType), + *getState(_options._state)); + ns.setDescription(_options._message); + req->GetParams()->AddString(address.str().c_str()); + req->GetParams()->AddString(ns.toString(false).c_str()); + } else { + req->SetMethodName("getSystemState"); + } + target->InvokeSync(req, 10.0); + failed = (req->GetErrorCode() != FRTE_NO_ERROR); + if (failed) { + std::cerr << "Failed RPC call against " + << specs.front().second + << ".\nError " << req->GetErrorCode() << " : " + << req->GetErrorMessage() << "\n"; + break; + } else { + bool friendly = !_options._nonfriendlyOutput; + if (_options._mode == GETNODESTATE) { + lib::NodeState current( + req->GetReturn()->GetValue(0)._string._str); + lib::NodeState reported( + req->GetReturn()->GetValue(1)._string._str); + lib::NodeState wanted( + req->GetReturn()->GetValue(2)._string._str); + std::cout << "Node state of " + << _options._cluster.getConfigId() << "/" << nodeType + << "/" << indexes[j]; + std::cout << "\nCurrent state: "; + current.print(std::cout, friendly, " "); + std::cout << "\nReported state "; + reported.print(std::cout, friendly, " "); + std::cout << "\nWanted state: "; + wanted.print(std::cout, friendly, " "); + std::cout << "\n\n"; + } else if (_options._mode == SETNODESTATE) { + std::string result( + req->GetReturn()->GetValue(0)._string._str); + if (result != "") { + std::cout << result << "\n"; + } + } else { + std::string rawstate( + req->GetReturn()->GetValue(1)._string._str); + lib::ClusterState state(rawstate); + if (friendly) { + state.printStateGroupwise(std::cout, distribution, + true, ""); + } else { + std::cout << rawstate << "\n"; + } + std::cout << "\n"; + } + } + req->SubRef(); + } + } + target->SubRef(); + supervisor.ShutDown(true); + return (failed ? 1 : 0); + } +}; + +} // storage + +int +main(int argc, char **argv) +{ + assert(argc > 0); + storage::StateApp client(argv[0]); + return client.Entry(argc, argv); +} + diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/.gitignore b/vespaclient/src/vespa/vespaclient/vespadoclocator/.gitignore new file mode 100644 index 00000000000..82da15ee406 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +vespadoclocator +vespadoclocator-bin diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/CMakeLists.txt b/vespaclient/src/vespa/vespaclient/vespadoclocator/CMakeLists.txt new file mode 100644 index 00000000000..bd0f36f3896 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespaclient_vespadoclocator_app + SOURCES + application.cpp + locator.cpp + main.cpp + OUTPUT_NAME vespadoclocator-bin + INSTALL bin + DEPENDS +) +vespa_add_target_system_dependency(vespaclient_vespadoclocator_app boost boost_program_options-mt-d) diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp new file mode 100644 index 00000000000..2719a4a5e29 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.cpp @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("vespadoclocator"); + +#include <boost/program_options.hpp> +#include <vespa/config/common/exceptions.h> +#include "application.h" + +bool +Application::printDocumentLocation(Locator &locator, const std::string &docIdStr) +{ + try { + document::DocumentId docId(docIdStr); + std::cout << "DocumentId(" << docIdStr << ") " + << "BucketId(" << locator.getBucketId(docId).getId() << ") " + << "SearchColumn(" << locator.getSearchColumn(docId) << ")" + << std::endl; + } catch (document::IdParseException &e) { + std::cerr << e.getMessage() << std::endl; + return false; + } catch (vespalib::IllegalArgumentException &e) { + std::cerr << e.getMessage() << std::endl; + return false; + } + return true; +} + +int +Application::Main() +{ + // Configure locator object. + using namespace boost::program_options; + + uint32_t numColumns = 0; + std::string configId; + std::string clusterName; + std::vector<std::string> docIds; + + options_description desc("This is a tool for resolving the target column number of a document." + "\n\n" + "The options are"); + desc.add_options() + ( "config-id,i", + value<std::string>(&configId)->default_value("client"), + "The identifier to use when subscribing to configuration." ) + + ( "cluster-name,c", + value<std::string>(&clusterName), + "The name of the search cluster in which to resolve document location." ) + + ( "document-id,d", + value< std::vector<std::string> >(&docIds), + "The identifiers of the documents to locate. " + "These can also be passed as arguments without the option prefix. " + "If none is given, this tool parses identifiers from standard in." ) + + ( "help,h", + "Shows this help page." ) + + ( "num-columns,n", + value<uint32_t>(&numColumns), + "The number of columns in the search cluster. By providing this, no configuration " + "is required, meaning you can run this tool outside of a vespa cluster." ); + + positional_options_description pos; + pos.add("document-id", -1); + + variables_map vm; + try { + store(command_line_parser(_argc, _argv).options(desc).positional(pos).run(), vm); + notify(vm); + } catch (unknown_option &e) { + std::cout << e.what() << std::endl; + return EXIT_FAILURE; + } + + if (vm.count("help") != 0) { + std::cout << desc << std::endl; + return EXIT_SUCCESS; + } + + Locator locator(numColumns); + if (vm.count("num-columns") == 0) { + try { + locator.configure(configId, clusterName); + } catch (config::InvalidConfigException &e) { + std::cerr << e.getMessage() << std::endl; + return EXIT_FAILURE; + } + } + + // Locate the documents provided. + if (docIds.empty()) { + char buf[4096]; + while (!std::cin.getline(buf, 4096).eof()) { + std::string in(buf); + if (!printDocumentLocation(locator, in)) { + return EXIT_FAILURE; + } + } + } else { + for (std::vector<std::string>::iterator it = docIds.begin(); + it != docIds.end(); ++it) + { + if (!printDocumentLocation(locator, *it)) { + return EXIT_FAILURE; + } + } + } + return EXIT_SUCCESS; +} diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/application.h b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.h new file mode 100644 index 00000000000..c443d2f03a1 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/application.h @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "locator.h" + +class Application : public FastOS_Application { +private: + /** + * Locates and outputs the whereabouts of the given document id. If there is a problem parsing the given + * document identifier, this method returns false. + * + * @param locator The locator to use. + * @param docId The document to locate. + * @return True if the document was located. + */ + bool printDocumentLocation(Locator &locator, const std::string &docId); + +public: + // Implements FastOS_Application. + int Main(); +}; + diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp new file mode 100644 index 00000000000..6bca0db2991 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.cpp @@ -0,0 +1,142 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <boost/tokenizer.hpp> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/messagebus/configagent.h> +#include <vespa/messagebus/iconfighandler.h> +#include <vespa/messagebus/routing/routingspec.h> +#include <vespa/vdslib/bucketdistribution.h> +#include <vespa/messagebus/config-messagebus.h> +#include <vespa/config/helper/configgetter.h> + +#include "locator.h" + +typedef std::map<std::string, uint32_t> ClusterMap; +using namespace config; + +namespace { + + void + processHop(const mbus::HopSpec &hop, ClusterMap &clusters) + { + typedef boost::char_separator<char> CharSeparator; + typedef boost::tokenizer<CharSeparator> Tokenizer; + + int colIdx = -1; + for (uint32_t r = 0, len = hop.getNumRecipients(); r < len; ++r) { + Tokenizer tokens(hop.getRecipient(r), CharSeparator("/")); + Tokenizer::iterator token = tokens.begin(); + for (uint32_t t = 0; t < 2 && token != tokens.end(); ++t, ++token) { + // empty + } + if (token != tokens.end()) { + colIdx = std::max(colIdx, atoi(&token->c_str()[1])); + } + } + if (colIdx < 0) { + throw config::InvalidConfigException(vespalib::make_string("Failed to process cluster '%s'.", + hop.getName().c_str())); + } + clusters.insert(ClusterMap::value_type(hop.getName().substr(15), colIdx + 1)); + } + + void + processTable(const mbus::RoutingTableSpec &table, ClusterMap &clusters) + { + clusters.clear(); + for (uint32_t i = 0, len = table.getNumHops(); i < len; ++i) { + const mbus::HopSpec &hop = table.getHop(i); + if (hop.getName().find("search/cluster.") == 0) { + processHop(hop, clusters); + } + } + if (clusters.empty()) { + throw config::InvalidConfigException("No search clusters found to resolve document location for."); + } + } + + void + processRouting(const mbus::RoutingSpec &routing, ClusterMap &clusters) + { + const mbus::RoutingTableSpec *table = NULL; + for (uint32_t i = 0, len = routing.getNumTables(); i < len; ++i) { + const mbus::RoutingTableSpec &ref = routing.getTable(i); + if (ref.getProtocol() == documentapi::DocumentProtocol::NAME) { + table = &ref; + break; + } + } + if (table == NULL) { + throw config::InvalidConfigException("No routing table available to derive config from."); + } + processTable(*table, clusters); + } + + uint32_t + getNumColumns(const mbus::RoutingSpec &routing, const std::string &clusterName) + { + ClusterMap clusters; + processRouting(routing, clusters); + + if (clusterName.empty() && clusters.size() == 1) { + return clusters.begin()->second; + } + + ClusterMap::iterator it = clusters.find(clusterName); + if (it == clusters.end()) { + std::string str = "Cluster name must be one of "; + int i = 0, len = clusters.size(); + for (it = clusters.begin(); it != clusters.end(); ++it, ++i) + { + str.append("'").append(it->first).append("'"); + if (i < len - 2) { + str.append(", "); + } else if (i == len - 2) { + str.append(" or "); + } + } + str.append("."); + throw config::InvalidConfigException(str); + } + + return it->second; + } +} + +Locator::Locator(uint32_t numColumns) : + _factory(), + _numColumns(numColumns) +{ + // empty +} + +void +Locator::configure(const std::string &configId, const std::string &clusterName) +{ + config::ConfigUri configUri(configId); + // Configure by inspecting routing config. + struct MyCB : public mbus::IConfigHandler { + mbus::RoutingSpec mySpec; + MyCB() : mySpec() {} + bool setupRouting(const mbus::RoutingSpec &spec) { + mySpec = spec; + return true; + } + } myCB; + mbus::ConfigAgent agent(myCB); + agent.configure(ConfigGetter<messagebus::MessagebusConfig>::getConfig(configUri.getConfigId(), configUri.getContext())); + _numColumns = getNumColumns(myCB.mySpec, clusterName); +} + +document::BucketId +Locator::getBucketId(document::DocumentId &docId) +{ + return _factory.getBucketId(docId); +} + +uint32_t +Locator::getSearchColumn(document::DocumentId &docId) +{ + vdslib::BucketDistribution dist(_numColumns, 16u); + return dist.getColumn(getBucketId(docId)); +} diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.h b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.h new file mode 100644 index 00000000000..24af000e667 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/locator.h @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/fastos.h> +#include <vespa/document/base/documentid.h> +#include <vespa/document/bucket/bucketidfactory.h> + +class Locator { +private: + document::BucketIdFactory _factory; + uint32_t _numColumns; + +public: + /** + * Constructs a new locator object. + */ + Locator(uint32_t numColumns = 0); + + /** + * Configures this locator using the supplied configuration id and cluster name. This method will + * subscribe to some known config and attempt to retrieve the number of columns of the given search + * cluster from that. + * + * This method throws an exception if it could not be configured. + * + * @param configId The config identifier to subscribe to. + * @param clusterName The name of the search cluster to resolve locations in. + */ + void configure(const std::string &configId, + const std::string &clusterName); + + /** + * Returns the bucket id to which a document id belongs. + * + * @param docId The document id to resolve. + * @return The corresponding bucket id. + */ + document::BucketId getBucketId(document::DocumentId &docId); + + /** + * Returns the column in which the given document id belongs. + * + * @param docId The document id to resolve. + * @return The corresponding column. + */ + uint32_t getSearchColumn(document::DocumentId &docId); +}; + diff --git a/vespaclient/src/vespa/vespaclient/vespadoclocator/main.cpp b/vespaclient/src/vespa/vespaclient/vespadoclocator/main.cpp new file mode 100644 index 00000000000..1be39364efa --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vespadoclocator/main.cpp @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "application.h" + +int +main(int argc, char **argv) +{ + Application app; + return app.Entry(argc, argv); +} diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/.gitignore b/vespaclient/src/vespa/vespaclient/vesparoute/.gitignore new file mode 100644 index 00000000000..4d072420886 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vesparoute/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +vesparoute +vesparoute-bin diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/CMakeLists.txt b/vespaclient/src/vespa/vespaclient/vesparoute/CMakeLists.txt new file mode 100644 index 00000000000..bb9391331fc --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vesparoute/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespaclient_vesparoute_app + SOURCES + application.cpp + main.cpp + mynetwork.cpp + params.cpp + OUTPUT_NAME vesparoute-bin + INSTALL bin + DEPENDS +) diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp new file mode 100644 index 00000000000..7d4db8732c0 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp @@ -0,0 +1,570 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("vesparoute"); + +#include "application.h" + +#include "params.h" +#include <vespa/config/helper/configgetter.h> +#include <vespa/document/config/config-documenttypes.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/util/stringutil.h> +#include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/messagebus/configagent.h> +#include <vespa/messagebus/network/rpcsendv1.h> +#include <vespa/messagebus/routing/policydirective.h> +#include <vespa/messagebus/routing/routedirective.h> +#include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/config-messagebus.h> +#include <vespa/slobrok/sbmirror.h> +#include <algorithm> +#include <vector> + +using config::ConfigGetter; +using document::DocumenttypesConfig; +using messagebus::MessagebusConfig; +using document::DocumentTypeRepo; + +namespace vesparoute { + +Application::Application() : + _loadTypes(), + _net(), + _mbus(), + _params() +{ + // empty +} + +int +Application::Main() +{ + try { + if (_argc == 1) { + _params.setListRoutes(true); + _params.setListHops(true); + } else if (!parseArgs()) { + return EXIT_SUCCESS; + } + + DocumentTypeRepo::SP repo( + new DocumentTypeRepo( + *ConfigGetter<DocumenttypesConfig>::getConfig(_params.getDocumentTypesConfigId()))); + _net.reset(new MyNetwork(_params.getRPCNetworkParams())); + _mbus.reset( + new mbus::MessageBus( + *_net, + mbus::MessageBusParams() + .setRetryPolicy(mbus::IRetryPolicy::SP()) + .addProtocol(mbus::IProtocol::SP( + new documentapi::DocumentProtocol( + _loadTypes, repo))))); + mbus::ConfigAgent cfg(*_mbus); + cfg.configure(ConfigGetter<MessagebusConfig>::getConfig(_params.getRoutingConfigId())); + + // _P_A_R_A_N_O_I_A_ + mbus::RoutingTable::SP table = _mbus->getRoutingTable(_params.getProtocol()); + if (table.get() == NULL) { + throw config::InvalidConfigException(vespalib::make_string("There is no routing table for protocol '%s'.", + _params.getProtocol().c_str())); + } + for (std::vector<std::string>::iterator it = _params.getHops().begin(); + it != _params.getHops().end(); ++it) + { + if (table->getHop(*it) == NULL) { + throw config::InvalidConfigException(vespalib::make_string("There is no hop named '%s' for protocol '%s'.", + it->c_str(), _params.getProtocol().c_str())); + } + } + + // Perform requested action. + if (_params.getDump()) { + printDump(); + return EXIT_SUCCESS; + } + if (_params.getListRoutes()) { + listRoutes(); + } + if (_params.getListHops()) { + listHops(); + } + if (!_params.getRoutes().empty()) { + printRoutes(); + } + if (!_params.getHops().empty()) { + printHops(); + } + if (_params.getListServices()) { + printServices(); + } + + _mbus.reset(); + _net.reset(); + } catch(std::exception &e) { + std::string err(e.what()); + printf("ERROR: %s\n", err.substr(0, err.find_first_of('\n')).c_str()); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +bool +Application::parseArgs() +{ + for (int arg = 1; arg < _argc; arg++) { + if (strcasecmp(_argv[arg], "--documenttypesconfigid") == 0) { + if (++arg < _argc) { + _params.setDocumentTypesConfigId(_argv[arg]); + } else { + throw config::InvalidConfigException("Missing value for parameter 'documenttypesconfigid'."); + } + } else if (strcasecmp(_argv[arg], "--dump") == 0) { + _params.setDump(true); + } else if (strcasecmp(_argv[arg], "--help") == 0 || + strcasecmp(_argv[arg], "-h") == 0) { + printHelp(); + return false; + } else if (strcasecmp(_argv[arg], "--hop") == 0) { + if (++arg < _argc) { + _params.getHops().push_back(_argv[arg]); + } else { + throw config::InvalidConfigException("Missing value for parameter 'hop'."); + } + } else if (strcasecmp(_argv[arg], "--hops") == 0) { + _params.setListHops(true); + } else if (strcasecmp(_argv[arg], "--identity") == 0) { + if (++arg < _argc) { + _params.getRPCNetworkParams().setIdentity(mbus::Identity(_argv[arg])); + } else { + throw config::InvalidConfigException("Missing value for parameter 'identity'."); + } + } else if (strcasecmp(_argv[arg], "--listenport") == 0) { + if (++arg < _argc) { + _params.getRPCNetworkParams().setListenPort(atoi(_argv[arg])); + } else { + throw config::InvalidConfigException("Missing value for parameter 'listenport'."); + } + } else if (strcasecmp(_argv[arg], "--oosserverpattern") == 0) { + if (++arg < _argc) { + _params.getRPCNetworkParams().setOOSServerPattern(_argv[arg]); + } else { + throw config::InvalidConfigException("Missing value for parameter 'oosserverpattern'."); + } + } else if (strcasecmp(_argv[arg], "--protocol") == 0) { + if (++arg < _argc) { + _params.setProtocol(_argv[arg]); + } else { + throw config::InvalidConfigException("Missing value for parameter 'protocol'."); + } + } else if (strcasecmp(_argv[arg], "--route") == 0) { + if (++arg < _argc) { + _params.getRoutes().push_back(_argv[arg]); + } else { + throw config::InvalidConfigException("Missing value for parameter 'route'."); + } + } else if (strcasecmp(_argv[arg], "--routes") == 0) { + _params.setListRoutes(true); + } else if (strcasecmp(_argv[arg], "--routingconfigid") == 0) { + if (++arg < _argc) { + _params.setRoutingConfigId(_argv[arg]); + } else { + throw config::InvalidConfigException("Missing value for parameter 'routingconfigid'."); + } + } else if (strcasecmp(_argv[arg], "--services") == 0) { + _params.setListServices(true); + } else if (strcasecmp(_argv[arg], "--slobrokconfigid") == 0) { + if (++arg < _argc) { + _params.getRPCNetworkParams().setSlobrokConfig(_argv[arg]); + } else { + throw config::InvalidConfigException("Missing value for parameter 'slobrokconfigid'."); + } + } else if (strcasecmp(_argv[arg], "--verify") == 0) { + _params.setVerify(true); + } else { + throw config::InvalidConfigException(vespalib::make_string("Unknown option '%s'.", _argv[arg])); + } + } + return true; +} + +void +Application::printHelp() const +{ + printf("Usage: vesparoute [OPTION]...\n" + "Options:\n" + " --documenttypesconfigid <id> Sets the config id that supplies document configuration.\n" + " --dump Prints the complete content of the routing table.\n" + " --help Prints this help.\n" + " --hop <name> Prints detailed information about hop <name>.\n" + " --hops Prints a list of all available hops.\n" + " --identity <id> Sets the identity of message bus.\n" + " --listenport <num> Sets the port message bus will listen to.\n" + " --oosserverpattern <id> Sets the out-of-service server pattern for message bus.\n" + " --protocol <name> Sets the name of the protocol whose routing to inspect.\n" + " --route <name> Prints detailed information about route <name>.\n" + " --routes Prints a list of all available routes.\n" + " --routingconfigid <id> Sets the config id that supplies the routing tables.\n" + " --services Prints a list of all available services.\n" + " --slobrokconfigid <id> Sets the config id that supplies the slobrok server list.\n" + " --verify All hops and routes are verified when routing.\n"); +} + +bool +Application::verifyRoute(const mbus::Route &route, std::set<std::string> &errors) const +{ + for (uint32_t i = 0; i < route.getNumHops(); ++i) { + std::string str = route.getHop(i).toString(); + mbus::HopBlueprint hop = getHop(str); + std::set<std::string> hopErrors; + std::vector<std::string> services, oos; + if (!verifyHop(hop, services, oos, hopErrors)) { + for (std::set<std::string>::iterator err = hopErrors.begin(); + err != hopErrors.end(); ++err) + { + errors.insert(vespalib::make_string("for hop '%s', %s", + str.c_str(), + err->c_str())); + } + } + } + return errors.empty(); +} + +bool +Application::verifyHop(const mbus::HopBlueprint &hop, std::vector<std::string> &services, + std::vector<std::string> &oos, std::set<std::string> &errors) const +{ + // _P_A_R_A_N_O_I_A_ + if (!hop.hasDirectives()) { + errors.insert("is empty"); + return false; + } + + // Look for a policy directive. + for (uint32_t i = 0; i < hop.getNumDirectives(); ++i) { + if (hop.getDirective(i)->getType() == mbus::IHopDirective::TYPE_POLICY) { + return true; // can do whatever + } + } + if (hop.hasRecipients()) { + errors.insert("has recipients but no policy"); + } + + // Look for route or hop names. + const mbus::RoutingTable &table = *_mbus->getRoutingTable(_params.getProtocol()); + if (hop.getDirective(0)->getType() == mbus::IHopDirective::TYPE_ROUTE) { + const mbus::RouteDirective &dir = static_cast<const mbus::RouteDirective &>(*hop.getDirective(0)); + if (table.getRoute(dir.getName()) == NULL) { + errors.insert(vespalib::make_string("route '%s' not found", + dir.getName().c_str())); + return false; + } else { + return true; + } + } + + std::string selector = hop.create()->toString(); + if (table.getHop(selector) != NULL) { + return true; + } else if (table.getRoute(selector) != NULL) { + return true; + } + + // Must be service pattern, perform slobrok lookup. + slobrok::api::IMirrorAPI::SpecList lst = _net->getMirror().lookup(selector); + if (lst.empty()) { + errors.insert("no matching services"); + return false; + } + + // Check OOS status of all matches. + for (slobrok::api::IMirrorAPI::SpecList::iterator it = lst.begin(); + it != lst.end(); ++it) + { + services.push_back(it->first); + if (_net->verifyOOS(it->first)) { + oos.push_back(it->first); + } + } + if (oos.size() == lst.size()) { + errors.insert("matching service(s) out of service"); + } + return errors.empty(); +} + +void +Application::printDump() const +{ + const mbus::RoutingTable &table = *_mbus->getRoutingTable(_params.getProtocol()); + printf("<protocol name='%s'>\n", _params.getProtocol().c_str()); + for (mbus::RoutingTable::HopIterator it = table.getHopIterator(); + it.isValid(); it.next()) + { + std::set<std::string> errors; + std::vector<std::string> services, oos; + bool ok = verifyHop(it.getHop(), services, oos, errors); + + printf(" <hop name='%s' selector='%s'", it.getName().c_str(), it.getHop().create()->toString().c_str()); + if (it.getHop().getIgnoreResult()) { + printf(" ignore-result='true'"); + } + if (ok && !it.getHop().hasRecipients()) { + printf(" />\n"); + } else { + printf(">\n"); + for (uint32_t r = 0; r < it.getHop().getNumRecipients(); ++r) { + printf(" <recipient session='%s' />\n", it.getHop().getRecipient(r).toString().c_str()); + } + for (std::set<std::string>::iterator err = errors.begin(); + err != errors.end(); ++err) { + printf(" <error>%s</error>\n", err->c_str()); + } + printf(" </hop>\n"); + } + } + for (mbus::RoutingTable::RouteIterator it = table.getRouteIterator(); + it.isValid(); it.next()) + { + std::set<std::string> errors; + bool ok = verifyRoute(it.getRoute(), errors); + printf(" <route name='%s' hops='%s'", it.getName().c_str(), it.getRoute().toString().c_str()); + if (ok) { + printf(" />\n"); + } else { + printf(">\n"); + for (std::set<std::string>::iterator err = errors.begin(); + err != errors.end(); ++err) { + printf(" <error>%s</error>\n", err->c_str()); + } + printf(" </route>\n"); + } + + } + printf("</protocol>\n"); + + slobrok::api::IMirrorAPI::SpecList services; + getServices(services); + printf("<services>\n"); + for (slobrok::api::IMirrorAPI::SpecList::iterator it = services.begin(); + it != services.end(); ++it) + { + printf(" <service name='%s' spec='%s' %s/>\n", + it->first.c_str(), it->second.c_str(), + _net->verifyOOS(it->first) ? "state='oos' " : ""); + } + printf("</services>\n"); +} + +void +Application::listHops() const +{ + const mbus::RoutingTable &table = *_mbus->getRoutingTable(_params.getProtocol()); + if (table.hasHops()) { + printf("There are %d hop(s):\n", table.getNumHops()); + + uint32_t hop = 0; + for (mbus::RoutingTable::HopIterator it = table.getHopIterator(); + it.isValid(); it.next()) + { + printf("%5d. %s\n", ++hop, it.getName().c_str()); + } + } else { + printf("There are no hops configured.\n"); + } + printf("\n"); +} + +void +Application::printHops() const +{ + const mbus::RoutingTable &table = *_mbus->getRoutingTable(_params.getProtocol()); + const std::vector<std::string> &hops = _params.getHops(); + for (uint32_t i = 0; i < hops.size(); ++i) { + const mbus::HopBlueprint &hop = *table.getHop(hops[i]); + printf("The hop '%s' has selector:\n %s", + hops[i].c_str(), hop.create()->toString().c_str()); + + std::set<std::string> errors; + std::vector<std::string> services, oos; + if (_params.getVerify() && verifyHop(hop, services, oos, errors)) { + printf(" (verified)\n"); + } else { + printf("\n"); + } + + if (hop.hasRecipients()) { + printf("And %d recipient(s):\n", hop.getNumRecipients()); + for (uint32_t r = 0; r < hop.getNumRecipients(); ++r) { + std::string service = hop.getRecipient(r).toString(); + printf("%5d. %s\n", r + 1, service.c_str()); + } + } + + if (hop.getIgnoreResult()) { + printf("Any results from routing through this hop are ignored.\n"); + } + + if (!errors.empty()) { + printf("It has %zd error(s):\n", errors.size()); + uint32_t err = 1; + for (std::set<std::string>::iterator it = errors.begin(); + it != errors.end(); ++err, ++it) + { + printf("%5d. %s\n", err, it->c_str()); + } + } + printf("\n"); + } +} + +void +Application::listRoutes() const +{ + const mbus::RoutingTable &table = *_mbus->getRoutingTable(_params.getProtocol()); + if (table.hasRoutes()) { + printf("There are %d route(s):\n", table.getNumRoutes()); + + uint32_t route = 0; + for (mbus::RoutingTable::RouteIterator it = table.getRouteIterator(); + it.isValid(); it.next()) + { + printf("%5d. %s\n", ++route, it.getName().c_str()); + } + } else { + printf("There are no routes configured.\n"); + } + printf("\n"); +} + +void +Application::printRoutes() const +{ + const std::vector<std::string> &routes = _params.getRoutes(); + for (uint32_t i = 0; i < routes.size(); ++i) { + std::set<std::string> errors; + + mbus::Route route = getRoute(routes[i]); + printf("The route '%s' has %d hop(s):\n", + routes[i].c_str(), route.getNumHops()); + for (uint32_t hop = 0; hop < route.getNumHops(); ++hop) { + std::string str = route.getHop(hop).toString(); + if (_params.getVerify() && verifyRoute(route, errors)) { + str += " (verified)"; + } + printf("%5d. %s\n", hop + 1, str.c_str()); + } + if (!errors.empty()) { + printf("It has %zd error(s):\n", errors.size()); + uint32_t err = 1; + for (std::set<std::string>::iterator it = errors.begin(); + it != errors.end(); ++err, ++it) + { + printf("%5d. %s\n", err, it->c_str()); + } + } + printf("\n"); + } +} + +void +Application::printServices() const +{ + slobrok::api::IMirrorAPI::SpecList services; + getServices(services); + if (!services.empty()) { + std::set<std::string> lst; + for (slobrok::api::IMirrorAPI::SpecList::iterator it = services.begin(); + it != services.end(); ++it) + { + lst.insert(it->first); + } + printf("There are %zd service(s):\n", services.size()); + uint32_t service = 1; + for (std::set<std::string>::iterator it = lst.begin(); + it != lst.end(); ++it, ++service) + { + printf("%5d. %s\n", service, it->c_str()); + } + } else { + printf("There are no services available.\n"); + } + printf("\n"); +} + +void +Application::getServices(slobrok::api::IMirrorAPI::SpecList &ret, uint32_t depth) const +{ + FRT_Supervisor frt; + frt.Start(); + + std::string pattern = "*"; + for (uint32_t i = 0; i < depth; ++i) { + slobrok::api::IMirrorAPI::SpecList lst = _net->getMirror().lookup(pattern); + for (slobrok::api::IMirrorAPI::SpecList::iterator it = lst.begin(); + it != lst.end(); ++it) + { + if (isService(frt, it->second)) { + ret.push_back(*it); + } + } + pattern.append("/*"); + } + + frt.ShutDown(true); +} + +bool +Application::isService(FRT_Supervisor &frt, const std::string &spec) const +{ + FRT_Target *target = frt.GetTarget(spec.c_str()); + if (target == NULL) { + return false; + } + FRT_RPCRequest *req = frt.AllocRPCRequest(); + req->SetMethodName("frt.rpc.getMethodList"); + target->InvokeSync(req, 5.0); + + bool ret = false; + if (!req->IsError()) { + uint32_t numMethods = req->GetReturn()->GetValue(0)._string_array._len; + FRT_StringValue *methods = req->GetReturn()->GetValue(0)._string_array._pt; + FRT_StringValue *argList = req->GetReturn()->GetValue(1)._string_array._pt; + FRT_StringValue *retList = req->GetReturn()->GetValue(2)._string_array._pt; + + for (uint32_t i = 0; i < numMethods; ++i) { + if (strcmp(methods[i]._str, mbus::RPCSendV1::METHOD_NAME) == 0 && + strcmp(argList[i]._str, mbus::RPCSendV1::METHOD_PARAMS) == 0 && + strcmp(retList[i]._str, mbus::RPCSendV1::METHOD_RETURN) == 0) { + ret = true; + break; + } + } + } + + req->SubRef(); + target->SubRef(); + return ret; +} + +mbus::HopBlueprint +Application::getHop(const std::string &str) const +{ + const mbus::HopBlueprint *ret = _mbus->getRoutingTable(_params.getProtocol())->getHop(str); + if (ret == NULL) { + return mbus::HopBlueprint(mbus::HopSpec("anonymous", str)); + } + return *ret; +} + +mbus::Route +Application::getRoute(const std::string &str) const +{ + const mbus::Route *ret = _mbus->getRoutingTable(_params.getProtocol())->getRoute(str); + if (ret != NULL) { + return *ret; + } + return mbus::Route::parse(str); +} + +} diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/application.h b/vespaclient/src/vespa/vespaclient/vesparoute/application.h new file mode 100644 index 00000000000..bdfbd8d3c92 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vesparoute/application.h @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/app.h> +#include <vespa/messagebus/messagebus.h> +#include "mynetwork.h" +#include "params.h" +#include <vespa/documentapi/loadtypes/loadtypeset.h> + +namespace vesparoute { + +/** + * Command-line feeder running on document api. + */ +class Application : public FastOS_Application { +private: + documentapi::LoadTypeSet _loadTypes; + std::unique_ptr<MyNetwork> _net; + std::unique_ptr<mbus::MessageBus> _mbus; + Params _params; + + /** Parses the arguments of this application into the given params object. */ + bool parseArgs(); + + /** Prints help for this application. */ + void printHelp() const; + + /** Prints the full content of the given table. */ + void printDump() const; + + /** Prints information about all hops in the given table. */ + void listHops() const; + + /** Prints detailed information about the named hops. */ + void printHops() const; + + /** Prints information about all routes in the given table. */ + void listRoutes() const; + + /** Prints detailed information about the named routes. */ + void printRoutes() const; + + /** Prints information about all routable services. */ + void printServices() const; + + /** Fills the given spec list with all available routable services. */ + void getServices(slobrok::api::IMirrorAPI::SpecList &ret, uint32_t depth = 10) const; + + /** Returns whether or not the given spec resolves to a mbus service. */ + bool isService(FRT_Supervisor &frt, const std::string &spec) const; + + /** Returns a route corresponding to the given string. */ + mbus::Route getRoute(const std::string &str) const; + + /** Returns a hop corresponding to the given string. */ + mbus::HopBlueprint getHop(const std::string &str) const; + + /** Verifies the content of the given route. */ + bool verifyRoute(const mbus::Route &route, std::set<std::string> &errors) const; + + /** Verifies the content of the given hop. */ + bool verifyHop(const mbus::HopBlueprint &hop, std::vector<std::string> &services, + std::vector<std::string> &oos, std::set<std::string> &errors) const; + +public: + /** Null member variables. */ + Application(); + + // Inherit doc from FastOS_Application. + int Main(); +}; + +} + diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/main.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/main.cpp new file mode 100644 index 00000000000..2d3fffe66b8 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vesparoute/main.cpp @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/signalhandler.h> +#include "application.h" + +int +main(int argc, char** argv) +{ + vespalib::SignalHandler::PIPE.ignore(); + vesparoute::Application app; + int ret = app.Entry(argc, argv); + if (ret) { + printf("Non-zero exit status: %d\n", ret); + } + return ret; +} + diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp new file mode 100644 index 00000000000..573ede4e6dc --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".testframe"); + +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/sendproxy.h> +#include "mynetwork.h" + +class MyServiceAddress : public mbus::IServiceAddress { +private: + std::string _address; + +public: + MyServiceAddress(const std::string &address) : + _address(address) { + // empty + } + + const std::string &getAddress() { + return _address; + } +}; + +MyNetwork::MyNetwork(const mbus::RPCNetworkParams ¶ms) : + mbus::RPCNetwork(params), + _nodes() +{ + // empty +} + + +bool +MyNetwork::allocServiceAddress(mbus::RoutingNode &recipient) +{ + recipient.setServiceAddress(mbus::IServiceAddress::UP(new MyServiceAddress(recipient.getRoute().getHop(0).toString()))); + return true; +} + +void +MyNetwork::freeServiceAddress(mbus::RoutingNode &recipient) +{ + recipient.setServiceAddress(mbus::IServiceAddress::UP()); +} + +bool +MyNetwork::verifyOOS(const std::string &address) +{ + return getOOSManager().isOOS(address); +} + +void +MyNetwork::send(const mbus::Message &msg, const std::vector<mbus::RoutingNode*> &nodes) +{ + (void)msg; + _nodes.insert(_nodes.begin(), nodes.begin(), nodes.end()); +} + +void +MyNetwork::removeNodes(std::vector<mbus::RoutingNode*> &nodes) +{ + nodes.insert(nodes.begin(), _nodes.begin(), _nodes.end()); + _nodes.clear(); +} diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.h b/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.h new file mode 100644 index 00000000000..c8d959bb4ad --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.h @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/messagebus/network/rpcnetwork.h> + +/** + * Implements a dummy network on top of an rpc network, blocking anything from reaching the actual transmit + * steps in the base class. + */ +class MyNetwork : public mbus::RPCNetwork { +private: + std::vector<mbus::RoutingNode*> _nodes; + +public: + /** + * Constructs a new network object. + * + * @param params The parameter object to pass to the rpc network. + */ + MyNetwork(const mbus::RPCNetworkParams ¶ms); + + // Overrides RPCNetwork. + bool allocServiceAddress(mbus::RoutingNode &recipient); + + // Overrides RPCNetwork. + void freeServiceAddress(mbus::RoutingNode &recipient); + + // Overrides RPCNetwork. + void send(const mbus::Message &msg, const std::vector<mbus::RoutingNode*> &recipients); + + /** + * Returns whether or not the given address is actually out of service. + * + * @param address The address to check. + * @return True if the address is out of service. + */ + bool verifyOOS(const std::string &address); + + /** + * Removes and returns the list of recipients that was most recently sent to. + * + * @param contexts The list to move the contexts to. + */ + void removeNodes(std::vector<mbus::RoutingNode*> &nodes); +}; + diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/params.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/params.cpp new file mode 100644 index 00000000000..5a085381a87 --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vesparoute/params.cpp @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "params.h" + +namespace vesparoute { + +Params::Params() : + _rpcParams(), + _hops(), + _routes(), + _documentTypesConfigId("client"), + _routingConfigId("client"), + _protocol("document"), + _lstHops(false), + _lstRoutes(false), + _lstServices(false), + _dump(false), + _verify(false) +{ + _rpcParams.setOOSServerPattern("search/*/rtx/*/clustercontroller"); // magic +} + +Params::~Params() +{ + // empty +} + +} + diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/params.h b/vespaclient/src/vespa/vespaclient/vesparoute/params.h new file mode 100644 index 00000000000..7bd9ad47d7b --- /dev/null +++ b/vespaclient/src/vespa/vespaclient/vesparoute/params.h @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <string> +#include <vector> +#include <vespa/messagebus/network/rpcnetworkparams.h> + +namespace vesparoute { + +/** + * All parameters for application is contained in this class to simplify the api for parsing them. + */ +class Params { +public: + /** Constructs a new parameter object. */ + Params(); + + /** Destructor. Frees any allocated resources. */ + virtual ~Params(); + + /** Returns the rpc network params object. */ + mbus::RPCNetworkParams &getRPCNetworkParams() { return _rpcParams; } + + /** Returns a const reference to the rpc network params object. */ + const mbus::RPCNetworkParams &getRPCNetworkParams() const { return _rpcParams; } + + /** Returns the list of hops to print. */ + std::vector<std::string> &getHops() { return _hops; } + + /** Returns a const reference to the list of hops to print. */ + const std::vector<std::string> &getHops() const { return _hops; } + + /** Returns the list of routes to print. */ + std::vector<std::string> &getRoutes() { return _routes; } + + /** Returns a const reference the list of routes to print. */ + const std::vector<std::string> &getRoutes() const { return _routes; } + + /** Sets the config id to use for document types. */ + void + setDocumentTypesConfigId(const std::string &configId) + { + _documentTypesConfigId = configId; + } + + /** Returns the config id to use for the document manager. */ + const std::string & + getDocumentTypesConfigId() + { + return _documentTypesConfigId; + } + + /** Sets the config id to use for routing. */ + void setRoutingConfigId(const std::string &configId) { _routingConfigId = configId; } + + /** Returns the config id to use for routing. */ + const std::string &getRoutingConfigId() { return _routingConfigId; } + + /** Sets the name of the protocol whose routing table to use. */ + void setProtocol(const std::string &protocol) { _protocol = protocol; } + + /** Returns the name of the protocol whose routing table to use. */ + const std::string &getProtocol() const { return _protocol; } + + /** Sets wether or not to print all hops. */ + void setListHops(bool lst) { _lstHops = lst; } + + /** Returns wether or not to print all hops. */ + bool getListHops() const { return _lstHops; } + + /** Sets wether or not to print all routes. */ + void setListRoutes(bool lst) { _lstRoutes = lst; } + + /** Returns wether or not to print all routes. */ + bool getListRoutes() const { return _lstRoutes; } + + /** Sets wether or not to print all services. */ + void setListServices(bool lst) { _lstServices = lst; } + + /** Returns wether or not to print all services. */ + bool getListServices() const { return _lstServices; } + + /** Sets wether or not to print the full routing table content. */ + void setDump(bool dump) { _dump = dump; } + + /** Returns wether or not to print the full routing table content. */ + bool getDump() const { return _dump; } + + /** Sets wether or not to verify service names. */ + void setVerify(bool verify) { _verify = verify; } + + /** Returns wether or not to verify service names. */ + bool getVerify() const { return _verify; } + +private: + mbus::RPCNetworkParams _rpcParams; + std::vector<std::string> _hops; + std::vector<std::string> _routes; + std::string _documentTypesConfigId; + std::string _routingConfigId; + std::string _protocol; + bool _lstHops; + bool _lstRoutes; + bool _lstServices; + bool _dump; + bool _verify; +}; + +} + |