path: root/config-model/src/main/perl/vespa-deploy

#!/usr/bin/env perl
# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
# This script is for uploading, preparing, activating and fetching
# application packages to a cloud config server

# BEGIN perl environment bootstrap section
# Do not edit between here and END as this section should stay identical in all scripts

use File::Basename;
use File::Path;

sub findpath {
    my $myfullname = ${0};
    my($myname, $mypath) = fileparse($myfullname);

    return $mypath if ( $mypath && -d $mypath );
    $mypath=`pwd`;

    my $pwdfullname = $mypath . "/" . $myname;
    return $mypath if ( -f $pwdfullname );
    return 0;
}

# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise
sub is_vespa_home {
    my($VESPA_HOME) = shift;
    my $COMMON_ENV="libexec/vespa/common-env.sh";
    if ( $VESPA_HOME && -d $VESPA_HOME ) {
        my $common_env = $VESPA_HOME . "/" . $COMMON_ENV;
        return $VESPA_HOME if -f $common_env;
    }
    return 0;
}

# Returns the home of Vespa, or dies if it cannot
sub findhome {
    # Try the VESPA_HOME env variable
    return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'});
    if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly
        die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n";
    }

    # Try the ROOT env variable
    $ROOT = $ENV{'ROOT'};
    return $ROOT if is_vespa_home($ROOT);

    # Try the script location or current dir
    my $mypath = findpath();
    if ($mypath) {
        while ( $mypath =~ s|/[^/]*$|| ) {
            return $mypath if is_vespa_home($mypath);
        }
    }
    die "FATAL: Missing VESPA_HOME environment variable\n";
}

sub findhost {
    my $tmp = $ENV{'VESPA_HOSTNAME'};
    my $bin = $ENV{'VESPA_HOME'} . "/bin";
    if (!defined $tmp) {
        $tmp = `${bin}/vespa-detect-hostname || hostname -f || hostname || echo "localhost"`;
        chomp $tmp;
    }
    my $validate = "${bin}/vespa-validate-hostname";
    if (-f "${validate}") {
       system("${validate} $tmp");
       ( $? == 0 ) or die "Could not validate hostname\n";
    }
    return $tmp;
}

BEGIN {
    my $tmp = findhome();
    $ENV{'VESPA_HOME'} = $tmp;
    $tmp = findhost();
    $ENV{'VESPA_HOSTNAME'} = $tmp;
}
my $VESPA_HOME = $ENV{'VESPA_HOME'};

# END perl environment bootstrap section

use lib $ENV{'VESPA_HOME'} . '/lib/perl5/site_perl';
use lib $ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl';
use Yahoo::Vespa::Defaults;
readConfFile();

use strict;
use warnings;
use feature qw(switch say);
use vars qw/ $opt_c $opt_h $opt_n $opt_v $opt_f $opt_t $opt_a $opt_e $opt_E $opt_r $opt_i $opt_p $opt_H $opt_R $opt_F $opt_V /;
use Env qw($HOME);
use JSON;
use Getopt::Std;
use File::Path qw(make_path);
use Scalar::Util qw(looks_like_number);

my $cloudconfig_dir = "$HOME/.cloudconfig";
my $session_id_file;
my $configsource_url_used_file = "$cloudconfig_dir/deploy-configsource-url-used";



my $pathPrefix;
my $tenant = "default";
my $application = "default";
my $environment = "prod";
my $region = "default";
my $instance = "default";
my $version = "v2";
my $configserver = "";
my $port = "19071";
getopts('c:fhnt:ve:E:r:a:i:p:HR:F:V:');

if ($opt_h) {
  usage();
  exit 0;
}

if ($opt_c) {
    $configserver = $opt_c;
}

if ($opt_e) {
  $tenant = $opt_e;
}

if ($opt_r) {
  $region = $opt_r;
}

if ($opt_E) {
  $environment = $opt_E;
}

if ($opt_a) {
  $application = $opt_a;
}

if ($opt_i) {
  $instance = $opt_i;
}

if ($opt_p) {
  $port = $opt_p;
}

$pathPrefix = "/application/v2/tenant/$tenant/session";

create_cloudconfig_dir();

$session_id_file = "$cloudconfig_dir/$tenant/deploy-session-id";

my $command = shift;
$command ||= "help";

my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200';

my $CURL_PUT = $curl_command . ' --write-out \%{http_code} --request PUT';
my $CURL_GET = $curl_command . ' --request GET';
my $GZIP = "gzip";
my $CURL_POST_WITH_HEADERS = $curl_command . ' -i --request POST --header "Content-Type: application/x-gzip" --data-binary @- -D /tmp/http-headers';
my $CURL_POST = $curl_command . ' --write-out \%{http_code} --request POST --header "Content-Type: application/x-gzip" --data-binary @-';
my $CURL_POST_ZIP = $curl_command . ' --write-out \%{http_code} --request POST --header "Content-Type: application/zip" --data-binary @-';

if ($command eq "upload") {
  my $application_package = shift;
  if (!$opt_F) {
    if (!$application_package) {
      print "Command failed. No application package specified\n";
      usage("upload");
      exit 1;
    }
    if (!(-e $application_package)) {
      print "Command failed. No such directory found: '$application_package'\n";
      exit 1;
    }
    check_application_directory($application_package);
  }

  do_http_request("upload", $application_package);
} elsif ($command eq "prepare") {
    my $arg = shift;
    if ($arg && looks_like_number($arg) && !(-d $arg)) {
      do_http_request("prepare", "", $arg);
    } elsif ($arg) {
      check_application_directory($arg);
      do_http_request("upload", $arg);
      do_http_request("prepare");
    } else {
      do_http_request("prepare");
    }
} elsif ($command eq "activate") {
    my $session_id = shift;
    do_http_request("activate", "", $session_id);
} elsif ($command eq "fetch") {
    my $arg = shift;
    if ($arg) {
      fetch_active_application($arg);
    } else {
      usage("fetch", $arg);
    }
} elsif ($command eq "help") {
  my $arg = shift;
  usage($command, $arg);
} else {
    usage($command);
}


sub check_application_directory {
  my ($application_package) = shift;

  if (-d $application_package) {
    # OK
  } elsif ((-f $application_package) && ($application_package =~ /.*\.zip/ )) {
    # OK
  } else {
    print "Command failed. No directory or zip file found: '$application_package'\n";
    exit 1;
  }
}

sub usage {
  my ($command, $arg) = @_;

  if ($command && $command eq "help") {
    $command = $arg;
  }
  $command ||= "help";

  if ($command eq "upload") {
      usage_upload();
  } elsif ($command eq "prepare") {
      usage_prepare();
  } elsif ($command eq "activate") {
      usage_activate();
  } elsif ($command eq "fetch") {
      usage_fetch();
  } else {
    print "Usage: vespa-deploy [-h] [-v] [-f] [-t] [-c] [-p] [-z] [-V] [<command>] [args]\n";
    print "Supported commands: 'upload', 'prepare', 'activate', 'fetch' and 'help'\n";
    print "Supported options: '-h' (help), '-v' (verbose), '-f' (force/ignore validation errors), '-t' (timeout in seconds), '-p' (config server http port)\n";
    print "    '-h' (help)\n";
    print "    '-v' (verbose)\n";
    print "    '-n' (dry-run)\n";
    print "    '-f' (force/ignore validation errors)\n";
    print "    '-t <timeout>' (timeout in seconds)\n";
    print "    '-c <server>' (config server hostname)\n";
    print "    '-p <port>' (config server http port)\n";

    print "Try 'vespa-deploy help <command>' to get more help\n";
  }
}

sub usage_upload {
  print "Usage: vespa-deploy upload <application package>\n";
}

sub usage_prepare {
  print  "Usage: vespa-deploy prepare [<session_id> | <application package>]\n";
}

sub usage_activate {
  print "Usage: vespa-deploy activate [<session_id>]\n";
}

sub usage_fetch {
  print "Usage: vespa-deploy fetch <output directory>\n";
}

sub fetch_active_application {
  my ($outputdir) = @_;
  my ($configsource_url, @configsources) = get_configsource_url("fetch");
  my $url = "$configsource_url$pathPrefix/active/content/";
  if ($version eq "v2") {
    $url = $configsource_url
         . "/application/v2"
         . "/tenant/${tenant}"
         . "/application/${application}"
         . "/environment/${environment}"
         . "/region/${region}"
         . "/instance/${instance}"
         . "/content/";
  }
  my $output = http_content($url);
  my $exitcode = $? >> 8;
  if ($exitcode != 0) {
    print_request_failed($exitcode, $configsource_url);
    exit 1;
  } else {
    print "Writing active application to $outputdir\n";
    `mkdir -p $outputdir`;
    die "$outputdir is not writeable. Please check permissions\n" if (! -w $outputdir);
    my $json_text = get_json($output);
    if(ref($json_text) eq 'ARRAY'){
      fetch_directory($json_text, $outputdir);
    } else {
      print "Error response: $json_text->{message}\n";
      exit 1;
    }
  }
}

sub fetch_directory {
  my ($json, $outputdir) = @_;
  `mkdir -p $outputdir`;
  foreach my $entry (@{$json}) {
    my $name = "$outputdir/";
    if ($entry =~ /\/([^\/]+\/?)$/) {
      $name .= $1;
    }
    if ($name =~ /(.*)\/$/) {
      my $dir = $1;
      my $output = http_content($entry);
      my $json_text = get_json($output);
      fetch_directory($json_text, "$dir");
    } else {
      my $output = http_content($entry);
      open(FH, ">$name");
      print FH $output;
      close(FH);
    }
  }
}

sub get_configsource_url {
  my ($command) = @_;

  my @configsources;
  if ($configserver and $configserver ne "") {
      @configsources = ('http://' . $configserver . ':' . $port . '/');
  } else {
      @configsources = split(' ', `$VESPA_HOME/bin/vespa-print-default configservers_http`);
  }

  my $configsource_url = shift(@configsources);
  if (!$configsource_url) {
    die "Could not get url to config server, make sure that VESPA_HOME and VESPA_CONFIGSERVERS are set\n";
  }
  chomp($configsource_url);
  my @temp = split(':', $configsource_url, 3);
  $configsource_url = $temp[0] . ":" . $temp[1] . ":" . $port;
  if (!$configsource_url) {
    print "Could not get url to config server, make sure that VESPA_CONFIGSERVERS is set\n";
    exit 1;
  }

  # configsource_url to be used by prepare and activate
  if ($command eq "prepare" || $command eq "activate") {
    my $temp = get_configsource_url_used();
    if ($temp and $temp ne "") {
      $configsource_url = $temp;
      debug("Using config server URL " . $configsource_url . " read from file\n");
    } else {
      print "Could not read config server URL used for previous upload of an application package, trying to use $configsource_url\n";
    }
  }
  return ($configsource_url, @configsources);
}

sub do_http_request {
  my ($command, $application_package, $supplied_session_id) = @_;
  my ($configsource_url, @configsources) = get_configsource_url($command);

  my $output;
  my $exitcode = 1;
  if ($command eq "upload") {
    ($exitcode, $output) = http_upload(\@configsources, $configsource_url, $application_package);
  } elsif ($command eq "prepare")  {
    $output = http_prepare($configsource_url, $supplied_session_id);
    $exitcode = $? >> 8;
  } elsif ($command eq "activate") {
    $output = http_activate($configsource_url, $supplied_session_id);
    $exitcode = $? >> 8;
  }

  my $response;
  if ($exitcode != 0) {
    print_request_failed($exitcode, $configsource_url);
    exit 1;
  } else {
    my $status_code;
    ($status_code, $response) = parse_http_response($output);
    if ($status_code != 200) {
      print "Request failed. HTTP status code: $status_code\n";
      print_response($response);
      exit 1;
    }
    print_response($response);
  }
}

sub http_upload {
  my ($temp, $configsource_url, $application_package) = @_;
  my @configsources = @{$temp};

  my $output;
  my $exitcode = 0;
  my $retry = 0;
  my $configsource_url_used = $configsource_url;
  LOOP: {
      do {
        my $status_code;
        my $response;
        $output = http_upload_lowlevel($configsource_url, $application_package);
        ($status_code, $response) = parse_http_response($output);
        $exitcode = $? >> 8;
        last LOOP if ($exitcode == 0 && $status_code == 200);

        debug("exitcode=$exitcode\n");
        debug("output=$output\n");
        $configsource_url = shift(@configsources);
        if ($configsource_url) {
          $configsource_url_used = $configsource_url;
          $retry = 1;
          print_request_failed($exitcode, $configsource_url_used);
          print "Retrying with another config server\n";
        } else {
          if ($exitcode != 0) {
            print_request_failed($exitcode, $configsource_url_used);
            exit 1;
          } else { # Non-200 HTTP status code
            print "Request failed. HTTP status code: $status_code\n";
            print_response($response);
            exit 1;
          }
        }
      } while ($retry);
    }

  write_session_id($output);
  write_configsource_url_used($configsource_url_used);

  return ($exitcode, $output);
}

sub http_upload_lowlevel {
  my ($source, $app) = @_;

  my $url = $source . $pathPrefix;
  $url = add_url_property_from_flag($url, $opt_v, "verbose");
  if ($opt_F) {
    $url = add_url_property_from_option($url, $opt_F, "from");
    `$CURL_POST $url`;
  } else {
    my $TAR="tar -C $app --dereference --exclude='.[a-zA-Z0-9]*' --exclude=ext -cf - . --transform=\"s#^#application/#\" ";
    print "Uploading application '$app' using $url\n";
    if (-f $app) {
      `cat $app | $CURL_POST_ZIP $url`;
    } else {
      `$TAR | $GZIP | $CURL_POST $url`;
    }
  }
}

sub http_prepare {
  my $source = shift;
  my $session_id = shift || get_session_id();

  my $url = $source . $pathPrefix . "/$session_id/prepared";
  $url = add_url_property_from_flag($url, $opt_f, "ignoreValidationErrors");
  $url = add_url_property_from_flag($url, $opt_n, "dryrun");
  $url = add_url_property_from_flag($url, $opt_v, "verbose");
  $url = add_url_property_from_flag($url, $opt_H, "hostedVespa");
  $url = add_url_property_from_option($url, $opt_a, "applicationName");
  $url = add_url_property_from_option($url, $opt_i, "instance");
  $url = add_url_property_from_option($url, $opt_t, "timeout");
  $url = add_url_property_from_option($url, $opt_R, "rotations");
  $url = add_url_property_from_option($url, $opt_V, "vespaVersion");
  print "Preparing session $session_id using $url\n";
  `$CURL_PUT \"$url\"`;
}

sub http_content {
  my $url = shift;
  print "Getting content using $url\n";
  `$CURL_GET \"$url\"`;
}

sub http_activate {
  my $source = shift;
  my $session_id = shift || get_session_id();

  my $url = $source . $pathPrefix . "/$session_id/active";
  $url = add_url_property_from_flag($url, $opt_v, "verbose");
  $url = add_url_property_from_option($url, $opt_t, "timeout");
  print "Activating session $session_id using $url\n";
  `$CURL_PUT \"$url\"`;
}

sub get_session_id {
  my $session_id = `cat $session_id_file 2>/dev/null`;
  unless ($session_id) {
    print "Could not read session id from file, and no session id supplied as argument. Exiting.\n";
    exit 1
  }
  $session_id;
}

sub get_session_id_from_response {
  my ($response) = @_;

  my $new_session_id;
  if ($response =~ /.*"session-id":"(\d+)".*/) { $new_session_id = $1; }
  $new_session_id;
}

sub print_response {
  my ($response) = @_;
  chomp($response);
  debug("$response\n");
  if ($response) {
    my $json_text = get_json($response);
    my $error = $json_text->{error};
    if ($error) {
      print "$json_text->{error}\n";
    }
    my $status = $json_text->{status};
    foreach my $log_message (@{$json_text->{log}}) {
      print "$log_message->{level}: $log_message->{message}\n";
    }
    my $message = $json_text->{message};
    if ($message) {
      print "$message\n";
    }
    my $metadata_deploy = $json_text->{deploy};
    if ($metadata_deploy) {
      my $timestamp = $metadata_deploy->{timestamp};
      my $metadata_application = $json_text->{application};
      my $checksum = $metadata_application->{checksum};
      my $generation = $metadata_application->{generation};
      print "Checksum:   $checksum\n";
      print "Timestamp:  $timestamp\n";
      print "Generation: $generation\n";
    }
  } else {
    print "Empty response";
  }
}

sub get_json {
  my ($response) = @_;

  my $json_dir = JSON->new;
  return $json_dir->utf8->decode($response);
}

# extend $url with $url_property=true if $flag is set
sub add_url_property_from_flag {
   my ($url, $flag, $url_property) = @_;
   return $url unless $flag;
   return add_url_property($url, "$url_property=true");
}

# extend $url with $url_property=$opt if $opt is set
sub add_url_property_from_option {
  my ($url, $opt, $url_property) = @_;
  return $url unless $opt;
  add_url_property($url, "$url_property=$opt");
}

sub add_url_property {
  my ($url, $url_property) = @_;
  if ($url =~ /\?/) {
    $url = $url . "&" . $url_property;
  } else {
    $url = $url . "?" . $url_property;
  }
  $url;
}

sub write_session_id {
  my ($response) = @_;

  my $new_session_id = get_session_id_from_response($response);
  if ($new_session_id) {
    open(my $fh, '>', $session_id_file) or die "Could not open file '$session_id_file' $!";
    print $fh $new_session_id;
    close $fh;
  }
}

sub write_configsource_url_used {
  my ($configsource_url) = @_;

  if ($configsource_url) {
    open(my $fh2, '>', $configsource_url_used_file) or die "Could not open file '$configsource_url_used_file' $!";
    print $fh2 $configsource_url;
    close $fh2;
  }
}

sub get_configsource_url_used {
  my $configsource_url = `cat $configsource_url_used_file 2>/dev/null` || "";
  $configsource_url;
}

sub print_request_failed {
  my ($exitcode, $configsource_url) = @_;

  my $message = "HTTP request failed";
  if ($exitcode == 7) {
    $message .= ". Could not connect to $configsource_url";
  } else {
    $message .= " with curl exit code $exitcode";
  }

  print $message . "\n";
}


sub debug {
  my ($message) = @_;

  if ($opt_v) {
    print "$message";
  }
}

sub parse_http_response {
  my ($response) = @_;

  my $message = "";
  my $status_code = 500;

  if ($response =~ /(.*)(\d\d\d)/) {
      $message = $1;
      $status_code = int($2);
    }
  return ($status_code, $message);
}

sub create_cloudconfig_dir {
  my $path = "$cloudconfig_dir/$tenant";
  if (-e $path) {
    check_dir_permissions($path);
  } else {
      make_path($path);
  }
}


sub check_dir_permissions {
  my ($dir) = @_;

  if (!(-d $dir)) {
    print "$dir is not a directory, please fix\n";
  }
  if (!(-r $dir)) {
    print "$dir is not readable, please fix\n";
  }
}