Publish

4 files changed, 271 insertions, 0 deletions

diff --git a/filedistribution/src/apps/status/.gitignore b/filedistribution/src/apps/status/.gitignore
new file mode 100644
index 00000000000..6dc1c1fff5d
--- /dev/null
+++ b/filedistribution/src/apps/status/.gitignore
@@ -0,0 +1,2 @@
+/status-filedistribution
+filedistribution_status-filedistribution_app
diff --git a/filedistribution/src/apps/status/CMakeLists.txt b/filedistribution/src/apps/status/CMakeLists.txt
new file mode 100644
index 00000000000..fb0b76af55b
--- /dev/null
+++ b/filedistribution/src/apps/status/CMakeLists.txt
@@ -0,0 +1,16 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(filedistribution_status-filedistribution_app
+    SOURCES
+    status-filedistribution.cpp
+    INSTALL bin
+    DEPENDS
+    filedistribution_filedistributionmodel
+    filedistribution_common
+)
+target_compile_options(filedistribution_status-filedistribution_app PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED)
+vespa_add_target_system_dependency(filedistribution_status-filedistribution_app boost boost_system-mt-d)
+vespa_add_target_system_dependency(filedistribution_status-filedistribution_app boost boost_thread-mt-d)
+vespa_add_target_system_dependency(filedistribution_status-filedistribution_app boost boost_program_options-mt-d)
+vespa_add_target_system_dependency(filedistribution_status-filedistribution_app boost boost_filesystem-mt-d)
+vespa_add_target_system_dependency(filedistribution_status-filedistribution_app boost boost_unit_test_framework-mt-d)
+vespa_install_script(vespa-status-filedistribution.sh vespa-status-filedistribution bin)
diff --git a/filedistribution/src/apps/status/status-filedistribution.cpp b/filedistribution/src/apps/status/status-filedistribution.cpp
new file mode 100644
index 00000000000..90c21623016
--- /dev/null
+++ b/filedistribution/src/apps/status/status-filedistribution.cpp
@@ -0,0 +1,185 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("status-filedistribution");
+
+#include <iostream>
+#include <map>
+
+#include <boost/program_options.hpp>
+#include <boost/foreach.hpp>
+#include <boost/thread.hpp>
+
+#include <vespa/filedistribution/common/exceptionrethrower.h>
+#include <vespa/filedistribution/model/zkfacade.h>
+#include <vespa/filedistribution/model/filedistributionmodel.h>
+#include <vespa/filedistribution/model/filedistributionmodelimpl.h>
+#include <zookeeper/zookeeper.h>
+
+using namespace filedistribution;
+
+std::string
+plural(size_t size)
+{
+    if (size == 1)
+        return "";
+    else
+        return "s";
+}
+
+template <class CONT>
+std::string
+plural(const CONT& cont)
+{
+    size_t size = cont.size();
+    return plural(size);
+}
+
+typedef FileDBModel::HostStatus HostStatus;
+typedef std::map<std::string, HostStatus> StatusByHostName;
+
+void
+printWaitingForHosts(const StatusByHostName& notFinishedHosts)
+{
+    std::cout <<"Waiting for the following host" <<plural(notFinishedHosts) <<":" <<std::endl;
+    BOOST_FOREACH(const StatusByHostName::value_type hostNameAndStatus, notFinishedHosts) {
+        std::cout <<hostNameAndStatus.first <<"  (";
+
+        const HostStatus& hostStatus = hostNameAndStatus.second;
+        if (hostStatus._state == HostStatus::notStarted) {
+            std::cout <<"Not started";
+        } else {
+            std::cout <<"Downloading, "
+                      <<hostStatus._numFilesFinished <<"/" <<hostStatus._numFilesToDownload
+                      <<" file" <<plural(hostStatus._numFilesToDownload) <<" completed";
+        }
+        std::cout <<")" <<std::endl;
+    }
+}
+
+//TODO:refactor
+int printStatus(const std::string& zkservers)
+{
+    boost::shared_ptr<ExceptionRethrower> exceptionRethrower;
+    boost::shared_ptr<ZKFacade> zk(new ZKFacade(zkservers, exceptionRethrower));
+
+    boost::shared_ptr<FileDBModel> model(new ZKFileDBModel(zk));
+
+    std::vector<std::string> hosts = model->getHosts();
+
+    StatusByHostName notFinishedHosts;
+    StatusByHostName finishedHosts;
+    bool hasStarted = false;
+
+    BOOST_FOREACH(std::string host, hosts) {
+        HostStatus hostStatus = model->getHostStatus(host);
+        switch (hostStatus._state) {
+          case HostStatus::finished:
+            hasStarted = true;
+            finishedHosts[host] = hostStatus;
+            break;
+          case HostStatus::inProgress:
+            hasStarted = true;
+          case HostStatus::notStarted:
+            notFinishedHosts[host] = hostStatus;
+            break;
+        }
+    }
+
+    if (notFinishedHosts.empty()) {
+        std::cout <<"Finished distributing files to all hosts." <<std::endl;
+        return 0;
+    } else if (!hasStarted) {
+        std::cout <<"File distribution has not yet started." <<std::endl;
+        return 0;
+    } else {
+        printWaitingForHosts(notFinishedHosts);
+        return 5;
+    }
+}
+
+int
+printStatusRetryIfZKProblem(const std::string& zkservers, const std::string& zkLogFile)
+{
+    FILE* file = fopen(zkLogFile.c_str(), "w");
+    if (file == NULL) {
+         std::cerr <<"Could not open file " <<zkLogFile <<std::endl;
+    } else {
+         zoo_set_log_stream(file);
+    }
+    zoo_set_debug_level(ZOO_LOG_LEVEL_ERROR);
+
+    for (int i = 0; i < 5; ++i) {
+        try {
+            return printStatus(zkservers);
+        } catch (ZKNodeDoesNotExistsException& e) {
+            LOG(debug, "Node does not exists, assuming concurrent update. %s", boost::diagnostic_information(e).c_str());
+
+        } catch (ZKSessionExpired& e) {
+            LOG(debug, "Session expired.");
+        }
+        boost::this_thread::sleep(boost::posix_time::milliseconds(500));
+    }
+    return 4;
+}
+
+
+//TODO: move to common
+struct ProgramOptionException {
+    std::string _msg;
+    ProgramOptionException(const std::string & msg)
+        : _msg(msg)
+    {}
+};
+
+bool exists(const std::string& optionName, const boost::program_options::variables_map& map) {
+    return map.find(optionName) != map.end();
+}
+
+void ensureExists(const std::string& optionName, const boost::program_options::variables_map& map \
+                  ) {
+    if (!exists(optionName, map)) {
+        throw ProgramOptionException("Error: Missing option " + optionName);
+    }
+}
+//END: move to common
+
+
+int
+main(int argc, char** argv) {
+    const char
+        *zkstring = "zkstring",
+        *zkLogFile = "zkLogFile",
+        *help = "help";
+
+    namespace po = boost::program_options;
+    boost::program_options::options_description description;
+    description.add_options()
+        (zkstring, po::value<std::string > (), "The zookeeper servers to connect to, separated by comma")
+        (zkLogFile, po::value<std::string >() -> default_value("/dev/null"), "Zookeeper log file")
+        (help, "help");
+
+    try {
+        boost::program_options::variables_map values;
+        po::store(
+                boost::program_options::parse_command_line(argc, argv, description),
+                values);
+
+        if (exists(help, values)) {
+            std::cout <<description;
+            return 0;
+        }
+
+        ensureExists(zkstring, values);
+
+        return printStatusRetryIfZKProblem(
+                values[zkstring] .as<std::string>(),
+                values[zkLogFile].as<std::string>());
+    } catch (const ProgramOptionException& e) {
+        std::cerr <<e._msg <<std::endl;
+        return 3;
+    } catch(const std::exception& e) {
+        std::cerr <<"Error: " <<e.what() <<std::endl;
+        return 3;
+    }
+}
diff --git a/filedistribution/src/apps/status/vespa-status-filedistribution.sh b/filedistribution/src/apps/status/vespa-status-filedistribution.sh
new file mode 100644
index 00000000000..477e4f6bff5
--- /dev/null
+++ b/filedistribution/src/apps/status/vespa-status-filedistribution.sh
@@ -0,0 +1,68 @@
+#!/bin/sh
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+    myname=${0}
+    mypath=${myname%/*}
+    myname=${myname##*/}
+    if [ "$mypath" ] && [ -d "$mypath" ]; then
+        return
+    fi
+    mypath=$(pwd)
+    if [ -f "${mypath}/${myname}" ]; then
+        return
+    fi
+    echo "FATAL: Could not figure out the path where $myname lives from $0"
+    exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+    if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+        # ensure it ends with "/" :
+        VESPA_HOME=${VESPA_HOME%/}/
+        export VESPA_HOME
+        common_env=$VESPA_HOME/$COMMON_ENV
+        if [ -f "$common_env" ]; then
+            . $common_env
+            return
+        fi
+    fi
+    return 1
+}
+
+findroot () {
+    source_common_env && return
+    if [ "$VESPA_HOME" ]; then
+        echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+        exit 1
+    fi
+    if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+        VESPA_HOME="$ROOT"
+        source_common_env && return
+    fi
+    findpath
+    while [ "$mypath" ]; do
+        VESPA_HOME=${mypath}
+        source_common_env && return
+        mypath=${mypath%/*}
+    done
+    echo "FATAL: missing VESPA_HOME environment variable"
+    echo "Could not locate $COMMON_ENV anywhere"
+    exit 1
+}
+
+findroot
+
+# END environment bootstrap section
+
+ROOT=$VESPA_HOME
+
+ZKSTRING=$($ROOT/libexec/vespa/vespa-config.pl -zkstring)
+test -z "$VESPA_LOG_LEVEL" && VESPA_LOG_LEVEL=warning
+export VESPA_LOG_LEVEL
+exec $ROOT/libexec/vespa/status-filedistribution --zkstring "$ZKSTRING" $@