diff options
author | freva <valerijf@yahoo-inc.com> | 2016-10-27 13:55:41 +0200 |
---|---|---|
committer | freva <valerijf@yahoo-inc.com> | 2016-10-27 13:55:41 +0200 |
commit | a3b52baf7e3bac1071bf2644aae99b85552ca976 (patch) | |
tree | e6432b5cce27f2f4910747fb858cce8f62d8548a /node-admin/scripts | |
parent | cf5078fceb2c9cc7305f8e1b297cf01e7bd6826c (diff) |
Removed unused files
Diffstat (limited to 'node-admin/scripts')
-rwxr-xr-x | node-admin/scripts/app.sh | 156 | ||||
-rw-r--r-- | node-admin/scripts/common-vm.sh | 13 | ||||
-rw-r--r-- | node-admin/scripts/common.sh | 180 | ||||
-rwxr-xr-x | node-admin/scripts/config-server.sh | 141 | ||||
-rwxr-xr-x | node-admin/scripts/configure-container-networking.py | 273 | ||||
-rwxr-xr-x | node-admin/scripts/etc-hosts.sh | 43 | ||||
-rwxr-xr-x | node-admin/scripts/make-host-like-container.sh | 52 | ||||
-rwxr-xr-x | node-admin/scripts/network-bridge.sh | 63 | ||||
-rwxr-xr-x | node-admin/scripts/node-admin.sh | 73 | ||||
-rwxr-xr-x | node-admin/scripts/populate-noderepo-with-local-nodes.sh | 44 | ||||
-rwxr-xr-x | node-admin/scripts/route-osx.sh | 16 | ||||
-rwxr-xr-x | node-admin/scripts/setup-docker.sh | 176 | ||||
-rwxr-xr-x | node-admin/scripts/setup-route-and-hosts-osx.sh | 20 | ||||
-rwxr-xr-x | node-admin/scripts/vm.sh | 77 | ||||
-rwxr-xr-x | node-admin/scripts/zone.sh | 80 |
15 files changed, 14 insertions, 1393 deletions
diff --git a/node-admin/scripts/app.sh b/node-admin/scripts/app.sh deleted file mode 100755 index d3eb6996ab4..00000000000 --- a/node-admin/scripts/app.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -source "${0%/*}"/common.sh - -declare SCRIPTS_DIR="${0%/*}" - -declare -r APP_DIR_NAME_UNDER_SHARED=app - -function Usage { - UsageHelper "$@" <<EOF -Usage: $SCRIPT_NAME <command> [<app-dir>] -Deploy (or undeploy) application rooted at <app-dir> on localhost Config Server. - -The local zone must be up and running. <app-dir> should point to -e.g. vespa/basic-search-on-docker/target/application. -EOF -} - -function RunOnConfigServer { - docker exec config-server "$@" -} - -function VerifyApp { - local app_dir="$1" - - # Sanity-check app_dir - if ! [ -d "$app_dir" ] - then - Fail "<app-dir> '$app_dir' is not a directory" - fi - - local services_xml="$app_dir"/services.xml - if ! [ -f "$services_xml" ] - then - Fail "Failed to find services.xml in <app-dir> '$app_dir'" - fi - - # Verify there's no <admin> element. - if grep -qE '<admin[ >]' "$services_xml" - then - Fail "services.xml cannot contain an <admin> element in hosted Vespa" - fi - - # Verify <nodes> seems to be correctly specified (warning: this test is - # incomplete). - if grep -qE "<nodes>" "$services_xml" || - ! grep -qE "<nodes (.* )?docker-image=" "$services_xml" || - ! grep -qE "<nodes (.* )?flavor=[\"']docker[\"']" "$services_xml" - then - Fail "You must specify the <nodes> element in the following form" \ - "in hosted Vespa w/Docker:" \ - " <nodes count=\"2\" flavor=\"docker\" docker-image=\"IMAGE\" />" \ - "where IMAGE is e.g. vespa-local:latest." - fi -} - -# Copies the application rooted at $1 to a directory tree shared with the -# Config Server. -function CopyToSharedDir { - local app_dir="$1" - - local shared_dir_on_localhost="$APPLICATION_STORAGE_ROOT/$CONFIG_SERVER_CONTAINER_NAME/$ROOT_DIR_SHARED_WITH_HOST" - if ! [ -d "$shared_dir_on_localhost" ] - then - Fail "Failed to find the Config Server's shared directory on" \ - "localhost '$shared_dir_on_localhost', has the" \ - "$CONFIG_SERVER_CONTAINER_NAME container been started?" - fi - - - local shared_app_dir_on_localhost="$shared_dir_on_localhost/$APP_DIR_NAME_UNDER_SHARED" - if [ "$shared_app_dir_on_localhost" != /home/docker/container-storage/config-server/shared/app ] - then - # This duplication of code is a safety-guard against 'rm -rf' unknown - # directories. - Fail "We're about to remove '$shared_app_dir_on_localhost', but it's" \ - "pointing to something unexpected, refusing to proceed..." - fi - - echo -n "Copying application to '$shared_app_dir_on_localhost'... " - rm -rf "$shared_app_dir_on_localhost" - cp -r "$app_dir" "$shared_app_dir_on_localhost" - echo done -} - -function DeployApp { - if (($# != 1)) - then - Usage - fi - - local app_dir="$1" - - CopyToSharedDir "$app_dir" - - # Create tenant - echo -n "Creating tenant... " - local create_tenant_response - if create_tenant_response=$(curl --silent --show-error -X PUT "http://$CONFIG_SERVER_HOSTNAME:$VESPA_WEB_SERVICE_PORT/application/v2/tenant/$TENANT_NAME" 2>&1) - then - if ! [[ "$create_tenant_response" =~ "Tenant $TENANT_NAME created" ]] && - ! [[ "$create_tenant_response" =~ "already exists" ]] - then - echo - Fail "May have failed to create the tenant: '$create_tenant_response'" - fi - else - echo - Fail "Failed to create the tenant: $?: '$create_tenant_response'" - fi - echo done - - # Deploy app - local app_dir_on_config_server="/$ROOT_DIR_SHARED_WITH_HOST/$APP_DIR_NAME_UNDER_SHARED" - RunOnConfigServer $VESPA_HOME/bin/deploy -e "$TENANT_NAME" prepare "$app_dir_on_config_server" - echo "Activating application" - RunOnConfigServer $VESPA_HOME/bin/deploy -e "$TENANT_NAME" activate -} - -function UndeployApp { - if (($# != 0)) - then - Usage "undeploy takes no arguments" - fi - - local app_name=default - local output - echo -n "Removing application $TENANT_NAME:$app_name... " - if ! output=$(curl --silent --show-error -X DELETE "http://$CONFIG_SERVER_HOSTNAME:$VESPA_WEB_SERVICE_PORT/application/v2/tenant/$TENANT_NAME/application/$app_name") - then - echo - Fail "Failed to remove application: $output" - fi - - echo done -} - -function Main { - if (($# == 0)) - then - Usage "Missing command" - fi - local command="$1" - shift - - case "$command" in - deploy) DeployApp "$@" ;; - undeploy) UndeployApp "$@" ;; - *) Usage "Unknown command '$command'" ;; - esac -} - -Main "$@" diff --git a/node-admin/scripts/common-vm.sh b/node-admin/scripts/common-vm.sh deleted file mode 100644 index c91c75e1404..00000000000 --- a/node-admin/scripts/common-vm.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -source "${0%/*}/common.sh" - -# VM configuration -declare -r DOCKER_VM_NAME=vespa # Don't put spaces in the name -declare -r DOCKER_VM_DISK_SIZE_IN_MB=40000 -declare -r DOCKER_VM_MEMORY_SIZE_IN_MB=4096 -declare -r DOCKER_VM_CPU_COUNT=1 -declare -r DOCKER_VM_HOST_CIDR=172.21.46.1/24 diff --git a/node-admin/scripts/common.sh b/node-admin/scripts/common.sh deleted file mode 100644 index 6a10fb71a99..00000000000 --- a/node-admin/scripts/common.sh +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Common variables and functions that may be useful for scripts IN THIS -# DIRECTORY. Should be sourced as follows: -# -# source "${0%/*}/common.sh" -# -# WARNING: Some system variables, like the Config Server's, are also hardcoded -# in the Docker image startup scripts. - -declare -r SCRIPT_NAME="${0##*/}" -declare -r SCRIPT_DIR="${0%/*}" - -# TODO: Find a better name. Consider having separate images for config-server -# and node-admin. -declare -r DOCKER_IMAGE="vespa-local:latest" -declare -r APPLICATION_STORAGE_ROOT="/home/docker/container-storage" -declare -r ROOT_DIR_SHARED_WITH_HOST=shared - -# The 172.18.0.0/16 network is in IPDB. -declare -r NETWORK_PREFIX=172.18 -declare -r NETWORK_PREFIX_BITLENGTH=16 - -# Hostnames, IP addresses, names, etc of the infrastructure containers. -declare -r HOST_BRIDGE_INTERFACE=vespa -declare -r HOST_BRIDGE_IP="$NETWORK_PREFIX.0.1" -declare -r HOST_BRIDGE_NETWORK="$NETWORK_PREFIX.0.0/$NETWORK_PREFIX_BITLENGTH" -declare -r NODE_ADMIN_CONTAINER_NAME=node-admin -declare -r CONFIG_SERVER_CONTAINER_NAME=config-server -declare -r CONFIG_SERVER_HOSTNAME="$CONFIG_SERVER_CONTAINER_NAME" -declare -r CONFIG_SERVER_IP="$NETWORK_PREFIX.1.1" -declare -r VESPA_WEB_SERVICE_PORT=4080 # E.g. config server port - -declare -r DEFAULT_HOSTED_VESPA_REGION=local-region -declare -r DEFAULT_HOSTED_VESPA_ENVIRONMENT=prod - -# Hostnames, IP addresses, names, etc of the application containers. Hostname -# and container names are of the form $PREFIX$N, where N is a number between 1 -# and $NUM_APP_CONTAINERS. The IP is $APP_NETWORK_PREFIX.$N. -declare -r APP_NETWORK_PREFIX="$NETWORK_PREFIX.2" -declare -r APP_CONTAINER_NAME_PREFIX=cnode- -declare -r APP_HOSTNAME_PREFIX="$APP_CONTAINER_NAME_PREFIX" -declare -r DEFAULT_NUM_APP_CONTAINERS=20 # Statically allocated number of nodes. -declare -r TENANT_NAME=localtenant - -# May be 'vm' if docker hosts runs within a VM (osx). Default is native/Fedora. -declare -r NETWORK_TYPE="${NETWORK_TYPE:-local}" - -# Allowed program opions -declare OPTION_NUM_NODES # Set from --num-nodes or DEFAULT_NUM_APP_CONTAINERS, see Main. -declare OPTION_WAIT # Set from --wait or true, see Main. -declare OPTION_HV_REGION # Set from --hv-region or DEFAULT_HOSTED_VESPA_REGION, see Main. -declare OPTION_HV_ENV # Set from --hv-env or DEFAULT_HOSTED_VESPA_ENVIRONMENT, see Main. - -declare NUM_APP_CONTAINERS # Set from OPTION_NUM_NODES or DEFAULT_NUM_APP_CONTAINERS, see Main. - -function Fail { - printf "%s\n" "$@" >&2 - exit 1 -} - -# Used to help scripts with implementing the Usage function. The intended usage -# is: -# -# function Usage { -# UsageHelper "$@" <<EOF -# Usage: $SCRIPT_NAME ... -# ... -# EOF -# } -# -# When Usage is called, any arguments passed will be printed to stderr, then -# the usage-string will be printed (on stdin for UsageHelper), then the process -# will exit with code 1. -function UsageHelper { - exec >&2 - - if (($# > 0)) - then - printf "%s\n\n" "$*" - fi - - # Print to stdout (which has been redirected to stderr) what's on - # stdin. This will print the usage-string. - cat - - exit 1 -} - -# See Main -function Restart { - Stop - Start "$@" -} - -# Use Main as follows: -# -# Pass all script arguments to Main: -# -# Main "$@" -# -# Main will parse the arguments as follows. It assumes the arguments have -# the following form: -# -# script.sh <command> [<arg> | <option>]... -# -# where <command> is one of start, stop, or restart: -# start: The script MUST define a Start function. -# stop: The script MUST define a Stop function. -# restart: common.sh defines a Restart function to mean Stop, then Start. -# -# <arg> cannot start with a dash, and will get passed as argument to the Start -# function (if applicable). -# -# <option> is either of the form --<name>=<value> or --<name> <value>. -# <name>/<value> denotes a set of options. For each option, it sets one of the -# predefined global OPTION_* options. -# -# Having parsed the arguments, Main then calls Start, Restart, or Stop, -# depending on the command. These functions must be defined by the script. -# -# A function Usage must also be defined, which will be called when there's a -# usage error. -function Main { - # Default command is start - local command=start - if (($# > 0)) && ! [[ "$1" =~ ^- ]] - then - command="$1" - shift - fi - - local -a args=() - - while (($# > 0)) - do - if [[ "$1" =~ ^--([a-z0-9][a-z0-9-]*)(=(.*))?$ ]] - then - # Option argument - local name="${BASH_REMATCH[1]}" - shift - - if ((${#BASH_REMATCH[2]} > 0)) - then - local value="${BASH_REMATCH[3]}" - else - if (($# == 0)) - then - Usage "Option '$name' missing value" - fi - - value="$1" - shift - fi - - case "$name" in - num-nodes) OPTION_NUM_NODES="$value" ;; - wait) OPTION_WAIT="$value" ;; - hv-region) OPTION_HV_REGION="$value" ;; - hv-env) OPTION_HV_ENV="$value" ;; - esac - elif [[ "$1" =~ ^[^-] ]] - then - # Non-option argument - args+=("$1") - shift - else - Usage "Bad argument '$1'" - fi - done - - NUM_APP_CONTAINERS="${OPTION_NUM_NODES:-$DEFAULT_NUM_APP_CONTAINERS}" - - case "$command" in - help) Usage ;; - stop) Stop ;; - start) Start "${args[@]}" ;; - restart) Restart "${args[@]}" ;; - *) Usage "Unknown command '$command'" - esac -} diff --git a/node-admin/scripts/config-server.sh b/node-admin/scripts/config-server.sh deleted file mode 100755 index 0806f4374c6..00000000000 --- a/node-admin/scripts/config-server.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -source "${0%/*}/common.sh" - -declare CONTAINER_ROOT_DIR="$APPLICATION_STORAGE_ROOT/$CONFIG_SERVER_CONTAINER_NAME" - -function Usage { - UsageHelper "$@" <<EOF -Usage: $SCRIPT_NAME <command> [--wait] -Manage the Config Server - -Commands: - start Start the Config Server in a Docker container - stop Remove the Config Server container - restart Stop, then start - -Options: - --hv-env <env> - Start the config server with the given hosted Vespa environment - name. Must be one of prod, dev, test, staging, etc. Default is - $DEFAULT_HOSTED_VESPA_ENVIRONMENT. - --hv-region <region> - Start the config server with the given hosted Vespa region name. - Default is $DEFAULT_HOSTED_VESPA_REGION. - --wait true - Make start wait until the Config Server is healthy -EOF -} - -function Stop { - # Prime sudo - sudo true - - echo -n "Removing $CONFIG_SERVER_CONTAINER_NAME container... " - docker rm -f "$CONFIG_SERVER_CONTAINER_NAME" &>/dev/null || true - echo done - - if [ -d "$CONTAINER_ROOT_DIR" ] - then - # Double-check we're not 'rm -rf' something unexpected! - if ! [[ "$CONTAINER_ROOT_DIR" =~ ^/home/docker/container-storage/ ]] - then - Fail "DANGEROUS: Almost removed '$CONTAINER_ROOT_DIR'..." - fi - - echo -n "Removing container dir $CONTAINER_ROOT_DIR... " - sudo rm -rf "$CONTAINER_ROOT_DIR" - # The next two statements will prune empty parent directories. - sudo mkdir "$CONTAINER_ROOT_DIR" - sudo rmdir --ignore-fail-on-non-empty -p "$CONTAINER_ROOT_DIR" - echo done - fi -} - -function Start { - # Prime sudo - sudo true - - local wait="${OPTION_WAIT:-true}" - case "$wait" in - true|false) : ;; - *) Usage "--wait should only be set to true or false" ;; - esac - - local region="${OPTION_HV_REGION:-$DEFAULT_HOSTED_VESPA_REGION}" - local environment="${OPTION_HV_ENV:-$DEFAULT_HOSTED_VESPA_ENVIRONMENT}" - - echo -n "Creating container dir $CONTAINER_ROOT_DIR... " - local shared_dir_on_localhost="$APPLICATION_STORAGE_ROOT/$CONFIG_SERVER_CONTAINER_NAME/$ROOT_DIR_SHARED_WITH_HOST" - sudo mkdir -p "$shared_dir_on_localhost" - sudo chmod a+wt "$shared_dir_on_localhost" - echo done - - # Start config server - echo -n "Making $CONFIG_SERVER_CONTAINER_NAME container... " - local config_server_container_id - config_server_container_id=$(\ - docker run \ - --detach \ - --cap-add=NET_ADMIN \ - --net=none \ - --hostname "$CONFIG_SERVER_HOSTNAME" \ - --name "$CONFIG_SERVER_CONTAINER_NAME" \ - --volume "/etc/hosts:/etc/hosts" \ - --volume "$shared_dir_on_localhost:/$ROOT_DIR_SHARED_WITH_HOST" \ - --env "HOSTED_VESPA_REGION=$region" \ - --env "HOSTED_VESPA_ENVIRONMENT=$environment" \ - --env "CONFIG_SERVER_HOSTNAME=$CONFIG_SERVER_HOSTNAME" \ - --env "VESPA_HOME=$VESPA_HOME" \ - --env "HOST_BRIDGE_IP=$HOST_BRIDGE_IP" \ - --entrypoint /usr/local/bin/start-config-server.sh \ - "$DOCKER_IMAGE") - echo done - - echo -n "Verifying that $CONFIG_SERVER_CONTAINER_NAME container is running... " - local config_server_container_pid - config_server_container_pid=$(docker inspect -f '{{.State.Pid}}' "$CONFIG_SERVER_CONTAINER_NAME") - - echo -n "(pid $config_server_container_pid) " - - # TODO: Use .State.Status instead (only supported from version 1.9). - local config_server_container_running - config_server_container_running=$(docker inspect -f '{{.State.Running}}' "$CONFIG_SERVER_CONTAINER_NAME") - - if [ "$config_server_container_pid" == 0 -o "$config_server_container_running" != true ] - then - echo "failed" - Fail "The Config Server is not running anymore, consider looking" \ - "at the logs with 'docker logs $CONFIG_SERVER_CONTAINER_NAME'" - fi - echo "done" - - echo -n "Setting up the $CONFIG_SERVER_CONTAINER_NAME container network of type $NETWORK_TYPE... " - if ! script_out=$(sudo ./configure-container-networking.py --"$NETWORK_TYPE" "$config_server_container_pid" "$CONFIG_SERVER_IP" 2>&1); then - echo "failed" - echo "$script_out" - exit - fi - echo "done" - - if [ "$wait" == true ] - then - # Wait for config server to come up - echo -n "Waiting for healthy Config Server (~30s)" - local url="http://$CONFIG_SERVER_HOSTNAME:$VESPA_WEB_SERVICE_PORT/state/v1/health" - while ! curl --silent --fail --max-time 1 "$url" >/dev/null - do - echo -n . - sleep 2 - done - echo " done" - fi -} - -# Makes it easier to access scripts in the same 'scripts' directory -cd "$SCRIPT_DIR" - -Main "$@" diff --git a/node-admin/scripts/configure-container-networking.py b/node-admin/scripts/configure-container-networking.py index 516d223494f..2efe0891e00 100755 --- a/node-admin/scripts/configure-container-networking.py +++ b/node-admin/scripts/configure-container-networking.py @@ -1,32 +1,21 @@ #!/usr/bin/env python # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Quick and dirty script to set up routable ip address for docker container -# Remove when docker releases a plugin api to configure networking. -# TODO: Refactor for readability - +# Quick and dirty script to specify the default gateway on docker interface from __future__ import print_function - -import hashlib -import ipaddress import os import sys - -from pyroute2 import IPRoute from pyroute2 import NetNS -from pyroute2.netlink import NetlinkError -from socket import gethostname -from socket import AF_INET from socket import AF_INET6 def create_directory_ignore_exists(path, permissions): if not os.path.isdir(path): os.mkdir(path, permissions) - + def create_symlink_ignore_exists(path_to_point_to, symlink_location): if not os.path.islink(symlink_location): os.symlink(path_to_point_to, symlink_location) @@ -44,32 +33,9 @@ def get_attribute(struct_with_attrs, name): except Exception as e: raise RuntimeError("Couldn't find attribute %s for value: %s" % (name, struct_with_attrs), e) -def network(ip_address): - ip = ipaddress.ip_network(unicode(get_attribute(ip_address, 'IFA_ADDRESS'))) - prefix = int(ip_address['prefixlen']) - return ip.supernet(new_prefix = prefix) - def net_namespace_path(pid): return "/host/proc/%d/ns/net" % pid -def generate_mac_address(base_host_name, ip_address): - hash = hashlib.sha1() - hash.update(base_host_name) - hash.update(ip_address) - digest = hash.digest() - # For a mac address, we only need six bytes. - six_byte_digest = digest[:6] - mac_address_bytes = bytearray(six_byte_digest) - - # Set 'unicast' - mac_address_bytes[0] &= 0b11111110 - - # Set 'local' - mac_address_bytes[0] |= 0b00000010 - - mac_address = ':'.join('%02x' % n for n in mac_address_bytes) - return mac_address - def get_net_namespace_for_pid(pid): net_ns_path = net_namespace_path(pid) if not os.path.isfile(net_ns_path): @@ -78,88 +44,6 @@ def get_net_namespace_for_pid(pid): create_symlink_ignore_exists(net_ns_path, "/var/run/netns/%d" % pid) return NetNS(str(pid)) -# ip address format: { -# 'index': 3, -# 'family': 2, -# 'header': { -# 'pid': 15, -# 'length': 88, -# 'flags': 2, -# 'error': None, -# 'type': 20, -# 'sequence_number': 256 -# }, -# 'flags': 128, -# 'attrs': [ -# ['IFA_ADDRESS', '10.0.2.15'], -# ['IFA_LOCAL', '10.0.2.15'], -# ['IFA_BROADCAST', '10.0.2.255'], -# ['IFA_LABEL', 'eth0'], -# ['IFA_FLAGS', 128], -# [ -# 'IFA_CACHEINFO', -# { -# 'ifa_valid': 4294967295, -# 'tstamp': 2448, -# 'cstamp': 2448, -# 'ifa_prefered': 4294967295 -# } -# ] -# ], -# 'prefixlen': 24, -# 'scope': 0, -# 'event': 'RTM_NEWADDR' -# } -def ip_with_most_specific_network_for_address(address, ips): - ips_with_network_matching_address = [ip for ip in ips if address in network(ip)] - ip_best_match_for_address = None - for ip in ips_with_network_matching_address: - if not ip_best_match_for_address: - ip_best_match_for_address = ip - elif ip['prefixlen'] < ip_best_match_for_address['prefixlen']: - ip_best_match_for_address = ip - if not ip_best_match_for_address: - raise RuntimeError("No matching ip address for %s, candidates are on networks %s" % (address, ', '.join([str(network(ip)) for ip in ips]))) - return ip_best_match_for_address - -ipr = IPRoute() - -def delete_interface_by_name(interface_name): - for interface_index in ipr.link_lookup(ifname=interface_name): - ipr.link('delete', index=interface_index) - -def create_interface_in_namespace(network_namespace, ip_address_textual, interface_name, link_device_index): - mac_address = generate_mac_address( - base_host_name=gethostname(), - ip_address=ip_address_textual) - - # For traceability. - with open('/tmp/container_mac_address_' + ip_address_textual, 'w') as f: - f.write(mac_address) - - # result = [{ - # 'header': { - # 'pid': 240, - # 'length': 36, - # 'flags': 0, - # 'error': None, - # 'type': 2, - # 'sequence_number': 256 - # }, - # 'event': 'NLMSG_ERROR' - # }] - result = network_namespace.link_create( - ifname=interface_name, - kind='macvlan', - link=link_device_index, - macvlan_mode='bridge', - address=mac_address) - if result[0]['header']['error']: - raise RuntimeError("Failed creating link, result = %s" % result ) - - index_of_created_interface = network_namespace.link_lookup(ifname=interface_name)[0] - return index_of_created_interface - def index_of_interface_in_namespace(interface_name, namespace): interface_index_list = namespace.link_lookup(ifname=interface_name) if not interface_index_list: @@ -167,45 +51,6 @@ def index_of_interface_in_namespace(interface_name, namespace): assert len(interface_index_list) == 1 return interface_index_list[0] -def move_interface(src_interface_index, dest_namespace, dest_namespace_pid, dest_interface_name): - ipr.link('set', - index=src_interface_index, - net_ns_fd=str(dest_namespace_pid), - ifname=dest_interface_name) - - new_interface_index = index_of_interface_in_namespace(interface_name=dest_interface_name, - namespace=dest_namespace) - if not new_interface_index: - raise RuntimeError("Concurrent modification to network interfaces") - return new_interface_index - -def set_ip_address(net_namespace, interface_index, ip_address, network_prefix_length): - ip_already_configured = False - for existing_ip in net_namespace.get_addr(index=interface_index): - existing_ip_address = get_attribute(existing_ip, 'IFA_ADDRESS') - existing_ip_prefixlen = existing_ip['prefixlen'] - is_same_address = ipaddress.ip_address(unicode(existing_ip_address)) == ip_address - is_same_netmask = existing_ip_prefixlen == network_prefix_length - if is_same_address and is_same_netmask: - ip_already_configured = True - else: - # TODO Should we remove auto assigned ipv6 address (fe80:*) that is constructed from mac address? - print("Deleting old ip address. %s/%s" % (existing_ip_address, existing_ip_prefixlen)) - net_namespace.addr('remove', - index=interface_index, - address=existing_ip_address, - mask=existing_ip_prefixlen) - - if not ip_already_configured: - try: - net_namespace.addr('add', - index=interface_index, - address=str(ip_address), - # broadcast='192.168.59.255', - mask=network_prefix_length) - except NetlinkError as e: - if e.code == 17: # File exists, i.e. address is already added - pass def get_default_route(net_namespace, family): # route format: { @@ -239,102 +84,9 @@ def get_default_route(net_namespace, family): return route raise RuntimeError("Couldn't find default route: " + str(default_routes)) -def setup_container_networking(local_mode, vm_mode): - if len(sys.argv) != 3: - raise RuntimeError("Usage: %s <container-pid> <ip>" % sys.argv[0]) - - container_pid_arg = sys.argv[1] - container_ip_arg = sys.argv[2] - - try: - container_pid = int(container_pid_arg) - except ValueError: - raise RuntimeError("Container pid must be an integer, got %s" % container_pid_arg) - container_ip = ipaddress.ip_address(unicode(container_ip_arg)) - family = AF_INET6 if container_ip.version == 6 else AF_INET - - # Done parsing arguments, now let's get to work. - - host_ns = get_net_namespace_for_pid(1) - container_ns = get_net_namespace_for_pid(container_pid) - - all_host_ips = host_ns.get_addr() - host_ip_best_match_for_container = ip_with_most_specific_network_for_address(address=container_ip, - ips=all_host_ips) - host_device_index_for_container = host_ip_best_match_for_container['index'] - container_network_prefix_length = host_ip_best_match_for_container['prefixlen'] - - - # Create new interface for the container. - - # The interface to the vespa network are all (in the end) named "vespa". However, - # the container interfaces are prepared in the host network namespace, and so they - # need temporary names to avoid name-clash. - temporary_interface_name_while_in_host_ns = "vespa-tmp-" + container_pid_arg - assert len(temporary_interface_name_while_in_host_ns) <= 15 # linux requirement - - container_interface_name = "vespa" - assert len(container_interface_name) <= 15 # linux requirement - - # Clean up any leftovers from the past. - delete_interface_by_name(temporary_interface_name_while_in_host_ns) - - container_interface_index = index_of_interface_in_namespace(interface_name=container_interface_name, - namespace=container_ns) - if not container_interface_index: - # Must be created in the host_ns to have the same lifetime as the host. - # Otherwise, it will be deleted when the node-admin container stops. - # (Only temporarily there, moved to the container namespace later.) - # - # TODO: Here we're linking against the device with the best matching network. - # For the sake of argument, as of 2015-12-17, this device is always named - # 'vespa'. 'vespa' is itself a macvlan bridge linked to the default route's - # interface (typically eth0 or em1). So could we link against eth0 or em1 - # (or whatever) instead here? What's the difference? - temporary_interface_index = create_interface_in_namespace(network_namespace=host_ns, - ip_address_textual=container_ip_arg, - interface_name=temporary_interface_name_while_in_host_ns, - link_device_index=host_device_index_for_container) - - # Move interface from host namespace to container namespace, and change name from temporary name. - # Exploit that node_admin docker container shares net namespace with host: - container_interface_index = move_interface(src_interface_index=temporary_interface_index, - dest_namespace=container_ns, - dest_namespace_pid=container_pid, - dest_interface_name=container_interface_name) - - - # Set ip address on interface in container namespace. - set_ip_address(net_namespace=container_ns, - interface_index=container_interface_index, - ip_address=container_ip, - network_prefix_length=container_network_prefix_length) - - - # Activate container interface. - - container_ns.link('set', index=container_interface_index, state='up', name=container_interface_name) - - - if local_mode: - pass - elif vm_mode: - # Set the default route to the IP of the host vespa interface (e.g. osx) - # TODO: What about idempotency? This does not check for existing. Re-does work every time. - container_ns.route("add", gateway=get_attribute(host_ip_best_match_for_container, 'IFA_ADDRESS')) - else: - # Set up default route/gateway in container. - - host_default_route = get_default_route(net_namespace=host_ns, family=family) - host_default_route_device_index = get_attribute(host_default_route, 'RTA_OIF') - if host_device_index_for_container != host_default_route_device_index: - raise RuntimeError("Container's ip address is not on the same network as the host's default route." - " Could not set up default route for the container.") - host_default_route_gateway = get_attribute(host_default_route, 'RTA_GATEWAY') - container_ns.route(command="replace", gateway=host_default_route_gateway, index=container_interface_index, family=family) - # There is a bug in the Docker networking setup which requires us to manually specify the default gateway +# https://github.com/docker/libnetwork/issues/1443 def set_docker_gateway_on_docker_interface(): if len(sys.argv) != 2: raise RuntimeError("Usage: %s --fix-docker-gateway <container-pid>" % sys.argv[0]) @@ -352,29 +104,12 @@ def set_docker_gateway_on_docker_interface(): # Parse arguments -flag_local_mode = "--local" -local_mode = flag_local_mode in sys.argv -if local_mode: - sys.argv.remove(flag_local_mode) - -flag_vm_mode = "--vm" -vm_mode = flag_vm_mode in sys.argv -if vm_mode: - sys.argv.remove(flag_vm_mode) - flag_fix_docker_gateway = "--fix-docker-gateway" fix_docker_gateway = flag_fix_docker_gateway in sys.argv if fix_docker_gateway: sys.argv.remove(flag_fix_docker_gateway) -if fix_docker_gateway and (local_mode or vm_mode): - raise RuntimeError("Cannot use %s with %s or %s" % (flag_fix_docker_gateway, flag_local_mode, flag_vm_mode)) - -if local_mode and vm_mode: - raise RuntimeError("Cannot specify both %s and %s" % (flag_local_mode, flag_vm_mode)) - - if fix_docker_gateway: set_docker_gateway_on_docker_interface() else: - setup_container_networking(local_mode, vm_mode) + raise RuntimeError("Only valid flag is %s, got %s" % (flag_fix_docker_gateway, sys.argv[1]))
\ No newline at end of file diff --git a/node-admin/scripts/etc-hosts.sh b/node-admin/scripts/etc-hosts.sh index a588d3cc4e6..68c6d0e1b34 100755 --- a/node-admin/scripts/etc-hosts.sh +++ b/node-admin/scripts/etc-hosts.sh @@ -3,27 +3,17 @@ set -e -source "${0%/*}/common.sh" +declare -r NETWORK_PREFIX=172.18 +declare -r CONFIG_SERVER_HOSTNAME=config-server +declare -r CONFIG_SERVER_IP="$NETWORK_PREFIX.1.1" +declare -r APP_HOSTNAME_PREFIX=cnode- +declare -r APP_NETWORK_PREFIX="$NETWORK_PREFIX.2" + +declare -r NUM_APP_CONTAINERS=20 # Statically allocated number of nodes. declare -r HOSTS_FILE=/etc/hosts declare -r HOSTS_LINE_SUFFIX=" # Managed by etc-hosts.sh" -function Usage { - UsageHelper "$@" <<EOF -Usage: $SCRIPT_NAME <command> [--num-nodes <num-nodes>] -Manage Docker container DNS<->IP resolution ($HOSTS_FILE). - -Commands: - start Add Docker containers to $HOSTS_FILE - stop Remove Docker containers from $HOSTS_FILE (not implemented) - restart Stop, then start - -Options: - --num-nodes <num-nodes> - Add <num-nodes> hosts instead of the default $DEFAULT_NUM_APP_CONTAINERS. -EOF -} - function IsInHostsAlready { local ip="$1" local hostname="$2" @@ -58,7 +48,8 @@ function IsInHostsAlready { then return 1 else - Fail "$file contains a conflicting host specification for $hostname/$ip" + printf "$file contains a conflicting host specification for $hostname/$ip" + exit 1 fi } @@ -83,11 +74,6 @@ function Stop { } function StartAsRoot { - if (($# != 0)) - then - Usage - fi - # May need sudo if [ ! -w "$HOSTS_FILE" ] then @@ -105,13 +91,4 @@ function StartAsRoot { done } -function Start { - if [ "$(id -u)" != 0 ] - then - sudo "$0" "$@" - else - StartAsRoot "$@" - fi -} - -Main "$@" +StartAsRoot "$@" diff --git a/node-admin/scripts/make-host-like-container.sh b/node-admin/scripts/make-host-like-container.sh deleted file mode 100755 index 7e88e89b125..00000000000 --- a/node-admin/scripts/make-host-like-container.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -source "${0%/*}/common.sh" - -function Usage { - UsageHelper "$@" <<EOF -Usage: $SCRIPT_NAME <command> -Make localhost look like a container for the purpose of various other scripts. - -Commands: - start Make /host/proc point to /proc - stop Remove /host directory - restart Stop, then start -EOF -} - -function MakeHostDirectory { - if ! [ -e /host ] - then - echo "Created directory /host" - sudo mkdir /host - if ! [ -e /host/proc ] - then - echo "Created symbolic link from /host/proc to /proc" - sudo ln -s /proc /host/proc - fi - fi -} - -function RemoveHostDirectory { - if [ -d /host ] - then - echo "Removed /host directory" - sudo rm -rf /host - fi -} - -function Stop { - sudo true # Prime sudo - - RemoveHostDirectory -} - -function Start { - sudo true # Prime sudo - MakeHostDirectory -} - -Main "$@" diff --git a/node-admin/scripts/network-bridge.sh b/node-admin/scripts/network-bridge.sh deleted file mode 100755 index 705591c425a..00000000000 --- a/node-admin/scripts/network-bridge.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -source "${0%/*}/common.sh" - -declare -r DUMMY_KERNEL_NETWORK_MODULE="dummy" -declare -r DUMMY_NETWORK_INTERFACE="dummy0" - -function Usage { - UsageHelper "$@" <<EOF -Usage: $SCRIPT_NAME <command> -Manages the network bridge to the Docker container network - -Commands: - start Set up the network bridge - stop Tear down the network bridge - restart Stop, then start -EOF -} - -function Stop { - echo -n "Removing bridge $HOST_BRIDGE_INTERFACE... " - sudo ip link del "$HOST_BRIDGE_INTERFACE" &>/dev/null || true - - if sudo lsmod | grep -q "$DUMMY_KERNEL_NETWORK_MODULE" - then - sudo rmmod "$DUMMY_KERNEL_NETWORK_MODULE" - fi - - echo done -} - -function MakeBridge { - local ip="$1" - local prefix_bitlength="$2" - local name="$3" - - if ip link show dev "$name" up &>/dev/null - then - # TODO: Verify it is indeed set up correctly. - echo "Bridge '$name' already exists, will assume it has been set up correctly" - else - echo -n "Adding bridge $name ($ip) to the container network... " - - # Check if the $DUMMY_NETWORK_INTERFACE module is loaded and load if it is not - if ! sudo ip link show $DUMMY_NETWORK_INTERFACE &> /dev/null; then - sudo modprobe "$DUMMY_KERNEL_NETWORK_MODULE" - fi - sudo ip link set "$DUMMY_NETWORK_INTERFACE" up - sudo ip link add dev "$name" link "$DUMMY_NETWORK_INTERFACE" type macvlan mode bridge - sudo ip addr add dev "$name" "$ip/$prefix_bitlength" - sudo ip link set dev "$name" up - echo done - fi -} - -function Start { - MakeBridge "$HOST_BRIDGE_IP" "$NETWORK_PREFIX_BITLENGTH" "$HOST_BRIDGE_INTERFACE" -} - -Main "$@" diff --git a/node-admin/scripts/node-admin.sh b/node-admin/scripts/node-admin.sh deleted file mode 100755 index 405480e6d02..00000000000 --- a/node-admin/scripts/node-admin.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -source "${0%/*}/common.sh" - -function Usage { - UsageHelper "$@" <<EOF -Usage: $SCRIPT_NAME <command> -Manage the Node Admin - -Commands: - start Start the Node Admin in a Docker container - stop Remove the Node Admin container - restart Stop, then start -EOF -} - -function Stop { - # Prime sudo to avoid password prompt in the middle of the script. - sudo true - - echo -n "Removing $NODE_ADMIN_CONTAINER_NAME container... " - docker rm -f "$NODE_ADMIN_CONTAINER_NAME" &>/dev/null || true - echo done -} - -function Start { - # Prime sudo to avoid password prompt in the middle of the script. - sudo true - - echo -n "Making directory $APPLICATION_STORAGE_ROOT... " - sudo mkdir -p $APPLICATION_STORAGE_ROOT - echo done - - # Start node-admin - echo -n "Making $NODE_ADMIN_CONTAINER_NAME container... " - docker run \ - --detach \ - --privileged \ - --cap-add ALL \ - --name "$NODE_ADMIN_CONTAINER_NAME" \ - --net=host \ - --volume "$CONTAINER_CERT_PATH:/host/docker/certs" \ - --volume "/proc:/host/proc" \ - --volume "$APPLICATION_STORAGE_ROOT:/host$APPLICATION_STORAGE_ROOT" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/logs:$VESPA_HOME/logs" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/cache:$VESPA_HOME/var/cache" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/crash:$VESPA_HOME/var/crash" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/db/jdisc:$VESPA_HOME/var/db/jdisc" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/db/vespa:$VESPA_HOME/var/db/vespa" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/jdisc_container:$VESPA_HOME/var/jdisc_container" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/jdisc_core:$VESPA_HOME/var/jdisc_core" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/logstash-forwarder:$VESPA_HOME/var/logstash-forwarder" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/maven:$VESPA_HOME/var/maven" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/run:$VESPA_HOME/var/run" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/scoreboards:$VESPA_HOME/var/scoreboards" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/service:$VESPA_HOME/var/service" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/share:$VESPA_HOME/var/share" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/spool:$VESPA_HOME/var/spool" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/vespa:$VESPA_HOME/var/vespa" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/yca:$VESPA_HOME/var/yca" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/ycore++:$VESPA_HOME/var/ycore++" \ - --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/zookeeper:$VESPA_HOME/var/zookeeper" \ - --env "CONFIG_SERVER_ADDRESS=$CONFIG_SERVER_HOSTNAME" \ - --env "NETWORK_TYPE=$NETWORK_TYPE" \ - --entrypoint=/usr/local/bin/start-node-admin.sh \ - "$DOCKER_IMAGE" >/dev/null - echo done -} - -Main "$@" diff --git a/node-admin/scripts/populate-noderepo-with-local-nodes.sh b/node-admin/scripts/populate-noderepo-with-local-nodes.sh deleted file mode 100755 index 6d9a789426d..00000000000 --- a/node-admin/scripts/populate-noderepo-with-local-nodes.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -source "${0%/*}/common.sh" - -# Used to return response from RunCurl -declare CURL_RESPONSE - -function Usage { - UsageHelper "$@" <<EOF -Usage: $SCRIPT_NAME <command> [--num-nodes <num-nodes>] -Add Docker containers as nodes in the node repo, and activate them - -Commands: - start Add and activate nodes - stop Remove nodes (not implemented) - restart Stop, then start - -Options: - --num-nodes <num-nodes> - Activate <num-nodes> instead of the default $DEFAULT_NUM_APP_CONTAINERS. -EOF -} - -function Stop { - # TODO: Implement removal of the Docker containers from the node repo - : -} - -function Start { - local -a hostnames=() - - local -i i=1 - for ((; i <= $NUM_APP_CONTAINERS; ++i)); do - hostnames+=("$APP_HOSTNAME_PREFIX$i") - done - - ./node-repo.sh add -c "$CONFIG_SERVER_HOSTNAME" -p "$HOSTNAME" \ - "${hostnames[@]}" -} - -Main "$@" diff --git a/node-admin/scripts/route-osx.sh b/node-admin/scripts/route-osx.sh deleted file mode 100755 index 780d69f741e..00000000000 --- a/node-admin/scripts/route-osx.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -source "${0%/*}/common-vm.sh" - -VESPA_DOCKER_MACHINE_IP=$(docker-machine ip "$DOCKER_VM_NAME") -if [ $? -ne 0 ]; then - echo "Could not get the IP of the docker-machine $DOCKER_VM_NAME" - exit 1 -fi - -# Setup the route -sudo route delete "$HOST_BRIDGE_NETWORK" &> /dev/null -sudo route add "$HOST_BRIDGE_NETWORK" "$VESPA_DOCKER_MACHINE_IP" diff --git a/node-admin/scripts/setup-docker.sh b/node-admin/scripts/setup-docker.sh deleted file mode 100755 index 3e4b10dbd74..00000000000 --- a/node-admin/scripts/setup-docker.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# WARNING: Please double-check with the documentation in node-admin/README* -# whether these commands are in fact correct. If they are, this saves a bunch -# of typing... -# -# See HelpAndExit below for usage. - -set -ex - -declare DAYS_VALID=3650 - -# Note regarding the file names: Some are renamed from what you get from -# following the recipe in the docker documentation. Here, we've used -# underscores exclusively, never dashes. Some files have been renamed for -# explicitness, clarity and consistency (e.g. 'key' is renamed 'client_key'). -declare CERTS_DIR=~/.docker-certs -declare CA_FILE="$CERTS_DIR"/ca_cert.pem -declare CA_KEY_FILE="$CERTS_DIR"/ca_key.pem -declare CLIENT_CERT_FILE="$CERTS_DIR"/client_cert.pem -declare CLIENT_KEY_FILE="$CERTS_DIR"/client_key.pem -declare SERVER_CERT_FILE="$CERTS_DIR"/server_cert.pem -declare SERVER_KEY_FILE="$CERTS_DIR"/server_key.pem - -declare GROUP=users -declare YAHOO_GROUP="$GROUP" - -function HelpAndExit { - cat <<EOF -Usage: ${0##*/} <command>... -Setup Docker. - -Commands: - all Setup docker home and TLS certificates/keys. - Same as following commands: home certs - certs Generate and install TLS keys. - Same as following commands: generate-certs install-certs - generate-certs Generate TLS-related certificates and keys to - $CERTS_DIR - help Print this message. - install-certs Install TLS-related certificates and keys in - $CERTS_DIR - to /etc/dockercert_{daemon,cli,container}. - home Add docker user and make symbolic links from Docker dirs in - /var to dirs below ~docker. -EOF - - exit 0 -} - -function GenerateCertificates { - rm -rf "$CERTS_DIR" - mkdir -p "$CERTS_DIR" - - # Generate CA private and public keys - echo "We're about to generate a CA key, please use a secure password." - echo "You will be prompted for this password many times in what follows..." - openssl genrsa -aes256 -out "$CA_KEY_FILE" 4096 - openssl req -new -x509 -days "$DAYS_VALID" -key "$CA_KEY_FILE" -sha256 \ - -out "$CA_FILE" - - # Generate server key and certificate signing request (CSR) - openssl genrsa -out "$SERVER_KEY_FILE" 4096 - local server_csr_file="$CERTS_DIR"/server.csr - openssl req -subj "/CN=$HOSTNAME" -sha256 -new -key "$SERVER_KEY_FILE" \ - -out "$server_csr_file" - - # Sign server's public key with CA - local server_config_file="$CERTS_DIR"/server.cnf - echo "subjectAltName = IP:127.0.0.1" > "$server_config_file" - openssl x509 -req -days "$DAYS_VALID" -sha256 -in "$server_csr_file" \ - -CA "$CA_FILE" -CAkey "$CA_KEY_FILE" -CAcreateserial \ - -out "$SERVER_CERT_FILE" -extfile "$server_config_file" - - # Generate client key and certificate signing request (CSR) - openssl genrsa -out "$CLIENT_KEY_FILE" 4096 - local client_csr_file="$CERTS_DIR"/client.csr - openssl req -subj '/CN=client' -new -key "$CLIENT_KEY_FILE" \ - -out "$client_csr_file" - - # Sign client's public key with CA - local client_config_file="$CERTS_DIR"/client.cnf - echo extendedKeyUsage = clientAuth > "$client_config_file" - openssl x509 -req -days "$DAYS_VALID" -sha256 -in "$client_csr_file" \ - -CA "$CA_FILE" -CAkey "$CA_KEY_FILE" -CAcreateserial \ - -out "$CLIENT_CERT_FILE" -extfile "$client_config_file" - - # CSR and config files no longer needed - rm "$client_csr_file" "$server_csr_file" - rm "$server_config_file" "$client_config_file" - - # Avoid accidental writes - chmod 0400 "$CA_KEY_FILE" "$CLIENT_KEY_FILE" "$SERVER_KEY_FILE" - chmod 0444 "$CA_FILE" "$SERVER_CERT_FILE" "$CLIENT_CERT_FILE" -} - -function InstallCertificates { - # The files you end up with after GenerateKeys will be used by three - # parties: The docker daemon, the docker CLI, and the docker client in Node - # Admin. None of these parties need (nor should they have) access to all - # these files. Also, the three parties will run as different users. Since - # these files should not be world-readable, one solution is to create - # separate directories for the three usages, so each directory may contain - # only the needed files, with the correct owner and permissions. - - sudo mkdir -p /etc/dockercert_daemon - sudo chown yahoo:users /etc/dockercert_daemon - sudo cp "$CA_FILE" "$SERVER_CERT_FILE" "$SERVER_KEY_FILE" /etc/dockercert_daemon - sudo chown root:root /etc/dockercert_daemon/* - - # The docker client looks for files with certain names (you can only - # configure the path to the directory containing the files), so the - # "original" file names are used. - sudo mkdir -p /etc/dockercert_cli - sudo chown yahoo:users /etc/dockercert_cli - sudo cp "$CA_FILE" /etc/dockercert_cli/ca.pem - sudo cp "$CLIENT_CERT_FILE" /etc/dockercert_cli/cert.pem - sudo cp "$CLIENT_KEY_FILE" /etc/dockercert_cli/key.pem - sudo chown $USER:$GROUP /etc/dockercert_cli/* - - sudo mkdir -p /etc/dockercert_container - sudo chown yahoo:$YAHOO_GROUP /etc/dockercert_container - # These filenames must match the config given in - # src/main/application/services.xml. - sudo cp "$CA_FILE" "$CLIENT_CERT_FILE" "$CLIENT_KEY_FILE" /etc/dockercert_container - sudo chown yahoo:$YAHOO_GROUP /etc/dockercert_container/* - - echo "Note: Consider reloading & restarting the docker daemon to pick up" - echo "the new certificates and keys:" - echo " sudo systemctl daemon-reload" - echo " sudo systemctl restart docker" -} - -function SetupDockerHome { - # Assume an error means the docker user already exists - sudo useradd -g docker docker || true - - sudo mkdir -p ~docker/lib ~docker/run - sudo chmod +rx ~docker ~docker/lib ~docker/run - sudo systemctl stop docker - sudo rm -rf /var/{run,lib}/docker - sudo ln -s ~docker/run /var/run/docker - sudo ln -s ~docker/lib /var/lib/docker - sudo systemctl daemon-reload - sudo systemctl restart docker -} - -function Main { - # Prime sudo - sudo true - - if (($# == 0)) - then - HelpAndExit - fi - - local command - for command in "$@" - do - case "$command" in - all) Main home certs ;; - certs) - GenerateCertificates - InstallCertificates - ;; - generate-certs) GenerateCertificates ;; - help) HelpAndExit ;; - home) SetupDockerHome ;; - install-certs) InstallCertificates ;; - *) Fail "Unknown command '$command'" ;; - esac - done -} - -Main "$@" diff --git a/node-admin/scripts/setup-route-and-hosts-osx.sh b/node-admin/scripts/setup-route-and-hosts-osx.sh deleted file mode 100755 index dcfcfc0f121..00000000000 --- a/node-admin/scripts/setup-route-and-hosts-osx.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -echo "This will alter your routing table and /etc/hosts file. Continue ?" -select yn in "Yes" "No"; do - case $yn in - Yes ) break;; - No ) echo "Exiting."; exit;; - esac -done - -# Setup the route -cd "$SCRIPT_DIR" -./route-osx.sh - -# Setup the hosts file -cd "$SCRIPT_DIR" -./etc-hosts.sh diff --git a/node-admin/scripts/vm.sh b/node-admin/scripts/vm.sh deleted file mode 100755 index 19542a7c392..00000000000 --- a/node-admin/scripts/vm.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -set -e - -source "${0%/*}/common-vm.sh" - -DOCKER_VM_WAS_STARTED=false - -if ! docker-machine status "$DOCKER_VM_NAME" &> /dev/null; then - # Machine does not exist and we have to create and start - docker-machine create -d virtualbox \ - --virtualbox-disk-size "$DOCKER_VM_DISK_SIZE_IN_MB" \ - --virtualbox-memory "$DOCKER_VM_MEMORY_SIZE_IN_MB" \ - --virtualbox-cpu-count "$DOCKER_VM_CPU_COUNT" \ - --virtualbox-hostonly-cidr "$DOCKER_VM_HOST_CIDR" \ - "$DOCKER_VM_NAME" - - eval $(docker-machine env "$DOCKER_VM_NAME") - - # Node admin expects different names for the certificates. Just symlink docker has - # generated for us to match those in node-admin/src/main/application/services.xml. - ( - cd "$DOCKER_CERT_PATH" - ln -s ca.pem ca_cert.pem - ln -s key.pem client_key.pem - ln -s cert.pem client_cert.pem - ) - DOCKER_VM_WAS_STARTED=true -fi - - -VESPA_VM_STATUS=$(docker-machine status "$DOCKER_VM_NAME") -if [ "$VESPA_VM_STATUS" == "Stopped" ]; then - docker-machine start "$DOCKER_VM_NAME" - DOCKER_VM_WAS_STARTED=true - VESPA_VM_STATUS=$(docker-machine status "$DOCKER_VM_NAME") -fi - -if [ "$VESPA_VM_STATUS" != "Running" ]; then - echo "Unable to get Docker machine $DOCKER_VM_NAME up and running." - echo "You can try to manually remove the machine: docker-machine rm -y $DOCKER_VM_NAME " - echo " and then rerun this script." - echo "Exiting." - exit 1 -fi - -if $DOCKER_VM_WAS_STARTED; then - # Put anything that is not persisted between VM restarts in here. - # Set up NAT for the $HOST_BRIDGE_INTERFACE interface so that we can connect directly from OS X. - docker-machine ssh "$DOCKER_VM_NAME" sudo /usr/local/sbin/iptables -t nat -A POSTROUTING -s "$HOST_BRIDGE_NETWORK" ! -o "$HOST_BRIDGE_INTERFACE" -j MASQUERADE - docker-machine ssh "$DOCKER_VM_NAME" sudo /usr/local/sbin/iptables -A FORWARD -o "$HOST_BRIDGE_INTERFACE" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - - # Install dependencies used by setup scripts - docker-machine ssh "$DOCKER_VM_NAME" tce-load -wi python bash -fi - -# Get the environment for our VM -eval $(docker-machine env "$DOCKER_VM_NAME") - -if [ $# -ge 1 ]; then - declare -r ARG_SCRIPT=$1 - shift - - declare -r ARG_SCRIPT_BASE=$(basename "$ARG_SCRIPT") - declare -r ARG_SCRIPT_DIR=$(cd $(dirname "$ARG_SCRIPT") && pwd -P) - declare -r ARG_SCRIPT_ABS="$ARG_SCRIPT_DIR/$ARG_SCRIPT_BASE" - - if ! docker-machine ssh "$DOCKER_VM_NAME" which "$ARG_SCRIPT_ABS" &> /dev/null; then - echo "Provided script file does not exist or is not executable in VM : $ARG_SCRIPT_ABS" - echo "Usage: $0 [SCRIPT] [SCRIPT_ARGS...]" - exit 1 - fi - - # Start the provided script. This works because the $HOME directory is mapped in the same location in the VM. - docker-machine ssh "$DOCKER_VM_NAME" "CONTAINER_CERT_PATH=$DOCKER_CERT_PATH NETWORK_TYPE=vm $ARG_SCRIPT_ABS $*" -fi - diff --git a/node-admin/scripts/zone.sh b/node-admin/scripts/zone.sh deleted file mode 100755 index b35f367ba59..00000000000 --- a/node-admin/scripts/zone.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -set -e - -source "${0%/*}/common.sh" - -function Usage { - UsageHelper "$@" <<EOF -Usage: $SCRIPT_NAME <command> [<option>]... -Manage Hosted Vespa zone on localhost using Docker. - -The Docker daemon must already be running, and the Docker image must have been -built. The node-admin module must have been packaged. - -Commands: - start Start zone (start Config Server, Node Admin, etc) - stop Stop zone (take down Node Admin, Config Server, etc) - restart Stop, then start - -Options: - --hv-env <env> - Make a zone with this Hosted Vespa environment. Must be one of - prod, dev, test, staging, etc. Default is $DEFAULT_HOSTED_VESPA_ENVIRONMENT. - --hv-region <region> - Make a zone with this Hosted Vespa region. Default is $DEFAULT_HOSTED_VESPA_REGION. - --num-nodes <num-nodes> - Make a zone with <num-nodes> Docker nodes instead of the default $DEFAULT_NUM_APP_CONTAINERS. -EOF -} - -function Stop { - if (($# != 0)) - then - Usage - fi - - # Prime sudo to avoid password prompt in the middle of the script. - sudo true - - ./node-admin.sh stop - - # TODO: Stop and remove existing vespa node containers. - - # There's no need to stop populate-noderepo-with-local-nodes.sh, as the - # whole node repo is going down when the config server is stopped. - # - # ./populate-noderepo-with-local-nodes.sh stop - - ./config-server.sh stop - ./make-host-like-container.sh stop - ./network-bridge.sh stop - ./etc-hosts.sh stop -} - -function Start { - if (($# != 0)) - then - Usage - fi - - # Prime sudo to avoid password prompt in the middle of the script. - sudo true - - ./etc-hosts.sh --num-nodes "$NUM_APP_CONTAINERS" - ./network-bridge.sh - ./make-host-like-container.sh - - local region="${OPTION_HV_REGION:-$DEFAULT_HOSTED_VESPA_REGION}" - local env="${OPTION_HV_ENV:-$DEFAULT_HOSTED_VESPA_ENVIRONMENT}" - ./config-server.sh --wait=true --hv-region="$region" --hv-env="$env" - - ./populate-noderepo-with-local-nodes.sh --num-nodes "$NUM_APP_CONTAINERS" - ./node-admin.sh -} - -# Makes it easier to access scripts in the same 'scripts' directory -cd "$SCRIPT_DIR" - -Main "$@" |