summaryrefslogtreecommitdiffstats
path: root/node-admin/scripts
diff options
context:
space:
mode:
authorfreva <valerijf@yahoo-inc.com>2016-10-27 13:55:41 +0200
committerfreva <valerijf@yahoo-inc.com>2016-10-27 13:55:41 +0200
commita3b52baf7e3bac1071bf2644aae99b85552ca976 (patch)
treee6432b5cce27f2f4910747fb858cce8f62d8548a /node-admin/scripts
parentcf5078fceb2c9cc7305f8e1b297cf01e7bd6826c (diff)
Removed unused files
Diffstat (limited to 'node-admin/scripts')
-rwxr-xr-xnode-admin/scripts/app.sh156
-rw-r--r--node-admin/scripts/common-vm.sh13
-rw-r--r--node-admin/scripts/common.sh180
-rwxr-xr-xnode-admin/scripts/config-server.sh141
-rwxr-xr-xnode-admin/scripts/configure-container-networking.py273
-rwxr-xr-xnode-admin/scripts/etc-hosts.sh43
-rwxr-xr-xnode-admin/scripts/make-host-like-container.sh52
-rwxr-xr-xnode-admin/scripts/network-bridge.sh63
-rwxr-xr-xnode-admin/scripts/node-admin.sh73
-rwxr-xr-xnode-admin/scripts/populate-noderepo-with-local-nodes.sh44
-rwxr-xr-xnode-admin/scripts/route-osx.sh16
-rwxr-xr-xnode-admin/scripts/setup-docker.sh176
-rwxr-xr-xnode-admin/scripts/setup-route-and-hosts-osx.sh20
-rwxr-xr-xnode-admin/scripts/vm.sh77
-rwxr-xr-xnode-admin/scripts/zone.sh80
15 files changed, 14 insertions, 1393 deletions
diff --git a/node-admin/scripts/app.sh b/node-admin/scripts/app.sh
deleted file mode 100755
index d3eb6996ab4..00000000000
--- a/node-admin/scripts/app.sh
+++ /dev/null
@@ -1,156 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-set -e
-
-source "${0%/*}"/common.sh
-
-declare SCRIPTS_DIR="${0%/*}"
-
-declare -r APP_DIR_NAME_UNDER_SHARED=app
-
-function Usage {
- UsageHelper "$@" <<EOF
-Usage: $SCRIPT_NAME <command> [<app-dir>]
-Deploy (or undeploy) application rooted at <app-dir> on localhost Config Server.
-
-The local zone must be up and running. <app-dir> should point to
-e.g. vespa/basic-search-on-docker/target/application.
-EOF
-}
-
-function RunOnConfigServer {
- docker exec config-server "$@"
-}
-
-function VerifyApp {
- local app_dir="$1"
-
- # Sanity-check app_dir
- if ! [ -d "$app_dir" ]
- then
- Fail "<app-dir> '$app_dir' is not a directory"
- fi
-
- local services_xml="$app_dir"/services.xml
- if ! [ -f "$services_xml" ]
- then
- Fail "Failed to find services.xml in <app-dir> '$app_dir'"
- fi
-
- # Verify there's no <admin> element.
- if grep -qE '<admin[ >]' "$services_xml"
- then
- Fail "services.xml cannot contain an <admin> element in hosted Vespa"
- fi
-
- # Verify <nodes> seems to be correctly specified (warning: this test is
- # incomplete).
- if grep -qE "<nodes>" "$services_xml" ||
- ! grep -qE "<nodes (.* )?docker-image=" "$services_xml" ||
- ! grep -qE "<nodes (.* )?flavor=[\"']docker[\"']" "$services_xml"
- then
- Fail "You must specify the <nodes> element in the following form" \
- "in hosted Vespa w/Docker:" \
- " <nodes count=\"2\" flavor=\"docker\" docker-image=\"IMAGE\" />" \
- "where IMAGE is e.g. vespa-local:latest."
- fi
-}
-
-# Copies the application rooted at $1 to a directory tree shared with the
-# Config Server.
-function CopyToSharedDir {
- local app_dir="$1"
-
- local shared_dir_on_localhost="$APPLICATION_STORAGE_ROOT/$CONFIG_SERVER_CONTAINER_NAME/$ROOT_DIR_SHARED_WITH_HOST"
- if ! [ -d "$shared_dir_on_localhost" ]
- then
- Fail "Failed to find the Config Server's shared directory on" \
- "localhost '$shared_dir_on_localhost', has the" \
- "$CONFIG_SERVER_CONTAINER_NAME container been started?"
- fi
-
-
- local shared_app_dir_on_localhost="$shared_dir_on_localhost/$APP_DIR_NAME_UNDER_SHARED"
- if [ "$shared_app_dir_on_localhost" != /home/docker/container-storage/config-server/shared/app ]
- then
- # This duplication of code is a safety-guard against 'rm -rf' unknown
- # directories.
- Fail "We're about to remove '$shared_app_dir_on_localhost', but it's" \
- "pointing to something unexpected, refusing to proceed..."
- fi
-
- echo -n "Copying application to '$shared_app_dir_on_localhost'... "
- rm -rf "$shared_app_dir_on_localhost"
- cp -r "$app_dir" "$shared_app_dir_on_localhost"
- echo done
-}
-
-function DeployApp {
- if (($# != 1))
- then
- Usage
- fi
-
- local app_dir="$1"
-
- CopyToSharedDir "$app_dir"
-
- # Create tenant
- echo -n "Creating tenant... "
- local create_tenant_response
- if create_tenant_response=$(curl --silent --show-error -X PUT "http://$CONFIG_SERVER_HOSTNAME:$VESPA_WEB_SERVICE_PORT/application/v2/tenant/$TENANT_NAME" 2>&1)
- then
- if ! [[ "$create_tenant_response" =~ "Tenant $TENANT_NAME created" ]] &&
- ! [[ "$create_tenant_response" =~ "already exists" ]]
- then
- echo
- Fail "May have failed to create the tenant: '$create_tenant_response'"
- fi
- else
- echo
- Fail "Failed to create the tenant: $?: '$create_tenant_response'"
- fi
- echo done
-
- # Deploy app
- local app_dir_on_config_server="/$ROOT_DIR_SHARED_WITH_HOST/$APP_DIR_NAME_UNDER_SHARED"
- RunOnConfigServer $VESPA_HOME/bin/deploy -e "$TENANT_NAME" prepare "$app_dir_on_config_server"
- echo "Activating application"
- RunOnConfigServer $VESPA_HOME/bin/deploy -e "$TENANT_NAME" activate
-}
-
-function UndeployApp {
- if (($# != 0))
- then
- Usage "undeploy takes no arguments"
- fi
-
- local app_name=default
- local output
- echo -n "Removing application $TENANT_NAME:$app_name... "
- if ! output=$(curl --silent --show-error -X DELETE "http://$CONFIG_SERVER_HOSTNAME:$VESPA_WEB_SERVICE_PORT/application/v2/tenant/$TENANT_NAME/application/$app_name")
- then
- echo
- Fail "Failed to remove application: $output"
- fi
-
- echo done
-}
-
-function Main {
- if (($# == 0))
- then
- Usage "Missing command"
- fi
- local command="$1"
- shift
-
- case "$command" in
- deploy) DeployApp "$@" ;;
- undeploy) UndeployApp "$@" ;;
- *) Usage "Unknown command '$command'" ;;
- esac
-}
-
-Main "$@"
diff --git a/node-admin/scripts/common-vm.sh b/node-admin/scripts/common-vm.sh
deleted file mode 100644
index c91c75e1404..00000000000
--- a/node-admin/scripts/common-vm.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-set -e
-
-source "${0%/*}/common.sh"
-
-# VM configuration
-declare -r DOCKER_VM_NAME=vespa # Don't put spaces in the name
-declare -r DOCKER_VM_DISK_SIZE_IN_MB=40000
-declare -r DOCKER_VM_MEMORY_SIZE_IN_MB=4096
-declare -r DOCKER_VM_CPU_COUNT=1
-declare -r DOCKER_VM_HOST_CIDR=172.21.46.1/24
diff --git a/node-admin/scripts/common.sh b/node-admin/scripts/common.sh
deleted file mode 100644
index 6a10fb71a99..00000000000
--- a/node-admin/scripts/common.sh
+++ /dev/null
@@ -1,180 +0,0 @@
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-# Common variables and functions that may be useful for scripts IN THIS
-# DIRECTORY. Should be sourced as follows:
-#
-# source "${0%/*}/common.sh"
-#
-# WARNING: Some system variables, like the Config Server's, are also hardcoded
-# in the Docker image startup scripts.
-
-declare -r SCRIPT_NAME="${0##*/}"
-declare -r SCRIPT_DIR="${0%/*}"
-
-# TODO: Find a better name. Consider having separate images for config-server
-# and node-admin.
-declare -r DOCKER_IMAGE="vespa-local:latest"
-declare -r APPLICATION_STORAGE_ROOT="/home/docker/container-storage"
-declare -r ROOT_DIR_SHARED_WITH_HOST=shared
-
-# The 172.18.0.0/16 network is in IPDB.
-declare -r NETWORK_PREFIX=172.18
-declare -r NETWORK_PREFIX_BITLENGTH=16
-
-# Hostnames, IP addresses, names, etc of the infrastructure containers.
-declare -r HOST_BRIDGE_INTERFACE=vespa
-declare -r HOST_BRIDGE_IP="$NETWORK_PREFIX.0.1"
-declare -r HOST_BRIDGE_NETWORK="$NETWORK_PREFIX.0.0/$NETWORK_PREFIX_BITLENGTH"
-declare -r NODE_ADMIN_CONTAINER_NAME=node-admin
-declare -r CONFIG_SERVER_CONTAINER_NAME=config-server
-declare -r CONFIG_SERVER_HOSTNAME="$CONFIG_SERVER_CONTAINER_NAME"
-declare -r CONFIG_SERVER_IP="$NETWORK_PREFIX.1.1"
-declare -r VESPA_WEB_SERVICE_PORT=4080 # E.g. config server port
-
-declare -r DEFAULT_HOSTED_VESPA_REGION=local-region
-declare -r DEFAULT_HOSTED_VESPA_ENVIRONMENT=prod
-
-# Hostnames, IP addresses, names, etc of the application containers. Hostname
-# and container names are of the form $PREFIX$N, where N is a number between 1
-# and $NUM_APP_CONTAINERS. The IP is $APP_NETWORK_PREFIX.$N.
-declare -r APP_NETWORK_PREFIX="$NETWORK_PREFIX.2"
-declare -r APP_CONTAINER_NAME_PREFIX=cnode-
-declare -r APP_HOSTNAME_PREFIX="$APP_CONTAINER_NAME_PREFIX"
-declare -r DEFAULT_NUM_APP_CONTAINERS=20 # Statically allocated number of nodes.
-declare -r TENANT_NAME=localtenant
-
-# May be 'vm' if docker hosts runs within a VM (osx). Default is native/Fedora.
-declare -r NETWORK_TYPE="${NETWORK_TYPE:-local}"
-
-# Allowed program opions
-declare OPTION_NUM_NODES # Set from --num-nodes or DEFAULT_NUM_APP_CONTAINERS, see Main.
-declare OPTION_WAIT # Set from --wait or true, see Main.
-declare OPTION_HV_REGION # Set from --hv-region or DEFAULT_HOSTED_VESPA_REGION, see Main.
-declare OPTION_HV_ENV # Set from --hv-env or DEFAULT_HOSTED_VESPA_ENVIRONMENT, see Main.
-
-declare NUM_APP_CONTAINERS # Set from OPTION_NUM_NODES or DEFAULT_NUM_APP_CONTAINERS, see Main.
-
-function Fail {
- printf "%s\n" "$@" >&2
- exit 1
-}
-
-# Used to help scripts with implementing the Usage function. The intended usage
-# is:
-#
-# function Usage {
-# UsageHelper "$@" <<EOF
-# Usage: $SCRIPT_NAME ...
-# ...
-# EOF
-# }
-#
-# When Usage is called, any arguments passed will be printed to stderr, then
-# the usage-string will be printed (on stdin for UsageHelper), then the process
-# will exit with code 1.
-function UsageHelper {
- exec >&2
-
- if (($# > 0))
- then
- printf "%s\n\n" "$*"
- fi
-
- # Print to stdout (which has been redirected to stderr) what's on
- # stdin. This will print the usage-string.
- cat
-
- exit 1
-}
-
-# See Main
-function Restart {
- Stop
- Start "$@"
-}
-
-# Use Main as follows:
-#
-# Pass all script arguments to Main:
-#
-# Main "$@"
-#
-# Main will parse the arguments as follows. It assumes the arguments have
-# the following form:
-#
-# script.sh <command> [<arg> | <option>]...
-#
-# where <command> is one of start, stop, or restart:
-# start: The script MUST define a Start function.
-# stop: The script MUST define a Stop function.
-# restart: common.sh defines a Restart function to mean Stop, then Start.
-#
-# <arg> cannot start with a dash, and will get passed as argument to the Start
-# function (if applicable).
-#
-# <option> is either of the form --<name>=<value> or --<name> <value>.
-# <name>/<value> denotes a set of options. For each option, it sets one of the
-# predefined global OPTION_* options.
-#
-# Having parsed the arguments, Main then calls Start, Restart, or Stop,
-# depending on the command. These functions must be defined by the script.
-#
-# A function Usage must also be defined, which will be called when there's a
-# usage error.
-function Main {
- # Default command is start
- local command=start
- if (($# > 0)) && ! [[ "$1" =~ ^- ]]
- then
- command="$1"
- shift
- fi
-
- local -a args=()
-
- while (($# > 0))
- do
- if [[ "$1" =~ ^--([a-z0-9][a-z0-9-]*)(=(.*))?$ ]]
- then
- # Option argument
- local name="${BASH_REMATCH[1]}"
- shift
-
- if ((${#BASH_REMATCH[2]} > 0))
- then
- local value="${BASH_REMATCH[3]}"
- else
- if (($# == 0))
- then
- Usage "Option '$name' missing value"
- fi
-
- value="$1"
- shift
- fi
-
- case "$name" in
- num-nodes) OPTION_NUM_NODES="$value" ;;
- wait) OPTION_WAIT="$value" ;;
- hv-region) OPTION_HV_REGION="$value" ;;
- hv-env) OPTION_HV_ENV="$value" ;;
- esac
- elif [[ "$1" =~ ^[^-] ]]
- then
- # Non-option argument
- args+=("$1")
- shift
- else
- Usage "Bad argument '$1'"
- fi
- done
-
- NUM_APP_CONTAINERS="${OPTION_NUM_NODES:-$DEFAULT_NUM_APP_CONTAINERS}"
-
- case "$command" in
- help) Usage ;;
- stop) Stop ;;
- start) Start "${args[@]}" ;;
- restart) Restart "${args[@]}" ;;
- *) Usage "Unknown command '$command'"
- esac
-}
diff --git a/node-admin/scripts/config-server.sh b/node-admin/scripts/config-server.sh
deleted file mode 100755
index 0806f4374c6..00000000000
--- a/node-admin/scripts/config-server.sh
+++ /dev/null
@@ -1,141 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-set -e
-
-source "${0%/*}/common.sh"
-
-declare CONTAINER_ROOT_DIR="$APPLICATION_STORAGE_ROOT/$CONFIG_SERVER_CONTAINER_NAME"
-
-function Usage {
- UsageHelper "$@" <<EOF
-Usage: $SCRIPT_NAME <command> [--wait]
-Manage the Config Server
-
-Commands:
- start Start the Config Server in a Docker container
- stop Remove the Config Server container
- restart Stop, then start
-
-Options:
- --hv-env <env>
- Start the config server with the given hosted Vespa environment
- name. Must be one of prod, dev, test, staging, etc. Default is
- $DEFAULT_HOSTED_VESPA_ENVIRONMENT.
- --hv-region <region>
- Start the config server with the given hosted Vespa region name.
- Default is $DEFAULT_HOSTED_VESPA_REGION.
- --wait true
- Make start wait until the Config Server is healthy
-EOF
-}
-
-function Stop {
- # Prime sudo
- sudo true
-
- echo -n "Removing $CONFIG_SERVER_CONTAINER_NAME container... "
- docker rm -f "$CONFIG_SERVER_CONTAINER_NAME" &>/dev/null || true
- echo done
-
- if [ -d "$CONTAINER_ROOT_DIR" ]
- then
- # Double-check we're not 'rm -rf' something unexpected!
- if ! [[ "$CONTAINER_ROOT_DIR" =~ ^/home/docker/container-storage/ ]]
- then
- Fail "DANGEROUS: Almost removed '$CONTAINER_ROOT_DIR'..."
- fi
-
- echo -n "Removing container dir $CONTAINER_ROOT_DIR... "
- sudo rm -rf "$CONTAINER_ROOT_DIR"
- # The next two statements will prune empty parent directories.
- sudo mkdir "$CONTAINER_ROOT_DIR"
- sudo rmdir --ignore-fail-on-non-empty -p "$CONTAINER_ROOT_DIR"
- echo done
- fi
-}
-
-function Start {
- # Prime sudo
- sudo true
-
- local wait="${OPTION_WAIT:-true}"
- case "$wait" in
- true|false) : ;;
- *) Usage "--wait should only be set to true or false" ;;
- esac
-
- local region="${OPTION_HV_REGION:-$DEFAULT_HOSTED_VESPA_REGION}"
- local environment="${OPTION_HV_ENV:-$DEFAULT_HOSTED_VESPA_ENVIRONMENT}"
-
- echo -n "Creating container dir $CONTAINER_ROOT_DIR... "
- local shared_dir_on_localhost="$APPLICATION_STORAGE_ROOT/$CONFIG_SERVER_CONTAINER_NAME/$ROOT_DIR_SHARED_WITH_HOST"
- sudo mkdir -p "$shared_dir_on_localhost"
- sudo chmod a+wt "$shared_dir_on_localhost"
- echo done
-
- # Start config server
- echo -n "Making $CONFIG_SERVER_CONTAINER_NAME container... "
- local config_server_container_id
- config_server_container_id=$(\
- docker run \
- --detach \
- --cap-add=NET_ADMIN \
- --net=none \
- --hostname "$CONFIG_SERVER_HOSTNAME" \
- --name "$CONFIG_SERVER_CONTAINER_NAME" \
- --volume "/etc/hosts:/etc/hosts" \
- --volume "$shared_dir_on_localhost:/$ROOT_DIR_SHARED_WITH_HOST" \
- --env "HOSTED_VESPA_REGION=$region" \
- --env "HOSTED_VESPA_ENVIRONMENT=$environment" \
- --env "CONFIG_SERVER_HOSTNAME=$CONFIG_SERVER_HOSTNAME" \
- --env "VESPA_HOME=$VESPA_HOME" \
- --env "HOST_BRIDGE_IP=$HOST_BRIDGE_IP" \
- --entrypoint /usr/local/bin/start-config-server.sh \
- "$DOCKER_IMAGE")
- echo done
-
- echo -n "Verifying that $CONFIG_SERVER_CONTAINER_NAME container is running... "
- local config_server_container_pid
- config_server_container_pid=$(docker inspect -f '{{.State.Pid}}' "$CONFIG_SERVER_CONTAINER_NAME")
-
- echo -n "(pid $config_server_container_pid) "
-
- # TODO: Use .State.Status instead (only supported from version 1.9).
- local config_server_container_running
- config_server_container_running=$(docker inspect -f '{{.State.Running}}' "$CONFIG_SERVER_CONTAINER_NAME")
-
- if [ "$config_server_container_pid" == 0 -o "$config_server_container_running" != true ]
- then
- echo "failed"
- Fail "The Config Server is not running anymore, consider looking" \
- "at the logs with 'docker logs $CONFIG_SERVER_CONTAINER_NAME'"
- fi
- echo "done"
-
- echo -n "Setting up the $CONFIG_SERVER_CONTAINER_NAME container network of type $NETWORK_TYPE... "
- if ! script_out=$(sudo ./configure-container-networking.py --"$NETWORK_TYPE" "$config_server_container_pid" "$CONFIG_SERVER_IP" 2>&1); then
- echo "failed"
- echo "$script_out"
- exit
- fi
- echo "done"
-
- if [ "$wait" == true ]
- then
- # Wait for config server to come up
- echo -n "Waiting for healthy Config Server (~30s)"
- local url="http://$CONFIG_SERVER_HOSTNAME:$VESPA_WEB_SERVICE_PORT/state/v1/health"
- while ! curl --silent --fail --max-time 1 "$url" >/dev/null
- do
- echo -n .
- sleep 2
- done
- echo " done"
- fi
-}
-
-# Makes it easier to access scripts in the same 'scripts' directory
-cd "$SCRIPT_DIR"
-
-Main "$@"
diff --git a/node-admin/scripts/configure-container-networking.py b/node-admin/scripts/configure-container-networking.py
index 516d223494f..2efe0891e00 100755
--- a/node-admin/scripts/configure-container-networking.py
+++ b/node-admin/scripts/configure-container-networking.py
@@ -1,32 +1,21 @@
#!/usr/bin/env python
# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-# Quick and dirty script to set up routable ip address for docker container
-# Remove when docker releases a plugin api to configure networking.
-# TODO: Refactor for readability
-
+# Quick and dirty script to specify the default gateway on docker interface
from __future__ import print_function
-
-import hashlib
-import ipaddress
import os
import sys
-
-from pyroute2 import IPRoute
from pyroute2 import NetNS
-from pyroute2.netlink import NetlinkError
-from socket import gethostname
-from socket import AF_INET
from socket import AF_INET6
def create_directory_ignore_exists(path, permissions):
if not os.path.isdir(path):
os.mkdir(path, permissions)
-
+
def create_symlink_ignore_exists(path_to_point_to, symlink_location):
if not os.path.islink(symlink_location):
os.symlink(path_to_point_to, symlink_location)
@@ -44,32 +33,9 @@ def get_attribute(struct_with_attrs, name):
except Exception as e:
raise RuntimeError("Couldn't find attribute %s for value: %s" % (name, struct_with_attrs), e)
-def network(ip_address):
- ip = ipaddress.ip_network(unicode(get_attribute(ip_address, 'IFA_ADDRESS')))
- prefix = int(ip_address['prefixlen'])
- return ip.supernet(new_prefix = prefix)
-
def net_namespace_path(pid):
return "/host/proc/%d/ns/net" % pid
-def generate_mac_address(base_host_name, ip_address):
- hash = hashlib.sha1()
- hash.update(base_host_name)
- hash.update(ip_address)
- digest = hash.digest()
- # For a mac address, we only need six bytes.
- six_byte_digest = digest[:6]
- mac_address_bytes = bytearray(six_byte_digest)
-
- # Set 'unicast'
- mac_address_bytes[0] &= 0b11111110
-
- # Set 'local'
- mac_address_bytes[0] |= 0b00000010
-
- mac_address = ':'.join('%02x' % n for n in mac_address_bytes)
- return mac_address
-
def get_net_namespace_for_pid(pid):
net_ns_path = net_namespace_path(pid)
if not os.path.isfile(net_ns_path):
@@ -78,88 +44,6 @@ def get_net_namespace_for_pid(pid):
create_symlink_ignore_exists(net_ns_path, "/var/run/netns/%d" % pid)
return NetNS(str(pid))
-# ip address format: {
-# 'index': 3,
-# 'family': 2,
-# 'header': {
-# 'pid': 15,
-# 'length': 88,
-# 'flags': 2,
-# 'error': None,
-# 'type': 20,
-# 'sequence_number': 256
-# },
-# 'flags': 128,
-# 'attrs': [
-# ['IFA_ADDRESS', '10.0.2.15'],
-# ['IFA_LOCAL', '10.0.2.15'],
-# ['IFA_BROADCAST', '10.0.2.255'],
-# ['IFA_LABEL', 'eth0'],
-# ['IFA_FLAGS', 128],
-# [
-# 'IFA_CACHEINFO',
-# {
-# 'ifa_valid': 4294967295,
-# 'tstamp': 2448,
-# 'cstamp': 2448,
-# 'ifa_prefered': 4294967295
-# }
-# ]
-# ],
-# 'prefixlen': 24,
-# 'scope': 0,
-# 'event': 'RTM_NEWADDR'
-# }
-def ip_with_most_specific_network_for_address(address, ips):
- ips_with_network_matching_address = [ip for ip in ips if address in network(ip)]
- ip_best_match_for_address = None
- for ip in ips_with_network_matching_address:
- if not ip_best_match_for_address:
- ip_best_match_for_address = ip
- elif ip['prefixlen'] < ip_best_match_for_address['prefixlen']:
- ip_best_match_for_address = ip
- if not ip_best_match_for_address:
- raise RuntimeError("No matching ip address for %s, candidates are on networks %s" % (address, ', '.join([str(network(ip)) for ip in ips])))
- return ip_best_match_for_address
-
-ipr = IPRoute()
-
-def delete_interface_by_name(interface_name):
- for interface_index in ipr.link_lookup(ifname=interface_name):
- ipr.link('delete', index=interface_index)
-
-def create_interface_in_namespace(network_namespace, ip_address_textual, interface_name, link_device_index):
- mac_address = generate_mac_address(
- base_host_name=gethostname(),
- ip_address=ip_address_textual)
-
- # For traceability.
- with open('/tmp/container_mac_address_' + ip_address_textual, 'w') as f:
- f.write(mac_address)
-
- # result = [{
- # 'header': {
- # 'pid': 240,
- # 'length': 36,
- # 'flags': 0,
- # 'error': None,
- # 'type': 2,
- # 'sequence_number': 256
- # },
- # 'event': 'NLMSG_ERROR'
- # }]
- result = network_namespace.link_create(
- ifname=interface_name,
- kind='macvlan',
- link=link_device_index,
- macvlan_mode='bridge',
- address=mac_address)
- if result[0]['header']['error']:
- raise RuntimeError("Failed creating link, result = %s" % result )
-
- index_of_created_interface = network_namespace.link_lookup(ifname=interface_name)[0]
- return index_of_created_interface
-
def index_of_interface_in_namespace(interface_name, namespace):
interface_index_list = namespace.link_lookup(ifname=interface_name)
if not interface_index_list:
@@ -167,45 +51,6 @@ def index_of_interface_in_namespace(interface_name, namespace):
assert len(interface_index_list) == 1
return interface_index_list[0]
-def move_interface(src_interface_index, dest_namespace, dest_namespace_pid, dest_interface_name):
- ipr.link('set',
- index=src_interface_index,
- net_ns_fd=str(dest_namespace_pid),
- ifname=dest_interface_name)
-
- new_interface_index = index_of_interface_in_namespace(interface_name=dest_interface_name,
- namespace=dest_namespace)
- if not new_interface_index:
- raise RuntimeError("Concurrent modification to network interfaces")
- return new_interface_index
-
-def set_ip_address(net_namespace, interface_index, ip_address, network_prefix_length):
- ip_already_configured = False
- for existing_ip in net_namespace.get_addr(index=interface_index):
- existing_ip_address = get_attribute(existing_ip, 'IFA_ADDRESS')
- existing_ip_prefixlen = existing_ip['prefixlen']
- is_same_address = ipaddress.ip_address(unicode(existing_ip_address)) == ip_address
- is_same_netmask = existing_ip_prefixlen == network_prefix_length
- if is_same_address and is_same_netmask:
- ip_already_configured = True
- else:
- # TODO Should we remove auto assigned ipv6 address (fe80:*) that is constructed from mac address?
- print("Deleting old ip address. %s/%s" % (existing_ip_address, existing_ip_prefixlen))
- net_namespace.addr('remove',
- index=interface_index,
- address=existing_ip_address,
- mask=existing_ip_prefixlen)
-
- if not ip_already_configured:
- try:
- net_namespace.addr('add',
- index=interface_index,
- address=str(ip_address),
- # broadcast='192.168.59.255',
- mask=network_prefix_length)
- except NetlinkError as e:
- if e.code == 17: # File exists, i.e. address is already added
- pass
def get_default_route(net_namespace, family):
# route format: {
@@ -239,102 +84,9 @@ def get_default_route(net_namespace, family):
return route
raise RuntimeError("Couldn't find default route: " + str(default_routes))
-def setup_container_networking(local_mode, vm_mode):
- if len(sys.argv) != 3:
- raise RuntimeError("Usage: %s <container-pid> <ip>" % sys.argv[0])
-
- container_pid_arg = sys.argv[1]
- container_ip_arg = sys.argv[2]
-
- try:
- container_pid = int(container_pid_arg)
- except ValueError:
- raise RuntimeError("Container pid must be an integer, got %s" % container_pid_arg)
- container_ip = ipaddress.ip_address(unicode(container_ip_arg))
- family = AF_INET6 if container_ip.version == 6 else AF_INET
-
- # Done parsing arguments, now let's get to work.
-
- host_ns = get_net_namespace_for_pid(1)
- container_ns = get_net_namespace_for_pid(container_pid)
-
- all_host_ips = host_ns.get_addr()
- host_ip_best_match_for_container = ip_with_most_specific_network_for_address(address=container_ip,
- ips=all_host_ips)
- host_device_index_for_container = host_ip_best_match_for_container['index']
- container_network_prefix_length = host_ip_best_match_for_container['prefixlen']
-
-
- # Create new interface for the container.
-
- # The interface to the vespa network are all (in the end) named "vespa". However,
- # the container interfaces are prepared in the host network namespace, and so they
- # need temporary names to avoid name-clash.
- temporary_interface_name_while_in_host_ns = "vespa-tmp-" + container_pid_arg
- assert len(temporary_interface_name_while_in_host_ns) <= 15 # linux requirement
-
- container_interface_name = "vespa"
- assert len(container_interface_name) <= 15 # linux requirement
-
- # Clean up any leftovers from the past.
- delete_interface_by_name(temporary_interface_name_while_in_host_ns)
-
- container_interface_index = index_of_interface_in_namespace(interface_name=container_interface_name,
- namespace=container_ns)
- if not container_interface_index:
- # Must be created in the host_ns to have the same lifetime as the host.
- # Otherwise, it will be deleted when the node-admin container stops.
- # (Only temporarily there, moved to the container namespace later.)
- #
- # TODO: Here we're linking against the device with the best matching network.
- # For the sake of argument, as of 2015-12-17, this device is always named
- # 'vespa'. 'vespa' is itself a macvlan bridge linked to the default route's
- # interface (typically eth0 or em1). So could we link against eth0 or em1
- # (or whatever) instead here? What's the difference?
- temporary_interface_index = create_interface_in_namespace(network_namespace=host_ns,
- ip_address_textual=container_ip_arg,
- interface_name=temporary_interface_name_while_in_host_ns,
- link_device_index=host_device_index_for_container)
-
- # Move interface from host namespace to container namespace, and change name from temporary name.
- # Exploit that node_admin docker container shares net namespace with host:
- container_interface_index = move_interface(src_interface_index=temporary_interface_index,
- dest_namespace=container_ns,
- dest_namespace_pid=container_pid,
- dest_interface_name=container_interface_name)
-
-
- # Set ip address on interface in container namespace.
- set_ip_address(net_namespace=container_ns,
- interface_index=container_interface_index,
- ip_address=container_ip,
- network_prefix_length=container_network_prefix_length)
-
-
- # Activate container interface.
-
- container_ns.link('set', index=container_interface_index, state='up', name=container_interface_name)
-
-
- if local_mode:
- pass
- elif vm_mode:
- # Set the default route to the IP of the host vespa interface (e.g. osx)
- # TODO: What about idempotency? This does not check for existing. Re-does work every time.
- container_ns.route("add", gateway=get_attribute(host_ip_best_match_for_container, 'IFA_ADDRESS'))
- else:
- # Set up default route/gateway in container.
-
- host_default_route = get_default_route(net_namespace=host_ns, family=family)
- host_default_route_device_index = get_attribute(host_default_route, 'RTA_OIF')
- if host_device_index_for_container != host_default_route_device_index:
- raise RuntimeError("Container's ip address is not on the same network as the host's default route."
- " Could not set up default route for the container.")
- host_default_route_gateway = get_attribute(host_default_route, 'RTA_GATEWAY')
- container_ns.route(command="replace", gateway=host_default_route_gateway, index=container_interface_index, family=family)
-
# There is a bug in the Docker networking setup which requires us to manually specify the default gateway
+# https://github.com/docker/libnetwork/issues/1443
def set_docker_gateway_on_docker_interface():
if len(sys.argv) != 2:
raise RuntimeError("Usage: %s --fix-docker-gateway <container-pid>" % sys.argv[0])
@@ -352,29 +104,12 @@ def set_docker_gateway_on_docker_interface():
# Parse arguments
-flag_local_mode = "--local"
-local_mode = flag_local_mode in sys.argv
-if local_mode:
- sys.argv.remove(flag_local_mode)
-
-flag_vm_mode = "--vm"
-vm_mode = flag_vm_mode in sys.argv
-if vm_mode:
- sys.argv.remove(flag_vm_mode)
-
flag_fix_docker_gateway = "--fix-docker-gateway"
fix_docker_gateway = flag_fix_docker_gateway in sys.argv
if fix_docker_gateway:
sys.argv.remove(flag_fix_docker_gateway)
-if fix_docker_gateway and (local_mode or vm_mode):
- raise RuntimeError("Cannot use %s with %s or %s" % (flag_fix_docker_gateway, flag_local_mode, flag_vm_mode))
-
-if local_mode and vm_mode:
- raise RuntimeError("Cannot specify both %s and %s" % (flag_local_mode, flag_vm_mode))
-
-
if fix_docker_gateway:
set_docker_gateway_on_docker_interface()
else:
- setup_container_networking(local_mode, vm_mode)
+ raise RuntimeError("Only valid flag is %s, got %s" % (flag_fix_docker_gateway, sys.argv[1])) \ No newline at end of file
diff --git a/node-admin/scripts/etc-hosts.sh b/node-admin/scripts/etc-hosts.sh
index a588d3cc4e6..68c6d0e1b34 100755
--- a/node-admin/scripts/etc-hosts.sh
+++ b/node-admin/scripts/etc-hosts.sh
@@ -3,27 +3,17 @@
set -e
-source "${0%/*}/common.sh"
+declare -r NETWORK_PREFIX=172.18
+declare -r CONFIG_SERVER_HOSTNAME=config-server
+declare -r CONFIG_SERVER_IP="$NETWORK_PREFIX.1.1"
+declare -r APP_HOSTNAME_PREFIX=cnode-
+declare -r APP_NETWORK_PREFIX="$NETWORK_PREFIX.2"
+
+declare -r NUM_APP_CONTAINERS=20 # Statically allocated number of nodes.
declare -r HOSTS_FILE=/etc/hosts
declare -r HOSTS_LINE_SUFFIX=" # Managed by etc-hosts.sh"
-function Usage {
- UsageHelper "$@" <<EOF
-Usage: $SCRIPT_NAME <command> [--num-nodes <num-nodes>]
-Manage Docker container DNS<->IP resolution ($HOSTS_FILE).
-
-Commands:
- start Add Docker containers to $HOSTS_FILE
- stop Remove Docker containers from $HOSTS_FILE (not implemented)
- restart Stop, then start
-
-Options:
- --num-nodes <num-nodes>
- Add <num-nodes> hosts instead of the default $DEFAULT_NUM_APP_CONTAINERS.
-EOF
-}
-
function IsInHostsAlready {
local ip="$1"
local hostname="$2"
@@ -58,7 +48,8 @@ function IsInHostsAlready {
then
return 1
else
- Fail "$file contains a conflicting host specification for $hostname/$ip"
+ printf "$file contains a conflicting host specification for $hostname/$ip"
+ exit 1
fi
}
@@ -83,11 +74,6 @@ function Stop {
}
function StartAsRoot {
- if (($# != 0))
- then
- Usage
- fi
-
# May need sudo
if [ ! -w "$HOSTS_FILE" ]
then
@@ -105,13 +91,4 @@ function StartAsRoot {
done
}
-function Start {
- if [ "$(id -u)" != 0 ]
- then
- sudo "$0" "$@"
- else
- StartAsRoot "$@"
- fi
-}
-
-Main "$@"
+StartAsRoot "$@"
diff --git a/node-admin/scripts/make-host-like-container.sh b/node-admin/scripts/make-host-like-container.sh
deleted file mode 100755
index 7e88e89b125..00000000000
--- a/node-admin/scripts/make-host-like-container.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-set -e
-
-source "${0%/*}/common.sh"
-
-function Usage {
- UsageHelper "$@" <<EOF
-Usage: $SCRIPT_NAME <command>
-Make localhost look like a container for the purpose of various other scripts.
-
-Commands:
- start Make /host/proc point to /proc
- stop Remove /host directory
- restart Stop, then start
-EOF
-}
-
-function MakeHostDirectory {
- if ! [ -e /host ]
- then
- echo "Created directory /host"
- sudo mkdir /host
- if ! [ -e /host/proc ]
- then
- echo "Created symbolic link from /host/proc to /proc"
- sudo ln -s /proc /host/proc
- fi
- fi
-}
-
-function RemoveHostDirectory {
- if [ -d /host ]
- then
- echo "Removed /host directory"
- sudo rm -rf /host
- fi
-}
-
-function Stop {
- sudo true # Prime sudo
-
- RemoveHostDirectory
-}
-
-function Start {
- sudo true # Prime sudo
- MakeHostDirectory
-}
-
-Main "$@"
diff --git a/node-admin/scripts/network-bridge.sh b/node-admin/scripts/network-bridge.sh
deleted file mode 100755
index 705591c425a..00000000000
--- a/node-admin/scripts/network-bridge.sh
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-set -e
-
-source "${0%/*}/common.sh"
-
-declare -r DUMMY_KERNEL_NETWORK_MODULE="dummy"
-declare -r DUMMY_NETWORK_INTERFACE="dummy0"
-
-function Usage {
- UsageHelper "$@" <<EOF
-Usage: $SCRIPT_NAME <command>
-Manages the network bridge to the Docker container network
-
-Commands:
- start Set up the network bridge
- stop Tear down the network bridge
- restart Stop, then start
-EOF
-}
-
-function Stop {
- echo -n "Removing bridge $HOST_BRIDGE_INTERFACE... "
- sudo ip link del "$HOST_BRIDGE_INTERFACE" &>/dev/null || true
-
- if sudo lsmod | grep -q "$DUMMY_KERNEL_NETWORK_MODULE"
- then
- sudo rmmod "$DUMMY_KERNEL_NETWORK_MODULE"
- fi
-
- echo done
-}
-
-function MakeBridge {
- local ip="$1"
- local prefix_bitlength="$2"
- local name="$3"
-
- if ip link show dev "$name" up &>/dev/null
- then
- # TODO: Verify it is indeed set up correctly.
- echo "Bridge '$name' already exists, will assume it has been set up correctly"
- else
- echo -n "Adding bridge $name ($ip) to the container network... "
-
- # Check if the $DUMMY_NETWORK_INTERFACE module is loaded and load if it is not
- if ! sudo ip link show $DUMMY_NETWORK_INTERFACE &> /dev/null; then
- sudo modprobe "$DUMMY_KERNEL_NETWORK_MODULE"
- fi
- sudo ip link set "$DUMMY_NETWORK_INTERFACE" up
- sudo ip link add dev "$name" link "$DUMMY_NETWORK_INTERFACE" type macvlan mode bridge
- sudo ip addr add dev "$name" "$ip/$prefix_bitlength"
- sudo ip link set dev "$name" up
- echo done
- fi
-}
-
-function Start {
- MakeBridge "$HOST_BRIDGE_IP" "$NETWORK_PREFIX_BITLENGTH" "$HOST_BRIDGE_INTERFACE"
-}
-
-Main "$@"
diff --git a/node-admin/scripts/node-admin.sh b/node-admin/scripts/node-admin.sh
deleted file mode 100755
index 405480e6d02..00000000000
--- a/node-admin/scripts/node-admin.sh
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-set -e
-
-source "${0%/*}/common.sh"
-
-function Usage {
- UsageHelper "$@" <<EOF
-Usage: $SCRIPT_NAME <command>
-Manage the Node Admin
-
-Commands:
- start Start the Node Admin in a Docker container
- stop Remove the Node Admin container
- restart Stop, then start
-EOF
-}
-
-function Stop {
- # Prime sudo to avoid password prompt in the middle of the script.
- sudo true
-
- echo -n "Removing $NODE_ADMIN_CONTAINER_NAME container... "
- docker rm -f "$NODE_ADMIN_CONTAINER_NAME" &>/dev/null || true
- echo done
-}
-
-function Start {
- # Prime sudo to avoid password prompt in the middle of the script.
- sudo true
-
- echo -n "Making directory $APPLICATION_STORAGE_ROOT... "
- sudo mkdir -p $APPLICATION_STORAGE_ROOT
- echo done
-
- # Start node-admin
- echo -n "Making $NODE_ADMIN_CONTAINER_NAME container... "
- docker run \
- --detach \
- --privileged \
- --cap-add ALL \
- --name "$NODE_ADMIN_CONTAINER_NAME" \
- --net=host \
- --volume "$CONTAINER_CERT_PATH:/host/docker/certs" \
- --volume "/proc:/host/proc" \
- --volume "$APPLICATION_STORAGE_ROOT:/host$APPLICATION_STORAGE_ROOT" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/logs:$VESPA_HOME/logs" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/cache:$VESPA_HOME/var/cache" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/crash:$VESPA_HOME/var/crash" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/db/jdisc:$VESPA_HOME/var/db/jdisc" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/db/vespa:$VESPA_HOME/var/db/vespa" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/jdisc_container:$VESPA_HOME/var/jdisc_container" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/jdisc_core:$VESPA_HOME/var/jdisc_core" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/logstash-forwarder:$VESPA_HOME/var/logstash-forwarder" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/maven:$VESPA_HOME/var/maven" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/run:$VESPA_HOME/var/run" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/scoreboards:$VESPA_HOME/var/scoreboards" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/service:$VESPA_HOME/var/service" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/share:$VESPA_HOME/var/share" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/spool:$VESPA_HOME/var/spool" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/vespa:$VESPA_HOME/var/vespa" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/yca:$VESPA_HOME/var/yca" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/ycore++:$VESPA_HOME/var/ycore++" \
- --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/zookeeper:$VESPA_HOME/var/zookeeper" \
- --env "CONFIG_SERVER_ADDRESS=$CONFIG_SERVER_HOSTNAME" \
- --env "NETWORK_TYPE=$NETWORK_TYPE" \
- --entrypoint=/usr/local/bin/start-node-admin.sh \
- "$DOCKER_IMAGE" >/dev/null
- echo done
-}
-
-Main "$@"
diff --git a/node-admin/scripts/populate-noderepo-with-local-nodes.sh b/node-admin/scripts/populate-noderepo-with-local-nodes.sh
deleted file mode 100755
index 6d9a789426d..00000000000
--- a/node-admin/scripts/populate-noderepo-with-local-nodes.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-set -e
-
-source "${0%/*}/common.sh"
-
-# Used to return response from RunCurl
-declare CURL_RESPONSE
-
-function Usage {
- UsageHelper "$@" <<EOF
-Usage: $SCRIPT_NAME <command> [--num-nodes <num-nodes>]
-Add Docker containers as nodes in the node repo, and activate them
-
-Commands:
- start Add and activate nodes
- stop Remove nodes (not implemented)
- restart Stop, then start
-
-Options:
- --num-nodes <num-nodes>
- Activate <num-nodes> instead of the default $DEFAULT_NUM_APP_CONTAINERS.
-EOF
-}
-
-function Stop {
- # TODO: Implement removal of the Docker containers from the node repo
- :
-}
-
-function Start {
- local -a hostnames=()
-
- local -i i=1
- for ((; i <= $NUM_APP_CONTAINERS; ++i)); do
- hostnames+=("$APP_HOSTNAME_PREFIX$i")
- done
-
- ./node-repo.sh add -c "$CONFIG_SERVER_HOSTNAME" -p "$HOSTNAME" \
- "${hostnames[@]}"
-}
-
-Main "$@"
diff --git a/node-admin/scripts/route-osx.sh b/node-admin/scripts/route-osx.sh
deleted file mode 100755
index 780d69f741e..00000000000
--- a/node-admin/scripts/route-osx.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-set -e
-
-source "${0%/*}/common-vm.sh"
-
-VESPA_DOCKER_MACHINE_IP=$(docker-machine ip "$DOCKER_VM_NAME")
-if [ $? -ne 0 ]; then
- echo "Could not get the IP of the docker-machine $DOCKER_VM_NAME"
- exit 1
-fi
-
-# Setup the route
-sudo route delete "$HOST_BRIDGE_NETWORK" &> /dev/null
-sudo route add "$HOST_BRIDGE_NETWORK" "$VESPA_DOCKER_MACHINE_IP"
diff --git a/node-admin/scripts/setup-docker.sh b/node-admin/scripts/setup-docker.sh
deleted file mode 100755
index 3e4b10dbd74..00000000000
--- a/node-admin/scripts/setup-docker.sh
+++ /dev/null
@@ -1,176 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-# WARNING: Please double-check with the documentation in node-admin/README*
-# whether these commands are in fact correct. If they are, this saves a bunch
-# of typing...
-#
-# See HelpAndExit below for usage.
-
-set -ex
-
-declare DAYS_VALID=3650
-
-# Note regarding the file names: Some are renamed from what you get from
-# following the recipe in the docker documentation. Here, we've used
-# underscores exclusively, never dashes. Some files have been renamed for
-# explicitness, clarity and consistency (e.g. 'key' is renamed 'client_key').
-declare CERTS_DIR=~/.docker-certs
-declare CA_FILE="$CERTS_DIR"/ca_cert.pem
-declare CA_KEY_FILE="$CERTS_DIR"/ca_key.pem
-declare CLIENT_CERT_FILE="$CERTS_DIR"/client_cert.pem
-declare CLIENT_KEY_FILE="$CERTS_DIR"/client_key.pem
-declare SERVER_CERT_FILE="$CERTS_DIR"/server_cert.pem
-declare SERVER_KEY_FILE="$CERTS_DIR"/server_key.pem
-
-declare GROUP=users
-declare YAHOO_GROUP="$GROUP"
-
-function HelpAndExit {
- cat <<EOF
-Usage: ${0##*/} <command>...
-Setup Docker.
-
-Commands:
- all Setup docker home and TLS certificates/keys.
- Same as following commands: home certs
- certs Generate and install TLS keys.
- Same as following commands: generate-certs install-certs
- generate-certs Generate TLS-related certificates and keys to
- $CERTS_DIR
- help Print this message.
- install-certs Install TLS-related certificates and keys in
- $CERTS_DIR
- to /etc/dockercert_{daemon,cli,container}.
- home Add docker user and make symbolic links from Docker dirs in
- /var to dirs below ~docker.
-EOF
-
- exit 0
-}
-
-function GenerateCertificates {
- rm -rf "$CERTS_DIR"
- mkdir -p "$CERTS_DIR"
-
- # Generate CA private and public keys
- echo "We're about to generate a CA key, please use a secure password."
- echo "You will be prompted for this password many times in what follows..."
- openssl genrsa -aes256 -out "$CA_KEY_FILE" 4096
- openssl req -new -x509 -days "$DAYS_VALID" -key "$CA_KEY_FILE" -sha256 \
- -out "$CA_FILE"
-
- # Generate server key and certificate signing request (CSR)
- openssl genrsa -out "$SERVER_KEY_FILE" 4096
- local server_csr_file="$CERTS_DIR"/server.csr
- openssl req -subj "/CN=$HOSTNAME" -sha256 -new -key "$SERVER_KEY_FILE" \
- -out "$server_csr_file"
-
- # Sign server's public key with CA
- local server_config_file="$CERTS_DIR"/server.cnf
- echo "subjectAltName = IP:127.0.0.1" > "$server_config_file"
- openssl x509 -req -days "$DAYS_VALID" -sha256 -in "$server_csr_file" \
- -CA "$CA_FILE" -CAkey "$CA_KEY_FILE" -CAcreateserial \
- -out "$SERVER_CERT_FILE" -extfile "$server_config_file"
-
- # Generate client key and certificate signing request (CSR)
- openssl genrsa -out "$CLIENT_KEY_FILE" 4096
- local client_csr_file="$CERTS_DIR"/client.csr
- openssl req -subj '/CN=client' -new -key "$CLIENT_KEY_FILE" \
- -out "$client_csr_file"
-
- # Sign client's public key with CA
- local client_config_file="$CERTS_DIR"/client.cnf
- echo extendedKeyUsage = clientAuth > "$client_config_file"
- openssl x509 -req -days "$DAYS_VALID" -sha256 -in "$client_csr_file" \
- -CA "$CA_FILE" -CAkey "$CA_KEY_FILE" -CAcreateserial \
- -out "$CLIENT_CERT_FILE" -extfile "$client_config_file"
-
- # CSR and config files no longer needed
- rm "$client_csr_file" "$server_csr_file"
- rm "$server_config_file" "$client_config_file"
-
- # Avoid accidental writes
- chmod 0400 "$CA_KEY_FILE" "$CLIENT_KEY_FILE" "$SERVER_KEY_FILE"
- chmod 0444 "$CA_FILE" "$SERVER_CERT_FILE" "$CLIENT_CERT_FILE"
-}
-
-function InstallCertificates {
- # The files you end up with after GenerateKeys will be used by three
- # parties: The docker daemon, the docker CLI, and the docker client in Node
- # Admin. None of these parties need (nor should they have) access to all
- # these files. Also, the three parties will run as different users. Since
- # these files should not be world-readable, one solution is to create
- # separate directories for the three usages, so each directory may contain
- # only the needed files, with the correct owner and permissions.
-
- sudo mkdir -p /etc/dockercert_daemon
- sudo chown yahoo:users /etc/dockercert_daemon
- sudo cp "$CA_FILE" "$SERVER_CERT_FILE" "$SERVER_KEY_FILE" /etc/dockercert_daemon
- sudo chown root:root /etc/dockercert_daemon/*
-
- # The docker client looks for files with certain names (you can only
- # configure the path to the directory containing the files), so the
- # "original" file names are used.
- sudo mkdir -p /etc/dockercert_cli
- sudo chown yahoo:users /etc/dockercert_cli
- sudo cp "$CA_FILE" /etc/dockercert_cli/ca.pem
- sudo cp "$CLIENT_CERT_FILE" /etc/dockercert_cli/cert.pem
- sudo cp "$CLIENT_KEY_FILE" /etc/dockercert_cli/key.pem
- sudo chown $USER:$GROUP /etc/dockercert_cli/*
-
- sudo mkdir -p /etc/dockercert_container
- sudo chown yahoo:$YAHOO_GROUP /etc/dockercert_container
- # These filenames must match the config given in
- # src/main/application/services.xml.
- sudo cp "$CA_FILE" "$CLIENT_CERT_FILE" "$CLIENT_KEY_FILE" /etc/dockercert_container
- sudo chown yahoo:$YAHOO_GROUP /etc/dockercert_container/*
-
- echo "Note: Consider reloading & restarting the docker daemon to pick up"
- echo "the new certificates and keys:"
- echo " sudo systemctl daemon-reload"
- echo " sudo systemctl restart docker"
-}
-
-function SetupDockerHome {
- # Assume an error means the docker user already exists
- sudo useradd -g docker docker || true
-
- sudo mkdir -p ~docker/lib ~docker/run
- sudo chmod +rx ~docker ~docker/lib ~docker/run
- sudo systemctl stop docker
- sudo rm -rf /var/{run,lib}/docker
- sudo ln -s ~docker/run /var/run/docker
- sudo ln -s ~docker/lib /var/lib/docker
- sudo systemctl daemon-reload
- sudo systemctl restart docker
-}
-
-function Main {
- # Prime sudo
- sudo true
-
- if (($# == 0))
- then
- HelpAndExit
- fi
-
- local command
- for command in "$@"
- do
- case "$command" in
- all) Main home certs ;;
- certs)
- GenerateCertificates
- InstallCertificates
- ;;
- generate-certs) GenerateCertificates ;;
- help) HelpAndExit ;;
- home) SetupDockerHome ;;
- install-certs) InstallCertificates ;;
- *) Fail "Unknown command '$command'" ;;
- esac
- done
-}
-
-Main "$@"
diff --git a/node-admin/scripts/setup-route-and-hosts-osx.sh b/node-admin/scripts/setup-route-and-hosts-osx.sh
deleted file mode 100755
index dcfcfc0f121..00000000000
--- a/node-admin/scripts/setup-route-and-hosts-osx.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-set -e
-
-echo "This will alter your routing table and /etc/hosts file. Continue ?"
-select yn in "Yes" "No"; do
- case $yn in
- Yes ) break;;
- No ) echo "Exiting."; exit;;
- esac
-done
-
-# Setup the route
-cd "$SCRIPT_DIR"
-./route-osx.sh
-
-# Setup the hosts file
-cd "$SCRIPT_DIR"
-./etc-hosts.sh
diff --git a/node-admin/scripts/vm.sh b/node-admin/scripts/vm.sh
deleted file mode 100755
index 19542a7c392..00000000000
--- a/node-admin/scripts/vm.sh
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-set -e
-
-source "${0%/*}/common-vm.sh"
-
-DOCKER_VM_WAS_STARTED=false
-
-if ! docker-machine status "$DOCKER_VM_NAME" &> /dev/null; then
- # Machine does not exist and we have to create and start
- docker-machine create -d virtualbox \
- --virtualbox-disk-size "$DOCKER_VM_DISK_SIZE_IN_MB" \
- --virtualbox-memory "$DOCKER_VM_MEMORY_SIZE_IN_MB" \
- --virtualbox-cpu-count "$DOCKER_VM_CPU_COUNT" \
- --virtualbox-hostonly-cidr "$DOCKER_VM_HOST_CIDR" \
- "$DOCKER_VM_NAME"
-
- eval $(docker-machine env "$DOCKER_VM_NAME")
-
- # Node admin expects different names for the certificates. Just symlink docker has
- # generated for us to match those in node-admin/src/main/application/services.xml.
- (
- cd "$DOCKER_CERT_PATH"
- ln -s ca.pem ca_cert.pem
- ln -s key.pem client_key.pem
- ln -s cert.pem client_cert.pem
- )
- DOCKER_VM_WAS_STARTED=true
-fi
-
-
-VESPA_VM_STATUS=$(docker-machine status "$DOCKER_VM_NAME")
-if [ "$VESPA_VM_STATUS" == "Stopped" ]; then
- docker-machine start "$DOCKER_VM_NAME"
- DOCKER_VM_WAS_STARTED=true
- VESPA_VM_STATUS=$(docker-machine status "$DOCKER_VM_NAME")
-fi
-
-if [ "$VESPA_VM_STATUS" != "Running" ]; then
- echo "Unable to get Docker machine $DOCKER_VM_NAME up and running."
- echo "You can try to manually remove the machine: docker-machine rm -y $DOCKER_VM_NAME "
- echo " and then rerun this script."
- echo "Exiting."
- exit 1
-fi
-
-if $DOCKER_VM_WAS_STARTED; then
- # Put anything that is not persisted between VM restarts in here.
- # Set up NAT for the $HOST_BRIDGE_INTERFACE interface so that we can connect directly from OS X.
- docker-machine ssh "$DOCKER_VM_NAME" sudo /usr/local/sbin/iptables -t nat -A POSTROUTING -s "$HOST_BRIDGE_NETWORK" ! -o "$HOST_BRIDGE_INTERFACE" -j MASQUERADE
- docker-machine ssh "$DOCKER_VM_NAME" sudo /usr/local/sbin/iptables -A FORWARD -o "$HOST_BRIDGE_INTERFACE" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-
- # Install dependencies used by setup scripts
- docker-machine ssh "$DOCKER_VM_NAME" tce-load -wi python bash
-fi
-
-# Get the environment for our VM
-eval $(docker-machine env "$DOCKER_VM_NAME")
-
-if [ $# -ge 1 ]; then
- declare -r ARG_SCRIPT=$1
- shift
-
- declare -r ARG_SCRIPT_BASE=$(basename "$ARG_SCRIPT")
- declare -r ARG_SCRIPT_DIR=$(cd $(dirname "$ARG_SCRIPT") && pwd -P)
- declare -r ARG_SCRIPT_ABS="$ARG_SCRIPT_DIR/$ARG_SCRIPT_BASE"
-
- if ! docker-machine ssh "$DOCKER_VM_NAME" which "$ARG_SCRIPT_ABS" &> /dev/null; then
- echo "Provided script file does not exist or is not executable in VM : $ARG_SCRIPT_ABS"
- echo "Usage: $0 [SCRIPT] [SCRIPT_ARGS...]"
- exit 1
- fi
-
- # Start the provided script. This works because the $HOME directory is mapped in the same location in the VM.
- docker-machine ssh "$DOCKER_VM_NAME" "CONTAINER_CERT_PATH=$DOCKER_CERT_PATH NETWORK_TYPE=vm $ARG_SCRIPT_ABS $*"
-fi
-
diff --git a/node-admin/scripts/zone.sh b/node-admin/scripts/zone.sh
deleted file mode 100755
index b35f367ba59..00000000000
--- a/node-admin/scripts/zone.sh
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-set -e
-
-source "${0%/*}/common.sh"
-
-function Usage {
- UsageHelper "$@" <<EOF
-Usage: $SCRIPT_NAME <command> [<option>]...
-Manage Hosted Vespa zone on localhost using Docker.
-
-The Docker daemon must already be running, and the Docker image must have been
-built. The node-admin module must have been packaged.
-
-Commands:
- start Start zone (start Config Server, Node Admin, etc)
- stop Stop zone (take down Node Admin, Config Server, etc)
- restart Stop, then start
-
-Options:
- --hv-env <env>
- Make a zone with this Hosted Vespa environment. Must be one of
- prod, dev, test, staging, etc. Default is $DEFAULT_HOSTED_VESPA_ENVIRONMENT.
- --hv-region <region>
- Make a zone with this Hosted Vespa region. Default is $DEFAULT_HOSTED_VESPA_REGION.
- --num-nodes <num-nodes>
- Make a zone with <num-nodes> Docker nodes instead of the default $DEFAULT_NUM_APP_CONTAINERS.
-EOF
-}
-
-function Stop {
- if (($# != 0))
- then
- Usage
- fi
-
- # Prime sudo to avoid password prompt in the middle of the script.
- sudo true
-
- ./node-admin.sh stop
-
- # TODO: Stop and remove existing vespa node containers.
-
- # There's no need to stop populate-noderepo-with-local-nodes.sh, as the
- # whole node repo is going down when the config server is stopped.
- #
- # ./populate-noderepo-with-local-nodes.sh stop
-
- ./config-server.sh stop
- ./make-host-like-container.sh stop
- ./network-bridge.sh stop
- ./etc-hosts.sh stop
-}
-
-function Start {
- if (($# != 0))
- then
- Usage
- fi
-
- # Prime sudo to avoid password prompt in the middle of the script.
- sudo true
-
- ./etc-hosts.sh --num-nodes "$NUM_APP_CONTAINERS"
- ./network-bridge.sh
- ./make-host-like-container.sh
-
- local region="${OPTION_HV_REGION:-$DEFAULT_HOSTED_VESPA_REGION}"
- local env="${OPTION_HV_ENV:-$DEFAULT_HOSTED_VESPA_ENVIRONMENT}"
- ./config-server.sh --wait=true --hv-region="$region" --hv-env="$env"
-
- ./populate-noderepo-with-local-nodes.sh --num-nodes "$NUM_APP_CONTAINERS"
- ./node-admin.sh
-}
-
-# Makes it easier to access scripts in the same 'scripts' directory
-cd "$SCRIPT_DIR"
-
-Main "$@"