summaryrefslogtreecommitdiffstats
path: root/node-admin
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
committerJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
commit72231250ed81e10d66bfe70701e64fa5fe50f712 (patch)
tree2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /node-admin
Publish
Diffstat (limited to 'node-admin')
-rw-r--r--node-admin/.gitignore3
-rw-r--r--node-admin/Dockerfile.template36
-rw-r--r--node-admin/OWNERS2
-rw-r--r--node-admin/README.md111
-rw-r--r--node-admin/README_LINUX.md129
-rw-r--r--node-admin/README_MAC.md73
-rwxr-xr-xnode-admin/build.sh36
-rw-r--r--node-admin/include/config-ctl.patch5
-rwxr-xr-xnode-admin/include/deploy-music-app.sh85
-rw-r--r--node-admin/include/music-on-docker-services.xml18
-rw-r--r--node-admin/include/node-flavors.xml15
-rwxr-xr-xnode-admin/include/nodectl-instance.sh138
-rw-r--r--node-admin/include/root-bashrc1
-rwxr-xr-xnode-admin/include/start-config-server.sh138
-rwxr-xr-xnode-admin/include/start-services.sh98
-rw-r--r--node-admin/pom.xml162
-rwxr-xr-xnode-admin/scripts/app.sh217
-rw-r--r--node-admin/scripts/common-vm.sh13
-rw-r--r--node-admin/scripts/common.sh180
-rwxr-xr-xnode-admin/scripts/config-server.sh140
-rwxr-xr-xnode-admin/scripts/configure-container-networking.py311
-rwxr-xr-xnode-admin/scripts/etc-hosts.sh117
-rw-r--r--node-admin/scripts/ipaddress.py2411
-rwxr-xr-xnode-admin/scripts/make-host-like-container.sh52
-rwxr-xr-xnode-admin/scripts/network-bridge.sh63
-rwxr-xr-xnode-admin/scripts/node-admin.sh135
-rwxr-xr-xnode-admin/scripts/node-repo.sh318
-rwxr-xr-xnode-admin/scripts/populate-noderepo-with-local-nodes.sh44
-rw-r--r--node-admin/scripts/pyroute2/__init__.py95
-rw-r--r--node-admin/scripts/pyroute2/arp.py69
-rw-r--r--node-admin/scripts/pyroute2/common.py288
-rw-r--r--node-admin/scripts/pyroute2/config.py10
-rw-r--r--node-admin/scripts/pyroute2/debugger.py85
-rw-r--r--node-admin/scripts/pyroute2/dhcp/__init__.py300
-rw-r--r--node-admin/scripts/pyroute2/dhcp/dhcp4msg.py60
-rw-r--r--node-admin/scripts/pyroute2/dhcp/dhcp4socket.py135
-rw-r--r--node-admin/scripts/pyroute2/ipdb/__init__.py981
-rw-r--r--node-admin/scripts/pyroute2/ipdb/common.py51
-rw-r--r--node-admin/scripts/pyroute2/ipdb/interface.py709
-rw-r--r--node-admin/scripts/pyroute2/ipdb/linkedset.py134
-rw-r--r--node-admin/scripts/pyroute2/ipdb/route.py354
-rw-r--r--node-admin/scripts/pyroute2/ipdb/transactional.py402
-rw-r--r--node-admin/scripts/pyroute2/iproute.py888
-rw-r--r--node-admin/scripts/pyroute2/ipset.py149
-rw-r--r--node-admin/scripts/pyroute2/iwutil.py355
-rw-r--r--node-admin/scripts/pyroute2/netlink/__init__.py1349
-rw-r--r--node-admin/scripts/pyroute2/netlink/generic/__init__.py74
-rw-r--r--node-admin/scripts/pyroute2/netlink/ipq/__init__.py131
-rw-r--r--node-admin/scripts/pyroute2/netlink/nfnetlink/__init__.py33
-rw-r--r--node-admin/scripts/pyroute2/netlink/nfnetlink/ipset.py77
-rw-r--r--node-admin/scripts/pyroute2/netlink/nl80211/__init__.py609
-rw-r--r--node-admin/scripts/pyroute2/netlink/nlsocket.py856
-rw-r--r--node-admin/scripts/pyroute2/netlink/rtnl/__init__.py156
-rw-r--r--node-admin/scripts/pyroute2/netlink/rtnl/errmsg.py11
-rw-r--r--node-admin/scripts/pyroute2/netlink/rtnl/fibmsg.py60
-rw-r--r--node-admin/scripts/pyroute2/netlink/rtnl/ifaddrmsg.py96
-rw-r--r--node-admin/scripts/pyroute2/netlink/rtnl/ifinfmsg.py1068
-rw-r--r--node-admin/scripts/pyroute2/netlink/rtnl/iprsocket.py164
-rw-r--r--node-admin/scripts/pyroute2/netlink/rtnl/iw_event.py85
-rw-r--r--node-admin/scripts/pyroute2/netlink/rtnl/ndmsg.py61
-rw-r--r--node-admin/scripts/pyroute2/netlink/rtnl/req.py182
-rw-r--r--node-admin/scripts/pyroute2/netlink/rtnl/rtmsg.py90
-rw-r--r--node-admin/scripts/pyroute2/netlink/rtnl/tcmsg.py917
-rw-r--r--node-admin/scripts/pyroute2/netlink/taskstats/__init__.py167
-rw-r--r--node-admin/scripts/pyroute2/netns/__init__.py123
-rw-r--r--node-admin/scripts/pyroute2/netns/nslink.py310
-rw-r--r--node-admin/scripts/pyroute2/netns/process/__init__.py39
-rw-r--r--node-admin/scripts/pyroute2/netns/process/base_p2.py7
-rw-r--r--node-admin/scripts/pyroute2/netns/process/base_p3.py7
-rw-r--r--node-admin/scripts/pyroute2/netns/process/proxy.py163
-rw-r--r--node-admin/scripts/pyroute2/protocols/__init__.py234
-rw-r--r--node-admin/scripts/pyroute2/protocols/rawsocket.py70
-rw-r--r--node-admin/scripts/pyroute2/proxy.py65
-rwxr-xr-xnode-admin/scripts/route-osx.sh16
-rwxr-xr-xnode-admin/scripts/setup-docker.sh176
-rwxr-xr-xnode-admin/scripts/setup-route-and-hosts-osx.sh20
-rwxr-xr-xnode-admin/scripts/vm.sh77
-rwxr-xr-xnode-admin/scripts/zone.sh80
-rw-r--r--node-admin/src/main/application/services.xml17
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java93
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java155
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java144
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java33
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java404
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/Container.java54
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ContainerName.java44
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/Docker.java49
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImage.java44
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java589
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java44
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/package-info.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java28
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java150
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeState.java7
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetNodesResponse.java73
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java35
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesRequestBody.java31
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesResponse.java17
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java21
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorException.java9
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java101
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java15
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java20
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java44
-rw-r--r--node-admin/src/main/resources/configdefinitions/docker.def7
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminTest.java194
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImplTest.java1073
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java322
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResultTest.java25
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java98
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java50
111 files changed, 21684 insertions, 0 deletions
diff --git a/node-admin/.gitignore b/node-admin/.gitignore
new file mode 100644
index 00000000000..51bc5191fc6
--- /dev/null
+++ b/node-admin/.gitignore
@@ -0,0 +1,3 @@
+/dependencies
+/Dockerfile
+/**/*.pyc
diff --git a/node-admin/Dockerfile.template b/node-admin/Dockerfile.template
new file mode 100644
index 00000000000..485518cb86a
--- /dev/null
+++ b/node-admin/Dockerfile.template
@@ -0,0 +1,36 @@
+# NOTE: Whenever the vespa-base version changes, review pom.xml. A fixed version of vespa-base contains components
+# built from a certain vespa revision, while the docker image you will build with this Dockerfile contains components
+# (i.e. the node-admin jar file) built from HEAD. Since dependencies change over time, there may be mismatches.
+# Therefore, one of the following may be necessary for one or more dependencies in pom.xml:
+# a) Hard-coding a version override.
+# b) Changing a hard-coded version override.
+# c) Removing a hard-coded version override.
+#
+# See build.sh for the meaning of $NODE_ADMIN_FROM_IMAGE.
+FROM $NODE_ADMIN_FROM_IMAGE
+
+# Be aware that this Dockerfile is not being used in any pipelines and has no relation to production environments etc.
+# It is here for developers' convenience - it allows building and experimenting with node-admin locally.
+
+# Things for convenience.
+RUN yum install -y tcpdump
+ADD include/root-bashrc /root/.bashrc
+
+# Override what's in the base image with local versions:
+ADD target/node-admin-jar-with-dependencies.jar $VESPA_HOME/lib/jars/node-admin-jar-with-dependencies.jar
+ADD src/main/application/services.xml $VESPA_HOME/conf/node-admin-app/services.xml
+ADD scripts/configure-container-networking.py $VESPA_HOME/libexec/vespa/node-admin/configure-container-networking.py
+
+# For deploying sample application.
+ADD include/deploy-music-app.sh /usr/local/bin/deploy-music-app.sh
+ADD include/music-on-docker-services.xml $VESPA_HOME/share/vespa/sampleapps/search/music/services.xml
+
+# Entrypoint for running config server in a container.
+ADD include/start-config-server.sh /usr/local/bin/start-config-server.sh
+
+# Included in base image, but here overridden with local modifications.
+# TODO: Update the source instead.
+ADD include/start-services.sh /usr/local/bin/start-services.sh
+
+# Make config-server aware of node flavor 'docker'.
+ADD include/node-flavors.xml $VESPA_HOME/conf/configserver-app/node-flavors.xml
diff --git a/node-admin/OWNERS b/node-admin/OWNERS
new file mode 100644
index 00000000000..9ecc8472a21
--- /dev/null
+++ b/node-admin/OWNERS
@@ -0,0 +1,2 @@
+bakksjo
+hakon
diff --git a/node-admin/README.md b/node-admin/README.md
new file mode 100644
index 00000000000..b727dde7bb8
--- /dev/null
+++ b/node-admin/README.md
@@ -0,0 +1,111 @@
+# Node Admin
+
+## Setup
+
+Set up Docker on your machine according to the instructions in README_LINUX or README_MAC, depending on your hardware.
+
+You should have the docker daemon running and the following environment variables set:
+```
+DOCKER_HOST
+CONTAINER_CERT_PATH
+```
+
+## Building
+
+Build Node Admin and include it (and other local modifications) in the Docker image ```vespa-local```:
+```
+mvn clean package
+./build.sh
+```
+
+## Running
+
+TODO: Outdated! Update this section with info on how to run everything locally.
+
+Start the container for the config server (TODO). Set the CONFIG_SERVER_ADDRESS
+variable to the hostname of the config server.
+
+Start the container
+```
+docker run -t -i --privileged \
+ -p 4080:4080 \
+ -v $CONTAINER_CERT_PATH:/host/docker/certs \
+ -e "DOCKER_HOST=$DOCKER_HOST" \
+ -e "CONFIG_SERVER_ADDRESS=$CONFIG_SERVER_ADDRESS" \
+ vespa-local:latest
+```
+
+This will map the client certificate/key files to the path where Node Admin looks for them (as configured in
+services.xml), and enable Node Admin to talk to the docker daemon. You can invoke Node Admin's REST APIs on port 4080
+from both inside the container and the outside host.
+
+## Using
+
+Trigger the incredibly rich and complex node-admin REST API(s)
+```
+curl localhost:4080/test/ping
+```
+
+## Troubleshooting
+
+If the container doesn't start, it can be helpful to look at the jdisc log. First, find the container id:
+```
+docker ps -a
+```
+
+Then, find the log files:
+```
+docker diff <container id>| grep $VESPA_HOME/logs
+```
+
+View the log file (`-L` follows the symbolic link):
+```
+docker cp -L <container id>:$VESPA_HOME/logs/jdisc_core/jdisc_core.log - | less
+```
+
+## Developing
+
+We will describe how you can build a Docker image for Vespa which will be used
+to set up a local Docker container with the Node Admin, and a local container
+with the Config Server.
+
+Then, we'll show how you bring up this local zone. And finally, how you can
+deploy a local Vespa application to this zone.
+
+### Building Local Docker Image
+
+A Dockerfile exists in the module's root directory. This Dockerfile is not used
+in production or any pipelines, it is here for convenience so you can build
+images and experiment locally. See build.sh for how to build.
+
+The image created by the Dockerfile will be used to run Node Admin or a Config
+Server.
+
+### Starting a Local Zone
+
+To start a local zone, ensure your operating system ignores ```config-server```
+and ```node-admin``` for proxying. Then issue the following command:
+
+```
+scripts/zone.sh start
+```
+
+The Node Admin and Config Server now runs in the ```node-admin``` and
+```config-server``` Docker containers. These containers have their own IP
+addresses and hostnames (also ```node-admin``` and ```config-server```).
+
+### Deploying a Local Application
+
+To deploy an application, use ```scripts/app.sh```. Assuming you have checked
+out ```vespa/basic-search-for-docker``` to ```~```, and packaged it with ```mvn
+package```, you can deploy the application with:
+
+```
+scripts/app.sh deploy ~/vespa/basic-search-for-docker/target/application
+```
+
+You can undeploy it with
+
+```
+scripts/app.sh undeploy
+```
diff --git a/node-admin/README_LINUX.md b/node-admin/README_LINUX.md
new file mode 100644
index 00000000000..b369044eef5
--- /dev/null
+++ b/node-admin/README_LINUX.md
@@ -0,0 +1,129 @@
+# Setting up Docker on a linux machine
+
+First, install Docker. With Fedora 21 you should follow
+https://docs.docker.com/installation/fedora/, which describes how to install
+Docker, start the Docker daemon, verify you can download images and run them,
+and making your user run docker instances.
+
+```
+sudo systemctl enable docker
+```
+
+## Set up yahoo user
+
+The Vespa docker containers will run images that need to access the host file
+system as the user 'yahoo' with UID 1000, e.g. the Node Admin runs as this
+user. If this UID is not already taken, you can create a yahoo user as follows:
+
+```
+sudo useradd -u 1000 -g 100 -s /dev/null yahoo
+```
+
+If the UID is already in use you should move the user to a new UID first.
+Alternatively, it might be possible to reuse that user, but this is confusing
+and may lead to errors later on (and has not been tested). In the following
+example we move the 'landesk' user from UID 1000 to 1010, keeping its GID 1001.
+
+```
+sudo usermod -u 1010 landesk
+sudo find / -user 1000 -exec chown -h 1010 {} \;
+```
+
+## Set up image storage (aka don't break your machine)
+
+Docker will by default download (huge) images to a directory under /var. On our
+Fedora machines, /var is part of the root filesystem, which does not have a lot
+of free space. Since docker runs as root, it is allowed to completely fill up
+the filesystem, and it will happily do so. Fedora works very poorly with a full
+root filesystem. You won't even be able to log in and clean up the disk usage
+once it's happened.
+
+So you'll want to store images somewhere else. An obvious choice is /home,
+which typically has a lot more room. Make Docker use directories in the docker
+user's home directory. Run the following script to do this:
+
+```
+scripts/setup-docker.sh home
+```
+
+## Set up TLS
+
+By default, the docker CLI communicates with the docker daemon via a unix
+socket. This is fine in itself, but not suffficient for our use. Node Admin,
+itself running in a container, will talk to the docker daemon to start
+containers for vespa nodes. Node Admin uses a Java library for communication
+with the docker daemon, and this library depends on JNI (native) code for unix
+socket communication. We don't want that, so that dependency has been
+excluded. Therefore, Node Admin uses TLS over IP to communicate with the docker
+daemon. You must therefore set up docker with TLS. Mostly, you can follow the
+instructions at https://docs.docker.com/articles/https/.
+
+The commands can be run with
+
+```
+scripts/setup-docker.sh certs
+```
+
+Note the following:
+
+ - You will be asked to generate a key, and will repeatedly be asked for it.
+ - Use your fully qualified domain name for Common Name.
+
+Now, you need to tell the docker daemon to use TLS. Edit the file ```/lib/systemd/system/docker.service``` and change
+the ExecStart line so it includes the following arguments:
+```
+--tlsverify --tlscacert=/etc/dockercert_daemon/ca_cert.pem --tlscert=/etc/dockercert_daemon/server_cert.pem --tlskey=/etc/dockercert_daemon/server_key.pem -H=0.0.0.0:2376
+```
+
+Then restart docker:
+```
+sudo systemctl daemon-reload
+sudo systemctl restart docker
+```
+
+Now tell the docker CLI how to communicate with the docker daemon:
+```
+export DOCKER_HOST=tcp://127.0.0.1:2376
+export DOCKER_TLS_VERIFY=1
+export DOCKER_CERT_PATH=/etc/dockercert_cli
+```
+
+(You might want to add this to your .bashrc file.)
+
+Now, test that the docker cli can talk to the docker daemon:
+```
+docker version
+docker run --rm hello-world
+```
+
+These should run without errors. Finally, to run Node Admin locally, it needs access to the certificate/key files.
+```
+export CONTAINER_CERT_PATH=/etc/dockercert_container
+```
+
+This environment variable will be used when starting the container, which is decribed in the platform-independent
+README file.
+
+While docker can and should be run as your user, it's nice to make it possible
+to run docker under root too. To enable this you must make sure sudo doesn't
+strip off the environment variables, otherwise certain docker commands may
+hang. Add a file /etc/sudoers.d/passthrough-docker-env with the content:
+
+```
+Defaults env_keep += "DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH CONTAINER_CERT_PATH"
+```
+
+You are now done with the linux-specific setup work.
+
+## Other
+
+For more information on how to configure the docker daemon, see https://docs.docker.com/articles/systemd/.
+
+## Upgrade of Docker
+
+When Docker upgrades it may overwrite /lib/systemd/system/docker.service. The
+symptom is that any docker command will fail with the error message "Cannot
+connect to the Docker daemon. Is the docker daemon running on this host?".
+
+Once you have updated docker.service according to this document, and restarted
+the Docker daemon, Docker should work again.
diff --git a/node-admin/README_MAC.md b/node-admin/README_MAC.md
new file mode 100644
index 00000000000..75a67f6a29c
--- /dev/null
+++ b/node-admin/README_MAC.md
@@ -0,0 +1,73 @@
+# Setting up Docker on OS X
+Install Docker Toolbox according to the procedure on [https://docs.docker.com/mac/step_one](https://docs.docker.com/mac/step_one).
+
+# Running Vespa on OS X
+
+## Starting the VM
+On OS X the docker daemon is running inside a VM called boot2docker. This VM is running using the VirtualBox virtualization software. To setup and start the VM, execute the following script:
+
+```
+scripts/vm.sh
+```
+You should now have a Docker machine up and running. This can be verified with:
+
+```
+docker-machine ls
+```
+Which should list the running vespa machine.
+
+## Building the Vespa Docker image
+Building node-admin requires that the vespa Docker machine is up and running. This is because the building of the Docker image actually happens inside the VM.
+
+First we need to make sure that some environment variables are set so that the ```docker``` command knows how to communicate with the VM:
+
+```
+eval $(docker-machine env vespa)
+```
+
+To build the image, follow the instructions in [README.md](README.md).
+
+The Vespa Docker image will be persisted inside the VM and it is not necessary to rebuild the image if you stop and restart the VM. However, if you remove the VM with ```docker-machine rm vespa```, the image must be rebuilt.
+
+## Running Vespa with the node-admin scripts
+The scripts that are used for starting local zones and deploying applications in Linux can be used in OS X by prefixing them with ```scripts/vm.sh ```.
+
+Follow the instructions in [README.md](README.md) for starting local zones and deploying applications.
+
+## Accessing Vespa directly from OS X
+The ```scripts/vm.sh``` script does some of the network setup inside the VM that is required for this to work. However, it is necessary set up routing and the ```/etc/hosts``` file to get this working. To automatically do this, execute the script:
+
+```
+scripts/setup-route-and-hosts-osx.sh
+```
+The script will prompt you to continue as this will alter your routing table and /etc/hosts file. If your local zone is up and running, the config server should respond to this:
+
+```
+curl config-server:19071
+```
+
+If you don't want your `/etc/hosts` file to be changed, the
+`scripts/route-osx.sh` script can be used instead. This means that you must
+inspect `/etc/hosts` inside the VM to find the IP address of each container:
+`docker-machine ssh cat /etc/hosts`
+
+## Useful Docker commands
+Obtain the values for the required environment variables with:
+
+```
+eval $(docker-machine env vespa)
+```
+
+How to log onto the Docker base host:
+
+```
+docker-machine ssh vespa
+```
+
+Regular ```docker``` commands works as in Linux when you have the environment variables set.
+Look in [README.md](README.md) for useful docker commands.
+
+## Issues
+* Accessing Vespa from OS X while on a Cisco VPN connection does not work. This is because the VPN client will protect the routing table on OS X.
+ * Workaround is to use ```docker-machine ssh vespa``` and then execute everything from inside the VM.
+
diff --git a/node-admin/build.sh b/node-admin/build.sh
new file mode 100755
index 00000000000..f0476c6a30b
--- /dev/null
+++ b/node-admin/build.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -e
+
+function Usage {
+ cat <<EOF >&2
+$*
+
+Usage: build.sh
+Builds the local node-admin docker image used in the local zone.
+
+You must set the NODE_ADMIN_FROM_IMAGE environment variable to point to the
+base image (FROM-line in Dockerfile) you'd like to build the node admin image
+on.
+EOF
+ exit 1
+}
+
+if [ -z "$NODE_ADMIN_FROM_IMAGE" ]
+then
+ Usage "NODE_ADMIN_FROM_IMAGE environment variable is not set."
+elif [[ "$NODE_ADMIN_FROM_IMAGE" =~ % ]]
+then
+ Usage "NODE_ADMIN_FROM_IMAGE environment variable cannot contain the %-character."
+elif [ -z "$VESPA_HOME" ]
+then
+ Usage "VESPA_HOME environment variable is not set."
+fi
+
+cat Dockerfile.template | \
+ sed 's%$NODE_ADMIN_FROM_IMAGE%'"$NODE_ADMIN_FROM_IMAGE%g" | \
+ sed 's%$VESPA_HOME%'"$VESPA_HOME%g" \
+ > Dockerfile
+
+docker build --tag="vespa-local:latest" .
diff --git a/node-admin/include/config-ctl.patch b/node-admin/include/config-ctl.patch
new file mode 100644
index 00000000000..fdb6882a375
--- /dev/null
+++ b/node-admin/include/config-ctl.patch
@@ -0,0 +1,5 @@
+36,39d35
+< case $hname in
+< *.*.*) ;;
+< *) echo "The hostname must be a FQDN, was: $hname" ; exit 1 ;;
+< esac
diff --git a/node-admin/include/deploy-music-app.sh b/node-admin/include/deploy-music-app.sh
new file mode 100755
index 00000000000..1a13725213d
--- /dev/null
+++ b/node-admin/include/deploy-music-app.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#
+# Usage: docker exec config-server /usr/local/bin/deploy-music-app.sh
+# Deploy app to config-server running in local Docker zone
+#
+# You must build the vespa-local:latest (vespa/vespa/node-admin) image and
+# (re)start the local zone, before running deploy-music-app.sh.
+#
+# See also app.sh
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+ myname=${0}
+ mypath=${myname%/*}
+ myname=${myname##*/}
+ if [ "$mypath" ] && [ -d "$mypath" ]; then
+ return
+ fi
+ mypath=$(pwd)
+ if [ -f "${mypath}/${myname}" ]; then
+ return
+ fi
+ echo "FATAL: Could not figure out the path where $myname lives from $0"
+ exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+ if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+ # ensure it ends with "/" :
+ VESPA_HOME=${VESPA_HOME%/}/
+ export VESPA_HOME
+ common_env=$VESPA_HOME/$COMMON_ENV
+ if [ -f "$common_env" ]; then
+ . $common_env
+ return
+ fi
+ fi
+ return 1
+}
+
+findroot () {
+ source_common_env && return
+ if [ "$VESPA_HOME" ]; then
+ echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+ exit 1
+ fi
+ if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+ VESPA_HOME="$ROOT"
+ source_common_env && return
+ fi
+ findpath
+ while [ "$mypath" ]; do
+ VESPA_HOME=${mypath}
+ source_common_env && return
+ mypath=${mypath%/*}
+ done
+ echo "FATAL: missing VESPA_HOME environment variable"
+ echo "Could not locate $COMMON_ENV anywhere"
+ exit 1
+}
+
+findroot
+
+# END environment bootstrap section
+
+set -e
+set -x
+
+declare -r CONFIG_SERVER_HOSTNAME=config-server
+declare -r CONFIG_SERVER_PORT=19071
+declare -r TENANT_NAME=localtenant
+# TODO: Make it possible to deploy any app from host context, not only this app (from within container).
+declare -r VESPA_APP=$VESPA_HOME/share/vespa/sampleapps/search/music
+
+# Create tenant
+curl -X PUT $CONFIG_SERVER_HOSTNAME:$CONFIG_SERVER_PORT/application/v2/tenant/$TENANT_NAME
+
+# Deploy sample app
+deploy -e "$TENANT_NAME" prepare $VESPA_APP
+deploy -e "$TENANT_NAME" activate
diff --git a/node-admin/include/music-on-docker-services.xml b/node-admin/include/music-on-docker-services.xml
new file mode 100644
index 00000000000..0bb37950c98
--- /dev/null
+++ b/node-admin/include/music-on-docker-services.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<services version="1.0">
+
+ <jdisc version="1.0" id="container">
+ <search />
+ <nodes count="2" flavor="docker" docker-image="vespa-local:latest" />
+ </jdisc>
+
+ <content id="music" version="1.0">
+ <redundancy>1</redundancy>
+ <documents>
+ <document type="music" mode="index" />
+ </documents>
+ <nodes count="2" flavor="docker" docker-image="vespa-local:latest" />
+ </content>
+
+</services>
diff --git a/node-admin/include/node-flavors.xml b/node-admin/include/node-flavors.xml
new file mode 100644
index 00000000000..419ea348493
--- /dev/null
+++ b/node-admin/include/node-flavors.xml
@@ -0,0 +1,15 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<config name="vespa.config.nodes.node-repository">
+ <flavor>
+ <item>
+ <!-- Note: This flavor does NOT match the production 'docker' flavor -->
+ <name>docker</name>
+ <environment>DOCKER_CONTAINER</environment>
+ <minCpuCores>1.0</minCpuCores>
+ <minMainMemoryAvailableGb>3.0</minMainMemoryAvailableGb>
+ <minDiskAvailableGb>10.0</minDiskAvailableGb>
+ <description>DOCKER_CONTAINER with 1.0 CPUs, 3.0 Gb memory and 10.0 Gb disk</description>
+ </item>
+ </flavor>
+</config>
+
diff --git a/node-admin/include/nodectl-instance.sh b/node-admin/include/nodectl-instance.sh
new file mode 100755
index 00000000000..5a6665dbdc7
--- /dev/null
+++ b/node-admin/include/nodectl-instance.sh
@@ -0,0 +1,138 @@
+#!/bin/sh
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# WARNING: This script should be kept in sync with the file used by Chef:
+# vespa-cookbooks/hosted/files/default/prepost-instance.sh
+# TODO: Remove the above cookbook file (with the down-side that a new script
+# requires a new vespa release, instead of just a hosted release).
+
+# Usage: nodectl-instance.sh [start|stop]
+#
+# start: Set the node "in service" by e.g. undraining container traffic.
+# start can be assumed to have completed successfully.
+#
+# stop: Prepare for a short suspension, e.g. there's a pending upgrade. Set the
+# node "out of service" by draining container traffic, and flush index for a
+# quick start after the suspension. There's no need to stop.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+ myname=${0}
+ mypath=${myname%/*}
+ myname=${myname##*/}
+ if [ "$mypath" ] && [ -d "$mypath" ]; then
+ return
+ fi
+ mypath=$(pwd)
+ if [ -f "${mypath}/${myname}" ]; then
+ return
+ fi
+ echo "FATAL: Could not figure out the path where $myname lives from $0"
+ exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+ if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+ # ensure it ends with "/" :
+ VESPA_HOME=${VESPA_HOME%/}/
+ export VESPA_HOME
+ common_env=$VESPA_HOME/$COMMON_ENV
+ if [ -f "$common_env" ]; then
+ . $common_env
+ return
+ fi
+ fi
+ return 1
+}
+
+findroot () {
+ source_common_env && return
+ if [ "$VESPA_HOME" ]; then
+ echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+ exit 1
+ fi
+ if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+ VESPA_HOME="$ROOT"
+ source_common_env && return
+ fi
+ findpath
+ while [ "$mypath" ]; do
+ VESPA_HOME=${mypath}
+ source_common_env && return
+ mypath=${mypath%/*}
+ done
+ echo "FATAL: missing VESPA_HOME environment variable"
+ echo "Could not locate $COMMON_ENV anywhere"
+ exit 1
+}
+
+findroot
+
+# END environment bootstrap section
+
+has_servicename() {
+ local name="$1"
+ $VESPA_HOME/bin/vespa-model-inspect host $(hostname) | grep -q "$name @ "
+ return $?
+}
+
+has_container() {
+ has_servicename container || has_servicename qrserver
+}
+
+has_searchnode() {
+ has_servicename searchnode
+}
+
+container_drain() {
+ # TODO: Implement proper draining
+ sleep 60
+}
+
+start() {
+ # Always start vip for now
+ $echo $VESPA_HOME/bin/vespa-routing vip -u chef in
+}
+
+stop() {
+ # Always stop vip for now
+ $echo $VESPA_HOME/bin/vespa-routing vip -u chef out
+
+ if has_searchnode; then
+ $echo $VESPA_HOME/bin/vespa-proton-cmd --local triggerFlush
+ fi
+
+ if has_container; then
+ $echo container_drain
+ fi
+}
+
+main() {
+ if [ $# -lt 1 ]; then
+ echo "Usage: $0 [-e] start|stop" >&2
+ exit 1
+ fi
+
+ echo=""
+ if [ "$1" = "-e" ]; then
+ echo=echo
+ shift
+ fi
+
+ action="$1"
+
+ if [ "$action" = "start" ]; then
+ start
+ elif [ "$action" = "stop" ]; then
+ stop
+ else
+ echo "Unknown action: $action" >&2
+ exit 1
+ fi
+}
+
+main "$@"
diff --git a/node-admin/include/root-bashrc b/node-admin/include/root-bashrc
new file mode 100644
index 00000000000..fda1752c85f
--- /dev/null
+++ b/node-admin/include/root-bashrc
@@ -0,0 +1 @@
+export TERM=xterm
diff --git a/node-admin/include/start-config-server.sh b/node-admin/include/start-config-server.sh
new file mode 100755
index 00000000000..a84b0454db3
--- /dev/null
+++ b/node-admin/include/start-config-server.sh
@@ -0,0 +1,138 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+ myname=${0}
+ mypath=${myname%/*}
+ myname=${myname##*/}
+ if [ "$mypath" ] && [ -d "$mypath" ]; then
+ return
+ fi
+ mypath=$(pwd)
+ if [ -f "${mypath}/${myname}" ]; then
+ return
+ fi
+ echo "FATAL: Could not figure out the path where $myname lives from $0"
+ exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+ if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+ # ensure it ends with "/" :
+ VESPA_HOME=${VESPA_HOME%/}/
+ export VESPA_HOME
+ common_env=$VESPA_HOME/$COMMON_ENV
+ if [ -f "$common_env" ]; then
+ . $common_env
+ return
+ fi
+ fi
+ return 1
+}
+
+findroot () {
+ source_common_env && return
+ if [ "$VESPA_HOME" ]; then
+ echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+ exit 1
+ fi
+ if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+ VESPA_HOME="$ROOT"
+ source_common_env && return
+ fi
+ findpath
+ while [ "$mypath" ]; do
+ VESPA_HOME=${mypath}
+ source_common_env && return
+ mypath=${mypath%/*}
+ done
+ echo "FATAL: missing VESPA_HOME environment variable"
+ echo "Could not locate $COMMON_ENV anywhere"
+ exit 1
+}
+
+findroot
+
+# END environment bootstrap section
+
+export LC_ALL=C
+
+function WaitUntilHostIsReachable {
+ # Address may be IP or hostname.
+ local address="$1"
+
+ echo -n "Will wait until $address is reachable... "
+ while ! ping -q -c 1 -W 3 "$address" &>/dev/null
+ do
+ echo "not done (will retry)"
+ sleep 1
+ done
+ echo "Done"
+}
+
+function VerifyRequiredEnvironmentVariablesAreSet {
+ if [ -z "$HOSTED_VESPA_REGION" ]
+ then
+ Fail "Environment variable HOSTED_VESPA_REGION is not set"
+ fi
+ if [ -z "$CONFIG_SERVER_HOSTNAME" ]
+ then
+ Fail "Environment variable CONFIG_SERVER_HOSTNAME is not set"
+ fi
+ if [ -z "$HOST_BRIDGE_IP" ]
+ then
+ Fail "Environment variable HOST_BRIDGE_IP is not set"
+ fi
+
+ case "$HOSTED_VESPA_ENVIRONMENT" in
+ prod|test|dev|staging|perf) : ;;
+ *) Fail "The HOSTED_VESPA_ENVIRONMENT environment variable must be one of prod, test, dev, staging, or perf" ;;
+ esac
+}
+
+function InternalMain {
+ VerifyRequiredEnvironmentVariablesAreSet
+
+ mkdir -p $VESPA_HOME/logs
+ chmod 1777 $VESPA_HOME/logs
+ mkdir -p $VESPA_HOME/logs/jdisc_core
+
+ rm -rf $VESPA_HOME/var/vespa/bundlecache/standalone
+
+ yinst set \
+ cloudconfig_server.multitenant=true \
+ cloudconfig_server.region="$HOSTED_VESPA_REGION" \
+ cloudconfig_server.autostart=on \
+ cloudconfig_server.default_flavor=docker \
+ cloudconfig_server.environment="$HOSTED_VESPA_ENVIRONMENT" \
+ cloudconfig_server.hosted_vespa=true \
+ services.addr_configserver="$CONFIG_SERVER_HOSTNAME"
+
+ # Can also set jvmargs if necessary:
+ # set cloudconfig_server.jvmargs=-Dvespa.freezedetector.disable=true -XX:NewRatio=1 -verbose:gc -XX:+PrintGCDateStamps -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 -Xms6g -Xmx6g
+
+ # The network is set up asynchronously and from outside of this
+ # container. Wait until it's done.
+ WaitUntilHostIsReachable "$HOST_BRIDGE_IP"
+
+ yinst start cloudconfig_server
+
+ touch $VESPA_HOME/logs/jdisc_core/jdisc_core.log
+ $VESPA_HOME/bin/logfmt -N -f $VESPA_HOME/logs/jdisc_core/jdisc_core.log
+}
+
+function Main {
+ # Prefix each line to stdout/stderr with a timestamp to make it easier to
+ # understand the progress.
+ InternalMain |& while read -r
+ do
+ printf "%s %s\n" "$(date +%FT%T)" "$REPLY"
+ done
+}
+
+Main "$@"
diff --git a/node-admin/include/start-services.sh b/node-admin/include/start-services.sh
new file mode 100755
index 00000000000..0fe5c1c0724
--- /dev/null
+++ b/node-admin/include/start-services.sh
@@ -0,0 +1,98 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# TODO: This file duplicates a file in the base image (with local modifications).
+# Update the file there.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+ myname=${0}
+ mypath=${myname%/*}
+ myname=${myname##*/}
+ if [ "$mypath" ] && [ -d "$mypath" ]; then
+ return
+ fi
+ mypath=$(pwd)
+ if [ -f "${mypath}/${myname}" ]; then
+ return
+ fi
+ echo "FATAL: Could not figure out the path where $myname lives from $0"
+ exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+ if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+ # ensure it ends with "/" :
+ VESPA_HOME=${VESPA_HOME%/}/
+ export VESPA_HOME
+ common_env=$VESPA_HOME/$COMMON_ENV
+ if [ -f "$common_env" ]; then
+ . $common_env
+ return
+ fi
+ fi
+ return 1
+}
+
+findroot () {
+ source_common_env && return
+ if [ "$VESPA_HOME" ]; then
+ echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+ exit 1
+ fi
+ if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+ VESPA_HOME="$ROOT"
+ source_common_env && return
+ fi
+ findpath
+ while [ "$mypath" ]; do
+ VESPA_HOME=${mypath}
+ source_common_env && return
+ mypath=${mypath%/*}
+ done
+ echo "FATAL: missing VESPA_HOME environment variable"
+ echo "Could not locate $COMMON_ENV anywhere"
+ exit 1
+}
+
+findroot
+
+# END environment bootstrap section
+
+export LC_ALL=C
+
+function wait_for_network_up {
+ while true
+ do
+ for config_server_host in $(echo $CONFIG_SERVER_ADDRESS | tr "," " ")
+ do
+ ping -c 1 -W 3 $config_server_host && return
+ sleep 1
+ done
+ done
+}
+
+if [ -z $CONFIG_SERVER_ADDRESS ]
+then
+ echo "CONFIG_SERVER_ADDRESS must be set."
+ exit -1
+fi
+
+chown yahoo $VESPA_HOME/var/jdisc_container
+
+# Local modification; fixes ownership issues for vespa node running in container.
+chown yahoo $VESPA_HOME/var/zookeeper
+
+if [ -d "$VESPA_HOME/logs" ]
+then
+ chmod 1777 $VESPA_HOME/logs
+fi
+
+yinst set services.addr_configserver=$CONFIG_SERVER_ADDRESS
+wait_for_network_up
+yinst start services
+logfmt -n -f
diff --git a/node-admin/pom.xml b/node-admin/pom.xml
new file mode 100644
index 00000000000..c89e8524fc3
--- /dev/null
+++ b/node-admin/pom.xml
@@ -0,0 +1,162 @@
+<?xml version="1.0"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>parent</artifactId>
+ <version>6-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
+ </parent>
+
+ <artifactId>node-admin</artifactId>
+ <version>6-SNAPSHOT</version>
+ <packaging>container-plugin</packaging>
+ <name>${project.artifactId}</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>node-repository</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>defaults</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-junit</artifactId>
+ <version>2.0.0.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-dev</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.spotify</groupId>
+ <artifactId>docker-client</artifactId>
+ <version>3.5.12</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-json-provider</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>javax.ws.rs-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
+ </exclusion>
+ <exclusion> <!-- Using network sockets instead of unix domain sockets -->
+ <artifactId>jnr-unixsocket</artifactId>
+ <groupId>com.github.jnr</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>4.4.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>orchestrator-restapi</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>jaxrs_utils</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>jaxrs_client_utils</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>application-model</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>application</artifactId>
+ <scope>test</scope>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <compilerArgs>
+ <arg>-Xlint:all</arg>
+ <arg>-Werror</arg>
+ </compilerArgs>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/node-admin/scripts/app.sh b/node-admin/scripts/app.sh
new file mode 100755
index 00000000000..8f1787118ed
--- /dev/null
+++ b/node-admin/scripts/app.sh
@@ -0,0 +1,217 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+ myname=${0}
+ mypath=${myname%/*}
+ myname=${myname##*/}
+ if [ "$mypath" ] && [ -d "$mypath" ]; then
+ return
+ fi
+ mypath=$(pwd)
+ if [ -f "${mypath}/${myname}" ]; then
+ return
+ fi
+ echo "FATAL: Could not figure out the path where $myname lives from $0"
+ exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+ if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+ # ensure it ends with "/" :
+ VESPA_HOME=${VESPA_HOME%/}/
+ export VESPA_HOME
+ common_env=$VESPA_HOME/$COMMON_ENV
+ if [ -f "$common_env" ]; then
+ . $common_env
+ return
+ fi
+ fi
+ return 1
+}
+
+findroot () {
+ source_common_env && return
+ if [ "$VESPA_HOME" ]; then
+ echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+ exit 1
+ fi
+ if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+ VESPA_HOME="$ROOT"
+ source_common_env && return
+ fi
+ findpath
+ while [ "$mypath" ]; do
+ VESPA_HOME=${mypath}
+ source_common_env && return
+ mypath=${mypath%/*}
+ done
+ echo "FATAL: missing VESPA_HOME environment variable"
+ echo "Could not locate $COMMON_ENV anywhere"
+ exit 1
+}
+
+findroot
+
+# END environment bootstrap section
+
+set -e
+
+source "${0%/*}"/common.sh
+
+declare SCRIPTS_DIR="${0%/*}"
+
+declare -r APP_DIR_NAME_UNDER_SHARED=app
+
+function Usage {
+ UsageHelper "$@" <<EOF
+Usage: $SCRIPT_NAME <command> [<app-dir>]
+Deploy (or undeploy) application rooted at <app-dir> on localhost Config Server.
+
+The local zone must be up and running. <app-dir> should point to
+e.g. vespa/basic-search-on-docker/target/application.
+EOF
+}
+
+function RunOnConfigServer {
+ docker exec config-server "$@"
+}
+
+function VerifyApp {
+ local app_dir="$1"
+
+ # Sanity-check app_dir
+ if ! [ -d "$app_dir" ]
+ then
+ Fail "<app-dir> '$app_dir' is not a directory"
+ fi
+
+ local services_xml="$app_dir"/services.xml
+ if ! [ -f "$services_xml" ]
+ then
+ Fail "Failed to find services.xml in <app-dir> '$app_dir'"
+ fi
+
+ # Verify there's no <admin> element.
+ if grep -qE '<admin[ >]' "$services_xml"
+ then
+ Fail "services.xml cannot contain an <admin> element in hosted Vespa"
+ fi
+
+ # Verify <nodes> seems to be correctly specified (warning: this test is
+ # incomplete).
+ if grep -qE "<nodes>" "$services_xml" ||
+ ! grep -qE "<nodes (.* )?docker-image=" "$services_xml" ||
+ ! grep -qE "<nodes (.* )?flavor=[\"']docker[\"']" "$services_xml"
+ then
+ Fail "You must specify the <nodes> element in the following form" \
+ "in hosted Vespa w/Docker:" \
+ " <nodes count=\"2\" flavor=\"docker\" docker-image=\"IMAGE\" />" \
+ "where IMAGE is e.g. vespa-local:latest."
+ fi
+}
+
+# Copies the application rooted at $1 to a directory tree shared with the
+# Config Server.
+function CopyToSharedDir {
+ local app_dir="$1"
+
+ local shared_dir_on_localhost="$APPLICATION_STORAGE_ROOT/$CONFIG_SERVER_CONTAINER_NAME/$ROOT_DIR_SHARED_WITH_HOST"
+ if ! [ -d "$shared_dir_on_localhost" ]
+ then
+ Fail "Failed to find the Config Server's shared directory on" \
+ "localhost '$shared_dir_on_localhost', has the" \
+ "$CONFIG_SERVER_CONTAINER_NAME container been started?"
+ fi
+
+
+ local shared_app_dir_on_localhost="$shared_dir_on_localhost/$APP_DIR_NAME_UNDER_SHARED"
+ if [ "$shared_app_dir_on_localhost" != /home/docker/container-storage/config-server/shared/app ]
+ then
+ # This duplication of code is a safety-guard against 'rm -rf' unknown
+ # directories.
+ Fail "We're about to remove '$shared_app_dir_on_localhost', but it's" \
+ "pointing to something unexpected, refusing to proceed..."
+ fi
+
+ echo -n "Copying application to '$shared_app_dir_on_localhost'... "
+ rm -rf "$shared_app_dir_on_localhost"
+ cp -r "$app_dir" "$shared_app_dir_on_localhost"
+ echo done
+}
+
+function DeployApp {
+ if (($# != 1))
+ then
+ Usage
+ fi
+
+ local app_dir="$1"
+
+ VerifyApp "$app_dir"
+
+ CopyToSharedDir "$app_dir"
+
+ # Create tenant
+ echo -n "Creating tenant... "
+ local create_tenant_response
+ if create_tenant_response=$(curl --silent --show-error -X PUT "http://$CONFIG_SERVER_HOSTNAME:$CONFIG_SERVER_PORT/application/v2/tenant/$TENANT_NAME" 2>&1)
+ then
+ if ! [[ "$create_tenant_response" =~ "Tenant $TENANT_NAME created" ]] &&
+ ! [[ "$create_tenant_response" =~ "already exists" ]]
+ then
+ echo
+ Fail "May have failed to create the tenant: '$create_tenant_response'"
+ fi
+ else
+ echo
+ Fail "Failed to create the tenant: $?: '$create_tenant_response'"
+ fi
+ echo done
+
+ # Deploy app
+ local app_dir_on_config_server="/$ROOT_DIR_SHARED_WITH_HOST/$APP_DIR_NAME_UNDER_SHARED"
+ RunOnConfigServer $VESPA_HOME/bin/deploy -e "$TENANT_NAME" prepare "$app_dir_on_config_server"
+ echo "Activating application"
+ RunOnConfigServer $VESPA_HOME/bin/deploy -e "$TENANT_NAME" activate
+}
+
+function UndeployApp {
+ if (($# != 0))
+ then
+ Usage "undeploy takes no arguments"
+ fi
+
+ local app_name=default
+ local output
+ echo -n "Removing application $TENANT_NAME:$app_name... "
+ if ! output=$(curl --silent --show-error -X DELETE "http://$CONFIG_SERVER_HOSTNAME:$CONFIG_SERVER_PORT/application/v2/tenant/$TENANT_NAME/application/$app_name")
+ then
+ echo
+ Fail "Failed to remove application: $output"
+ fi
+
+ echo done
+}
+
+function Main {
+ if (($# == 0))
+ then
+ Usage "Missing command"
+ fi
+ local command="$1"
+ shift
+
+ case "$command" in
+ deploy) DeployApp "$@" ;;
+ undeploy) UndeployApp "$@" ;;
+ *) Usage "Unknown command '$command'" ;;
+ esac
+}
+
+Main "$@"
diff --git a/node-admin/scripts/common-vm.sh b/node-admin/scripts/common-vm.sh
new file mode 100644
index 00000000000..c91c75e1404
--- /dev/null
+++ b/node-admin/scripts/common-vm.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -e
+
+source "${0%/*}/common.sh"
+
+# VM configuration
+declare -r DOCKER_VM_NAME=vespa # Don't put spaces in the name
+declare -r DOCKER_VM_DISK_SIZE_IN_MB=40000
+declare -r DOCKER_VM_MEMORY_SIZE_IN_MB=4096
+declare -r DOCKER_VM_CPU_COUNT=1
+declare -r DOCKER_VM_HOST_CIDR=172.21.46.1/24
diff --git a/node-admin/scripts/common.sh b/node-admin/scripts/common.sh
new file mode 100644
index 00000000000..d07b4adcc5a
--- /dev/null
+++ b/node-admin/scripts/common.sh
@@ -0,0 +1,180 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+# Common variables and functions that may be useful for scripts IN THIS
+# DIRECTORY. Should be sourced as follows:
+#
+# source "${0%/*}/common.sh"
+#
+# WARNING: Some system variables, like the Config Server's, are also hardcoded
+# in the Docker image startup scripts.
+
+declare -r SCRIPT_NAME="${0##*/}"
+declare -r SCRIPT_DIR="${0%/*}"
+
+# TODO: Find a better name. Consider having separate images for config-server
+# and node-admin.
+declare -r DOCKER_IMAGE="vespa-local:latest"
+declare -r APPLICATION_STORAGE_ROOT="/home/docker/container-storage"
+declare -r ROOT_DIR_SHARED_WITH_HOST=shared
+
+# The 172.18.0.0/16 network is in IPDB.
+declare -r NETWORK_PREFIX=172.18
+declare -r NETWORK_PREFIX_BITLENGTH=16
+
+# Hostnames, IP addresses, names, etc of the infrastructure containers.
+declare -r HOST_BRIDGE_INTERFACE=vespa
+declare -r HOST_BRIDGE_IP="$NETWORK_PREFIX.0.1"
+declare -r HOST_BRIDGE_NETWORK="$NETWORK_PREFIX.0.0/$NETWORK_PREFIX_BITLENGTH"
+declare -r NODE_ADMIN_CONTAINER_NAME=node-admin
+declare -r CONFIG_SERVER_CONTAINER_NAME=config-server
+declare -r CONFIG_SERVER_HOSTNAME="$CONFIG_SERVER_CONTAINER_NAME"
+declare -r CONFIG_SERVER_IP="$NETWORK_PREFIX.1.1"
+declare -r CONFIG_SERVER_PORT=19071
+
+declare -r DEFAULT_HOSTED_VESPA_REGION=local-region
+declare -r DEFAULT_HOSTED_VESPA_ENVIRONMENT=prod
+
+# Hostnames, IP addresses, names, etc of the application containers. Hostname
+# and container names are of the form $PREFIX$N, where N is a number between 1
+# and $NUM_APP_CONTAINERS. The IP is $APP_NETWORK_PREFIX.$N.
+declare -r APP_NETWORK_PREFIX="$NETWORK_PREFIX.2"
+declare -r APP_CONTAINER_NAME_PREFIX=cnode-
+declare -r APP_HOSTNAME_PREFIX="$APP_CONTAINER_NAME_PREFIX"
+declare -r DEFAULT_NUM_APP_CONTAINERS=20 # Statically allocated number of nodes.
+declare -r TENANT_NAME=localtenant
+
+# May be 'vm' if docker hosts runs within a VM (osx). Default is native/Fedora.
+declare -r NETWORK_TYPE="${NETWORK_TYPE:-local}"
+
+# Allowed program opions
+declare OPTION_NUM_NODES # Set from --num-nodes or DEFAULT_NUM_APP_CONTAINERS, see Main.
+declare OPTION_WAIT # Set from --wait or true, see Main.
+declare OPTION_HV_REGION # Set from --hv-region or DEFAULT_HOSTED_VESPA_REGION, see Main.
+declare OPTION_HV_ENV # Set from --hv-env or DEFAULT_HOSTED_VESPA_ENVIRONMENT, see Main.
+
+declare NUM_APP_CONTAINERS # Set from OPTION_NUM_NODES or DEFAULT_NUM_APP_CONTAINERS, see Main.
+
+function Fail {
+ printf "%s\n" "$@" >&2
+ exit 1
+}
+
+# Used to help scripts with implementing the Usage function. The intended usage
+# is:
+#
+# function Usage {
+# UsageHelper "$@" <<EOF
+# Usage: $SCRIPT_NAME ...
+# ...
+# EOF
+# }
+#
+# When Usage is called, any arguments passed will be printed to stderr, then
+# the usage-string will be printed (on stdin for UsageHelper), then the process
+# will exit with code 1.
+function UsageHelper {
+ exec >&2
+
+ if (($# > 0))
+ then
+ printf "%s\n\n" "$*"
+ fi
+
+ # Print to stdout (which has been redirected to stderr) what's on
+ # stdin. This will print the usage-string.
+ cat
+
+ exit 1
+}
+
+# See Main
+function Restart {
+ Stop
+ Start "$@"
+}
+
+# Use Main as follows:
+#
+# Pass all script arguments to Main:
+#
+# Main "$@"
+#
+# Main will parse the arguments as follows. It assumes the arguments have
+# the following form:
+#
+# script.sh <command> [<arg> | <option>]...
+#
+# where <command> is one of start, stop, or restart:
+# start: The script MUST define a Start function.
+# stop: The script MUST define a Stop function.
+# restart: common.sh defines a Restart function to mean Stop, then Start.
+#
+# <arg> cannot start with a dash, and will get passed as argument to the Start
+# function (if applicable).
+#
+# <option> is either of the form --<name>=<value> or --<name> <value>.
+# <name>/<value> denotes a set of options. For each option, it sets one of the
+# predefined global OPTION_* options.
+#
+# Having parsed the arguments, Main then calls Start, Restart, or Stop,
+# depending on the command. These functions must be defined by the script.
+#
+# A function Usage must also be defined, which will be called when there's a
+# usage error.
+function Main {
+ # Default command is start
+ local command=start
+ if (($# > 0)) && ! [[ "$1" =~ ^- ]]
+ then
+ command="$1"
+ shift
+ fi
+
+ local -a args=()
+
+ while (($# > 0))
+ do
+ if [[ "$1" =~ ^--([a-z0-9][a-z0-9-]*)(=(.*))?$ ]]
+ then
+ # Option argument
+ local name="${BASH_REMATCH[1]}"
+ shift
+
+ if ((${#BASH_REMATCH[2]} > 0))
+ then
+ local value="${BASH_REMATCH[3]}"
+ else
+ if (($# == 0))
+ then
+ Usage "Option '$name' missing value"
+ fi
+
+ value="$1"
+ shift
+ fi
+
+ case "$name" in
+ num-nodes) OPTION_NUM_NODES="$value" ;;
+ wait) OPTION_WAIT="$value" ;;
+ hv-region) OPTION_HV_REGION="$value" ;;
+ hv-env) OPTION_HV_ENV="$value" ;;
+ esac
+ elif [[ "$1" =~ ^[^-] ]]
+ then
+ # Non-option argument
+ args+=("$1")
+ shift
+ else
+ Usage "Bad argument '$1'"
+ fi
+ done
+
+ NUM_APP_CONTAINERS="${OPTION_NUM_NODES:-$DEFAULT_NUM_APP_CONTAINERS}"
+
+ case "$command" in
+ help) Usage ;;
+ stop) Stop ;;
+ start) Start "${args[@]}" ;;
+ restart) Restart "${args[@]}" ;;
+ *) Usage "Unknown command '$command'"
+ esac
+}
diff --git a/node-admin/scripts/config-server.sh b/node-admin/scripts/config-server.sh
new file mode 100755
index 00000000000..60b05d4b3cd
--- /dev/null
+++ b/node-admin/scripts/config-server.sh
@@ -0,0 +1,140 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -e
+
+source "${0%/*}/common.sh"
+
+declare CONTAINER_ROOT_DIR="$APPLICATION_STORAGE_ROOT/$CONFIG_SERVER_CONTAINER_NAME"
+
+function Usage {
+ UsageHelper "$@" <<EOF
+Usage: $SCRIPT_NAME <command> [--wait]
+Manage the Config Server
+
+Commands:
+ start Start the Config Server in a Docker container
+ stop Remove the Config Server container
+ restart Stop, then start
+
+Options:
+ --hv-env <env>
+ Start the config server with the given hosted Vespa environment
+ name. Must be one of prod, dev, test, staging, etc. Default is
+ $DEFAULT_HOSTED_VESPA_ENVIRONMENT.
+ --hv-region <region>
+ Start the config server with the given hosted Vespa region name.
+ Default is $DEFAULT_HOSTED_VESPA_REGION.
+ --wait true
+ Make start wait until the Config Server is healthy
+EOF
+}
+
+function Stop {
+ # Prime sudo
+ sudo true
+
+ echo -n "Removing $CONFIG_SERVER_CONTAINER_NAME container... "
+ docker rm -f "$CONFIG_SERVER_CONTAINER_NAME" &>/dev/null || true
+ echo done
+
+ if [ -d "$CONTAINER_ROOT_DIR" ]
+ then
+ # Double-check we're not 'rm -rf' something unexpected!
+ if ! [[ "$CONTAINER_ROOT_DIR" =~ ^/home/docker/container-storage/ ]]
+ then
+ Fail "DANGEROUS: Almost removed '$CONTAINER_ROOT_DIR'..."
+ fi
+
+ echo -n "Removing container dir $CONTAINER_ROOT_DIR... "
+ sudo rm -rf "$CONTAINER_ROOT_DIR"
+ # The next two statements will prune empty parent directories.
+ sudo mkdir "$CONTAINER_ROOT_DIR"
+ sudo rmdir --ignore-fail-on-non-empty -p "$CONTAINER_ROOT_DIR"
+ echo done
+ fi
+}
+
+function Start {
+ # Prime sudo
+ sudo true
+
+ local wait="${OPTION_WAIT:-true}"
+ case "$wait" in
+ true|false) : ;;
+ *) Usage "--wait should only be set to true or false" ;;
+ esac
+
+ local region="${OPTION_HV_REGION:-$DEFAULT_HOSTED_VESPA_REGION}"
+ local environment="${OPTION_HV_ENV:-$DEFAULT_HOSTED_VESPA_ENVIRONMENT}"
+
+ echo -n "Creating container dir $CONTAINER_ROOT_DIR... "
+ local shared_dir_on_localhost="$APPLICATION_STORAGE_ROOT/$CONFIG_SERVER_CONTAINER_NAME/$ROOT_DIR_SHARED_WITH_HOST"
+ sudo mkdir -p "$shared_dir_on_localhost"
+ sudo chmod a+wt "$shared_dir_on_localhost"
+ echo done
+
+ # Start config server
+ echo -n "Making $CONFIG_SERVER_CONTAINER_NAME container... "
+ local config_server_container_id
+ config_server_container_id=$(\
+ docker run \
+ --detach \
+ --cap-add=NET_ADMIN \
+ --net=none \
+ --hostname "$CONFIG_SERVER_HOSTNAME" \
+ --name "$CONFIG_SERVER_CONTAINER_NAME" \
+ --volume "/etc/hosts:/etc/hosts" \
+ --volume "$shared_dir_on_localhost:/$ROOT_DIR_SHARED_WITH_HOST" \
+ --env "HOSTED_VESPA_REGION=$region" \
+ --env "HOSTED_VESPA_ENVIRONMENT=$environment" \
+ --env "CONFIG_SERVER_HOSTNAME=$CONFIG_SERVER_HOSTNAME" \
+ --env "HOST_BRIDGE_IP=$HOST_BRIDGE_IP" \
+ --entrypoint /usr/local/bin/start-config-server.sh \
+ "$DOCKER_IMAGE")
+ echo done
+
+ echo -n "Verifying that $CONFIG_SERVER_CONTAINER_NAME container is running... "
+ local config_server_container_pid
+ config_server_container_pid=$(docker inspect -f '{{.State.Pid}}' "$CONFIG_SERVER_CONTAINER_NAME")
+
+ echo -n "(pid $config_server_container_pid) "
+
+ # TODO: Use .State.Status instead (only supported from version 1.9).
+ local config_server_container_running
+ config_server_container_running=$(docker inspect -f '{{.State.Running}}' "$CONFIG_SERVER_CONTAINER_NAME")
+
+ if [ "$config_server_container_pid" == 0 -o "$config_server_container_running" != true ]
+ then
+ echo "failed"
+ Fail "The Config Server is not running anymore, consider looking" \
+ "at the logs with 'docker logs $CONFIG_SERVER_CONTAINER_NAME'"
+ fi
+ echo "done"
+
+ echo -n "Setting up the $CONFIG_SERVER_CONTAINER_NAME container network of type $NETWORK_TYPE... "
+ if ! script_out=$(sudo ./configure-container-networking.py --"$NETWORK_TYPE" "$config_server_container_pid" "$CONFIG_SERVER_IP" 2>&1); then
+ echo "failed"
+ echo "$script_out"
+ exit
+ fi
+ echo "done"
+
+ if [ "$wait" == true ]
+ then
+ # Wait for config server to come up
+ echo -n "Waiting for healthy Config Server (~30s)"
+ local url="http://$CONFIG_SERVER_HOSTNAME:19071/state/v1/health"
+ while ! curl --silent --fail --max-time 1 "$url" >/dev/null
+ do
+ echo -n .
+ sleep 2
+ done
+ echo " done"
+ fi
+}
+
+# Makes it easier to access scripts in the same 'scripts' directory
+cd "$SCRIPT_DIR"
+
+Main "$@"
diff --git a/node-admin/scripts/configure-container-networking.py b/node-admin/scripts/configure-container-networking.py
new file mode 100755
index 00000000000..29ff6aa46ba
--- /dev/null
+++ b/node-admin/scripts/configure-container-networking.py
@@ -0,0 +1,311 @@
+#!/usr/bin/env python
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# Quick and dirty script to set up routable ip address for docker container
+# Remove when docker releases a plugin api to configure networking.
+# TODO: Refactor for readability
+
+
+from __future__ import print_function
+
+
+import hashlib
+import ipaddress
+import os
+import sys
+
+
+from pyroute2 import IPRoute
+from pyroute2 import NetNS
+from pyroute2.netlink import NetlinkError
+from socket import AF_INET
+from socket import gethostname
+
+
+def create_directory_ignore_exists(path, permissions):
+ if not os.path.isdir(path):
+ os.mkdir(path, permissions)
+
+
+def create_symlink_ignore_exists(path_to_point_to, symlink_location):
+ if not os.path.islink(symlink_location):
+ os.symlink(path_to_point_to, symlink_location)
+
+
+def get_attribute(struct_with_attrs, name):
+ try:
+ matching_values = [attribute[1] for attribute in struct_with_attrs['attrs'] if attribute[0] == name]
+ if len(matching_values) == 0:
+ raise RuntimeError("No such attribute: %s" % name)
+ elif len(matching_values) != 1:
+ raise RuntimeError("Multiple values for attribute %s" %name)
+ else:
+ return matching_values[0]
+ except Exception as e:
+ raise RuntimeError("Couldn't find attribute %s for value: %s" % (name, struct_with_attrs), e)
+
+def network(ipv4_address):
+ ip = ipaddress.ip_network(unicode(get_attribute(ipv4_address, 'IFA_ADDRESS')))
+ prefix = int(ipv4_address['prefixlen'])
+ return ip.supernet(new_prefix = prefix)
+
+def net_namespace_path(pid):
+ return "/host/proc/%d/ns/net" % pid
+
+def generate_mac_address(base_host_name, ip_address):
+ hash = hashlib.sha1()
+ hash.update(base_host_name)
+ hash.update(ip_address)
+ digest = hash.digest()
+ # For a mac address, we only need six bytes.
+ six_byte_digest = digest[:6]
+ mac_address_bytes = bytearray(six_byte_digest)
+
+ # Set 'unicast'
+ mac_address_bytes[0] &= 0b11111110
+
+ # Set 'local'
+ mac_address_bytes[0] |= 0b00000010
+
+ mac_address = ':'.join('%02x' % n for n in mac_address_bytes)
+ return mac_address
+
+
+flag_local_mode = "--local"
+local_mode = flag_local_mode in sys.argv
+if local_mode:
+ sys.argv.remove(flag_local_mode)
+
+flag_vm_mode = "--vm"
+vm_mode = flag_vm_mode in sys.argv
+if vm_mode:
+ sys.argv.remove(flag_vm_mode)
+
+if local_mode and vm_mode:
+ raise RuntimeError("Cannot specify both --local and --vm")
+
+if len(sys.argv) != 3:
+ raise RuntimeError("Usage: %s <container-pid> <ip>" % sys.argv[0])
+
+container_pid_arg = sys.argv[1]
+container_ip_arg = sys.argv[2]
+
+host_ns_name = "docker-host"
+try:
+ container_pid = int(container_pid_arg)
+except ValueError:
+ raise RuntimeError("Container pid must be an integer, got %s" % container_pid_arg)
+
+container_net_ns_path = net_namespace_path(container_pid)
+if not os.path.isfile(container_net_ns_path):
+ raise RuntimeError("No such net namespace %s" % container_net_ns_path )
+
+container_ip = ipaddress.ip_address(unicode(container_ip_arg))
+
+create_directory_ignore_exists("/var/run/netns", 0766)
+create_symlink_ignore_exists(net_namespace_path(1), "/var/run/netns/%s" % host_ns_name)
+create_symlink_ignore_exists(container_net_ns_path, "/var/run/netns/%d" % container_pid)
+
+host_ns = NetNS(host_ns_name)
+container_ns = NetNS(str(container_pid))
+
+# ipv4 address format: {
+# 'index': 3,
+# 'family': 2,
+# 'header': {
+# 'pid': 15,
+# 'length': 88,
+# 'flags': 2,
+# 'error': None,
+# 'type': 20,
+# 'sequence_number': 256
+# },
+# 'flags': 128,
+# 'attrs': [
+# ['IFA_ADDRESS', '10.0.2.15'],
+# ['IFA_LOCAL', '10.0.2.15'],
+# ['IFA_BROADCAST', '10.0.2.255'],
+# ['IFA_LABEL', 'eth0'],
+# ['IFA_FLAGS', 128],
+# [
+# 'IFA_CACHEINFO',
+# {
+# 'ifa_valid': 4294967295,
+# 'tstamp': 2448,
+# 'cstamp': 2448,
+# 'ifa_prefered': 4294967295
+# }
+# ]
+# ],
+# 'prefixlen': 24,
+# 'scope': 0,
+# 'event': 'RTM_NEWADDR'
+# }
+# Note: This only fetches ipv4 addresses
+host_ips = host_ns.get_addr(family=AF_INET)
+
+host_ips_with_network_matching_container_ip = [host_ip for host_ip in host_ips if container_ip in network(host_ip)]
+
+host_ip_best_match_for_container = None
+for host_ip in host_ips_with_network_matching_container_ip:
+ if not host_ip_best_match_for_container:
+ host_ip_best_match_for_container = host_ip
+ elif host_ip['prefixlen'] < host_ip_best_match_for_container['prefixlen']:
+ host_ip_best_match_for_container = host_ip
+
+if not host_ip_best_match_for_container:
+ raise RuntimeError("No matching ip address for %s, candidates are on networks %s" % (container_ip, ', '.join([str(network(host_ip)) for host_ip in host_ips])))
+
+host_device_index_for_container = host_ip_best_match_for_container['index']
+container_network_prefix_length = host_ip_best_match_for_container['prefixlen']
+
+ipr = IPRoute()
+
+
+# Create new interface for the container.
+
+# The interface to the vespa network are all named "vespa". However, the
+# container interfaces are prepared in the host network namespace, and so it
+# needs a temporary name to avoid name-clash.
+temporary_host_interface_name = "vespa-tmp-" + container_pid_arg
+assert len(temporary_host_interface_name) <= 15 # linux requirement
+
+container_interface_name = "vespa"
+assert len(container_interface_name) <= 15 # linux requirement
+
+for interface_index in ipr.link_lookup(ifname=temporary_host_interface_name):
+ ipr.link('delete', index=interface_index)
+
+if not container_ns.link_lookup(ifname=container_interface_name):
+
+ mac_address = generate_mac_address(
+ gethostname(),
+ container_ip_arg)
+
+ # For traceability.
+ with open('/tmp/container_mac_address_' + container_ip_arg, 'w') as f:
+ f.write(mac_address)
+
+ # Must be created in the host_ns to have the same lifetime as the host.
+ # Otherwise, it will be deleted when the node-admin container stops.
+ # (Only temporarily there, moved to the container namespace later.)
+ # result = [{
+ # 'header': {
+ # 'pid': 240,
+ # 'length': 36,
+ # 'flags': 0,
+ # 'error': None,
+ # 'type': 2,
+ # 'sequence_number': 256
+ # },
+ # 'event': 'NLMSG_ERROR'
+ # }]
+ #
+ # TODO: Here we're linking against the most_specific_address device. For
+ # the sake of argument, as of 2015-12-17, this device is always named
+ # 'vespa'. 'vespa' is itself a macvlan bridge linked to the default route's
+ # interface (typically eth0 or em1). So could we link against eth0 or em1
+ # (or whatever) instead here? What's the difference?
+ result = host_ns.link_create(
+ ifname=temporary_host_interface_name,
+ kind='macvlan',
+ link=host_device_index_for_container,
+ macvlan_mode='bridge',
+ address=mac_address)
+ if result[0]['header']['error']:
+ raise RuntimeError("Failed creating link, result = %s" % result )
+
+ interface_index = host_ns.link_lookup(ifname=temporary_host_interface_name)[0]
+
+ # Move interface from host namespace to container namespace, and change name from temporary name.
+ # exploit that node_admin docker container shares net namespace with host:
+ ipr.link('set', index=interface_index, net_ns_fd=str(container_pid),
+ ifname=container_interface_name)
+
+
+# Find index of interface now in container namespace.
+
+container_interface_index_list = container_ns.link_lookup(ifname=container_interface_name)
+if not container_interface_index_list:
+ raise RuntimeError("Concurrent modification to network interfaces in container")
+
+assert len(container_interface_index_list) == 1
+container_interface_index = container_interface_index_list[0]
+
+
+# Set ip address on interface in container namespace.
+
+ip_already_configured = False
+
+for host_ip in container_ns.get_addr(index=container_interface_index, family = AF_INET):
+ if ipaddress.ip_address(unicode(get_attribute(host_ip, 'IFA_ADDRESS'))) == container_ip and host_ip['prefixlen'] == container_network_prefix_length:
+ ip_already_configured = True
+ else:
+ print("Deleting old ip address. %s/%s" % (get_attribute(host_ip, 'IFA_ADDRESS'), host_ip['prefixlen']))
+ print(container_ns.addr('remove',
+ index=container_interface_index,
+ address=get_attribute(host_ip, 'IFA_ADDRESS'),
+ mask=host_ip['prefixlen']))
+
+if not ip_already_configured:
+ try:
+ container_ns.addr('add',
+ index=container_interface_index,
+ address=str(container_ip),
+ #broadcast='192.168.59.255',
+ mask=container_network_prefix_length)
+ except NetlinkError as e:
+ if e.code == 17: # File exists, i.e. address is already added
+ pass
+
+
+# Activate container interface.
+
+container_ns.link('set', index=container_interface_index, state='up', name=container_interface_name)
+
+
+if local_mode:
+ pass
+elif vm_mode:
+ # Set the default route to the IP of the host vespa interface (e.g. osx)
+ container_ns.route("add", gateway=get_attribute(host_ip_best_match_for_container, 'IFA_ADDRESS'))
+else:
+ # Set up default route/gateway in container.
+
+ # route format: {
+ # 'family': 2,
+ # 'dst_len': 0,
+ # 'proto': 3,
+ # 'tos': 0,
+ # 'event': 'RTM_NEWROUTE',
+ # 'header': {
+ # 'pid': 43,
+ # 'length': 52,
+ # 'flags': 2,
+ # 'error': None,
+ # 'type': 24,
+ # 'sequence_number': 255
+ # },
+ # 'flags': 0,
+ # 'attrs': [
+ # ['RTA_TABLE', 254],
+ # ['RTA_GATEWAY', '172.17.42.1'],
+ # ['RTA_OIF', 18]
+ # ],
+ # 'table': 254,
+ # 'src_len': 0,
+ # 'type': 1,
+ # 'scope': 0
+ # }
+ host_default_routes = host_ns.get_default_routes(family = AF_INET)
+ if len(host_default_routes) != 1:
+ raise RuntimeError("Couldn't find default route: " + str(host_default_routes))
+ default_route = host_default_routes[0]
+
+ host_default_route_device_index = get_attribute(default_route, 'RTA_OIF')
+ host_default_gateway = get_attribute(default_route, 'RTA_GATEWAY')
+ if host_device_index_for_container != host_default_route_device_index:
+ raise RuntimeError("Container's ip address is not on the same network as the host's default route."
+ " Could not set up default route for the container.")
+ container_gateway = host_default_gateway
+ container_ns.route("replace", gateway=container_gateway, index=container_interface_index)
diff --git a/node-admin/scripts/etc-hosts.sh b/node-admin/scripts/etc-hosts.sh
new file mode 100755
index 00000000000..a588d3cc4e6
--- /dev/null
+++ b/node-admin/scripts/etc-hosts.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -e
+
+source "${0%/*}/common.sh"
+
+declare -r HOSTS_FILE=/etc/hosts
+declare -r HOSTS_LINE_SUFFIX=" # Managed by etc-hosts.sh"
+
+function Usage {
+ UsageHelper "$@" <<EOF
+Usage: $SCRIPT_NAME <command> [--num-nodes <num-nodes>]
+Manage Docker container DNS<->IP resolution ($HOSTS_FILE).
+
+Commands:
+ start Add Docker containers to $HOSTS_FILE
+ stop Remove Docker containers from $HOSTS_FILE (not implemented)
+ restart Stop, then start
+
+Options:
+ --num-nodes <num-nodes>
+ Add <num-nodes> hosts instead of the default $DEFAULT_NUM_APP_CONTAINERS.
+EOF
+}
+
+function IsInHostsAlready {
+ local ip="$1"
+ local hostname="$2"
+ local file="$3"
+
+ # TODO: Escape $ip to make sure it's matched as a literal in the regex.
+ local matching_ip_line
+ matching_ip_line=$(grep -E "^$ip[ \\t]" "$file")
+
+ local -i num_ip_lines=0
+ # This 'if' is needed because wc -l <<< "" is 1.
+ if [ -n "$matching_ip_line" ]
+ then
+ num_ip_lines=$(wc -l <<< "$matching_ip_line")
+ fi
+
+ local matching_hostname_line
+ matching_hostname_line=$(grep -E "^[^#]*[ \\t]$hostname(\$|[ \\t])" "$file")
+
+ local -i num_hostname_lines=0
+ # This 'if' is needed because wc -l <<< "" is 1.
+ if [ -n "$matching_hostname_line" ]
+ then
+ num_hostname_lines=$(wc -l <<< "$matching_hostname_line")
+ fi
+
+ if ((num_ip_lines == 1)) && ((num_hostname_lines == 1)) &&
+ [ "$matching_ip_line" == "$matching_hostname_line" ]
+ then
+ return 0
+ elif ((num_ip_lines == 0)) && ((num_hostname_lines == 0))
+ then
+ return 1
+ else
+ Fail "$file contains a conflicting host specification for $hostname/$ip"
+ fi
+}
+
+function AddHost {
+ local ip="$1"
+ local hostname="$2"
+ local file="$3"
+
+ if IsInHostsAlready "$ip" "$hostname" "$file"
+ then
+ return
+ fi
+
+ echo -n "Adding host $hostname ($ip) to $file... "
+ printf "%-11s %s%s\n" "$ip" "$hostname" "$HOSTS_LINE_SUFFIX" >> "$file"
+ echo done
+}
+
+function Stop {
+ # TODO: Remove entries.
+ :
+}
+
+function StartAsRoot {
+ if (($# != 0))
+ then
+ Usage
+ fi
+
+ # May need sudo
+ if [ ! -w "$HOSTS_FILE" ]
+ then
+ Fail "$HOSTS_FILE is not writeable (run script with sudo)"
+ fi
+
+ AddHost "$CONFIG_SERVER_IP" "$CONFIG_SERVER_HOSTNAME" "$HOSTS_FILE"
+
+ local -i index=1
+ for ((; index <= NUM_APP_CONTAINERS; ++index))
+ do
+ local ip="$APP_NETWORK_PREFIX.$index"
+ local container_name="$APP_HOSTNAME_PREFIX$index"
+ AddHost "$ip" "$container_name" "$HOSTS_FILE"
+ done
+}
+
+function Start {
+ if [ "$(id -u)" != 0 ]
+ then
+ sudo "$0" "$@"
+ else
+ StartAsRoot "$@"
+ fi
+}
+
+Main "$@"
diff --git a/node-admin/scripts/ipaddress.py b/node-admin/scripts/ipaddress.py
new file mode 100644
index 00000000000..a08eb743285
--- /dev/null
+++ b/node-admin/scripts/ipaddress.py
@@ -0,0 +1,2411 @@
+# Copyright 2007 Google Inc.
+# Licensed to PSF under a Contributor Agreement.
+
+"""A fast, lightweight IPv4/IPv6 manipulation library in Python.
+
+This library is used to create/poke/manipulate IPv4 and IPv6 addresses
+and networks.
+
+"""
+
+from __future__ import unicode_literals
+
+
+import itertools
+import struct
+
+__version__ = '1.0.14'
+
+# Compatibility functions
+_compat_int_types = (int,)
+try:
+ _compat_int_types = (int, long)
+except NameError:
+ pass
+try:
+ _compat_str = unicode
+except NameError:
+ _compat_str = str
+ assert bytes != str
+if b'\0'[0] == 0: # Python 3 semantics
+ def _compat_bytes_to_byte_vals(byt):
+ return byt
+else:
+ def _compat_bytes_to_byte_vals(byt):
+ return [struct.unpack(b'!B', b)[0] for b in byt]
+try:
+ _compat_int_from_byte_vals = int.from_bytes
+except AttributeError:
+ def _compat_int_from_byte_vals(bytvals, endianess):
+ assert endianess == 'big'
+ res = 0
+ for bv in bytvals:
+ assert isinstance(bv, _compat_int_types)
+ res = (res << 8) + bv
+ return res
+
+
+def _compat_to_bytes(intval, length, endianess):
+ assert isinstance(intval, _compat_int_types)
+ assert endianess == 'big'
+ if length == 4:
+ if intval < 0 or intval >= 2 ** 32:
+ raise struct.error("integer out of range for 'I' format code")
+ return struct.pack(b'!I', intval)
+ elif length == 16:
+ if intval < 0 or intval >= 2 ** 128:
+ raise struct.error("integer out of range for 'QQ' format code")
+ return struct.pack(b'!QQ', intval >> 64, intval & 0xffffffffffffffff)
+ else:
+ raise NotImplementedError()
+if hasattr(int, 'bit_length'):
+ # Not int.bit_length , since that won't work in 2.7 where long exists
+ def _compat_bit_length(i):
+ return i.bit_length()
+else:
+ def _compat_bit_length(i):
+ for res in itertools.count():
+ if i >> res == 0:
+ return res
+
+
+def _compat_range(start, end, step=1):
+ assert step > 0
+ i = start
+ while i < end:
+ yield i
+ i += step
+
+
+class _TotalOrderingMixin(object):
+ __slots__ = ()
+
+ # Helper that derives the other comparison operations from
+ # __lt__ and __eq__
+ # We avoid functools.total_ordering because it doesn't handle
+ # NotImplemented correctly yet (http://bugs.python.org/issue10042)
+ def __eq__(self, other):
+ raise NotImplementedError
+
+ def __ne__(self, other):
+ equal = self.__eq__(other)
+ if equal is NotImplemented:
+ return NotImplemented
+ return not equal
+
+ def __lt__(self, other):
+ raise NotImplementedError
+
+ def __le__(self, other):
+ less = self.__lt__(other)
+ if less is NotImplemented or not less:
+ return self.__eq__(other)
+ return less
+
+ def __gt__(self, other):
+ less = self.__lt__(other)
+ if less is NotImplemented:
+ return NotImplemented
+ equal = self.__eq__(other)
+ if equal is NotImplemented:
+ return NotImplemented
+ return not (less or equal)
+
+ def __ge__(self, other):
+ less = self.__lt__(other)
+ if less is NotImplemented:
+ return NotImplemented
+ return not less
+
+
+IPV4LENGTH = 32
+IPV6LENGTH = 128
+
+
+class AddressValueError(ValueError):
+ """A Value Error related to the address."""
+
+
+class NetmaskValueError(ValueError):
+ """A Value Error related to the netmask."""
+
+
+def ip_address(address):
+ """Take an IP string/int and return an object of the correct type.
+
+ Args:
+ address: A string or integer, the IP address. Either IPv4 or
+ IPv6 addresses may be supplied; integers less than 2**32 will
+ be considered to be IPv4 by default.
+
+ Returns:
+ An IPv4Address or IPv6Address object.
+
+ Raises:
+ ValueError: if the *address* passed isn't either a v4 or a v6
+ address
+
+ """
+ try:
+ return IPv4Address(address)
+ except (AddressValueError, NetmaskValueError):
+ pass
+
+ try:
+ return IPv6Address(address)
+ except (AddressValueError, NetmaskValueError):
+ pass
+
+ if isinstance(address, bytes):
+ raise AddressValueError(
+ '%r does not appear to be an IPv4 or IPv6 address. '
+ 'Did you pass in a bytes (str in Python 2) instead of'
+ ' a unicode object?' % address)
+
+ raise ValueError('%r does not appear to be an IPv4 or IPv6 address' %
+ address)
+
+
+def ip_network(address, strict=True):
+ """Take an IP string/int and return an object of the correct type.
+
+ Args:
+ address: A string or integer, the IP network. Either IPv4 or
+ IPv6 networks may be supplied; integers less than 2**32 will
+ be considered to be IPv4 by default.
+
+ Returns:
+ An IPv4Network or IPv6Network object.
+
+ Raises:
+ ValueError: if the string passed isn't either a v4 or a v6
+ address. Or if the network has host bits set.
+
+ """
+ try:
+ return IPv4Network(address, strict)
+ except (AddressValueError, NetmaskValueError):
+ pass
+
+ try:
+ return IPv6Network(address, strict)
+ except (AddressValueError, NetmaskValueError):
+ pass
+
+ raise ValueError('%r does not appear to be an IPv4 or IPv6 network' %
+ address)
+
+
+def ip_interface(address):
+ """Take an IP string/int and return an object of the correct type.
+
+ Args:
+ address: A string or integer, the IP address. Either IPv4 or
+ IPv6 addresses may be supplied; integers less than 2**32 will
+ be considered to be IPv4 by default.
+
+ Returns:
+ An IPv4Interface or IPv6Interface object.
+
+ Raises:
+ ValueError: if the string passed isn't either a v4 or a v6
+ address.
+
+ Notes:
+ The IPv?Interface classes describe an Address on a particular
+ Network, so they're basically a combination of both the Address
+ and Network classes.
+
+ """
+ try:
+ return IPv4Interface(address)
+ except (AddressValueError, NetmaskValueError):
+ pass
+
+ try:
+ return IPv6Interface(address)
+ except (AddressValueError, NetmaskValueError):
+ pass
+
+ raise ValueError('%r does not appear to be an IPv4 or IPv6 interface' %
+ address)
+
+
+def v4_int_to_packed(address):
+ """Represent an address as 4 packed bytes in network (big-endian) order.
+
+ Args:
+ address: An integer representation of an IPv4 IP address.
+
+ Returns:
+ The integer address packed as 4 bytes in network (big-endian) order.
+
+ Raises:
+ ValueError: If the integer is negative or too large to be an
+ IPv4 IP address.
+
+ """
+ try:
+ return _compat_to_bytes(address, 4, 'big')
+ except (struct.error, OverflowError):
+ raise ValueError("Address negative or too large for IPv4")
+
+
+def v6_int_to_packed(address):
+ """Represent an address as 16 packed bytes in network (big-endian) order.
+
+ Args:
+ address: An integer representation of an IPv6 IP address.
+
+ Returns:
+ The integer address packed as 16 bytes in network (big-endian) order.
+
+ """
+ try:
+ return _compat_to_bytes(address, 16, 'big')
+ except (struct.error, OverflowError):
+ raise ValueError("Address negative or too large for IPv6")
+
+
+def _split_optional_netmask(address):
+ """Helper to split the netmask and raise AddressValueError if needed"""
+ addr = _compat_str(address).split('/')
+ if len(addr) > 2:
+ raise AddressValueError("Only one '/' permitted in %r" % address)
+ return addr
+
+
+def _find_address_range(addresses):
+ """Find a sequence of sorted deduplicated IPv#Address.
+
+ Args:
+ addresses: a list of IPv#Address objects.
+
+ Yields:
+ A tuple containing the first and last IP addresses in the sequence.
+
+ """
+ it = iter(addresses)
+ first = last = next(it)
+ for ip in it:
+ if ip._ip != last._ip + 1:
+ yield first, last
+ first = ip
+ last = ip
+ yield first, last
+
+
+def _count_righthand_zero_bits(number, bits):
+ """Count the number of zero bits on the right hand side.
+
+ Args:
+ number: an integer.
+ bits: maximum number of bits to count.
+
+ Returns:
+ The number of zero bits on the right hand side of the number.
+
+ """
+ if number == 0:
+ return bits
+ return min(bits, _compat_bit_length(~number & (number - 1)))
+
+
+def summarize_address_range(first, last):
+ """Summarize a network range given the first and last IP addresses.
+
+ Example:
+ >>> list(summarize_address_range(IPv4Address('192.0.2.0'),
+ ... IPv4Address('192.0.2.130')))
+ ... #doctest: +NORMALIZE_WHITESPACE
+ [IPv4Network('192.0.2.0/25'), IPv4Network('192.0.2.128/31'),
+ IPv4Network('192.0.2.130/32')]
+
+ Args:
+ first: the first IPv4Address or IPv6Address in the range.
+ last: the last IPv4Address or IPv6Address in the range.
+
+ Returns:
+ An iterator of the summarized IPv(4|6) network objects.
+
+ Raise:
+ TypeError:
+ If the first and last objects are not IP addresses.
+ If the first and last objects are not the same version.
+ ValueError:
+ If the last object is not greater than the first.
+ If the version of the first address is not 4 or 6.
+
+ """
+ if (not (isinstance(first, _BaseAddress) and
+ isinstance(last, _BaseAddress))):
+ raise TypeError('first and last must be IP addresses, not networks')
+ if first.version != last.version:
+ raise TypeError("%s and %s are not of the same version" % (
+ first, last))
+ if first > last:
+ raise ValueError('last IP address must be greater than first')
+
+ if first.version == 4:
+ ip = IPv4Network
+ elif first.version == 6:
+ ip = IPv6Network
+ else:
+ raise ValueError('unknown IP version')
+
+ ip_bits = first._max_prefixlen
+ first_int = first._ip
+ last_int = last._ip
+ while first_int <= last_int:
+ nbits = min(_count_righthand_zero_bits(first_int, ip_bits),
+ _compat_bit_length(last_int - first_int + 1) - 1)
+ net = ip((first_int, ip_bits - nbits))
+ yield net
+ first_int += 1 << nbits
+ if first_int - 1 == ip._ALL_ONES:
+ break
+
+
+def _collapse_addresses_internal(addresses):
+ """Loops through the addresses, collapsing concurrent netblocks.
+
+ Example:
+
+ ip1 = IPv4Network('192.0.2.0/26')
+ ip2 = IPv4Network('192.0.2.64/26')
+ ip3 = IPv4Network('192.0.2.128/26')
+ ip4 = IPv4Network('192.0.2.192/26')
+
+ _collapse_addresses_internal([ip1, ip2, ip3, ip4]) ->
+ [IPv4Network('192.0.2.0/24')]
+
+ This shouldn't be called directly; it is called via
+ collapse_addresses([]).
+
+ Args:
+ addresses: A list of IPv4Network's or IPv6Network's
+
+ Returns:
+ A list of IPv4Network's or IPv6Network's depending on what we were
+ passed.
+
+ """
+ # First merge
+ to_merge = list(addresses)
+ subnets = {}
+ while to_merge:
+ net = to_merge.pop()
+ supernet = net.supernet()
+ existing = subnets.get(supernet)
+ if existing is None:
+ subnets[supernet] = net
+ elif existing != net:
+ # Merge consecutive subnets
+ del subnets[supernet]
+ to_merge.append(supernet)
+ # Then iterate over resulting networks, skipping subsumed subnets
+ last = None
+ for net in sorted(subnets.values()):
+ if last is not None:
+ # Since they are sorted,
+ # last.network_address <= net.network_address is a given.
+ if last.broadcast_address >= net.broadcast_address:
+ continue
+ yield net
+ last = net
+
+
+def collapse_addresses(addresses):
+ """Collapse a list of IP objects.
+
+ Example:
+ collapse_addresses([IPv4Network('192.0.2.0/25'),
+ IPv4Network('192.0.2.128/25')]) ->
+ [IPv4Network('192.0.2.0/24')]
+
+ Args:
+ addresses: An iterator of IPv4Network or IPv6Network objects.
+
+ Returns:
+ An iterator of the collapsed IPv(4|6)Network objects.
+
+ Raises:
+ TypeError: If passed a list of mixed version objects.
+
+ """
+ addrs = []
+ ips = []
+ nets = []
+
+ # split IP addresses and networks
+ for ip in addresses:
+ if isinstance(ip, _BaseAddress):
+ if ips and ips[-1]._version != ip._version:
+ raise TypeError("%s and %s are not of the same version" % (
+ ip, ips[-1]))
+ ips.append(ip)
+ elif ip._prefixlen == ip._max_prefixlen:
+ if ips and ips[-1]._version != ip._version:
+ raise TypeError("%s and %s are not of the same version" % (
+ ip, ips[-1]))
+ try:
+ ips.append(ip.ip)
+ except AttributeError:
+ ips.append(ip.network_address)
+ else:
+ if nets and nets[-1]._version != ip._version:
+ raise TypeError("%s and %s are not of the same version" % (
+ ip, nets[-1]))
+ nets.append(ip)
+
+ # sort and dedup
+ ips = sorted(set(ips))
+
+ # find consecutive address ranges in the sorted sequence and summarize them
+ if ips:
+ for first, last in _find_address_range(ips):
+ addrs.extend(summarize_address_range(first, last))
+
+ return _collapse_addresses_internal(addrs + nets)
+
+
+def get_mixed_type_key(obj):
+ """Return a key suitable for sorting between networks and addresses.
+
+ Address and Network objects are not sortable by default; they're
+ fundamentally different so the expression
+
+ IPv4Address('192.0.2.0') <= IPv4Network('192.0.2.0/24')
+
+ doesn't make any sense. There are some times however, where you may wish
+ to have ipaddress sort these for you anyway. If you need to do this, you
+ can use this function as the key= argument to sorted().
+
+ Args:
+ obj: either a Network or Address object.
+ Returns:
+ appropriate key.
+
+ """
+ if isinstance(obj, _BaseNetwork):
+ return obj._get_networks_key()
+ elif isinstance(obj, _BaseAddress):
+ return obj._get_address_key()
+ return NotImplemented
+
+
+class _IPAddressBase(_TotalOrderingMixin):
+
+ """The mother class."""
+
+ __slots__ = ()
+
+ @property
+ def exploded(self):
+ """Return the longhand version of the IP address as a string."""
+ return self._explode_shorthand_ip_string()
+
+ @property
+ def compressed(self):
+ """Return the shorthand version of the IP address as a string."""
+ return _compat_str(self)
+
+ @property
+ def reverse_pointer(self):
+ """The name of the reverse DNS pointer for the IP address, e.g.:
+ >>> ipaddress.ip_address("127.0.0.1").reverse_pointer
+ '1.0.0.127.in-addr.arpa'
+ >>> ipaddress.ip_address("2001:db8::1").reverse_pointer
+ '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa'
+
+ """
+ return self._reverse_pointer()
+
+ @property
+ def version(self):
+ msg = '%200s has no version specified' % (type(self),)
+ raise NotImplementedError(msg)
+
+ def _check_int_address(self, address):
+ if address < 0:
+ msg = "%d (< 0) is not permitted as an IPv%d address"
+ raise AddressValueError(msg % (address, self._version))
+ if address > self._ALL_ONES:
+ msg = "%d (>= 2**%d) is not permitted as an IPv%d address"
+ raise AddressValueError(msg % (address, self._max_prefixlen,
+ self._version))
+
+ def _check_packed_address(self, address, expected_len):
+ address_len = len(address)
+ if address_len != expected_len:
+ msg = (
+ '%r (len %d != %d) is not permitted as an IPv%d address. '
+ 'Did you pass in a bytes (str in Python 2) instead of'
+ ' a unicode object?'
+ )
+ raise AddressValueError(msg % (address, address_len,
+ expected_len, self._version))
+
+ @classmethod
+ def _ip_int_from_prefix(cls, prefixlen):
+ """Turn the prefix length into a bitwise netmask
+
+ Args:
+ prefixlen: An integer, the prefix length.
+
+ Returns:
+ An integer.
+
+ """
+ return cls._ALL_ONES ^ (cls._ALL_ONES >> prefixlen)
+
+ @classmethod
+ def _prefix_from_ip_int(cls, ip_int):
+ """Return prefix length from the bitwise netmask.
+
+ Args:
+ ip_int: An integer, the netmask in expanded bitwise format
+
+ Returns:
+ An integer, the prefix length.
+
+ Raises:
+ ValueError: If the input intermingles zeroes & ones
+ """
+ trailing_zeroes = _count_righthand_zero_bits(ip_int,
+ cls._max_prefixlen)
+ prefixlen = cls._max_prefixlen - trailing_zeroes
+ leading_ones = ip_int >> trailing_zeroes
+ all_ones = (1 << prefixlen) - 1
+ if leading_ones != all_ones:
+ byteslen = cls._max_prefixlen // 8
+ details = _compat_to_bytes(ip_int, byteslen, 'big')
+ msg = 'Netmask pattern %r mixes zeroes & ones'
+ raise ValueError(msg % details)
+ return prefixlen
+
+ @classmethod
+ def _report_invalid_netmask(cls, netmask_str):
+ msg = '%r is not a valid netmask' % netmask_str
+ raise NetmaskValueError(msg)
+
+ @classmethod
+ def _prefix_from_prefix_string(cls, prefixlen_str):
+ """Return prefix length from a numeric string
+
+ Args:
+ prefixlen_str: The string to be converted
+
+ Returns:
+ An integer, the prefix length.
+
+ Raises:
+ NetmaskValueError: If the input is not a valid netmask
+ """
+ # int allows a leading +/- as well as surrounding whitespace,
+ # so we ensure that isn't the case
+ if not _BaseV4._DECIMAL_DIGITS.issuperset(prefixlen_str):
+ cls._report_invalid_netmask(prefixlen_str)
+ try:
+ prefixlen = int(prefixlen_str)
+ except ValueError:
+ cls._report_invalid_netmask(prefixlen_str)
+ if not (0 <= prefixlen <= cls._max_prefixlen):
+ cls._report_invalid_netmask(prefixlen_str)
+ return prefixlen
+
+ @classmethod
+ def _prefix_from_ip_string(cls, ip_str):
+ """Turn a netmask/hostmask string into a prefix length
+
+ Args:
+ ip_str: The netmask/hostmask to be converted
+
+ Returns:
+ An integer, the prefix length.
+
+ Raises:
+ NetmaskValueError: If the input is not a valid netmask/hostmask
+ """
+ # Parse the netmask/hostmask like an IP address.
+ try:
+ ip_int = cls._ip_int_from_string(ip_str)
+ except AddressValueError:
+ cls._report_invalid_netmask(ip_str)
+
+ # Try matching a netmask (this would be /1*0*/ as a bitwise regexp).
+ # Note that the two ambiguous cases (all-ones and all-zeroes) are
+ # treated as netmasks.
+ try:
+ return cls._prefix_from_ip_int(ip_int)
+ except ValueError:
+ pass
+
+ # Invert the bits, and try matching a /0+1+/ hostmask instead.
+ ip_int ^= cls._ALL_ONES
+ try:
+ return cls._prefix_from_ip_int(ip_int)
+ except ValueError:
+ cls._report_invalid_netmask(ip_str)
+
+ def __reduce__(self):
+ return self.__class__, (_compat_str(self),)
+
+
+class _BaseAddress(_IPAddressBase):
+
+ """A generic IP object.
+
+ This IP class contains the version independent methods which are
+ used by single IP addresses.
+ """
+
+ __slots__ = ()
+
+ def __int__(self):
+ return self._ip
+
+ def __eq__(self, other):
+ try:
+ return (self._ip == other._ip and
+ self._version == other._version)
+ except AttributeError:
+ return NotImplemented
+
+ def __lt__(self, other):
+ if not isinstance(other, _IPAddressBase):
+ return NotImplemented
+ if not isinstance(other, _BaseAddress):
+ raise TypeError('%s and %s are not of the same type' % (
+ self, other))
+ if self._version != other._version:
+ raise TypeError('%s and %s are not of the same version' % (
+ self, other))
+ if self._ip != other._ip:
+ return self._ip < other._ip
+ return False
+
+ # Shorthand for Integer addition and subtraction. This is not
+ # meant to ever support addition/subtraction of addresses.
+ def __add__(self, other):
+ if not isinstance(other, _compat_int_types):
+ return NotImplemented
+ return self.__class__(int(self) + other)
+
+ def __sub__(self, other):
+ if not isinstance(other, _compat_int_types):
+ return NotImplemented
+ return self.__class__(int(self) - other)
+
+ def __repr__(self):
+ return '%s(%r)' % (self.__class__.__name__, _compat_str(self))
+
+ def __str__(self):
+ return _compat_str(self._string_from_ip_int(self._ip))
+
+ def __hash__(self):
+ return hash(hex(int(self._ip)))
+
+ def _get_address_key(self):
+ return (self._version, self)
+
+ def __reduce__(self):
+ return self.__class__, (self._ip,)
+
+
+class _BaseNetwork(_IPAddressBase):
+
+ """A generic IP network object.
+
+ This IP class contains the version independent methods which are
+ used by networks.
+
+ """
+ def __init__(self, address):
+ self._cache = {}
+
+ def __repr__(self):
+ return '%s(%r)' % (self.__class__.__name__, _compat_str(self))
+
+ def __str__(self):
+ return '%s/%d' % (self.network_address, self.prefixlen)
+
+ def hosts(self):
+ """Generate Iterator over usable hosts in a network.
+
+ This is like __iter__ except it doesn't return the network
+ or broadcast addresses.
+
+ """
+ network = int(self.network_address)
+ broadcast = int(self.broadcast_address)
+ for x in _compat_range(network + 1, broadcast):
+ yield self._address_class(x)
+
+ def __iter__(self):
+ network = int(self.network_address)
+ broadcast = int(self.broadcast_address)
+ for x in _compat_range(network, broadcast + 1):
+ yield self._address_class(x)
+
+ def __getitem__(self, n):
+ network = int(self.network_address)
+ broadcast = int(self.broadcast_address)
+ if n >= 0:
+ if network + n > broadcast:
+ raise IndexError
+ return self._address_class(network + n)
+ else:
+ n += 1
+ if broadcast + n < network:
+ raise IndexError
+ return self._address_class(broadcast + n)
+
+ def __lt__(self, other):
+ if not isinstance(other, _IPAddressBase):
+ return NotImplemented
+ if not isinstance(other, _BaseNetwork):
+ raise TypeError('%s and %s are not of the same type' % (
+ self, other))
+ if self._version != other._version:
+ raise TypeError('%s and %s are not of the same version' % (
+ self, other))
+ if self.network_address != other.network_address:
+ return self.network_address < other.network_address
+ if self.netmask != other.netmask:
+ return self.netmask < other.netmask
+ return False
+
+ def __eq__(self, other):
+ try:
+ return (self._version == other._version and
+ self.network_address == other.network_address and
+ int(self.netmask) == int(other.netmask))
+ except AttributeError:
+ return NotImplemented
+
+ def __hash__(self):
+ return hash(int(self.network_address) ^ int(self.netmask))
+
+ def __contains__(self, other):
+ # always false if one is v4 and the other is v6.
+ if self._version != other._version:
+ return False
+ # dealing with another network.
+ if isinstance(other, _BaseNetwork):
+ return False
+ # dealing with another address
+ else:
+ # address
+ return (int(self.network_address) <= int(other._ip) <=
+ int(self.broadcast_address))
+
+ def overlaps(self, other):
+ """Tell if self is partly contained in other."""
+ return self.network_address in other or (
+ self.broadcast_address in other or (
+ other.network_address in self or (
+ other.broadcast_address in self)))
+
+ @property
+ def broadcast_address(self):
+ x = self._cache.get('broadcast_address')
+ if x is None:
+ x = self._address_class(int(self.network_address) |
+ int(self.hostmask))
+ self._cache['broadcast_address'] = x
+ return x
+
+ @property
+ def hostmask(self):
+ x = self._cache.get('hostmask')
+ if x is None:
+ x = self._address_class(int(self.netmask) ^ self._ALL_ONES)
+ self._cache['hostmask'] = x
+ return x
+
+ @property
+ def with_prefixlen(self):
+ return '%s/%d' % (self.network_address, self._prefixlen)
+
+ @property
+ def with_netmask(self):
+ return '%s/%s' % (self.network_address, self.netmask)
+
+ @property
+ def with_hostmask(self):
+ return '%s/%s' % (self.network_address, self.hostmask)
+
+ @property
+ def num_addresses(self):
+ """Number of hosts in the current subnet."""
+ return int(self.broadcast_address) - int(self.network_address) + 1
+
+ @property
+ def _address_class(self):
+ # Returning bare address objects (rather than interfaces) allows for
+ # more consistent behaviour across the network address, broadcast
+ # address and individual host addresses.
+ msg = '%200s has no associated address class' % (type(self),)
+ raise NotImplementedError(msg)
+
+ @property
+ def prefixlen(self):
+ return self._prefixlen
+
+ def address_exclude(self, other):
+ """Remove an address from a larger block.
+
+ For example:
+
+ addr1 = ip_network('192.0.2.0/28')
+ addr2 = ip_network('192.0.2.1/32')
+ addr1.address_exclude(addr2) =
+ [IPv4Network('192.0.2.0/32'), IPv4Network('192.0.2.2/31'),
+ IPv4Network('192.0.2.4/30'), IPv4Network('192.0.2.8/29')]
+
+ or IPv6:
+
+ addr1 = ip_network('2001:db8::1/32')
+ addr2 = ip_network('2001:db8::1/128')
+ addr1.address_exclude(addr2) =
+ [ip_network('2001:db8::1/128'),
+ ip_network('2001:db8::2/127'),
+ ip_network('2001:db8::4/126'),
+ ip_network('2001:db8::8/125'),
+ ...
+ ip_network('2001:db8:8000::/33')]
+
+ Args:
+ other: An IPv4Network or IPv6Network object of the same type.
+
+ Returns:
+ An iterator of the IPv(4|6)Network objects which is self
+ minus other.
+
+ Raises:
+ TypeError: If self and other are of differing address
+ versions, or if other is not a network object.
+ ValueError: If other is not completely contained by self.
+
+ """
+ if not self._version == other._version:
+ raise TypeError("%s and %s are not of the same version" % (
+ self, other))
+
+ if not isinstance(other, _BaseNetwork):
+ raise TypeError("%s is not a network object" % other)
+
+ if not other.subnet_of(self):
+ raise ValueError('%s not contained in %s' % (other, self))
+ if other == self:
+ return
+
+ # Make sure we're comparing the network of other.
+ other = other.__class__('%s/%s' % (other.network_address,
+ other.prefixlen))
+
+ s1, s2 = self.subnets()
+ while s1 != other and s2 != other:
+ if other.subnet_of(s1):
+ yield s2
+ s1, s2 = s1.subnets()
+ elif other.subnet_of(s2):
+ yield s1
+ s1, s2 = s2.subnets()
+ else:
+ # If we got here, there's a bug somewhere.
+ raise AssertionError('Error performing exclusion: '
+ 's1: %s s2: %s other: %s' %
+ (s1, s2, other))
+ if s1 == other:
+ yield s2
+ elif s2 == other:
+ yield s1
+ else:
+ # If we got here, there's a bug somewhere.
+ raise AssertionError('Error performing exclusion: '
+ 's1: %s s2: %s other: %s' %
+ (s1, s2, other))
+
+ def compare_networks(self, other):
+ """Compare two IP objects.
+
+ This is only concerned about the comparison of the integer
+ representation of the network addresses. This means that the
+ host bits aren't considered at all in this method. If you want
+ to compare host bits, you can easily enough do a
+ 'HostA._ip < HostB._ip'
+
+ Args:
+ other: An IP object.
+
+ Returns:
+ If the IP versions of self and other are the same, returns:
+
+ -1 if self < other:
+ eg: IPv4Network('192.0.2.0/25') < IPv4Network('192.0.2.128/25')
+ IPv6Network('2001:db8::1000/124') <
+ IPv6Network('2001:db8::2000/124')
+ 0 if self == other
+ eg: IPv4Network('192.0.2.0/24') == IPv4Network('192.0.2.0/24')
+ IPv6Network('2001:db8::1000/124') ==
+ IPv6Network('2001:db8::1000/124')
+ 1 if self > other
+ eg: IPv4Network('192.0.2.128/25') > IPv4Network('192.0.2.0/25')
+ IPv6Network('2001:db8::2000/124') >
+ IPv6Network('2001:db8::1000/124')
+
+ Raises:
+ TypeError if the IP versions are different.
+
+ """
+ # does this need to raise a ValueError?
+ if self._version != other._version:
+ raise TypeError('%s and %s are not of the same type' % (
+ self, other))
+ # self._version == other._version below here:
+ if self.network_address < other.network_address:
+ return -1
+ if self.network_address > other.network_address:
+ return 1
+ # self.network_address == other.network_address below here:
+ if self.netmask < other.netmask:
+ return -1
+ if self.netmask > other.netmask:
+ return 1
+ return 0
+
+ def _get_networks_key(self):
+ """Network-only key function.
+
+ Returns an object that identifies this address' network and
+ netmask. This function is a suitable "key" argument for sorted()
+ and list.sort().
+
+ """
+ return (self._version, self.network_address, self.netmask)
+
+ def subnets(self, prefixlen_diff=1, new_prefix=None):
+ """The subnets which join to make the current subnet.
+
+ In the case that self contains only one IP
+ (self._prefixlen == 32 for IPv4 or self._prefixlen == 128
+ for IPv6), yield an iterator with just ourself.
+
+ Args:
+ prefixlen_diff: An integer, the amount the prefix length
+ should be increased by. This should not be set if
+ new_prefix is also set.
+ new_prefix: The desired new prefix length. This must be a
+ larger number (smaller prefix) than the existing prefix.
+ This should not be set if prefixlen_diff is also set.
+
+ Returns:
+ An iterator of IPv(4|6) objects.
+
+ Raises:
+ ValueError: The prefixlen_diff is too small or too large.
+ OR
+ prefixlen_diff and new_prefix are both set or new_prefix
+ is a smaller number than the current prefix (smaller
+ number means a larger network)
+
+ """
+ if self._prefixlen == self._max_prefixlen:
+ yield self
+ return
+
+ if new_prefix is not None:
+ if new_prefix < self._prefixlen:
+ raise ValueError('new prefix must be longer')
+ if prefixlen_diff != 1:
+ raise ValueError('cannot set prefixlen_diff and new_prefix')
+ prefixlen_diff = new_prefix - self._prefixlen
+
+ if prefixlen_diff < 0:
+ raise ValueError('prefix length diff must be > 0')
+ new_prefixlen = self._prefixlen + prefixlen_diff
+
+ if new_prefixlen > self._max_prefixlen:
+ raise ValueError(
+ 'prefix length diff %d is invalid for netblock %s' % (
+ new_prefixlen, self))
+
+ start = int(self.network_address)
+ end = int(self.broadcast_address)
+ step = (int(self.hostmask) + 1) >> prefixlen_diff
+ for new_addr in _compat_range(start, end, step):
+ current = self.__class__((new_addr, new_prefixlen))
+ yield current
+
+ def supernet(self, prefixlen_diff=1, new_prefix=None):
+ """The supernet containing the current network.
+
+ Args:
+ prefixlen_diff: An integer, the amount the prefix length of
+ the network should be decreased by. For example, given a
+ /24 network and a prefixlen_diff of 3, a supernet with a
+ /21 netmask is returned.
+
+ Returns:
+ An IPv4 network object.
+
+ Raises:
+ ValueError: If self.prefixlen - prefixlen_diff < 0. I.e., you have
+ a negative prefix length.
+ OR
+ If prefixlen_diff and new_prefix are both set or new_prefix is a
+ larger number than the current prefix (larger number means a
+ smaller network)
+
+ """
+ if self._prefixlen == 0:
+ return self
+
+ if new_prefix is not None:
+ if new_prefix > self._prefixlen:
+ raise ValueError('new prefix must be shorter')
+ if prefixlen_diff != 1:
+ raise ValueError('cannot set prefixlen_diff and new_prefix')
+ prefixlen_diff = self._prefixlen - new_prefix
+
+ new_prefixlen = self.prefixlen - prefixlen_diff
+ if new_prefixlen < 0:
+ raise ValueError(
+ 'current prefixlen is %d, cannot have a prefixlen_diff of %d' %
+ (self.prefixlen, prefixlen_diff))
+ return self.__class__((
+ int(self.network_address) & (int(self.netmask) << prefixlen_diff),
+ new_prefixlen
+ ))
+
+ @property
+ def is_multicast(self):
+ """Test if the address is reserved for multicast use.
+
+ Returns:
+ A boolean, True if the address is a multicast address.
+ See RFC 2373 2.7 for details.
+
+ """
+ return (self.network_address.is_multicast and
+ self.broadcast_address.is_multicast)
+
+ def subnet_of(self, other):
+ # always false if one is v4 and the other is v6.
+ if self._version != other._version:
+ return False
+ # dealing with another network.
+ if (hasattr(other, 'network_address') and
+ hasattr(other, 'broadcast_address')):
+ return (other.network_address <= self.network_address and
+ other.broadcast_address >= self.broadcast_address)
+ # dealing with another address
+ else:
+ raise TypeError('Unable to test subnet containment with element '
+ 'of type %s' % type(other))
+
+ def supernet_of(self, other):
+ # always false if one is v4 and the other is v6.
+ if self._version != other._version:
+ return False
+ # dealing with another network.
+ if (hasattr(other, 'network_address') and
+ hasattr(other, 'broadcast_address')):
+ return (other.network_address >= self.network_address and
+ other.broadcast_address <= self.broadcast_address)
+ # dealing with another address
+ else:
+ raise TypeError('Unable to test subnet containment with element '
+ 'of type %s' % type(other))
+
+ @property
+ def is_reserved(self):
+ """Test if the address is otherwise IETF reserved.
+
+ Returns:
+ A boolean, True if the address is within one of the
+ reserved IPv6 Network ranges.
+
+ """
+ return (self.network_address.is_reserved and
+ self.broadcast_address.is_reserved)
+
+ @property
+ def is_link_local(self):
+ """Test if the address is reserved for link-local.
+
+ Returns:
+ A boolean, True if the address is reserved per RFC 4291.
+
+ """
+ return (self.network_address.is_link_local and
+ self.broadcast_address.is_link_local)
+
+ @property
+ def is_private(self):
+ """Test if this address is allocated for private networks.
+
+ Returns:
+ A boolean, True if the address is reserved per
+ iana-ipv4-special-registry or iana-ipv6-special-registry.
+
+ """
+ return (self.network_address.is_private and
+ self.broadcast_address.is_private)
+
+ @property
+ def is_global(self):
+ """Test if this address is allocated for public networks.
+
+ Returns:
+ A boolean, True if the address is not reserved per
+ iana-ipv4-special-registry or iana-ipv6-special-registry.
+
+ """
+ return not self.is_private
+
+ @property
+ def is_unspecified(self):
+ """Test if the address is unspecified.
+
+ Returns:
+ A boolean, True if this is the unspecified address as defined in
+ RFC 2373 2.5.2.
+
+ """
+ return (self.network_address.is_unspecified and
+ self.broadcast_address.is_unspecified)
+
+ @property
+ def is_loopback(self):
+ """Test if the address is a loopback address.
+
+ Returns:
+ A boolean, True if the address is a loopback address as defined in
+ RFC 2373 2.5.3.
+
+ """
+ return (self.network_address.is_loopback and
+ self.broadcast_address.is_loopback)
+
+
+class _BaseV4(object):
+
+ """Base IPv4 object.
+
+ The following methods are used by IPv4 objects in both single IP
+ addresses and networks.
+
+ """
+
+ __slots__ = ()
+ _version = 4
+ # Equivalent to 255.255.255.255 or 32 bits of 1's.
+ _ALL_ONES = (2 ** IPV4LENGTH) - 1
+ _DECIMAL_DIGITS = frozenset('0123456789')
+
+ # the valid octets for host and netmasks. only useful for IPv4.
+ _valid_mask_octets = frozenset([255, 254, 252, 248, 240, 224, 192, 128, 0])
+
+ _max_prefixlen = IPV4LENGTH
+ # There are only a handful of valid v4 netmasks, so we cache them all
+ # when constructed (see _make_netmask()).
+ _netmask_cache = {}
+
+ def _explode_shorthand_ip_string(self):
+ return _compat_str(self)
+
+ @classmethod
+ def _make_netmask(cls, arg):
+ """Make a (netmask, prefix_len) tuple from the given argument.
+
+ Argument can be:
+ - an integer (the prefix length)
+ - a string representing the prefix length (e.g. "24")
+ - a string representing the prefix netmask (e.g. "255.255.255.0")
+ """
+ if arg not in cls._netmask_cache:
+ if isinstance(arg, _compat_int_types):
+ prefixlen = arg
+ else:
+ try:
+ # Check for a netmask in prefix length form
+ prefixlen = cls._prefix_from_prefix_string(arg)
+ except NetmaskValueError:
+ # Check for a netmask or hostmask in dotted-quad form.
+ # This may raise NetmaskValueError.
+ prefixlen = cls._prefix_from_ip_string(arg)
+ netmask = IPv4Address(cls._ip_int_from_prefix(prefixlen))
+ cls._netmask_cache[arg] = netmask, prefixlen
+ return cls._netmask_cache[arg]
+
+ @classmethod
+ def _ip_int_from_string(cls, ip_str):
+ """Turn the given IP string into an integer for comparison.
+
+ Args:
+ ip_str: A string, the IP ip_str.
+
+ Returns:
+ The IP ip_str as an integer.
+
+ Raises:
+ AddressValueError: if ip_str isn't a valid IPv4 Address.
+
+ """
+ if not ip_str:
+ raise AddressValueError('Address cannot be empty')
+
+ octets = ip_str.split('.')
+ if len(octets) != 4:
+ raise AddressValueError("Expected 4 octets in %r" % ip_str)
+
+ try:
+ return _compat_int_from_byte_vals(
+ map(cls._parse_octet, octets), 'big')
+ except ValueError as exc:
+ raise AddressValueError("%s in %r" % (exc, ip_str))
+
+ @classmethod
+ def _parse_octet(cls, octet_str):
+ """Convert a decimal octet into an integer.
+
+ Args:
+ octet_str: A string, the number to parse.
+
+ Returns:
+ The octet as an integer.
+
+ Raises:
+ ValueError: if the octet isn't strictly a decimal from [0..255].
+
+ """
+ if not octet_str:
+ raise ValueError("Empty octet not permitted")
+ # Whitelist the characters, since int() allows a lot of bizarre stuff.
+ if not cls._DECIMAL_DIGITS.issuperset(octet_str):
+ msg = "Only decimal digits permitted in %r"
+ raise ValueError(msg % octet_str)
+ # We do the length check second, since the invalid character error
+ # is likely to be more informative for the user
+ if len(octet_str) > 3:
+ msg = "At most 3 characters permitted in %r"
+ raise ValueError(msg % octet_str)
+ # Convert to integer (we know digits are legal)
+ octet_int = int(octet_str, 10)
+ # Any octets that look like they *might* be written in octal,
+ # and which don't look exactly the same in both octal and
+ # decimal are rejected as ambiguous
+ if octet_int > 7 and octet_str[0] == '0':
+ msg = "Ambiguous (octal/decimal) value in %r not permitted"
+ raise ValueError(msg % octet_str)
+ if octet_int > 255:
+ raise ValueError("Octet %d (> 255) not permitted" % octet_int)
+ return octet_int
+
+ @classmethod
+ def _string_from_ip_int(cls, ip_int):
+ """Turns a 32-bit integer into dotted decimal notation.
+
+ Args:
+ ip_int: An integer, the IP address.
+
+ Returns:
+ The IP address as a string in dotted decimal notation.
+
+ """
+ return '.'.join(_compat_str(struct.unpack(b'!B', b)[0]
+ if isinstance(b, bytes)
+ else b)
+ for b in _compat_to_bytes(ip_int, 4, 'big'))
+
+ def _is_hostmask(self, ip_str):
+ """Test if the IP string is a hostmask (rather than a netmask).
+
+ Args:
+ ip_str: A string, the potential hostmask.
+
+ Returns:
+ A boolean, True if the IP string is a hostmask.
+
+ """
+ bits = ip_str.split('.')
+ try:
+ parts = [x for x in map(int, bits) if x in self._valid_mask_octets]
+ except ValueError:
+ return False
+ if len(parts) != len(bits):
+ return False
+ if parts[0] < parts[-1]:
+ return True
+ return False
+
+ def _reverse_pointer(self):
+ """Return the reverse DNS pointer name for the IPv4 address.
+
+ This implements the method described in RFC1035 3.5.
+
+ """
+ reverse_octets = _compat_str(self).split('.')[::-1]
+ return '.'.join(reverse_octets) + '.in-addr.arpa'
+
+ @property
+ def max_prefixlen(self):
+ return self._max_prefixlen
+
+ @property
+ def version(self):
+ return self._version
+
+
+class IPv4Address(_BaseV4, _BaseAddress):
+
+ """Represent and manipulate single IPv4 Addresses."""
+
+ __slots__ = ('_ip', '__weakref__')
+
+ def __init__(self, address):
+
+ """
+ Args:
+ address: A string or integer representing the IP
+
+ Additionally, an integer can be passed, so
+ IPv4Address('192.0.2.1') == IPv4Address(3221225985).
+ or, more generally
+ IPv4Address(int(IPv4Address('192.0.2.1'))) ==
+ IPv4Address('192.0.2.1')
+
+ Raises:
+ AddressValueError: If ipaddress isn't a valid IPv4 address.
+
+ """
+ # Efficient constructor from integer.
+ if isinstance(address, _compat_int_types):
+ self._check_int_address(address)
+ self._ip = address
+ return
+
+ # Constructing from a packed address
+ if isinstance(address, bytes):
+ self._check_packed_address(address, 4)
+ bvs = _compat_bytes_to_byte_vals(address)
+ self._ip = _compat_int_from_byte_vals(bvs, 'big')
+ return
+
+ # Assume input argument to be string or any object representation
+ # which converts into a formatted IP string.
+ addr_str = _compat_str(address)
+ if '/' in addr_str:
+ raise AddressValueError("Unexpected '/' in %r" % address)
+ self._ip = self._ip_int_from_string(addr_str)
+
+ @property
+ def packed(self):
+ """The binary representation of this address."""
+ return v4_int_to_packed(self._ip)
+
+ @property
+ def is_reserved(self):
+ """Test if the address is otherwise IETF reserved.
+
+ Returns:
+ A boolean, True if the address is within the
+ reserved IPv4 Network range.
+
+ """
+ return self in self._constants._reserved_network
+
+ @property
+ def is_private(self):
+ """Test if this address is allocated for private networks.
+
+ Returns:
+ A boolean, True if the address is reserved per
+ iana-ipv4-special-registry.
+
+ """
+ return any(self in net for net in self._constants._private_networks)
+
+ @property
+ def is_multicast(self):
+ """Test if the address is reserved for multicast use.
+
+ Returns:
+ A boolean, True if the address is multicast.
+ See RFC 3171 for details.
+
+ """
+ return self in self._constants._multicast_network
+
+ @property
+ def is_unspecified(self):
+ """Test if the address is unspecified.
+
+ Returns:
+ A boolean, True if this is the unspecified address as defined in
+ RFC 5735 3.
+
+ """
+ return self == self._constants._unspecified_address
+
+ @property
+ def is_loopback(self):
+ """Test if the address is a loopback address.
+
+ Returns:
+ A boolean, True if the address is a loopback per RFC 3330.
+
+ """
+ return self in self._constants._loopback_network
+
+ @property
+ def is_link_local(self):
+ """Test if the address is reserved for link-local.
+
+ Returns:
+ A boolean, True if the address is link-local per RFC 3927.
+
+ """
+ return self in self._constants._linklocal_network
+
+
+class IPv4Interface(IPv4Address):
+
+ def __init__(self, address):
+ if isinstance(address, (bytes, _compat_int_types)):
+ IPv4Address.__init__(self, address)
+ self.network = IPv4Network(self._ip)
+ self._prefixlen = self._max_prefixlen
+ return
+
+ if isinstance(address, tuple):
+ IPv4Address.__init__(self, address[0])
+ if len(address) > 1:
+ self._prefixlen = int(address[1])
+ else:
+ self._prefixlen = self._max_prefixlen
+
+ self.network = IPv4Network(address, strict=False)
+ self.netmask = self.network.netmask
+ self.hostmask = self.network.hostmask
+ return
+
+ addr = _split_optional_netmask(address)
+ IPv4Address.__init__(self, addr[0])
+
+ self.network = IPv4Network(address, strict=False)
+ self._prefixlen = self.network._prefixlen
+
+ self.netmask = self.network.netmask
+ self.hostmask = self.network.hostmask
+
+ def __str__(self):
+ return '%s/%d' % (self._string_from_ip_int(self._ip),
+ self.network.prefixlen)
+
+ def __eq__(self, other):
+ address_equal = IPv4Address.__eq__(self, other)
+ if not address_equal or address_equal is NotImplemented:
+ return address_equal
+ try:
+ return self.network == other.network
+ except AttributeError:
+ # An interface with an associated network is NOT the
+ # same as an unassociated address. That's why the hash
+ # takes the extra info into account.
+ return False
+
+ def __lt__(self, other):
+ address_less = IPv4Address.__lt__(self, other)
+ if address_less is NotImplemented:
+ return NotImplemented
+ try:
+ return self.network < other.network
+ except AttributeError:
+ # We *do* allow addresses and interfaces to be sorted. The
+ # unassociated address is considered less than all interfaces.
+ return False
+
+ def __hash__(self):
+ return self._ip ^ self._prefixlen ^ int(self.network.network_address)
+
+ __reduce__ = _IPAddressBase.__reduce__
+
+ @property
+ def ip(self):
+ return IPv4Address(self._ip)
+
+ @property
+ def with_prefixlen(self):
+ return '%s/%s' % (self._string_from_ip_int(self._ip),
+ self._prefixlen)
+
+ @property
+ def with_netmask(self):
+ return '%s/%s' % (self._string_from_ip_int(self._ip),
+ self.netmask)
+
+ @property
+ def with_hostmask(self):
+ return '%s/%s' % (self._string_from_ip_int(self._ip),
+ self.hostmask)
+
+
+class IPv4Network(_BaseV4, _BaseNetwork):
+
+ """This class represents and manipulates 32-bit IPv4 network + addresses..
+
+ Attributes: [examples for IPv4Network('192.0.2.0/27')]
+ .network_address: IPv4Address('192.0.2.0')
+ .hostmask: IPv4Address('0.0.0.31')
+ .broadcast_address: IPv4Address('192.0.2.32')
+ .netmask: IPv4Address('255.255.255.224')
+ .prefixlen: 27
+
+ """
+ # Class to use when creating address objects
+ _address_class = IPv4Address
+
+ def __init__(self, address, strict=True):
+
+ """Instantiate a new IPv4 network object.
+
+ Args:
+ address: A string or integer representing the IP [& network].
+ '192.0.2.0/24'
+ '192.0.2.0/255.255.255.0'
+ '192.0.0.2/0.0.0.255'
+ are all functionally the same in IPv4. Similarly,
+ '192.0.2.1'
+ '192.0.2.1/255.255.255.255'
+ '192.0.2.1/32'
+ are also functionally equivalent. That is to say, failing to
+ provide a subnetmask will create an object with a mask of /32.
+
+ If the mask (portion after the / in the argument) is given in
+ dotted quad form, it is treated as a netmask if it starts with a
+ non-zero field (e.g. /255.0.0.0 == /8) and as a hostmask if it
+ starts with a zero field (e.g. 0.255.255.255 == /8), with the
+ single exception of an all-zero mask which is treated as a
+ netmask == /0. If no mask is given, a default of /32 is used.
+
+ Additionally, an integer can be passed, so
+ IPv4Network('192.0.2.1') == IPv4Network(3221225985)
+ or, more generally
+ IPv4Interface(int(IPv4Interface('192.0.2.1'))) ==
+ IPv4Interface('192.0.2.1')
+
+ Raises:
+ AddressValueError: If ipaddress isn't a valid IPv4 address.
+ NetmaskValueError: If the netmask isn't valid for
+ an IPv4 address.
+ ValueError: If strict is True and a network address is not
+ supplied.
+
+ """
+ _BaseNetwork.__init__(self, address)
+
+ # Constructing from a packed address or integer
+ if isinstance(address, (_compat_int_types, bytes)):
+ self.network_address = IPv4Address(address)
+ self.netmask, self._prefixlen = self._make_netmask(
+ self._max_prefixlen)
+ # fixme: address/network test here.
+ return
+
+ if isinstance(address, tuple):
+ if len(address) > 1:
+ arg = address[1]
+ else:
+ # We weren't given an address[1]
+ arg = self._max_prefixlen
+ self.network_address = IPv4Address(address[0])
+ self.netmask, self._prefixlen = self._make_netmask(arg)
+ packed = int(self.network_address)
+ if packed & int(self.netmask) != packed:
+ if strict:
+ raise ValueError('%s has host bits set' % self)
+ else:
+ self.network_address = IPv4Address(packed &
+ int(self.netmask))
+ return
+
+ # Assume input argument to be string or any object representation
+ # which converts into a formatted IP prefix string.
+ addr = _split_optional_netmask(address)
+ self.network_address = IPv4Address(self._ip_int_from_string(addr[0]))
+
+ if len(addr) == 2:
+ arg = addr[1]
+ else:
+ arg = self._max_prefixlen
+ self.netmask, self._prefixlen = self._make_netmask(arg)
+
+ if strict:
+ if (IPv4Address(int(self.network_address) & int(self.netmask)) !=
+ self.network_address):
+ raise ValueError('%s has host bits set' % self)
+ self.network_address = IPv4Address(int(self.network_address) &
+ int(self.netmask))
+
+ if self._prefixlen == (self._max_prefixlen - 1):
+ self.hosts = self.__iter__
+
+ @property
+ def is_global(self):
+ """Test if this address is allocated for public networks.
+
+ Returns:
+ A boolean, True if the address is not reserved per
+ iana-ipv4-special-registry.
+
+ """
+ return (not (self.network_address in IPv4Network('100.64.0.0/10') and
+ self.broadcast_address in IPv4Network('100.64.0.0/10')) and
+ not self.is_private)
+
+
+class _IPv4Constants(object):
+
+ _linklocal_network = IPv4Network('169.254.0.0/16')
+
+ _loopback_network = IPv4Network('127.0.0.0/8')
+
+ _multicast_network = IPv4Network('224.0.0.0/4')
+
+ _private_networks = [
+ IPv4Network('0.0.0.0/8'),
+ IPv4Network('10.0.0.0/8'),
+ IPv4Network('127.0.0.0/8'),
+ IPv4Network('169.254.0.0/16'),
+ IPv4Network('172.16.0.0/12'),
+ IPv4Network('192.0.0.0/29'),
+ IPv4Network('192.0.0.170/31'),
+ IPv4Network('192.0.2.0/24'),
+ IPv4Network('192.168.0.0/16'),
+ IPv4Network('198.18.0.0/15'),
+ IPv4Network('198.51.100.0/24'),
+ IPv4Network('203.0.113.0/24'),
+ IPv4Network('240.0.0.0/4'),
+ IPv4Network('255.255.255.255/32'),
+ ]
+
+ _reserved_network = IPv4Network('240.0.0.0/4')
+
+ _unspecified_address = IPv4Address('0.0.0.0')
+
+
+IPv4Address._constants = _IPv4Constants
+
+
+class _BaseV6(object):
+
+ """Base IPv6 object.
+
+ The following methods are used by IPv6 objects in both single IP
+ addresses and networks.
+
+ """
+
+ __slots__ = ()
+ _version = 6
+ _ALL_ONES = (2 ** IPV6LENGTH) - 1
+ _HEXTET_COUNT = 8
+ _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef')
+ _max_prefixlen = IPV6LENGTH
+
+ # There are only a bunch of valid v6 netmasks, so we cache them all
+ # when constructed (see _make_netmask()).
+ _netmask_cache = {}
+
+ @classmethod
+ def _make_netmask(cls, arg):
+ """Make a (netmask, prefix_len) tuple from the given argument.
+
+ Argument can be:
+ - an integer (the prefix length)
+ - a string representing the prefix length (e.g. "24")
+ - a string representing the prefix netmask (e.g. "255.255.255.0")
+ """
+ if arg not in cls._netmask_cache:
+ if isinstance(arg, _compat_int_types):
+ prefixlen = arg
+ else:
+ prefixlen = cls._prefix_from_prefix_string(arg)
+ netmask = IPv6Address(cls._ip_int_from_prefix(prefixlen))
+ cls._netmask_cache[arg] = netmask, prefixlen
+ return cls._netmask_cache[arg]
+
+ @classmethod
+ def _ip_int_from_string(cls, ip_str):
+ """Turn an IPv6 ip_str into an integer.
+
+ Args:
+ ip_str: A string, the IPv6 ip_str.
+
+ Returns:
+ An int, the IPv6 address
+
+ Raises:
+ AddressValueError: if ip_str isn't a valid IPv6 Address.
+
+ """
+ if not ip_str:
+ raise AddressValueError('Address cannot be empty')
+
+ parts = ip_str.split(':')
+
+ # An IPv6 address needs at least 2 colons (3 parts).
+ _min_parts = 3
+ if len(parts) < _min_parts:
+ msg = "At least %d parts expected in %r" % (_min_parts, ip_str)
+ raise AddressValueError(msg)
+
+ # If the address has an IPv4-style suffix, convert it to hexadecimal.
+ if '.' in parts[-1]:
+ try:
+ ipv4_int = IPv4Address(parts.pop())._ip
+ except AddressValueError as exc:
+ raise AddressValueError("%s in %r" % (exc, ip_str))
+ parts.append('%x' % ((ipv4_int >> 16) & 0xFFFF))
+ parts.append('%x' % (ipv4_int & 0xFFFF))
+
+ # An IPv6 address can't have more than 8 colons (9 parts).
+ # The extra colon comes from using the "::" notation for a single
+ # leading or trailing zero part.
+ _max_parts = cls._HEXTET_COUNT + 1
+ if len(parts) > _max_parts:
+ msg = "At most %d colons permitted in %r" % (
+ _max_parts - 1, ip_str)
+ raise AddressValueError(msg)
+
+ # Disregarding the endpoints, find '::' with nothing in between.
+ # This indicates that a run of zeroes has been skipped.
+ skip_index = None
+ for i in _compat_range(1, len(parts) - 1):
+ if not parts[i]:
+ if skip_index is not None:
+ # Can't have more than one '::'
+ msg = "At most one '::' permitted in %r" % ip_str
+ raise AddressValueError(msg)
+ skip_index = i
+
+ # parts_hi is the number of parts to copy from above/before the '::'
+ # parts_lo is the number of parts to copy from below/after the '::'
+ if skip_index is not None:
+ # If we found a '::', then check if it also covers the endpoints.
+ parts_hi = skip_index
+ parts_lo = len(parts) - skip_index - 1
+ if not parts[0]:
+ parts_hi -= 1
+ if parts_hi:
+ msg = "Leading ':' only permitted as part of '::' in %r"
+ raise AddressValueError(msg % ip_str) # ^: requires ^::
+ if not parts[-1]:
+ parts_lo -= 1
+ if parts_lo:
+ msg = "Trailing ':' only permitted as part of '::' in %r"
+ raise AddressValueError(msg % ip_str) # :$ requires ::$
+ parts_skipped = cls._HEXTET_COUNT - (parts_hi + parts_lo)
+ if parts_skipped < 1:
+ msg = "Expected at most %d other parts with '::' in %r"
+ raise AddressValueError(msg % (cls._HEXTET_COUNT - 1, ip_str))
+ else:
+ # Otherwise, allocate the entire address to parts_hi. The
+ # endpoints could still be empty, but _parse_hextet() will check
+ # for that.
+ if len(parts) != cls._HEXTET_COUNT:
+ msg = "Exactly %d parts expected without '::' in %r"
+ raise AddressValueError(msg % (cls._HEXTET_COUNT, ip_str))
+ if not parts[0]:
+ msg = "Leading ':' only permitted as part of '::' in %r"
+ raise AddressValueError(msg % ip_str) # ^: requires ^::
+ if not parts[-1]:
+ msg = "Trailing ':' only permitted as part of '::' in %r"
+ raise AddressValueError(msg % ip_str) # :$ requires ::$
+ parts_hi = len(parts)
+ parts_lo = 0
+ parts_skipped = 0
+
+ try:
+ # Now, parse the hextets into a 128-bit integer.
+ ip_int = 0
+ for i in range(parts_hi):
+ ip_int <<= 16
+ ip_int |= cls._parse_hextet(parts[i])
+ ip_int <<= 16 * parts_skipped
+ for i in range(-parts_lo, 0):
+ ip_int <<= 16
+ ip_int |= cls._parse_hextet(parts[i])
+ return ip_int
+ except ValueError as exc:
+ raise AddressValueError("%s in %r" % (exc, ip_str))
+
+ @classmethod
+ def _parse_hextet(cls, hextet_str):
+ """Convert an IPv6 hextet string into an integer.
+
+ Args:
+ hextet_str: A string, the number to parse.
+
+ Returns:
+ The hextet as an integer.
+
+ Raises:
+ ValueError: if the input isn't strictly a hex number from
+ [0..FFFF].
+
+ """
+ # Whitelist the characters, since int() allows a lot of bizarre stuff.
+ if not cls._HEX_DIGITS.issuperset(hextet_str):
+ raise ValueError("Only hex digits permitted in %r" % hextet_str)
+ # We do the length check second, since the invalid character error
+ # is likely to be more informative for the user
+ if len(hextet_str) > 4:
+ msg = "At most 4 characters permitted in %r"
+ raise ValueError(msg % hextet_str)
+ # Length check means we can skip checking the integer value
+ return int(hextet_str, 16)
+
+ @classmethod
+ def _compress_hextets(cls, hextets):
+ """Compresses a list of hextets.
+
+ Compresses a list of strings, replacing the longest continuous
+ sequence of "0" in the list with "" and adding empty strings at
+ the beginning or at the end of the string such that subsequently
+ calling ":".join(hextets) will produce the compressed version of
+ the IPv6 address.
+
+ Args:
+ hextets: A list of strings, the hextets to compress.
+
+ Returns:
+ A list of strings.
+
+ """
+ best_doublecolon_start = -1
+ best_doublecolon_len = 0
+ doublecolon_start = -1
+ doublecolon_len = 0
+ for index, hextet in enumerate(hextets):
+ if hextet == '0':
+ doublecolon_len += 1
+ if doublecolon_start == -1:
+ # Start of a sequence of zeros.
+ doublecolon_start = index
+ if doublecolon_len > best_doublecolon_len:
+ # This is the longest sequence of zeros so far.
+ best_doublecolon_len = doublecolon_len
+ best_doublecolon_start = doublecolon_start
+ else:
+ doublecolon_len = 0
+ doublecolon_start = -1
+
+ if best_doublecolon_len > 1:
+ best_doublecolon_end = (best_doublecolon_start +
+ best_doublecolon_len)
+ # For zeros at the end of the address.
+ if best_doublecolon_end == len(hextets):
+ hextets += ['']
+ hextets[best_doublecolon_start:best_doublecolon_end] = ['']
+ # For zeros at the beginning of the address.
+ if best_doublecolon_start == 0:
+ hextets = [''] + hextets
+
+ return hextets
+
+ @classmethod
+ def _string_from_ip_int(cls, ip_int=None):
+ """Turns a 128-bit integer into hexadecimal notation.
+
+ Args:
+ ip_int: An integer, the IP address.
+
+ Returns:
+ A string, the hexadecimal representation of the address.
+
+ Raises:
+ ValueError: The address is bigger than 128 bits of all ones.
+
+ """
+ if ip_int is None:
+ ip_int = int(cls._ip)
+
+ if ip_int > cls._ALL_ONES:
+ raise ValueError('IPv6 address is too large')
+
+ hex_str = '%032x' % ip_int
+ hextets = ['%x' % int(hex_str[x:x + 4], 16) for x in range(0, 32, 4)]
+
+ hextets = cls._compress_hextets(hextets)
+ return ':'.join(hextets)
+
+ def _explode_shorthand_ip_string(self):
+ """Expand a shortened IPv6 address.
+
+ Args:
+ ip_str: A string, the IPv6 address.
+
+ Returns:
+ A string, the expanded IPv6 address.
+
+ """
+ if isinstance(self, IPv6Network):
+ ip_str = _compat_str(self.network_address)
+ elif isinstance(self, IPv6Interface):
+ ip_str = _compat_str(self.ip)
+ else:
+ ip_str = _compat_str(self)
+
+ ip_int = self._ip_int_from_string(ip_str)
+ hex_str = '%032x' % ip_int
+ parts = [hex_str[x:x + 4] for x in range(0, 32, 4)]
+ if isinstance(self, (_BaseNetwork, IPv6Interface)):
+ return '%s/%d' % (':'.join(parts), self._prefixlen)
+ return ':'.join(parts)
+
+ def _reverse_pointer(self):
+ """Return the reverse DNS pointer name for the IPv6 address.
+
+ This implements the method described in RFC3596 2.5.
+
+ """
+ reverse_chars = self.exploded[::-1].replace(':', '')
+ return '.'.join(reverse_chars) + '.ip6.arpa'
+
+ @property
+ def max_prefixlen(self):
+ return self._max_prefixlen
+
+ @property
+ def version(self):
+ return self._version
+
+
+class IPv6Address(_BaseV6, _BaseAddress):
+
+ """Represent and manipulate single IPv6 Addresses."""
+
+ __slots__ = ('_ip', '__weakref__')
+
+ def __init__(self, address):
+ """Instantiate a new IPv6 address object.
+
+ Args:
+ address: A string or integer representing the IP
+
+ Additionally, an integer can be passed, so
+ IPv6Address('2001:db8::') ==
+ IPv6Address(42540766411282592856903984951653826560)
+ or, more generally
+ IPv6Address(int(IPv6Address('2001:db8::'))) ==
+ IPv6Address('2001:db8::')
+
+ Raises:
+ AddressValueError: If address isn't a valid IPv6 address.
+
+ """
+ # Efficient constructor from integer.
+ if isinstance(address, _compat_int_types):
+ self._check_int_address(address)
+ self._ip = address
+ return
+
+ # Constructing from a packed address
+ if isinstance(address, bytes):
+ self._check_packed_address(address, 16)
+ bvs = _compat_bytes_to_byte_vals(address)
+ self._ip = _compat_int_from_byte_vals(bvs, 'big')
+ return
+
+ # Assume input argument to be string or any object representation
+ # which converts into a formatted IP string.
+ addr_str = _compat_str(address)
+ if '/' in addr_str:
+ raise AddressValueError("Unexpected '/' in %r" % address)
+ self._ip = self._ip_int_from_string(addr_str)
+
+ @property
+ def packed(self):
+ """The binary representation of this address."""
+ return v6_int_to_packed(self._ip)
+
+ @property
+ def is_multicast(self):
+ """Test if the address is reserved for multicast use.
+
+ Returns:
+ A boolean, True if the address is a multicast address.
+ See RFC 2373 2.7 for details.
+
+ """
+ return self in self._constants._multicast_network
+
+ @property
+ def is_reserved(self):
+ """Test if the address is otherwise IETF reserved.
+
+ Returns:
+ A boolean, True if the address is within one of the
+ reserved IPv6 Network ranges.
+
+ """
+ return any(self in x for x in self._constants._reserved_networks)
+
+ @property
+ def is_link_local(self):
+ """Test if the address is reserved for link-local.
+
+ Returns:
+ A boolean, True if the address is reserved per RFC 4291.
+
+ """
+ return self in self._constants._linklocal_network
+
+ @property
+ def is_site_local(self):
+ """Test if the address is reserved for site-local.
+
+ Note that the site-local address space has been deprecated by RFC 3879.
+ Use is_private to test if this address is in the space of unique local
+ addresses as defined by RFC 4193.
+
+ Returns:
+ A boolean, True if the address is reserved per RFC 3513 2.5.6.
+
+ """
+ return self in self._constants._sitelocal_network
+
+ @property
+ def is_private(self):
+ """Test if this address is allocated for private networks.
+
+ Returns:
+ A boolean, True if the address is reserved per
+ iana-ipv6-special-registry.
+
+ """
+ return any(self in net for net in self._constants._private_networks)
+
+ @property
+ def is_global(self):
+ """Test if this address is allocated for public networks.
+
+ Returns:
+ A boolean, true if the address is not reserved per
+ iana-ipv6-special-registry.
+
+ """
+ return not self.is_private
+
+ @property
+ def is_unspecified(self):
+ """Test if the address is unspecified.
+
+ Returns:
+ A boolean, True if this is the unspecified address as defined in
+ RFC 2373 2.5.2.
+
+ """
+ return self._ip == 0
+
+ @property
+ def is_loopback(self):
+ """Test if the address is a loopback address.
+
+ Returns:
+ A boolean, True if the address is a loopback address as defined in
+ RFC 2373 2.5.3.
+
+ """
+ return self._ip == 1
+
+ @property
+ def ipv4_mapped(self):
+ """Return the IPv4 mapped address.
+
+ Returns:
+ If the IPv6 address is a v4 mapped address, return the
+ IPv4 mapped address. Return None otherwise.
+
+ """
+ if (self._ip >> 32) != 0xFFFF:
+ return None
+ return IPv4Address(self._ip & 0xFFFFFFFF)
+
+ @property
+ def teredo(self):
+ """Tuple of embedded teredo IPs.
+
+ Returns:
+ Tuple of the (server, client) IPs or None if the address
+ doesn't appear to be a teredo address (doesn't start with
+ 2001::/32)
+
+ """
+ if (self._ip >> 96) != 0x20010000:
+ return None
+ return (IPv4Address((self._ip >> 64) & 0xFFFFFFFF),
+ IPv4Address(~self._ip & 0xFFFFFFFF))
+
+ @property
+ def sixtofour(self):
+ """Return the IPv4 6to4 embedded address.
+
+ Returns:
+ The IPv4 6to4-embedded address if present or None if the
+ address doesn't appear to contain a 6to4 embedded address.
+
+ """
+ if (self._ip >> 112) != 0x2002:
+ return None
+ return IPv4Address((self._ip >> 80) & 0xFFFFFFFF)
+
+
+class IPv6Interface(IPv6Address):
+
+ def __init__(self, address):
+ if isinstance(address, (bytes, _compat_int_types)):
+ IPv6Address.__init__(self, address)
+ self.network = IPv6Network(self._ip)
+ self._prefixlen = self._max_prefixlen
+ return
+ if isinstance(address, tuple):
+ IPv6Address.__init__(self, address[0])
+ if len(address) > 1:
+ self._prefixlen = int(address[1])
+ else:
+ self._prefixlen = self._max_prefixlen
+ self.network = IPv6Network(address, strict=False)
+ self.netmask = self.network.netmask
+ self.hostmask = self.network.hostmask
+ return
+
+ addr = _split_optional_netmask(address)
+ IPv6Address.__init__(self, addr[0])
+ self.network = IPv6Network(address, strict=False)
+ self.netmask = self.network.netmask
+ self._prefixlen = self.network._prefixlen
+ self.hostmask = self.network.hostmask
+
+ def __str__(self):
+ return '%s/%d' % (self._string_from_ip_int(self._ip),
+ self.network.prefixlen)
+
+ def __eq__(self, other):
+ address_equal = IPv6Address.__eq__(self, other)
+ if not address_equal or address_equal is NotImplemented:
+ return address_equal
+ try:
+ return self.network == other.network
+ except AttributeError:
+ # An interface with an associated network is NOT the
+ # same as an unassociated address. That's why the hash
+ # takes the extra info into account.
+ return False
+
+ def __lt__(self, other):
+ address_less = IPv6Address.__lt__(self, other)
+ if address_less is NotImplemented:
+ return NotImplemented
+ try:
+ return self.network < other.network
+ except AttributeError:
+ # We *do* allow addresses and interfaces to be sorted. The
+ # unassociated address is considered less than all interfaces.
+ return False
+
+ def __hash__(self):
+ return self._ip ^ self._prefixlen ^ int(self.network.network_address)
+
+ __reduce__ = _IPAddressBase.__reduce__
+
+ @property
+ def ip(self):
+ return IPv6Address(self._ip)
+
+ @property
+ def with_prefixlen(self):
+ return '%s/%s' % (self._string_from_ip_int(self._ip),
+ self._prefixlen)
+
+ @property
+ def with_netmask(self):
+ return '%s/%s' % (self._string_from_ip_int(self._ip),
+ self.netmask)
+
+ @property
+ def with_hostmask(self):
+ return '%s/%s' % (self._string_from_ip_int(self._ip),
+ self.hostmask)
+
+ @property
+ def is_unspecified(self):
+ return self._ip == 0 and self.network.is_unspecified
+
+ @property
+ def is_loopback(self):
+ return self._ip == 1 and self.network.is_loopback
+
+
+class IPv6Network(_BaseV6, _BaseNetwork):
+
+ """This class represents and manipulates 128-bit IPv6 networks.
+
+ Attributes: [examples for IPv6('2001:db8::1000/124')]
+ .network_address: IPv6Address('2001:db8::1000')
+ .hostmask: IPv6Address('::f')
+ .broadcast_address: IPv6Address('2001:db8::100f')
+ .netmask: IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff:ffff:fff0')
+ .prefixlen: 124
+
+ """
+
+ # Class to use when creating address objects
+ _address_class = IPv6Address
+
+ def __init__(self, address, strict=True):
+ """Instantiate a new IPv6 Network object.
+
+ Args:
+ address: A string or integer representing the IPv6 network or the
+ IP and prefix/netmask.
+ '2001:db8::/128'
+ '2001:db8:0000:0000:0000:0000:0000:0000/128'
+ '2001:db8::'
+ are all functionally the same in IPv6. That is to say,
+ failing to provide a subnetmask will create an object with
+ a mask of /128.
+
+ Additionally, an integer can be passed, so
+ IPv6Network('2001:db8::') ==
+ IPv6Network(42540766411282592856903984951653826560)
+ or, more generally
+ IPv6Network(int(IPv6Network('2001:db8::'))) ==
+ IPv6Network('2001:db8::')
+
+ strict: A boolean. If true, ensure that we have been passed
+ A true network address, eg, 2001:db8::1000/124 and not an
+ IP address on a network, eg, 2001:db8::1/124.
+
+ Raises:
+ AddressValueError: If address isn't a valid IPv6 address.
+ NetmaskValueError: If the netmask isn't valid for
+ an IPv6 address.
+ ValueError: If strict was True and a network address was not
+ supplied.
+
+ """
+ _BaseNetwork.__init__(self, address)
+
+ # Efficient constructor from integer or packed address
+ if isinstance(address, (bytes, _compat_int_types)):
+ self.network_address = IPv6Address(address)
+ self.netmask, self._prefixlen = self._make_netmask(
+ self._max_prefixlen)
+ return
+
+ if isinstance(address, tuple):
+ if len(address) > 1:
+ arg = address[1]
+ else:
+ arg = self._max_prefixlen
+ self.netmask, self._prefixlen = self._make_netmask(arg)
+ self.network_address = IPv6Address(address[0])
+ packed = int(self.network_address)
+ if packed & int(self.netmask) != packed:
+ if strict:
+ raise ValueError('%s has host bits set' % self)
+ else:
+ self.network_address = IPv6Address(packed &
+ int(self.netmask))
+ return
+
+ # Assume input argument to be string or any object representation
+ # which converts into a formatted IP prefix string.
+ addr = _split_optional_netmask(address)
+
+ self.network_address = IPv6Address(self._ip_int_from_string(addr[0]))
+
+ if len(addr) == 2:
+ arg = addr[1]
+ else:
+ arg = self._max_prefixlen
+ self.netmask, self._prefixlen = self._make_netmask(arg)
+
+ if strict:
+ if (IPv6Address(int(self.network_address) & int(self.netmask)) !=
+ self.network_address):
+ raise ValueError('%s has host bits set' % self)
+ self.network_address = IPv6Address(int(self.network_address) &
+ int(self.netmask))
+
+ if self._prefixlen == (self._max_prefixlen - 1):
+ self.hosts = self.__iter__
+
+ def hosts(self):
+ """Generate Iterator over usable hosts in a network.
+
+ This is like __iter__ except it doesn't return the
+ Subnet-Router anycast address.
+
+ """
+ network = int(self.network_address)
+ broadcast = int(self.broadcast_address)
+ for x in _compat_range(network + 1, broadcast + 1):
+ yield self._address_class(x)
+
+ @property
+ def is_site_local(self):
+ """Test if the address is reserved for site-local.
+
+ Note that the site-local address space has been deprecated by RFC 3879.
+ Use is_private to test if this address is in the space of unique local
+ addresses as defined by RFC 4193.
+
+ Returns:
+ A boolean, True if the address is reserved per RFC 3513 2.5.6.
+
+ """
+ return (self.network_address.is_site_local and
+ self.broadcast_address.is_site_local)
+
+
+class _IPv6Constants(object):
+
+ _linklocal_network = IPv6Network('fe80::/10')
+
+ _multicast_network = IPv6Network('ff00::/8')
+
+ _private_networks = [
+ IPv6Network('::1/128'),
+ IPv6Network('::/128'),
+ IPv6Network('::ffff:0:0/96'),
+ IPv6Network('100::/64'),
+ IPv6Network('2001::/23'),
+ IPv6Network('2001:2::/48'),
+ IPv6Network('2001:db8::/32'),
+ IPv6Network('2001:10::/28'),
+ IPv6Network('fc00::/7'),
+ IPv6Network('fe80::/10'),
+ ]
+
+ _reserved_networks = [
+ IPv6Network('::/8'), IPv6Network('100::/8'),
+ IPv6Network('200::/7'), IPv6Network('400::/6'),
+ IPv6Network('800::/5'), IPv6Network('1000::/4'),
+ IPv6Network('4000::/3'), IPv6Network('6000::/3'),
+ IPv6Network('8000::/3'), IPv6Network('A000::/3'),
+ IPv6Network('C000::/3'), IPv6Network('E000::/4'),
+ IPv6Network('F000::/5'), IPv6Network('F800::/6'),
+ IPv6Network('FE00::/9'),
+ ]
+
+ _sitelocal_network = IPv6Network('fec0::/10')
+
+
+IPv6Address._constants = _IPv6Constants
diff --git a/node-admin/scripts/make-host-like-container.sh b/node-admin/scripts/make-host-like-container.sh
new file mode 100755
index 00000000000..7e88e89b125
--- /dev/null
+++ b/node-admin/scripts/make-host-like-container.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -e
+
+source "${0%/*}/common.sh"
+
+function Usage {
+ UsageHelper "$@" <<EOF
+Usage: $SCRIPT_NAME <command>
+Make localhost look like a container for the purpose of various other scripts.
+
+Commands:
+ start Make /host/proc point to /proc
+ stop Remove /host directory
+ restart Stop, then start
+EOF
+}
+
+function MakeHostDirectory {
+ if ! [ -e /host ]
+ then
+ echo "Created directory /host"
+ sudo mkdir /host
+ if ! [ -e /host/proc ]
+ then
+ echo "Created symbolic link from /host/proc to /proc"
+ sudo ln -s /proc /host/proc
+ fi
+ fi
+}
+
+function RemoveHostDirectory {
+ if [ -d /host ]
+ then
+ echo "Removed /host directory"
+ sudo rm -rf /host
+ fi
+}
+
+function Stop {
+ sudo true # Prime sudo
+
+ RemoveHostDirectory
+}
+
+function Start {
+ sudo true # Prime sudo
+ MakeHostDirectory
+}
+
+Main "$@"
diff --git a/node-admin/scripts/network-bridge.sh b/node-admin/scripts/network-bridge.sh
new file mode 100755
index 00000000000..535b7d30711
--- /dev/null
+++ b/node-admin/scripts/network-bridge.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -e
+
+source "${0%/*}/common.sh"
+
+declare -r DUMMY_KERNEL_NETWORK_MODULE="dummy"
+declare -r DUMMY_NETWORK_INTERFACE="dummy0"
+
+function Usage {
+ UsageHelper "$@" <<EOF
+Usage: $SCRIPT_NAME <command>
+Manages the network bridge to the Docker container network
+
+Commands:
+ start Set up the network bridge
+ stop Tear down the network bridge
+ restart Stop, then start
+EOF
+}
+
+function Stop {
+ echo -n "Removing bridge $HOST_BRIDGE_INTERFACE... "
+ sudo ip link del "$HOST_BRIDGE_INTERFACE" &>/dev/null || true
+
+ if sudo lsmod | grep -q "$DUMMY_KERNEL_NETWORK_MODULE"
+ then
+ sudo rmmod "$DUMMY_KERNEL_NETWORK_MODULE"
+ fi
+
+ echo done
+}
+
+function MakeBridge {
+ local ip="$1"
+ local prefix_bitlength="$2"
+ local name="$3"
+
+ if ip link show dev "$name" up &>/dev/null
+ then
+ # TODO: Verify it is indeed set up correctly.
+ echo "Bridge '$name' already exists, will assume it has been set up correctly"
+ else
+ echo -n "Adding bridge $name ($ip) to the container network... "
+
+ # Check if the $DUMMY_NETWORK_INTERFACE module is loaded and load if it is not
+ if ! sudo ip link show $DUMMY_NETWORK_INTERFACE &> /dev/null; then
+ sudo modprobe "$DUMMY_KERNEL_NETWORK_MODULE"
+ fi
+ sudo ip link set "$DUMMY_NETWORK_INTERFACE" up
+ sudo ip link add dev "$name" link "$DUMMY_NETWORK_INTERFACE" type macvlan mode bridge
+ sudo ip addr add dev "$name" "$ip/$prefix_bitlength" broadcast +
+ sudo ip link set dev "$name" up
+ echo done
+ fi
+}
+
+function Start {
+ MakeBridge "$HOST_BRIDGE_IP" "$NETWORK_PREFIX_BITLENGTH" "$HOST_BRIDGE_INTERFACE"
+}
+
+Main "$@"
diff --git a/node-admin/scripts/node-admin.sh b/node-admin/scripts/node-admin.sh
new file mode 100755
index 00000000000..ae0fa94029b
--- /dev/null
+++ b/node-admin/scripts/node-admin.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+ myname=${0}
+ mypath=${myname%/*}
+ myname=${myname##*/}
+ if [ "$mypath" ] && [ -d "$mypath" ]; then
+ return
+ fi
+ mypath=$(pwd)
+ if [ -f "${mypath}/${myname}" ]; then
+ return
+ fi
+ echo "FATAL: Could not figure out the path where $myname lives from $0"
+ exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+ if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+ # ensure it ends with "/" :
+ VESPA_HOME=${VESPA_HOME%/}/
+ export VESPA_HOME
+ common_env=$VESPA_HOME/$COMMON_ENV
+ if [ -f "$common_env" ]; then
+ . $common_env
+ return
+ fi
+ fi
+ return 1
+}
+
+findroot () {
+ source_common_env && return
+ if [ "$VESPA_HOME" ]; then
+ echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+ exit 1
+ fi
+ if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+ VESPA_HOME="$ROOT"
+ source_common_env && return
+ fi
+ findpath
+ while [ "$mypath" ]; do
+ VESPA_HOME=${mypath}
+ source_common_env && return
+ mypath=${mypath%/*}
+ done
+ echo "FATAL: missing VESPA_HOME environment variable"
+ echo "Could not locate $COMMON_ENV anywhere"
+ exit 1
+}
+
+findroot
+
+# END environment bootstrap section
+
+ROOT=$VESPA_HOME
+
+set -e
+
+source "${0%/*}/common.sh"
+
+function Usage {
+ UsageHelper "$@" <<EOF
+Usage: $SCRIPT_NAME <command>
+Manage the Node Admin
+
+Commands:
+ start Start the Node Admin in a Docker container
+ stop Remove the Node Admin container
+ restart Stop, then start
+EOF
+}
+
+function Stop {
+ # Prime sudo to avoid password prompt in the middle of the script.
+ sudo true
+
+ echo -n "Removing $NODE_ADMIN_CONTAINER_NAME container... "
+ docker rm -f "$NODE_ADMIN_CONTAINER_NAME" &>/dev/null || true
+ echo done
+}
+
+function Start {
+ # Prime sudo to avoid password prompt in the middle of the script.
+ sudo true
+
+ echo -n "Making directory $APPLICATION_STORAGE_ROOT... "
+ sudo mkdir -p $APPLICATION_STORAGE_ROOT
+ echo done
+
+ # Start node-admin
+ echo -n "Making $NODE_ADMIN_CONTAINER_NAME container... "
+ docker run \
+ --detach \
+ --privileged \
+ --cap-add ALL \
+ --name "$NODE_ADMIN_CONTAINER_NAME" \
+ --net=host \
+ --volume "$CONTAINER_CERT_PATH:/host/docker/certs" \
+ --volume "/proc:/host/proc" \
+ --volume "$APPLICATION_STORAGE_ROOT:/host$APPLICATION_STORAGE_ROOT" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/logs:$VESPA_HOME/logs" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/cache:$VESPA_HOME/var/cache" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/crash:$VESPA_HOME/var/crash" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/db/jdisc:$VESPA_HOME/var/db/jdisc" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/db/vespa:$VESPA_HOME/var/db/vespa" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/jdisc_container:$VESPA_HOME/var/jdisc_container" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/jdisc_core:$VESPA_HOME/var/jdisc_core" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/logstash-forwarder:$VESPA_HOME/var/logstash-forwarder" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/maven:$VESPA_HOME/var/maven" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/run:$VESPA_HOME/var/run" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/scoreboards:$VESPA_HOME/var/scoreboards" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/service:$VESPA_HOME/var/service" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/share:$VESPA_HOME/var/share" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/spool:$VESPA_HOME/var/spool" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/vespa:$VESPA_HOME/var/vespa" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/yca:$VESPA_HOME/var/yca" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/ycore++:$VESPA_HOME/var/ycore++" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/ymon:$VESPA_HOME/var/ymon" \
+ --volume "/home/docker/container-storage/node-admin$VESPA_HOME/var/zookeeper:$VESPA_HOME/var/zookeeper" \
+ --env "CONFIG_SERVER_ADDRESS=$CONFIG_SERVER_HOSTNAME" \
+ --env "NETWORK_TYPE=$NETWORK_TYPE" \
+ --entrypoint=/usr/local/bin/start-node-admin.sh \
+ "$DOCKER_IMAGE" >/dev/null
+ echo done
+}
+
+Main "$@"
diff --git a/node-admin/scripts/node-repo.sh b/node-admin/scripts/node-repo.sh
new file mode 100755
index 00000000000..38b30a4d662
--- /dev/null
+++ b/node-admin/scripts/node-repo.sh
@@ -0,0 +1,318 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -e
+
+# Output from InnerCurlNodeRepo, see there for details.
+declare CURL_RESPONSE
+
+function Usage {
+ cat <<EOF
+Usage: ${0##*/} <command> [<args>...]
+Script for manipulating the Node Repository.
+
+Commands
+ add [-c <configserverhost>] -p <parenthostname> <hostname>...
+ Provision node <hostname> in node repo with flavor "docker".
+ reprovision [-c <configserverhost>] -p <parenthostname> <hostname>...
+ Fail node <hostname>, then rm and add.
+ rm [-c <configserverhost>] <hostname>...
+ Remove nodes from node repo.
+ set-state [-c <configserverhost>] <state> <hostname>...
+ Set the node repo state.
+
+By default, <configserverhost> is config-server.
+
+Example
+ To remove docker-1--1 through docker-1--5 from the node repo at configserver.com,
+
+ ${0##*/} rm -c configserver.com \
+ docker-1--{1,2,3,4,5}.dockerhosts.com
+EOF
+
+ exit 1
+}
+
+function Fail {
+ printf "%s\nRun '${0##*/} help' for usage\n" "$*"
+ exit 1
+}
+
+# Invoke curl such that:
+#
+# - Arguments to this function are passed to curl.
+#
+# - Additional arguments are passed to curl to filter noise (--silent
+# --show-error).
+#
+# - The curl stdout (i.e. the response) is stored in the global CURL_RESPONSE
+# variable on return of the function.
+#
+# - If curl returns 0 (i.e. a successful HTTP response) with a JSON response
+# that contains an "error-code" key, this function will instead return 22. 22
+# does not conflict with a curl return code, because curl only ever returns
+# 22 when --fail is specified.
+#
+# Except, if the JSON response contains a "message" of the form "Cannot add
+# tmp-cnode-0: A node with this name already exists", InnerCurlNodeRepo
+# returns 0 instead of 22, even if the HTTP response status code is an error.
+#
+# Note: Why not use --fail with curl? Because as of 2015-11-24, if the node
+# exists when provisioning a node, the node repo returns a 400 Bad Request
+# with a JSON body containing a "message" field as described above. With
+# --fail, this would result in curl exiting with code 22, which is
+# indistinguishable from other HTTP errors. Can the output from --show-error
+# be used in combination with --fail? No, because that ends up saying "curl:
+# (22) The requested URL returned error: 400 Bad Request" when the node
+# exists, making it indistinguishable from other malformed request error
+# messages.
+#
+# TODO: Make node repo return a unique HTTP error code when node already
+# exists. It's also fragile to test for the error message in the response.
+function InnerCurlNodeRepo {
+ # --show-error causes error message to be printed on error, even with
+ # --silent, which is useful when we print the error message to Fail.
+ local -a command=(curl --silent --show-error "$@")
+
+ # We need the 'if' here, because a non-zero exit code of a command will
+ # exit the process, with 'set -e'.
+ if CURL_RESPONSE=$("${command[@]}" 2>&1)
+ then
+ # Match a JSON of the form:
+ # {
+ # "error-code": "BAD_REQUEST",
+ # "message": "Cannot add cnode-0: A node with this name already exists"
+ # }
+ if [[ "$CURL_RESPONSE" =~ '"error-code"' ]]
+ then
+ if [[ "$CURL_RESPONSE" =~ '"message"'[^\"]*\"(.*)\" ]]
+ then
+ local message="${BASH_REMATCH[1]}"
+ if [[ "$message" =~ 'already exists' ]]
+ then
+ return 0
+ fi
+ fi
+
+ return 22
+ fi
+
+ return 0
+ else
+ # Do not move this statement outside of this else: $? gets cleared when
+ # the execution passes out of the else-fi block.
+ return $?
+ fi
+}
+
+function CurlOrFail {
+ if InnerCurlNodeRepo "$@"
+ then
+ : # This form of if-else is used to preserve $?.
+ else
+ local error_code=$?
+
+ # Terminate the current progress-bar-like line
+ printf ' failed\n'
+
+ Fail "Error ($error_code) from the node repo at '$url': '$CURL_RESPONSE'"
+ fi
+}
+
+function ProvisionDockerNode {
+ local config_server_hostname="$1"
+ local container_hostname="$2"
+ local parent_hostname="$3"
+
+ local url="http://$config_server_hostname:19071/nodes/v2/node"
+
+ local json="[
+ {
+ \"hostname\":\"$container_hostname\",
+ \"parentHostname\":\"$parent_hostname\",
+ \"openStackId\":\"fake-$container_hostname\",
+ \"flavor\":\"docker\"
+ }
+ ]"
+
+ CurlOrFail -H "Content-Type: application/json" -X POST -d "$json" "$url"
+}
+
+function SetNodeState {
+ local config_server_hostname="$1"
+ local hostname="$2"
+ local state="$3"
+
+ local url="http://$config_server_hostname:19071/nodes/v2/state/$state/$hostname"
+ CurlOrFail -X PUT "$url"
+}
+
+function AddCommand {
+ local config_server_hostname=config-server
+ local parent_hostname=
+
+ OPTIND=1
+ local option
+ while getopts "c:p:" option
+ do
+ case "$option" in
+ c) config_server_hostname="$OPTARG" ;;
+ p) parent_hostname="$OPTARG" ;;
+ ?) exit 1 ;; # E.g. option lacks argument, in case error has been
+ # already been printed
+ *) Fail "Unknown option '$option' with value '$OPTARG'"
+ esac
+ done
+
+ if [ -z "$parent_hostname" ]
+ then
+ Fail "Parent hostname not specified (-d)"
+ fi
+
+ shift $((OPTIND - 1))
+
+ if (($# == 0))
+ then
+ Fail "No node hostnames were specified"
+ fi
+
+ echo -n "Provisioning $# nodes"
+
+ local container_hostname
+ for container_hostname in "$@"
+ do
+ ProvisionDockerNode "$config_server_hostname" \
+ "$container_hostname" \
+ "$parent_hostname"
+ echo -n .
+ done
+
+ echo " done"
+}
+
+function ReprovisionCommand {
+ local config_server_hostname=config-server
+ local parent_hostname=
+
+ OPTIND=1
+ local option
+ while getopts "c:p:" option
+ do
+ case "$option" in
+ c) config_server_hostname="$OPTARG" ;;
+ p) parent_hostname="$OPTARG" ;;
+ ?) exit 1 ;; # E.g. option lacks argument, in case error has been
+ # already been printed
+ *) Fail "Unknown option '$option' with value '$OPTARG'"
+ esac
+ done
+
+ if [ -z "$parent_hostname" ]
+ then
+ Fail "Parent hostname not specified (-p)"
+ fi
+
+ shift $((OPTIND - 1))
+
+ if (($# == 0))
+ then
+ Fail "No node hostnames were specified"
+ fi
+
+ # Simulate calls to the following commands.
+ SetStateCommand -c "$config_server_hostname" failed "$@"
+ RemoveCommand -c "$config_server_hostname" "$@"
+ AddCommand -c "$config_server_hostname" -p "$parent_hostname" "$@"
+}
+
+function RemoveCommand {
+ local config_server_hostname=config-server
+
+ OPTIND=1
+ local option
+ while getopts "c:" option
+ do
+ case "$option" in
+ c) config_server_hostname="$OPTARG" ;;
+ ?) exit 1 ;; # E.g. option lacks argument, in case error has been
+ # already been printed
+ *) Fail "Unknown option '$option' with value '$OPTARG'"
+ esac
+ done
+
+ shift $((OPTIND - 1))
+
+ if (($# == 0))
+ then
+ Fail "No nodes were specified"
+ fi
+
+ echo -n "Removing $# nodes"
+
+ local hostname
+ for hostname in "$@"
+ do
+ local url="http://$config_server_hostname:19071/nodes/v2/node/$hostname"
+ CurlOrFail -X DELETE "$url"
+ echo -n .
+ done
+
+ echo " done"
+}
+
+function SetStateCommand {
+ local config_server_hostname=config-server
+
+ OPTIND=1
+ local option
+ while getopts "c:" option
+ do
+ case "$option" in
+ c) config_server_hostname="$OPTARG" ;;
+ ?) exit 1 ;; # E.g. option lacks argument, in case error has been
+ # already been printed
+ *) Fail "Unknown option '$option' with value '$OPTARG'"
+ esac
+ done
+
+ shift $((OPTIND - 1))
+
+ if (($# <= 1))
+ then
+ Fail "Too few arguments"
+ fi
+
+ local state="$1"
+ shift
+
+ echo -n "Setting $# nodes to $state"
+
+ local hostname
+ for hostname in "$@"
+ do
+ SetNodeState "$config_server_hostname" "$hostname" "$state"
+ echo -n .
+ done
+
+ echo " done"
+}
+
+function Main {
+ if (($# == 0))
+ then
+ Usage
+ fi
+ local command="$1"
+ shift
+
+ case "$command" in
+ add) AddCommand "$@" ;;
+ reprovision) ReprovisionCommand "$@" ;;
+ rm) RemoveCommand "$@" ;;
+ set-state) SetStateCommand "$@" ;;
+ help) Usage "$@" ;;
+ *) Usage ;;
+ esac
+}
+
+Main "$@"
diff --git a/node-admin/scripts/populate-noderepo-with-local-nodes.sh b/node-admin/scripts/populate-noderepo-with-local-nodes.sh
new file mode 100755
index 00000000000..6d9a789426d
--- /dev/null
+++ b/node-admin/scripts/populate-noderepo-with-local-nodes.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -e
+
+source "${0%/*}/common.sh"
+
+# Used to return response from RunCurl
+declare CURL_RESPONSE
+
+function Usage {
+ UsageHelper "$@" <<EOF
+Usage: $SCRIPT_NAME <command> [--num-nodes <num-nodes>]
+Add Docker containers as nodes in the node repo, and activate them
+
+Commands:
+ start Add and activate nodes
+ stop Remove nodes (not implemented)
+ restart Stop, then start
+
+Options:
+ --num-nodes <num-nodes>
+ Activate <num-nodes> instead of the default $DEFAULT_NUM_APP_CONTAINERS.
+EOF
+}
+
+function Stop {
+ # TODO: Implement removal of the Docker containers from the node repo
+ :
+}
+
+function Start {
+ local -a hostnames=()
+
+ local -i i=1
+ for ((; i <= $NUM_APP_CONTAINERS; ++i)); do
+ hostnames+=("$APP_HOSTNAME_PREFIX$i")
+ done
+
+ ./node-repo.sh add -c "$CONFIG_SERVER_HOSTNAME" -p "$HOSTNAME" \
+ "${hostnames[@]}"
+}
+
+Main "$@"
diff --git a/node-admin/scripts/pyroute2/__init__.py b/node-admin/scripts/pyroute2/__init__.py
new file mode 100644
index 00000000000..014651fccf9
--- /dev/null
+++ b/node-admin/scripts/pyroute2/__init__.py
@@ -0,0 +1,95 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+##
+# Defer all root imports
+#
+# This allows to safely import config, change it, and
+# only after that actually run imports, though the
+# import statement can be on the top of the file
+#
+# Viva PEP8, morituri te salutant!
+#
+# Surely, you still can import modules directly from their
+# places, like `from pyroute2.iproute import IPRoute`
+##
+__all__ = []
+_modules = {'IPRoute': 'pyroute2.iproute',
+ 'IPSet': 'pyroute2.ipset',
+ 'IPDB': 'pyroute2.ipdb',
+ 'IW': 'pyroute2.iwutil',
+ 'NetNS': 'pyroute2.netns.nslink',
+ 'NSPopen': 'pyroute2.netns.process.proxy',
+ 'IPRSocket': 'pyroute2.netlink.rtnl.iprsocket',
+ 'TaskStats': 'pyroute2.netlink.taskstats',
+ 'NL80211': 'pyroute2.netlink.nl80211',
+ 'IPQSocket': 'pyroute2.netlink.ipq',
+ 'GenericNetlinkSocket': 'pyroute2.netlink.generic',
+ 'NetlinkError': 'pyroute2.netlink'}
+
+_DISCLAIMER = '''\n\nNotice:\n
+This is a proxy class. To read full docs, please run
+the `help()` method on the instance instead.
+
+Usage of the proxy allows to postpone the module load,
+thus providing a safe way to substitute base classes,
+if it is required. More details see in the `pyroute2.config`
+module.
+\n'''
+
+
+class _ProxyMeta(type):
+ '''
+ All this metaclass alchemy is implemented to provide a
+ reasonable, though not exhaustive documentation on the
+ proxy classes.
+ '''
+
+ def __init__(cls, name, bases, dct):
+
+ class doc(str):
+ def __repr__(self):
+ return repr(cls.proxy['doc'])
+
+ def __str__(self):
+ return str(cls.proxy['doc'])
+
+ def expandtabs(self, ts=4):
+ return cls.proxy['doc'].expandtabs(ts)
+
+ class proxy(object):
+ def __init__(self):
+ self.target = {}
+
+ def __getitem__(self, key):
+ if not self.target:
+ module = __import__(_modules[cls.name],
+ globals(),
+ locals(),
+ [cls.name], 0)
+ self.target['constructor'] = getattr(module, cls.name)
+ self.target['doc'] = self.target['constructor'].__doc__
+ try:
+ self.target['doc'] += _DISCLAIMER
+ except TypeError:
+ # ignore cases, when __doc__ is not a string, e.g. None
+ pass
+ return self.target[key]
+
+ def __call__(self, *argv, **kwarg):
+ '''
+ Actually load the module and call the constructor.
+ '''
+ return self.proxy['constructor'](*argv, **kwarg)
+
+ cls.name = name
+ cls.proxy = proxy()
+ cls.__call__ = __call__
+ cls.__doc__ = doc()
+
+ super(_ProxyMeta, cls).__init__(name, bases, dct)
+
+
+for name in _modules:
+
+ f = _ProxyMeta(name, (), {})()
+ globals()[name] = f
+ __all__.append(name)
diff --git a/node-admin/scripts/pyroute2/arp.py b/node-admin/scripts/pyroute2/arp.py
new file mode 100644
index 00000000000..aaa83c7503c
--- /dev/null
+++ b/node-admin/scripts/pyroute2/arp.py
@@ -0,0 +1,69 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+from pyroute2.common import map_namespace
+
+# ARP protocol HARDWARE identifiers.
+ARPHRD_NETROM = 0 # from KA9Q: NET/ROM pseudo
+ARPHRD_ETHER = 1 # Ethernet 10Mbps
+ARPHRD_EETHER = 2 # Experimental Ethernet
+ARPHRD_AX25 = 3 # AX.25 Level 2
+ARPHRD_PRONET = 4 # PROnet token ring
+ARPHRD_CHAOS = 5 # Chaosnet
+ARPHRD_IEEE802 = 6 # IEEE 802.2 Ethernet/TR/TB
+ARPHRD_ARCNET = 7 # ARCnet
+ARPHRD_APPLETLK = 8 # APPLEtalk
+ARPHRD_DLCI = 15 # Frame Relay DLCI
+ARPHRD_ATM = 19 # ATM
+ARPHRD_METRICOM = 23 # Metricom STRIP (new IANA id)
+ARPHRD_IEEE1394 = 24 # IEEE 1394 IPv4 - RFC 2734
+ARPHRD_EUI64 = 27 # EUI-64
+ARPHRD_INFINIBAND = 32 # InfiniBand
+
+# Dummy types for non ARP hardware
+ARPHRD_SLIP = 256
+ARPHRD_CSLIP = 257
+ARPHRD_SLIP6 = 258
+ARPHRD_CSLIP6 = 259
+ARPHRD_RSRVD = 260 # Notional KISS type
+ARPHRD_ADAPT = 264
+ARPHRD_ROSE = 270
+ARPHRD_X25 = 271 # CCITT X.25
+ARPHRD_HWX25 = 272 # Boards with X.25 in firmware
+ARPHRD_PPP = 512
+ARPHRD_CISCO = 513 # Cisco HDLC
+ARPHRD_HDLC = ARPHRD_CISCO
+ARPHRD_LAPB = 516 # LAPB
+ARPHRD_DDCMP = 517 # Digital's DDCMP protocol
+ARPHRD_RAWHDLC = 518 # Raw HDLC
+
+ARPHRD_TUNNEL = 768 # IPIP tunnel
+ARPHRD_TUNNEL6 = 769 # IP6IP6 tunnel
+ARPHRD_FRAD = 770 # Frame Relay Access Device
+ARPHRD_SKIP = 771 # SKIP vif
+ARPHRD_LOOPBACK = 772 # Loopback device
+ARPHRD_LOCALTLK = 773 # Localtalk device
+ARPHRD_FDDI = 774 # Fiber Distributed Data Interface
+ARPHRD_BIF = 775 # AP1000 BIF
+ARPHRD_SIT = 776 # sit0 device - IPv6-in-IPv4
+ARPHRD_IPDDP = 777 # IP over DDP tunneller
+ARPHRD_IPGRE = 778 # GRE over IP
+ARPHRD_PIMREG = 779 # PIMSM register interface
+ARPHRD_HIPPI = 780 # High Performance Parallel Interface
+ARPHRD_ASH = 781 # Nexus 64Mbps Ash
+ARPHRD_ECONET = 782 # Acorn Econet
+ARPHRD_IRDA = 783 # Linux-IrDA
+# ARP works differently on different FC media .. so
+ARPHRD_FCPP = 784 # Point to point fibrechannel
+ARPHRD_FCAL = 785 # Fibrechannel arbitrated loop
+ARPHRD_FCPL = 786 # Fibrechannel public loop
+ARPHRD_FCFABRIC = 787 # Fibrechannel fabric
+# 787->799 reserved for fibrechannel media types
+ARPHRD_IEEE802_TR = 800 # Magic type ident for TR
+ARPHRD_IEEE80211 = 801 # IEEE 802.11
+ARPHRD_IEEE80211_PRISM = 802 # IEEE 802.11 + Prism2 header
+ARPHRD_IEEE80211_RADIOTAP = 803 # IEEE 802.11 + radiotap header
+ARPHRD_MPLS_TUNNEL = 899 # MPLS Tunnel Interface
+
+ARPHRD_VOID = 0xFFFF # Void type, nothing is known
+ARPHRD_NONE = 0xFFFE # zero header length
+
+(ARPHRD_NAMES, ARPHRD_VALUES) = map_namespace("ARPHRD_", globals())
diff --git a/node-admin/scripts/pyroute2/common.py b/node-admin/scripts/pyroute2/common.py
new file mode 100644
index 00000000000..60eae5b88df
--- /dev/null
+++ b/node-admin/scripts/pyroute2/common.py
@@ -0,0 +1,288 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+# -*- coding: utf-8 -*-
+'''
+Common utilities
+'''
+import re
+import os
+import sys
+import types
+import struct
+import platform
+import threading
+
+from socket import inet_aton
+
+try:
+ basestring = basestring
+except NameError:
+ basestring = (str, bytes)
+
+AF_PIPE = 255 # Right now AF_MAX == 40
+DEFAULT_RCVBUF = 16384
+ANCIENT = (platform.dist()[0] in ('redhat', 'centos') and
+ platform.dist()[1].startswith('6.') or
+ os.environ.get('PYROUTE2_ANCIENT', False))
+
+size_suffixes = {'b': 1,
+ 'k': 1024,
+ 'kb': 1024,
+ 'm': 1024 * 1024,
+ 'mb': 1024 * 1024,
+ 'g': 1024 * 1024 * 1024,
+ 'gb': 1024 * 1024 * 1024,
+ 'kbit': 1024 / 8,
+ 'mbit': 1024 * 1024 / 8,
+ 'gbit': 1024 * 1024 * 1024 / 8}
+
+
+time_suffixes = {'s': 1,
+ 'sec': 1,
+ 'secs': 1,
+ 'ms': 1000,
+ 'msec': 1000,
+ 'msecs': 1000,
+ 'us': 1000000,
+ 'usec': 1000000,
+ 'usecs': 1000000}
+
+rate_suffixes = {'bit': 1,
+ 'Kibit': 1024,
+ 'kbit': 1000,
+ 'mibit': 1024 * 1024,
+ 'mbit': 1000000,
+ 'gibit': 1024 * 1024 * 1024,
+ 'gbit': 1000000000,
+ 'tibit': 1024 * 1024 * 1024 * 1024,
+ 'tbit': 1000000000000,
+ 'Bps': 8,
+ 'KiBps': 8 * 1024,
+ 'KBps': 8000,
+ 'MiBps': 8 * 1024 * 1024,
+ 'MBps': 8000000,
+ 'GiBps': 8 * 1024 * 1024 * 1024,
+ 'GBps': 8000000000,
+ 'TiBps': 8 * 1024 * 1024 * 1024 * 1024,
+ 'TBps': 8000000000000}
+
+
+##
+# General purpose
+#
+class Dotkeys(dict):
+ '''
+ This is a sick-minded hack of dict, intended to be an eye-candy.
+ It allows to get dict's items byt dot reference:
+
+ ipdb["lo"] == ipdb.lo
+ ipdb["eth0"] == ipdb.eth0
+
+ Obviously, it will not work for some cases, like unicode names
+ of interfaces and so on. Beside of that, it introduces some
+ complexity.
+
+ But it simplifies live for old-school admins, who works with good
+ old "lo", "eth0", and like that naming schemes.
+ '''
+ var_name = re.compile('^[a-zA-Z_]+[a-zA-Z_0-9]*$')
+
+ def __dir__(self):
+ return [i for i in self if
+ type(i) == str and self.var_name.match(i)]
+
+ def __getattribute__(self, key, *argv):
+ try:
+ return dict.__getattribute__(self, key)
+ except AttributeError as e:
+ if key == '__deepcopy__':
+ raise e
+ return self[key]
+
+ def __setattr__(self, key, value):
+ if key in self:
+ self[key] = value
+ else:
+ dict.__setattr__(self, key, value)
+
+ def __delattr__(self, key):
+ if key in self:
+ del self[key]
+ else:
+ dict.__delattr__(self, key)
+
+
+def map_namespace(prefix, ns, normalize=None):
+ '''
+ Take the namespace prefix, list all constants and build two
+ dictionaries -- straight and reverse mappings. E.g.:
+
+ ## neighbor attributes
+ NDA_UNSPEC = 0
+ NDA_DST = 1
+ NDA_LLADDR = 2
+ NDA_CACHEINFO = 3
+ NDA_PROBES = 4
+ (NDA_NAMES, NDA_VALUES) = map_namespace('NDA', globals())
+
+ Will lead to::
+
+ NDA_NAMES = {'NDA_UNSPEC': 0,
+ ...
+ 'NDA_PROBES': 4}
+ NDA_VALUES = {0: 'NDA_UNSPEC',
+ ...
+ 4: 'NDA_PROBES'}
+
+ The `normalize` parameter can be:
+
+ - None — no name transformation will be done
+ - True — cut the prefix and `lower()` the rest
+ - lambda x: … — apply the function to every name
+
+ '''
+ nmap = {None: lambda x: x,
+ True: lambda x: x[len(prefix):].lower()}
+
+ if not isinstance(normalize, types.FunctionType):
+ normalize = nmap[normalize]
+
+ by_name = dict([(normalize(i), ns[i]) for i in ns.keys()
+ if i.startswith(prefix)])
+ by_value = dict([(ns[i], normalize(i)) for i in ns.keys()
+ if i.startswith(prefix)])
+ return (by_name, by_value)
+
+
+def dqn2int(mask):
+ '''
+ IPv4 dotted quad notation to int mask conversion
+ '''
+ return bin(struct.unpack('>L', inet_aton(mask))[0]).count('1')
+
+
+def hexdump(payload, length=0):
+ '''
+ Represent byte string as hex -- for debug purposes
+ '''
+ if sys.version[0] == '3':
+ return ':'.join('{0:02x}'.format(c)
+ for c in payload[:length] or payload)
+ else:
+ return ':'.join('{0:02x}'.format(ord(c))
+ for c in payload[:length] or payload)
+
+
+class AddrPool(object):
+ '''
+ Address pool
+ '''
+ cell = 0xffffffffffffffff
+
+ def __init__(self,
+ minaddr=0xf,
+ maxaddr=0xffffff,
+ reverse=False,
+ release=False):
+ self.cell_size = 0 # in bits
+ mx = self.cell
+ self.reverse = reverse
+ self.release = release
+ self.allocated = 0
+ if self.release:
+ assert isinstance(self.release, int)
+ self.ban = []
+ while mx:
+ mx >>= 8
+ self.cell_size += 1
+ self.cell_size *= 8
+ # calculate, how many ints we need to bitmap all addresses
+ self.cells = int((maxaddr - minaddr) / self.cell_size + 1)
+ # initial array
+ self.addr_map = [self.cell]
+ self.minaddr = minaddr
+ self.maxaddr = maxaddr
+ self.lock = threading.RLock()
+
+ def alloc(self):
+ with self.lock:
+ # gc self.ban:
+ for item in tuple(self.ban):
+ if item['counter'] == 0:
+ self.free(item['addr'])
+ self.ban.remove(item)
+ else:
+ item['counter'] -= 1
+
+ # iterate through addr_map
+ base = 0
+ for cell in self.addr_map:
+ if cell:
+ # not allocated addr
+ bit = 0
+ while True:
+ if (1 << bit) & self.addr_map[base]:
+ self.addr_map[base] ^= 1 << bit
+ break
+ bit += 1
+ ret = (base * self.cell_size + bit)
+
+ if self.reverse:
+ ret = self.maxaddr - ret
+ else:
+ ret = ret + self.minaddr
+
+ if self.minaddr <= ret <= self.maxaddr:
+ if self.release:
+ self.free(ret, ban=self.release)
+ self.allocated += 1
+ return ret
+ else:
+ self.free(ret)
+ raise KeyError('no free address available')
+
+ base += 1
+ # no free address available
+ if len(self.addr_map) < self.cells:
+ # create new cell to allocate address from
+ self.addr_map.append(self.cell)
+ return self.alloc()
+ else:
+ raise KeyError('no free address available')
+
+ def locate(self, addr):
+ if self.reverse:
+ addr = self.maxaddr - addr
+ else:
+ addr -= self.minaddr
+ base = addr // self.cell_size
+ bit = addr % self.cell_size
+ try:
+ is_allocated = not self.addr_map[base] & (1 << bit)
+ except IndexError:
+ is_allocated = False
+ return (base, bit, is_allocated)
+
+ def setaddr(self, addr, value):
+ assert value in ('free', 'allocated')
+ with self.lock:
+ base, bit, is_allocated = self.locate(addr)
+ if value == 'free' and is_allocated:
+ self.allocated -= 1
+ self.addr_map[base] |= 1 << bit
+ elif value == 'allocated' and not is_allocated:
+ self.allocated += 1
+ self.addr_map[base] &= ~(1 << bit)
+
+ def free(self, addr, ban=0):
+ with self.lock:
+ if ban != 0:
+ self.ban.append({'addr': addr,
+ 'counter': ban})
+ else:
+ base, bit, is_allocated = self.locate(addr)
+ if len(self.addr_map) <= base:
+ raise KeyError('address is not allocated')
+ if self.addr_map[base] & (1 << bit):
+ raise KeyError('address is not allocated')
+ self.allocated -= 1
+ self.addr_map[base] ^= 1 << bit
diff --git a/node-admin/scripts/pyroute2/config.py b/node-admin/scripts/pyroute2/config.py
new file mode 100644
index 00000000000..0dbbbae6089
--- /dev/null
+++ b/node-admin/scripts/pyroute2/config.py
@@ -0,0 +1,10 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+import socket
+import multiprocessing
+
+SocketBase = socket.socket
+MpPipe = multiprocessing.Pipe
+MpQueue = multiprocessing.Queue
+MpProcess = multiprocessing.Process
+
+commit_barrier = 0.2
diff --git a/node-admin/scripts/pyroute2/debugger.py b/node-admin/scripts/pyroute2/debugger.py
new file mode 100644
index 00000000000..1e63644cada
--- /dev/null
+++ b/node-admin/scripts/pyroute2/debugger.py
@@ -0,0 +1,85 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+import select
+import socket
+import struct
+import threading
+from pyroute2.iproute import IPRoute
+try:
+ from Queue import Queue
+except ImportError:
+ from queue import Queue
+
+
+class Server(object):
+
+ def __init__(self, addr='0.0.0.0', port=3546):
+ self.addr = addr
+ self.port = port
+
+ def run(self):
+ nat = {}
+ clients = []
+
+ srv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ srv.bind((self.addr, self.port))
+ ipr = IPRoute()
+ ipr.bind()
+
+ poll = select.poll()
+ poll.register(ipr, select.POLLIN | select.POLLPRI)
+ poll.register(srv, select.POLLIN | select.POLLPRI)
+
+ while True:
+ events = poll.poll()
+ for (fd, event) in events:
+ if fd == ipr.fileno():
+ bufsize = ipr.getsockopt(socket.SOL_SOCKET,
+ socket.SO_RCVBUF) // 2
+ data = ipr.recv(bufsize)
+ cookie = struct.unpack('I', data[8:12])[0]
+ if cookie == 0:
+ for address in clients:
+ srv.sendto(data, address)
+ else:
+ srv.sendto(data, nat[cookie])
+ else:
+ data, address = srv.recvfrom(16384)
+ if data is None:
+ clients.remove(address)
+ continue
+ cookie = struct.unpack('I', data[8:12])[0]
+ nat[cookie] = address
+ ipr.sendto(data, (0, 0))
+
+
+class Client(IPRoute):
+
+ def __init__(self, addr):
+ IPRoute.__init__(self)
+ self.proxy = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ self.proxy.bind(('0.0.0.0', 3547))
+ self.proxy_addr = addr
+ self.proxy_queue = Queue()
+
+ def recv():
+ while True:
+ (data, addr) = self.proxy.recvfrom(16384)
+ self.proxy_queue.put(data)
+
+ self.pthread = threading.Thread(target=recv)
+ self.pthread.setDaemon(True)
+ self.pthread.start()
+
+ def sendto(buf, *argv, **kwarg):
+ return self.proxy.sendto(buf, (self.proxy_addr, 3546))
+
+ def recv(*argv, **kwarg):
+ return self.proxy_queue.get()
+
+ self._sendto = sendto
+ self._recv = recv
+
+ def close(self):
+ self.proxy.close()
+ self.recv = lambda *x, **y: None
+ self.proxy_queue.put(None)
diff --git a/node-admin/scripts/pyroute2/dhcp/__init__.py b/node-admin/scripts/pyroute2/dhcp/__init__.py
new file mode 100644
index 00000000000..46dc26dadef
--- /dev/null
+++ b/node-admin/scripts/pyroute2/dhcp/__init__.py
@@ -0,0 +1,300 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+DHCP protocol
+=============
+
+The DHCP implementation here is far from complete, but
+already provides some basic functionality. Later it will
+be extended with IPv6 support and more DHCP options
+will be added.
+
+Right now it can be interesting mostly to developers,
+but not users and/or system administrators. So, the
+development hints first.
+
+The packet structure description is intentionally
+implemented as for netlink packets. Later these two
+parsers, netlink and generic, can be merged, so the
+syntax is more or less compatible.
+
+Packet fields
+-------------
+
+There are two big groups of items within any DHCP packet.
+First, there are BOOTP/DHCP packet fields, they're defined
+with the `fields` attribute::
+
+ class dhcp4msg(msg):
+ fields = ((name, format, policy),
+ (name, format, policy),
+ ...
+ (name, format, policy))
+
+The `name` can be any literal. Format should be specified
+as for the struct module, like `B` for `uint8`, or `i` for
+`int32`, or `>Q` for big-endian uint64. There are also
+aliases defined, so one can write `uint8` or `be16`, or
+like that. Possible aliases can be seen in the
+`pyroute2.protocols` module.
+
+The `policy` is a bit complicated. It can be a number or
+literal, and it will mean that it is a default value, that
+should be encoded if no other value is given.
+
+But when the `policy` is a dictionary, it can contain keys
+as follows::
+
+ 'l2addr': {'format': '6B',
+ 'decode': ...,
+ 'encode': ...}
+
+Keys `encode` and `decode` should contain filters to be used
+in decoding and encoding procedures. The encoding filter
+should accept the value from user's definition and should
+return a value that can be packed using `format`. The decoding
+filter should accept a value, decoded according to `format`,
+and should return value that can be used by a user.
+
+The `struct` module can not decode IP addresses etc, so they
+should be decoded as `4s`, e.g. Further transformation from
+4 bytes string to a string like '10.0.0.1' performs the filter.
+
+DHCP options
+------------
+
+DHCP options are described in a similar way::
+
+ options = ((code, name, format),
+ (code, name, format),
+ ...
+ (code, name, format))
+
+Code is a `uint8` value, name can be any string literal. Format
+is a string, that must have a corresponding class, inherited from
+`pyroute2.dhcp.option`. One can find these classes in
+`pyroute2.dhcp` (more generic) or in `pyroute2.dhcp.dhcp4msg`
+(IPv4-specific). The option class must reside within dhcp message
+class.
+
+Every option class can be decoded in two ways. If it has fixed
+width fields, it can be decoded with ordinary `msg` routines, and
+in this case it can look like that::
+
+ class client_id(option):
+ fields = (('type', 'uint8'),
+ ('key', 'l2addr'))
+
+If it must be decoded by some custom rules, one can define the
+policy just like for the fields above::
+
+ class array8(option):
+ policy = {'format': 'string',
+ 'encode': lambda x: array('B', x).tobytes(),
+ 'decode': lambda x: array('B', x).tolist()}
+
+In the corresponding modules, like in `pyroute2.dhcp.dhcp4msg`,
+one can define as many custom DHCP options, as one need. Just
+be sure, that they are compatible with the DHCP server and all
+fit into 1..254 (`uint8`) -- the 0 code is used for padding and
+the code 255 is the end of options code.
+'''
+
+import sys
+import struct
+from array import array
+from pyroute2.common import basestring
+from pyroute2.protocols import msg
+
+BOOTREQUEST = 1
+BOOTREPLY = 2
+
+DHCPDISCOVER = 1
+DHCPOFFER = 2
+DHCPREQUEST = 3
+DHCPDECLINE = 4
+DHCPACK = 5
+DHCPNAK = 6
+DHCPRELEASE = 7
+DHCPINFORM = 8
+
+
+if not hasattr(array, 'tobytes'):
+ # Python2 and Python3 versions of array differ,
+ # but we need here a consistent API w/o warnings
+ class array(array):
+ tobytes = array.tostring
+
+
+class option(msg):
+
+ code = 0
+ data_length = 0
+ policy = None
+ value = None
+
+ def __init__(self, content=None, buf=b'', offset=0, value=None, code=0):
+ msg.__init__(self, content=content, buf=buf,
+ offset=offset, value=value)
+ self.code = code
+
+ @property
+ def length(self):
+ if self.data_length is None:
+ return None
+ if self.data_length == 0:
+ return 1
+ else:
+ return self.data_length + 2
+
+ def encode(self):
+ # pack code
+ self.buf += struct.pack('B', self.code)
+ if self.code in (0, 255):
+ return self
+ # save buf
+ save = self.buf
+ self.buf = b''
+ # pack data into the new buf
+ if self.policy is not None:
+ value = self.policy.get('encode', lambda x: x)(self.value)
+ if self.policy['format'] == 'string':
+ fmt = '%is' % len(value)
+ else:
+ fmt = self.policy['format']
+ if sys.version_info[0] == 3 and isinstance(value, str):
+ value = value.encode('utf-8')
+ self.buf = struct.pack(fmt, value)
+ else:
+ msg.encode(self)
+ # get the length
+ data = self.buf
+ self.buf = save
+ self.buf += struct.pack('B', len(data))
+ # attach the packed data
+ self.buf += data
+ return self
+
+ def decode(self):
+ if self.policy is not None:
+ self.data_length = struct.unpack('B', self.buf[self.offset + 1:
+ self.offset + 2])[0]
+ if self.policy['format'] == 'string':
+ fmt = '%is' % self.data_length
+ else:
+ fmt = self.policy['format']
+ value = struct.unpack(fmt, self.buf[self.offset + 2:
+ self.offset + 2 +
+ self.data_length])
+ if len(value) == 1:
+ value = value[0]
+ value = self.policy.get('decode', lambda x: x)(value)
+ if isinstance(value, basestring) and \
+ self.policy['format'] == 'string':
+ value = value[:value.find('\x00')]
+ self.value = value
+ else:
+ msg.decode(self)
+ return self
+
+
+class dhcpmsg(msg):
+ options = ()
+ l2addr = None
+ _encode_map = {}
+ _decode_map = {}
+
+ def _register_options(self):
+ for option in self.options:
+ code, name, fmt = option[:3]
+ self._decode_map[code] =\
+ self._encode_map[name] = {'name': name,
+ 'code': code,
+ 'format': fmt}
+
+ def decode(self):
+ msg.decode(self)
+ self._register_options()
+ self['options'] = {}
+ while self.offset < len(self.buf):
+ code = struct.unpack('B', self.buf[self.offset:self.offset + 1])[0]
+ if code == 0:
+ self.offset += 1
+ continue
+ if code == 255:
+ return self
+ # code is unknown -- bypass it
+ if code not in self._decode_map:
+ length = struct.unpack('B', self.buf[self.offset + 1:
+ self.offset + 2])[0]
+ self.offset += length + 2
+ continue
+
+ # code is known, work on it
+ option_class = getattr(self, self._decode_map[code]['format'])
+ option = option_class(buf=self.buf, offset=self.offset)
+ option.decode()
+ self.offset += option.length
+ if option.value is not None:
+ value = option.value
+ else:
+ value = option
+ self['options'][self._decode_map[code]['name']] = value
+ return self
+
+ def encode(self):
+ msg.encode(self)
+ self._register_options()
+ # put message type
+ options = self.get('options') or {'message_type': DHCPDISCOVER,
+ 'parameter_list': [1, 3, 6,
+ 12, 15, 28]}
+
+ self.buf += self.uint8(code=53,
+ value=options['message_type']).encode().buf
+ self.buf += self.client_id({'type': 1,
+ 'key': self['chaddr']},
+ code=61).encode().buf
+ self.buf += self.string(code=60, value='pyroute2').encode().buf
+
+ for (name, value) in options.items():
+ if name in ('message_type', 'client_id', 'vendor_id'):
+ continue
+ fmt = self._encode_map.get(name, {'format': None})['format']
+ if fmt is None:
+ continue
+ # name is known, ok
+ option_class = getattr(self, fmt)
+ if isinstance(value, dict):
+ option = option_class(value,
+ code=self._encode_map[name]['code'])
+ else:
+ option = option_class(code=self._encode_map[name]['code'],
+ value=value)
+ self.buf += option.encode().buf
+
+ self.buf += self.none(code=255).encode().buf
+ return self
+
+ class none(option):
+ pass
+
+ class be16(option):
+ policy = {'format': '>H'}
+
+ class be32(option):
+ policy = {'format': '>I'}
+
+ class uint8(option):
+ policy = {'format': 'B'}
+
+ class string(option):
+ policy = {'format': 'string'}
+
+ class array8(option):
+ policy = {'format': 'string',
+ 'encode': lambda x: array('B', x).tobytes(),
+ 'decode': lambda x: array('B', x).tolist()}
+
+ class client_id(option):
+ fields = (('type', 'uint8'),
+ ('key', 'l2addr'))
diff --git a/node-admin/scripts/pyroute2/dhcp/dhcp4msg.py b/node-admin/scripts/pyroute2/dhcp/dhcp4msg.py
new file mode 100644
index 00000000000..4cd27c5d75c
--- /dev/null
+++ b/node-admin/scripts/pyroute2/dhcp/dhcp4msg.py
@@ -0,0 +1,60 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+from socket import inet_pton
+from socket import inet_ntop
+from socket import AF_INET
+from pyroute2.dhcp import dhcpmsg
+from pyroute2.dhcp import option
+
+
+class dhcp4msg(dhcpmsg):
+ #
+ # https://www.ietf.org/rfc/rfc2131.txt
+ #
+ fields = (('op', 'uint8', 1), # request
+ ('htype', 'uint8', 1), # ethernet
+ ('hlen', 'uint8', 6), # ethernet addr len
+ ('hops', 'uint8'),
+ ('xid', 'uint32'),
+ ('secs', 'uint16'),
+ ('flags', 'uint16'),
+ ('ciaddr', 'ip4addr'),
+ ('yiaddr', 'ip4addr'),
+ ('siaddr', 'ip4addr'),
+ ('giaddr', 'ip4addr'),
+ ('chaddr', 'l2paddr'),
+ ('sname', '64s'),
+ ('file', '128s'),
+ ('cookie', '4s', b'c\x82Sc'))
+ #
+ # https://www.ietf.org/rfc/rfc2132.txt
+ #
+ options = ((0, 'pad', 'none'),
+ (1, 'subnet_mask', 'ip4addr'),
+ (2, 'time_offset', 'be32'),
+ (3, 'router', 'ip4list'),
+ (4, 'time_server', 'ip4list'),
+ (5, 'ien_name_server', 'ip4list'),
+ (6, 'name_server', 'ip4list'),
+ (7, 'log_server', 'ip4list'),
+ (8, 'cookie_server', 'ip4list'),
+ (9, 'lpr_server', 'ip4list'),
+ (50, 'requested_ip', 'ip4addr'),
+ (53, 'message_type', 'uint8'),
+ (54, 'server_id', 'ip4addr'),
+ (55, 'parameter_list', 'array8'),
+ (57, 'messagi_size', 'be16'),
+ (60, 'vendor_id', 'string'),
+ (61, 'client_id', 'client_id'),
+ (255, 'end', 'none'))
+
+ class ip4addr(option):
+ policy = {'format': '4s',
+ 'encode': lambda x: inet_pton(AF_INET, x),
+ 'decode': lambda x: inet_ntop(AF_INET, x)}
+
+ class ip4list(option):
+ policy = {'format': 'string',
+ 'encode': lambda x: ''.join([inet_pton(AF_INET, i) for i
+ in x]),
+ 'decode': lambda x: [inet_ntop(AF_INET, x[i*4:i*4+4]) for i
+ in range(len(x)//4)]}
diff --git a/node-admin/scripts/pyroute2/dhcp/dhcp4socket.py b/node-admin/scripts/pyroute2/dhcp/dhcp4socket.py
new file mode 100644
index 00000000000..7928d5a0745
--- /dev/null
+++ b/node-admin/scripts/pyroute2/dhcp/dhcp4socket.py
@@ -0,0 +1,135 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+IPv4 DHCP socket
+================
+
+'''
+from pyroute2.common import AddrPool
+from pyroute2.protocols import udpmsg
+from pyroute2.protocols import udp4_pseudo_header
+from pyroute2.protocols import ethmsg
+from pyroute2.protocols import ip4msg
+from pyroute2.protocols.rawsocket import RawSocket
+from pyroute2.dhcp.dhcp4msg import dhcp4msg
+
+
+def listen_udp_port(port=68):
+ # pre-scripted BPF code that matches UDP port
+ bpf_code = [[40, 0, 0, 12],
+ [21, 0, 8, 2048],
+ [48, 0, 0, 23],
+ [21, 0, 6, 17],
+ [40, 0, 0, 20],
+ [69, 4, 0, 8191],
+ [177, 0, 0, 14],
+ [72, 0, 0, 16],
+ [21, 0, 1, port],
+ [6, 0, 0, 65535],
+ [6, 0, 0, 0]]
+ return bpf_code
+
+
+class DHCP4Socket(RawSocket):
+ '''
+ Parameters:
+
+ * ifname -- interface name to work on
+
+ This raw socket binds to an interface and installs BPF filter
+ to get only its UDP port. It can be used in poll/select and
+ provides also the context manager protocol, so can be used in
+ `with` statements.
+
+ It does not provide any DHCP state machine, and does not inspect
+ DHCP packets, it is totally up to you. No default values are
+ provided here, except `xid` -- DHCP transaction ID. If `xid` is
+ not provided, DHCP4Socket generates it for outgoing messages.
+ '''
+
+ def __init__(self, ifname, port=68):
+ RawSocket.__init__(self, ifname, listen_udp_port(port))
+ self.port = port
+ # Create xid pool
+ #
+ # Every allocated xid will be released automatically after 1024
+ # alloc() calls, there is no need to call free(). Minimal xid == 16
+ self.xid_pool = AddrPool(minaddr=16, release=1024)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.close()
+
+ def put(self, msg=None, dport=67):
+ '''
+ Put DHCP message. Parameters:
+
+ * msg -- dhcp4msg instance
+ * dport -- DHCP server port
+
+ If `msg` is not provided, it is constructed as default
+ BOOTREQUEST + DHCPDISCOVER.
+
+ Examples::
+
+ sock.put(dhcp4msg({'op': BOOTREQUEST,
+ 'chaddr': 'ff:11:22:33:44:55',
+ 'options': {'message_type': DHCPREQUEST,
+ 'parameter_list': [1, 3, 6, 12, 15],
+ 'requested_ip': '172.16.101.2',
+ 'server_id': '172.16.101.1'}}))
+
+ The method returns dhcp4msg that was sent, so one can get from
+ there `xid` (transaction id) and other details.
+ '''
+ # DHCP layer
+ dhcp = msg or dhcp4msg({'chaddr': self.l2addr})
+
+ # dhcp transaction id
+ if dhcp['xid'] is None:
+ dhcp['xid'] = self.xid_pool.alloc()
+
+ data = dhcp.encode().buf
+
+ # UDP layer
+ udp = udpmsg({'sport': self.port,
+ 'dport': dport,
+ 'len': 8 + len(data)})
+ udph = udp4_pseudo_header({'dst': '255.255.255.255',
+ 'len': 8 + len(data)})
+ udp['csum'] = self.csum(udph.encode().buf + udp.encode().buf + data)
+ udp.reset()
+
+ # IPv4 layer
+ ip4 = ip4msg({'len': 20 + 8 + len(data),
+ 'proto': 17,
+ 'dst': '255.255.255.255'})
+ ip4['csum'] = self.csum(ip4.encode().buf)
+ ip4.reset()
+
+ # MAC layer
+ eth = ethmsg({'dst': 'ff:ff:ff:ff:ff:ff',
+ 'src': self.l2addr,
+ 'type': 0x800})
+
+ data = eth.encode().buf +\
+ ip4.encode().buf +\
+ udp.encode().buf +\
+ data
+ self.send(data)
+ dhcp.reset()
+ return dhcp
+
+ def get(self):
+ '''
+ Get the next incoming packet from the socket and try
+ to decode it as IPv4 DHCP. No analysis is done here,
+ only MAC/IPv4/UDP headers are stripped out, and the
+ rest is interpreted as DHCP.
+ '''
+ (data, addr) = self.recvfrom(4096)
+ eth = ethmsg(buf=data).decode()
+ ip4 = ip4msg(buf=data, offset=eth.offset).decode()
+ udp = udpmsg(buf=data, offset=ip4.offset).decode()
+ return dhcp4msg(buf=data, offset=udp.offset).decode()
diff --git a/node-admin/scripts/pyroute2/ipdb/__init__.py b/node-admin/scripts/pyroute2/ipdb/__init__.py
new file mode 100644
index 00000000000..5887bb61bb2
--- /dev/null
+++ b/node-admin/scripts/pyroute2/ipdb/__init__.py
@@ -0,0 +1,981 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+# -*- coding: utf-8 -*-
+'''
+IPDB module
+===========
+
+Basically, IPDB is a transactional database, containing records,
+representing network stack objects. Any change in the database
+is not reflected immediately in OS (unless you ask for that
+explicitly), but waits until `commit()` is called. One failed
+operation during `commit()` rolls back all the changes, has been
+made so far. Moreover, IPDB has commit hooks API, that allows
+you to roll back changes depending on your own function calls,
+e.g. when a host or a network becomes unreachable.
+
+IPDB vs. IPRoute
+----------------
+
+These two modules, IPRoute and IPDB, use completely different
+approaches. The first one, IPRoute, is synchronous by default,
+and can be used in the same way, as usual Linux utilities. It
+doesn't spawn any additional threads or processes, until you
+explicitly ask for that.
+
+The latter, IPDB, is an asynchronously updated database, that
+starts several additional threads by default. If your project's
+policy doesn't allow implicit threads, keep it in mind.
+
+The choice depends on your project's workflow. If you plan to
+retrieve the system info not too often (or even once), or you
+are sure there will be not too many network object, it is better
+to use IPRoute. If you plan to lookup the network info on a
+regular basis and there can be loads of network objects, it is
+better to use IPDB. Why?
+
+IPRoute just loads what you ask -- and loads all the information
+you ask to. While IPDB loads all the info upon startup, and
+later is just updated by asynchronous broadcast netlink messages.
+Assume you want to lookup ARP cache that contains hundreds or
+even thousands of objects. Using IPRoute, you have to load all
+the ARP cache every time you want to make a lookup. While IPDB
+will load all the cache once, and then maintain it up-to-date
+just inserting new records or removing them by one.
+
+So, IPRoute is much simpler when you need to make a call and
+then exit. While IPDB is cheaper in terms of CPU performance
+if you implement a long-running program like a daemon. Later
+it can change, if there will be (an optional) cache for IPRoute
+too.
+
+quickstart
+----------
+
+Simple tutorial::
+
+ from pyroute2 import IPDB
+ # several IPDB instances are supported within on process
+ ip = IPDB()
+
+ # commit is called automatically upon the exit from `with`
+ # statement
+ with ip.interfaces.eth0 as i:
+ i.address = '00:11:22:33:44:55'
+ i.ifname = 'bala'
+ i.txqlen = 2000
+
+ # basic routing support
+ ip.routes.add({'dst': 'default', 'gateway': '10.0.0.1'}).commit()
+
+ # do not forget to shutdown IPDB
+ ip.release()
+
+Please, notice `ip.release()` call in the end. Though it is
+not forced in an interactive python session for the better
+user experience, it is required in the scripts to sync the
+IPDB state before exit.
+
+IPDB uses IPRoute as a transport, and monitors all broadcast
+netlink messages from the kernel, thus keeping the database
+up-to-date in an asynchronous manner. IPDB inherits `dict`
+class, and has two keys::
+
+ >>> from pyroute2 import IPDB
+ >>> ip = IPDB()
+ >>> ip.by_name.keys()
+ ['bond0', 'lo', 'em1', 'wlan0', 'dummy0', 'virbr0-nic', 'virbr0']
+ >>> ip.by_index.keys()
+ [32, 1, 2, 3, 4, 5, 8]
+ >>> ip.interfaces.keys()
+ [32,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 8,
+ 'lo',
+ 'em1',
+ 'wlan0',
+ 'bond0',
+ 'dummy0',
+ 'virbr0-nic',
+ 'virbr0']
+ >>> ip.interfaces['em1']['address']
+ 'f0:de:f1:93:94:0d'
+ >>> ip.interfaces['em1']['ipaddr']
+ [('10.34.131.210', 23),
+ ('2620:52:0:2282:f2de:f1ff:fe93:940d', 64),
+ ('fe80::f2de:f1ff:fe93:940d', 64)]
+ >>>
+
+One can address objects in IPDB not only with dict notation, but
+with dot notation also::
+
+ >>> ip.interfaces.em1.address
+ 'f0:de:f1:93:94:0d'
+ >>> ip.interfaces.em1.ipaddr
+ [('10.34.131.210', 23),
+ ('2620:52:0:2282:f2de:f1ff:fe93:940d', 64),
+ ('fe80::f2de:f1ff:fe93:940d', 64)]
+ ```
+
+It is up to you, which way to choose. The former, being more flexible,
+is better for developers, the latter, the shorter form -- for system
+administrators.
+
+
+The library has also IPDB module. It is a database synchronized with
+the kernel, containing some of the information. It can be used also
+to set up IP settings in a transactional manner:
+
+ >>> from pyroute2 import IPDB
+ >>> from pprint import pprint
+ >>> ip = IPDB()
+ >>> pprint(ip.by_name.keys())
+ ['bond0',
+ 'lo',
+ 'vnet0',
+ 'em1',
+ 'wlan0',
+ 'macvtap0',
+ 'dummy0',
+ 'virbr0-nic',
+ 'virbr0']
+ >>> ip.interfaces.lo
+ {'promiscuity': 0,
+ 'operstate': 'UNKNOWN',
+ 'qdisc': 'noqueue',
+ 'group': 0,
+ 'family': 0,
+ 'index': 1,
+ 'linkmode': 0,
+ 'ipaddr': [('127.0.0.1', 8), ('::1', 128)],
+ 'mtu': 65536,
+ 'broadcast': '00:00:00:00:00:00',
+ 'num_rx_queues': 1,
+ 'txqlen': 0,
+ 'ifi_type': 772,
+ 'address': '00:00:00:00:00:00',
+ 'flags': 65609,
+ 'ifname': 'lo',
+ 'num_tx_queues': 1,
+ 'ports': [],
+ 'change': 0}
+ >>>
+
+transaction modes
+-----------------
+IPDB has several operating modes:
+
+ - 'direct' -- any change goes immediately to the OS level
+ - 'implicit' (default) -- the first change starts an implicit
+ transaction, that have to be committed
+ - 'explicit' -- you have to begin() a transaction prior to
+ make any change
+ - 'snapshot' -- no changes will go to the OS in any case
+
+The default is to use implicit transaction. This behaviour can
+be changed in the future, so use 'mode' argument when creating
+IPDB instances.
+
+The sample session with explicit transactions::
+
+ In [1]: from pyroute2 import IPDB
+ In [2]: ip = IPDB(mode='explicit')
+ In [3]: ifdb = ip.interfaces
+ In [4]: ifdb.tap0.begin()
+ Out[3]: UUID('7a637a44-8935-4395-b5e7-0ce40d31d937')
+ In [5]: ifdb.tap0.up()
+ In [6]: ifdb.tap0.address = '00:11:22:33:44:55'
+ In [7]: ifdb.tap0.add_ip('10.0.0.1', 24)
+ In [8]: ifdb.tap0.add_ip('10.0.0.2', 24)
+ In [9]: ifdb.tap0.review()
+ Out[8]:
+ {'+ipaddr': set([('10.0.0.2', 24), ('10.0.0.1', 24)]),
+ '-ipaddr': set([]),
+ 'address': '00:11:22:33:44:55',
+ 'flags': 4099}
+ In [10]: ifdb.tap0.commit()
+
+
+Note, that you can `review()` the `last()` transaction, and
+`commit()` or `drop()` it. Also, multiple `self._transactions`
+are supported, use uuid returned by `begin()` to identify them.
+
+Actually, the form like 'ip.tap0.address' is an eye-candy. The
+IPDB objects are dictionaries, so you can write the code above
+as that::
+
+ ip.interfaces['tap0'].down()
+ ip.interfaces['tap0']['address'] = '00:11:22:33:44:55'
+ ...
+
+context managers
+----------------
+
+Also, interface objects in transactional mode can operate as
+context managers::
+
+ with ip.interfaces.tap0 as i:
+ i.address = '00:11:22:33:44:55'
+ i.ifname = 'vpn'
+ i.add_ip('10.0.0.1', 24)
+ i.add_ip('10.0.0.1', 24)
+
+On exit, the context manager will authomatically `commit()` the
+transaction.
+
+create interfaces
+-----------------
+
+IPDB can also create interfaces::
+
+ with ip.create(kind='bridge', ifname='control') as i:
+ i.add_port(ip.interfaces.eth1)
+ i.add_port(ip.interfaces.eth2)
+ i.add_ip('10.0.0.1/24') # the same as i.add_ip('10.0.0.1', 24)
+
+IPDB supports many interface types, see docs below for the
+`IPDB.create()` method.
+
+routing management
+------------------
+
+IPDB has a simple yet useful routing management interface.
+To add a route, one can use almost any syntax::
+
+ # spec as a dictionary
+ spec = {'dst': '172.16.1.0/24',
+ 'oif': 4,
+ 'gateway': '192.168.122.60',
+ 'metrics': {'mtu': 1400,
+ 'advmss': 500}}
+
+ # pass spec as is
+ ip.routes.add(spec).commit()
+
+ # pass spec as kwargs
+ ip.routes.add(**spec).commit()
+
+ # use keyword arguments explicitly
+ ip.routes.add(dst='172.16.1.0/24', oif=4, ...).commit()
+
+To access and change the routes, one can use notations as follows::
+
+ # default table (254)
+ #
+ # change the route gateway and mtu
+ #
+ with ip.routes['172.16.1.0/24'] as route:
+ route.gateway = '192.168.122.60'
+ route.metrics.mtu = 1500
+
+ # access the default route
+ print(ip.routes['default])
+
+ # change the default gateway
+ with ip.routes['default'] as route:
+ route.gateway = '10.0.0.1'
+
+ # list automatic routes keys
+ print(ip.routes.tables[255].keys())
+
+
+performance issues
+------------------
+
+In the case of bursts of Netlink broadcast messages, all
+the activity of the pyroute2-based code in the async mode
+becomes suppressed to leave more CPU resources to the
+packet reader thread. So please be ready to cope with
+delays in the case of Netlink broadcast storms. It means
+also, that IPDB state will be synchronized with OS also
+after some delay.
+
+classes
+-------
+'''
+import sys
+import atexit
+import logging
+import traceback
+import threading
+
+from socket import AF_INET
+from socket import AF_INET6
+from pyroute2.common import Dotkeys
+from pyroute2.iproute import IPRoute
+from pyroute2.netlink.rtnl import RTM_GETLINK
+from pyroute2.ipdb.common import CreateException
+from pyroute2.ipdb.interface import Interface
+from pyroute2.ipdb.linkedset import LinkedSet
+from pyroute2.ipdb.linkedset import IPaddrSet
+from pyroute2.ipdb.common import compat
+from pyroute2.ipdb.common import SYNC_TIMEOUT
+from pyroute2.ipdb.route import RoutingTableSet
+
+
+def get_addr_nla(msg):
+ '''
+ Utility function to get NLA, containing the interface
+ address.
+
+ Incosistency in Linux IP addressing scheme is that
+ IPv4 uses IFA_LOCAL to store interface's ip address,
+ and IPv6 uses for the same IFA_ADDRESS.
+
+ IPv4 sets IFA_ADDRESS to == IFA_LOCAL or to a
+ tunneling endpoint.
+
+ Args:
+ - msg (nlmsg): RTM\_.*ADDR message
+
+ Returns:
+ - nla (nla): IFA_LOCAL for IPv4 and IFA_ADDRESS for IPv6
+ '''
+ nla = None
+ if msg['family'] == AF_INET:
+ nla = msg.get_attr('IFA_LOCAL')
+ elif msg['family'] == AF_INET6:
+ nla = msg.get_attr('IFA_ADDRESS')
+ return nla
+
+
+class Watchdog(object):
+ def __init__(self, ipdb, action, kwarg):
+ self.event = threading.Event()
+ self.ipdb = ipdb
+
+ def cb(ipdb, msg, _action):
+ if _action != action:
+ return
+
+ for key in kwarg:
+ if (msg.get(key, None) != kwarg[key]) and \
+ (msg.get_attr(msg.name2nla(key)) != kwarg[key]):
+ return
+ self.event.set()
+ self.cb = cb
+ # register callback prior to other things
+ self.ipdb.register_callback(self.cb)
+
+ def wait(self, timeout=SYNC_TIMEOUT):
+ self.event.wait(timeout=timeout)
+ self.cancel()
+
+ def cancel(self):
+ self.ipdb.unregister_callback(self.cb)
+
+
+class IPDB(object):
+ '''
+ The class that maintains information about network setup
+ of the host. Monitoring netlink events allows it to react
+ immediately. It uses no polling.
+ '''
+
+ def __init__(self, nl=None, mode='implicit',
+ restart_on_error=None):
+ '''
+ Parameters:
+ - nl -- IPRoute() reference
+ - mode -- (implicit, explicit, direct)
+ - iclass -- the interface class type
+
+ If you do not provide iproute instance, ipdb will
+ start it automatically.
+ '''
+ self.mode = mode
+ self.iclass = Interface
+ self._stop = False
+ # see also 'register_callback'
+ self._post_callbacks = []
+ self._pre_callbacks = []
+ self._cb_threads = set()
+
+ # locks and events
+ self._links_event = threading.Event()
+ self.exclusive = threading.RLock()
+ self._shutdown_lock = threading.Lock()
+
+ # load information
+ self.restart_on_error = restart_on_error if \
+ restart_on_error is not None else nl is None
+ self.initdb(nl)
+
+ # start monitoring thread
+ self._mthread = threading.Thread(target=self.serve_forever)
+ if hasattr(sys, 'ps1') and self.nl.__class__.__name__ != 'Client':
+ self._mthread.setDaemon(True)
+ self._mthread.start()
+ #
+ atexit.register(self.release)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.release()
+
+ def initdb(self, nl=None):
+ '''
+ Restart IPRoute channel, and create all the DB
+ from scratch. Can be used when sync is lost.
+ '''
+ self.nl = nl or IPRoute()
+ self.nl.monitor = True
+ self.nl.bind(async=True)
+
+ # resolvers
+ self.interfaces = Dotkeys()
+ self.routes = RoutingTableSet(ipdb=self)
+ self.by_name = Dotkeys()
+ self.by_index = Dotkeys()
+
+ # caches
+ self.ipaddr = {}
+ self.neighbors = {}
+
+ # load information
+ links = self.nl.get_links()
+ for link in links:
+ self.device_put(link, skip_slaves=True)
+ for link in links:
+ self.update_slaves(link)
+ self.update_addr(self.nl.get_addr())
+ self.update_neighbors(self.nl.get_neighbors())
+ routes4 = self.nl.get_routes(family=AF_INET)
+ routes6 = self.nl.get_routes(family=AF_INET6)
+ self.update_routes(routes4)
+ self.update_routes(routes6)
+
+ def register_callback(self, callback, mode='post'):
+ '''
+ IPDB callbacks are routines executed on a RT netlink
+ message arrival. There are two types of callbacks:
+ "post" and "pre" callbacks.
+
+ ...
+
+ "Post" callbacks are executed after the message is
+ processed by IPDB and all corresponding objects are
+ created or deleted. Using ipdb reference in "post"
+ callbacks you will access the most up-to-date state
+ of the IP database.
+
+ "Post" callbacks are executed asynchronously in
+ separate threads. These threads can work as long
+ as you want them to. Callback threads are joined
+ occasionally, so for a short time there can exist
+ stopped threads.
+
+ ...
+
+ "Pre" callbacks are synchronous routines, executed
+ before the message gets processed by IPDB. It gives
+ you the way to patch arriving messages, but also
+ places a restriction: until the callback exits, the
+ main event IPDB loop is blocked.
+
+ Normally, only "post" callbacks are required. But in
+ some specific cases "pre" also can be useful.
+
+ ...
+
+ The routine, `register_callback()`, takes two arguments:
+ - callback function
+ - mode (optional, default="post")
+
+ The callback should be a routine, that accepts three
+ arguments::
+
+ cb(ipdb, msg, action)
+
+ Arguments are:
+
+ - **ipdb** is a reference to IPDB instance, that invokes
+ the callback.
+ - **msg** is a message arrived
+ - **action** is just a msg['event'] field
+
+ E.g., to work on a new interface, you should catch
+ action == 'RTM_NEWLINK' and with the interface index
+ (arrived in msg['index']) get it from IPDB::
+
+ index = msg['index']
+ interface = ipdb.interfaces[index]
+ '''
+ lock = threading.Lock()
+
+ def safe(*argv, **kwarg):
+ with lock:
+ callback(*argv, **kwarg)
+
+ safe.hook = callback
+ if mode == 'post':
+ self._post_callbacks.append(safe)
+ elif mode == 'pre':
+ self._pre_callbacks.append(safe)
+
+ def unregister_callback(self, callback, mode='post'):
+ if mode == 'post':
+ cbchain = self._post_callbacks
+ elif mode == 'pre':
+ cbchain = self._pre_callbacks
+ else:
+ raise KeyError('Unknown callback mode')
+ for cb in tuple(cbchain):
+ if callback == cb.hook:
+ for t in tuple(self._cb_threads):
+ t.join(3)
+ return cbchain.pop(cbchain.index(cb))
+
+ def release(self):
+ '''
+ Shutdown IPDB instance and sync the state. Since
+ IPDB is asyncronous, some operations continue in the
+ background, e.g. callbacks. So, prior to exit the
+ script, it is required to properly shutdown IPDB.
+
+ The shutdown sequence is not forced in an interactive
+ python session, since it is easier for users and there
+ is enough time to sync the state. But for the scripts
+ the `release()` call is required.
+ '''
+ with self._shutdown_lock:
+ if self._stop:
+ return
+
+ self._stop = True
+ try:
+ self.nl.put({'index': 1}, RTM_GETLINK)
+ self._mthread.join()
+ except Exception:
+ # Just give up.
+ # We can not handle this case
+ pass
+ self.nl.close()
+ self.nl = None
+
+ # flush all the objects
+ # -- interfaces
+ for key in tuple(self.interfaces.keys()):
+ self.detach(key)
+ # -- routes
+ for key in tuple(self.routes.tables.keys()):
+ del self.routes.tables[key]
+ self.routes.tables[254] = None
+ # -- ipaddr
+ for key in tuple(self.ipaddr.keys()):
+ del self.ipaddr[key]
+ # -- neighbors
+ for key in tuple(self.neighbors.keys()):
+ del self.neighbors[key]
+
+ def create(self, kind, ifname, reuse=False, **kwarg):
+ '''
+ Create an interface. Arguments 'kind' and 'ifname' are
+ required.
+
+ - kind — interface type, can be of:
+ - bridge
+ - bond
+ - vlan
+ - tun
+ - dummy
+ - veth
+ - macvlan
+ - macvtap
+ - gre
+ - team
+ - ovs-bridge
+ - ifname — interface name
+ - reuse — if such interface exists, return it anyway
+
+ Different interface kinds can require different
+ arguments for creation.
+
+ â–º **veth**
+
+ To properly create `veth` interface, one should specify
+ `peer` also, since `veth` interfaces are created in pairs::
+
+ with ip.create(ifname='v1p0', kind='veth', peer='v1p1') as i:
+ i.add_ip('10.0.0.1/24')
+ i.add_ip('10.0.0.2/24')
+
+ The code above creates two interfaces, `v1p0` and `v1p1`, and
+ adds two addresses to `v1p0`.
+
+ â–º **macvlan**
+
+ Macvlan interfaces act like VLANs within OS. The macvlan driver
+ provides an ability to add several MAC addresses on one interface,
+ where every MAC address is reflected with a virtual interface in
+ the system.
+
+ In some setups macvlan interfaces can replace bridge interfaces,
+ providing more simple and at the same time high-performance
+ solution::
+
+ ip.create(ifname='mvlan0',
+ kind='macvlan',
+ link=ip.interfaces.em1,
+ macvlan_mode='private').commit()
+
+ Several macvlan modes are available: 'private', 'vepa', 'bridge',
+ 'passthru'. Ususally the default is 'vepa'.
+
+ â–º **macvtap**
+
+ Almost the same as macvlan, but creates also a character tap device::
+
+ ip.create(ifname='mvtap0',
+ kind='macvtap',
+ link=ip.interfaces.em1,
+ macvtap_mode='vepa').commit()
+
+ Will create a device file `"/dev/tap%s" % ip.interfaces.mvtap0.index`
+
+ â–º **gre**
+
+ Create GRE tunnel::
+
+ with ip.create(ifname='grex',
+ kind='gre',
+ gre_local='172.16.0.1',
+ gre_remote='172.16.0.101',
+ gre_ttl=16) as i:
+ i.add_ip('192.168.0.1/24')
+ i.up()
+
+
+ â–º **vlan**
+
+ VLAN interfaces require additional parameters, `vlan_id` and
+ `link`, where `link` is a master interface to create VLAN on::
+
+ ip.create(ifname='v100',
+ kind='vlan',
+ link=ip.interfaces.eth0,
+ vlan_id=100)
+
+ ip.create(ifname='v100',
+ kind='vlan',
+ link=1,
+ vlan_id=100)
+
+ The `link` parameter should be either integer, interface id, or
+ an interface object. VLAN id must be integer.
+
+ â–º **vxlan**
+
+ VXLAN interfaces are like VLAN ones, but require a bit more
+ parameters::
+
+ ip.create(ifname='vx101',
+ kind='vxlan',
+ vxlan_link=ip.interfaces.eth0,
+ vxlan_id=101,
+ vxlan_group='239.1.1.1',
+ vxlan_ttl=16)
+
+ All possible vxlan parameters are listed in the module
+ `pyroute2.netlink.rtnl.ifinfmsg:... vxlan_data`.
+
+ â–º **tuntap**
+
+ Possible `tuntap` keywords:
+
+ - `mode` — "tun" or "tap"
+ - `uid` — integer
+ - `gid` — integer
+ - `ifr` — dict of tuntap flags (see tuntapmsg.py)
+ '''
+ with self.exclusive:
+ # check for existing interface
+ if ifname in self.interfaces:
+ if self.interfaces[ifname]._flicker or reuse:
+ device = self.interfaces[ifname]
+ device._flicker = False
+ else:
+ raise CreateException("interface %s exists" %
+ ifname)
+ else:
+ device = \
+ self.by_name[ifname] = \
+ self.interfaces[ifname] = \
+ self.iclass(ipdb=self, mode='snapshot')
+ device.update(kwarg)
+ if isinstance(kwarg.get('link', None), Interface):
+ device['link'] = kwarg['link']['index']
+ if isinstance(kwarg.get('vxlan_link', None), Interface):
+ device['vxlan_link'] = kwarg['vxlan_link']['index']
+ device['kind'] = kind
+ device['index'] = kwarg.get('index', 0)
+ device['ifname'] = ifname
+ device._mode = self.mode
+ tid = device.begin()
+ #
+ # All the device methods are handled via `transactional.update()`
+ # except of the very creation.
+ #
+ # Commit the changes in the 'direct' mode, since this call is not
+ # decorated.
+ if self.mode == 'direct':
+ device.commit(tid)
+ return device
+
+ def device_del(self, msg):
+ # check for flicker devices
+ if (msg.get('index', None) in self.interfaces) and \
+ self.interfaces[msg['index']]._flicker:
+ self.interfaces[msg['index']].sync()
+ return
+ try:
+ self.update_slaves(msg)
+ if msg['change'] == 0xffffffff:
+ # FIXME catch exception
+ ifname = self.interfaces[msg['index']]['ifname']
+ self.interfaces[msg['index']].sync()
+ del self.by_name[ifname]
+ del self.by_index[msg['index']]
+ del self.interfaces[ifname]
+ del self.interfaces[msg['index']]
+ del self.ipaddr[msg['index']]
+ del self.neighbors[msg['index']]
+ except KeyError:
+ pass
+
+ def device_put(self, msg, skip_slaves=False):
+ # check, if a record exists
+ index = msg.get('index', None)
+ ifname = msg.get_attr('IFLA_IFNAME', None)
+ # scenario #1: no matches for both: new interface
+ # scenario #2: ifname exists, index doesn't: index changed
+ # scenario #3: index exists, ifname doesn't: name changed
+ # scenario #4: both exist: assume simple update and
+ # an optional name change
+ if ((index not in self.interfaces) and
+ (ifname not in self.interfaces)):
+ # scenario #1, new interface
+ if compat.fix_check_link(self.nl, index):
+ return
+ device = \
+ self.by_index[index] = \
+ self.interfaces[index] = \
+ self.interfaces[ifname] = \
+ self.by_name[ifname] = self.iclass(ipdb=self)
+ elif ((index not in self.interfaces) and
+ (ifname in self.interfaces)):
+ # scenario #2, index change
+ old_index = self.interfaces[ifname]['index']
+ device = \
+ self.interfaces[index] = \
+ self.by_index[index] = self.interfaces[ifname]
+ if old_index in self.interfaces:
+ del self.interfaces[old_index]
+ del self.by_index[old_index]
+ if old_index in self.ipaddr:
+ self.ipaddr[index] = self.ipaddr[old_index]
+ del self.ipaddr[old_index]
+ if old_index in self.neighbors:
+ self.neighbors[index] = self.neighbors[old_index]
+ del self.neighbors[old_index]
+ else:
+ # scenario #3, interface rename
+ # scenario #4, assume rename
+ old_name = self.interfaces[index]['ifname']
+ if old_name != ifname:
+ # unlink old name
+ del self.interfaces[old_name]
+ del self.by_name[old_name]
+ device = \
+ self.interfaces[ifname] = \
+ self.by_name[ifname] = self.interfaces[index]
+
+ if index not in self.ipaddr:
+ # for interfaces, created by IPDB
+ self.ipaddr[index] = IPaddrSet()
+
+ if index not in self.neighbors:
+ self.neighbors[index] = LinkedSet()
+
+ device.load_netlink(msg)
+
+ if not skip_slaves:
+ self.update_slaves(msg)
+
+ def detach(self, item):
+ with self.exclusive:
+ if item in self.interfaces:
+ del self.interfaces[item]
+ if item in self.by_name:
+ del self.by_name[item]
+ if item in self.by_index:
+ del self.by_index[item]
+
+ def watchdog(self, action='RTM_NEWLINK', **kwarg):
+ return Watchdog(self, action, kwarg)
+
+ def update_routes(self, routes):
+ for msg in routes:
+ self.routes.load_netlink(msg)
+
+ def _lookup_master(self, msg):
+ master = None
+ # lookup for IFLA_OVS_MASTER_IFNAME
+ li = msg.get_attr('IFLA_LINKINFO')
+ if li:
+ data = li.get_attr('IFLA_INFO_DATA')
+ if data:
+ try:
+ master = data.get_attr('IFLA_OVS_MASTER_IFNAME')
+ except AttributeError:
+ # IFLA_INFO_DATA can be undecoded, in that case
+ # it will be just a string with a hex dump
+ pass
+ # lookup for IFLA_MASTER
+ if master is None:
+ master = msg.get_attr('IFLA_MASTER')
+ # pls keep in mind, that in the case of IFLA_MASTER
+ # lookup is done via interface index, while in the case
+ # of IFLA_OVS_MASTER_IFNAME lookup is done via ifname
+ return self.interfaces.get(master, None)
+
+ def update_slaves(self, msg):
+ # Update slaves list -- only after update IPDB!
+
+ master = self._lookup_master(msg)
+ index = msg['index']
+ # there IS a master for the interface
+ if master is not None:
+ if msg['event'] == 'RTM_NEWLINK':
+ # TODO tags: ipdb
+ # The code serves one particular case, when
+ # an enslaved interface is set to belong to
+ # another master. In this case there will be
+ # no 'RTM_DELLINK', only 'RTM_NEWLINK', and
+ # we can end up in a broken state, when two
+ # masters refers to the same slave
+ for device in self.by_index:
+ if index in self.interfaces[device]['ports']:
+ self.interfaces[device].del_port(index,
+ direct=True)
+ master.add_port(index, direct=True)
+ elif msg['event'] == 'RTM_DELLINK':
+ if index in master['ports']:
+ master.del_port(index, direct=True)
+ # there is NO masters for the interface, clean them if any
+ else:
+ device = self.interfaces[msg['index']]
+
+ # clean device from ports
+ for master in self.by_index:
+ if index in self.interfaces[master]['ports']:
+ self.interfaces[master].del_port(index,
+ direct=True)
+ master = device.if_master
+ if master is not None:
+ if 'master' in device:
+ device.del_item('master')
+ if (master in self.interfaces) and \
+ (msg['index'] in self.interfaces[master].ports):
+ self.interfaces[master].del_port(msg['index'],
+ direct=True)
+
+ def update_addr(self, addrs, action='add'):
+ # Update address list of an interface.
+
+ for addr in addrs:
+ nla = get_addr_nla(addr)
+ if nla is not None:
+ try:
+ method = getattr(self.ipaddr[addr['index']], action)
+ method(key=(nla, addr['prefixlen']), raw=addr)
+ except:
+ pass
+
+ def update_neighbors(self, neighs, action='add'):
+
+ for neigh in neighs:
+ nla = neigh.get_attr('NDA_DST')
+ if nla is not None:
+ try:
+ method = getattr(self.neighbors[neigh['ifindex']], action)
+ method(key=nla, raw=neigh)
+ except:
+ pass
+
+ def serve_forever(self):
+ '''
+ Main monitoring cycle. It gets messages from the
+ default iproute queue and updates objects in the
+ database.
+
+ .. note::
+ Should not be called manually.
+ '''
+ while not self._stop:
+ try:
+ messages = self.nl.get()
+ ##
+ # Check it again
+ #
+ # NOTE: one should not run callbacks or
+ # anything like that after setting the
+ # _stop flag, since IPDB is not valid
+ # anymore
+ if self._stop:
+ break
+ except:
+ logging.error('Restarting IPDB instance after '
+ 'error:\n%s', traceback.format_exc())
+ if self.restart_on_error:
+ self.initdb()
+ continue
+ else:
+ raise RuntimeError('Emergency shutdown')
+ for msg in messages:
+ # Run pre-callbacks
+ # NOTE: pre-callbacks are synchronous
+ for cb in self._pre_callbacks:
+ try:
+ cb(self, msg, msg['event'])
+ except:
+ pass
+
+ with self.exclusive:
+ # FIXME: refactor it to a dict
+ if msg.get('event', None) == 'RTM_NEWLINK':
+ self.device_put(msg)
+ self._links_event.set()
+ elif msg.get('event', None) == 'RTM_DELLINK':
+ self.device_del(msg)
+ elif msg.get('event', None) == 'RTM_NEWADDR':
+ self.update_addr([msg], 'add')
+ elif msg.get('event', None) == 'RTM_DELADDR':
+ self.update_addr([msg], 'remove')
+ elif msg.get('event', None) == 'RTM_NEWNEIGH':
+ self.update_neighbors([msg], 'add')
+ elif msg.get('event', None) == 'RTM_DELNEIGH':
+ self.update_neighbors([msg], 'remove')
+ elif msg.get('event', None) in ('RTM_NEWROUTE'
+ 'RTM_DELROUTE'):
+ self.update_routes([msg])
+
+ # run post-callbacks
+ # NOTE: post-callbacks are asynchronous
+ for cb in self._post_callbacks:
+ t = threading.Thread(name="callback %s" % (id(cb)),
+ target=cb,
+ args=(self, msg, msg['event']))
+ t.start()
+ self._cb_threads.add(t)
+
+ # occasionally join cb threads
+ for t in tuple(self._cb_threads):
+ t.join(0)
+ if not t.is_alive():
+ self._cb_threads.remove(t)
diff --git a/node-admin/scripts/pyroute2/ipdb/common.py b/node-admin/scripts/pyroute2/ipdb/common.py
new file mode 100644
index 00000000000..0e915e47c64
--- /dev/null
+++ b/node-admin/scripts/pyroute2/ipdb/common.py
@@ -0,0 +1,51 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+import time
+import errno
+from pyroute2.common import ANCIENT
+from pyroute2.netlink import NetlinkError
+# How long should we wait on EACH commit() checkpoint: for ipaddr,
+# ports etc. That's not total commit() timeout.
+SYNC_TIMEOUT = 5
+
+
+class DeprecationException(Exception):
+ pass
+
+
+class CommitException(Exception):
+ pass
+
+
+class CreateException(Exception):
+ pass
+
+
+def bypass(f):
+ if ANCIENT:
+ return f
+ else:
+ return staticmethod(lambda *x, **y: None)
+
+
+class compat(object):
+ '''
+ A namespace to keep all compat-related methods.
+ '''
+ @bypass
+ @staticmethod
+ def fix_timeout(timeout):
+ time.sleep(timeout)
+
+ @bypass
+ @staticmethod
+ def fix_check_link(nl, index):
+ # check, if the link really exists --
+ # on some old kernels you can receive
+ # broadcast RTM_NEWLINK after the link
+ # was deleted
+ try:
+ nl.get_links(index)
+ except NetlinkError as e:
+ if e.code == errno.ENODEV: # No such device
+ # just drop this message then
+ return True
diff --git a/node-admin/scripts/pyroute2/ipdb/interface.py b/node-admin/scripts/pyroute2/ipdb/interface.py
new file mode 100644
index 00000000000..a2c4f72ec1b
--- /dev/null
+++ b/node-admin/scripts/pyroute2/ipdb/interface.py
@@ -0,0 +1,709 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+import time
+import errno
+import socket
+import threading
+import traceback
+from pyroute2 import config
+from pyroute2.common import basestring
+from pyroute2.common import dqn2int
+from pyroute2.netlink import NetlinkError
+from pyroute2.netlink.rtnl.req import IPLinkRequest
+from pyroute2.netlink.rtnl.ifinfmsg import IFF_MASK
+from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg
+from pyroute2.ipdb.transactional import Transactional
+from pyroute2.ipdb.transactional import update
+from pyroute2.ipdb.linkedset import LinkedSet
+from pyroute2.ipdb.linkedset import IPaddrSet
+from pyroute2.ipdb.common import CommitException
+from pyroute2.ipdb.common import SYNC_TIMEOUT
+from pyroute2.ipdb.common import compat
+
+
+class Interface(Transactional):
+ '''
+ Objects of this class represent network interface and
+ all related objects:
+ * addresses
+ * (todo) neighbors
+ * (todo) routes
+
+ Interfaces provide transactional model and can act as
+ context managers. Any attribute change implicitly
+ starts a transaction. The transaction can be managed
+ with three methods:
+ * review() -- review changes
+ * rollback() -- drop all the changes
+ * commit() -- try to apply changes
+
+ If anything will go wrong during transaction commit,
+ it will be rolled back authomatically and an
+ exception will be raised. Failed transaction review
+ will be attached to the exception.
+ '''
+ _fields_cmp = {'flags': lambda x, y: x & y & IFF_MASK == y & IFF_MASK}
+
+ def __init__(self, ipdb, mode=None, parent=None, uid=None):
+ '''
+ Parameters:
+ * ipdb -- ipdb() reference
+ * mode -- transaction mode
+ '''
+ Transactional.__init__(self, ipdb, mode)
+ self.cleanup = ('header',
+ 'linkinfo',
+ 'af_spec',
+ 'attrs',
+ 'event',
+ 'map',
+ 'stats',
+ 'stats64',
+ '__align')
+ self.ingress = None
+ self.egress = None
+ self._exists = False
+ self._flicker = False
+ self._exception = None
+ self._tb = None
+ self._virtual_fields = ('removal', 'flicker', 'state')
+ self._xfields = {'common': [ifinfmsg.nla2name(i[0]) for i
+ in ifinfmsg.nla_map]}
+ self._xfields['common'].append('index')
+ self._xfields['common'].append('flags')
+ self._xfields['common'].append('mask')
+ self._xfields['common'].append('change')
+ self._xfields['common'].append('kind')
+ self._xfields['common'].append('peer')
+ self._xfields['common'].append('vlan_id')
+ self._xfields['common'].append('bond_mode')
+
+ for data in ('bridge_data',
+ 'bond_data',
+ 'tuntap_data',
+ 'vxlan_data',
+ 'gre_data',
+ 'macvlan_data',
+ 'macvtap_data'):
+ msg = getattr(ifinfmsg.ifinfo, data)
+ self._xfields['common'].extend([msg.nla2name(i[0]) for i
+ in msg.nla_map])
+ for ftype in self._xfields:
+ self._fields += self._xfields[ftype]
+ self._fields.extend(self._virtual_fields)
+ self._load_event = threading.Event()
+ self._linked_sets.add('ipaddr')
+ self._linked_sets.add('ports')
+ self._freeze = None
+ # 8<-----------------------------------
+ # local setup: direct state is required
+ with self._direct_state:
+ self['ipaddr'] = IPaddrSet()
+ self['ports'] = LinkedSet()
+ for i in self._fields:
+ self[i] = None
+ for i in ('state', 'change', 'mask'):
+ del self[i]
+ # 8<-----------------------------------
+
+ def __hash__(self):
+ return self['index']
+
+ @property
+ def if_master(self):
+ '''
+ [property] Link to the parent interface -- if it exists
+ '''
+ return self.get('master', None)
+
+ def freeze(self):
+ dump = self.dump()
+
+ def cb(ipdb, msg, action):
+ if msg.get('index', -1) == dump['index']:
+ tr = self.load(dump)
+ for _ in range(3):
+ try:
+ self.commit(transaction=tr)
+ except (CommitException, RuntimeError):
+ # ignore here both CommitExceptions
+ # and RuntimeErrors (aka rollback errors),
+ # since ususally it is a races between
+ # 3d party setup and freeze; just
+ # sliently try again for several times
+ continue
+ except NetlinkError:
+ # on the netlink errors just give up
+ pass
+ break
+
+ self._freeze = cb
+ self.ipdb.register_callback(self._freeze)
+ return self
+
+ def unfreeze(self):
+ self.ipdb.unregister_callback(self._freeze)
+ self._freeze = None
+ return self
+
+ def load(self, data):
+ with self._write_lock:
+ template = self.__class__(ipdb=self.ipdb, mode='snapshot')
+ template.load_dict(data)
+ return template
+
+ def load_dict(self, data):
+ with self._direct_state:
+ for key in data:
+ if key == 'ipaddr':
+ for addr in data[key]:
+ if isinstance(addr, basestring):
+ addr = (addr, )
+ self.add_ip(*addr)
+ elif key == 'ports':
+ for port in data[key]:
+ self.add_port(port)
+ elif key == 'neighbors':
+ # ignore neighbors on load
+ pass
+ else:
+ self[key] = data[key]
+
+ def load_netlink(self, dev):
+ '''
+ Update the interface info from RTM_NEWLINK message.
+
+ This call always bypasses open transactions, loading
+ changes directly into the interface data.
+ '''
+ with self._direct_state:
+ self._exists = True
+ self.nlmsg = dev
+ for (name, value) in dev.items():
+ self[name] = value
+ for item in dev['attrs']:
+ name, value = item[:2]
+ norm = ifinfmsg.nla2name(name)
+ self[norm] = value
+ # load interface kind
+ linkinfo = dev.get_attr('IFLA_LINKINFO')
+ if linkinfo is not None:
+ kind = linkinfo.get_attr('IFLA_INFO_KIND')
+ if kind is not None:
+ self['kind'] = kind
+ if kind == 'vlan':
+ data = linkinfo.get_attr('IFLA_INFO_DATA')
+ self['vlan_id'] = data.get_attr('IFLA_VLAN_ID')
+ if kind in ('vxlan', 'macvlan', 'macvtap', 'gre'):
+ data = linkinfo.get_attr('IFLA_INFO_DATA')
+ for nla in data.get('attrs', []):
+ norm = ifinfmsg.nla2name(nla[0])
+ self[norm] = nla[1]
+ # get OVS master and override IFLA_MASTER value
+ try:
+ data = linkinfo.get_attr('IFLA_INFO_DATA')
+ master = data.get_attr('IFLA_OVS_MASTER_IFNAME')
+ self['master'] = self.ipdb.interfaces[master].index
+ except (AttributeError, KeyError):
+ pass
+ # the rest is possible only when interface
+ # is used in IPDB, not standalone
+ if self.ipdb is not None:
+ self['ipaddr'] = self.ipdb.ipaddr[self['index']]
+ self['neighbors'] = self.ipdb.neighbors[self['index']]
+ # finally, cleanup all not needed
+ for item in self.cleanup:
+ if item in self:
+ del self[item]
+
+ self.sync()
+
+ def sync(self):
+ self._load_event.set()
+
+ @update
+ def add_ip(self, direct, ip, mask=None,
+ brd=None, broadcast=None):
+ '''
+ Add IP address to an interface
+ '''
+ # split mask
+ if mask is None:
+ ip, mask = ip.split('/')
+ if mask.find('.') > -1:
+ mask = dqn2int(mask)
+ else:
+ mask = int(mask, 0)
+ elif isinstance(mask, basestring):
+ mask = dqn2int(mask)
+ brd = brd or broadcast
+ # FIXME: make it more generic
+ # skip IPv6 link-local addresses
+ if ip[:4] == 'fe80' and mask == 64:
+ return self
+ if not direct:
+ transaction = self.last()
+ transaction.add_ip(ip, mask, brd)
+ else:
+ self['ipaddr'].unlink((ip, mask))
+ if brd is not None:
+ raw = {'IFA_BROADCAST': brd}
+ self['ipaddr'].add((ip, mask), raw=raw)
+ else:
+ self['ipaddr'].add((ip, mask))
+ return self
+
+ @update
+ def del_ip(self, direct, ip, mask=None):
+ '''
+ Delete IP address from an interface
+ '''
+ if mask is None:
+ ip, mask = ip.split('/')
+ if mask.find('.') > -1:
+ mask = dqn2int(mask)
+ else:
+ mask = int(mask, 0)
+ if not direct:
+ transaction = self.last()
+ if (ip, mask) in transaction['ipaddr']:
+ transaction.del_ip(ip, mask)
+ else:
+ self['ipaddr'].unlink((ip, mask))
+ self['ipaddr'].remove((ip, mask))
+ return self
+
+ @update
+ def add_port(self, direct, port):
+ '''
+ Add a slave port to a bridge or bonding
+ '''
+ if isinstance(port, Interface):
+ port = port['index']
+ if not direct:
+ transaction = self.last()
+ transaction.add_port(port)
+ else:
+ self['ports'].unlink(port)
+ self['ports'].add(port)
+ return self
+
+ @update
+ def del_port(self, direct, port):
+ '''
+ Remove a slave port from a bridge or bonding
+ '''
+ if isinstance(port, Interface):
+ port = port['index']
+ if not direct:
+ transaction = self.last()
+ if port in transaction['ports']:
+ transaction.del_port(port)
+ else:
+ self['ports'].unlink(port)
+ self['ports'].remove(port)
+ return self
+
+ def reload(self):
+ '''
+ Reload interface information
+ '''
+ countdown = 3
+ while countdown:
+ links = self.nl.get_links(self['index'])
+ if links:
+ self.load_netlink(links[0])
+ break
+ else:
+ countdown -= 1
+ time.sleep(1)
+ return self
+
+ def filter(self, ftype):
+ ret = {}
+ for key in self:
+ if key in self._xfields[ftype]:
+ ret[key] = self[key]
+ return ret
+
+ def commit(self, tid=None, transaction=None, rollback=False, newif=False):
+ '''
+ Commit transaction. In the case of exception all
+ changes applied during commit will be reverted.
+ '''
+ error = None
+ added = None
+ removed = None
+ drop = True
+ if tid:
+ transaction = self._transactions[tid]
+ else:
+ if transaction:
+ drop = False
+ else:
+ transaction = self.last()
+
+ wd = None
+ with self._write_lock:
+ # if the interface does not exist, create it first ;)
+ if not self._exists:
+ request = IPLinkRequest(self.filter('common'))
+
+ # create watchdog
+ wd = self.ipdb.watchdog(ifname=self['ifname'])
+
+ newif = True
+ try:
+ # 8<----------------------------------------------------
+ # ACHTUNG: hack for old platforms
+ if request.get('address', None) == '00:00:00:00:00:00':
+ del request['address']
+ del request['broadcast']
+ # 8<----------------------------------------------------
+ try:
+ self.nl.link('add', **request)
+ except NetlinkError as x:
+ # File exists
+ if x.code == errno.EEXIST:
+ # A bit special case, could be one of two cases:
+ #
+ # 1. A race condition between two different IPDB
+ # processes
+ # 2. An attempt to create dummy0, gre0, bond0 when
+ # the corrseponding module is not loaded. Being
+ # loaded, the module creates a default interface
+ # by itself, causing the request to fail
+ #
+ # The exception in that case can cause the DB
+ # inconsistence, since there can be queued not only
+ # the interface creation, but also IP address
+ # changes etc.
+ #
+ # So we ignore this particular exception and try to
+ # continue, as it is created by us.
+ pass
+
+ # Operation not supported
+ elif x.code == errno.EOPNOTSUPP and \
+ request.get('index', 0) != 0:
+ # ACHTUNG: hack for old platforms
+ request = IPLinkRequest({'ifname': self['ifname'],
+ 'kind': self['kind'],
+ 'index': 0})
+ self.nl.link('add', **request)
+ else:
+ raise
+ except Exception as e:
+ # on failure, invalidate the interface and detach it
+ # from the parent
+ # 1. drop the IPRoute() link
+ self.nl = None
+ # 2. clean up ipdb
+ self.ipdb.detach(self['index'])
+ self.ipdb.detach(self['ifname'])
+ # 3. invalidate the interface
+ with self._direct_state:
+ for i in tuple(self.keys()):
+ del self[i]
+ # 4. the rest
+ self._mode = 'invalid'
+ self._exception = e
+ self._tb = traceback.format_exc()
+ # raise the exception
+ raise
+
+ if wd is not None:
+ wd.wait()
+
+ # now we have our index and IP set and all other stuff
+ snapshot = self.pick()
+
+ try:
+ removed = snapshot - transaction
+ added = transaction - snapshot
+
+ # 8<---------------------------------------------
+ # Interface slaves
+ self['ports'].set_target(transaction['ports'])
+
+ for i in removed['ports']:
+ # detach the port
+ port = self.ipdb.interfaces[i]
+ port.set_target('master', None)
+ port.mirror_target('master', 'link')
+ self.nl.link('set', index=port['index'], master=0)
+
+ for i in added['ports']:
+ # enslave the port
+ port = self.ipdb.interfaces[i]
+ port.set_target('master', self['index'])
+ port.mirror_target('master', 'link')
+ self.nl.link('set',
+ index=port['index'],
+ master=self['index'])
+
+ if removed['ports'] or added['ports']:
+ self.nl.get_links(*(removed['ports'] | added['ports']))
+ self['ports'].target.wait(SYNC_TIMEOUT)
+ if not self['ports'].target.is_set():
+ raise CommitException('ports target is not set')
+
+ # RHEL 6.5 compat fix -- an explicit timeout
+ # it gives a time for all the messages to pass
+ compat.fix_timeout(1)
+
+ # wait for proper targets on ports
+ for i in list(added['ports']) + list(removed['ports']):
+ port = self.ipdb.interfaces[i]
+ target = port._local_targets['master']
+ target.wait(SYNC_TIMEOUT)
+ del port._local_targets['master']
+ del port._local_targets['link']
+ if not target.is_set():
+ raise CommitException('master target failed')
+ if i in added['ports']:
+ assert port.if_master == self['index']
+ else:
+ assert port.if_master != self['index']
+
+ # 8<---------------------------------------------
+ # Interface changes
+ request = IPLinkRequest()
+ for key in added:
+ if key in self._xfields['common']:
+ request[key] = added[key]
+ request['index'] = self['index']
+
+ # apply changes only if there is something to apply
+ if any([request[item] is not None for item in request
+ if item != 'index']):
+ self.nl.link('set', **request)
+ # hardcoded pause -- if the interface was moved
+ # across network namespaces
+ if 'net_ns_fd' in request:
+ while True:
+ # wait until the interface will disappear
+ # from the main network namespace
+ try:
+ self.nl.get_links(self['index'])
+ except NetlinkError as e:
+ if e.code == errno.ENODEV:
+ break
+ raise
+ except Exception:
+ raise
+ time.sleep(0.1)
+
+ # 8<---------------------------------------------
+ # IP address changes
+ self['ipaddr'].set_target(transaction['ipaddr'])
+
+ for i in removed['ipaddr']:
+ # Ignore link-local IPv6 addresses
+ if i[0][:4] == 'fe80' and i[1] == 64:
+ continue
+ # When you remove a primary IP addr, all subnetwork
+ # can be removed. In this case you will fail, but
+ # it is OK, no need to roll back
+ try:
+ self.nl.addr('delete', self['index'], i[0], i[1])
+ except NetlinkError as x:
+ # bypass only errno 99, 'Cannot assign address'
+ if x.code != errno.EADDRNOTAVAIL:
+ raise
+ except socket.error as x:
+ # bypass illegal IP requests
+ if not x.args[0].startswith('illegal IP'):
+ raise
+
+ for i in added['ipaddr']:
+ # Ignore link-local IPv6 addresses
+ if i[0][:4] == 'fe80' and i[1] == 64:
+ continue
+ # Try to fetch additional address attributes
+ try:
+ kwarg = transaction.ipaddr[i]
+ except KeyError:
+ kwarg = None
+ self.nl.addr('add', self['index'], i[0], i[1],
+ **kwarg if kwarg else {})
+
+ # 8<--------------------------------------
+ # FIXME: kernel bug, sometimes `addr add` for
+ # bond interfaces returns success, but does
+ # really nothing
+
+ if self['kind'] == 'bond':
+ while True:
+ try:
+ # dirtiest hack, but we have to use it here
+ time.sleep(0.1)
+ self.nl.addr('add', self['index'], i[0], i[1])
+ # continue to try to add the address
+ # until the kernel reports `file exists`
+ #
+ # a stupid solution, but must help
+ except NetlinkError as e:
+ if e.code == errno.EEXIST:
+ break
+ else:
+ raise
+ except Exception:
+ raise
+ # 8<--------------------------------------
+
+ if removed['ipaddr'] or added['ipaddr']:
+ # 8<--------------------------------------
+ # bond and bridge interfaces do not send
+ # IPv6 address updates, when are down
+ #
+ # beside of that, bridge interfaces are
+ # down by default, so they never send
+ # address updates from beginning
+ #
+ # so if we need, force address load
+ #
+ # FIXME: probably, we should handle other
+ # types as well
+ if self['kind'] in ('bond', 'bridge', 'veth'):
+ self.nl.get_addr()
+ # 8<--------------------------------------
+ self['ipaddr'].target.wait(SYNC_TIMEOUT)
+ if not self['ipaddr'].target.is_set():
+ raise CommitException('ipaddr target is not set')
+
+ # 8<---------------------------------------------
+ # reload interface to hit targets
+ if transaction._targets:
+ try:
+ self.reload()
+ except NetlinkError as e:
+ if e.code == errno.ENODEV: # No such device
+ if ('net_ns_fd' in added) or \
+ ('net_ns_pid' in added):
+ # it means, that the device was moved
+ # to another netns; just give up
+ if drop:
+ self.drop(transaction)
+ return self
+
+ # wait for targets
+ transaction._wait_all_targets()
+
+ # 8<---------------------------------------------
+ # Interface removal
+ if added.get('removal') or \
+ added.get('flicker') or\
+ (newif and rollback):
+ wd = self.ipdb.watchdog(action='RTM_DELLINK',
+ ifname=self['ifname'])
+ if added.get('flicker'):
+ self._flicker = True
+ self.nl.link('delete', **self)
+ wd.wait()
+ if added.get('flicker'):
+ self._exists = False
+ if added.get('removal'):
+ self._mode = 'invalid'
+ if drop:
+ self.drop(transaction)
+ return self
+ # 8<---------------------------------------------
+
+ # Iterate callback chain
+ for ch in self._commit_hooks:
+ # An exception will rollback the transaction
+ ch(self.dump(), snapshot.dump(), transaction.dump())
+ # 8<---------------------------------------------
+
+ except Exception as e:
+ # something went wrong: roll the transaction back
+ if not rollback:
+ ret = self.commit(transaction=snapshot,
+ rollback=True,
+ newif=newif)
+ # if some error was returned by the internal
+ # closure, substitute the initial one
+ if isinstance(ret, Exception):
+ error = ret
+ else:
+ error = e
+ error.traceback = traceback.format_exc()
+ elif isinstance(e, NetlinkError) and \
+ getattr(e, 'code', 0) == errno.EPERM:
+ # It is <Operation not permitted>, catched in
+ # rollback. So return it -- see ~5 lines above
+ e.traceback = traceback.format_exc()
+ return e
+ else:
+ # somethig went wrong during automatic rollback.
+ # that's the worst case, but it is still possible,
+ # since we have no locks on OS level.
+ self['ipaddr'].set_target(None)
+ self['ports'].set_target(None)
+ # reload all the database -- it can take a long time,
+ # but it is required since we have no idea, what is
+ # the result of the failure
+ #
+ # ACHTUNG: database reload is asynchronous, so after
+ # getting RuntimeError() from commit(), take a seat
+ # and rest for a while. It is an extremal case, it
+ # should not became at all, and there is no sync.
+ self.nl.get_links()
+ self.nl.get_addr()
+ x = RuntimeError()
+ x.cause = e
+ x.traceback = traceback.format_exc()
+ raise x
+
+ # if it is not a rollback turn
+ if drop and not rollback:
+ # drop last transaction in any case
+ self.drop(transaction)
+
+ # raise exception for failed transaction
+ if error is not None:
+ error.transaction = transaction
+ raise error
+
+ time.sleep(config.commit_barrier)
+ return self
+
+ def up(self):
+ '''
+ Shortcut: change the interface state to 'up'.
+ '''
+ if self['flags'] is None:
+ self['flags'] = 1
+ else:
+ self['flags'] |= 1
+ return self
+
+ def down(self):
+ '''
+ Shortcut: change the interface state to 'down'.
+ '''
+ if self['flags'] is None:
+ self['flags'] = 0
+ else:
+ self['flags'] &= ~(self['flags'] & 1)
+ return self
+
+ def remove(self):
+ '''
+ Mark the interface for removal
+ '''
+ self['removal'] = True
+ return self
+
+ def shadow(self):
+ '''
+ Remove the interface from the OS, but leave it in the
+ database. When one will try to re-create interface with
+ the same name, all the old saved attributes will apply
+ to the new interface, incl. MAC-address and even the
+ interface index. Please be aware, that the interface
+ index can be reused by OS while the interface is "in the
+ shadow state", in this case re-creation will fail.
+ '''
+ self['flicker'] = True
+ return self
diff --git a/node-admin/scripts/pyroute2/ipdb/linkedset.py b/node-admin/scripts/pyroute2/ipdb/linkedset.py
new file mode 100644
index 00000000000..15c762f3670
--- /dev/null
+++ b/node-admin/scripts/pyroute2/ipdb/linkedset.py
@@ -0,0 +1,134 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+'''
+import threading
+
+
+class LinkedSet(set):
+ '''
+ Utility class, used by `Interface` to track ip addresses
+ and ports. Called "linked" as it automatically updates all
+ instances, linked with it.
+
+ Target filter is a function, that returns `True` if a set
+ member should be counted in target checks (target methods
+ see below), or `False` if it should be ignored.
+ '''
+ def target_filter(self, x):
+ return True
+
+ def __init__(self, *argv, **kwarg):
+ set.__init__(self, *argv, **kwarg)
+ self.lock = threading.RLock()
+ self.target = threading.Event()
+ self._ct = None
+ self.raw = {}
+ self.links = []
+ self.exclusive = set()
+
+ def __getitem__(self, key):
+ return self.raw[key]
+
+ def set_target(self, value):
+ '''
+ Set target state for the object and clear the target
+ event. Once the target is reached, the event will be
+ set, see also: `check_target()`
+
+ Args:
+ - value (set): the target state to compare with
+ '''
+ with self.lock:
+ if value is None:
+ self._ct = None
+ self.target.clear()
+ else:
+ self._ct = set(value)
+ self.target.clear()
+ # immediately check, if the target already
+ # reached -- otherwise you will miss the
+ # target forever
+ self.check_target()
+
+ def check_target(self):
+ '''
+ Check the target state and set the target event in the
+ case the state is reached. Called from mutators, `add()`
+ and `remove()`
+ '''
+ with self.lock:
+ if self._ct is not None:
+ if set(filter(self.target_filter, self)) == \
+ set(filter(self.target_filter, self._ct)):
+ self._ct = None
+ self.target.set()
+
+ def add(self, key, raw=None, cascade=False):
+ '''
+ Add an item to the set and all connected instances,
+ check the target state.
+
+ Args:
+ - key: any hashable object
+ - raw (optional): raw representation of the object
+
+ Raw representation is not required. It can be used, e.g.,
+ to store RTM_NEWADDR RTNL messages along with
+ human-readable ip addr representation.
+ '''
+ with self.lock:
+ if cascade and (key in self.exclusive):
+ return
+ if key not in self:
+ self.raw[key] = raw
+ set.add(self, key)
+ for link in self.links:
+ link.add(key, raw, cascade=True)
+ self.check_target()
+
+ def remove(self, key, raw=None, cascade=False):
+ '''
+ Remove an item from the set and all connected instances,
+ check the target state.
+ '''
+ with self.lock:
+ if cascade and (key in self.exclusive):
+ return
+ set.remove(self, key)
+ for link in self.links:
+ if key in link:
+ link.remove(key, cascade=True)
+ self.check_target()
+
+ def unlink(self, key):
+ '''
+ Exclude key from cascade updates.
+ '''
+ self.exclusive.add(key)
+
+ def relink(self, key):
+ '''
+ Do not ignore key on cascade updates.
+ '''
+ self.exclusive.remove(key)
+
+ def connect(self, link):
+ '''
+ Connect a LinkedSet instance to this one. Connected
+ sets will be updated together with this instance.
+ '''
+ assert isinstance(link, LinkedSet)
+ self.links.append(link)
+
+ def __repr__(self):
+ return repr(list(self))
+
+
+class IPaddrSet(LinkedSet):
+ '''
+ LinkedSet child class with different target filter. The
+ filter ignores link local IPv6 addresses when sets and checks
+ the target.
+ '''
+ def target_filter(self, x):
+ return not ((x[0][:4] == 'fe80') and (x[1] == 64))
diff --git a/node-admin/scripts/pyroute2/ipdb/route.py b/node-admin/scripts/pyroute2/ipdb/route.py
new file mode 100644
index 00000000000..86692f3c839
--- /dev/null
+++ b/node-admin/scripts/pyroute2/ipdb/route.py
@@ -0,0 +1,354 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+import logging
+import threading
+from socket import AF_UNSPEC
+from pyroute2.common import basestring
+from pyroute2.netlink import nlmsg
+from pyroute2.netlink.rtnl.rtmsg import rtmsg
+from pyroute2.netlink.rtnl.req import IPRouteRequest
+from pyroute2.ipdb.transactional import Transactional
+
+
+class Metrics(Transactional):
+
+ def __init__(self, *argv, **kwarg):
+ Transactional.__init__(self, *argv, **kwarg)
+ self._fields = [rtmsg.metrics.nla2name(i[0]) for i
+ in rtmsg.metrics.nla_map]
+
+
+class RouteKey(dict):
+ '''
+ Construct from a netlink message a key that can be used
+ to locate the route in the table
+ '''
+ def __init__(self, msg):
+ # calculate dst
+ if msg.get_attr('RTA_DST', None) is not None:
+ dst = '%s/%s' % (msg.get_attr('RTA_DST'),
+ msg['dst_len'])
+ else:
+ dst = 'default'
+ self['dst'] = dst
+ # use output | input interfaces as key also
+ for key in ('oif', 'iif'):
+ value = msg.get_attr(msg.name2nla(key))
+ if value:
+ self[key] = value
+
+
+class Route(Transactional):
+ '''
+ Persistent transactional route object
+ '''
+
+ def __init__(self, ipdb, mode=None, parent=None, uid=None):
+ Transactional.__init__(self, ipdb, mode, parent, uid)
+ self._exists = False
+ self._load_event = threading.Event()
+ self._fields = [rtmsg.nla2name(i[0]) for i in rtmsg.nla_map]
+ self._fields.append('flags')
+ self._fields.append('src_len')
+ self._fields.append('dst_len')
+ self._fields.append('table')
+ self._fields.append('removal')
+ self.cleanup = ('attrs',
+ 'header',
+ 'event')
+ with self._direct_state:
+ self['metrics'] = Metrics(parent=self)
+
+ def load_netlink(self, msg):
+ with self._direct_state:
+ self._exists = True
+ self.update(msg)
+
+ # re-init metrics
+ metrics = self.get('metrics', Metrics(parent=self))
+ with metrics._direct_state:
+ for metric in tuple(metrics.keys()):
+ del metrics[metric]
+ self['metrics'] = metrics
+
+ # merge key
+ for (name, value) in msg['attrs']:
+ norm = rtmsg.nla2name(name)
+ # normalize RTAX
+ if norm == 'metrics':
+ with self['metrics']._direct_state:
+ for (rtax, rtax_value) in value['attrs']:
+ rtax_norm = rtmsg.metrics.nla2name(rtax)
+ self['metrics'][rtax_norm] = rtax_value
+ else:
+ self[norm] = value
+
+ if msg.get_attr('RTA_DST', None) is not None:
+ dst = '%s/%s' % (msg.get_attr('RTA_DST'),
+ msg['dst_len'])
+ else:
+ dst = 'default'
+ self['dst'] = dst
+ # finally, cleanup all not needed
+ for item in self.cleanup:
+ if item in self:
+ del self[item]
+
+ self.sync()
+
+ def sync(self):
+ self._load_event.set()
+
+ def reload(self):
+ # do NOT call get_routes() here, it can cause race condition
+ self._load_event.wait()
+ return self
+
+ def commit(self, tid=None, transaction=None, rollback=False):
+ self._load_event.clear()
+ error = None
+
+ if tid:
+ transaction = self._transactions[tid]
+ else:
+ transaction = transaction or self.last()
+
+ # create a new route
+ if not self._exists:
+ try:
+ self.nl.route('add', **IPRouteRequest(self))
+ except Exception:
+ self.nl = None
+ self.ipdb.routes.remove(self)
+ raise
+
+ # work on existing route
+ snapshot = self.pick()
+ try:
+ # route set
+ request = IPRouteRequest(transaction - snapshot)
+ if any([request[x] not in (None, {'attrs': []}) for x in request]):
+ self.nl.route('set', **IPRouteRequest(transaction))
+
+ if transaction.get('removal'):
+ self.nl.route('delete', **IPRouteRequest(snapshot))
+
+ except Exception as e:
+ if not rollback:
+ ret = self.commit(transaction=snapshot, rollback=True)
+ if isinstance(ret, Exception):
+ error = ret
+ else:
+ error = e
+ else:
+ self.drop()
+ x = RuntimeError()
+ x.cause = e
+ raise x
+
+ if not rollback:
+ self.drop()
+
+ if error is not None:
+ error.transaction = transaction
+ raise error
+
+ if not rollback:
+ self.reload()
+
+ return self
+
+ def remove(self):
+ self['removal'] = True
+ return self
+
+
+class RoutingTable(object):
+
+ def __init__(self, ipdb, prime=None):
+ self.ipdb = ipdb
+ self.records = prime or []
+
+ def __repr__(self):
+ return repr(self.records)
+
+ def __len__(self):
+ return len(self.records)
+
+ def __iter__(self):
+ for record in tuple(self.records):
+ yield record
+
+ def keys(self, key='dst'):
+ return [x[key] for x in self.records]
+
+ def describe(self, target, forward=True):
+ if isinstance(target, int):
+ return {'route': self.records[target],
+ 'index': target}
+ if isinstance(target, basestring):
+ target = {'dst': target}
+ if not isinstance(target, dict):
+ raise TypeError('unsupported key type')
+
+ for record in self.records:
+ for key in target:
+ # skip non-existing keys
+ #
+ # it's a hack, but newly-created routes
+ # don't contain all the fields that are
+ # in the netlink message
+ if record.get(key) is None:
+ continue
+ # if any key doesn't match
+ if target[key] != record[key]:
+ break
+ else:
+ # if all keys match
+ return {'route': record,
+ 'index': self.records.index(record)}
+
+ if not forward:
+ raise KeyError('route not found')
+
+ # split masks
+ if target.get('dst', '').find('/') >= 0:
+ dst = target['dst'].split('/')
+ target['dst'] = dst[0]
+ target['dst_len'] = int(dst[1])
+
+ if target.get('src', '').find('/') >= 0:
+ src = target['src'].split('/')
+ target['src'] = src[0]
+ target['src_len'] = int(src[1])
+
+ # load and return the route, if exists
+ route = Route(self.ipdb)
+ route.load_netlink(self.ipdb.nl.get_routes(**target)[0])
+ return {'route': route,
+ 'index': None}
+
+ def __delitem__(self, key):
+ self.records.pop(self.describe(key, forward=False)['index'])
+
+ def __setitem__(self, key, value):
+ try:
+ record = self.describe(key, forward=False)
+ except KeyError:
+ record = {'route': Route(self.ipdb),
+ 'index': None}
+
+ if isinstance(value, nlmsg):
+ record['route'].load_netlink(value)
+ elif isinstance(value, Route):
+ record['route'] = value
+ elif isinstance(value, dict):
+ with record['route']._direct_state:
+ record['route'].update(value)
+
+ if record['index'] is None:
+ self.records.append(record['route'])
+ else:
+ self.records[record['index']] = record['route']
+
+ def __getitem__(self, key):
+ return self.describe(key, forward=True)['route']
+
+ def __contains__(self, key):
+ try:
+ self.describe(key, forward=False)
+ return True
+ except KeyError:
+ return False
+
+
+class RoutingTableSet(object):
+
+ def __init__(self, ipdb):
+ self.ipdb = ipdb
+ self.tables = {254: RoutingTable(self.ipdb)}
+
+ def add(self, spec=None, **kwarg):
+ '''
+ Create a route from a dictionary
+ '''
+ spec = spec or kwarg
+ table = spec.get('table', 254)
+ assert 'dst' in spec
+ if table not in self.tables:
+ self.tables[table] = RoutingTable(self.ipdb)
+ route = Route(self.ipdb)
+ metrics = spec.pop('metrics', {})
+ route.update(spec)
+ route.metrics.update(metrics)
+ self.tables[table][route['dst']] = route
+ route.begin()
+ return route
+
+ def load_netlink(self, msg):
+ '''
+ Loads an existing route from a rtmsg
+ '''
+ table = msg.get('table', 254)
+ # construct a key
+ # FIXME: temporary solution
+ # FIXME: can `Route()` be used as a key?
+ key = RouteKey(msg)
+
+ # RTM_DELROUTE
+ if msg['event'] == 'RTM_DELROUTE':
+ try:
+ # locate the record
+ record = self.tables[table][key]
+ # delete the record
+ del self.tables[table][key]
+ # sync ???
+ record.sync()
+ except Exception as e:
+ logging.debug(e)
+ logging.debug(msg)
+ return
+
+ # RTM_NEWROUTE
+ if table not in self.tables:
+ self.tables[table] = RoutingTable(self.ipdb)
+ self.tables[table][key] = msg
+ return self.tables[table][key]
+
+ def remove(self, route, table=None):
+ if isinstance(route, Route):
+ table = route.get('table', 254)
+ route = route.get('dst', 'default')
+ else:
+ table = table or 254
+ del self.tables[table][route]
+
+ def describe(self, spec, table=254):
+ return self.tables[table].describe(spec)
+
+ def get(self, dst, table=None):
+ table = table or 254
+ return self.tables[table][dst]
+
+ def keys(self, table=254, family=AF_UNSPEC):
+ return [x['dst'] for x in self.tables[table]
+ if (x['family'] == family)
+ or (family == AF_UNSPEC)]
+
+ def has_key(self, key, table=254):
+ return key in self.tables[table]
+
+ def __contains__(self, key):
+ return key in self.tables[254]
+
+ def __getitem__(self, key):
+ return self.get(key)
+
+ def __setitem__(self, key, value):
+ assert key == value['dst']
+ return self.add(value)
+
+ def __delitem__(self, key):
+ return self.remove(key)
+
+ def __repr__(self):
+ return repr(self.tables[254])
diff --git a/node-admin/scripts/pyroute2/ipdb/transactional.py b/node-admin/scripts/pyroute2/ipdb/transactional.py
new file mode 100644
index 00000000000..533f3b9fd7f
--- /dev/null
+++ b/node-admin/scripts/pyroute2/ipdb/transactional.py
@@ -0,0 +1,402 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+'''
+import uuid
+import threading
+from pyroute2.common import Dotkeys
+from pyroute2.ipdb.common import SYNC_TIMEOUT
+from pyroute2.ipdb.common import CommitException
+from pyroute2.ipdb.common import DeprecationException
+from pyroute2.ipdb.linkedset import LinkedSet
+
+
+class State(object):
+
+ def __init__(self, lock=None):
+ self.lock = lock or threading.Lock()
+ self.flag = 0
+
+ def acquire(self):
+ self.lock.acquire()
+ self.flag += 1
+
+ def release(self):
+ assert self.flag > 0
+ self.flag -= 1
+ self.lock.release()
+
+ def is_set(self):
+ return self.flag
+
+ def __enter__(self):
+ self.acquire()
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.release()
+
+
+def update(f):
+ def decorated(self, *argv, **kwarg):
+ # obtain update lock
+ ret = None
+ tid = None
+ direct = True
+ with self._write_lock:
+ dcall = kwarg.pop('direct', False)
+ if dcall:
+ self._direct_state.acquire()
+
+ direct = self._direct_state.is_set()
+ if not direct:
+ # 1. begin transaction for 'direct' type
+ if self._mode == 'direct':
+ tid = self.begin()
+ # 2. begin transaction, if there is none
+ elif self._mode == 'implicit':
+ if not self._tids:
+ self.begin()
+ # 3. require open transaction for 'explicit' type
+ elif self._mode == 'explicit':
+ if not self._tids:
+ raise TypeError('start a transaction first')
+ # 4. transactions can not require transactions :)
+ elif self._mode == 'snapshot':
+ direct = True
+ # do not support other modes
+ else:
+ raise TypeError('transaction mode not supported')
+ # now that the transaction _is_ open
+ ret = f(self, direct, *argv, **kwarg)
+
+ if dcall:
+ self._direct_state.release()
+
+ if tid:
+ # close the transaction for 'direct' type
+ self.commit(tid)
+
+ return ret
+ decorated.__doc__ = f.__doc__
+ return decorated
+
+
+class Transactional(Dotkeys):
+ '''
+ Utility class that implements common transactional logic.
+ '''
+ _fields_cmp = {}
+
+ def __init__(self, ipdb=None, mode=None, parent=None, uid=None):
+ #
+ if ipdb is not None:
+ self.nl = ipdb.nl
+ self.ipdb = ipdb
+ else:
+ self.nl = None
+ self.ipdb = None
+ #
+ self._parent = None
+ if parent is not None:
+ self._mode = mode or parent._mode
+ self._parent = parent
+ elif ipdb is not None:
+ self._mode = mode or ipdb.mode
+ else:
+ self._mode = mode or 'implicit'
+ #
+ self.nlmsg = None
+ self.uid = uid or uuid.uuid4()
+ self.last_error = None
+ self._commit_hooks = []
+ self._fields = []
+ self._sids = []
+ self._ts = threading.local()
+ self._snapshots = {}
+ self._targets = {}
+ self._local_targets = {}
+ self._write_lock = threading.RLock()
+ self._direct_state = State(self._write_lock)
+ self._linked_sets = set()
+
+ @property
+ def _tids(self):
+ if not hasattr(self._ts, 'tids'):
+ self._ts.tids = []
+ return self._ts.tids
+
+ @property
+ def _transactions(self):
+ if not hasattr(self._ts, 'transactions'):
+ self._ts.transactions = {}
+ return self._ts.transactions
+
+ def register_callback(self, callback):
+ raise DeprecationException("deprecated since 0.2.15;"
+ "use `register_commit_hook()`")
+
+ def register_commit_hook(self, hook):
+ # FIXME: write docs
+ self._commit_hooks.append(hook)
+
+ def unregister_callback(self, callback):
+ raise DeprecationException("deprecated since 0.2.15;"
+ "use `unregister_commit_hook()`")
+
+ def unregister_commit_hook(self, hook):
+ # FIXME: write docs
+ with self._write_lock:
+ for cb in tuple(self._commit_hooks):
+ if hook == cb:
+ self._commit_hooks.pop(self._commit_hooks.index(cb))
+
+ def pick(self, detached=True, uid=None, parent=None, forge_tids=False):
+ '''
+ Get a snapshot of the object. Can be of two
+ types:
+ * detached=True -- (default) "true" snapshot
+ * detached=False -- keep ip addr set updated from OS
+
+ Please note, that "updated" doesn't mean "in sync".
+ The reason behind this logic is that snapshots can be
+ used as transactions.
+ '''
+ with self._write_lock:
+ res = self.__class__(ipdb=self.ipdb,
+ mode='snapshot',
+ parent=parent,
+ uid=uid)
+ for (key, value) in self.items():
+ if key in self._fields:
+ if isinstance(value, Transactional):
+ t = value.pick(detached=detached,
+ uid=res.uid,
+ parent=self)
+ if forge_tids:
+ # forge the transaction for nested objects
+ value._transactions[res.uid] = t
+ value._tids.append(res.uid)
+ res[key] = t
+ else:
+ res[key] = self[key]
+ for key in self._linked_sets:
+ res[key] = LinkedSet(self[key])
+ if not detached:
+ self[key].connect(res[key])
+ return res
+
+ def __enter__(self):
+ # FIXME: use a bitmask?
+ if self._mode not in ('implicit', 'explicit'):
+ raise TypeError('context managers require a transactional mode')
+ if not self._tids:
+ self.begin()
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ # apply transaction only if there was no error
+ if exc_type is None:
+ try:
+ self.commit()
+ except Exception as e:
+ self.last_error = e
+ raise
+
+ def __repr__(self):
+ res = {}
+ for i in self:
+ if self[i] is not None:
+ res[i] = self[i]
+ return res.__repr__()
+
+ def __sub__(self, vs):
+ res = self.__class__(ipdb=self.ipdb, mode='snapshot')
+ with self._direct_state:
+ # simple keys
+ for key in self:
+ if (key in self._fields) and \
+ ((key not in vs) or (self[key] != vs[key])):
+ res[key] = self[key]
+ for key in self._linked_sets:
+ diff = LinkedSet(self[key] - vs[key])
+ if diff:
+ res[key] = diff
+ return res
+
+ def dump(self, not_none=True):
+ with self._write_lock:
+ res = {}
+ for key in self:
+ if self[key] is not None and key[0] != '_':
+ if isinstance(self[key], Transactional):
+ res[key] = self[key].dump()
+ elif isinstance(self[key], LinkedSet):
+ res[key] = tuple(self[key])
+ else:
+ res[key] = self[key]
+ return res
+
+ def load(self, data):
+ pass
+
+ def commit(self, *args, **kwarg):
+ pass
+
+ def last_snapshot_id(self):
+ return self._sids[-1]
+
+ def revert(self, sid):
+ with self._write_lock:
+ self._transactions[sid] = self._snapshots[sid]
+ self._tids.append(sid)
+ self._sids.remove(sid)
+ del self._snapshots[sid]
+ return self
+
+ def snapshot(self):
+ '''
+ Create new snapshot
+ '''
+ return self._begin(mapping=self._snapshots,
+ ids=self._sids,
+ detached=True)
+
+ def begin(self):
+ '''
+ Start new transaction
+ '''
+ if self._parent is not None:
+ self._parent.begin()
+ else:
+ return self._begin(mapping=self._transactions,
+ ids=self._tids,
+ detached=False)
+
+ def _begin(self, mapping, ids, detached):
+ # keep snapshot's ip addr set updated from the OS
+ # it is required by the commit logic
+ if (self.ipdb is not None) and self.ipdb._stop:
+ raise RuntimeError("Can't start transaction on released IPDB")
+ t = self.pick(detached=detached, forge_tids=True)
+ mapping[t.uid] = t
+ ids.append(t.uid)
+ return t.uid
+
+ def last_snapshot(self):
+ if not self._sids:
+ raise TypeError('create a snapshot first')
+ return self._snapshots[self._sids[-1]]
+
+ def last(self):
+ '''
+ Return last open transaction
+ '''
+ with self._write_lock:
+ if not self._tids:
+ raise TypeError('start a transaction first')
+
+ return self._transactions[self._tids[-1]]
+
+ def review(self):
+ '''
+ Review last open transaction
+ '''
+ if not self._tids:
+ raise TypeError('start a transaction first')
+
+ with self._write_lock:
+ added = self.last() - self
+ removed = self - self.last()
+ for key in self._linked_sets:
+ added['-%s' % (key)] = removed[key]
+ added['+%s' % (key)] = added[key]
+ del added[key]
+ return added
+
+ def drop(self, tid=None):
+ '''
+ Drop a transaction.
+ '''
+ with self._write_lock:
+ if isinstance(tid, Transactional):
+ tid = tid.uid
+ elif tid is None:
+ tid = self._tids[-1]
+ self._tids.remove(tid)
+ del self._transactions[tid]
+ for (key, value) in self.items():
+ if isinstance(value, Transactional):
+ try:
+ value.drop(tid)
+ except KeyError:
+ pass
+
+ @update
+ def __setitem__(self, direct, key, value):
+ with self._write_lock:
+ if not direct:
+ # automatically set target on the last transaction,
+ # which must be started prior to that call
+ transaction = self.last()
+ transaction[key] = value
+ transaction._targets[key] = threading.Event()
+ else:
+ # set the item
+ Dotkeys.__setitem__(self, key, value)
+
+ # update on local targets
+ if key in self._local_targets:
+ func = self._fields_cmp.get(key, lambda x, y: x == y)
+ if func(value, self._local_targets[key].value):
+ self._local_targets[key].set()
+
+ # cascade update on nested targets
+ for tn in tuple(self._transactions.values()):
+ if (key in tn._targets) and (key in tn):
+ if self._fields_cmp.\
+ get(key, lambda x, y: x == y)(value, tn[key]):
+ tn._targets[key].set()
+
+ @update
+ def __delitem__(self, direct, key):
+ with self._write_lock:
+ # firstly set targets
+ self[key] = None
+
+ # then continue with delete
+ if not direct:
+ transaction = self.last()
+ if key in transaction:
+ del transaction[key]
+ else:
+ Dotkeys.__delitem__(self, key)
+
+ def option(self, key, value):
+ self[key] = value
+ return self
+
+ def unset(self, key):
+ del self[key]
+ return self
+
+ def _wait_all_targets(self):
+ for key, target in self._targets.items():
+ if key not in self._virtual_fields:
+ target.wait(SYNC_TIMEOUT)
+ if not target.is_set():
+ raise CommitException('target %s is not set' % key)
+
+ def set_target(self, key, value):
+ self._local_targets[key] = threading.Event()
+ self._local_targets[key].value = value
+
+ def mirror_target(self, key_from, key_to):
+ self._local_targets[key_to] = self._local_targets[key_from]
+
+ def set_item(self, key, value):
+ with self._direct_state:
+ self[key] = value
+
+ def del_item(self, key):
+ with self._direct_state:
+ del self[key]
diff --git a/node-admin/scripts/pyroute2/iproute.py b/node-admin/scripts/pyroute2/iproute.py
new file mode 100644
index 00000000000..09312945979
--- /dev/null
+++ b/node-admin/scripts/pyroute2/iproute.py
@@ -0,0 +1,888 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+# -*- coding: utf-8 -*-
+'''
+IPRoute module
+==============
+
+iproute quickstart
+------------------
+
+**IPRoute** in two words::
+
+ $ sudo pip install pyroute2
+
+ $ cat example.py
+ from pyroute2 import IPRoute
+ ip = IPRoute()
+ print([x.get_attr('IFLA_IFNAME') for x in ip.get_links()])
+
+ $ python example.py
+ ['lo', 'p6p1', 'wlan0', 'virbr0', 'virbr0-nic']
+
+threaded vs. threadless architecture
+------------------------------------
+
+Since v0.3.2, IPRoute class is threadless by default.
+It spawns no additional threads, and receives only
+responses to own requests, no broadcast messages. So,
+if you prefer not to cope with implicit threading, you
+can safely use this module.
+
+To get broadcast messages, use `IPRoute.bind()` call.
+Please notice, that after calling `IPRoute.bind()` you
+MUST get all the messages in time. In the case of the
+kernel buffer overflow, you will have to restart the
+socket.
+
+With `IPRoute.bind(async=True)` one can launch async
+message receiver thread with `Queue`-based buffer. The
+buffer is thread-safe and completely transparent from
+the programmer's perspective. Please read also
+`NetlinkSocket` documentation to know more about async
+mode.
+
+think about IPDB
+----------------
+
+If you plan to regularly fetch loads of objects, think
+about IPDB also. Unlike to IPRoute, IPDB does not fetch
+all the objects from OS every time you request them, but
+keeps a cache that is asynchronously updated by the netlink
+broadcasts. For a long-term running programs, that often
+retrieve info about hundreds or thousands of objects, it
+can be better to use IPDB as it will load CPU significantly
+less.
+
+classes
+-------
+'''
+
+from socket import htons
+from socket import AF_INET
+from socket import AF_INET6
+from socket import AF_UNSPEC
+from pyroute2.netlink import NLMSG_ERROR
+from pyroute2.netlink import NLM_F_ATOMIC
+from pyroute2.netlink import NLM_F_ROOT
+from pyroute2.netlink import NLM_F_REPLACE
+from pyroute2.netlink import NLM_F_REQUEST
+from pyroute2.netlink import NLM_F_ACK
+from pyroute2.netlink import NLM_F_DUMP
+from pyroute2.netlink import NLM_F_CREATE
+from pyroute2.netlink import NLM_F_EXCL
+from pyroute2.netlink.rtnl import RTM_NEWADDR
+from pyroute2.netlink.rtnl import RTM_GETADDR
+from pyroute2.netlink.rtnl import RTM_DELADDR
+from pyroute2.netlink.rtnl import RTM_NEWLINK
+from pyroute2.netlink.rtnl import RTM_GETLINK
+from pyroute2.netlink.rtnl import RTM_DELLINK
+from pyroute2.netlink.rtnl import RTM_NEWQDISC
+from pyroute2.netlink.rtnl import RTM_GETQDISC
+from pyroute2.netlink.rtnl import RTM_DELQDISC
+from pyroute2.netlink.rtnl import RTM_NEWTFILTER
+from pyroute2.netlink.rtnl import RTM_GETTFILTER
+from pyroute2.netlink.rtnl import RTM_DELTFILTER
+from pyroute2.netlink.rtnl import RTM_NEWTCLASS
+from pyroute2.netlink.rtnl import RTM_GETTCLASS
+from pyroute2.netlink.rtnl import RTM_DELTCLASS
+from pyroute2.netlink.rtnl import RTM_GETNEIGH
+from pyroute2.netlink.rtnl import RTM_NEWRULE
+from pyroute2.netlink.rtnl import RTM_GETRULE
+from pyroute2.netlink.rtnl import RTM_DELRULE
+from pyroute2.netlink.rtnl import RTM_NEWROUTE
+from pyroute2.netlink.rtnl import RTM_GETROUTE
+from pyroute2.netlink.rtnl import RTM_DELROUTE
+from pyroute2.netlink.rtnl import RTM_SETLINK
+from pyroute2.netlink.rtnl import TC_H_INGRESS
+from pyroute2.netlink.rtnl import TC_H_ROOT
+from pyroute2.netlink.rtnl import rtprotos
+from pyroute2.netlink.rtnl import rtypes
+from pyroute2.netlink.rtnl import rtscopes
+from pyroute2.netlink.rtnl.req import IPLinkRequest
+from pyroute2.netlink.rtnl.tcmsg import get_htb_parameters
+from pyroute2.netlink.rtnl.tcmsg import get_htb_class_parameters
+from pyroute2.netlink.rtnl.tcmsg import get_tbf_parameters
+from pyroute2.netlink.rtnl.tcmsg import get_sfq_parameters
+from pyroute2.netlink.rtnl.tcmsg import get_u32_parameters
+from pyroute2.netlink.rtnl.tcmsg import get_netem_parameters
+from pyroute2.netlink.rtnl.tcmsg import get_fw_parameters
+from pyroute2.netlink.rtnl.tcmsg import tcmsg
+from pyroute2.netlink.rtnl.rtmsg import rtmsg
+from pyroute2.netlink.rtnl.ndmsg import ndmsg
+from pyroute2.netlink.rtnl.fibmsg import fibmsg
+from pyroute2.netlink.rtnl.fibmsg import FR_ACT_NAMES
+from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg
+from pyroute2.netlink.rtnl.ifaddrmsg import ifaddrmsg
+from pyroute2.netlink.rtnl.iprsocket import IPRSocket
+
+from pyroute2.common import basestring
+
+DEFAULT_TABLE = 254
+
+
+def transform_handle(handle):
+ if isinstance(handle, basestring):
+ (major, minor) = [int(x if x else '0', 16) for x in handle.split(':')]
+ handle = (major << 8 * 2) | minor
+ return handle
+
+
+class IPRouteMixin(object):
+ '''
+ `IPRouteMixin` should not be instantiated by itself. It is intended
+ to be used as a mixin class that provides iproute2-like API. You
+ should use `IPRoute` or `NetNS` classes.
+
+ All following info you can consider as IPRoute info as well.
+
+ It is an old-school API, that provides access to rtnetlink as is.
+ It helps you to retrieve and change almost all the data, available
+ through rtnetlink::
+
+ from pyroute2 import IPRoute
+ ipr = IPRoute()
+ # lookup interface by name
+ dev = ipr.link_lookup(ifname='tap0')[0]
+ # bring it down
+ ipr.link('set', dev, state='down')
+ # change interface MAC address and rename it
+ ipr.link('set', dev, address='00:11:22:33:44:55', ifname='vpn')
+ # add primary IP address
+ ipr.addr('add', dev, address='10.0.0.1', mask=24)
+ # add secondary IP address
+ ipr.addr('add', dev, address='10.0.0.2', mask=24)
+ # bring it up
+ ipr.link('set', dev, state='up')
+
+ '''
+
+ # 8<---------------------------------------------------------------
+ #
+ # Listing methods
+ #
+ def get_qdiscs(self, index=None):
+ '''
+ Get all queue disciplines for all interfaces or for specified
+ one.
+ '''
+ msg = tcmsg()
+ msg['family'] = AF_UNSPEC
+ ret = self.nlm_request(msg, RTM_GETQDISC)
+ if index is None:
+ return ret
+ else:
+ return [x for x in ret if x['index'] == index]
+
+ def get_filters(self, index=0, handle=0, parent=0):
+ '''
+ Get filters for specified interface, handle and parent.
+ '''
+ msg = tcmsg()
+ msg['family'] = AF_UNSPEC
+ msg['index'] = index
+ msg['handle'] = handle
+ msg['parent'] = parent
+ return self.nlm_request(msg, RTM_GETTFILTER)
+
+ def get_classes(self, index=0):
+ '''
+ Get classes for specified interface.
+ '''
+ msg = tcmsg()
+ msg['family'] = AF_UNSPEC
+ msg['index'] = index
+ return self.nlm_request(msg, RTM_GETTCLASS)
+
+ def get_links(self, *argv, **kwarg):
+ '''
+ Get network interfaces.
+
+ By default returns all interfaces. Arguments vector
+ can contain interface indices or a special keyword
+ 'all'::
+
+ ip.get_links()
+ ip.get_links('all')
+ ip.get_links(1, 2, 3)
+
+ interfaces = [1, 2, 3]
+ ip.get_links(*interfaces)
+ '''
+ result = []
+ links = argv or ['all']
+ msg_flags = NLM_F_REQUEST | NLM_F_DUMP
+ for index in links:
+ msg = ifinfmsg()
+ msg['family'] = kwarg.get('family', AF_UNSPEC)
+ if index != 'all':
+ msg['index'] = index
+ msg_flags = NLM_F_REQUEST
+ result.extend(self.nlm_request(msg, RTM_GETLINK, msg_flags))
+ return result
+
+ def get_neighbors(self, family=AF_UNSPEC):
+ '''
+ Retrieve ARP cache records.
+ '''
+ msg = ndmsg()
+ msg['family'] = family
+ return self.nlm_request(msg, RTM_GETNEIGH)
+
+ def get_addr(self, family=AF_UNSPEC, index=None):
+ '''
+ Get addresses::
+ ip.get_addr() # get all addresses
+ ip.get_addr(index=2) # get addresses for the 2nd interface
+ '''
+ msg = ifaddrmsg()
+ msg['family'] = family
+ ret = self.nlm_request(msg, RTM_GETADDR)
+ if index is not None:
+ return [x for x in ret if x.get('index') == index]
+ else:
+ return ret
+
+ def get_rules(self, family=AF_UNSPEC):
+ '''
+ Get all rules.
+ You can specify inet family, by default return rules for all families.
+
+ Example::
+ ip.get_rules() # get all the rules for all families
+ ip.get_routes(family=AF_INET6) # get only IPv6 rules
+ '''
+ msg = fibmsg()
+ msg['family'] = family
+ msg_flags = NLM_F_REQUEST | NLM_F_ROOT | NLM_F_ATOMIC
+ return self.nlm_request(msg, RTM_GETRULE, msg_flags)
+
+ def get_routes(self, family=AF_INET, **kwarg):
+ '''
+ Get all routes. You can specify the table. There
+ are 255 routing classes (tables), and the kernel
+ returns all the routes on each request. So the
+ routine filters routes from full output.
+
+ Example::
+
+ ip.get_routes() # get all the routes for all families
+ ip.get_routes(family=AF_INET6) # get only IPv6 routes
+ ip.get_routes(table=254) # get routes from 254 table
+ '''
+
+ msg_flags = NLM_F_DUMP | NLM_F_REQUEST
+ msg = rtmsg()
+ # you can specify the table here, but the kernel
+ # will ignore this setting
+ table = kwarg.get('table', DEFAULT_TABLE)
+ msg['table'] = table if table <= 255 else 252
+
+ # explicitly look for IPv6
+ if any([kwarg.get(x, '').find(':') >= 0 for x
+ in ('dst', 'src', 'gateway', 'prefsrc')]):
+ family = AF_INET6
+ msg['family'] = family
+
+ # get a particular route
+ if kwarg.get('dst', None) is not None:
+ dlen = 32 if family == AF_INET else \
+ 128 if family == AF_INET6 else 0
+ msg_flags = NLM_F_REQUEST
+ msg['dst_len'] = kwarg.get('dst_len', dlen)
+
+ for key in kwarg:
+ nla = rtmsg.name2nla(key)
+ if kwarg[key] is not None:
+ msg['attrs'].append([nla, kwarg[key]])
+
+ routes = self.nlm_request(msg, RTM_GETROUTE, msg_flags)
+ return [x for x in routes
+ if x.get_attr('RTA_TABLE') == table or
+ kwarg.get('table', None) is None]
+ # 8<---------------------------------------------------------------
+
+ # 8<---------------------------------------------------------------
+ #
+ # Shortcuts
+ #
+ # addr_add(), addr_del(), route_add(), route_del() shortcuts are
+ # removed due to redundancy. Only link shortcuts are left here for
+ # now. Possibly, they should be moved to a separate module.
+ #
+ def get_default_routes(self, family=AF_UNSPEC, table=DEFAULT_TABLE):
+ '''
+ Get default routes
+ '''
+ # according to iproute2/ip/iproute.c:print_route()
+ return [x for x in self.get_routes(family, table=table)
+ if (x.get_attr('RTA_DST', None) is None and
+ x['dst_len'] == 0)]
+
+ def link_create(self, **kwarg):
+ '''
+ Create a link. The method parameters will be
+ passed to the `IPLinkRequest()` constructor as
+ a dictionary.
+
+ Examples::
+
+ ip.link_create(ifname='very_dummy', kind='dummy')
+ ip.link_create(ifname='br0', kind='bridge')
+ ip.link_create(ifname='v101', kind='vlan', vlan_id=101, link=1)
+ '''
+ return self.link('add', **IPLinkRequest(kwarg))
+
+ def link_up(self, index):
+ '''
+ Switch an interface up unconditionally.
+ '''
+ self.link('set', index=index, state='up')
+
+ def link_down(self, index):
+ '''
+ Switch an interface down unconditilnally.
+ '''
+ self.link('set', index=index, state='down')
+
+ def link_rename(self, index, name):
+ '''
+ Rename an interface. Please note, that the interface must be
+ in the `DOWN` state in order to be renamed, otherwise you
+ will get an error.
+ '''
+ self.link('set', index=index, ifname=name)
+
+ def link_remove(self, index):
+ '''
+ Remove an interface
+ '''
+ self.link('delete', index=index)
+
+ def link_lookup(self, **kwarg):
+ '''
+ Lookup interface index (indeces) by first level NLA
+ value.
+
+ Example::
+
+ ip.link_lookup(address="52:54:00:9d:4e:3d")
+ ip.link_lookup(ifname="lo")
+ ip.link_lookup(operstate="UP")
+
+ Please note, that link_lookup() returns list, not one
+ value.
+ '''
+ name = tuple(kwarg.keys())[0]
+ value = kwarg[name]
+
+ name = str(name).upper()
+ if not name.startswith('IFLA_'):
+ name = 'IFLA_%s' % (name)
+
+ return [k['index'] for k in
+ [i for i in self.get_links() if 'attrs' in i] if
+ [l for l in k['attrs'] if l[0] == name and l[1] == value]]
+
+ def flush_routes(self, *argv, **kwarg):
+ '''
+ Flush routes -- purge route records from a table.
+ Arguments are the same as for `get_routes()`
+ routine. Actually, this routine implements a pipe from
+ `get_routes()` to `nlm_request()`.
+ '''
+ flags = NLM_F_ACK | NLM_F_CREATE | NLM_F_EXCL | NLM_F_REQUEST
+ ret = []
+ kwarg['table'] = kwarg.get('table', DEFAULT_TABLE)
+ for route in self.get_routes(*argv, **kwarg):
+ ret.append(self.nlm_request(route,
+ msg_type=RTM_DELROUTE,
+ msg_flags=flags))
+ return ret
+ # 8<---------------------------------------------------------------
+
+ # 8<---------------------------------------------------------------
+ #
+ # General low-level configuration methods
+ #
+ def link(self, command, **kwarg):
+ '''
+ Link operations.
+
+ * command -- set, add or delete
+ * index -- device index
+ * \*\*kwarg -- keywords, NLA
+
+ Example::
+
+ x = 62 # interface index
+ ip.link("set", index=x, state="down")
+ ip.link("set", index=x, address="00:11:22:33:44:55", name="bala")
+ ip.link("set", index=x, mtu=1000, txqlen=2000)
+ ip.link("set", index=x, state="up")
+
+ Keywords "state", "flags" and "mask" are reserved. State can
+ be "up" or "down", it is a shortcut::
+
+ state="up": flags=1, mask=1
+ state="down": flags=0, mask=0
+
+ For more flags grep IFF in the kernel code, until we write
+ human-readable flag resolver.
+
+ Other keywords are from ifinfmsg.nla_map, look into the
+ corresponding module. You can use the form "ifname" as well
+ as "IFLA_IFNAME" and so on, so that's equal::
+
+ ip.link("set", index=x, mtu=1000)
+ ip.link("set", index=x, IFLA_MTU=1000)
+
+ You can also delete interface with::
+
+ ip.link("delete", index=x)
+ '''
+
+ commands = {'set': RTM_SETLINK,
+ 'add': RTM_NEWLINK,
+ 'del': RTM_DELLINK,
+ 'remove': RTM_DELLINK,
+ 'delete': RTM_DELLINK}
+ command = commands.get(command, command)
+
+ msg_flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE | NLM_F_EXCL
+ msg = ifinfmsg()
+ # index is required
+ msg['index'] = kwarg.get('index')
+
+ flags = kwarg.pop('flags', 0) or 0
+ mask = kwarg.pop('mask', 0) or kwarg.pop('change', 0) or 0
+
+ if 'state' in kwarg:
+ mask = 1 # IFF_UP mask
+ if kwarg['state'].lower() == 'up':
+ flags = 1 # 0 (down) or 1 (up)
+ del kwarg['state']
+
+ msg['flags'] = flags
+ msg['change'] = mask
+
+ for key in kwarg:
+ nla = type(msg).name2nla(key)
+ if kwarg[key] is not None:
+ msg['attrs'].append([nla, kwarg[key]])
+
+ return self.nlm_request(msg, msg_type=command, msg_flags=msg_flags)
+
+ def addr(self, command, index, address, mask=24,
+ family=None, scope=0, **kwarg):
+ '''
+ Address operations
+
+ * command -- add, delete
+ * index -- device index
+ * address -- IPv4 or IPv6 address
+ * mask -- address mask
+ * family -- socket.AF_INET for IPv4 or socket.AF_INET6 for IPv6
+ * scope -- the address scope, see /etc/iproute2/rt_scopes
+
+ Example::
+
+ index = 62
+ ip.addr("add", index, address="10.0.0.1", mask=24)
+ ip.addr("add", index, address="10.0.0.2", mask=24)
+ '''
+
+ commands = {'add': RTM_NEWADDR,
+ 'del': RTM_DELADDR,
+ 'remove': RTM_DELADDR,
+ 'delete': RTM_DELADDR}
+ command = commands.get(command, command)
+
+ flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE | NLM_F_EXCL
+
+ # try to guess family, if it is not forced
+ if family is None:
+ if address.find(":") > -1:
+ family = AF_INET6
+ else:
+ family = AF_INET
+
+ msg = ifaddrmsg()
+ msg['index'] = index
+ msg['family'] = family
+ msg['prefixlen'] = mask
+ msg['scope'] = scope
+ if family == AF_INET:
+ msg['attrs'] = [['IFA_LOCAL', address],
+ ['IFA_ADDRESS', address]]
+ elif family == AF_INET6:
+ msg['attrs'] = [['IFA_ADDRESS', address]]
+ for key in kwarg:
+ nla = ifaddrmsg.name2nla(key)
+ if kwarg[key] is not None:
+ msg['attrs'].append([nla, kwarg[key]])
+ return self.nlm_request(msg,
+ msg_type=command,
+ msg_flags=flags,
+ terminate=lambda x: x['header']['type'] ==
+ NLMSG_ERROR)
+
+ def tc(self, command, kind, index, handle=0, **kwarg):
+ '''
+ "Swiss knife" for traffic control. With the method you can
+ add, delete or modify qdiscs, classes and filters.
+
+ * command -- add or delete qdisc, class, filter.
+ * kind -- a string identifier -- "sfq", "htb", "u32" and so on.
+ * handle -- integer or string
+
+ Command can be one of ("add", "del", "add-class", "del-class",
+ "add-filter", "del-filter") (see `commands` dict in the code).
+
+ Handle notice: traditional iproute2 notation, like "1:0", actually
+ represents two parts in one four-bytes integer::
+
+ 1:0 -> 0x10000
+ 1:1 -> 0x10001
+ ff:0 -> 0xff0000
+ ffff:1 -> 0xffff0001
+
+ For pyroute2 tc() you can use both forms: integer like 0xffff0000
+ or string like 'ffff:0000'. By default, handle is 0, so you can add
+ simple classless queues w/o need to specify handle. Ingress queue
+ causes handle to be 0xffff0000.
+
+ So, to set up sfq queue on interface 1, the function call
+ will be like that::
+
+ ip = IPRoute()
+ ip.tc("add", "sfq", 1)
+
+ Instead of string commands ("add", "del"...), you can use also
+ module constants, `RTM_NEWQDISC`, `RTM_DELQDISC` and so on::
+
+ ip = IPRoute()
+ ip.tc(RTM_NEWQDISC, "sfq", 1)
+
+ More complex example with htb qdisc, lets assume eth0 == 2::
+
+ # u32 --> +--> htb 1:10 --> sfq 10:0
+ # | |
+ # | |
+ # eth0 -- htb 1:0 -- htb 1:1
+ # | |
+ # | |
+ # u32 --> +--> htb 1:20 --> sfq 20:0
+
+ eth0 = 2
+ # add root queue 1:0
+ ip.tc("add", "htb", eth0, 0x10000, default=0x200000)
+
+ # root class 1:1
+ ip.tc("add-class", "htb", eth0, 0x10001,
+ parent=0x10000,
+ rate="256kbit",
+ burst=1024 * 6)
+
+ # two branches: 1:10 and 1:20
+ ip.tc("add-class", "htb", eth0, 0x10010,
+ parent=0x10001,
+ rate="192kbit",
+ burst=1024 * 6,
+ prio=1)
+ ip.tc("add-class", "htb", eht0, 0x10020,
+ parent=0x10001,
+ rate="128kbit",
+ burst=1024 * 6,
+ prio=2)
+
+ # two leaves: 10:0 and 20:0
+ ip.tc("add", "sfq", eth0, 0x100000,
+ parent=0x10010,
+ perturb=10)
+ ip.tc("add", "sfq", eth0, 0x200000,
+ parent=0x10020,
+ perturb=10)
+
+ # two filters: one to load packets into 1:10 and the
+ # second to 1:20
+ ip.tc("add-filter", "u32", eth0,
+ parent=0x10000,
+ prio=10,
+ protocol=socket.AF_INET,
+ target=0x10010,
+ keys=["0x0006/0x00ff+8", "0x0000/0xffc0+2"])
+ ip.tc("add-filter", "u32", eth0,
+ parent=0x10000,
+ prio=10,
+ protocol=socket.AF_INET,
+ target=0x10020,
+ keys=["0x5/0xf+0", "0x10/0xff+33"])
+ '''
+
+ commands = {'add': RTM_NEWQDISC,
+ 'del': RTM_DELQDISC,
+ 'remove': RTM_DELQDISC,
+ 'delete': RTM_DELQDISC,
+ 'add-class': RTM_NEWTCLASS,
+ 'del-class': RTM_DELTCLASS,
+ 'add-filter': RTM_NEWTFILTER,
+ 'del-filter': RTM_DELTFILTER}
+ command = commands.get(command, command)
+ flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE | NLM_F_EXCL
+ msg = tcmsg()
+ # transform handle, parent and target, if needed:
+ handle = transform_handle(handle)
+ for item in ('parent', 'target', 'default'):
+ if item in kwarg and kwarg[item] is not None:
+ kwarg[item] = transform_handle(kwarg[item])
+ msg['index'] = index
+ msg['handle'] = handle
+ opts = kwarg.get('opts', None)
+ if kind == 'ingress':
+ msg['parent'] = TC_H_INGRESS
+ msg['handle'] = 0xffff0000
+ elif kind == 'tbf':
+ msg['parent'] = TC_H_ROOT
+ if kwarg:
+ opts = get_tbf_parameters(kwarg)
+ elif kind == 'htb':
+ msg['parent'] = kwarg.get('parent', TC_H_ROOT)
+ if kwarg:
+ if command in (RTM_NEWQDISC, RTM_DELQDISC):
+ opts = get_htb_parameters(kwarg)
+ elif command in (RTM_NEWTCLASS, RTM_DELTCLASS):
+ opts = get_htb_class_parameters(kwarg)
+ elif kind == 'netem':
+ msg['parent'] = kwarg.get('parent', TC_H_ROOT)
+ if kwarg:
+ opts = get_netem_parameters(kwarg)
+ elif kind == 'sfq':
+ msg['parent'] = kwarg.get('parent', TC_H_ROOT)
+ if kwarg:
+ opts = get_sfq_parameters(kwarg)
+ elif kind == 'u32':
+ msg['parent'] = kwarg.get('parent')
+ msg['info'] = htons(kwarg.get('protocol', 0) & 0xffff) |\
+ ((kwarg.get('prio', 0) << 16) & 0xffff0000)
+ if kwarg:
+ opts = get_u32_parameters(kwarg)
+ elif kind == 'fw':
+ msg['parent'] = kwarg.get('parent')
+ msg['info'] = htons(kwarg.get('protocol', 0) & 0xffff) |\
+ ((kwarg.get('prio', 0) << 16) & 0xffff0000)
+ if kwarg:
+ opts = get_fw_parameters(kwarg)
+ else:
+ msg['parent'] = kwarg.get('parent', TC_H_ROOT)
+
+ if kind is not None:
+ msg['attrs'] = [['TCA_KIND', kind]]
+ if opts is not None:
+ msg['attrs'].append(['TCA_OPTIONS', opts])
+ return self.nlm_request(msg, msg_type=command, msg_flags=flags)
+
+ def route(self, command,
+ rtype='RTN_UNICAST',
+ rtproto='RTPROT_STATIC',
+ rtscope='RT_SCOPE_UNIVERSE',
+ **kwarg):
+ '''
+ Route operations
+
+ * command -- add, delete, change, replace
+ * prefix -- route prefix
+ * mask -- route prefix mask
+ * rtype -- route type (default: "RTN_UNICAST")
+ * rtproto -- routing protocol (default: "RTPROT_STATIC")
+ * rtscope -- routing scope (default: "RT_SCOPE_UNIVERSE")
+ * family -- socket.AF_INET (default) or socket.AF_INET6
+
+ `pyroute2/netlink/rtnl/rtmsg.py` rtmsg.nla_map:
+
+ * table -- routing table to use (default: 254)
+ * gateway -- via address
+ * prefsrc -- preferred source IP address
+ * dst -- the same as `prefix`
+ * src -- source address
+ * iif -- incoming traffic interface
+ * oif -- outgoing traffic interface
+
+ etc.
+
+ Example::
+
+ ip.route("add", dst="10.0.0.0", mask=24, gateway="192.168.0.1")
+
+ Commands `change` and `replace` have the same meanings, as
+ in ip-route(8): `change` modifies only existing route, while
+ `replace` creates a new one, if there is no such route yet.
+ '''
+
+ # 8<----------------------------------------------------
+ # FIXME
+ # flags should be moved to some more general place
+ flags_base = NLM_F_REQUEST | NLM_F_ACK
+ flags_make = flags_base | NLM_F_CREATE | NLM_F_EXCL
+ flags_change = flags_base | NLM_F_REPLACE
+ flags_replace = flags_change | NLM_F_CREATE
+ # 8<----------------------------------------------------
+ commands = {'add': (RTM_NEWROUTE, flags_make),
+ 'set': (RTM_NEWROUTE, flags_replace),
+ 'replace': (RTM_NEWROUTE, flags_replace),
+ 'change': (RTM_NEWROUTE, flags_change),
+ 'del': (RTM_DELROUTE, flags_make),
+ 'remove': (RTM_DELROUTE, flags_make),
+ 'delete': (RTM_DELROUTE, flags_make)}
+ (command, flags) = commands.get(command, command)
+ msg = rtmsg()
+ # table is mandatory; by default == 254
+ # if table is not defined in kwarg, save it there
+ # also for nla_attr:
+ table = kwarg.get('table', 254)
+ msg['table'] = table if table <= 255 else 252
+ msg['family'] = kwarg.get('family', AF_INET)
+ msg['proto'] = rtprotos[rtproto]
+ msg['type'] = rtypes[rtype]
+ msg['scope'] = rtscopes[rtscope]
+ msg['dst_len'] = kwarg.get('dst_len', None) or \
+ kwarg.get('mask', 0)
+ msg['attrs'] = []
+ # FIXME
+ # deprecated "prefix" support:
+ if 'prefix' in kwarg:
+ kwarg['dst'] = kwarg['prefix']
+
+ for key in kwarg:
+ nla = rtmsg.name2nla(key)
+ if kwarg[key] is not None:
+ msg['attrs'].append([nla, kwarg[key]])
+
+ return self.nlm_request(msg, msg_type=command,
+ msg_flags=flags)
+
+ def rule(self, command, table, priority=32000,
+ action='FR_ACT_NOP', family=AF_INET,
+ src=None, src_len=None,
+ dst=None, dst_len=None,
+ fwmark=None, iifname=None, oifname=None):
+ '''
+ Rule operations
+
+ - command — add, delete
+ - table — 0 < table id < 253
+ - priority — 0 < rule's priority < 32766
+ - action — type of rule, default 'FR_ACT_NOP' (see fibmsg.py)
+ - rtscope — routing scope, default RT_SCOPE_UNIVERSE
+ `(RT_SCOPE_UNIVERSE|RT_SCOPE_SITE|\
+ RT_SCOPE_LINK|RT_SCOPE_HOST|RT_SCOPE_NOWHERE)`
+ - family — rule's family (socket.AF_INET (default) or
+ socket.AF_INET6)
+ - src — IP source for Source Based (Policy Based) routing's rule
+ - dst — IP for Destination Based (Policy Based) routing's rule
+ - src_len — Mask for Source Based (Policy Based) routing's rule
+ - dst_len — Mask for Destination Based (Policy Based) routing's
+ rule
+ - iifname — Input interface for Interface Based (Policy Based)
+ routing's rule
+ - oifname — Output interface for Interface Based (Policy Based)
+ routing's rule
+
+ Example::
+ ip.rule('add', 10, 32000)
+
+ Will create::
+ #ip ru sh
+ ...
+ 32000: from all lookup 10
+ ....
+
+ Example::
+ iproute.rule('add', 11, 32001, 'FR_ACT_UNREACHABLE')
+
+ Will create::
+ #ip ru sh
+ ...
+ 32001: from all lookup 11 unreachable
+ ....
+
+ Example::
+ iproute.rule('add', 14, 32004, src='10.64.75.141')
+
+ Will create::
+ #ip ru sh
+ ...
+ 32004: from 10.64.75.141 lookup 14
+ ...
+
+ Example::
+ iproute.rule('add', 15, 32005, dst='10.64.75.141', dst_len=24)
+
+ Will create::
+ #ip ru sh
+ ...
+ 32005: from 10.64.75.141/24 lookup 15
+ ...
+
+ Example::
+ iproute.rule('add', 15, 32006, dst='10.64.75.141', fwmark=10)
+
+ Will create::
+ #ip ru sh
+ ...
+ 32006: from 10.64.75.141 fwmark 0xa lookup 15
+ ...
+ '''
+ if table < 0:
+ raise ValueError('unsupported table number')
+
+ commands = {'add': RTM_NEWRULE,
+ 'del': RTM_DELRULE,
+ 'remove': RTM_DELRULE,
+ 'delete': RTM_DELRULE}
+ command = commands.get(command, command)
+
+ msg_flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE | NLM_F_EXCL
+ msg = fibmsg()
+ msg['table'] = table if table <= 255 else 252
+ msg['family'] = family
+ msg['action'] = FR_ACT_NAMES[action]
+ msg['attrs'] = [['FRA_TABLE', table]]
+ msg['attrs'].append(['FRA_PRIORITY', priority])
+ if fwmark is not None:
+ msg['attrs'].append(['FRA_FWMARK', fwmark])
+ addr_len = {AF_INET6: 128, AF_INET: 32}[family]
+ if(dst_len is not None and dst_len >= 0 and dst_len <= addr_len):
+ msg['dst_len'] = dst_len
+ else:
+ msg['dst_len'] = 0
+ if(src_len is not None and src_len >= 0 and src_len <= addr_len):
+ msg['src_len'] = src_len
+ else:
+ msg['src_len'] = 0
+ if src is not None:
+ msg['attrs'].append(['FRA_SRC', src])
+ if src_len is None:
+ msg['src_len'] = addr_len
+ if dst is not None:
+ msg['attrs'].append(['FRA_DST', dst])
+ if dst_len is None:
+ msg['dst_len'] = addr_len
+ if iifname is not None:
+ msg['attrs'].append(['FRA_IIFNAME', iifname])
+ if oifname is not None:
+ msg['attrs'].append(['FRA_OIFNAME', oifname])
+
+ return self.nlm_request(msg, msg_type=command,
+ msg_flags=msg_flags)
+ # 8<---------------------------------------------------------------
+
+
+class IPRoute(IPRouteMixin, IPRSocket):
+ '''
+ Production class that provides iproute API over normal Netlink
+ socket.
+
+ You can think of this class in some way as of plain old iproute2
+ utility.
+ '''
+ pass
diff --git a/node-admin/scripts/pyroute2/ipset.py b/node-admin/scripts/pyroute2/ipset.py
new file mode 100644
index 00000000000..e12b0551357
--- /dev/null
+++ b/node-admin/scripts/pyroute2/ipset.py
@@ -0,0 +1,149 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+IPSet module
+============
+
+The very basic ipset support.
+
+Right now it is tested only for hash:ip and doesn't support
+many useful options. But it can be easily extended, so you
+are welcome to help with that.
+'''
+import socket
+from pyroute2.netlink import NLMSG_ERROR
+from pyroute2.netlink import NLM_F_REQUEST
+from pyroute2.netlink import NLM_F_DUMP
+from pyroute2.netlink import NLM_F_ACK
+from pyroute2.netlink import NLM_F_EXCL
+from pyroute2.netlink import NETLINK_NETFILTER
+from pyroute2.netlink.nlsocket import NetlinkSocket
+from pyroute2.netlink.nfnetlink import NFNL_SUBSYS_IPSET
+from pyroute2.netlink.nfnetlink.ipset import IPSET_CMD_PROTOCOL
+from pyroute2.netlink.nfnetlink.ipset import IPSET_CMD_CREATE
+from pyroute2.netlink.nfnetlink.ipset import IPSET_CMD_DESTROY
+from pyroute2.netlink.nfnetlink.ipset import IPSET_CMD_SWAP
+from pyroute2.netlink.nfnetlink.ipset import IPSET_CMD_LIST
+from pyroute2.netlink.nfnetlink.ipset import IPSET_CMD_ADD
+from pyroute2.netlink.nfnetlink.ipset import IPSET_CMD_DEL
+from pyroute2.netlink.nfnetlink.ipset import ipset_msg
+
+
+def _nlmsg_error(msg):
+ return msg['header']['type'] == NLMSG_ERROR
+
+
+class IPSet(NetlinkSocket):
+ '''
+ NFNetlink socket (family=NETLINK_NETFILTER).
+
+ Implements API to the ipset functionality.
+ '''
+
+ policy = {IPSET_CMD_PROTOCOL: ipset_msg,
+ IPSET_CMD_LIST: ipset_msg}
+
+ def __init__(self, version=6, attr_revision=2, nfgen_family=2):
+ super(IPSet, self).__init__(family=NETLINK_NETFILTER)
+ policy = dict([(x | (NFNL_SUBSYS_IPSET << 8), y)
+ for (x, y) in self.policy.items()])
+ self.register_policy(policy)
+ self._proto_version = version
+ self._attr_revision = attr_revision
+ self._nfgen_family = nfgen_family
+
+ def request(self, msg, msg_type,
+ msg_flags=NLM_F_REQUEST | NLM_F_DUMP,
+ terminate=None):
+ msg['nfgen_family'] = self._nfgen_family
+ return self.nlm_request(msg,
+ msg_type | (NFNL_SUBSYS_IPSET << 8),
+ msg_flags, terminate=terminate)
+
+ def list(self, name=None):
+ '''
+ List installed ipsets. If `name` is provided, list
+ the named ipset or return an empty list.
+
+ It looks like nfnetlink doesn't return an error,
+ when requested ipset doesn't exist.
+ '''
+ msg = ipset_msg()
+ msg['attrs'] = [['IPSET_ATTR_PROTOCOL', self._proto_version]]
+ if name is not None:
+ msg['attrs'].append(['IPSET_ATTR_SETNAME', name])
+ return self.request(msg, IPSET_CMD_LIST)
+
+ def destroy(self, name):
+ '''
+ Destroy an ipset
+ '''
+ msg = ipset_msg()
+ msg['attrs'] = [['IPSET_ATTR_PROTOCOL', self._proto_version],
+ ['IPSET_ATTR_SETNAME', name]]
+ return self.request(msg, IPSET_CMD_DESTROY,
+ msg_flags=NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL,
+ terminate=_nlmsg_error)
+
+ def create(self, name, stype='hash:ip', family=socket.AF_INET,
+ exclusive=True):
+ '''
+ Create an ipset `name` of type `stype`, by default
+ `hash:ip`.
+
+ Very simple and stupid method, should be extended
+ to support ipset options.
+ '''
+ excl_flag = NLM_F_EXCL if exclusive else 0
+ msg = ipset_msg()
+ msg['attrs'] = [['IPSET_ATTR_PROTOCOL', self._proto_version],
+ ['IPSET_ATTR_SETNAME', name],
+ ['IPSET_ATTR_TYPENAME', stype],
+ ['IPSET_ATTR_FAMILY', family],
+ ['IPSET_ATTR_REVISION', self._attr_revision]]
+
+ return self.request(msg, IPSET_CMD_CREATE,
+ msg_flags=NLM_F_REQUEST | NLM_F_ACK | excl_flag,
+ terminate=_nlmsg_error)
+
+ def _add_delete(self, name, entry, family, cmd, exclusive):
+ if family == socket.AF_INET:
+ entry_type = 'IPSET_ATTR_IPADDR_IPV4'
+ elif family == socket.AF_INET6:
+ entry_type = 'IPSET_ATTR_IPADDR_IPV6'
+ else:
+ raise TypeError('unknown family')
+ excl_flag = NLM_F_EXCL if exclusive else 0
+
+ msg = ipset_msg()
+ msg['attrs'] = [['IPSET_ATTR_PROTOCOL', self._proto_version],
+ ['IPSET_ATTR_SETNAME', name],
+ ['IPSET_ATTR_DATA',
+ {'attrs': [['IPSET_ATTR_IP',
+ {'attrs': [[entry_type, entry]]}]]}]]
+ return self.request(msg, cmd,
+ msg_flags=NLM_F_REQUEST | NLM_F_ACK | excl_flag,
+ terminate=_nlmsg_error)
+
+ def add(self, name, entry, family=socket.AF_INET, exclusive=True):
+ '''
+ Add a member to the ipset
+ '''
+ return self._add_delete(name, entry, family, IPSET_CMD_ADD, exclusive)
+
+ def delete(self, name, entry, family=socket.AF_INET, exclusive=True):
+ '''
+ Delete a member from the ipset
+ '''
+ return self._add_delete(name, entry, family, IPSET_CMD_DEL, exclusive)
+
+ def swap(self, set_a, set_b):
+ '''
+ Swap two ipsets
+ '''
+ msg = ipset_msg()
+ msg['attrs'] = [['IPSET_ATTR_PROTOCOL', self._proto_version],
+ ['IPSET_ATTR_SETNAME', set_a],
+ ['IPSET_ATTR_TYPENAME', set_b]]
+ return self.request(msg, IPSET_CMD_SWAP,
+ msg_flags=NLM_F_REQUEST | NLM_F_ACK,
+ terminate=_nlmsg_error)
diff --git a/node-admin/scripts/pyroute2/iwutil.py b/node-admin/scripts/pyroute2/iwutil.py
new file mode 100644
index 00000000000..835f5fc773a
--- /dev/null
+++ b/node-admin/scripts/pyroute2/iwutil.py
@@ -0,0 +1,355 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+# -*- coding: utf-8 -*-
+'''
+IW module
+=========
+
+Experimental wireless module — nl80211 support.
+
+Disclaimer
+----------
+
+Unlike IPRoute, which is mostly usable, though is far from
+complete yet, the IW module is in the very initial state.
+Neither the module itself, nor the message class cover the
+nl80211 functionality reasonably enough. So if you're
+going to use it, brace yourself — debug is coming.
+
+Messages
+--------
+
+nl80211 messages are defined here::
+
+ pyroute2/netlink/nl80211/__init__.py
+
+Pls notice NLAs of type `hex`. On the early development stage
+`hex` allows to inspect incoming data as a hex dump and,
+occasionally, even make requests with such NLAs. But it's
+not a production way.
+
+The type `hex` in the NLA definitions means that this
+particular NLA is not handled yet properly. If you want to
+use some NLA which is defined as `hex` yet, pls find out a
+specific type, patch the message class and submit your pull
+request on github.
+
+If you're not familiar with NLA types, take a look at RTNL
+definitions::
+
+ pyroute2/netlink/rtnl/ndmsg.py
+
+and so on.
+
+Communication with the kernel
+-----------------------------
+
+There are several methods of the communication with the kernel.
+
+ * `sendto()` — lowest possible, send a raw binary data
+ * `put()` — send a netlink message
+ * `nlm_request()` — send a message, return the response
+ * `get()` — get a netlink message
+ * `recv()` — get a raw binary data from the kernel
+
+There are no errors on `put()` usually. Any `permission denied`,
+any `invalid value` errors are returned from the kernel with
+netlink also. So if you do `put()`, but don't do `get()`, be
+prepared to miss errors.
+
+The preferred method for the communication is `nlm_request()`.
+It tracks the message ID, returns the corresponding response.
+In the case of errors `nlm_request()` raises an exception.
+To get the response on any operation with nl80211, use flag
+`NLM_F_ACK`.
+
+Reverse it
+----------
+
+If you're too lazy to read the kernel sources, but still need
+something not implemented here, you can use reverse engineering
+on a reference implementation. E.g.::
+
+ # strace -e trace=network -f -x -s 4096 \\
+ iw phy phy0 interface add test type monitor
+
+Will dump all the netlink traffic between the program `iw` and
+the kernel. Three first packets are the generic netlink protocol
+discovery, you can ignore them. All that follows, is the
+nl80211 traffic::
+
+ sendmsg(3, {msg_name(12)={sa_family=AF_NETLINK, ... },
+ msg_iov(1)=[{"\\x30\\x00\\x00\\x00\\x1b\\x00\\x05 ...", 48}],
+ msg_controllen=0, msg_flags=0}, 0) = 48
+ recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, ... },
+ msg_iov(1)=[{"\\x58\\x00\\x00\\x00\\x1b\\x00\\x00 ...", 16384}],
+ msg_controllen=0, msg_flags=0}, 0) = 88
+ ...
+
+With `-s 4096` you will get the full dump. Then copy the strings
+from `msg_iov` to a file, let's say `data`, and run the decoder::
+
+ $ pwd
+ /home/user/Projects/pyroute2
+ $ export PYTHONPATH=`pwd`
+ $ python scripts/decoder.py pyroute2.netlink.nl80211.nl80211cmd data
+
+You will get the session decoded::
+
+ {'attrs': [['NL80211_ATTR_WIPHY', 0],
+ ['NL80211_ATTR_IFNAME', 'test'],
+ ['NL80211_ATTR_IFTYPE', 6]],
+ 'cmd': 7,
+ 'header': {'flags': 5,
+ 'length': 48,
+ 'pid': 3292542647,
+ 'sequence_number': 1430426434,
+ 'type': 27},
+ 'reserved': 0,
+ 'version': 0}
+ {'attrs': [['NL80211_ATTR_IFINDEX', 23811],
+ ['NL80211_ATTR_IFNAME', 'test'],
+ ['NL80211_ATTR_WIPHY', 0],
+ ['NL80211_ATTR_IFTYPE', 6],
+ ['NL80211_ATTR_WDEV', 4],
+ ['NL80211_ATTR_MAC', 'a4:4e:31:43:1c:7c'],
+ ['NL80211_ATTR_GENERATION', '02:00:00:00']],
+ 'cmd': 7,
+ 'header': {'flags': 0,
+ 'length': 88,
+ 'pid': 3292542647,
+ 'sequence_number': 1430426434,
+ 'type': 27},
+ 'reserved': 0,
+ 'version': 1}
+
+Now you know, how to do a request and what you will get as a
+response. Sample collected data is in the `scripts` directory.
+
+Submit changes
+--------------
+
+Please do not hesitate to submit the changes on github. Without
+your patches this module will not evolve.
+'''
+from pyroute2.netlink import NLM_F_ACK
+from pyroute2.netlink import NLM_F_REQUEST
+from pyroute2.netlink import NLM_F_DUMP
+from pyroute2.netlink.nl80211 import NL80211
+from pyroute2.netlink.nl80211 import nl80211cmd
+from pyroute2.netlink.nl80211 import NL80211_NAMES
+from pyroute2.netlink.nl80211 import IFTYPE_NAMES
+
+
+class IW(NL80211):
+
+ def __init__(self, *argv, **kwarg):
+ # get specific groups kwarg
+ if 'groups' in kwarg:
+ groups = kwarg['groups']
+ del kwarg['groups']
+ else:
+ groups = None
+
+ # get specific async kwarg
+ if 'async' in kwarg:
+ async = kwarg['async']
+ del kwarg['async']
+ else:
+ async = False
+
+ # align groups with async
+ if groups is None:
+ groups = ~0 if async else 0
+
+ # continue with init
+ super(IW, self).__init__(*argv, **kwarg)
+
+ # do automatic bind
+ # FIXME: unfortunately we can not omit it here
+ self.bind(groups, async)
+
+ def del_interface(self, dev):
+ '''
+ Delete a virtual interface
+
+ - dev — device index
+ '''
+ msg = nl80211cmd()
+ msg['cmd'] = NL80211_NAMES['NL80211_CMD_DEL_INTERFACE']
+ msg['attrs'] = [['NL80211_ATTR_IFINDEX', dev]]
+ self.nlm_request(msg,
+ msg_type=self.prid,
+ msg_flags=NLM_F_REQUEST | NLM_F_ACK)
+
+ def add_interface(self, ifname, iftype, dev=None, phy=0):
+ '''
+ Create a virtual interface
+
+ - ifname — name of the interface to create
+ - iftype — interface type to create
+ - dev — device index
+ - phy — phy index
+
+ One should specify `dev` (device index) or `phy`
+ (phy index). If no one specified, phy == 0.
+
+ `iftype` can be integer or string:
+
+ 1. adhoc
+ 2. station
+ 3. ap
+ 4. ap_vlan
+ 5. wds
+ 6. monitor
+ 7. mesh_point
+ 8. p2p_client
+ 9. p2p_go
+ 10. p2p_device
+ 11. ocb
+ '''
+ # lookup the interface type
+ iftype = IFTYPE_NAMES.get(iftype, iftype)
+ assert isinstance(iftype, int)
+
+ msg = nl80211cmd()
+ msg['cmd'] = NL80211_NAMES['NL80211_CMD_NEW_INTERFACE']
+ msg['attrs'] = [['NL80211_ATTR_IFNAME', ifname],
+ ['NL80211_ATTR_IFTYPE', iftype]]
+ if dev is not None:
+ msg['attrs'].append(['NL80211_ATTR_IFINDEX', dev])
+ elif phy is not None:
+ msg['attrs'].append(['NL80211_ATTR_WIPHY', phy])
+ else:
+ raise TypeError('no device specified')
+ self.nlm_request(msg,
+ msg_type=self.prid,
+ msg_flags=NLM_F_REQUEST | NLM_F_ACK)
+
+ def list_wiphy(self):
+ '''
+ Get all list of phy device
+ '''
+ msg = nl80211cmd()
+ msg['cmd'] = NL80211_NAMES['NL80211_CMD_GET_WIPHY']
+ return self.nlm_request(msg,
+ msg_type=self.prid,
+ msg_flags=NLM_F_REQUEST | NLM_F_DUMP)
+
+ def _get_phy_name(self, attr):
+ return 'phy%i' % attr.get_attr('NL80211_ATTR_WIPHY')
+
+ def _get_frequency(self, attr):
+ try:
+ return attr.get_attr('NL80211_ATTR_WIPHY_FREQ') + 2304
+ except:
+ return 0
+
+ def get_interfaces_dict(self):
+ '''
+ Get interfaces dictionary
+ '''
+ ret = {}
+ for wif in self.get_interfaces_dump():
+ chan_width = wif.get_attr('NL80211_ATTR_CHANNEL_WIDTH')
+ freq = self._get_frequency(wif) if chan_width is not None else 0
+ wifname = wif.get_attr('NL80211_ATTR_IFNAME')
+ ret[wifname] = [wif.get_attr('NL80211_ATTR_IFINDEX'),
+ self._get_phy_name(wif),
+ wif.get_attr('NL80211_ATTR_MAC'),
+ freq, chan_width]
+ return ret
+
+ def get_interfaces_dump(self):
+ '''
+ Get interfaces dump
+ '''
+ msg = nl80211cmd()
+ msg['cmd'] = NL80211_NAMES['NL80211_CMD_GET_INTERFACE']
+ return self.nlm_request(msg,
+ msg_type=self.prid,
+ msg_flags=NLM_F_REQUEST | NLM_F_DUMP)
+
+ def get_interface_by_phy(self, attr):
+ '''
+ Get interface by phy ( use x.get_attr('NL80211_ATTR_WIPHY') )
+ '''
+ msg = nl80211cmd()
+ msg['cmd'] = NL80211_NAMES['NL80211_CMD_GET_INTERFACE']
+ msg['attrs'] = [['NL80211_ATTR_WIPHY', attr]]
+ return self.nlm_request(msg,
+ msg_type=self.prid,
+ msg_flags=NLM_F_REQUEST | NLM_F_DUMP)
+
+ def get_interface_by_ifindex(self, ifindex):
+ '''
+ Get interface by ifindex ( use x.get_attr('NL80211_ATTR_IFINDEX')
+ '''
+ msg = nl80211cmd()
+ msg['cmd'] = NL80211_NAMES['NL80211_CMD_GET_INTERFACE']
+ msg['attrs'] = [['NL80211_ATTR_IFINDEX', ifindex]]
+ return self.nlm_request(msg,
+ msg_type=self.prid,
+ msg_flags=NLM_F_REQUEST)
+
+ def connect(self, ifindex, ssid, bssid=None):
+ '''
+ Connect to the ap with ssid and bssid
+ '''
+ msg = nl80211cmd()
+ msg['cmd'] = NL80211_NAMES['NL80211_CMD_CONNECT']
+ msg['attrs'] = [['NL80211_ATTR_IFINDEX', ifindex],
+ ['NL80211_ATTR_SSID', ssid]]
+ if bssid is not None:
+ msg['attrs'].append(['NL80211_ATTR_MAC', bssid])
+
+ self.nlm_request(msg,
+ msg_type=self.prid,
+ msg_flags=NLM_F_REQUEST | NLM_F_ACK)
+
+ def disconnect(self, ifindex):
+ '''
+ Disconnect the device
+ '''
+ msg = nl80211cmd()
+ msg['cmd'] = NL80211_NAMES['NL80211_CMD_DISCONNECT']
+ msg['attrs'] = [['NL80211_ATTR_IFINDEX', ifindex]]
+ self.nlm_request(msg,
+ msg_type=self.prid,
+ msg_flags=NLM_F_REQUEST | NLM_F_ACK)
+
+ def scan(self, ifindex):
+ '''
+ Scan wifi
+ '''
+ # Prepare a second netlink socket to get the scan results.
+ # The issue is that the kernel can send the results notification
+ # before we get answer for the NL80211_CMD_TRIGGER_SCAN
+ nsock = NL80211()
+ nsock.bind()
+ nsock.add_membership('scan')
+
+ # send scan request
+ msg = nl80211cmd()
+ msg['cmd'] = NL80211_NAMES['NL80211_CMD_TRIGGER_SCAN']
+ msg['attrs'] = [['NL80211_ATTR_IFINDEX', ifindex]]
+ self.nlm_request(msg,
+ msg_type=self.prid,
+ msg_flags=NLM_F_REQUEST | NLM_F_ACK)
+
+ # monitor the results notification on the secondary socket
+ scanResultNotFound = True
+ while scanResultNotFound:
+ listMsg = nsock.get()
+ for msg in listMsg:
+ if msg["event"] == "NL80211_CMD_NEW_SCAN_RESULTS":
+ scanResultNotFound = False
+ break
+ # close the secondary socket
+ nsock.close()
+
+ # request the results
+ msg2 = nl80211cmd()
+ msg2['cmd'] = NL80211_NAMES['NL80211_CMD_GET_SCAN']
+ msg2['attrs'] = [['NL80211_ATTR_IFINDEX', ifindex]]
+ return self.nlm_request(msg2, msg_type=self.prid,
+ msg_flags=NLM_F_REQUEST | NLM_F_DUMP)
diff --git a/node-admin/scripts/pyroute2/netlink/__init__.py b/node-admin/scripts/pyroute2/netlink/__init__.py
new file mode 100644
index 00000000000..499c7ab79d3
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/__init__.py
@@ -0,0 +1,1349 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+Netlink basics
+==============
+
+General netlink packet structure::
+
+ nlmsg packet:
+ + header
+ + data
+
+Generic netlink message header::
+
+ nlmsg header:
+ + uint32 length
+ + uint16 type
+ + uint16 flags
+ + uint32 sequence number
+ + uint32 pid
+
+The `length` field is the length of all the packet, including
+data and header. The `type` field is used to distinguish different
+message types, commands etc. Please note, that there is no
+explicit protocol field -- you choose a netlink protocol, when
+you create a socket.
+
+The `sequence number` is very important. Netlink is an asynchronous
+protocol -- it means, that the packet order doesn't matter and is
+not guaranteed. But responses to a request are always marked with
+the same sequence number, so you can treat it as a cookie.
+
+Please keep in mind, that a netlink request can initiate a
+cascade of events, and netlink messages from these events can
+carry sequence number == 0. E.g., it is so when you remove a
+primary IP addr from an interface, when `promote_secondaries`
+sysctl is set.
+
+Beside of incapsulated headers and other protocol-specific data,
+netlink messages can carry NLA (netlink attributes). NLA
+structure is as follows::
+
+ NLA header:
+ + uint16 length
+ + uint16 type
+ NLA data:
+ + data-specific struct
+ # optional:
+ + NLA
+ + NLA
+ + ...
+
+So, NLA structures can be nested, forming a tree.
+
+Complete structure of a netlink packet::
+
+ nlmsg header:
+ + uint32 length
+ + uint16 type
+ + uint16 flags
+ + uint32 sequence number
+ + uint32 pid
+ [ optional protocol-specific data ]
+ [ optional NLA tree ]
+
+More information about netlink protocol you can find in
+the man pages.
+
+Pyroute2 and netlink
+====================
+
+packets
+-------
+
+To simplify the development, pyroute2 provides an easy way to
+describe packet structure. As an example, you can take the
+ifaddrmsg description -- `pyroute2/netlink/rtnl/ifaddrmsg.py`.
+
+To describe a packet, you need to inherit from `nlmsg` class::
+
+ from pyroute2.netlink import nlmsg
+
+ class foo_msg(nlmsg):
+ fields = ( ... )
+ nla_map = ( ... )
+
+NLA are described in the same way, but the parent class should be
+`nla`, instead of `nlmsg`. And yes, it is important to use the
+proper parent class -- it affects the header structure.
+
+fields attribute
+----------------
+
+The `fields` attribute describes the structure of the
+protocol-specific data. It is a tuple of tuples, where each
+member contains a field name and its data format.
+
+Field data format should be specified as for Python `struct`
+module. E.g., ifaddrmsg structure::
+
+ ifaddrmsg structure:
+ + unsigned char ifa_family
+ + unsigned char ifa_prefixlen
+ + unsigned char ifa_flags
+ + unsigned char ifa_scope
+ + int ifa_index
+
+should be described as follows::
+
+ class ifaddrmsg(nlmsg):
+ fields = (('family', 'B'),
+ ('prefixlen', 'B'),
+ ('flags', 'B'),
+ ('scope', 'B'),
+ ('index', 'I'))
+
+Format strings are passed directly to the `struct` module,
+so you can use all the notations like `>I`, `16s` etc. All
+fields are parsed from the stream separately, so if you
+want to explicitly fix alignemt, as if it were C struct,
+use the `pack` attribute::
+
+ class tstats(nla):
+ pack = 'struct'
+ fields = (('version', 'H'),
+ ('ac_exitcode', 'I'),
+ ('ac_flag', 'B'),
+ ...)
+
+Explicit padding bytes also can be used, when struct
+packing doesn't work well::
+
+ class ipq_mode_msg(nlmsg):
+ pack = 'struct'
+ fields = (('value', 'B'),
+ ('__pad', '7x'),
+ ('range', 'I'),
+ ('__pad', '12x'))
+
+
+nla_map attribute
+-----------------
+
+The `nla_map` attribute is a tuple of NLA descriptions. Each
+description is also a tuple in two different forms: either
+two fields, name and format, or three fields: type, name and
+format.
+
+Please notice, that the format field is a string name of
+corresponding NLA class::
+
+ class ifaddrmsg(nlmsg):
+ ...
+ nla_map = (('IFA_UNSPEC', 'hex'),
+ ('IFA_ADDRESS', 'ipaddr'),
+ ('IFA_LOCAL', 'ipaddr'),
+ ...)
+
+This code will create mapping, where IFA_ADDRESS NLA will be of
+type 1 and IFA_LOCAL -- of type 2, etc. Both NLA will be decoded
+as IP addresses (class `ipaddr`). IFA_UNSPEC will be of type 0,
+and if it will be in the NLA tree, it will be just dumped in hex.
+
+NLA class names are should be specified as strings, since they
+are resolved in runtime.
+
+There are several pre-defined NLA types, that you will get with
+`nla` class:
+
+ - none # forces pyroute2 just to skip this NLA
+ - uint8
+ - uint16
+ - uint32 # there are dedicated NLA of these types as well
+ - ipaddr # IP address, IPv4 or IPv6, depending on the socket
+ - l2addr # MAC address
+ - hex # hex dump as a string -- useful for debugging
+ - cdata # just a binary string
+ - asciiz # zero-terminated ASCII string
+
+Please refer to `pyroute2/netlink/__init__.py` for details.
+
+You can also make your own NLA descriptions::
+
+ class ifaddrmsg(nlmsg):
+ ...
+ nla_map = (...
+ ('IFA_CACHEINFO', 'cacheinfo'),
+ ...)
+
+ class cacheinfo(nla):
+ fields = (('ifa_prefered', 'I'),
+ ('ifa_valid', 'I'),
+ ('cstamp', 'I'),
+ ('tstamp', 'I'))
+
+Custom NLA descriptions should be defined in the same class,
+where they are used.
+
+Also, it is possible to use not autogenerated type numbers, as
+for ifaddrmsg, but specify them explicitly::
+
+ class iw_event(nla):
+ ...
+ nla_map = ((0x8B00, 'SIOCSIWCOMMIT', 'hex'),
+ (0x8B01, 'SIOCGIWNAME', 'hex'),
+ (0x8B02, 'SIOCSIWNWID', 'hex'),
+ (0x8B03, 'SIOCGIWNWID', 'hex'),
+ ...)
+
+Here you can see custom NLA type numbers -- 0x8B00, 0x8B01 etc.
+It is not permitted to mix these two forms in one class: you should
+use ether autogenerated type numbers (two fields tuples), or
+explicit numbers (three fields typles).
+
+parsed netlink message
+----------------------
+
+Netlink messages are represented by pyroute2 as dictionaries
+as follows::
+
+ {'header': {'pid': ...,
+ 'length: ...,
+ 'flags': ...,
+ 'error': None, # if you are lucky
+ 'type': ...,
+ 'sequence_number': ...},
+
+ # fields attributes
+ 'field_name1': value,
+ ...
+ 'field_nameX': value,
+
+ # nla tree
+ 'attrs': [['NLA_NAME1', value],
+ ...
+ ['NLA_NAMEX', value],
+ ['NLA_NAMEY', {'field_name1': value,
+ ...
+ 'field_nameX': value,
+ 'attrs': [['NLA_NAME.... ]]}]]}
+
+As an example, a message from the wireless subsystem about new
+scan event::
+
+ {'index': 4,
+ 'family': 0,
+ '__align': 0,
+ 'header': {'pid': 0,
+ 'length': 64,
+ 'flags': 0,
+ 'error': None,
+ 'type': 16,
+ 'sequence_number': 0},
+ 'flags': 69699,
+ 'ifi_type': 1,
+ 'event': 'RTM_NEWLINK',
+ 'change': 0,
+ 'attrs': [['IFLA_IFNAME', 'wlp3s0'],
+ ['IFLA_WIRELESS',
+ {'attrs': [['SIOCGIWSCAN',
+ '00:00:00:00:00:00:00:00:00:00:00:00']]}]]}
+
+create and send messages
+------------------------
+
+Using high-level interfaces like `IPRoute` or `IPDB`, you will never
+need to manually construct and send netlink messages. But in the case
+you really need it, it is simple as well.
+
+Having a description class, like `ifaddrmsg` from above, you need to:
+
+ - instantiate it
+ - fill the fields
+ - encode the packet
+ - send the encoded data
+
+The code::
+
+ from pyroute2.netlink import NLM_F_REQUEST
+ from pyroute2.netlink import NLM_F_ACK
+ from pyroute2.netlink import NLM_F_CREATE
+ from pyroute2.netlink import NLM_F_EXCL
+ from pyroute2.iproute import RTM_NEWADDR
+ from pyroute2.netlink.rtnl.ifaddrmsg import ifaddrmsg
+
+ ##
+ # add an addr to an interface
+ #
+
+ # create the message
+ msg = ifaddrmsg()
+
+ # fill the protocol-specific fields
+ msg['index'] = index # index of the interface
+ msg['family'] = AF_INET # address family
+ msg['prefixlen'] = 24 # the address mask
+ msg['scope'] = scope # see /etc/iproute2/rt_scopes
+
+ # attach NLA -- it MUST be a list / mutable
+ msg['attrs'] = [['IFA_LOCAL', '192.168.0.1'],
+ ['IFA_ADDRESS', '192.162.0.1']]
+
+ # fill generic netlink fields
+ msg['header']['sequence_number'] = nonce # an unique seq number
+ msg['header']['pid'] = os.getpid()
+ msg['header']['type'] = RTM_NEWADDR
+ msg['header']['flags'] = NLM_F_REQUEST |\\
+ NLM_F_ACK |\\
+ NLM_F_CREATE |\\
+ NLM_F_EXCL
+
+ # encode the packet
+ msg.encode()
+
+ # send the buffer
+ nlsock.sendto(msg.buf.getvalue(), (0, 0))
+
+Please notice, that NLA list *MUST* be mutable.
+
+------------------
+
+Module contents:
+'''
+
+import traceback
+import logging
+import socket
+import struct
+import types
+import sys
+import io
+import re
+import os
+
+from pyroute2.common import hexdump
+from pyroute2.common import basestring
+
+_letters = re.compile('[A-Za-z]')
+_fmt_letters = re.compile('[^!><@=][!><@=]')
+
+##
+# That's a hack for the code linter, which works under
+# Python3, see unicode reference in the code below
+if sys.version[0] == '3':
+ unicode = str
+
+NLMSG_MIN_TYPE = 0x10
+
+GENL_NAMSIZ = 16 # length of family name
+GENL_MIN_ID = NLMSG_MIN_TYPE
+GENL_MAX_ID = 1023
+
+GENL_ADMIN_PERM = 0x01
+GENL_CMD_CAP_DO = 0x02
+GENL_CMD_CAP_DUMP = 0x04
+GENL_CMD_CAP_HASPOL = 0x08
+
+#
+# List of reserved static generic netlink identifiers:
+#
+GENL_ID_GENERATE = 0
+GENL_ID_CTRL = NLMSG_MIN_TYPE
+
+#
+# Controller
+#
+
+CTRL_CMD_UNSPEC = 0x0
+CTRL_CMD_NEWFAMILY = 0x1
+CTRL_CMD_DELFAMILY = 0x2
+CTRL_CMD_GETFAMILY = 0x3
+CTRL_CMD_NEWOPS = 0x4
+CTRL_CMD_DELOPS = 0x5
+CTRL_CMD_GETOPS = 0x6
+CTRL_CMD_NEWMCAST_GRP = 0x7
+CTRL_CMD_DELMCAST_GRP = 0x8
+CTRL_CMD_GETMCAST_GRP = 0x9 # unused
+
+
+CTRL_ATTR_UNSPEC = 0x0
+CTRL_ATTR_FAMILY_ID = 0x1
+CTRL_ATTR_FAMILY_NAME = 0x2
+CTRL_ATTR_VERSION = 0x3
+CTRL_ATTR_HDRSIZE = 0x4
+CTRL_ATTR_MAXATTR = 0x5
+CTRL_ATTR_OPS = 0x6
+CTRL_ATTR_MCAST_GROUPS = 0x7
+
+CTRL_ATTR_OP_UNSPEC = 0x0
+CTRL_ATTR_OP_ID = 0x1
+CTRL_ATTR_OP_FLAGS = 0x2
+
+CTRL_ATTR_MCAST_GRP_UNSPEC = 0x0
+CTRL_ATTR_MCAST_GRP_NAME = 0x1
+CTRL_ATTR_MCAST_GRP_ID = 0x2
+
+
+# Different Netlink families
+#
+NETLINK_ROUTE = 0 # Routing/device hook
+NETLINK_UNUSED = 1 # Unused number
+NETLINK_USERSOCK = 2 # Reserved for user mode socket protocols
+NETLINK_FIREWALL = 3 # Firewalling hook
+NETLINK_INET_DIAG = 4 # INET socket monitoring
+NETLINK_NFLOG = 5 # netfilter/iptables ULOG
+NETLINK_XFRM = 6 # ipsec
+NETLINK_SELINUX = 7 # SELinux event notifications
+NETLINK_ISCSI = 8 # Open-iSCSI
+NETLINK_AUDIT = 9 # auditing
+NETLINK_FIB_LOOKUP = 10
+NETLINK_CONNECTOR = 11
+NETLINK_NETFILTER = 12 # netfilter subsystem
+NETLINK_IP6_FW = 13
+NETLINK_DNRTMSG = 14 # DECnet routing messages
+NETLINK_KOBJECT_UEVENT = 15 # Kernel messages to userspace
+NETLINK_GENERIC = 16
+# leave room for NETLINK_DM (DM Events)
+NETLINK_SCSITRANSPORT = 18 # SCSI Transports
+
+# NLA flags
+NLA_F_NESTED = 1 << 15
+NLA_F_NET_BYTEORDER = 1 << 14
+
+NLMSG_ALIGNTO = 4
+
+
+class NetlinkError(Exception):
+ '''
+ Base netlink error
+ '''
+ def __init__(self, code, msg=None):
+ msg = msg or os.strerror(code)
+ super(NetlinkError, self).__init__(code, msg)
+ self.code = code
+
+
+class NetlinkDecodeError(Exception):
+ '''
+ Base decoding error class.
+
+ Incapsulates underlying error for the following analysis
+ '''
+ def __init__(self, exception):
+ self.exception = exception
+
+
+class NetlinkHeaderDecodeError(NetlinkDecodeError):
+ '''
+ The error occured while decoding a header
+ '''
+ pass
+
+
+class NetlinkDataDecodeError(NetlinkDecodeError):
+ '''
+ The error occured while decoding the message fields
+ '''
+ pass
+
+
+class NetlinkNLADecodeError(NetlinkDecodeError):
+ '''
+ The error occured while decoding NLA chain
+ '''
+ pass
+
+
+def NLMSG_ALIGN(l):
+ return (l + NLMSG_ALIGNTO - 1) & ~ (NLMSG_ALIGNTO - 1)
+
+
+class NotInitialized(Exception):
+ pass
+
+
+# Netlink message flags values (nlmsghdr.flags)
+#
+NLM_F_REQUEST = 1 # It is request message.
+NLM_F_MULTI = 2 # Multipart message, terminated by NLMSG_DONE
+NLM_F_ACK = 4 # Reply with ack, with zero or error code
+NLM_F_ECHO = 8 # Echo this request
+# Modifiers to GET request
+NLM_F_ROOT = 0x100 # specify tree root
+NLM_F_MATCH = 0x200 # return all matching
+NLM_F_ATOMIC = 0x400 # atomic GET
+NLM_F_DUMP = (NLM_F_ROOT | NLM_F_MATCH)
+# Modifiers to NEW request
+NLM_F_REPLACE = 0x100 # Override existing
+NLM_F_EXCL = 0x200 # Do not touch, if it exists
+NLM_F_CREATE = 0x400 # Create, if it does not exist
+NLM_F_APPEND = 0x800 # Add to end of list
+
+NLMSG_NOOP = 0x1 # Nothing
+NLMSG_ERROR = 0x2 # Error
+NLMSG_DONE = 0x3 # End of a dump
+NLMSG_OVERRUN = 0x4 # Data lost
+NLMSG_CONTROL = 0xe # Custom message type for messaging control
+NLMSG_TRANSPORT = 0xf # Custom message type for NL as a transport
+NLMSG_MIN_TYPE = 0x10 # < 0x10: reserved control messages
+NLMSG_MAX_LEN = 0xffff # Max message length
+
+mtypes = {1: 'NLMSG_NOOP',
+ 2: 'NLMSG_ERROR',
+ 3: 'NLMSG_DONE',
+ 4: 'NLMSG_OVERRUN'}
+
+IPRCMD_NOOP = 0
+IPRCMD_STOP = 1
+IPRCMD_ACK = 2
+IPRCMD_ERR = 3
+IPRCMD_REGISTER = 4
+IPRCMD_RELOAD = 5
+IPRCMD_ROUTE = 6
+IPRCMD_CONNECT = 7
+IPRCMD_DISCONNECT = 8
+IPRCMD_SERVE = 9
+IPRCMD_SHUTDOWN = 10
+IPRCMD_SUBSCRIBE = 11
+IPRCMD_UNSUBSCRIBE = 12
+IPRCMD_PROVIDE = 13
+IPRCMD_REMOVE = 14
+IPRCMD_DISCOVER = 15
+IPRCMD_UNREGISTER = 16
+
+SOL_NETLINK = 270
+
+NETLINK_ADD_MEMBERSHIP = 1
+NETLINK_DROP_MEMBERSHIP = 2
+NETLINK_PKTINFO = 3
+NETLINK_BROADCAST_ERROR = 4
+NETLINK_NO_ENOBUFS = 5
+NETLINK_RX_RING = 6
+NETLINK_TX_RING = 7
+
+
+class nlmsg_base(dict):
+ '''
+ Netlink base class. You do not need to inherit it directly, unless
+ you're inventing completely new protocol structure.
+
+ Use nlmsg or nla classes.
+
+ The class provides several methods, but often one need to customize
+ only `decode()` and `encode()`.
+ '''
+
+ fields = [] # data field names, to build a dictionary
+ header = None # optional header class
+ pack = None # pack pragma
+ array = False
+ nla_map = {} # NLA mapping
+ nla_flags = 0 # NLA flags
+ value_map = {}
+
+ def __init__(self, buf=None, length=None, parent=None, debug=False):
+ dict.__init__(self)
+ for i in self.fields:
+ self[i[0]] = 0 # FIXME: only for number values
+ self.raw = None
+ self.debug = debug
+ self.length = length or 0
+ self.parent = parent
+ self.offset = 0
+ self.prefix = None
+ self['attrs'] = []
+ self['value'] = NotInitialized
+ self.value = NotInitialized
+ self.register_nlas()
+ self.r_value_map = dict([(x[1], x[0]) for x in self.value_map.items()])
+ self.reset(buf)
+ self.clean_cbs = []
+ if self.header is not None:
+ self['header'] = self.header(self.buf)
+
+ def copy(self):
+ '''
+ Return a decoded copy of the netlink message. Works
+ correctly only if the message was encoded, or is
+ received from the socket.
+ '''
+ ret = type(self)(self.buf.getvalue())
+ ret.decode()
+ return ret
+
+ def reset(self, buf=None):
+ '''
+ Reset the message buffer. Optionally, set the message
+ from the `buf` parameter. This parameter can be either
+ string, or io.BytesIO, or dict instance.
+ '''
+ if isinstance(buf, basestring):
+ b = io.BytesIO()
+ b.write(buf)
+ b.seek(0)
+ buf = b
+ if isinstance(buf, dict):
+ self.setvalue(buf)
+ buf = None
+ self.buf = buf or io.BytesIO()
+ if 'header' in self:
+ self['header'].buf = self.buf
+
+ def register_clean_cb(self, cb):
+ if self.parent is not None:
+ return self.parent.register_clean_cb(cb)
+ else:
+ self.clean_cbs.append(cb)
+
+ def _strip_one(self, name):
+ for i in tuple(self['attrs']):
+ if i[0] == name:
+ self['attrs'].remove(i)
+ return self
+
+ def strip(self, attrs):
+ '''
+ Remove an NLA from the attrs chain. The `attrs`
+ parameter can be either string, or iterable. In
+ the latter case, will be stripped NLAs, specified
+ in the provided list.
+ '''
+ if isinstance(attrs, basestring):
+ self._strip_one(attrs)
+ else:
+ for name in attrs:
+ self._strip_one(name)
+ return self
+
+ def __ops(self, rvalue, op0, op1):
+ lvalue = self.getvalue()
+ res = self.__class__()
+ for key in lvalue:
+ if key not in ('header', 'attrs'):
+ if op0 == '__sub__':
+ # operator -, complement
+ if (key not in rvalue) or (lvalue[key] != rvalue[key]):
+ res[key] = lvalue[key]
+ elif op0 == '__and__':
+ # operator &, intersection
+ if (key in rvalue) and (lvalue[key] == rvalue[key]):
+ res[key] = lvalue[key]
+ if 'attrs' in lvalue:
+ res['attrs'] = []
+ for attr in lvalue['attrs']:
+ if isinstance(attr[1], nla):
+ diff = getattr(attr[1], op0)(rvalue.get_attr(attr[0]))
+ if diff is not None:
+ res['attrs'].append([attr[0], diff])
+ else:
+ if op0 == '__sub__':
+ # operator -, complement
+ if rvalue.get_attr(attr[0]) != attr[1]:
+ res['attrs'].append(attr)
+ elif op0 == '__and__':
+ # operator &, intersection
+ if rvalue.get_attr(attr[0]) == attr[1]:
+ res['attrs'].append(attr)
+ if not len(res):
+ return None
+ else:
+ if 'header' in res:
+ del res['header']
+ if 'value' in res:
+ del res['value']
+ if 'attrs' in res and not len(res['attrs']):
+ del res['attrs']
+ return res
+
+ def __sub__(self, rvalue):
+ '''
+ Subjunction operation.
+ '''
+ return self.__ops(rvalue, '__sub__', '__ne__')
+
+ def __and__(self, rvalue):
+ '''
+ Conjunction operation.
+ '''
+ return self.__ops(rvalue, '__and__', '__eq__')
+
+ def __eq__(self, rvalue):
+ '''
+ Having nla, we are able to use it in operations like::
+
+ if nla == 'some value':
+ ...
+ '''
+ lvalue = self.getvalue()
+ if lvalue is self:
+ for key in self:
+ try:
+ assert self.get(key) == rvalue.get(key)
+ except Exception:
+ # on any error -- is not equal
+ return False
+ return True
+ else:
+ return lvalue == rvalue
+
+ @classmethod
+ def get_size(self):
+ size = 0
+ for field in self.fields:
+ size += struct.calcsize(field[1])
+ return size
+
+ @classmethod
+ def nla2name(self, name):
+ '''
+ Convert NLA name into human-friendly name
+
+ Example: IFLA_ADDRESS -> address
+
+ Requires self.prefix to be set
+ '''
+ return name[(name.find(self.prefix) + 1) * len(self.prefix):].lower()
+
+ @classmethod
+ def name2nla(self, name):
+ '''
+ Convert human-friendly name into NLA name
+
+ Example: address -> IFLA_ADDRESS
+
+ Requires self.prefix to be set
+ '''
+ name = name.upper()
+ if name.find(self.prefix) == -1:
+ name = "%s%s" % (self.prefix, name)
+ return name
+
+ def reserve(self):
+ '''
+ Reserve space in the buffer for data. This can be used
+ to skip encoding of the header until some fields will
+ be known.
+ '''
+ size = 0
+ for i in self.fields:
+ size += struct.calcsize(i[1])
+ self.buf.seek(size, 1)
+
+ def decode(self):
+ '''
+ Decode the message. The message should have the `buf`
+ attribute initialized. e.g.::
+
+ data = sock.recv(16384)
+ msg = ifinfmsg(data)
+
+ If you want to customize the decoding process, override
+ the method, but don't forget to call parent's `decode()`::
+
+ class CustomMessage(nlmsg):
+
+ def decode(self):
+ nlmsg.decode(self)
+ ... # do some custom data tuning
+ '''
+ self.offset = self.buf.tell()
+ # decode the header
+ if self.header is not None:
+ try:
+ self['header'].decode()
+ # update length from header
+ # it can not be less than 4
+ self.length = max(self['header']['length'], 4)
+ save = self.buf.tell()
+ self.buf.seek(self.offset)
+ self.raw = self.buf.read(self.length)
+ self.buf.seek(save)
+ except Exception as e:
+ raise NetlinkHeaderDecodeError(e)
+ # handle the array case
+ if self.array:
+ self.setvalue([])
+ while self.buf.tell() < self.offset + self.length:
+ cell = type(self)(self.buf, parent=self, debug=self.debug)
+ cell.array = False
+ cell.decode()
+ self.value.append(cell)
+ # decode the data
+ try:
+ if self.pack == 'struct':
+ names = []
+ formats = []
+ for field in self.fields:
+ names.append(field[0])
+ formats.append(field[1])
+ fields = ((','.join(names), ''.join(formats)), )
+ else:
+ fields = self.fields
+
+ for field in fields:
+ name = field[0]
+ fmt = field[1]
+
+ # 's' and 'z' can be used only in connection with
+ # length, encoded in the header
+ if field[1] in ('s', 'z'):
+ fmt = '%is' % (self.length - 4)
+
+ size = struct.calcsize(fmt)
+ raw = self.buf.read(size)
+ actual_size = len(raw)
+
+ # FIXME: adjust string size again
+ if field[1] in ('s', 'z'):
+ size = actual_size
+ fmt = '%is' % (actual_size)
+ if size == actual_size:
+ value = struct.unpack(fmt, raw)
+ if len(value) == 1:
+ self[name] = value[0]
+ # cut zero-byte from z-strings
+ # 0x00 -- python3; '\0' -- python2
+ if field[1] == 'z' and self[name][-1] in (0x00, '\0'):
+ self[name] = self[name][:-1]
+ else:
+ if self.pack == 'struct':
+ names = name.split(',')
+ values = list(value)
+ for name in names:
+ if name[0] != '_':
+ self[name] = values.pop(0)
+ else:
+ self[name] = value
+
+ else:
+ # FIXME: log an error
+ pass
+
+ except Exception as e:
+ raise NetlinkDataDecodeError(e)
+ # decode NLA
+ try:
+ # align NLA chain start
+ self.buf.seek(NLMSG_ALIGN(self.buf.tell()))
+ # read NLA chain
+ if self.nla_map:
+ self.decode_nlas()
+ except Exception as e:
+ logging.warning(traceback.format_exc())
+ raise NetlinkNLADecodeError(e)
+ if len(self['attrs']) == 0:
+ del self['attrs']
+ if self['value'] is NotInitialized:
+ del self['value']
+
+ def encode(self):
+ '''
+ Encode the message into the binary buffer::
+
+ msg.encode()
+ sock.send(msg.buf.getvalue())
+
+ If you want to customize the encoding process, override
+ the method::
+
+ class CustomMessage(nlmsg):
+
+ def encode(self):
+ ... # do some custom data tuning
+ nlmsg.encode(self)
+ '''
+ init = self.buf.tell()
+ diff = 0
+ # reserve space for the header
+ if self.header is not None:
+ self['header'].reserve()
+
+ if self.getvalue() is not None:
+
+ payload = b''
+ for i in self.fields:
+ name = i[0]
+ fmt = i[1]
+ value = self[name]
+
+ if fmt == 's':
+ length = len(value)
+ fmt = '%is' % (length)
+ elif fmt == 'z':
+ length = len(value) + 1
+ fmt = '%is' % (length)
+
+ # in python3 we should force it
+ if sys.version[0] == '3':
+ if isinstance(value, str):
+ value = bytes(value, 'utf-8')
+ elif isinstance(value, float):
+ value = int(value)
+ elif sys.version[0] == '2':
+ if isinstance(value, unicode):
+ value = value.encode('utf-8')
+
+ try:
+ if fmt[-1] == 'x':
+ payload += struct.pack(fmt)
+ elif type(value) in (list, tuple, set):
+ payload += struct.pack(fmt, *value)
+ else:
+ payload += struct.pack(fmt, value)
+ except struct.error:
+ logging.error(traceback.format_exc())
+ logging.error("error pack: %s %s %s" %
+ (fmt, value, type(value)))
+ raise
+
+ diff = NLMSG_ALIGN(len(payload)) - len(payload)
+ self.buf.write(payload)
+ self.buf.write(b'\0' * diff)
+ # write NLA chain
+ if self.nla_map:
+ diff = 0
+ self.encode_nlas()
+ # calculate the size and write it
+ if self.header is not None:
+ self.update_length(init, diff)
+
+ def update_length(self, start, diff=0):
+ save = self.buf.tell()
+ self['header']['length'] = save - start - diff
+ self.buf.seek(start)
+ self['header'].encode()
+ self.buf.seek(save)
+
+ def setvalue(self, value):
+ if isinstance(value, dict):
+ self.update(value)
+ else:
+ try:
+ value = self.r_value_map.get(value, value)
+ except TypeError:
+ pass
+ self['value'] = value
+ self.value = value
+
+ def get_encoded(self, attr, default=None):
+ '''
+ Return the first encoded NLA by name
+ '''
+ return self.get_attr(attr, default, 'encoded')
+
+ def get_attr(self, attr, default=None, fmt='raw'):
+ '''
+ Return the first attr by name or None
+ '''
+ try:
+ attrs = self.get_attrs(attr, fmt)
+ except KeyError:
+ return default
+ if attrs:
+ return attrs[0]
+ else:
+ return default
+
+ def get_attrs(self, attr, fmt='raw'):
+ '''
+ Return attrs by name
+ '''
+ fmt_map = {'raw': 1,
+ 'encoded': 2}
+ return [i[fmt_map[fmt]] for i in self['attrs'] if i[0] == attr]
+
+ def getvalue(self):
+ '''
+ Atomic NLAs return their value in the 'value' field,
+ not as a dictionary. Complex NLAs return whole dictionary.
+ '''
+ if self.value != NotInitialized:
+ # value decoded by custom decoder
+ return self.value
+
+ if 'value' in self and self['value'] != NotInitialized:
+ # raw value got by generic decoder
+ return self.value_map.get(self['value'], self['value'])
+
+ return self
+
+ def register_nlas(self):
+ '''
+ Convert 'nla_map' tuple into two dictionaries for mapping
+ and reverse mapping of NLA types.
+
+ ex: given::
+
+ nla_map = (('TCA_HTB_UNSPEC', 'none'),
+ ('TCA_HTB_PARMS', 'htb_parms'),
+ ('TCA_HTB_INIT', 'htb_glob'))
+
+ creates::
+
+ t_nla_map = {0: (<class 'pyroute2...none'>, 'TCA_HTB_UNSPEC'),
+ 1: (<class 'pyroute2...htb_parms'>, 'TCA_HTB_PARMS'),
+ 2: (<class 'pyroute2...htb_glob'>, 'TCA_HTB_INIT')}
+ r_nla_map = {'TCA_HTB_UNSPEC': (<class 'pyroute2...none'>, 0),
+ 'TCA_HTB_PARMS': (<class 'pyroute2...htb_parms'>, 1),
+ 'TCA_HTB_INIT': (<class 'pyroute2...htb_glob'>, 2)}
+
+ nla_map format::
+
+ nla_map = (([ID, ] NAME, TYPE[, FLAGS]), ...)
+
+ Items in `[...]` are optional. If ID is not given, then the map will
+ be autonumerated from 0. If flags are not given, they are 0 by default.
+
+ '''
+ # clean up NLA mappings
+ self.t_nla_map = {}
+ self.r_nla_map = {}
+
+ # work only on non-empty mappings
+ if not self.nla_map:
+ return
+
+ # fix nla flags
+ nla_map = []
+ for item in self.nla_map:
+ if not isinstance(item[-1], int):
+ item = list(item)
+ item.append(0)
+ nla_map.append(item)
+
+ # detect, whether we have pre-defined keys
+ if not isinstance(nla_map[0][0], int):
+ # create enumeration
+ nla_types = enumerate((i[0] for i in nla_map))
+ # that's a little bit tricky, but to reduce
+ # the required amount of code in modules, we have
+ # to jump over the head
+ zipped = [(k[1][0], k[0][0], k[0][1], k[0][2]) for k in
+ zip(nla_map, nla_types)]
+ else:
+ zipped = nla_map
+
+ for (key, name, nla_class, nla_flags) in zipped:
+ # is it an array
+ if nla_class[0] == '*':
+ nla_class = nla_class[1:]
+ array = True
+ else:
+ array = False
+ # lookup NLA class
+ if nla_class == 'recursive':
+ nla_class = type(self)
+ else:
+ nla_class = getattr(self, nla_class)
+ # update mappings
+ self.t_nla_map[key] = (nla_class, name, nla_flags, array)
+ self.r_nla_map[name] = (nla_class, key, nla_flags, array)
+
+ def encode_nlas(self):
+ '''
+ Encode the NLA chain. Should not be called manually, since
+ it is called from `encode()` routine.
+ '''
+ for i in self['attrs']:
+ if i[0] in self.r_nla_map:
+ msg_class = self.r_nla_map[i[0]][0]
+ msg_type = self.r_nla_map[i[0]][1]
+ # is it a class or a function?
+ if isinstance(msg_class, types.MethodType):
+ # if it is a function -- use it to get the class
+ msg_class = msg_class()
+ # encode NLA
+ nla = msg_class(self.buf, parent=self)
+ nla.nla_flags |= self.r_nla_map[i[0]][2]
+ nla['header']['type'] = msg_type | nla.nla_flags
+ nla.setvalue(i[1])
+ try:
+ nla.encode()
+ except:
+ raise
+ else:
+ if len(i) == 2:
+ i.append(nla)
+ elif len(i) == 3:
+ i[2] = nla
+
+ def decode_nlas(self):
+ '''
+ Decode the NLA chain. Should not be called manually, since
+ it is called from `decode()` routine.
+ '''
+ while self.buf.tell() < (self.offset + self.length):
+ init = self.buf.tell()
+ nla = None
+ # pick the length and the type
+ (length, msg_type) = struct.unpack('HH', self.buf.read(4))
+ # first two bits of msg_type are flags:
+ msg_type = msg_type & ~(NLA_F_NESTED | NLA_F_NET_BYTEORDER)
+ # rewind to the beginning
+ self.buf.seek(init)
+ length = min(max(length, 4),
+ (self.length - self.buf.tell() + self.offset))
+
+ # we have a mapping for this NLA
+ if msg_type in self.t_nla_map:
+ # get the class
+ msg_class = self.t_nla_map[msg_type][0]
+ # is it a class or a function?
+ if isinstance(msg_class, types.MethodType):
+ # if it is a function -- use it to get the class
+ msg_class = msg_class(buf=self.buf, length=length)
+ # and the name
+ msg_name = self.t_nla_map[msg_type][1]
+ # is it an array?
+ msg_array = self.t_nla_map[msg_type][3]
+ # decode NLA
+ nla = msg_class(self.buf, length, self, debug=self.debug)
+ nla.array = msg_array
+ try:
+ nla.decode()
+ nla.nla_flags = msg_type & (NLA_F_NESTED |
+ NLA_F_NET_BYTEORDER)
+ except Exception:
+ logging.warning("decoding %s" % (msg_name))
+ logging.warning(traceback.format_exc())
+ self.buf.seek(init)
+ msg_name = 'UNDECODED'
+ msg_value = hexdump(self.buf.read(length))
+ else:
+ msg_value = nla.getvalue()
+ else:
+ msg_name = 'UNKNOWN'
+ msg_value = hexdump(self.buf.read(length))
+
+ self['attrs'].append([msg_name, msg_value])
+
+ # fix the offset
+ self.buf.seek(init + NLMSG_ALIGN(length))
+
+
+class nla_header(nlmsg_base):
+ '''
+ The NLA header structure: uin16 length and uint16 type.
+ '''
+ fields = (('length', 'H'),
+ ('type', 'H'))
+
+
+class nla_base(nlmsg_base):
+ '''
+ The NLA base class. Use `nla_header` class as the header.
+ '''
+ header = nla_header
+
+
+class nlmsg_header(nlmsg_base):
+ '''
+ Common netlink message header
+ '''
+ fields = (('length', 'I'),
+ ('type', 'H'),
+ ('flags', 'H'),
+ ('sequence_number', 'I'),
+ ('pid', 'I'))
+
+
+class nlmsg_atoms(nlmsg_base):
+ '''
+ A collection of base NLA types
+ '''
+ class none(nla_base):
+ '''
+ 'none' type is used to skip decoding of NLA. You can
+ also use 'hex' type to dump NLA's content.
+ '''
+ def decode(self):
+ nla_base.decode(self)
+ self.value = None
+
+ class uint8(nla_base):
+ fields = [('value', 'B')]
+
+ class uint16(nla_base):
+ fields = [('value', 'H')]
+
+ class uint32(nla_base):
+ fields = [('value', 'I')]
+
+ class uint64(nla_base):
+ fields = [('value', 'Q')]
+
+ class be8(nla_base):
+ fields = [('value', '>B')]
+
+ class be16(nla_base):
+ fields = [('value', '>H')]
+
+ class be32(nla_base):
+ fields = [('value', '>I')]
+
+ class be64(nla_base):
+ fields = [('value', '>Q')]
+
+ class ipXaddr(nla_base):
+ fields = [('value', 's')]
+ family = None
+
+ def encode(self):
+ self['value'] = socket.inet_pton(self.family,
+ self.value)
+ nla_base.encode(self)
+
+ def decode(self):
+ nla_base.decode(self)
+ self.value = socket.inet_ntop(self.family,
+ self['value'])
+
+ class ip4addr(ipXaddr):
+ '''
+ Explicit IPv4 address type class.
+ '''
+ family = socket.AF_INET
+
+ class ip6addr(ipXaddr):
+ '''
+ Explicit IPv6 address type class.
+ '''
+ family = socket.AF_INET6
+
+ class ipaddr(nla_base):
+ '''
+ This class is used to decode IP addresses according to
+ the family. Socket library currently supports only two
+ families, AF_INET and AF_INET6.
+
+ We do not specify here the string size, it will be
+ calculated in runtime.
+ '''
+ fields = [('value', 's')]
+ family_map = {socket.AF_INET: socket.AF_INET,
+ socket.AF_BRIDGE: socket.AF_INET,
+ socket.AF_INET6: socket.AF_INET6}
+
+ def encode(self):
+ family = self.family_map[self.parent['family']]
+ self['value'] = socket.inet_pton(family, self.value)
+ nla_base.encode(self)
+
+ def decode(self):
+ nla_base.decode(self)
+ family = self.family_map[self.parent['family']]
+ self.value = socket.inet_ntop(family, self['value'])
+
+ class l2addr(nla_base):
+ '''
+ Decode MAC address.
+ '''
+ fields = [('value', '=6s')]
+
+ def encode(self):
+ self['value'] = struct.pack('BBBBBB',
+ *[int(i, 16) for i in
+ self.value.split(':')])
+ nla_base.encode(self)
+
+ def decode(self):
+ nla_base.decode(self)
+ self.value = ':'.join('%02x' % (i) for i in
+ struct.unpack('BBBBBB', self['value']))
+
+ class hex(nla_base):
+ '''
+ Represent NLA's content with header as hex string.
+ '''
+ fields = [('value', 's')]
+
+ def decode(self):
+ nla_base.decode(self)
+ self.value = hexdump(self['value'])
+
+ class cdata(nla_base):
+ '''
+ Binary data
+ '''
+ fields = [('value', 's')]
+
+ class asciiz(nla_base):
+ '''
+ Zero-terminated string.
+ '''
+ # FIXME: move z-string hacks from general decode here?
+ fields = [('value', 'z')]
+
+ def encode(self):
+ if isinstance(self['value'], str) and sys.version[0] == '3':
+ self['value'] = bytes(self['value'], 'utf-8')
+ nla_base.encode(self)
+
+ def decode(self):
+ nla_base.decode(self)
+ try:
+ assert sys.version[0] == '3'
+ self.value = self['value'].decode('utf-8')
+ except (AssertionError, UnicodeDecodeError):
+ self.value = self['value']
+
+
+class nla(nla_base, nlmsg_atoms):
+ '''
+ Main NLA class
+ '''
+ def decode(self):
+ nla_base.decode(self)
+ if not self.debug:
+ del self['header']
+
+
+class nlmsg(nlmsg_atoms):
+ '''
+ Main netlink message class
+ '''
+ header = nlmsg_header
+
+
+class genlmsg(nlmsg):
+ '''
+ Generic netlink message
+ '''
+ fields = (('cmd', 'B'),
+ ('version', 'B'),
+ ('reserved', 'H'))
+
+
+class ctrlmsg(genlmsg):
+ '''
+ Netlink control message
+ '''
+ # FIXME: to be extended
+ nla_map = (('CTRL_ATTR_UNSPEC', 'none'),
+ ('CTRL_ATTR_FAMILY_ID', 'uint16'),
+ ('CTRL_ATTR_FAMILY_NAME', 'asciiz'),
+ ('CTRL_ATTR_VERSION', 'uint32'),
+ ('CTRL_ATTR_HDRSIZE', 'uint32'),
+ ('CTRL_ATTR_MAXATTR', 'uint32'),
+ ('CTRL_ATTR_OPS', '*ops'),
+ ('CTRL_ATTR_MCAST_GROUPS', '*mcast_groups'))
+
+ class ops(nla):
+ nla_map = (('CTRL_ATTR_OP_UNSPEC', 'none'),
+ ('CTRL_ATTR_OP_ID', 'uint32'),
+ ('CTRL_ATTR_OP_FLAGS', 'uint32'))
+
+ class mcast_groups(nla):
+ nla_map = (('CTRL_ATTR_MCAST_GRP_UNSPEC', 'none'),
+ ('CTRL_ATTR_MCAST_GRP_NAME', 'asciiz'),
+ ('CTRL_ATTR_MCAST_GRP_ID', 'uint32'))
diff --git a/node-admin/scripts/pyroute2/netlink/generic/__init__.py b/node-admin/scripts/pyroute2/netlink/generic/__init__.py
new file mode 100644
index 00000000000..8dfb8cc5fab
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/generic/__init__.py
@@ -0,0 +1,74 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+# -*- coding: utf-8 -*-
+'''
+Generic netlink
+===============
+
+Describe
+'''
+
+from pyroute2.netlink import CTRL_CMD_GETFAMILY
+from pyroute2.netlink import GENL_ID_CTRL
+from pyroute2.netlink import NLM_F_REQUEST
+from pyroute2.netlink import SOL_NETLINK
+from pyroute2.netlink import NETLINK_ADD_MEMBERSHIP
+from pyroute2.netlink import NETLINK_DROP_MEMBERSHIP
+from pyroute2.netlink import ctrlmsg
+from pyroute2.netlink.nlsocket import NetlinkSocket
+
+
+class GenericNetlinkSocket(NetlinkSocket):
+ '''
+ Low-level socket interface. Provides all the
+ usual socket does, can be used in poll/select,
+ doesn't create any implicit threads.
+ '''
+
+ mcast_groups = {}
+
+ def bind(self, proto, msg_class, groups=0, pid=0, async=False):
+ '''
+ Bind the socket and performs generic netlink
+ proto lookup. The `proto` parameter is a string,
+ like "TASKSTATS", `msg_class` is a class to
+ parse messages with.
+ '''
+ NetlinkSocket.bind(self, groups, pid, async)
+ self.marshal.msg_map[GENL_ID_CTRL] = ctrlmsg
+ msg = self.discovery(proto)
+ self.prid = msg.get_attr('CTRL_ATTR_FAMILY_ID')
+ self.mcast_groups = \
+ dict([(x.get_attr('CTRL_ATTR_MCAST_GRP_NAME'),
+ x.get_attr('CTRL_ATTR_MCAST_GRP_ID')) for x
+ in msg.get_attr('CTRL_ATTR_MCAST_GROUPS', [])])
+ self.marshal.msg_map[self.prid] = msg_class
+
+ def add_membership(self, group):
+ self.setsockopt(SOL_NETLINK,
+ NETLINK_ADD_MEMBERSHIP,
+ self.mcast_groups[group])
+
+ def drop_membership(self, group):
+ self.setsockopt(SOL_NETLINK,
+ NETLINK_DROP_MEMBERSHIP,
+ self.mcast_groups[group])
+
+ def discovery(self, proto):
+ '''
+ Resolve generic netlink protocol -- takes a string
+ as the only parameter, return protocol description
+ '''
+ msg = ctrlmsg()
+ msg['cmd'] = CTRL_CMD_GETFAMILY
+ msg['version'] = 1
+ msg['attrs'].append(['CTRL_ATTR_FAMILY_NAME', proto])
+ msg['header']['type'] = GENL_ID_CTRL
+ msg['header']['flags'] = NLM_F_REQUEST
+ msg['header']['pid'] = self.pid
+ msg.encode()
+ self.sendto(msg.buf.getvalue(), (0, 0))
+ msg = self.get()[0]
+ err = msg['header'].get('error', None)
+ if err is not None:
+ raise err
+ return msg
diff --git a/node-admin/scripts/pyroute2/netlink/ipq/__init__.py b/node-admin/scripts/pyroute2/netlink/ipq/__init__.py
new file mode 100644
index 00000000000..7408f50e877
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/ipq/__init__.py
@@ -0,0 +1,131 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+IPQ -- userspace firewall
+=========================
+
+Netlink family for dealing with `QUEUE` iptables
+target. All the packets routed to the target
+`QUEUE` should be handled by a userspace program
+and the program should response with a verdict.
+E.g., the verdict can be `NF_DROP` and in that
+case the packet will be silently dropped, or
+`NF_ACCEPT`, and the packet will be pass the
+rule.
+'''
+from pyroute2.netlink import NLM_F_REQUEST
+from pyroute2.netlink import nlmsg
+from pyroute2.netlink.nlsocket import NetlinkSocket
+from pyroute2.netlink.nlsocket import Marshal
+# constants
+IFNAMSIZ = 16
+IPQ_MAX_PAYLOAD = 0x800
+
+# IPQ messages
+IPQM_BASE = 0x10
+IPQM_MODE = IPQM_BASE + 1
+IPQM_VERDICT = IPQM_BASE + 2
+IPQM_PACKET = IPQM_BASE + 3
+
+# IPQ modes
+IPQ_COPY_NONE = 0
+IPQ_COPY_META = 1
+IPQ_COPY_PACKET = 2
+
+# verdict types
+NF_DROP = 0
+NF_ACCEPT = 1
+NF_STOLEN = 2
+NF_QUEUE = 3
+NF_REPEAT = 4
+NF_STOP = 5
+
+
+class ipq_base_msg(nlmsg):
+ def decode(self):
+ nlmsg.decode(self)
+ self['payload'] = self.buf.read(self['data_len'])
+
+ def encode(self):
+ init = self.buf.tell()
+ nlmsg.encode(self)
+ if 'payload' in self:
+ self.buf.write(self['payload'])
+ self.update_length(init)
+
+
+class ipq_packet_msg(ipq_base_msg):
+ fields = (('packet_id', 'L'),
+ ('mark', 'L'),
+ ('timestamp_sec', 'l'),
+ ('timestamp_usec', 'l'),
+ ('hook', 'I'),
+ ('indev_name', '%is' % IFNAMSIZ),
+ ('outdev_name', '%is' % IFNAMSIZ),
+ ('hw_protocol', '>H'),
+ ('hw_type', 'H'),
+ ('hw_addrlen', 'B'),
+ ('hw_addr', '6B'),
+ ('__pad', '9x'),
+ ('data_len', 'I'),
+ ('__pad', '4x'))
+
+
+class ipq_mode_msg(nlmsg):
+ pack = 'struct'
+ fields = (('value', 'B'),
+ ('__pad', '7x'),
+ ('range', 'I'),
+ ('__pad', '12x'))
+
+
+class ipq_verdict_msg(ipq_base_msg):
+ pack = 'struct'
+ fields = (('value', 'I'),
+ ('__pad', '4x'),
+ ('id', 'L'),
+ ('data_len', 'I'),
+ ('__pad', '4x'))
+
+
+class MarshalIPQ(Marshal):
+
+ msg_map = {IPQM_MODE: ipq_mode_msg,
+ IPQM_VERDICT: ipq_verdict_msg,
+ IPQM_PACKET: ipq_packet_msg}
+
+
+class IPQSocket(NetlinkSocket):
+ '''
+ Low-level socket interface. Provides all the
+ usual socket does, can be used in poll/select,
+ doesn't create any implicit threads.
+ '''
+
+ def bind(self, mode=IPQ_COPY_PACKET):
+ '''
+ Bind the socket and performs IPQ mode configuration.
+ The only parameter is mode, the default value is
+ IPQ_COPY_PACKET (copy all the packet data).
+ '''
+ NetlinkSocket.bind(self, groups=0, pid=0)
+ self.register_policy(MarshalIPQ.msg_map)
+ msg = ipq_mode_msg()
+ msg['value'] = mode
+ msg['range'] = IPQ_MAX_PAYLOAD
+ msg['header']['type'] = IPQM_MODE
+ msg['header']['flags'] = NLM_F_REQUEST
+ msg.encode()
+ self.sendto(msg.buf.getvalue(), (0, 0))
+
+ def verdict(self, seq, v):
+ '''
+ Issue a verdict `v` for a packet `seq`.
+ '''
+ msg = ipq_verdict_msg()
+ msg['value'] = v
+ msg['id'] = seq
+ msg['data_len'] = 0
+ msg['header']['type'] = IPQM_VERDICT
+ msg['header']['flags'] = NLM_F_REQUEST
+ msg.encode()
+ self.sendto(msg.buf.getvalue(), (0, 0))
diff --git a/node-admin/scripts/pyroute2/netlink/nfnetlink/__init__.py b/node-admin/scripts/pyroute2/netlink/nfnetlink/__init__.py
new file mode 100644
index 00000000000..682414a0ff7
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/nfnetlink/__init__.py
@@ -0,0 +1,33 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+Nfnetlink
+=========
+
+The support of nfnetlink families is now at the
+very beginning. So there is no public exports
+yet, but you can review the code. Work is in
+progress, stay tuned.
+
+nf-queue
+++++++++
+
+Netfilter protocol for NFQUEUE iptables target.
+'''
+
+from pyroute2.netlink import nlmsg
+
+
+NFNL_SUBSYS_NONE = 0
+NFNL_SUBSYS_CTNETLINK = 1
+NFNL_SUBSYS_CTNETLINK_EXP = 2
+NFNL_SUBSYS_QUEUE = 3
+NFNL_SUBSYS_ULOG = 4
+NFNL_SUBSYS_OSF = 5
+NFNL_SUBSYS_IPSET = 6
+NFNL_SUBSYS_COUNT = 7
+
+
+class nfgen_msg(nlmsg):
+ fields = (('nfgen_family', 'B'),
+ ('version', 'B'),
+ ('res_id', 'H'))
diff --git a/node-admin/scripts/pyroute2/netlink/nfnetlink/ipset.py b/node-admin/scripts/pyroute2/netlink/nfnetlink/ipset.py
new file mode 100644
index 00000000000..10c6d57024e
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/nfnetlink/ipset.py
@@ -0,0 +1,77 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+from pyroute2.netlink import nla
+from pyroute2.netlink import NLA_F_NESTED
+from pyroute2.netlink import NLA_F_NET_BYTEORDER
+from pyroute2.netlink.nfnetlink import nfgen_msg
+
+
+IPSET_MAXNAMELEN = 32
+
+IPSET_CMD_NONE = 0
+IPSET_CMD_PROTOCOL = 1 # Return protocol version
+IPSET_CMD_CREATE = 2 # Create a new (empty) set
+IPSET_CMD_DESTROY = 3 # Destroy a (empty) set
+IPSET_CMD_FLUSH = 4 # Remove all elements from a set
+IPSET_CMD_RENAME = 5 # Rename a set
+IPSET_CMD_SWAP = 6 # Swap two sets
+IPSET_CMD_LIST = 7 # List sets
+IPSET_CMD_SAVE = 8 # Save sets
+IPSET_CMD_ADD = 9 # Add an element to a set
+IPSET_CMD_DEL = 10 # Delete an element from a set
+IPSET_CMD_TEST = 11 # Test an element in a set
+IPSET_CMD_HEADER = 12 # Get set header data only
+IPSET_CMD_TYPE = 13 # 13: Get set type
+
+
+class ipset_msg(nfgen_msg):
+ '''
+ Since the support just begins to be developed,
+ many attrs are still in `hex` format -- just to
+ dump the content.
+ '''
+ nla_map = (('IPSET_ATTR_UNSPEC', 'none'),
+ ('IPSET_ATTR_PROTOCOL', 'uint8'),
+ ('IPSET_ATTR_SETNAME', 'asciiz'),
+ ('IPSET_ATTR_TYPENAME', 'asciiz'),
+ ('IPSET_ATTR_REVISION', 'uint8'),
+ ('IPSET_ATTR_FAMILY', 'uint8'),
+ ('IPSET_ATTR_FLAGS', 'hex'),
+ ('IPSET_ATTR_DATA', 'data'),
+ ('IPSET_ATTR_ADT', 'data'),
+ ('IPSET_ATTR_LINENO', 'hex'),
+ ('IPSET_ATTR_PROTOCOL_MIN', 'hex'))
+
+ class data(nla):
+ nla_flags = NLA_F_NESTED
+ nla_map = ((0, 'IPSET_ATTR_UNSPEC', 'none'),
+ (1, 'IPSET_ATTR_IP', 'ipset_ip'),
+ (1, 'IPSET_ATTR_IP_FROM', 'ipset_ip'),
+ (2, 'IPSET_ATTR_IP_TO', 'ipset_ip'),
+ (3, 'IPSET_ATTR_CIDR', 'hex'),
+ (4, 'IPSET_ATTR_PORT', 'hex'),
+ (4, 'IPSET_ATTR_PORT_FROM', 'hex'),
+ (5, 'IPSET_ATTR_PORT_TO', 'hex'),
+ (6, 'IPSET_ATTR_TIMEOUT', 'hex'),
+ (7, 'IPSET_ATTR_PROTO', 'recursive'),
+ (8, 'IPSET_ATTR_CADT_FLAGS', 'hex'),
+ (9, 'IPSET_ATTR_CADT_LINENO', 'be32'),
+ (10, 'IPSET_ATTR_MARK', 'hex'),
+ (11, 'IPSET_ATTR_MARKMASK', 'hex'),
+ (17, 'IPSET_ATTR_GC', 'hex'),
+ (18, 'IPSET_ATTR_HASHSIZE', 'be32'),
+ (19, 'IPSET_ATTR_MAXELEM', 'be32'),
+ (20, 'IPSET_ATTR_NETMASK', 'hex'),
+ (21, 'IPSET_ATTR_PROBES', 'hex'),
+ (22, 'IPSET_ATTR_RESIZE', 'hex'),
+ (23, 'IPSET_ATTR_SIZE', 'hex'),
+ (24, 'IPSET_ATTR_ELEMENTS', 'hex'),
+ (25, 'IPSET_ATTR_REFERENCES', 'be32'),
+ (26, 'IPSET_ATTR_MEMSIZE', 'be32'))
+
+ class ipset_ip(nla):
+ nla_flags = NLA_F_NESTED
+ nla_map = (('IPSET_ATTR_UNSPEC', 'none'),
+ ('IPSET_ATTR_IPADDR_IPV4', 'ip4addr',
+ NLA_F_NET_BYTEORDER),
+ ('IPSET_ATTR_IPADDR_IPV6', 'ip6addr',
+ NLA_F_NET_BYTEORDER))
diff --git a/node-admin/scripts/pyroute2/netlink/nl80211/__init__.py b/node-admin/scripts/pyroute2/netlink/nl80211/__init__.py
new file mode 100644
index 00000000000..a2946b06dc1
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/nl80211/__init__.py
@@ -0,0 +1,609 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+NL80211 module
+==============
+
+TODO
+'''
+from pyroute2.common import map_namespace
+from pyroute2.netlink import genlmsg
+from pyroute2.netlink.generic import GenericNetlinkSocket
+from pyroute2.netlink.nlsocket import Marshal
+from pyroute2.netlink import nla
+from pyroute2.netlink import nla_base
+
+
+# import pdb
+import struct
+from pyroute2.common import hexdump
+
+# nl80211 commands
+
+NL80211_CMD_UNSPEC = 0
+NL80211_CMD_GET_WIPHY = 1
+NL80211_CMD_SET_WIPHY = 2
+NL80211_CMD_NEW_WIPHY = 3
+NL80211_CMD_DEL_WIPHY = 4
+NL80211_CMD_GET_INTERFACE = 5
+NL80211_CMD_SET_INTERFACE = 6
+NL80211_CMD_NEW_INTERFACE = 7
+NL80211_CMD_DEL_INTERFACE = 8
+NL80211_CMD_GET_KEY = 9
+NL80211_CMD_SET_KEY = 10
+NL80211_CMD_NEW_KEY = 11
+NL80211_CMD_DEL_KEY = 12
+NL80211_CMD_GET_BEACON = 13
+NL80211_CMD_SET_BEACON = 14
+NL80211_CMD_START_AP = 15
+NL80211_CMD_NEW_BEACON = NL80211_CMD_START_AP
+NL80211_CMD_STOP_AP = 16
+NL80211_CMD_DEL_BEACON = NL80211_CMD_STOP_AP
+NL80211_CMD_GET_STATION = 17
+NL80211_CMD_SET_STATION = 18
+NL80211_CMD_NEW_STATION = 19
+NL80211_CMD_DEL_STATION = 20
+NL80211_CMD_GET_MPATH = 21
+NL80211_CMD_SET_MPATH = 22
+NL80211_CMD_NEW_MPATH = 23
+NL80211_CMD_DEL_MPATH = 24
+NL80211_CMD_SET_BSS = 25
+NL80211_CMD_SET_REG = 26
+NL80211_CMD_REQ_SET_REG = 27
+NL80211_CMD_GET_MESH_CONFIG = 28
+NL80211_CMD_SET_MESH_CONFIG = 29
+NL80211_CMD_SET_MGMT_EXTRA_IE = 30
+NL80211_CMD_GET_REG = 31
+NL80211_CMD_GET_SCAN = 32
+NL80211_CMD_TRIGGER_SCAN = 33
+NL80211_CMD_NEW_SCAN_RESULTS = 34
+NL80211_CMD_SCAN_ABORTED = 35
+NL80211_CMD_REG_CHANGE = 36
+NL80211_CMD_AUTHENTICATE = 37
+NL80211_CMD_ASSOCIATE = 38
+NL80211_CMD_DEAUTHENTICATE = 39
+NL80211_CMD_DISASSOCIATE = 40
+NL80211_CMD_MICHAEL_MIC_FAILURE = 41
+NL80211_CMD_REG_BEACON_HINT = 42
+NL80211_CMD_JOIN_IBSS = 43
+NL80211_CMD_LEAVE_IBSS = 44
+NL80211_CMD_TESTMODE = 45
+NL80211_CMD_CONNECT = 46
+NL80211_CMD_ROAM = 47
+NL80211_CMD_DISCONNECT = 48
+NL80211_CMD_SET_WIPHY_NETNS = 49
+NL80211_CMD_GET_SURVEY = 50
+NL80211_CMD_NEW_SURVEY_RESULTS = 51
+NL80211_CMD_SET_PMKSA = 52
+NL80211_CMD_DEL_PMKSA = 53
+NL80211_CMD_FLUSH_PMKSA = 54
+NL80211_CMD_REMAIN_ON_CHANNEL = 55
+NL80211_CMD_CANCEL_REMAIN_ON_CHANNEL = 56
+NL80211_CMD_SET_TX_BITRATE_MASK = 57
+NL80211_CMD_REGISTER_FRAME = 58
+NL80211_CMD_REGISTER_ACTION = NL80211_CMD_REGISTER_FRAME
+NL80211_CMD_FRAME = 59
+NL80211_CMD_ACTION = NL80211_CMD_FRAME
+NL80211_CMD_FRAME_TX_STATUS = 60
+NL80211_CMD_ACTION_TX_STATUS = NL80211_CMD_FRAME_TX_STATUS
+NL80211_CMD_SET_POWER_SAVE = 61
+NL80211_CMD_GET_POWER_SAVE = 62
+NL80211_CMD_SET_CQM = 63
+NL80211_CMD_NOTIFY_CQM = 64
+NL80211_CMD_SET_CHANNEL = 65
+NL80211_CMD_SET_WDS_PEER = 66
+NL80211_CMD_FRAME_WAIT_CANCEL = 67
+NL80211_CMD_JOIN_MESH = 68
+NL80211_CMD_LEAVE_MESH = 69
+NL80211_CMD_UNPROT_DEAUTHENTICATE = 70
+NL80211_CMD_UNPROT_DISASSOCIATE = 71
+NL80211_CMD_NEW_PEER_CANDIDATE = 72
+NL80211_CMD_GET_WOWLAN = 73
+NL80211_CMD_SET_WOWLAN = 74
+NL80211_CMD_START_SCHED_SCAN = 75
+NL80211_CMD_STOP_SCHED_SCAN = 76
+NL80211_CMD_SCHED_SCAN_RESULTS = 77
+NL80211_CMD_SCHED_SCAN_STOPPED = 78
+NL80211_CMD_SET_REKEY_OFFLOAD = 79
+NL80211_CMD_PMKSA_CANDIDATE = 80
+NL80211_CMD_TDLS_OPER = 81
+NL80211_CMD_TDLS_MGMT = 82
+NL80211_CMD_UNEXPECTED_FRAME = 83
+NL80211_CMD_PROBE_CLIENT = 84
+NL80211_CMD_REGISTER_BEACONS = 85
+NL80211_CMD_UNEXPECTED_4ADDR_FRAME = 86
+NL80211_CMD_SET_NOACK_MAP = 87
+NL80211_CMD_CH_SWITCH_NOTIFY = 88
+NL80211_CMD_START_P2P_DEVICE = 89
+NL80211_CMD_STOP_P2P_DEVICE = 90
+NL80211_CMD_CONN_FAILED = 91
+NL80211_CMD_SET_MCAST_RATE = 92
+NL80211_CMD_SET_MAC_ACL = 93
+NL80211_CMD_RADAR_DETECT = 94
+NL80211_CMD_GET_PROTOCOL_FEATURES = 95
+NL80211_CMD_UPDATE_FT_IES = 96
+NL80211_CMD_FT_EVENT = 97
+NL80211_CMD_CRIT_PROTOCOL_START = 98
+NL80211_CMD_CRIT_PROTOCOL_STOP = 99
+NL80211_CMD_GET_COALESCE = 100
+NL80211_CMD_SET_COALESCE = 101
+NL80211_CMD_CHANNEL_SWITCH = 102
+NL80211_CMD_VENDOR = 103
+NL80211_CMD_SET_QOS_MAP = 104
+NL80211_CMD_ADD_TX_TS = 105
+NL80211_CMD_DEL_TX_TS = 106
+NL80211_CMD_GET_MPP = 107
+NL80211_CMD_JOIN_OCB = 108
+NL80211_CMD_LEAVE_OCB = 109
+NL80211_CMD_CH_SWITCH_STARTED_NOTIFY = 110
+NL80211_CMD_TDLS_CHANNEL_SWITCH = 111
+NL80211_CMD_TDLS_CANCEL_CHANNEL_SWITCH = 112
+NL80211_CMD_WIPHY_REG_CHANGE = 113
+NL80211_CMD_MAX = NL80211_CMD_WIPHY_REG_CHANGE
+(NL80211_NAMES, NL80211_VALUES) = map_namespace('NL80211_CMD_', globals())
+
+NL80211_BSS_ELEMENTS_SSID = 0
+NL80211_BSS_ELEMENTS_SUPPORTED_RATES = 1
+NL80211_BSS_ELEMENTS_CHANNEL = 3
+NL80211_BSS_ELEMENTS_VENDOR = 221
+
+BSS_MEMBERSHIP_SELECTOR_HT_PHY = 127
+BSS_MEMBERSHIP_SELECTOR_VHT_PHY = 126
+
+# interface types
+NL80211_IFTYPE_UNSPECIFIED = 0
+NL80211_IFTYPE_ADHOC = 1
+NL80211_IFTYPE_STATION = 2
+NL80211_IFTYPE_AP = 3
+NL80211_IFTYPE_AP_VLAN = 4
+NL80211_IFTYPE_WDS = 5
+NL80211_IFTYPE_MONITOR = 6
+NL80211_IFTYPE_MESH_POINT = 7
+NL80211_IFTYPE_P2P_CLIENT = 8
+NL80211_IFTYPE_P2P_GO = 9
+NL80211_IFTYPE_P2P_DEVICE = 10
+NL80211_IFTYPE_OCB = 11
+(IFTYPE_NAMES, IFTYPE_VALUES) = map_namespace('NL80211_IFTYPE_',
+ globals(),
+ normalize=True)
+
+
+class nl80211cmd(genlmsg):
+ nla_map = (('NL80211_ATTR_UNSPEC', 'none'),
+ ('NL80211_ATTR_WIPHY', 'uint32'),
+ ('NL80211_ATTR_WIPHY_NAME', 'asciiz'),
+ ('NL80211_ATTR_IFINDEX', 'uint32'),
+ ('NL80211_ATTR_IFNAME', 'asciiz'),
+ ('NL80211_ATTR_IFTYPE', 'uint32'),
+ ('NL80211_ATTR_MAC', 'l2addr'),
+ ('NL80211_ATTR_KEY_DATA', 'hex'),
+ ('NL80211_ATTR_KEY_IDX', 'hex'),
+ ('NL80211_ATTR_KEY_CIPHER', 'uint32'),
+ ('NL80211_ATTR_KEY_SEQ', 'hex'),
+ ('NL80211_ATTR_KEY_DEFAULT', 'hex'),
+ ('NL80211_ATTR_BEACON_INTERVAL', 'hex'),
+ ('NL80211_ATTR_DTIM_PERIOD', 'hex'),
+ ('NL80211_ATTR_BEACON_HEAD', 'hex'),
+ ('NL80211_ATTR_BEACON_TAIL', 'hex'),
+ ('NL80211_ATTR_STA_AID', 'hex'),
+ ('NL80211_ATTR_STA_FLAGS', 'hex'),
+ ('NL80211_ATTR_STA_LISTEN_INTERVAL', 'hex'),
+ ('NL80211_ATTR_STA_SUPPORTED_RATES', 'hex'),
+ ('NL80211_ATTR_STA_VLAN', 'hex'),
+ ('NL80211_ATTR_STA_INFO', 'hex'),
+ ('NL80211_ATTR_WIPHY_BANDS', 'hex'),
+ ('NL80211_ATTR_MNTR_FLAGS', 'hex'),
+ ('NL80211_ATTR_MESH_ID', 'hex'),
+ ('NL80211_ATTR_STA_PLINK_ACTION', 'hex'),
+ ('NL80211_ATTR_MPATH_NEXT_HOP', 'hex'),
+ ('NL80211_ATTR_MPATH_INFO', 'hex'),
+ ('NL80211_ATTR_BSS_CTS_PROT', 'hex'),
+ ('NL80211_ATTR_BSS_SHORT_PREAMBLE', 'hex'),
+ ('NL80211_ATTR_BSS_SHORT_SLOT_TIME', 'hex'),
+ ('NL80211_ATTR_HT_CAPABILITY', 'hex'),
+ ('NL80211_ATTR_SUPPORTED_IFTYPES', 'hex'),
+ ('NL80211_ATTR_REG_ALPHA2', 'hex'),
+ ('NL80211_ATTR_REG_RULES', 'hex'),
+ ('NL80211_ATTR_MESH_CONFIG', 'hex'),
+ ('NL80211_ATTR_BSS_BASIC_RATES', 'hex'),
+ ('NL80211_ATTR_WIPHY_TXQ_PARAMS', 'hex'),
+ ('NL80211_ATTR_WIPHY_FREQ', 'uint32'),
+ ('NL80211_ATTR_WIPHY_CHANNEL_TYPE', 'hex'),
+ ('NL80211_ATTR_KEY_DEFAULT_MGMT', 'hex'),
+ ('NL80211_ATTR_MGMT_SUBTYPE', 'hex'),
+ ('NL80211_ATTR_IE', 'hex'),
+ ('NL80211_ATTR_MAX_NUM_SCAN_SSIDS', 'hex'),
+ ('NL80211_ATTR_SCAN_FREQUENCIES', 'hex'),
+ ('NL80211_ATTR_SCAN_SSIDS', 'hex'),
+ ('NL80211_ATTR_GENERATION', 'hex'),
+ ('NL80211_ATTR_BSS', 'bss'),
+ ('NL80211_ATTR_REG_INITIATOR', 'hex'),
+ ('NL80211_ATTR_REG_TYPE', 'hex'),
+ ('NL80211_ATTR_SUPPORTED_COMMANDS', 'hex'),
+ ('NL80211_ATTR_FRAME', 'hex'),
+ ('NL80211_ATTR_SSID', 'hex'),
+ ('NL80211_ATTR_AUTH_TYPE', 'hex'),
+ ('NL80211_ATTR_REASON_CODE', 'hex'),
+ ('NL80211_ATTR_KEY_TYPE', 'hex'),
+ ('NL80211_ATTR_MAX_SCAN_IE_LEN', 'hex'),
+ ('NL80211_ATTR_CIPHER_SUITES', 'hex'),
+ ('NL80211_ATTR_FREQ_BEFORE', 'hex'),
+ ('NL80211_ATTR_FREQ_AFTER', 'hex'),
+ ('NL80211_ATTR_FREQ_FIXED', 'hex'),
+ ('NL80211_ATTR_WIPHY_RETRY_SHORT', 'hex'),
+ ('NL80211_ATTR_WIPHY_RETRY_LONG', 'hex'),
+ ('NL80211_ATTR_WIPHY_FRAG_THRESHOLD', 'hex'),
+ ('NL80211_ATTR_WIPHY_RTS_THRESHOLD', 'hex'),
+ ('NL80211_ATTR_TIMED_OUT', 'hex'),
+ ('NL80211_ATTR_USE_MFP', 'hex'),
+ ('NL80211_ATTR_STA_FLAGS2', 'hex'),
+ ('NL80211_ATTR_CONTROL_PORT', 'hex'),
+ ('NL80211_ATTR_TESTDATA', 'hex'),
+ ('NL80211_ATTR_PRIVACY', 'hex'),
+ ('NL80211_ATTR_DISCONNECTED_BY_AP', 'hex'),
+ ('NL80211_ATTR_STATUS_CODE', 'hex'),
+ ('NL80211_ATTR_CIPHER_SUITES_PAIRWISE', 'hex'),
+ ('NL80211_ATTR_CIPHER_SUITE_GROUP', 'hex'),
+ ('NL80211_ATTR_WPA_VERSIONS', 'hex'),
+ ('NL80211_ATTR_AKM_SUITES', 'hex'),
+ ('NL80211_ATTR_REQ_IE', 'hex'),
+ ('NL80211_ATTR_RESP_IE', 'hex'),
+ ('NL80211_ATTR_PREV_BSSID', 'hex'),
+ ('NL80211_ATTR_KEY', 'hex'),
+ ('NL80211_ATTR_KEYS', 'hex'),
+ ('NL80211_ATTR_PID', 'hex'),
+ ('NL80211_ATTR_4ADDR', 'hex'),
+ ('NL80211_ATTR_SURVEY_INFO', 'hex'),
+ ('NL80211_ATTR_PMKID', 'hex'),
+ ('NL80211_ATTR_MAX_NUM_PMKIDS', 'hex'),
+ ('NL80211_ATTR_DURATION', 'hex'),
+ ('NL80211_ATTR_COOKIE', 'hex'),
+ ('NL80211_ATTR_WIPHY_COVERAGE_CLASS', 'hex'),
+ ('NL80211_ATTR_TX_RATES', 'hex'),
+ ('NL80211_ATTR_FRAME_MATCH', 'hex'),
+ ('NL80211_ATTR_ACK', 'hex'),
+ ('NL80211_ATTR_PS_STATE', 'hex'),
+ ('NL80211_ATTR_CQM', 'hex'),
+ ('NL80211_ATTR_LOCAL_STATE_CHANGE', 'hex'),
+ ('NL80211_ATTR_AP_ISOLATE', 'hex'),
+ ('NL80211_ATTR_WIPHY_TX_POWER_SETTING', 'hex'),
+ ('NL80211_ATTR_WIPHY_TX_POWER_LEVEL', 'hex'),
+ ('NL80211_ATTR_TX_FRAME_TYPES', 'hex'),
+ ('NL80211_ATTR_RX_FRAME_TYPES', 'hex'),
+ ('NL80211_ATTR_FRAME_TYPE', 'hex'),
+ ('NL80211_ATTR_CONTROL_PORT_ETHERTYPE', 'hex'),
+ ('NL80211_ATTR_CONTROL_PORT_NO_ENCRYPT', 'hex'),
+ ('NL80211_ATTR_SUPPORT_IBSS_RSN', 'hex'),
+ ('NL80211_ATTR_WIPHY_ANTENNA_TX', 'hex'),
+ ('NL80211_ATTR_WIPHY_ANTENNA_RX', 'hex'),
+ ('NL80211_ATTR_MCAST_RATE', 'hex'),
+ ('NL80211_ATTR_OFFCHANNEL_TX_OK', 'hex'),
+ ('NL80211_ATTR_BSS_HT_OPMODE', 'hex'),
+ ('NL80211_ATTR_KEY_DEFAULT_TYPES', 'hex'),
+ ('NL80211_ATTR_MAX_REMAIN_ON_CHANNEL_DURATION', 'hex'),
+ ('NL80211_ATTR_MESH_SETUP', 'hex'),
+ ('NL80211_ATTR_WIPHY_ANTENNA_AVAIL_TX', 'hex'),
+ ('NL80211_ATTR_WIPHY_ANTENNA_AVAIL_RX', 'hex'),
+ ('NL80211_ATTR_SUPPORT_MESH_AUTH', 'hex'),
+ ('NL80211_ATTR_STA_PLINK_STATE', 'hex'),
+ ('NL80211_ATTR_WOWLAN_TRIGGERS', 'hex'),
+ ('NL80211_ATTR_WOWLAN_TRIGGERS_SUPPORTED', 'hex'),
+ ('NL80211_ATTR_SCHED_SCAN_INTERVAL', 'hex'),
+ ('NL80211_ATTR_INTERFACE_COMBINATIONS', 'hex'),
+ ('NL80211_ATTR_SOFTWARE_IFTYPES', 'hex'),
+ ('NL80211_ATTR_REKEY_DATA', 'hex'),
+ ('NL80211_ATTR_MAX_NUM_SCHED_SCAN_SSIDS', 'hex'),
+ ('NL80211_ATTR_MAX_SCHED_SCAN_IE_LEN', 'hex'),
+ ('NL80211_ATTR_SCAN_SUPP_RATES', 'hex'),
+ ('NL80211_ATTR_HIDDEN_SSID', 'hex'),
+ ('NL80211_ATTR_IE_PROBE_RESP', 'hex'),
+ ('NL80211_ATTR_IE_ASSOC_RESP', 'hex'),
+ ('NL80211_ATTR_STA_WME', 'hex'),
+ ('NL80211_ATTR_SUPPORT_AP_UAPSD', 'hex'),
+ ('NL80211_ATTR_ROAM_SUPPORT', 'hex'),
+ ('NL80211_ATTR_SCHED_SCAN_MATCH', 'hex'),
+ ('NL80211_ATTR_MAX_MATCH_SETS', 'hex'),
+ ('NL80211_ATTR_PMKSA_CANDIDATE', 'hex'),
+ ('NL80211_ATTR_TX_NO_CCK_RATE', 'hex'),
+ ('NL80211_ATTR_TDLS_ACTION', 'hex'),
+ ('NL80211_ATTR_TDLS_DIALOG_TOKEN', 'hex'),
+ ('NL80211_ATTR_TDLS_OPERATION', 'hex'),
+ ('NL80211_ATTR_TDLS_SUPPORT', 'hex'),
+ ('NL80211_ATTR_TDLS_EXTERNAL_SETUP', 'hex'),
+ ('NL80211_ATTR_DEVICE_AP_SME', 'hex'),
+ ('NL80211_ATTR_DONT_WAIT_FOR_ACK', 'hex'),
+ ('NL80211_ATTR_FEATURE_FLAGS', 'hex'),
+ ('NL80211_ATTR_PROBE_RESP_OFFLOAD', 'hex'),
+ ('NL80211_ATTR_PROBE_RESP', 'hex'),
+ ('NL80211_ATTR_DFS_REGION', 'hex'),
+ ('NL80211_ATTR_DISABLE_HT', 'hex'),
+ ('NL80211_ATTR_HT_CAPABILITY_MASK', 'hex'),
+ ('NL80211_ATTR_NOACK_MAP', 'hex'),
+ ('NL80211_ATTR_INACTIVITY_TIMEOUT', 'hex'),
+ ('NL80211_ATTR_RX_SIGNAL_DBM', 'hex'),
+ ('NL80211_ATTR_BG_SCAN_PERIOD', 'hex'),
+ ('NL80211_ATTR_WDEV', 'uint32'),
+ ('NL80211_ATTR_USER_REG_HINT_TYPE', 'hex'),
+ ('NL80211_ATTR_CONN_FAILED_REASON', 'hex'),
+ ('NL80211_ATTR_SAE_DATA', 'hex'),
+ ('NL80211_ATTR_VHT_CAPABILITY', 'hex'),
+ ('NL80211_ATTR_SCAN_FLAGS', 'hex'),
+ ('NL80211_ATTR_CHANNEL_WIDTH', 'uint32'),
+ ('NL80211_ATTR_CENTER_FREQ1', 'hex'),
+ ('NL80211_ATTR_CENTER_FREQ2', 'hex'),
+ ('NL80211_ATTR_P2P_CTWINDOW', 'hex'),
+ ('NL80211_ATTR_P2P_OPPPS', 'hex'),
+ ('NL80211_ATTR_LOCAL_MESH_POWER_MODE', 'hex'),
+ ('NL80211_ATTR_ACL_POLICY', 'hex'),
+ ('NL80211_ATTR_MAC_ADDRS', 'hex'),
+ ('NL80211_ATTR_MAC_ACL_MAX', 'hex'),
+ ('NL80211_ATTR_RADAR_EVENT', 'hex'),
+ ('NL80211_ATTR_EXT_CAPA', 'hex'),
+ ('NL80211_ATTR_EXT_CAPA_MASK', 'hex'),
+ ('NL80211_ATTR_STA_CAPABILITY', 'hex'),
+ ('NL80211_ATTR_STA_EXT_CAPABILITY', 'hex'),
+ ('NL80211_ATTR_PROTOCOL_FEATURES', 'hex'),
+ ('NL80211_ATTR_SPLIT_WIPHY_DUMP', 'hex'),
+ ('NL80211_ATTR_DISABLE_VHT', 'hex'),
+ ('NL80211_ATTR_VHT_CAPABILITY_MASK', 'hex'),
+ ('NL80211_ATTR_MDID', 'hex'),
+ ('NL80211_ATTR_IE_RIC', 'hex'),
+ ('NL80211_ATTR_CRIT_PROT_ID', 'hex'),
+ ('NL80211_ATTR_MAX_CRIT_PROT_DURATION', 'hex'),
+ ('NL80211_ATTR_PEER_AID', 'hex'),
+ ('NL80211_ATTR_COALESCE_RULE', 'hex'),
+ ('NL80211_ATTR_CH_SWITCH_COUNT', 'hex'),
+ ('NL80211_ATTR_CH_SWITCH_BLOCK_TX', 'hex'),
+ ('NL80211_ATTR_CSA_IES', 'hex'),
+ ('NL80211_ATTR_CSA_C_OFF_BEACON', 'hex'),
+ ('NL80211_ATTR_CSA_C_OFF_PRESP', 'hex'),
+ ('NL80211_ATTR_RXMGMT_FLAGS', 'hex'),
+ ('NL80211_ATTR_STA_SUPPORTED_CHANNELS', 'hex'),
+ ('NL80211_ATTR_STA_SUPPORTED_OPER_CLASSES', 'hex'),
+ ('NL80211_ATTR_HANDLE_DFS', 'hex'),
+ ('NL80211_ATTR_SUPPORT_5_MHZ', 'hex'),
+ ('NL80211_ATTR_SUPPORT_10_MHZ', 'hex'),
+ ('NL80211_ATTR_OPMODE_NOTIF', 'hex'),
+ ('NL80211_ATTR_VENDOR_ID', 'hex'),
+ ('NL80211_ATTR_VENDOR_SUBCMD', 'hex'),
+ ('NL80211_ATTR_VENDOR_DATA', 'hex'),
+ ('NL80211_ATTR_VENDOR_EVENTS', 'hex'),
+ ('NL80211_ATTR_QOS_MAP', 'hex'),
+ ('NL80211_ATTR_MAC_HINT', 'hex'),
+ ('NL80211_ATTR_WIPHY_FREQ_HINT', 'hex'),
+ ('NL80211_ATTR_MAX_AP_ASSOC_STA', 'hex'),
+ ('NL80211_ATTR_TDLS_PEER_CAPABILITY', 'hex'),
+ ('NL80211_ATTR_SOCKET_OWNER', 'hex'),
+ ('NL80211_ATTR_CSA_C_OFFSETS_TX', 'hex'),
+ ('NL80211_ATTR_MAX_CSA_COUNTERS', 'hex'),
+ ('NL80211_ATTR_TDLS_INITIATOR', 'hex'),
+ ('NL80211_ATTR_USE_RRM', 'hex'),
+ ('NL80211_ATTR_WIPHY_DYN_ACK', 'hex'),
+ ('NL80211_ATTR_TSID', 'hex'),
+ ('NL80211_ATTR_USER_PRIO', 'hex'),
+ ('NL80211_ATTR_ADMITTED_TIME', 'hex'),
+ ('NL80211_ATTR_SMPS_MODE', 'hex'),
+ ('NL80211_ATTR_OPER_CLASS', 'hex'),
+ ('NL80211_ATTR_MAC_MASK', 'hex'),
+ ('NL80211_ATTR_WIPHY_SELF_MANAGED_REG', 'hex'),
+ ('NUM_NL80211_ATTR', 'hex'))
+
+ class bss(nla):
+ class elementsBinary(nla_base):
+
+ def binary_supported_rates(self, rawdata):
+ # pdb.set_trace()
+ string = ""
+ for byteRaw in rawdata:
+ (byte,) = struct.unpack("B", byteRaw)
+ r = byte & 0x7f
+
+ if r == BSS_MEMBERSHIP_SELECTOR_VHT_PHY and byte & 0x80:
+ string += "VHT"
+ elif r == BSS_MEMBERSHIP_SELECTOR_HT_PHY and byte & 0x80:
+ string += "HT"
+ else:
+ string += "%d.%d" % (r / 2, 5 * (r & 1))
+
+ string += "%s " % ("*" if byte & 0x80 else "")
+
+ return string
+
+ def binary_vendor(self, rawdata):
+ '''
+ Extract vendor data
+ '''
+ vendor = {}
+# pdb.set_trace()
+ size = len(rawdata)
+ # if len > 4 and rawdata[0] == ms_oui[0]
+ # and rawdata[1] == ms_oui[1] and rawdata[2] == ms_oui[2]
+
+ if size < 3:
+ vendor["VENDOR_NAME"] = "Vendor specific: <too short data:"
+ + hexdump(rawdata)
+ return vendor
+
+ def decode_nlas(self):
+
+ return
+
+ def decode(self):
+ nla_base.decode(self)
+
+ self.value = {}
+
+ init = self.buf.tell()
+
+ while (self.buf.tell()-init) < (self.length-4):
+ (msg_type, length) = struct.unpack('BB', self.buf.read(2))
+ data = self.buf.read(length)
+ if msg_type == NL80211_BSS_ELEMENTS_SSID:
+ self.value["SSID"] = data
+
+ if msg_type == NL80211_BSS_ELEMENTS_SUPPORTED_RATES:
+ supported_rates = self.binary_supported_rates(data)
+ self.value["SUPPORTED_RATES"] = supported_rates
+
+ if msg_type == NL80211_BSS_ELEMENTS_CHANNEL:
+ (channel,) = struct.unpack("B", data[0])
+ self.value["CHANNEL"] = channel
+
+ if msg_type == NL80211_BSS_ELEMENTS_VENDOR:
+ self.binary_vendor(data)
+
+ self.buf.seek(init)
+
+ prefix = 'NL80211_BSS_'
+ nla_map = (('NL80211_BSS_UNSPEC', 'none'),
+ ('NL80211_BSS_BSSID', 'hex'),
+ ('NL80211_BSS_FREQUENCY', 'uint32'),
+ ('NL80211_BSS_TSF', 'uint64'),
+ ('NL80211_BSS_BEACON_INTERVAL', 'uint16'),
+ ('NL80211_BSS_CAPABILITY', 'uint8'),
+ ('NL80211_BSS_INFORMATION_ELEMENTS', 'elementsBinary'),
+ ('NL80211_BSS_SIGNAL_MBM', 'uint32'),
+ ('NL80211_BSS_STATUS', 'uint32'),
+ ('NL80211_BSS_SEEN_MS_AGO', 'uint32'),
+ ('NL80211_BSS_BEACON_IES', 'hex'),
+ ('NL80211_BSS_CHAN_WIDTH', 'uint32'),
+ ('NL80211_BSS_BEACON_TSF', 'uint64')
+ )
+
+
+class MarshalNl80211(Marshal):
+ msg_map = {NL80211_CMD_UNSPEC: nl80211cmd,
+ NL80211_CMD_GET_WIPHY: nl80211cmd,
+ NL80211_CMD_SET_WIPHY: nl80211cmd,
+ NL80211_CMD_NEW_WIPHY: nl80211cmd,
+ NL80211_CMD_DEL_WIPHY: nl80211cmd,
+ NL80211_CMD_GET_INTERFACE: nl80211cmd,
+ NL80211_CMD_SET_INTERFACE: nl80211cmd,
+ NL80211_CMD_NEW_INTERFACE: nl80211cmd,
+ NL80211_CMD_DEL_INTERFACE: nl80211cmd,
+ NL80211_CMD_GET_KEY: nl80211cmd,
+ NL80211_CMD_SET_KEY: nl80211cmd,
+ NL80211_CMD_NEW_KEY: nl80211cmd,
+ NL80211_CMD_DEL_KEY: nl80211cmd,
+ NL80211_CMD_GET_BEACON: nl80211cmd,
+ NL80211_CMD_SET_BEACON: nl80211cmd,
+ NL80211_CMD_START_AP: nl80211cmd,
+ NL80211_CMD_NEW_BEACON: nl80211cmd,
+ NL80211_CMD_STOP_AP: nl80211cmd,
+ NL80211_CMD_DEL_BEACON: nl80211cmd,
+ NL80211_CMD_GET_STATION: nl80211cmd,
+ NL80211_CMD_SET_STATION: nl80211cmd,
+ NL80211_CMD_NEW_STATION: nl80211cmd,
+ NL80211_CMD_DEL_STATION: nl80211cmd,
+ NL80211_CMD_GET_MPATH: nl80211cmd,
+ NL80211_CMD_SET_MPATH: nl80211cmd,
+ NL80211_CMD_NEW_MPATH: nl80211cmd,
+ NL80211_CMD_DEL_MPATH: nl80211cmd,
+ NL80211_CMD_SET_BSS: nl80211cmd,
+ NL80211_CMD_SET_REG: nl80211cmd,
+ NL80211_CMD_REQ_SET_REG: nl80211cmd,
+ NL80211_CMD_GET_MESH_CONFIG: nl80211cmd,
+ NL80211_CMD_SET_MESH_CONFIG: nl80211cmd,
+ NL80211_CMD_SET_MGMT_EXTRA_IE: nl80211cmd,
+ NL80211_CMD_GET_REG: nl80211cmd,
+ NL80211_CMD_GET_SCAN: nl80211cmd,
+ NL80211_CMD_TRIGGER_SCAN: nl80211cmd,
+ NL80211_CMD_NEW_SCAN_RESULTS: nl80211cmd,
+ NL80211_CMD_SCAN_ABORTED: nl80211cmd,
+ NL80211_CMD_REG_CHANGE: nl80211cmd,
+ NL80211_CMD_AUTHENTICATE: nl80211cmd,
+ NL80211_CMD_ASSOCIATE: nl80211cmd,
+ NL80211_CMD_DEAUTHENTICATE: nl80211cmd,
+ NL80211_CMD_DISASSOCIATE: nl80211cmd,
+ NL80211_CMD_MICHAEL_MIC_FAILURE: nl80211cmd,
+ NL80211_CMD_REG_BEACON_HINT: nl80211cmd,
+ NL80211_CMD_JOIN_IBSS: nl80211cmd,
+ NL80211_CMD_LEAVE_IBSS: nl80211cmd,
+ NL80211_CMD_TESTMODE: nl80211cmd,
+ NL80211_CMD_CONNECT: nl80211cmd,
+ NL80211_CMD_ROAM: nl80211cmd,
+ NL80211_CMD_DISCONNECT: nl80211cmd,
+ NL80211_CMD_SET_WIPHY_NETNS: nl80211cmd,
+ NL80211_CMD_GET_SURVEY: nl80211cmd,
+ NL80211_CMD_NEW_SURVEY_RESULTS: nl80211cmd,
+ NL80211_CMD_SET_PMKSA: nl80211cmd,
+ NL80211_CMD_DEL_PMKSA: nl80211cmd,
+ NL80211_CMD_FLUSH_PMKSA: nl80211cmd,
+ NL80211_CMD_REMAIN_ON_CHANNEL: nl80211cmd,
+ NL80211_CMD_CANCEL_REMAIN_ON_CHANNEL: nl80211cmd,
+ NL80211_CMD_SET_TX_BITRATE_MASK: nl80211cmd,
+ NL80211_CMD_REGISTER_FRAME: nl80211cmd,
+ NL80211_CMD_REGISTER_ACTION: nl80211cmd,
+ NL80211_CMD_FRAME: nl80211cmd,
+ NL80211_CMD_ACTION: nl80211cmd,
+ NL80211_CMD_FRAME_TX_STATUS: nl80211cmd,
+ NL80211_CMD_ACTION_TX_STATUS: nl80211cmd,
+ NL80211_CMD_SET_POWER_SAVE: nl80211cmd,
+ NL80211_CMD_GET_POWER_SAVE: nl80211cmd,
+ NL80211_CMD_SET_CQM: nl80211cmd,
+ NL80211_CMD_NOTIFY_CQM: nl80211cmd,
+ NL80211_CMD_SET_CHANNEL: nl80211cmd,
+ NL80211_CMD_SET_WDS_PEER: nl80211cmd,
+ NL80211_CMD_FRAME_WAIT_CANCEL: nl80211cmd,
+ NL80211_CMD_JOIN_MESH: nl80211cmd,
+ NL80211_CMD_LEAVE_MESH: nl80211cmd,
+ NL80211_CMD_UNPROT_DEAUTHENTICATE: nl80211cmd,
+ NL80211_CMD_UNPROT_DISASSOCIATE: nl80211cmd,
+ NL80211_CMD_NEW_PEER_CANDIDATE: nl80211cmd,
+ NL80211_CMD_GET_WOWLAN: nl80211cmd,
+ NL80211_CMD_SET_WOWLAN: nl80211cmd,
+ NL80211_CMD_START_SCHED_SCAN: nl80211cmd,
+ NL80211_CMD_STOP_SCHED_SCAN: nl80211cmd,
+ NL80211_CMD_SCHED_SCAN_RESULTS: nl80211cmd,
+ NL80211_CMD_SCHED_SCAN_STOPPED: nl80211cmd,
+ NL80211_CMD_SET_REKEY_OFFLOAD: nl80211cmd,
+ NL80211_CMD_PMKSA_CANDIDATE: nl80211cmd,
+ NL80211_CMD_TDLS_OPER: nl80211cmd,
+ NL80211_CMD_TDLS_MGMT: nl80211cmd,
+ NL80211_CMD_UNEXPECTED_FRAME: nl80211cmd,
+ NL80211_CMD_PROBE_CLIENT: nl80211cmd,
+ NL80211_CMD_REGISTER_BEACONS: nl80211cmd,
+ NL80211_CMD_UNEXPECTED_4ADDR_FRAME: nl80211cmd,
+ NL80211_CMD_SET_NOACK_MAP: nl80211cmd,
+ NL80211_CMD_CH_SWITCH_NOTIFY: nl80211cmd,
+ NL80211_CMD_START_P2P_DEVICE: nl80211cmd,
+ NL80211_CMD_STOP_P2P_DEVICE: nl80211cmd,
+ NL80211_CMD_CONN_FAILED: nl80211cmd,
+ NL80211_CMD_SET_MCAST_RATE: nl80211cmd,
+ NL80211_CMD_SET_MAC_ACL: nl80211cmd,
+ NL80211_CMD_RADAR_DETECT: nl80211cmd,
+ NL80211_CMD_GET_PROTOCOL_FEATURES: nl80211cmd,
+ NL80211_CMD_UPDATE_FT_IES: nl80211cmd,
+ NL80211_CMD_FT_EVENT: nl80211cmd,
+ NL80211_CMD_CRIT_PROTOCOL_START: nl80211cmd,
+ NL80211_CMD_CRIT_PROTOCOL_STOP: nl80211cmd,
+ NL80211_CMD_GET_COALESCE: nl80211cmd,
+ NL80211_CMD_SET_COALESCE: nl80211cmd,
+ NL80211_CMD_CHANNEL_SWITCH: nl80211cmd,
+ NL80211_CMD_VENDOR: nl80211cmd,
+ NL80211_CMD_SET_QOS_MAP: nl80211cmd,
+ NL80211_CMD_ADD_TX_TS: nl80211cmd,
+ NL80211_CMD_DEL_TX_TS: nl80211cmd,
+ NL80211_CMD_GET_MPP: nl80211cmd,
+ NL80211_CMD_JOIN_OCB: nl80211cmd,
+ NL80211_CMD_LEAVE_OCB: nl80211cmd,
+ NL80211_CMD_CH_SWITCH_STARTED_NOTIFY: nl80211cmd,
+ NL80211_CMD_TDLS_CHANNEL_SWITCH: nl80211cmd,
+ NL80211_CMD_TDLS_CANCEL_CHANNEL_SWITCH: nl80211cmd,
+ NL80211_CMD_WIPHY_REG_CHANGE: nl80211cmd}
+
+ def fix_message(self, msg):
+ try:
+ msg['event'] = NL80211_VALUES[msg['cmd']]
+ except Exception:
+ pass
+
+
+class NL80211(GenericNetlinkSocket):
+
+ def __init__(self):
+ GenericNetlinkSocket.__init__(self)
+ self.marshal = MarshalNl80211()
+
+ def bind(self, groups=0, async=False):
+ GenericNetlinkSocket.bind(self, 'nl80211', nl80211cmd,
+ groups, None, async)
diff --git a/node-admin/scripts/pyroute2/netlink/nlsocket.py b/node-admin/scripts/pyroute2/netlink/nlsocket.py
new file mode 100644
index 00000000000..432c2a55c4b
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/nlsocket.py
@@ -0,0 +1,856 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+Base netlink socket and marshal
+===============================
+
+All the netlink providers are derived from the socket
+class, so they provide normal socket API, including
+`getsockopt()`, `setsockopt()`, they can be used in
+poll/select I/O loops etc.
+
+asynchronous I/O
+----------------
+
+To run async reader thread, one should call
+`NetlinkSocket.bind(async=True)`. In that case a
+background thread will be launched. The thread will
+automatically collect all the messages and store
+into a userspace buffer.
+
+.. note::
+ There is no need to turn on async I/O, if you
+ don't plan to receive broadcast messages.
+
+ENOBUF and async I/O
+--------------------
+
+When Netlink messages arrive faster than a program
+reads then from the socket, the messages overflow
+the socket buffer and one gets ENOBUF on `recv()`::
+
+ ... self.recv(bufsize)
+ error: [Errno 105] No buffer space available
+
+One way to avoid ENOBUF, is to use async I/O. Then the
+library not only reads and buffers all the messages, but
+also re-prioritizes threads. Suppressing the parser
+activity, the library increases the response delay, but
+spares CPU to read and enqueue arriving messages as
+fast, as it is possible.
+
+With logging level DEBUG you can notice messages, that
+the library started to calm down the parser thread::
+
+ DEBUG:root:Packet burst: the reader thread priority
+ is increased, beware of delays on netlink calls
+ Counters: delta=25 qsize=25 delay=0.1
+
+This state requires no immediate action, but just some
+more attention. When the delay between messages on the
+parser thread exceeds 1 second, DEBUG messages become
+WARNING ones::
+
+ WARNING:root:Packet burst: the reader thread priority
+ is increased, beware of delays on netlink calls
+ Counters: delta=2525 qsize=213536 delay=3
+
+This state means, that almost all the CPU resources are
+dedicated to the reader thread. It doesn't mean, that
+the reader thread consumes 100% CPU -- it means, that the
+CPU is reserved for the case of more intensive bursts. The
+library will return to the normal state only when the
+broadcast storm will be over, and then the CPU will be
+100% loaded with the parser for some time, when it will
+process all the messages queued so far.
+
+when async I/O doesn't help
+---------------------------
+
+Sometimes, even turning async I/O doesn't fix ENOBUF.
+Mostly it means, that in this particular case the Python
+performance is not enough even to read and store the raw
+data from the socket. There is no workaround for such
+cases, except of using something *not* Python-based.
+
+One can still play around with SO_RCVBUF socket option,
+but it doesn't help much. So keep it in mind, and if you
+expect massive broadcast Netlink storms, perform stress
+testing prior to deploy a solution in the production.
+
+classes
+-------
+'''
+
+import os
+import sys
+import time
+import select
+import struct
+import logging
+import traceback
+import threading
+
+from socket import AF_NETLINK
+from socket import SOCK_DGRAM
+from socket import MSG_PEEK
+from socket import SOL_SOCKET
+from socket import SO_RCVBUF
+from socket import SO_SNDBUF
+
+from pyroute2.config import SocketBase
+from pyroute2.common import AddrPool
+from pyroute2.common import DEFAULT_RCVBUF
+from pyroute2.netlink import nlmsg
+from pyroute2.netlink import mtypes
+from pyroute2.netlink import NetlinkError
+from pyroute2.netlink import NetlinkDecodeError
+from pyroute2.netlink import NetlinkHeaderDecodeError
+from pyroute2.netlink import NLMSG_ERROR
+from pyroute2.netlink import NLMSG_DONE
+from pyroute2.netlink import NETLINK_GENERIC
+from pyroute2.netlink import NLM_F_DUMP
+from pyroute2.netlink import NLM_F_MULTI
+from pyroute2.netlink import NLM_F_REQUEST
+
+try:
+ from Queue import Queue
+except ImportError:
+ from queue import Queue
+
+
+class Marshal(object):
+ '''
+ Generic marshalling class
+ '''
+
+ msg_map = {}
+ debug = False
+
+ def __init__(self):
+ self.lock = threading.Lock()
+ # one marshal instance can be used to parse one
+ # message at once
+ self.msg_map = self.msg_map or {}
+ self.defragmentation = {}
+
+ def parse(self, data):
+ '''
+ Parse string data.
+
+ At this moment all transport, except of the native
+ Netlink is deprecated in this library, so we should
+ not support any defragmentation on that level
+ '''
+ offset = 0
+ result = []
+ while offset < len(data):
+ # pick type and length
+ (length, msg_type) = struct.unpack('IH', data[offset:offset+6])
+ error = None
+ if msg_type == NLMSG_ERROR:
+ code = abs(struct.unpack('i', data[offset+16:offset+20])[0])
+ if code > 0:
+ error = NetlinkError(code)
+
+ msg_class = self.msg_map.get(msg_type, nlmsg)
+ msg = msg_class(data[offset:offset+length], debug=self.debug)
+
+ try:
+ msg.decode()
+ msg['header']['error'] = error
+ # try to decode encapsulated error message
+ if error is not None:
+ enc_type = struct.unpack('H', msg.raw[24:26])[0]
+ enc_class = self.msg_map.get(enc_type, nlmsg)
+ enc = enc_class(msg.raw[20:])
+ enc.decode()
+ msg['header']['errmsg'] = enc
+ except NetlinkHeaderDecodeError as e:
+ # in the case of header decoding error,
+ # create an empty message
+ msg = nlmsg()
+ msg['header']['error'] = e
+ except NetlinkDecodeError as e:
+ msg['header']['error'] = e
+
+ mtype = msg['header'].get('type', None)
+ if mtype in (1, 2, 3, 4):
+ msg['event'] = mtypes.get(mtype, 'none')
+ self.fix_message(msg)
+ offset += msg.length
+ result.append(msg)
+
+ return result
+
+ def fix_message(self, msg):
+ pass
+
+
+# 8<-----------------------------------------------------------
+# Singleton, containing possible modifiers to the NetlinkSocket
+# bind() call.
+#
+# Normally, you can open only one netlink connection for one
+# process, but there is a hack. Current PID_MAX_LIMIT is 2^22,
+# so we can use the rest to midify pid field.
+#
+# See also libnl library, lib/socket.c:generate_local_port()
+sockets = AddrPool(minaddr=0x0,
+ maxaddr=0x3ff,
+ reverse=True)
+# 8<-----------------------------------------------------------
+
+
+class LockProxy(object):
+
+ def __init__(self, factory, key):
+ self.factory = factory
+ self.refcount = 0
+ self.key = key
+ self.internal = threading.Lock()
+ self.lock = factory.klass()
+
+ def acquire(self, *argv, **kwarg):
+ with self.internal:
+ self.refcount += 1
+ return self.lock.acquire()
+
+ def release(self):
+ with self.internal:
+ self.refcount -= 1
+ if (self.refcount == 0) and (self.key != 0):
+ try:
+ del self.factory.locks[self.key]
+ except KeyError:
+ pass
+ return self.lock.release()
+
+ def __enter__(self):
+ self.acquire()
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.release()
+
+
+class LockFactory(object):
+
+ def __init__(self, klass=threading.RLock):
+ self.klass = klass
+ self.locks = {0: LockProxy(self, 0)}
+
+ def __enter__(self):
+ self.locks[0].acquire()
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.locks[0].release()
+
+ def __getitem__(self, key):
+ if key is None:
+ key = 0
+ if key not in self.locks:
+ self.locks[key] = LockProxy(self, key)
+ return self.locks[key]
+
+ def __delitem__(self, key):
+ del self.locks[key]
+
+
+class NetlinkMixin(object):
+ '''
+ Generic netlink socket
+ '''
+
+ def __init__(self,
+ family=NETLINK_GENERIC,
+ port=None,
+ pid=None,
+ fileno=None):
+ #
+ # That's a trick. Python 2 is not able to construct
+ # sockets from an open FD.
+ #
+ # So raise an exception, if the major version is < 3
+ # and fileno is not None.
+ #
+ # Do NOT use fileno in a core pyroute2 functionality,
+ # since the core should be both Python 2 and 3
+ # compatible.
+ #
+ super(NetlinkMixin, self).__init__()
+ if fileno is not None and sys.version_info[0] < 3:
+ raise NotImplementedError('fileno parameter is not supported '
+ 'on Python < 3.2')
+
+ # 8<-----------------------------------------
+ # PID init is here only for compatibility,
+ # later it will be completely moved to bind()
+ self.addr_pool = AddrPool(minaddr=0xff)
+ self.epid = None
+ self.port = 0
+ self.fixed = True
+ self.family = family
+ self._fileno = fileno
+ self.backlog = {0: []}
+ self.monitor = False
+ self.callbacks = [] # [(predicate, callback, args), ...]
+ self.clean_cbs = {} # {msg_seq: [callback, ...], ...}
+ self.pthread = None
+ self.closed = False
+ self.backlog_lock = threading.Lock()
+ self.read_lock = threading.Lock()
+ self.change_master = threading.Event()
+ self.lock = LockFactory()
+ self._sock = None
+ self._ctrl_read, self._ctrl_write = os.pipe()
+ self.buffer_queue = Queue()
+ self.qsize = 0
+ self.log = []
+ self.get_timeout = 30
+ self.get_timeout_exception = None
+ if pid is None:
+ self.pid = os.getpid() & 0x3fffff
+ self.port = port
+ self.fixed = self.port is not None
+ elif pid == 0:
+ self.pid = os.getpid()
+ else:
+ self.pid = pid
+ # 8<-----------------------------------------
+ self.groups = 0
+ self.marshal = Marshal()
+ # 8<-----------------------------------------
+ # Set defaults
+ self.post_init()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.close()
+
+ def release(self):
+ logging.warning("The `release()` call is deprecated")
+ logging.warning("Use `close()` instead")
+ self.close()
+
+ def register_callback(self, callback,
+ predicate=lambda x: True, args=None):
+ '''
+ Register a callback to run on a message arrival.
+
+ Callback is the function that will be called with the
+ message as the first argument. Predicate is the optional
+ callable object, that returns True or False. Upon True,
+ the callback will be called. Upon False it will not.
+ Args is a list or tuple of arguments.
+
+ Simplest example, assume ipr is the IPRoute() instance::
+
+ # create a simplest callback that will print messages
+ def cb(msg):
+ print(msg)
+
+ # register callback for any message:
+ ipr.register_callback(cb)
+
+ More complex example, with filtering::
+
+ # Set object's attribute after the message key
+ def cb(msg, obj):
+ obj.some_attr = msg["some key"]
+
+ # Register the callback only for the loopback device, index 1:
+ ipr.register_callback(cb,
+ lambda x: x.get('index', None) == 1,
+ (self, ))
+
+ Please note: you do **not** need to register the default 0 queue
+ to invoke callbacks on broadcast messages. Callbacks are
+ iterated **before** messages get enqueued.
+ '''
+ if args is None:
+ args = []
+ self.callbacks.append((predicate, callback, args))
+
+ def unregister_callback(self, callback):
+ '''
+ Remove the first reference to the function from the callback
+ register
+ '''
+ cb = tuple(self.callbacks)
+ for cr in cb:
+ if cr[1] == callback:
+ self.callbacks.pop(cb.index(cr))
+ return
+
+ def register_policy(self, policy, msg_class=None):
+ '''
+ Register netlink encoding/decoding policy. Can
+ be specified in two ways:
+ `nlsocket.register_policy(MSG_ID, msg_class)`
+ to register one particular rule, or
+ `nlsocket.register_policy({MSG_ID1: msg_class})`
+ to register several rules at once.
+ E.g.::
+
+ policy = {RTM_NEWLINK: ifinfmsg,
+ RTM_DELLINK: ifinfmsg,
+ RTM_NEWADDR: ifaddrmsg,
+ RTM_DELADDR: ifaddrmsg}
+ nlsocket.register_policy(policy)
+
+ One can call `register_policy()` as many times,
+ as one want to -- it will just extend the current
+ policy scheme, not replace it.
+ '''
+ if isinstance(policy, int) and msg_class is not None:
+ policy = {policy: msg_class}
+
+ assert isinstance(policy, dict)
+ for key in policy:
+ self.marshal.msg_map[key] = policy[key]
+
+ return self.marshal.msg_map
+
+ def unregister_policy(self, policy):
+ '''
+ Unregister policy. Policy can be:
+
+ - int -- then it will just remove one policy
+ - list or tuple of ints -- remove all given
+ - dict -- remove policies by keys from dict
+
+ In the last case the routine will ignore dict values,
+ it is implemented so just to make it compatible with
+ `get_policy_map()` return value.
+ '''
+ if isinstance(policy, int):
+ policy = [policy]
+ elif isinstance(policy, dict):
+ policy = list(policy)
+
+ assert isinstance(policy, (tuple, list, set))
+
+ for key in policy:
+ del self.marshal.msg_map[key]
+
+ return self.marshal.msg_map
+
+ def get_policy_map(self, policy=None):
+ '''
+ Return policy for a given message type or for all
+ message types. Policy parameter can be either int,
+ or a list of ints. Always return dictionary.
+ '''
+ if policy is None:
+ return self.marshal.msg_map
+
+ if isinstance(policy, int):
+ policy = [policy]
+
+ assert isinstance(policy, (list, tuple, set))
+
+ ret = {}
+ for key in policy:
+ ret[key] = self.marshal.msg_map[key]
+
+ return ret
+
+ def sendto(self, *argv, **kwarg):
+ return self._sendto(*argv, **kwarg)
+
+ def recv(self, *argv, **kwarg):
+ return self._recv(*argv, **kwarg)
+
+ def async_recv(self):
+ poll = select.poll()
+ poll.register(self._sock, select.POLLIN | select.POLLPRI)
+ poll.register(self._ctrl_read, select.POLLIN | select.POLLPRI)
+ sockfd = self._sock.fileno()
+ while True:
+ events = poll.poll()
+ for (fd, event) in events:
+ if fd == sockfd:
+ try:
+ self.buffer_queue.put(self._sock.recv(1024 * 1024))
+ except Exception as e:
+ self.buffer_queue.put(e)
+ else:
+ return
+
+ def put(self, msg, msg_type,
+ msg_flags=NLM_F_REQUEST,
+ addr=(0, 0),
+ msg_seq=0,
+ msg_pid=None):
+ '''
+ Construct a message from a dictionary and send it to
+ the socket. Parameters:
+
+ - msg -- the message in the dictionary format
+ - msg_type -- the message type
+ - msg_flags -- the message flags to use in the request
+ - addr -- `sendto()` addr, default `(0, 0)`
+ - msg_seq -- sequence number to use
+ - msg_pid -- pid to use, if `None` -- use os.getpid()
+
+ Example::
+
+ s = IPRSocket()
+ s.bind()
+ s.put({'index': 1}, RTM_GETLINK)
+ s.get()
+ s.close()
+
+ Please notice, that the return value of `s.get()` can be
+ not the result of `s.put()`, but any broadcast message.
+ To fix that, use `msg_seq` -- the response must contain the
+ same `msg['header']['sequence_number']` value.
+ '''
+ if msg_seq != 0:
+ self.lock[msg_seq].acquire()
+ try:
+ if msg_seq not in self.backlog:
+ self.backlog[msg_seq] = []
+ if not isinstance(msg, nlmsg):
+ msg_class = self.marshal.msg_map[msg_type]
+ msg = msg_class(msg)
+ if msg_pid is None:
+ msg_pid = os.getpid()
+ msg['header']['type'] = msg_type
+ msg['header']['flags'] = msg_flags
+ msg['header']['sequence_number'] = msg_seq
+ msg['header']['pid'] = msg_pid
+ msg.encode()
+ if msg_seq not in self.clean_cbs:
+ self.clean_cbs[msg_seq] = []
+ self.clean_cbs[msg_seq].extend(msg.clean_cbs)
+ self.sendto(msg.buf.getvalue(), addr)
+ except:
+ raise
+ finally:
+ if msg_seq != 0:
+ self.lock[msg_seq].release()
+
+ def get(self, bufsize=DEFAULT_RCVBUF, msg_seq=0, terminate=None):
+ '''
+ Get parsed messages list. If `msg_seq` is given, return
+ only messages with that `msg['header']['sequence_number']`,
+ saving all other messages into `self.backlog`.
+
+ The routine is thread-safe.
+
+ The `bufsize` parameter can be:
+
+ - -1: bufsize will be calculated from the first 4 bytes of
+ the network data
+ - 0: bufsize will be calculated from SO_RCVBUF sockopt
+ - int >= 0: just a bufsize
+ '''
+ ctime = time.time()
+
+ with self.lock[msg_seq]:
+ if bufsize == -1:
+ # get bufsize from the network data
+ bufsize = struct.unpack("I", self.recv(4, MSG_PEEK))[0]
+ elif bufsize == 0:
+ # get bufsize from SO_RCVBUF
+ bufsize = self.getsockopt(SOL_SOCKET, SO_RCVBUF) // 2
+
+ ret = []
+ enough = False
+ while not enough:
+ # 8<-----------------------------------------------------------
+ #
+ # This stage changes the backlog, so use mutex to
+ # prevent side changes
+ self.backlog_lock.acquire()
+ ##
+ # Stage 1. BEGIN
+ #
+ # 8<-----------------------------------------------------------
+ #
+ # Check backlog and return already collected
+ # messages.
+ #
+ if msg_seq == 0 and self.backlog[0]:
+ # Zero queue.
+ #
+ # Load the backlog, if there is valid
+ # content in it
+ ret.extend(self.backlog[0])
+ self.backlog[0] = []
+ # And just exit
+ self.backlog_lock.release()
+ break
+ elif self.backlog.get(msg_seq, None):
+ # Any other msg_seq.
+ #
+ # Collect messages up to the terminator.
+ # Terminator conditions:
+ # * NLMSG_ERROR != 0
+ # * NLMSG_DONE
+ # * terminate() function (if defined)
+ # * not NLM_F_MULTI
+ #
+ # Please note, that if terminator not occured,
+ # more `recv()` rounds CAN be required.
+ for msg in tuple(self.backlog[msg_seq]):
+
+ # Drop the message from the backlog, if any
+ self.backlog[msg_seq].remove(msg)
+
+ # If there is an error, raise exception
+ if msg['header'].get('error', None) is not None:
+ self.backlog[0].extend(self.backlog[msg_seq])
+ del self.backlog[msg_seq]
+ # The loop is done
+ self.backlog_lock.release()
+ raise msg['header']['error']
+
+ # If it is the terminator message, say "enough"
+ # and requeue all the rest into Zero queue
+ if (msg['header']['type'] == NLMSG_DONE) or \
+ (terminate is not None and terminate(msg)):
+ # The loop is done
+ enough = True
+
+ # If it is just a normal message, append it to
+ # the response
+ if not enough:
+ ret.append(msg)
+ # But finish the loop on single messages
+ if not msg['header']['flags'] & NLM_F_MULTI:
+ # but not multi -- so end the loop
+ enough = True
+
+ # Enough is enough, requeue the rest and delete
+ # our backlog
+ if enough:
+ self.backlog[0].extend(self.backlog[msg_seq])
+ del self.backlog[msg_seq]
+ break
+
+ # Next iteration
+ self.backlog_lock.release()
+ else:
+ # Stage 1. END
+ #
+ # 8<-------------------------------------------------------
+ #
+ # Stage 2. BEGIN
+ #
+ # 8<-------------------------------------------------------
+ #
+ # Receive the data from the socket and put the messages
+ # into the backlog
+ #
+ self.backlog_lock.release()
+ ##
+ #
+ # Control the timeout. We should not be within the
+ # function more than TIMEOUT seconds. All the locks
+ # MUST be released here.
+ #
+ if time.time() - ctime > self.get_timeout:
+ if self.get_timeout_exception:
+ raise self.get_timeout_exception()
+ else:
+ return ret
+ #
+ if self.read_lock.acquire(False):
+ self.change_master.clear()
+ # If the socket is free to read from, occupy
+ # it and wait for the data
+ #
+ # This is a time consuming process, so all the
+ # locks, except the read lock must be released
+ data = self.recv(bufsize)
+ # Parse data
+ msgs = self.marshal.parse(data)
+ # Reset ctime -- timeout should be measured
+ # for every turn separately
+ ctime = time.time()
+ #
+ current = self.buffer_queue.qsize()
+ delta = current - self.qsize
+ if delta > 10:
+ delay = min(3, max(0.1, float(current) / 60000))
+ message = ("Packet burst: the reader thread "
+ "priority is increased, beware of "
+ "delays on netlink calls\n\tCounters: "
+ "delta=%s qsize=%s delay=%s "
+ % (delta, current, delay))
+ if delay < 1:
+ logging.debug(message)
+ else:
+ logging.warning(message)
+ time.sleep(delay)
+ self.qsize = current
+
+ # We've got the data, lock the backlog again
+ self.backlog_lock.acquire()
+ for msg in msgs:
+ seq = msg['header']['sequence_number']
+ if seq in self.clean_cbs:
+ for cb in self.clean_cbs[seq]:
+ try:
+ cb()
+ except:
+ logging.warning("Cleanup callback"
+ "fail: %s" % (cb))
+ logging.warning(traceback.format_exc())
+ del self.clean_cbs[seq]
+ if seq not in self.backlog:
+ if msg['header']['type'] == NLMSG_ERROR:
+ # Drop orphaned NLMSG_ERROR messages
+ continue
+ seq = 0
+ # 8<-----------------------------------------------
+ # Callbacks section
+ for cr in self.callbacks:
+ try:
+ if cr[0](msg):
+ cr[1](msg, *cr[2])
+ except:
+ logging.warning("Callback fail: %s" % (cr))
+ logging.warning(traceback.format_exc())
+ # 8<-----------------------------------------------
+ self.backlog[seq].append(msg)
+ # Monitor mode:
+ if self.monitor and seq != 0:
+ self.backlog[0].append(msg)
+ # We finished with the backlog, so release the lock
+ self.backlog_lock.release()
+
+ # Now wake up other threads
+ self.change_master.set()
+
+ # Finally, release the read lock: all data processed
+ self.read_lock.release()
+ else:
+ # If the socket is occupied and there is still no
+ # data for us, wait for the next master change or
+ # for a timeout
+ self.change_master.wait(1)
+ # 8<-------------------------------------------------------
+ #
+ # Stage 2. END
+ #
+ # 8<-------------------------------------------------------
+
+ return ret
+
+ def nlm_request(self, msg, msg_type,
+ msg_flags=NLM_F_REQUEST | NLM_F_DUMP,
+ terminate=None):
+ msg_seq = self.addr_pool.alloc()
+ with self.lock[msg_seq]:
+ try:
+ self.put(msg, msg_type, msg_flags, msg_seq=msg_seq)
+ ret = self.get(msg_seq=msg_seq, terminate=terminate)
+ return ret
+ except:
+ raise
+ finally:
+ # Ban this msg_seq for 0xff rounds
+ #
+ # It's a long story. Modern kernels for RTM_SET.* operations
+ # always return NLMSG_ERROR(0) == success, even not setting
+ # NLM_F_MULTY flag on other response messages and thus w/o
+ # any NLMSG_DONE. So, how to detect the response end? One
+ # can not rely on NLMSG_ERROR on old kernels, but we have to
+ # support them too. Ty, we just ban msg_seq for several rounds,
+ # and NLMSG_ERROR, being received, will become orphaned and
+ # just dropped.
+ #
+ # Hack, but true.
+ self.addr_pool.free(msg_seq, ban=0xff)
+
+
+class NetlinkSocket(NetlinkMixin):
+
+ def post_init(self):
+ # recreate the underlying socket
+ with self.lock:
+ if self._sock is not None:
+ self._sock.close()
+ self._sock = SocketBase(AF_NETLINK,
+ SOCK_DGRAM,
+ self.family,
+ self._fileno)
+ for name in ('getsockname', 'getsockopt', 'makefile',
+ 'setsockopt', 'setblocking', 'settimeout',
+ 'gettimeout', 'shutdown', 'recvfrom',
+ 'recv_into', 'recvfrom_into', 'fileno'):
+ setattr(self, name, getattr(self._sock, name))
+
+ self._sendto = getattr(self._sock, 'sendto')
+ self._recv = getattr(self._sock, 'recv')
+
+ self.setsockopt(SOL_SOCKET, SO_SNDBUF, 32768)
+ self.setsockopt(SOL_SOCKET, SO_RCVBUF, 1024 * 1024)
+
+ def bind(self, groups=0, pid=None, async=False):
+ '''
+ Bind the socket to given multicast groups, using
+ given pid.
+
+ - If pid is None, use automatic port allocation
+ - If pid == 0, use process' pid
+ - If pid == <int>, use the value instead of pid
+ '''
+ if pid is not None:
+ self.port = 0
+ self.fixed = True
+ self.pid = pid or os.getpid()
+
+ self.groups = groups
+ # if we have pre-defined port, use it strictly
+ if self.fixed:
+ self.epid = self.pid + (self.port << 22)
+ self._sock.bind((self.epid, self.groups))
+ else:
+ for port in range(1024):
+ try:
+ self.port = port
+ self.epid = self.pid + (self.port << 22)
+ self._sock.bind((self.epid, self.groups))
+ break
+ except Exception:
+ # create a new underlying socket -- on kernel 4
+ # one failed bind() makes the socket useless
+ self.post_init()
+ else:
+ raise KeyError('no free address available')
+ # all is OK till now, so start async recv, if we need
+ if async:
+ def recv_plugin(*argv, **kwarg):
+ data = self.buffer_queue.get()
+ if isinstance(data, Exception):
+ raise data
+ else:
+ return data
+ self._recv = recv_plugin
+ self.pthread = threading.Thread(target=self.async_recv)
+ self.pthread.setDaemon(True)
+ self.pthread.start()
+
+ def close(self):
+ '''
+ Correctly close the socket and free all resources.
+ '''
+ with self.lock:
+ if self.closed:
+ return
+ self.closed = True
+
+ if self.pthread:
+ os.write(self._ctrl_write, b'exit')
+ self.pthread.join()
+
+ os.close(self._ctrl_write)
+ os.close(self._ctrl_read)
+
+ # Common shutdown procedure
+ self._sock.close()
diff --git a/node-admin/scripts/pyroute2/netlink/rtnl/__init__.py b/node-admin/scripts/pyroute2/netlink/rtnl/__init__.py
new file mode 100644
index 00000000000..fd4c4d03a96
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/rtnl/__init__.py
@@ -0,0 +1,156 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+RTNetlink: network setup
+========================
+
+RTNL is a netlink protocol, used to get and set information
+about different network objects -- addresses, routes, interfaces
+etc.
+
+RTNL protocol-specific data in messages depends on the object
+type. E.g., complete packet with the interface address information::
+
+ nlmsg header:
+ + uint32 length
+ + uint16 type
+ + uint16 flags
+ + uint32 sequence number
+ + uint32 pid
+ ifaddrmsg structure:
+ + unsigned char ifa_family
+ + unsigned char ifa_prefixlen
+ + unsigned char ifa_flags
+ + unsigned char ifa_scope
+ + uint32 ifa_index
+ [ optional NLA tree ]
+
+NLA for this kind of packets can be of type IFA_ADDRESS, IFA_LOCAL
+etc. -- please refer to the corresponding source.
+
+Other objects types require different structures, sometimes really
+complex. All these structures are described in sources.
+
+---------------------------
+
+Module contents:
+
+'''
+from pyroute2.common import map_namespace
+
+# RTnetlink multicast groups
+RTNLGRP_NONE = 0x0
+RTNLGRP_LINK = 0x1
+RTNLGRP_NOTIFY = 0x2
+RTNLGRP_NEIGH = 0x4
+RTNLGRP_TC = 0x8
+RTNLGRP_IPV4_IFADDR = 0x10
+RTNLGRP_IPV4_MROUTE = 0x20
+RTNLGRP_IPV4_ROUTE = 0x40
+RTNLGRP_IPV4_RULE = 0x80
+RTNLGRP_IPV6_IFADDR = 0x100
+RTNLGRP_IPV6_MROUTE = 0x200
+RTNLGRP_IPV6_ROUTE = 0x400
+RTNLGRP_IPV6_IFINFO = 0x800
+RTNLGRP_DECnet_IFADDR = 0x1000
+RTNLGRP_NOP2 = 0x2000
+RTNLGRP_DECnet_ROUTE = 0x4000
+RTNLGRP_DECnet_RULE = 0x8000
+RTNLGRP_NOP4 = 0x10000
+RTNLGRP_IPV6_PREFIX = 0x20000
+RTNLGRP_IPV6_RULE = 0x40000
+
+# Types of messages
+# RTM_BASE = 16
+RTM_NEWLINK = 16
+RTM_DELLINK = 17
+RTM_GETLINK = 18
+RTM_SETLINK = 19
+RTM_NEWADDR = 20
+RTM_DELADDR = 21
+RTM_GETADDR = 22
+RTM_NEWROUTE = 24
+RTM_DELROUTE = 25
+RTM_GETROUTE = 26
+RTM_NEWNEIGH = 28
+RTM_DELNEIGH = 29
+RTM_GETNEIGH = 30
+RTM_NEWRULE = 32
+RTM_DELRULE = 33
+RTM_GETRULE = 34
+RTM_NEWQDISC = 36
+RTM_DELQDISC = 37
+RTM_GETQDISC = 38
+RTM_NEWTCLASS = 40
+RTM_DELTCLASS = 41
+RTM_GETTCLASS = 42
+RTM_NEWTFILTER = 44
+RTM_DELTFILTER = 45
+RTM_GETTFILTER = 46
+RTM_NEWACTION = 48
+RTM_DELACTION = 49
+RTM_GETACTION = 50
+RTM_NEWPREFIX = 52
+RTM_GETMULTICAST = 58
+RTM_GETANYCAST = 62
+RTM_NEWNEIGHTBL = 64
+RTM_GETNEIGHTBL = 66
+RTM_SETNEIGHTBL = 67
+# custom message types
+RTM_GETBRIDGE = 88
+RTM_SETBRIDGE = 89
+RTM_GETBOND = 90
+RTM_SETBOND = 91
+(RTM_NAMES, RTM_VALUES) = map_namespace('RTM', globals())
+
+TC_H_INGRESS = 0xfffffff1
+TC_H_ROOT = 0xffffffff
+
+
+RTNL_GROUPS = RTNLGRP_IPV4_IFADDR |\
+ RTNLGRP_IPV6_IFADDR |\
+ RTNLGRP_IPV4_ROUTE |\
+ RTNLGRP_IPV6_ROUTE |\
+ RTNLGRP_NEIGH |\
+ RTNLGRP_LINK |\
+ RTNLGRP_TC
+
+
+rtypes = {'RTN_UNSPEC': 0,
+ 'RTN_UNICAST': 1, # Gateway or direct route
+ 'RTN_LOCAL': 2, # Accept locally
+ 'RTN_BROADCAST': 3, # Accept locally as broadcast
+ # send as broadcast
+ 'RTN_ANYCAST': 4, # Accept locally as broadcast,
+ # but send as unicast
+ 'RTN_MULTICAST': 5, # Multicast route
+ 'RTN_BLACKHOLE': 6, # Drop
+ 'RTN_UNREACHABLE': 7, # Destination is unreachable
+ 'RTN_PROHIBIT': 8, # Administratively prohibited
+ 'RTN_THROW': 9, # Not in this table
+ 'RTN_NAT': 10, # Translate this address
+ 'RTN_XRESOLVE': 11} # Use external resolver
+
+rtprotos = {'RTPROT_UNSPEC': 0,
+ 'RTPROT_REDIRECT': 1, # Route installed by ICMP redirects;
+ # not used by current IPv4
+ 'RTPROT_KERNEL': 2, # Route installed by kernel
+ 'RTPROT_BOOT': 3, # Route installed during boot
+ 'RTPROT_STATIC': 4, # Route installed by administrator
+ # Values of protocol >= RTPROT_STATIC are not
+ # interpreted by kernel;
+ # keep in sync with iproute2 !
+ 'RTPROT_GATED': 8, # gated
+ 'RTPROT_RA': 9, # RDISC/ND router advertisements
+ 'RTPROT_MRT': 10, # Merit MRT
+ 'RTPROT_ZEBRA': 11, # Zebra
+ 'RTPROT_BIRD': 12, # BIRD
+ 'RTPROT_DNROUTED': 13, # DECnet routing daemon
+ 'RTPROT_XORP': 14, # XORP
+ 'RTPROT_NTK': 15, # Netsukuku
+ 'RTPROT_DHCP': 16} # DHCP client
+
+rtscopes = {'RT_SCOPE_UNIVERSE': 0,
+ 'RT_SCOPE_SITE': 200,
+ 'RT_SCOPE_LINK': 253,
+ 'RT_SCOPE_HOST': 254,
+ 'RT_SCOPE_NOWHERE': 255}
diff --git a/node-admin/scripts/pyroute2/netlink/rtnl/errmsg.py b/node-admin/scripts/pyroute2/netlink/rtnl/errmsg.py
new file mode 100644
index 00000000000..2bea8091730
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/rtnl/errmsg.py
@@ -0,0 +1,11 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+from pyroute2.netlink import nlmsg
+
+
+class errmsg(nlmsg):
+ '''
+ Custom message type
+
+ Error ersatz-message
+ '''
+ fields = (('code', 'i'), )
diff --git a/node-admin/scripts/pyroute2/netlink/rtnl/fibmsg.py b/node-admin/scripts/pyroute2/netlink/rtnl/fibmsg.py
new file mode 100644
index 00000000000..686160da398
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/rtnl/fibmsg.py
@@ -0,0 +1,60 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+
+from pyroute2.common import map_namespace
+from pyroute2.netlink import nlmsg
+
+FR_ACT_UNSPEC = 0
+FR_ACT_TO_TBL = 1
+FR_ACT_GOTO = 2
+FR_ACT_NOP = 3
+FR_ACT_BLACKHOLE = 6
+FR_ACT_UNREACHABLE = 7
+FR_ACT_PROHIBIT = 8
+(FR_ACT_NAMES, FR_ACT_VALUES) = map_namespace('FR_ACT', globals())
+
+
+class fibmsg(nlmsg):
+ '''
+ IP rule message
+
+ C structure::
+
+ struct fib_rule_hdr {
+ __u8 family;
+ __u8 dst_len;
+ __u8 src_len;
+ __u8 tos;
+ __u8 table;
+ __u8 res1; /* reserved */
+ __u8 res2; /* reserved */
+ __u8 action;
+ __u32 flags;
+ };
+ '''
+ prefix = 'FRA_'
+
+ fields = (('family', 'B'),
+ ('dst_len', 'B'),
+ ('src_len', 'B'),
+ ('tos', 'B'),
+ ('table', 'B'),
+ ('res1', 'B'),
+ ('res2', 'B'),
+ ('action', 'B'),
+ ('flags', 'I'))
+
+ # fibmsg NLA numbers are not sequential, so
+ # give it here explicitly
+ nla_map = ((0, 'FRA_UNSPEC', 'none'),
+ (1, 'FRA_DST', 'ipaddr'),
+ (2, 'FRA_SRC', 'ipaddr'),
+ (3, 'FRA_IIFNAME', 'asciiz'),
+ (4, 'FRA_GOTO', 'uint32'),
+ (6, 'FRA_PRIORITY', 'uint32'),
+ (10, 'FRA_FWMARK', 'uint32'),
+ (11, 'FRA_FLOW', 'uint32'),
+ (13, 'FRA_SUPPRESS_IFGROUP', 'uint32'),
+ (14, 'FRA_SUPPRESS_PREFIXLEN', 'uint32'),
+ (15, 'FRA_TABLE', 'uint32'),
+ (16, 'FRA_FWMASK', 'uint32'),
+ (17, 'FRA_OIFNAME', 'asciiz'))
diff --git a/node-admin/scripts/pyroute2/netlink/rtnl/ifaddrmsg.py b/node-admin/scripts/pyroute2/netlink/rtnl/ifaddrmsg.py
new file mode 100644
index 00000000000..0ecb2273611
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/rtnl/ifaddrmsg.py
@@ -0,0 +1,96 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+import socket
+from pyroute2.common import map_namespace
+from pyroute2.netlink import nlmsg
+from pyroute2.netlink import nla
+
+# address attributes
+#
+# Important comment:
+# For IPv4, IFA_ADDRESS is a prefix address, not a local interface
+# address. It makes no difference for normal interfaces, but
+# for point-to-point ones IFA_ADDRESS means DESTINATION address,
+# and the local address is supplied in IFA_LOCAL attribute.
+#
+
+IFA_F_SECONDARY = 0x01
+# IFA_F_TEMPORARY IFA_F_SECONDARY
+IFA_F_NODAD = 0x02
+IFA_F_OPTIMISTIC = 0x04
+IFA_F_DADFAILED = 0x08
+IFA_F_HOMEADDRESS = 0x10
+IFA_F_DEPRECATED = 0x20
+IFA_F_TENTATIVE = 0x40
+IFA_F_PERMANENT = 0x80
+IFA_F_MANAGETEMPADDR = 0x100
+IFA_F_NOPREFIXROUTE = 0x200
+
+(IFA_F_NAMES, IFA_F_VALUES) = map_namespace('IFA_F', globals())
+# 8<----------------------------------------------
+IFA_F_TEMPORARY = IFA_F_SECONDARY
+IFA_F_NAMES['IFA_F_TEMPORARY'] = IFA_F_TEMPORARY
+IFA_F_VALUES6 = IFA_F_VALUES
+IFA_F_VALUES6[IFA_F_TEMPORARY] = 'IFA_F_TEMPORARY'
+# 8<----------------------------------------------
+
+
+class ifaddrmsg(nlmsg):
+ '''
+ IP address information
+
+ C structure::
+
+ struct ifaddrmsg {
+ unsigned char ifa_family; /* Address type */
+ unsigned char ifa_prefixlen; /* Prefixlength of address */
+ unsigned char ifa_flags; /* Address flags */
+ unsigned char ifa_scope; /* Address scope */
+ int ifa_index; /* Interface index */
+ };
+
+ '''
+ prefix = 'IFA_'
+
+ fields = (('family', 'B'),
+ ('prefixlen', 'B'),
+ ('flags', 'B'),
+ ('scope', 'B'),
+ ('index', 'I'))
+
+ nla_map = (('IFA_UNSPEC', 'hex'),
+ ('IFA_ADDRESS', 'ipaddr'),
+ ('IFA_LOCAL', 'ipaddr'),
+ ('IFA_LABEL', 'asciiz'),
+ ('IFA_BROADCAST', 'ipaddr'),
+ ('IFA_ANYCAST', 'ipaddr'),
+ ('IFA_CACHEINFO', 'cacheinfo'),
+ ('IFA_MULTICAST', 'ipaddr'),
+ ('IFA_FLAGS', 'uint32'))
+
+ class cacheinfo(nla):
+ fields = (('ifa_prefered', 'I'),
+ ('ifa_valid', 'I'),
+ ('cstamp', 'I'),
+ ('tstamp', 'I'))
+
+ @staticmethod
+ def flags2names(flags, family=socket.AF_INET):
+ if family == socket.AF_INET6:
+ ifa_f_values = IFA_F_VALUES6
+ else:
+ ifa_f_values = IFA_F_VALUES
+ ret = []
+ for f in ifa_f_values:
+ if f & flags:
+ ret.append(ifa_f_values[f])
+ return ret
+
+ @staticmethod
+ def names2flags(flags):
+ ret = 0
+ for f in flags:
+ if f[0] == '!':
+ f = f[1:]
+ else:
+ ret |= IFA_F_NAMES[f]
+ return ret
diff --git a/node-admin/scripts/pyroute2/netlink/rtnl/ifinfmsg.py b/node-admin/scripts/pyroute2/netlink/rtnl/ifinfmsg.py
new file mode 100644
index 00000000000..02bc1cf17a7
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/rtnl/ifinfmsg.py
@@ -0,0 +1,1068 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+import os
+import time
+import json
+import struct
+import logging
+import platform
+import subprocess
+from fcntl import ioctl
+from pyroute2.common import map_namespace
+from pyroute2.common import ANCIENT
+# from pyroute2.netlink import NLMSG_ERROR
+from pyroute2.netlink import nla
+from pyroute2.netlink import nlmsg
+from pyroute2.netlink import nlmsg_atoms
+from pyroute2.netlink.rtnl.iw_event import iw_event
+
+
+# it's simpler to double constants here, than to change all the
+# module layout; but it is a subject of the future refactoring
+RTM_NEWLINK = 16
+RTM_DELLINK = 17
+#
+
+_ANCIENT_BARRIER = 0.3
+_BONDING_MASTERS = '/sys/class/net/bonding_masters'
+_BONDING_SLAVES = '/sys/class/net/%s/bonding/slaves'
+_BRIDGE_MASTER = '/sys/class/net/%s/brport/bridge/ifindex'
+_BONDING_MASTER = '/sys/class/net/%s/master/ifindex'
+IFNAMSIZ = 16
+
+TUNDEV = '/dev/net/tun'
+arch = platform.machine()
+if arch == 'x86_64':
+ TUNSETIFF = 0x400454ca
+ TUNSETPERSIST = 0x400454cb
+ TUNSETOWNER = 0x400454cc
+ TUNSETGROUP = 0x400454ce
+elif arch in ('ppc64', 'mips'):
+ TUNSETIFF = 0x800454ca
+ TUNSETPERSIST = 0x800454cb
+ TUNSETOWNER = 0x800454cc
+ TUNSETGROUP = 0x800454ce
+else:
+ TUNSETIFF = None
+
+##
+#
+# tuntap flags
+#
+IFT_TUN = 0x0001
+IFT_TAP = 0x0002
+IFT_NO_PI = 0x1000
+IFT_ONE_QUEUE = 0x2000
+IFT_VNET_HDR = 0x4000
+IFT_TUN_EXCL = 0x8000
+IFT_MULTI_QUEUE = 0x0100
+IFT_ATTACH_QUEUE = 0x0200
+IFT_DETACH_QUEUE = 0x0400
+# read-only
+IFT_PERSIST = 0x0800
+IFT_NOFILTER = 0x1000
+
+##
+#
+# normal flags
+#
+IFF_UP = 0x1 # interface is up
+IFF_BROADCAST = 0x2 # broadcast address valid
+IFF_DEBUG = 0x4 # turn on debugging
+IFF_LOOPBACK = 0x8 # is a loopback net
+IFF_POINTOPOINT = 0x10 # interface is has p-p link
+IFF_NOTRAILERS = 0x20 # avoid use of trailers
+IFF_RUNNING = 0x40 # interface RFC2863 OPER_UP
+IFF_NOARP = 0x80 # no ARP protocol
+IFF_PROMISC = 0x100 # receive all packets
+IFF_ALLMULTI = 0x200 # receive all multicast packets
+IFF_MASTER = 0x400 # master of a load balancer
+IFF_SLAVE = 0x800 # slave of a load balancer
+IFF_MULTICAST = 0x1000 # Supports multicast
+IFF_PORTSEL = 0x2000 # can set media type
+IFF_AUTOMEDIA = 0x4000 # auto media select active
+IFF_DYNAMIC = 0x8000 # dialup device with changing addresses
+IFF_LOWER_UP = 0x10000 # driver signals L1 up
+IFF_DORMANT = 0x20000 # driver signals dormant
+IFF_ECHO = 0x40000 # echo sent packets
+
+(IFF_NAMES, IFF_VALUES) = map_namespace('IFF', globals())
+
+IFF_MASK = IFF_UP |\
+ IFF_DEBUG |\
+ IFF_NOTRAILERS |\
+ IFF_NOARP |\
+ IFF_PROMISC |\
+ IFF_ALLMULTI
+
+IFF_VOLATILE = IFF_LOOPBACK |\
+ IFF_POINTOPOINT |\
+ IFF_BROADCAST |\
+ IFF_ECHO |\
+ IFF_MASTER |\
+ IFF_SLAVE |\
+ IFF_RUNNING |\
+ IFF_LOWER_UP |\
+ IFF_DORMANT
+
+states = ('UNKNOWN',
+ 'NOTPRESENT',
+ 'DOWN',
+ 'LOWERLAYERDOWN',
+ 'TESTING',
+ 'DORMANT',
+ 'UP')
+state_by_name = dict(((i[1], i[0]) for i in enumerate(states)))
+state_by_code = dict(enumerate(states))
+stats_names = ('rx_packets',
+ 'tx_packets',
+ 'rx_bytes',
+ 'tx_bytes',
+ 'rx_errors',
+ 'tx_errors',
+ 'rx_dropped',
+ 'tx_dropped',
+ 'multicast',
+ 'collisions',
+ 'rx_length_errors',
+ 'rx_over_errors',
+ 'rx_crc_errors',
+ 'rx_frame_errors',
+ 'rx_fifo_errors',
+ 'rx_missed_errors',
+ 'tx_aborted_errors',
+ 'tx_carrier_errors',
+ 'tx_fifo_errors',
+ 'tx_heartbeat_errors',
+ 'tx_window_errors',
+ 'rx_compressed',
+ 'tx_compressed')
+
+
+class ifinfbase(object):
+ '''
+ Network interface message.
+
+ C structure::
+
+ struct ifinfomsg {
+ unsigned char ifi_family; /* AF_UNSPEC */
+ unsigned short ifi_type; /* Device type */
+ int ifi_index; /* Interface index */
+ unsigned int ifi_flags; /* Device flags */
+ unsigned int ifi_change; /* change mask */
+ };
+ '''
+ prefix = 'IFLA_'
+
+ fields = (('family', 'B'),
+ ('__align', 'B'),
+ ('ifi_type', 'H'),
+ ('index', 'i'),
+ ('flags', 'I'),
+ ('change', 'I'))
+
+ nla_map = (('IFLA_UNSPEC', 'none'),
+ ('IFLA_ADDRESS', 'l2addr'),
+ ('IFLA_BROADCAST', 'l2addr'),
+ ('IFLA_IFNAME', 'asciiz'),
+ ('IFLA_MTU', 'uint32'),
+ ('IFLA_LINK', 'uint32'),
+ ('IFLA_QDISC', 'asciiz'),
+ ('IFLA_STATS', 'ifstats'),
+ ('IFLA_COST', 'hex'),
+ ('IFLA_PRIORITY', 'hex'),
+ ('IFLA_MASTER', 'uint32'),
+ ('IFLA_WIRELESS', 'wireless'),
+ ('IFLA_PROTINFO', 'hex'),
+ ('IFLA_TXQLEN', 'uint32'),
+ ('IFLA_MAP', 'ifmap'),
+ ('IFLA_WEIGHT', 'hex'),
+ ('IFLA_OPERSTATE', 'state'),
+ ('IFLA_LINKMODE', 'uint8'),
+ ('IFLA_LINKINFO', 'ifinfo'),
+ ('IFLA_NET_NS_PID', 'uint32'),
+ ('IFLA_IFALIAS', 'asciiz'),
+ ('IFLA_NUM_VF', 'uint32'),
+ ('IFLA_VFINFO_LIST', 'hex'),
+ ('IFLA_STATS64', 'ifstats64'),
+ ('IFLA_VF_PORTS', 'hex'),
+ ('IFLA_PORT_SELF', 'hex'),
+ ('IFLA_AF_SPEC', 'af_spec'),
+ ('IFLA_GROUP', 'uint32'),
+ ('IFLA_NET_NS_FD', 'netns_fd'),
+ ('IFLA_EXT_MASK', 'hex'),
+ ('IFLA_PROMISCUITY', 'uint32'),
+ ('IFLA_NUM_TX_QUEUES', 'uint32'),
+ ('IFLA_NUM_RX_QUEUES', 'uint32'),
+ ('IFLA_CARRIER', 'uint8'),
+ ('IFLA_PHYS_PORT_ID', 'hex'),
+ ('IFLA_CARRIER_CHANGES', 'uint32'))
+
+ @staticmethod
+ def flags2names(flags, mask=0xffffffff):
+ ret = []
+ for flag in IFF_VALUES:
+ if (flag & mask & flags) == flag:
+ ret.append(IFF_VALUES[flag])
+ return ret
+
+ @staticmethod
+ def names2flags(flags):
+ ret = 0
+ mask = 0
+ for flag in flags:
+ if flag[0] == '!':
+ flag = flag[1:]
+ else:
+ ret |= IFF_NAMES[flag]
+ mask |= IFF_NAMES[flag]
+ return (ret, mask)
+
+ def encode(self):
+ # convert flags
+ if isinstance(self['flags'], (set, tuple, list)):
+ self['flags'], self['change'] = self.names2flags(self['flags'])
+ return super(ifinfbase, self).encode()
+
+ class netns_fd(nla):
+ fields = [('value', 'I')]
+ netns_run_dir = '/var/run/netns'
+ netns_fd = None
+
+ def encode(self):
+ self.close()
+ #
+ # There are two ways to specify netns
+ #
+ # 1. provide fd to an open file
+ # 2. provide a file name
+ #
+ # In the first case, the value is passed to the kernel
+ # as is. In the second case, the object opens appropriate
+ # file from `self.netns_run_dir` and closes it upon
+ # `__del__(self)`
+ if isinstance(self.value, int):
+ self['value'] = self.value
+ else:
+ self.netns_fd = os.open('%s/%s' % (self.netns_run_dir,
+ self.value), os.O_RDONLY)
+ self['value'] = self.netns_fd
+ nla.encode(self)
+ self.register_clean_cb(self.close)
+
+ def close(self):
+ if self.netns_fd is not None:
+ os.close(self.netns_fd)
+
+ class wireless(iw_event):
+ pass
+
+ class state(nla):
+ fields = (('value', 'B'), )
+
+ def encode(self):
+ self['value'] = state_by_name[self.value]
+ nla.encode(self)
+
+ def decode(self):
+ nla.decode(self)
+ self.value = state_by_code[self['value']]
+
+ class ifstats(nla):
+ fields = [(i, 'I') for i in stats_names]
+
+ class ifstats64(nla):
+ fields = [(i, 'Q') for i in stats_names]
+
+ class ifmap(nla):
+ fields = (('mem_start', 'Q'),
+ ('mem_end', 'Q'),
+ ('base_addr', 'Q'),
+ ('irq', 'H'),
+ ('dma', 'B'),
+ ('port', 'B'))
+
+ class ifinfo(nla):
+ nla_map = (('IFLA_INFO_UNSPEC', 'none'),
+ ('IFLA_INFO_KIND', 'asciiz'),
+ ('IFLA_INFO_DATA', 'info_data'),
+ ('IFLA_INFO_XSTATS', 'hex'),
+ ('IFLA_INFO_SLAVE_KIND', 'asciiz'),
+ ('IFLA_INFO_SLAVE_DATA', 'info_data'))
+
+ def info_data(self, *argv, **kwarg):
+ '''
+ The function returns appropriate IFLA_INFO_DATA
+ type according to IFLA_INFO_KIND info. Return
+ 'hex' type for all unknown kind's and when the
+ kind is not known.
+ '''
+ kind = self.get_attr('IFLA_INFO_KIND')
+ slave = self.get_attr('IFLA_INFO_SLAVE_KIND')
+ data_map = {'vlan': self.vlan_data,
+ 'vxlan': self.vxlan_data,
+ 'macvlan': self.macvlan_data,
+ 'macvtap': self.macvtap_data,
+ 'gre': self.gre_data,
+ 'bond': self.bond_data,
+ 'veth': self.veth_data,
+ 'tuntap': self.tuntap_data,
+ 'bridge': self.bridge_data}
+ slave_map = {'openvswitch': self.ovs_data}
+ return data_map.get(kind, slave_map.get(slave, self.hex))
+
+ class tuntap_data(nla):
+ '''
+ Fake data type
+ '''
+ prefix = 'IFTUN_'
+
+ nla_map = (('IFTUN_UNSPEC', 'none'),
+ ('IFTUN_MODE', 'asciiz'),
+ ('IFTUN_UID', 'uint32'),
+ ('IFTUN_GID', 'uint32'),
+ ('IFTUN_IFR', 'flags'))
+
+ class flags(nla):
+ fields = (('no_pi', 'B'),
+ ('one_queue', 'B'),
+ ('vnet_hdr', 'B'),
+ ('tun_excl', 'B'),
+ ('multi_queue', 'B'),
+ ('persist', 'B'),
+ ('nofilter', 'B'))
+
+ class veth_data(nla):
+ nla_map = (('VETH_INFO_UNSPEC', 'none'),
+ ('VETH_INFO_PEER', 'info_peer'))
+
+ def info_peer(self, *argv, **kwarg):
+ return ifinfveth
+
+ class ovs_data(nla):
+ prefix = 'IFLA_'
+ nla_map = (('IFLA_OVS_UNSPEC', 'none'),
+ ('IFLA_OVS_MASTER_IFNAME', 'asciiz'))
+
+ class vxlan_data(nla):
+ prefix = 'IFLA_'
+ nla_map = (('IFLA_VXLAN_UNSPEC', 'none'),
+ ('IFLA_VXLAN_ID', 'uint32'),
+ ('IFLA_VXLAN_GROUP', 'ip4addr'),
+ ('IFLA_VXLAN_LINK', 'uint32'),
+ ('IFLA_VXLAN_LOCAL', 'ip4addr'),
+ ('IFLA_VXLAN_TTL', 'uint8'),
+ ('IFLA_VXLAN_TOS', 'uint8'),
+ ('IFLA_VXLAN_LEARNING', 'uint8'),
+ ('IFLA_VXLAN_AGEING', 'uint32'),
+ ('IFLA_VXLAN_LIMIT', 'uint32'),
+ ('IFLA_VXLAN_PORT_RANGE', 'port_range'),
+ ('IFLA_VXLAN_PROXY', 'uint8'),
+ ('IFLA_VXLAN_RSC', 'uint8'),
+ ('IFLA_VXLAN_L2MISS', 'uint8'),
+ ('IFLA_VXLAN_L3MISS', 'uint8'),
+ ('IFLA_VXLAN_PORT', 'uint16'),
+ ('IFLA_VXLAN_GROUP6', 'ip6addr'),
+ ('IFLA_VXLAN_LOCAL6', 'ip6addr'),
+ ('IFLA_VXLAN_UDP_CSUM', 'uint8'),
+ ('IFLA_VXLAN_UDP_ZERO_CSUM6_TX', 'uint8'),
+ ('IFLA_VXLAN_UDP_ZERO_CSUM6_RX', 'uint8'))
+
+ class port_range(nla):
+ fields = (('low', '>H'),
+ ('high', '>H'))
+
+ class gre_data(nla):
+ prefix = 'IFLA_'
+
+ nla_map = (('IFLA_GRE_UNSPEC', 'none'),
+ ('IFLA_GRE_LINK', 'uint32'),
+ ('IFLA_GRE_IFLAGS', 'uint16'),
+ ('IFLA_GRE_OFLAGS', 'uint16'),
+ ('IFLA_GRE_IKEY', 'uint32'),
+ ('IFLA_GRE_OKEY', 'uint32'),
+ ('IFLA_GRE_LOCAL', 'ip4addr'),
+ ('IFLA_GRE_REMOTE', 'ip4addr'),
+ ('IFLA_GRE_TTL', 'uint8'),
+ ('IFLA_GRE_TOS', 'uint8'),
+ ('IFLA_GRE_PMTUDISC', 'uint8'),
+ ('IFLA_GRE_ENCAP_LIMIT', 'uint8'),
+ ('IFLA_GRE_FLOWINFO', 'uint32'),
+ ('IFLA_GRE_FLAGS', 'uint32'))
+
+ class macvx_data(nla):
+ prefix = 'IFLA_'
+
+ class mode(nlmsg_atoms.uint32):
+ value_map = {0: 'none',
+ 1: 'private',
+ 2: 'vepa',
+ 4: 'bridge',
+ 8: 'passthru'}
+
+ class flags(nlmsg_atoms.uint16):
+ value_map = {0: 'none',
+ 1: 'nopromisc'}
+
+ class macvtap_data(macvx_data):
+ nla_map = (('IFLA_MACVTAP_UNSPEC', 'none'),
+ ('IFLA_MACVTAP_MODE', 'mode'),
+ ('IFLA_MACVTAP_FLAGS', 'flags'))
+
+ class macvlan_data(macvx_data):
+ nla_map = (('IFLA_MACVLAN_UNSPEC', 'none'),
+ ('IFLA_MACVLAN_MODE', 'mode'),
+ ('IFLA_MACVLAN_FLAGS', 'flags'))
+
+ class vlan_data(nla):
+ nla_map = (('IFLA_VLAN_UNSPEC', 'none'),
+ ('IFLA_VLAN_ID', 'uint16'),
+ ('IFLA_VLAN_FLAGS', 'vlan_flags'),
+ ('IFLA_VLAN_EGRESS_QOS', 'hex'),
+ ('IFLA_VLAN_INGRESS_QOS', 'hex'))
+
+ class vlan_flags(nla):
+ fields = (('flags', 'I'),
+ ('mask', 'I'))
+
+ class bridge_data(nla):
+ prefix = 'IFLA_BRIDGE_'
+ nla_map = (('IFLA_BRIDGE_STP_STATE', 'uint32'),
+ ('IFLA_BRIDGE_MAX_AGE', 'uint32'))
+
+ class bond_data(nla):
+ prefix = 'IFLA_BOND_'
+ nla_map = (('IFLA_BOND_UNSPEC', 'none'),
+ ('IFLA_BOND_MODE', 'uint8'),
+ ('IFLA_BOND_ACTIVE_SLAVE', 'uint32'),
+ ('IFLA_BOND_MIIMON', 'uint32'),
+ ('IFLA_BOND_UPDELAY', 'uint32'),
+ ('IFLA_BOND_DOWNDELAY', 'uint32'),
+ ('IFLA_BOND_USE_CARRIER', 'uint8'),
+ ('IFLA_BOND_ARP_INTERVAL', 'uint32'),
+ ('IFLA_BOND_ARP_IP_TARGET', 'arp_ip_target'),
+ ('IFLA_BOND_ARP_VALIDATE', 'uint32'),
+ ('IFLA_BOND_ARP_ALL_TARGETS', 'uint32'),
+ ('IFLA_BOND_PRIMARY', 'uint32'),
+ ('IFLA_BOND_PRIMARY_RESELECT', 'uint8'),
+ ('IFLA_BOND_FAIL_OVER_MAC', 'uint8'),
+ ('IFLA_BOND_XMIT_HASH_POLICY', 'uint8'),
+ ('IFLA_BOND_RESEND_IGMP', 'uint32'),
+ ('IFLA_BOND_NUM_PEER_NOTIF', 'uint8'),
+ ('IFLA_BOND_ALL_SLAVES_ACTIVE', 'uint8'),
+ ('IFLA_BOND_MIN_LINKS', 'uint32'),
+ ('IFLA_BOND_LP_INTERVAL', 'uint32'),
+ ('IFLA_BOND_PACKETS_PER_SLAVE', 'uint32'),
+ ('IFLA_BOND_AD_LACP_RATE', 'uint8'),
+ ('IFLA_BOND_AD_SELECT', 'uint8'),
+ ('IFLA_BOND_AD_INFO', 'ad_info'))
+
+ class ad_info(nla):
+ nla_map = (('IFLA_BOND_AD_INFO_UNSPEC', 'none'),
+ ('IFLA_BOND_AD_INFO_AGGREGATOR', 'uint16'),
+ ('IFLA_BOND_AD_INFO_NUM_PORTS', 'uint16'),
+ ('IFLA_BOND_AD_INFO_ACTOR_KEY', 'uint16'),
+ ('IFLA_BOND_AD_INFO_PARTNER_KEY', 'uint16'),
+ ('IFLA_BOND_AD_INFO_PARTNER_MAC', 'l2addr'))
+
+ class arp_ip_target(nla):
+ fields = (('targets', '16I'), )
+
+ class af_spec(nla):
+ nla_map = (('AF_UNSPEC', 'none'),
+ ('AF_UNIX', 'hex'),
+ ('AF_INET', 'inet'),
+ ('AF_AX25', 'hex'),
+ ('AF_IPX', 'hex'),
+ ('AF_APPLETALK', 'hex'),
+ ('AF_NETROM', 'hex'),
+ ('AF_BRIDGE', 'hex'),
+ ('AF_ATMPVC', 'hex'),
+ ('AF_X25', 'hex'),
+ ('AF_INET6', 'inet6'))
+
+ class inet(nla):
+ # ./include/linux/inetdevice.h: struct ipv4_devconf
+ field_names = ('sysctl',
+ 'forwarding',
+ 'mc_forwarding',
+ 'proxy_arp',
+ 'accept_redirects',
+ 'secure_redirects',
+ 'send_redirects',
+ 'shared_media',
+ 'rp_filter',
+ 'accept_source_route',
+ 'bootp_relay',
+ 'log_martians',
+ 'tag',
+ 'arp_filter',
+ 'medium_id',
+ 'disable_xfrm',
+ 'disable_policy',
+ 'force_igmp_version',
+ 'arp_announce',
+ 'arp_ignore',
+ 'promote_secondaries',
+ 'arp_accept',
+ 'arp_notify',
+ 'accept_local',
+ 'src_valid_mark',
+ 'proxy_arp_pvlan',
+ 'route_localnet')
+ fields = [(i, 'I') for i in field_names]
+
+ class inet6(nla):
+ nla_map = (('IFLA_INET6_UNSPEC', 'none'),
+ ('IFLA_INET6_FLAGS', 'uint32'),
+ ('IFLA_INET6_CONF', 'ipv6_devconf'),
+ ('IFLA_INET6_STATS', 'ipv6_stats'),
+ ('IFLA_INET6_MCAST', 'hex'),
+ ('IFLA_INET6_CACHEINFO', 'ipv6_cache_info'),
+ ('IFLA_INET6_ICMP6STATS', 'icmp6_stats'),
+ ('IFLA_INET6_TOKEN', 'ip6addr'),
+ ('IFLA_INET6_ADDR_GEN_MODE', 'uint8'))
+
+ class ipv6_devconf(nla):
+ # ./include/uapi/linux/ipv6.h
+ # DEVCONF_
+ field_names = ('forwarding',
+ 'hop_limit',
+ 'mtu',
+ 'accept_ra',
+ 'accept_redirects',
+ 'autoconf',
+ 'dad_transmits',
+ 'router_solicitations',
+ 'router_solicitation_interval',
+ 'router_solicitation_delay',
+ 'use_tempaddr',
+ 'temp_valid_lft',
+ 'temp_prefered_lft',
+ 'regen_max_retry',
+ 'max_desync_factor',
+ 'max_addresses',
+ 'force_mld_version',
+ 'accept_ra_defrtr',
+ 'accept_ra_pinfo',
+ 'accept_ra_rtr_pref',
+ 'router_probe_interval',
+ 'accept_ra_rt_info_max_plen',
+ 'proxy_ndp',
+ 'optimistic_dad',
+ 'accept_source_route',
+ 'mc_forwarding',
+ 'disable_ipv6',
+ 'accept_dad',
+ 'force_tllao',
+ 'ndisc_notify')
+ fields = [(i, 'I') for i in field_names]
+
+ class ipv6_cache_info(nla):
+ # ./include/uapi/linux/if_link.h: struct ifla_cacheinfo
+ fields = (('max_reasm_len', 'I'),
+ ('tstamp', 'I'),
+ ('reachable_time', 'I'),
+ ('retrans_time', 'I'))
+
+ class ipv6_stats(nla):
+ field_names = ('inoctets',
+ 'fragcreates',
+ 'indiscards',
+ 'num',
+ 'outoctets',
+ 'outnoroutes',
+ 'inbcastoctets',
+ 'outforwdatagrams',
+ 'outpkts',
+ 'reasmtimeout',
+ 'inhdrerrors',
+ 'reasmreqds',
+ 'fragfails',
+ 'outmcastpkts',
+ 'inaddrerrors',
+ 'inmcastpkts',
+ 'reasmfails',
+ 'outdiscards',
+ 'outbcastoctets',
+ 'inmcastoctets',
+ 'inpkts',
+ 'fragoks',
+ 'intoobigerrors',
+ 'inunknownprotos',
+ 'intruncatedpkts',
+ 'outbcastpkts',
+ 'reasmoks',
+ 'inbcastpkts',
+ 'innoroutes',
+ 'indelivers',
+ 'outmcastoctets')
+ fields = [(i, 'I') for i in field_names]
+
+ class icmp6_stats(nla):
+ fields = (('num', 'Q'),
+ ('inerrors', 'Q'),
+ ('outmsgs', 'Q'),
+ ('outerrors', 'Q'),
+ ('inmsgs', 'Q'))
+
+
+class ifinfmsg(ifinfbase, nlmsg):
+ pass
+
+
+class ifinfveth(ifinfbase, nla):
+ pass
+
+
+def compat_fix_attrs(msg):
+ kind = None
+ ifname = msg.get_attr('IFLA_IFNAME')
+
+ # fix master
+ if ANCIENT:
+ master = compat_get_master(ifname)
+ if master is not None:
+ msg['attrs'].append(['IFLA_MASTER', master])
+
+ # fix linkinfo & kind
+ li = msg.get_attr('IFLA_LINKINFO')
+ if li is not None:
+ kind = li.get_attr('IFLA_INFO_KIND')
+ slave_kind = li.get_attr('IFLA_INFO_SLAVE_KIND')
+ if kind is None:
+ kind = get_interface_type(ifname)
+ li['attrs'].append(['IFLA_INFO_KIND', kind])
+ else:
+ kind = get_interface_type(ifname)
+ slave_kind = None
+ msg['attrs'].append(['IFLA_LINKINFO',
+ {'attrs': [['IFLA_INFO_KIND', kind]]}])
+
+ li = msg.get_attr('IFLA_LINKINFO')
+ # fetch specific interface data
+ if slave_kind == 'openvswitch':
+ # fix master for the OVS slave
+ proc = subprocess.Popen(['ovs-vsctl', 'iface-to-br', ifname],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ ret = proc.communicate()
+ if ret[1]:
+ logging.warning("ovs communication error: %s" % ret[1])
+ commands = [['IFLA_OVS_MASTER_IFNAME', ret[0].strip()]]
+ li['attrs'].append(['IFLA_INFO_DATA', {'attrs': commands}])
+
+ if (kind in ('bridge', 'bond')) and \
+ [x for x in li['attrs'] if x[0] == 'IFLA_INFO_DATA']:
+ if kind == 'bridge':
+ t = '/sys/class/net/%s/bridge/%s'
+ ifdata = ifinfmsg.ifinfo.bridge_data
+ elif kind == 'bond':
+ t = '/sys/class/net/%s/bonding/%s'
+ ifdata = ifinfmsg.ifinfo.bond_data
+
+ commands = []
+ for cmd, _ in ifdata.nla_map:
+ try:
+ with open(t % (ifname, ifdata.nla2name(cmd)), 'r') as f:
+ value = f.read()
+ if cmd == 'IFLA_BOND_MODE':
+ value = value.split()[1]
+ commands.append([cmd, int(value)])
+ except:
+ pass
+ if commands:
+ li['attrs'].append(['IFLA_INFO_DATA', {'attrs': commands}])
+
+
+def proxy_linkinfo(data, nl):
+ offset = 0
+ inbox = []
+ while offset < len(data):
+ msg = ifinfmsg(data[offset:])
+ msg.decode()
+ inbox.append(msg)
+ offset += msg['header']['length']
+
+ data = b''
+ for msg in inbox:
+ # Sysfs operations can require root permissions,
+ # but the script can be run under a normal user
+ # Bug-Url: https://github.com/svinota/pyroute2/issues/113
+ try:
+ compat_fix_attrs(msg)
+ except OSError:
+ # We can safely ignore here any OSError.
+ # In the worst case, we just return what we have got
+ # from the kernel via netlink
+ pass
+
+ msg.reset()
+ msg.encode()
+ data += msg.buf.getvalue()
+
+ return {'verdict': 'forward',
+ 'data': data}
+
+
+def proxy_setlink(data, nl):
+
+ def get_interface(index):
+ msg = nl.get_links(index)[0]
+ try:
+ ovs_master = msg.\
+ get_attr('IFLA_LINKINFO').\
+ get_attr('IFLA_INFO_DATA').\
+ get_attr('IFLA_OVS_MASTER_IFNAME')
+ except Exception:
+ ovs_master = None
+ return {'ifname': msg.get_attr('IFLA_IFNAME'),
+ 'master': msg.get_attr('IFLA_MASTER'),
+ 'ovs-master': ovs_master,
+ 'kind': msg.
+ get_attr('IFLA_LINKINFO').
+ get_attr('IFLA_INFO_KIND')}
+
+ msg = ifinfmsg(data)
+ msg.decode()
+ forward = True
+
+ kind = None
+ infodata = None
+
+ ifname = msg.get_attr('IFLA_IFNAME') or \
+ get_interface(msg['index'])['ifname']
+ linkinfo = msg.get_attr('IFLA_LINKINFO')
+ if linkinfo:
+ kind = linkinfo.get_attr('IFLA_INFO_KIND')
+ infodata = linkinfo.get_attr('IFLA_INFO_DATA')
+
+ if kind in ('bond', 'bridge'):
+ code = 0
+ #
+ if kind == 'bond':
+ func = compat_set_bond
+ elif kind == 'bridge':
+ func = compat_set_bridge
+ #
+ for (cmd, value) in infodata.get('attrs', []):
+ cmd = infodata.nla2name(cmd)
+ code = func(ifname, cmd, value) or code
+ #
+ if code:
+ err = OSError()
+ err.errno = code
+ raise err
+
+ # is it a port setup?
+ master = msg.get_attr('IFLA_MASTER')
+ if master is not None:
+
+ if master == 0:
+ # port delete
+ # 1. get the current master
+ iface = get_interface(msg['index'])
+ if iface['ovs-master'] is not None:
+ master = {'ifname': iface['ovs-master'],
+ 'kind': 'openvswitch'}
+ else:
+ master = get_interface(iface['master'])
+ cmd = 'del'
+ else:
+ # port add
+ # 1. get the master
+ master = get_interface(master)
+ cmd = 'add'
+
+ # 2. manage the port
+ forward_map = {'team': manage_team_port,
+ 'bridge': compat_bridge_port,
+ 'bond': compat_bond_port,
+ 'openvswitch': manage_ovs_port}
+ forward = forward_map[master['kind']](cmd, master['ifname'], ifname)
+
+ if forward is not None:
+ return {'verdict': 'forward',
+ 'data': data}
+
+
+def proxy_dellink(data, nl):
+ orig_msg = ifinfmsg(data)
+ orig_msg.decode()
+
+ # get full interface description
+ msg = nl.get_links(orig_msg['index'])[0]
+ msg['header']['type'] = orig_msg['header']['type']
+
+ # get the interface kind
+ kind = None
+ li = msg.get_attr('IFLA_LINKINFO')
+ if li is not None:
+ kind = li.get_attr('IFLA_INFO_KIND')
+
+ if kind in ('ovs-bridge', 'openvswitch'):
+ return manage_ovs(msg)
+
+ if ANCIENT and kind in ('bridge', 'bond'):
+ # route the request
+ if kind == 'bridge':
+ compat_del_bridge(msg.get_attr('IFLA_IFNAME'))
+ elif kind == 'bond':
+ compat_del_bond(msg.get_attr('IFLA_IFNAME'))
+ # while RTM_NEWLINK is not intercepted -- sleep
+ time.sleep(_ANCIENT_BARRIER)
+ return
+
+ return {'verdict': 'forward',
+ 'data': data}
+
+
+def proxy_newlink(data, nl):
+ msg = ifinfmsg(data)
+ msg.decode()
+ kind = None
+
+ # get the interface kind
+ linkinfo = msg.get_attr('IFLA_LINKINFO')
+ if linkinfo is not None:
+ kind = [x[1] for x in linkinfo['attrs']
+ if x[0] == 'IFLA_INFO_KIND']
+ if kind:
+ kind = kind[0]
+
+ if kind == 'tuntap':
+ return manage_tuntap(msg)
+ elif kind == 'team':
+ return manage_team(msg)
+ elif kind in ('ovs-bridge', 'openvswitch'):
+ return manage_ovs(msg)
+
+ if ANCIENT and kind in ('bridge', 'bond'):
+ # route the request
+ if kind == 'bridge':
+ compat_create_bridge(msg.get_attr('IFLA_IFNAME'))
+ elif kind == 'bond':
+ compat_create_bond(msg.get_attr('IFLA_IFNAME'))
+ # while RTM_NEWLINK is not intercepted -- sleep
+ time.sleep(_ANCIENT_BARRIER)
+ return
+
+ return {'verdict': 'forward',
+ 'data': data}
+
+
+def manage_team(msg):
+
+ assert msg['header']['type'] == RTM_NEWLINK
+
+ config = {'device': msg.get_attr('IFLA_IFNAME'),
+ 'runner': {'name': 'activebackup'},
+ 'link_watch': {'name': 'ethtool'}}
+
+ with open(os.devnull, 'w') as fnull:
+ subprocess.check_call(['teamd', '-d', '-n', '-c', json.dumps(config)],
+ stdout=fnull,
+ stderr=fnull)
+
+
+def manage_team_port(cmd, master, ifname):
+ with open(os.devnull, 'w') as fnull:
+ subprocess.check_call(['teamdctl', master, 'port',
+ 'remove' if cmd == 'del' else 'add', ifname],
+ stdout=fnull,
+ stderr=fnull)
+
+
+def manage_ovs_port(cmd, master, ifname):
+ with open(os.devnull, 'w') as fnull:
+ subprocess.check_call(['ovs-vsctl', '%s-port' % cmd, master, ifname],
+ stdout=fnull,
+ stderr=fnull)
+
+
+def manage_ovs(msg):
+ linkinfo = msg.get_attr('IFLA_LINKINFO')
+ ifname = msg.get_attr('IFLA_IFNAME')
+ kind = linkinfo.get_attr('IFLA_INFO_KIND')
+
+ # operations map
+ op_map = {RTM_NEWLINK: {'ovs-bridge': 'add-br',
+ 'openvswitch': 'add-br'},
+ RTM_DELLINK: {'ovs-bridge': 'del-br',
+ 'openvswitch': 'del-br'}}
+ op = op_map[msg['header']['type']][kind]
+
+ # make a call
+ with open(os.devnull, 'w') as fnull:
+ subprocess.check_call(['ovs-vsctl', op, ifname],
+ stdout=fnull,
+ stderr=fnull)
+
+
+def manage_tuntap(msg):
+
+ if TUNSETIFF is None:
+ raise Exception('unsupported arch')
+
+ if msg['header']['type'] != RTM_NEWLINK:
+ raise Exception('unsupported event')
+
+ ifru_flags = 0
+ linkinfo = msg.get_attr('IFLA_LINKINFO')
+ infodata = linkinfo.get_attr('IFLA_INFO_DATA')
+
+ flags = infodata.get_attr('IFTUN_IFR', None)
+ if infodata.get_attr('IFTUN_MODE') == 'tun':
+ ifru_flags |= IFT_TUN
+ elif infodata.get_attr('IFTUN_MODE') == 'tap':
+ ifru_flags |= IFT_TAP
+ else:
+ raise ValueError('invalid mode')
+ if flags is not None:
+ if flags['no_pi']:
+ ifru_flags |= IFT_NO_PI
+ if flags['one_queue']:
+ ifru_flags |= IFT_ONE_QUEUE
+ if flags['vnet_hdr']:
+ ifru_flags |= IFT_VNET_HDR
+ if flags['multi_queue']:
+ ifru_flags |= IFT_MULTI_QUEUE
+ ifr = msg.get_attr('IFLA_IFNAME')
+ if len(ifr) > IFNAMSIZ:
+ raise ValueError('ifname too long')
+ ifr += (IFNAMSIZ - len(ifr)) * '\0'
+ ifr = ifr.encode('ascii')
+ ifr += struct.pack('H', ifru_flags)
+
+ user = infodata.get_attr('IFTUN_UID')
+ group = infodata.get_attr('IFTUN_GID')
+ #
+ fd = os.open(TUNDEV, os.O_RDWR)
+ try:
+ ioctl(fd, TUNSETIFF, ifr)
+ if user is not None:
+ ioctl(fd, TUNSETOWNER, user)
+ if group is not None:
+ ioctl(fd, TUNSETGROUP, group)
+ ioctl(fd, TUNSETPERSIST, 1)
+ except Exception:
+ raise
+ finally:
+ os.close(fd)
+
+
+def compat_create_bridge(name):
+ with open(os.devnull, 'w') as fnull:
+ subprocess.check_call(['brctl', 'addbr', name],
+ stdout=fnull,
+ stderr=fnull)
+
+
+def compat_create_bond(name):
+ with open(_BONDING_MASTERS, 'w') as f:
+ f.write('+%s' % (name))
+
+
+def compat_set_bond(name, cmd, value):
+ # FIXME: join with bridge
+ # FIXME: use internal IO, not bash
+ t = 'echo %s >/sys/class/net/%s/bonding/%s'
+ with open(os.devnull, 'w') as fnull:
+ return subprocess.call(['bash', '-c', t % (value, name, cmd)],
+ stdout=fnull,
+ stderr=fnull)
+
+
+def compat_set_bridge(name, cmd, value):
+ t = 'echo %s >/sys/class/net/%s/bridge/%s'
+ with open(os.devnull, 'w') as fnull:
+ return subprocess.call(['bash', '-c', t % (value, name, cmd)],
+ stdout=fnull,
+ stderr=fnull)
+
+
+def compat_del_bridge(name):
+ with open(os.devnull, 'w') as fnull:
+ subprocess.check_call(['ip', 'link', 'set',
+ 'dev', name, 'down'])
+ subprocess.check_call(['brctl', 'delbr', name],
+ stdout=fnull,
+ stderr=fnull)
+
+
+def compat_del_bond(name):
+ subprocess.check_call(['ip', 'link', 'set',
+ 'dev', name, 'down'])
+ with open(_BONDING_MASTERS, 'w') as f:
+ f.write('-%s' % (name))
+
+
+def compat_bridge_port(cmd, master, port):
+ if not ANCIENT:
+ return True
+ with open(os.devnull, 'w') as fnull:
+ subprocess.check_call(['brctl', '%sif' % (cmd), master, port],
+ stdout=fnull,
+ stderr=fnull)
+
+
+def compat_bond_port(cmd, master, port):
+ if not ANCIENT:
+ return True
+ remap = {'add': '+',
+ 'del': '-'}
+ cmd = remap[cmd]
+ with open(_BONDING_SLAVES % (master), 'w') as f:
+ f.write('%s%s' % (cmd, port))
+
+
+def compat_get_master(name):
+ f = None
+
+ for i in (_BRIDGE_MASTER, _BONDING_MASTER):
+ try:
+ f = open(i % (name))
+ break
+ except IOError:
+ pass
+
+ if f is not None:
+ master = int(f.read())
+ f.close()
+ return master
+
+
+def get_interface_type(name):
+ '''
+ Utility function to get interface type.
+
+ Unfortunately, we can not rely on RTNL or even ioctl().
+ RHEL doesn't support interface type in RTNL and doesn't
+ provide extended (private) interface flags via ioctl().
+
+ Args:
+ * name (str): interface name
+
+ Returns:
+ * False -- sysfs info unavailable
+ * None -- type not known
+ * str -- interface type:
+ - 'bond'
+ - 'bridge'
+ '''
+ # FIXME: support all interface types? Right now it is
+ # not needed
+ try:
+ ifattrs = os.listdir('/sys/class/net/%s/' % (name))
+ except OSError as e:
+ if e.errno == 2:
+ return 'unknown'
+ else:
+ raise
+
+ if 'bonding' in ifattrs:
+ return 'bond'
+ elif 'bridge' in ifattrs:
+ return 'bridge'
+ else:
+ return 'unknown'
diff --git a/node-admin/scripts/pyroute2/netlink/rtnl/iprsocket.py b/node-admin/scripts/pyroute2/netlink/rtnl/iprsocket.py
new file mode 100644
index 00000000000..5f03649ab8b
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/rtnl/iprsocket.py
@@ -0,0 +1,164 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+
+from pyroute2.proxy import NetlinkProxy
+from pyroute2.common import ANCIENT
+from pyroute2.netlink import NETLINK_ROUTE
+from pyroute2.netlink.nlsocket import Marshal
+from pyroute2.netlink.nlsocket import NetlinkSocket
+from pyroute2.netlink import rtnl
+from pyroute2.netlink.rtnl.tcmsg import tcmsg
+from pyroute2.netlink.rtnl.rtmsg import rtmsg
+from pyroute2.netlink.rtnl.ndmsg import ndmsg
+from pyroute2.netlink.rtnl.fibmsg import fibmsg
+from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg
+from pyroute2.netlink.rtnl.ifinfmsg import proxy_newlink
+from pyroute2.netlink.rtnl.ifinfmsg import proxy_setlink
+from pyroute2.netlink.rtnl.ifinfmsg import proxy_dellink
+from pyroute2.netlink.rtnl.ifinfmsg import proxy_linkinfo
+from pyroute2.netlink.rtnl.ifaddrmsg import ifaddrmsg
+
+
+class MarshalRtnl(Marshal):
+ msg_map = {rtnl.RTM_NEWLINK: ifinfmsg,
+ rtnl.RTM_DELLINK: ifinfmsg,
+ rtnl.RTM_GETLINK: ifinfmsg,
+ rtnl.RTM_SETLINK: ifinfmsg,
+ rtnl.RTM_NEWADDR: ifaddrmsg,
+ rtnl.RTM_DELADDR: ifaddrmsg,
+ rtnl.RTM_GETADDR: ifaddrmsg,
+ rtnl.RTM_NEWROUTE: rtmsg,
+ rtnl.RTM_DELROUTE: rtmsg,
+ rtnl.RTM_GETROUTE: rtmsg,
+ rtnl.RTM_NEWRULE: fibmsg,
+ rtnl.RTM_DELRULE: fibmsg,
+ rtnl.RTM_GETRULE: fibmsg,
+ rtnl.RTM_NEWNEIGH: ndmsg,
+ rtnl.RTM_DELNEIGH: ndmsg,
+ rtnl.RTM_GETNEIGH: ndmsg,
+ rtnl.RTM_NEWQDISC: tcmsg,
+ rtnl.RTM_DELQDISC: tcmsg,
+ rtnl.RTM_GETQDISC: tcmsg,
+ rtnl.RTM_NEWTCLASS: tcmsg,
+ rtnl.RTM_DELTCLASS: tcmsg,
+ rtnl.RTM_GETTCLASS: tcmsg,
+ rtnl.RTM_NEWTFILTER: tcmsg,
+ rtnl.RTM_DELTFILTER: tcmsg,
+ rtnl.RTM_GETTFILTER: tcmsg}
+
+ def fix_message(self, msg):
+ # FIXME: pls do something with it
+ try:
+ msg['event'] = rtnl.RTM_VALUES[msg['header']['type']]
+ except:
+ pass
+
+
+class IPRSocketMixin(object):
+
+ def __init__(self, fileno=None):
+ super(IPRSocketMixin, self).__init__(NETLINK_ROUTE, fileno=fileno)
+ self.marshal = MarshalRtnl()
+ self.ancient = ANCIENT
+ self._s_channel = None
+ self._sproxy = NetlinkProxy(policy='return', nl=self)
+ self._sproxy.pmap = {rtnl.RTM_NEWLINK: proxy_newlink,
+ rtnl.RTM_SETLINK: proxy_setlink,
+ rtnl.RTM_DELLINK: proxy_dellink}
+ self._rproxy = NetlinkProxy(policy='forward', nl=self)
+ self._rproxy.pmap = {rtnl.RTM_NEWLINK: proxy_linkinfo}
+
+ def bind(self, groups=rtnl.RTNL_GROUPS, async=False):
+ super(IPRSocketMixin, self).bind(groups, async=async)
+
+ ##
+ # proxy-ng protocol
+ #
+ def sendto(self, data, address):
+ ret = self._sproxy.handle(data)
+ if ret is not None:
+ if ret['verdict'] == 'forward':
+ return self._sendto(ret['data'], address)
+ elif ret['verdict'] in ('return', 'error'):
+ if self._s_channel is not None:
+ return self._s_channel.send(ret['data'])
+ else:
+ msgs = self.marshal.parse(ret['data'])
+ for msg in msgs:
+ seq = msg['header']['sequence_number']
+ if seq in self.backlog:
+ self.backlog[seq].append(msg)
+ else:
+ self.backlog[seq] = [msg]
+ return len(ret['data'])
+ else:
+ ValueError('Incorrect verdict')
+
+ return self._sendto(data, address)
+
+ def recv(self, bufsize, flags=0):
+ data = self._recv(bufsize, flags)
+ ret = self._rproxy.handle(data)
+ if ret is not None:
+ if ret['verdict'] in ('forward', 'error'):
+ return ret['data']
+ else:
+ ValueError('Incorrect verdict')
+
+ return data
+
+
+class IPRSocket(IPRSocketMixin, NetlinkSocket):
+ '''
+ The simplest class, that connects together the netlink parser and
+ a generic Python socket implementation. Provides method get() to
+ receive the next message from netlink socket and parse it. It is
+ just simple socket-like class, it implements no buffering or
+ like that. It spawns no additional threads, leaving this up to
+ developers.
+
+ Please note, that netlink is an asynchronous protocol with
+ non-guaranteed delivery. You should be fast enough to get all the
+ messages in time. If the message flow rate is higher than the
+ speed you parse them with, exceeding messages will be dropped.
+
+ *Usage*
+
+ Threadless RT netlink monitoring with blocking I/O calls:
+
+ >>> from pyroute2 import IPRSocket
+ >>> from pprint import pprint
+ >>> s = IPRSocket()
+ >>> s.bind()
+ >>> pprint(s.get())
+ [{'attrs': [('RTA_TABLE', 254),
+ ('RTA_DST', '2a00:1450:4009:808::1002'),
+ ('RTA_GATEWAY', 'fe80:52:0:2282::1fe'),
+ ('RTA_OIF', 2),
+ ('RTA_PRIORITY', 0),
+ ('RTA_CACHEINFO', {'rta_clntref': 0,
+ 'rta_error': 0,
+ 'rta_expires': 0,
+ 'rta_id': 0,
+ 'rta_lastuse': 5926,
+ 'rta_ts': 0,
+ 'rta_tsage': 0,
+ 'rta_used': 1})],
+ 'dst_len': 128,
+ 'event': 'RTM_DELROUTE',
+ 'family': 10,
+ 'flags': 512,
+ 'header': {'error': None,
+ 'flags': 0,
+ 'length': 128,
+ 'pid': 0,
+ 'sequence_number': 0,
+ 'type': 25},
+ 'proto': 9,
+ 'scope': 0,
+ 'src_len': 0,
+ 'table': 254,
+ 'tos': 0,
+ 'type': 1}]
+ >>>
+ '''
+ pass
diff --git a/node-admin/scripts/pyroute2/netlink/rtnl/iw_event.py b/node-admin/scripts/pyroute2/netlink/rtnl/iw_event.py
new file mode 100644
index 00000000000..5a4e4ae5375
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/rtnl/iw_event.py
@@ -0,0 +1,85 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+from pyroute2.netlink import nla
+
+
+class iw_event(nla):
+
+ nla_map = ((0x8B00, 'SIOCSIWCOMMIT', 'hex'),
+ (0x8B01, 'SIOCGIWNAME', 'hex'),
+ # Basic operations
+ (0x8B02, 'SIOCSIWNWID', 'hex'),
+ (0x8B03, 'SIOCGIWNWID', 'hex'),
+ (0x8B04, 'SIOCSIWFREQ', 'hex'),
+ (0x8B05, 'SIOCGIWFREQ', 'hex'),
+ (0x8B06, 'SIOCSIWMODE', 'hex'),
+ (0x8B07, 'SIOCGIWMODE', 'hex'),
+ (0x8B08, 'SIOCSIWSENS', 'hex'),
+ (0x8B09, 'SIOCGIWSENS', 'hex'),
+ # Informative stuff
+ (0x8B0A, 'SIOCSIWRANGE', 'hex'),
+ (0x8B0B, 'SIOCGIWRANGE', 'hex'),
+ (0x8B0C, 'SIOCSIWPRIV', 'hex'),
+ (0x8B0D, 'SIOCGIWPRIV', 'hex'),
+ (0x8B0E, 'SIOCSIWSTATS', 'hex'),
+ (0x8B0F, 'SIOCGIWSTATS', 'hex'),
+ # Spy support (statistics per MAC address -
+ # used for Mobile IP support)
+ (0x8B10, 'SIOCSIWSPY', 'hex'),
+ (0x8B11, 'SIOCGIWSPY', 'hex'),
+ (0x8B12, 'SIOCSIWTHRSPY', 'hex'),
+ (0x8B13, 'SIOCGIWTHRSPY', 'hex'),
+ # Access Point manipulation
+ (0x8B14, 'SIOCSIWAP', 'hex'),
+ (0x8B15, 'SIOCGIWAP', 'hex'),
+ (0x8B17, 'SIOCGIWAPLIST', 'hex'),
+ (0x8B18, 'SIOCSIWSCAN', 'hex'),
+ (0x8B19, 'SIOCGIWSCAN', 'hex'),
+ # 802.11 specific support
+ (0x8B1A, 'SIOCSIWESSID', 'hex'),
+ (0x8B1B, 'SIOCGIWESSID', 'hex'),
+ (0x8B1C, 'SIOCSIWNICKN', 'hex'),
+ (0x8B1D, 'SIOCGIWNICKN', 'hex'),
+ # Other parameters useful in 802.11 and
+ # some other devices
+ (0x8B20, 'SIOCSIWRATE', 'hex'),
+ (0x8B21, 'SIOCGIWRATE', 'hex'),
+ (0x8B22, 'SIOCSIWRTS', 'hex'),
+ (0x8B23, 'SIOCGIWRTS', 'hex'),
+ (0x8B24, 'SIOCSIWFRAG', 'hex'),
+ (0x8B25, 'SIOCGIWFRAG', 'hex'),
+ (0x8B26, 'SIOCSIWTXPOW', 'hex'),
+ (0x8B27, 'SIOCGIWTXPOW', 'hex'),
+ (0x8B28, 'SIOCSIWRETRY', 'hex'),
+ (0x8B29, 'SIOCGIWRETRY', 'hex'),
+ # Encoding stuff (scrambling, hardware security, WEP...)
+ (0x8B2A, 'SIOCSIWENCODE', 'hex'),
+ (0x8B2B, 'SIOCGIWENCODE', 'hex'),
+ # Power saving stuff (power management, unicast
+ # and multicast)
+ (0x8B2C, 'SIOCSIWPOWER', 'hex'),
+ (0x8B2D, 'SIOCGIWPOWER', 'hex'),
+ # WPA : Generic IEEE 802.11 informatiom element
+ # (e.g., for WPA/RSN/WMM).
+ (0x8B30, 'SIOCSIWGENIE', 'hex'),
+ (0x8B31, 'SIOCGIWGENIE', 'hex'),
+ # WPA : IEEE 802.11 MLME requests
+ (0x8B16, 'SIOCSIWMLME', 'hex'),
+ # WPA : Authentication mode parameters
+ (0x8B32, 'SIOCSIWAUTH', 'hex'),
+ (0x8B33, 'SIOCGIWAUTH', 'hex'),
+ # WPA : Extended version of encoding configuration
+ (0x8B34, 'SIOCSIWENCODEEXT', 'hex'),
+ (0x8B35, 'SIOCGIWENCODEEXT', 'hex'),
+ # WPA2 : PMKSA cache management
+ (0x8B36, 'SIOCSIWPMKSA', 'hex'),
+ # Events s.str.
+ (0x8C00, 'IWEVTXDROP', 'hex'),
+ (0x8C01, 'IWEVQUAL', 'hex'),
+ (0x8C02, 'IWEVCUSTOM', 'hex'),
+ (0x8C03, 'IWEVREGISTERED', 'hex'),
+ (0x8C04, 'IWEVEXPIRED', 'hex'),
+ (0x8C05, 'IWEVGENIE', 'hex'),
+ (0x8C06, 'IWEVMICHAELMICFAILURE', 'hex'),
+ (0x8C07, 'IWEVASSOCREQIE', 'hex'),
+ (0x8C08, 'IWEVASSOCRESPIE', 'hex'),
+ (0x8C09, 'IWEVPMKIDCAND', 'hex'))
diff --git a/node-admin/scripts/pyroute2/netlink/rtnl/ndmsg.py b/node-admin/scripts/pyroute2/netlink/rtnl/ndmsg.py
new file mode 100644
index 00000000000..f7dcf836453
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/rtnl/ndmsg.py
@@ -0,0 +1,61 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+
+from pyroute2.netlink import nlmsg
+from pyroute2.netlink import nla
+
+
+class ndmsg(nlmsg):
+ '''
+ ARP cache update message
+
+ C structure::
+
+ struct ndmsg {
+ unsigned char ndm_family;
+ int ndm_ifindex; /* Interface index */
+ __u16 ndm_state; /* State */
+ __u8 ndm_flags; /* Flags */
+ __u8 ndm_type;
+ };
+
+ Cache info structure::
+
+ struct nda_cacheinfo {
+ __u32 ndm_confirmed;
+ __u32 ndm_used;
+ __u32 ndm_updated;
+ __u32 ndm_refcnt;
+ };
+ '''
+ fields = (('family', 'B'),
+ ('__pad', '3x'),
+ ('ifindex', 'i'),
+ ('state', 'H'),
+ ('flags', 'B'),
+ ('ndm_type', 'B'))
+
+ # Please note, that nla_map creates implicit
+ # enumeration. In this case it will be:
+ #
+ # NDA_UNSPEC = 0
+ # NDA_DST = 1
+ # NDA_LLADDR = 2
+ # NDA_CACHEINFO = 3
+ # NDA_PROBES = 4
+ # ...
+ #
+ nla_map = (('NDA_UNSPEC', 'none'),
+ ('NDA_DST', 'ipaddr'),
+ ('NDA_LLADDR', 'l2addr'),
+ ('NDA_CACHEINFO', 'cacheinfo'),
+ ('NDA_PROBES', 'uint32'),
+ ('NDA_VLAN', 'uint16'),
+ ('NDA_PORT', 'be16'),
+ ('NDA_VNI', 'be32'),
+ ('NDA_IFINDEX', 'uint32'))
+
+ class cacheinfo(nla):
+ fields = (('ndm_confirmed', 'I'),
+ ('ndm_used', 'I'),
+ ('ndm_updated', 'I'),
+ ('ndm_refcnt', 'I'))
diff --git a/node-admin/scripts/pyroute2/netlink/rtnl/req.py b/node-admin/scripts/pyroute2/netlink/rtnl/req.py
new file mode 100644
index 00000000000..268fd7ff604
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/rtnl/req.py
@@ -0,0 +1,182 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+from socket import AF_INET6
+from pyroute2.common import basestring
+from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg
+from pyroute2.netlink.rtnl.rtmsg import rtmsg
+
+
+class IPRequest(dict):
+
+ def __init__(self, obj=None):
+ dict.__init__(self)
+ if obj is not None:
+ self.update(obj)
+
+ def update(self, obj):
+ for key in obj:
+ if obj[key] is not None:
+ self[key] = obj[key]
+
+
+class IPRouteRequest(IPRequest):
+ '''
+ Utility class, that converts human-readable dictionary
+ into RTNL route request.
+ '''
+
+ def __setitem__(self, key, value):
+ # fix family
+ if isinstance(value, basestring) and value.find(':') >= 0:
+ self['family'] = AF_INET6
+ # work on the rest
+ if key == 'dst':
+ if value != 'default':
+ value = value.split('/')
+ if len(value) == 1:
+ dst = value[0]
+ mask = 0
+ elif len(value) == 2:
+ dst = value[0]
+ mask = int(value[1])
+ else:
+ raise ValueError('wrong destination')
+ dict.__setitem__(self, 'dst', dst)
+ dict.__setitem__(self, 'dst_len', mask)
+ elif key == 'metrics':
+ ret = {'attrs': []}
+ for name in value:
+ rtax = rtmsg.metrics.name2nla(name)
+ ret['attrs'].append([rtax, value[name]])
+ dict.__setitem__(self, 'metrics', ret)
+ else:
+ dict.__setitem__(self, key, value)
+
+
+class CBRequest(IPRequest):
+ '''
+ FIXME
+ '''
+ commands = None
+ msg = None
+
+ def __init__(self, *argv, **kwarg):
+ self['commands'] = {'attrs': []}
+
+ def __setitem__(self, key, value):
+ if value is None:
+ return
+ if key in self.commands:
+ self['commands']['attrs'].\
+ append([self.msg.name2nla(key), value])
+ else:
+ dict.__setitem__(self, key, value)
+
+
+class IPLinkRequest(IPRequest):
+ '''
+ Utility class, that converts human-readable dictionary
+ into RTNL link request.
+ '''
+ blacklist = ['carrier',
+ 'carrier_changes']
+
+ def __init__(self, *argv, **kwarg):
+ self.deferred = []
+ IPRequest.__init__(self, *argv, **kwarg)
+ if 'index' not in self:
+ self['index'] = 0
+
+ def __setitem__(self, key, value):
+ # ignore blacklisted attributes
+ if key in self.blacklist:
+ return
+
+ # there must be no "None" values in the request
+ if value is None:
+ return
+
+ # all the values must be in ascii
+ try:
+ if isinstance(value, unicode):
+ value = value.encode('ascii')
+ except NameError:
+ pass
+
+ # set up specific keys
+ if key == 'kind':
+ self['IFLA_LINKINFO'] = {'attrs': []}
+ linkinfo = self['IFLA_LINKINFO']['attrs']
+ linkinfo.append(['IFLA_INFO_KIND', value])
+ if value in ('vlan', 'bond', 'tuntap', 'veth',
+ 'vxlan', 'macvlan', 'macvtap', 'gre'):
+ linkinfo.append(['IFLA_INFO_DATA', {'attrs': []}])
+ elif key == 'vlan_id':
+ nla = ['IFLA_VLAN_ID', value]
+ # FIXME: we need to replace, not add
+ self.defer_nla(nla, ('IFLA_LINKINFO', 'IFLA_INFO_DATA'),
+ lambda x: x.get('kind', None) == 'vlan')
+ elif key == 'gid':
+ nla = ['IFTUN_UID', value]
+ self.defer_nla(nla, ('IFLA_LINKINFO', 'IFLA_INFO_DATA'),
+ lambda x: x.get('kind', None) == 'tuntap')
+ elif key == 'uid':
+ nla = ['IFTUN_UID', value]
+ self.defer_nla(nla, ('IFLA_LINKINFO', 'IFLA_INFO_DATA'),
+ lambda x: x.get('kind', None) == 'tuntap')
+ elif key == 'mode':
+ nla = ['IFTUN_MODE', value]
+ self.defer_nla(nla, ('IFLA_LINKINFO', 'IFLA_INFO_DATA'),
+ lambda x: x.get('kind', None) == 'tuntap')
+ nla = ['IFLA_BOND_MODE', value]
+ self.defer_nla(nla, ('IFLA_LINKINFO', 'IFLA_INFO_DATA'),
+ lambda x: x.get('kind', None) == 'bond')
+ elif key == 'ifr':
+ nla = ['IFTUN_IFR', value]
+ self.defer_nla(nla, ('IFLA_LINKINFO', 'IFLA_INFO_DATA'),
+ lambda x: x.get('kind', None) == 'tuntap')
+ elif key.startswith('macvtap'):
+ nla = [ifinfmsg.name2nla(key), value]
+ self.defer_nla(nla, ('IFLA_LINKINFO', 'IFLA_INFO_DATA'),
+ lambda x: x.get('kind', None) == 'macvtap')
+ elif key.startswith('macvlan'):
+ nla = [ifinfmsg.name2nla(key), value]
+ self.defer_nla(nla, ('IFLA_LINKINFO', 'IFLA_INFO_DATA'),
+ lambda x: x.get('kind', None) == 'macvlan')
+ elif key.startswith('gre'):
+ nla = [ifinfmsg.name2nla(key), value]
+ self.defer_nla(nla, ('IFLA_LINKINFO', 'IFLA_INFO_DATA'),
+ lambda x: x.get('kind', None) == 'gre')
+ elif key.startswith('vxlan'):
+ nla = [ifinfmsg.name2nla(key), value]
+ self.defer_nla(nla, ('IFLA_LINKINFO', 'IFLA_INFO_DATA'),
+ lambda x: x.get('kind', None) == 'vxlan')
+ elif key == 'peer':
+ nla = ['VETH_INFO_PEER', {'attrs': [['IFLA_IFNAME', value]]}]
+ self.defer_nla(nla, ('IFLA_LINKINFO', 'IFLA_INFO_DATA'),
+ lambda x: x.get('kind', None) == 'veth')
+ dict.__setitem__(self, key, value)
+ if self.deferred:
+ self.flush_deferred()
+
+ def flush_deferred(self):
+ deferred = []
+ for nla, path, predicate in self.deferred:
+ if predicate(self):
+ self.append_nla(nla, path)
+ else:
+ deferred.append((nla, path, predicate))
+ self.deferred = deferred
+
+ def append_nla(self, nla, path):
+ pwd = self
+ for step in path:
+ if step in pwd:
+ pwd = pwd[step]
+ else:
+ pwd = [x[1] for x in pwd['attrs']
+ if x[0] == step][0]['attrs']
+ pwd.append(nla)
+
+ def defer_nla(self, nla, path, predicate):
+ self.deferred.append((nla, path, predicate))
+ self.flush_deferred()
diff --git a/node-admin/scripts/pyroute2/netlink/rtnl/rtmsg.py b/node-admin/scripts/pyroute2/netlink/rtnl/rtmsg.py
new file mode 100644
index 00000000000..4983f0a405c
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/rtnl/rtmsg.py
@@ -0,0 +1,90 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+
+from pyroute2.netlink import nlmsg
+from pyroute2.netlink import nla
+
+
+class rtmsg(nlmsg):
+ '''
+ Route message
+
+ C structure::
+
+ struct rtmsg {
+ unsigned char rtm_family; /* Address family of route */
+ unsigned char rtm_dst_len; /* Length of destination */
+ unsigned char rtm_src_len; /* Length of source */
+ unsigned char rtm_tos; /* TOS filter */
+
+ unsigned char rtm_table; /* Routing table ID */
+ unsigned char rtm_protocol; /* Routing protocol; see below */
+ unsigned char rtm_scope; /* See below */
+ unsigned char rtm_type; /* See below */
+
+ unsigned int rtm_flags;
+ };
+ '''
+ prefix = 'RTA_'
+
+ fields = (('family', 'B'),
+ ('dst_len', 'B'),
+ ('src_len', 'B'),
+ ('tos', 'B'),
+ ('table', 'B'),
+ ('proto', 'B'),
+ ('scope', 'B'),
+ ('type', 'B'),
+ ('flags', 'I'))
+
+ nla_map = (('RTA_UNSPEC', 'none'),
+ ('RTA_DST', 'ipaddr'),
+ ('RTA_SRC', 'ipaddr'),
+ ('RTA_IIF', 'uint32'),
+ ('RTA_OIF', 'uint32'),
+ ('RTA_GATEWAY', 'ipaddr'),
+ ('RTA_PRIORITY', 'uint32'),
+ ('RTA_PREFSRC', 'ipaddr'),
+ ('RTA_METRICS', 'metrics'),
+ ('RTA_MULTIPATH', 'hex'),
+ ('RTA_PROTOINFO', 'uint32'),
+ ('RTA_FLOW', 'hex'),
+ ('RTA_CACHEINFO', 'cacheinfo'),
+ ('RTA_SESSION', 'hex'),
+ ('RTA_MP_ALGO', 'hex'),
+ ('RTA_TABLE', 'uint32'),
+ ('RTA_MARK', 'uint32'),
+ ('RTA_MFC_STATS', 'rta_mfc_stats'))
+
+ class rta_mfc_stats(nla):
+ fields = (('mfcs_packets', 'uint64'),
+ ('mfcs_bytes', 'uint64'),
+ ('mfcs_wrong_if', 'uint64'))
+
+ class metrics(nla):
+ prefix = 'RTAX_'
+ nla_map = (('RTAX_UNSPEC', 'none'),
+ ('RTAX_LOCK', 'uint32'),
+ ('RTAX_MTU', 'uint32'),
+ ('RTAX_WINDOW', 'uint32'),
+ ('RTAX_RTT', 'uint32'),
+ ('RTAX_RTTVAR', 'uint32'),
+ ('RTAX_SSTHRESH', 'uint32'),
+ ('RTAX_CWND', 'uint32'),
+ ('RTAX_ADVMSS', 'uint32'),
+ ('RTAX_REORDERING', 'uint32'),
+ ('RTAX_HOPLIMIT', 'uint32'),
+ ('RTAX_INITCWND', 'uint32'),
+ ('RTAX_FEATURES', 'uint32'),
+ ('RTAX_RTO_MIN', 'uint32'),
+ ('RTAX_INITRWND', 'uint32'),
+ ('RTAX_QUICKACK', 'uint32'))
+
+ class cacheinfo(nla):
+ fields = (('rta_clntref', 'I'),
+ ('rta_lastuse', 'I'),
+ ('rta_expires', 'i'),
+ ('rta_error', 'I'),
+ ('rta_used', 'I'),
+ ('rta_id', 'I'),
+ ('rta_ts', 'I'),
+ ('rta_tsage', 'I'))
diff --git a/node-admin/scripts/pyroute2/netlink/rtnl/tcmsg.py b/node-admin/scripts/pyroute2/netlink/rtnl/tcmsg.py
new file mode 100644
index 00000000000..32b6a762b5b
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/rtnl/tcmsg.py
@@ -0,0 +1,917 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+import re
+import os
+import struct
+
+from pyroute2.common import size_suffixes
+from pyroute2.common import time_suffixes
+from pyroute2.common import rate_suffixes
+from pyroute2.common import basestring
+from pyroute2.netlink import nlmsg
+from pyroute2.netlink import nla
+
+
+TCA_ACT_MAX_PRIO = 32
+
+LINKLAYER_UNSPEC = 0
+LINKLAYER_ETHERNET = 1
+LINKLAYER_ATM = 2
+
+ATM_CELL_SIZE = 53
+ATM_CELL_PAYLOAD = 48
+
+TC_RED_ECN = 1
+TC_RED_HARDDROP = 2
+TC_RED_ADAPTATIVE = 4
+
+TIME_UNITS_PER_SEC = 1000000
+
+_psched = open('/proc/net/psched', 'r')
+[_t2us,
+ _us2t,
+ _clock_res,
+ _wee] = [int(i, 16) for i in _psched.read().split()]
+_clock_factor = float(_clock_res) / TIME_UNITS_PER_SEC
+_tick_in_usec = float(_t2us) / _us2t * _clock_factor
+_first_letter = re.compile('[^0-9]+')
+_psched.close()
+
+
+def _get_hz():
+ if _clock_res == 1000000:
+ return _wee
+ else:
+ return os.environ.get('HZ', 1000)
+
+
+def _get_by_suffix(value, default, func):
+ if not isinstance(value, basestring):
+ return value
+ pos = _first_letter.search(value)
+ if pos is None:
+ suffix = default
+ else:
+ pos = pos.start()
+ value, suffix = value[:pos], value[pos:]
+ value = int(value)
+ return func(value, suffix)
+
+
+def _get_size(size):
+ return _get_by_suffix(size, 'b',
+ lambda x, y: x * size_suffixes[y])
+
+
+def _get_time(lat):
+ return _get_by_suffix(lat, 'ms',
+ lambda x, y: (x * TIME_UNITS_PER_SEC) /
+ time_suffixes[y])
+
+
+def _get_rate(rate):
+ return _get_by_suffix(rate, 'bit',
+ lambda x, y: (x * rate_suffixes[y]) / 8)
+
+
+def _time2tick(time):
+ # The current code is ported from tc utility
+ return int(time) * _tick_in_usec
+
+
+def _calc_xmittime(rate, size):
+ # The current code is ported from tc utility
+ return int(_time2tick(TIME_UNITS_PER_SEC * (float(size) / rate)))
+
+
+def _percent2u32(pct):
+ '''xlate a percentage to an uint32 value
+ 0% -> 0
+ 100% -> 2**32 - 1'''
+ return int((2**32 - 1)*pct/100)
+
+
+def _red_eval_ewma(qmin, burst, avpkt):
+ # The current code is ported from tc utility
+ wlog = 1
+ W = 0.5
+ a = float(burst) + 1 - float(qmin) / avpkt
+ assert a < 1
+
+ while wlog < 32:
+ wlog += 1
+ W /= 2
+ if (a <= (1 - pow(1 - W, burst)) / W):
+ return wlog
+ return -1
+
+
+def _red_eval_P(qmin, qmax, probability):
+ # The current code is ported from tc utility
+ i = qmax - qmin
+ assert i > 0
+ assert i < 32
+
+ probability /= i
+ while i < 32:
+ i += 1
+ if probability > 1:
+ break
+ probability *= 2
+ return i
+
+
+def _get_rate_parameters(kwarg):
+ # rate and burst are required
+ rate = _get_rate(kwarg['rate'])
+ burst = kwarg['burst']
+
+ # if peak, mtu is required
+ peak = _get_rate(kwarg.get('peak', 0))
+ mtu = kwarg.get('mtu', 0)
+ if peak:
+ assert mtu
+
+ # limit OR latency is required
+ limit = kwarg.get('limit', None)
+ latency = _get_time(kwarg.get('latency', None))
+ assert limit is not None or latency is not None
+
+ # calculate limit from latency
+ if limit is None:
+ rate_limit = rate * float(latency) /\
+ TIME_UNITS_PER_SEC + burst
+ if peak:
+ peak_limit = peak * float(latency) /\
+ TIME_UNITS_PER_SEC + mtu
+ if rate_limit > peak_limit:
+ rate_limit = peak_limit
+ limit = rate_limit
+
+ return {'rate': int(rate),
+ 'mtu': mtu,
+ 'buffer': _calc_xmittime(rate, burst),
+ 'limit': int(limit)}
+
+
+def get_tbf_parameters(kwarg):
+ parms = _get_rate_parameters(kwarg)
+ # fill parameters
+ return {'attrs': [['TCA_TBF_PARMS', parms],
+ ['TCA_TBF_RTAB', True]]}
+
+
+def _get_filter_police_parameter(kwarg):
+ # if no limit specified, set it to zero to make
+ # the next call happy
+ kwarg['limit'] = kwarg.get('limit', 0)
+ tbfp = _get_rate_parameters(kwarg)
+ # create an alias -- while TBF uses 'buffer', rate
+ # policy uses 'burst'
+ tbfp['burst'] = tbfp['buffer']
+ # action resolver
+ actions = nla_plus_police.police.police_tbf.actions
+ tbfp['action'] = actions[kwarg.get('action', 'reclassify')]
+ police = [['TCA_POLICE_TBF', tbfp],
+ ['TCA_POLICE_RATE', True]]
+ return police
+
+
+def get_u32_parameters(kwarg):
+ ret = {'attrs': []}
+
+ if kwarg.get('rate'):
+ ret['attrs'].append([
+ 'TCA_U32_POLICE',
+ {'attrs': _get_filter_police_parameter(kwarg)}
+ ])
+
+ ret['attrs'].append(['TCA_U32_CLASSID', kwarg['target']])
+ ret['attrs'].append(['TCA_U32_SEL', {'keys': kwarg['keys']}])
+
+ return ret
+
+
+def get_fw_parameters(kwarg):
+ ret = {'attrs': []}
+ attrs_map = (
+ ('classid', 'TCA_FW_CLASSID'),
+ ('act', 'TCA_FW_ACT'),
+ # ('police', 'TCA_FW_POLICE'),
+ # Handled in _get_filter_police_parameter
+ ('indev', 'TCA_FW_INDEV'),
+ ('mask', 'TCA_FW_MASK'),
+ )
+
+ if kwarg.get('rate'):
+ ret['attrs'].append([
+ 'TCA_FW_POLICE',
+ {'attrs': _get_filter_police_parameter(kwarg)}
+ ])
+
+ for k, v in attrs_map:
+ r = kwarg.get(k, None)
+ if r is not None:
+ ret['attrs'].append([v, r])
+
+ return ret
+
+
+def get_sfq_parameters(kwarg):
+ kwarg['quantum'] = _get_size(kwarg.get('quantum', 0))
+ kwarg['perturb_period'] = kwarg.get('perturb', 0) or \
+ kwarg.get('perturb_period', 0)
+ limit = kwarg['limit'] = kwarg.get('limit', 0) or \
+ kwarg.get('redflowlimit', 0)
+ qth_min = kwarg.get('min', 0)
+ qth_max = kwarg.get('max', 0)
+ avpkt = kwarg.get('avpkt', 1000)
+ probability = kwarg.get('probability', 0.02)
+ ecn = kwarg.get('ecn', False)
+ harddrop = kwarg.get('harddrop', False)
+ kwarg['flags'] = kwarg.get('flags', 0)
+ if ecn:
+ kwarg['flags'] |= TC_RED_ECN
+ if harddrop:
+ kwarg['flags'] |= TC_RED_HARDDROP
+ if kwarg.get('redflowlimit'):
+ qth_max = qth_max or limit / 4
+ qth_min = qth_min or qth_max / 3
+ kwarg['burst'] = kwarg['burst'] or \
+ (2 * qth_min + qth_max) / (3 * avpkt)
+ assert limit > qth_max
+ assert qth_max > qth_min
+ kwarg['qth_min'] = qth_min
+ kwarg['qth_max'] = qth_max
+ kwarg['Wlog'] = _red_eval_ewma(qth_min, kwarg['burst'], avpkt)
+ kwarg['Plog'] = _red_eval_P(qth_min, qth_max, probability)
+ assert kwarg['Wlog'] >= 0
+ assert kwarg['Plog'] >= 0
+ kwarg['max_P'] = int(probability * pow(2, 23))
+
+ return kwarg
+
+
+def get_htb_class_parameters(kwarg):
+ #
+ prio = kwarg.get('prio', 0)
+ mtu = kwarg.get('mtu', 1600)
+ mpu = kwarg.get('mpu', 0)
+ overhead = kwarg.get('overhead', 0)
+ quantum = kwarg.get('quantum', 0)
+ #
+ rate = _get_rate(kwarg.get('rate', None))
+ ceil = _get_rate(kwarg.get('ceil', 0)) or rate
+
+ burst = kwarg.get('burst', None) or \
+ kwarg.get('maxburst', None) or \
+ kwarg.get('buffer', None)
+
+ if rate is not None:
+ if burst is None:
+ burst = rate / _get_hz() + mtu
+ burst = _calc_xmittime(rate, burst)
+
+ cburst = kwarg.get('cburst', None) or \
+ kwarg.get('cmaxburst', None) or \
+ kwarg.get('cbuffer', None)
+
+ if ceil is not None:
+ if cburst is None:
+ cburst = ceil / _get_hz() + mtu
+ cburst = _calc_xmittime(ceil, cburst)
+
+ return {'attrs': [['TCA_HTB_PARMS', {'buffer': burst,
+ 'cbuffer': cburst,
+ 'quantum': quantum,
+ 'prio': prio,
+ 'rate': rate,
+ 'ceil': ceil,
+ 'ceil_overhead': overhead,
+ 'rate_overhead': overhead,
+ 'rate_mpu': mpu,
+ 'ceil_mpu': mpu}],
+ ['TCA_HTB_RTAB', True],
+ ['TCA_HTB_CTAB', True]]}
+
+
+def get_htb_parameters(kwarg):
+ rate2quantum = kwarg.get('r2q', 0xa)
+ version = kwarg.get('version', 3)
+ defcls = kwarg.get('default', 0x10)
+
+ return {'attrs': [['TCA_HTB_INIT', {'debug': 0,
+ 'defcls': defcls,
+ 'direct_pkts': 0,
+ 'rate2quantum': rate2quantum,
+ 'version': version}]]}
+
+
+def get_netem_parameters(kwarg):
+ delay = _time2tick(kwarg.get('delay', 0)) # in microsecond
+ limit = kwarg.get('limit', 1000) # fifo limit (packets) see netem.c:230
+ loss = _percent2u32(kwarg.get('loss', 0)) # int percentage
+ gap = kwarg.get('gap', 0)
+ duplicate = kwarg.get('duplicate', 0)
+ jitter = _time2tick(kwarg.get('jitter', 0)) # in microsecond
+
+ opts = {
+ 'delay': delay,
+ 'limit': limit,
+ 'loss': loss,
+ 'gap': gap,
+ 'duplicate': duplicate,
+ 'jitter': jitter,
+ 'attrs': []
+ }
+
+ # correlation (delay, loss, duplicate)
+ delay_corr = _percent2u32(kwarg.get('delay_corr', 0))
+ loss_corr = _percent2u32(kwarg.get('loss_corr', 0))
+ dup_corr = _percent2u32(kwarg.get('dup_corr', 0))
+ if delay_corr or loss_corr or dup_corr:
+ # delay_corr requires that both jitter and delay are != 0
+ if delay_corr and not (delay and jitter):
+ raise Exception('delay correlation requires delay'
+ ' and jitter to be set')
+ # loss correlation and loss
+ if loss_corr and not loss:
+ raise Exception('loss correlation requires loss to be set')
+ # duplicate correlation and duplicate
+ if dup_corr and not duplicate:
+ raise Exception('duplicate correlation requires '
+ 'duplicate to be set')
+
+ opts['attrs'].append(['TCA_NETEM_CORR', {'delay_corr': delay_corr,
+ 'loss_corr': loss_corr,
+ 'dup_corr': dup_corr}])
+
+ # reorder (probability, correlation)
+ prob_reorder = _percent2u32(kwarg.get('prob_reorder', 0))
+ corr_reorder = _percent2u32(kwarg.get('corr_reorder', 0))
+ if prob_reorder != 0:
+ # gap defaults to 1 if equal to 0
+ if gap == 0:
+ opts['gap'] = gap = 1
+ opts['attrs'].append(['TCA_NETEM_REORDER',
+ {'prob_reorder': prob_reorder,
+ 'corr_reorder': corr_reorder}])
+ else:
+ if gap != 0:
+ raise Exception('gap can only be set when prob_reorder is set')
+ elif corr_reorder != 0:
+ raise Exception('corr_reorder can only be set when '
+ 'prob_reorder is set')
+
+ # corrupt (probability, correlation)
+ prob_corrupt = _percent2u32(kwarg.get('prob_corrupt', 0))
+ corr_corrupt = _percent2u32(kwarg.get('corr_corrupt', 0))
+ if prob_corrupt:
+ opts['attrs'].append(['TCA_NETEM_CORRUPT',
+ {'prob_corrupt': prob_corrupt,
+ 'corr_corrupt': corr_corrupt}])
+ elif corr_corrupt != 0:
+ raise Exception('corr_corrupt can only be set when '
+ 'prob_corrupt is set')
+
+ # TODO
+ # delay distribution (dist_size, dist_data)
+ return opts
+
+
+class nla_plus_rtab(nla):
+ class parms(nla):
+ def adjust_size(self, size, mpu, linklayer):
+ # The current code is ported from tc utility
+ if size < mpu:
+ size = mpu
+
+ if linklayer == LINKLAYER_ATM:
+ cells = size / ATM_CELL_PAYLOAD
+ if size % ATM_CELL_PAYLOAD > 0:
+ cells += 1
+ size = cells * ATM_CELL_SIZE
+
+ return size
+
+ def calc_rtab(self, kind):
+ # The current code is ported from tc utility
+ rtab = []
+ mtu = self.get('mtu', 0) or 1600
+ cell_log = self['%s_cell_log' % (kind)]
+ mpu = self['%s_mpu' % (kind)]
+ rate = self.get(kind, 'rate')
+
+ # calculate cell_log
+ if cell_log == 0:
+ while (mtu >> cell_log) > 255:
+ cell_log += 1
+
+ # fill up the table
+ for i in range(256):
+ size = self.adjust_size((i + 1) << cell_log,
+ mpu,
+ LINKLAYER_ETHERNET)
+ rtab.append(_calc_xmittime(rate, size))
+
+ self['%s_cell_align' % (kind)] = -1
+ self['%s_cell_log' % (kind)] = cell_log
+ return rtab
+
+ def encode(self):
+ self.rtab = None
+ self.ptab = None
+ if self.get('rate', False):
+ self.rtab = self.calc_rtab('rate')
+ if self.get('peak', False):
+ self.ptab = self.calc_rtab('peak')
+ if self.get('ceil', False):
+ self.ctab = self.calc_rtab('ceil')
+ nla.encode(self)
+
+ class rtab(nla):
+ fields = (('value', 's'), )
+
+ def encode(self):
+ parms = self.parent.get_encoded('TCA_TBF_PARMS') or \
+ self.parent.get_encoded('TCA_HTB_PARMS') or \
+ self.parent.get_encoded('TCA_POLICE_TBF')
+ if parms is not None:
+ self.value = getattr(parms, self.__class__.__name__)
+ self['value'] = struct.pack('I' * 256,
+ *(int(x) for x in self.value))
+ nla.encode(self)
+
+ def decode(self):
+ nla.decode(self)
+ parms = self.parent.get_attr('TCA_TBF_PARMS') or \
+ self.parent.get_attr('TCA_HTB_PARMS') or \
+ self.parent.get_attr('TCA_POLICE_TBF')
+ if parms is not None:
+ rtab = struct.unpack('I' * (len(self['value']) / 4),
+ self['value'])
+ self.value = rtab
+ setattr(parms, self.__class__.__name__, rtab)
+
+ class ptab(rtab):
+ pass
+
+ class ctab(rtab):
+ pass
+
+
+class nla_plus_stats2(object):
+ class stats2(nla):
+ nla_map = (('TCA_STATS_UNSPEC', 'none'),
+ ('TCA_STATS_BASIC', 'basic'),
+ ('TCA_STATS_RATE_EST', 'rate_est'),
+ ('TCA_STATS_QUEUE', 'queue'),
+ ('TCA_STATS_APP', 'hex'))
+
+ class basic(nla):
+ fields = (('bytes', 'Q'),
+ ('packets', 'Q'))
+
+ class rate_est(nla):
+ fields = (('bps', 'I'),
+ ('pps', 'I'))
+
+ class queue(nla):
+ fields = (('qlen', 'I'),
+ ('backlog', 'I'),
+ ('drops', 'I'),
+ ('requeues', 'I'),
+ ('overlimits', 'I'))
+
+ class stats2_hfsc(stats2):
+ nla_map = (('TCA_STATS_UNSPEC', 'none'),
+ ('TCA_STATS_BASIC', 'basic'),
+ ('TCA_STATS_RATE_EST', 'rate_est'),
+ ('TCA_STATS_QUEUE', 'queue'),
+ ('TCA_STATS_APP', 'stats_app_hfsc'))
+
+ class stats_app_hfsc(nla):
+ fields = (('work', 'Q'), # total work done
+ ('rtwork', 'Q'), # total work done by real-time criteria
+ ('period', 'I'), # current period
+ ('level', 'I')) # class level in hierarchy
+
+
+class nla_plus_police(object):
+ class police(nla_plus_rtab):
+ nla_map = (('TCA_POLICE_UNSPEC', 'none'),
+ ('TCA_POLICE_TBF', 'police_tbf'),
+ ('TCA_POLICE_RATE', 'rtab'),
+ ('TCA_POLICE_PEAKRATE', 'ptab'),
+ ('TCA_POLICE_AVRATE', 'uint32'),
+ ('TCA_POLICE_RESULT', 'uint32'))
+
+ class police_tbf(nla_plus_rtab.parms):
+ fields = (('index', 'I'),
+ ('action', 'i'),
+ ('limit', 'I'),
+ ('burst', 'I'),
+ ('mtu', 'I'),
+ ('rate_cell_log', 'B'),
+ ('rate___reserved', 'B'),
+ ('rate_overhead', 'H'),
+ ('rate_cell_align', 'h'),
+ ('rate_mpu', 'H'),
+ ('rate', 'I'),
+ ('peak_cell_log', 'B'),
+ ('peak___reserved', 'B'),
+ ('peak_overhead', 'H'),
+ ('peak_cell_align', 'h'),
+ ('peak_mpu', 'H'),
+ ('peak', 'I'),
+ ('refcnt', 'i'),
+ ('bindcnt', 'i'),
+ ('capab', 'I'))
+
+ actions = {'unspec': -1, # TC_POLICE_UNSPEC
+ 'ok': 0, # TC_POLICE_OK
+ 'reclassify': 1, # TC_POLICE_RECLASSIFY
+ 'shot': 2, # TC_POLICE_SHOT
+ 'drop': 2, # TC_POLICE_SHOT
+ 'pipe': 3} # TC_POLICE_PIPE
+
+
+class tcmsg(nlmsg, nla_plus_stats2):
+
+ prefix = 'TCA_'
+
+ fields = (('family', 'B'),
+ ('pad1', 'B'),
+ ('pad2', 'H'),
+ ('index', 'i'),
+ ('handle', 'I'),
+ ('parent', 'I'),
+ ('info', 'I'))
+
+ nla_map = (('TCA_UNSPEC', 'none'),
+ ('TCA_KIND', 'asciiz'),
+ ('TCA_OPTIONS', 'get_options'),
+ ('TCA_STATS', 'stats'),
+ ('TCA_XSTATS', 'get_xstats'),
+ ('TCA_RATE', 'hex'),
+ ('TCA_FCNT', 'hex'),
+ ('TCA_STATS2', 'get_stats2'),
+ ('TCA_STAB', 'hex'))
+
+ class stats(nla):
+ fields = (('bytes', 'Q'),
+ ('packets', 'I'),
+ ('drop', 'I'),
+ ('overlimits', 'I'),
+ ('bps', 'I'),
+ ('pps', 'I'),
+ ('qlen', 'I'),
+ ('backlog', 'I'))
+
+ def get_stats2(self, *argv, **kwarg):
+ kind = self.get_attr('TCA_KIND')
+ if kind == 'hfsc':
+ return self.stats2_hfsc
+ return self.stats2
+
+ def get_xstats(self, *argv, **kwarg):
+ kind = self.get_attr('TCA_KIND')
+ if kind == 'htb':
+ return self.xstats_htb
+ return self.hex
+
+ class xstats_htb(nla):
+ fields = (('lends', 'I'),
+ ('borrows', 'I'),
+ ('giants', 'I'),
+ ('tokens', 'I'),
+ ('ctokens', 'I'))
+
+ def get_options(self, *argv, **kwarg):
+ kind = self.get_attr('TCA_KIND')
+ if kind == 'ingress':
+ return self.options_ingress
+ elif kind == 'pfifo_fast':
+ return self.options_pfifo_fast
+ elif kind == 'tbf':
+ return self.options_tbf
+ elif kind == 'sfq':
+ if kwarg.get('length', 0) >= self.options_sfq_v1.get_size():
+ return self.options_sfq_v1
+ else:
+ return self.options_sfq_v0
+ elif kind == 'hfsc':
+ return self.options_hfsc
+ elif kind == 'htb':
+ return self.options_htb
+ elif kind == 'netem':
+ return self.options_netem
+ elif kind == 'u32':
+ return self.options_u32
+ elif kind == 'fw':
+ return self.options_fw
+ return self.hex
+
+ class options_ingress(nla):
+ fields = (('value', 'I'), )
+
+ class options_hfsc(nla):
+ nla_map = (('TCA_HFSC_UNSPEC', 'hfsc_qopt'),
+ ('TCA_HFSC_RSC', 'hfsc_curve'), # real-time curve
+ ('TCA_HFSC_FSC', 'hfsc_curve'), # link-share curve
+ ('TCA_HFSC_USC', 'hfsc_curve')) # upper-limit curve
+
+ class hfsc_qopt(nla):
+ fields = (('defcls', 'H'),) # default class
+
+ class hfsc_curve(nla):
+ fields = (('m1', 'I'), # slope of the first segment in bps
+ ('d', 'I'), # x-projection of the first segment in us
+ ('m2', 'I')) # slope of the second segment in bps
+
+ class options_htb(nla_plus_rtab):
+ nla_map = (('TCA_HTB_UNSPEC', 'none'),
+ ('TCA_HTB_PARMS', 'htb_parms'),
+ ('TCA_HTB_INIT', 'htb_glob'),
+ ('TCA_HTB_CTAB', 'ctab'),
+ ('TCA_HTB_RTAB', 'rtab'))
+
+ class htb_glob(nla):
+ fields = (('version', 'I'),
+ ('rate2quantum', 'I'),
+ ('defcls', 'I'),
+ ('debug', 'I'),
+ ('direct_pkts', 'I'))
+
+ class htb_parms(nla_plus_rtab.parms):
+ fields = (('rate_cell_log', 'B'),
+ ('rate___reserved', 'B'),
+ ('rate_overhead', 'H'),
+ ('rate_cell_align', 'h'),
+ ('rate_mpu', 'H'),
+ ('rate', 'I'),
+ ('ceil_cell_log', 'B'),
+ ('ceil___reserved', 'B'),
+ ('ceil_overhead', 'H'),
+ ('ceil_cell_align', 'h'),
+ ('ceil_mpu', 'H'),
+ ('ceil', 'I'),
+ ('buffer', 'I'),
+ ('cbuffer', 'I'),
+ ('quantum', 'I'),
+ ('level', 'I'),
+ ('prio', 'I'))
+
+ class options_netem(nla):
+ nla_map = (('TCA_NETEM_UNSPEC', 'none'),
+ ('TCA_NETEM_CORR', 'netem_corr'),
+ ('TCA_NETEM_DELAY_DIST', 'none'),
+ ('TCA_NETEM_REORDER', 'netem_reorder'),
+ ('TCA_NETEM_CORRUPT', 'netem_corrupt'),
+ ('TCA_NETEM_LOSS', 'none'),
+ ('TCA_NETEM_RATE', 'none'))
+
+ fields = (('delay', 'I'),
+ ('limit', 'I'),
+ ('loss', 'I'),
+ ('gap', 'I'),
+ ('duplicate', 'I'),
+ ('jitter', 'I'))
+
+ class netem_corr(nla):
+ '''correlation'''
+ fields = (('delay_corr', 'I'),
+ ('loss_corr', 'I'),
+ ('dup_corr', 'I'))
+
+ class netem_reorder(nla):
+ '''reorder has probability and correlation'''
+ fields = (('prob_reorder', 'I'),
+ ('corr_reorder', 'I'))
+
+ class netem_corrupt(nla):
+ '''corruption has probability and correlation'''
+ fields = (('prob_corrupt', 'I'),
+ ('corr_corrupt', 'I'))
+
+ class options_fw(nla, nla_plus_police):
+ nla_map = (('TCA_FW_UNSPEC', 'none'),
+ ('TCA_FW_CLASSID', 'uint32'),
+ ('TCA_FW_POLICE', 'police'), # TODO string?
+ ('TCA_FW_INDEV', 'hex'), # TODO string
+ ('TCA_FW_ACT', 'hex'), # TODO
+ ('TCA_FW_MASK', 'uint32'))
+
+ class options_u32(nla, nla_plus_police):
+ nla_map = (('TCA_U32_UNSPEC', 'none'),
+ ('TCA_U32_CLASSID', 'uint32'),
+ ('TCA_U32_HASH', 'uint32'),
+ ('TCA_U32_LINK', 'hex'),
+ ('TCA_U32_DIVISOR', 'uint32'),
+ ('TCA_U32_SEL', 'u32_sel'),
+ ('TCA_U32_POLICE', 'police'),
+ ('TCA_U32_ACT', 'tca_act_prio'),
+ ('TCA_U32_INDEV', 'hex'),
+ ('TCA_U32_PCNT', 'u32_pcnt'),
+ ('TCA_U32_MARK', 'u32_mark'))
+
+ class tca_act_prio(nla):
+ nla_map = tuple([('TCA_ACT_PRIO_%i' % x, 'tca_act') for x
+ in range(TCA_ACT_MAX_PRIO)])
+
+ class tca_act(nla, nla_plus_police, nla_plus_stats2):
+ nla_map = (('TCA_ACT_UNSPEC', 'none'),
+ ('TCA_ACT_KIND', 'asciiz'),
+ ('TCA_ACT_OPTIONS', 'police'),
+ ('TCA_ACT_INDEX', 'hex'),
+ ('TCA_ACT_STATS', 'stats2'))
+
+ class u32_sel(nla):
+ fields = (('flags', 'B'),
+ ('offshift', 'B'),
+ ('nkeys', 'B'),
+ ('__align', 'B'),
+ ('offmask', '>H'),
+ ('off', 'H'),
+ ('offoff', 'h'),
+ ('hoff', 'h'),
+ ('hmask', '>I'))
+
+ class u32_key(nlmsg):
+ header = None
+ fields = (('key_mask', '>I'),
+ ('key_val', '>I'),
+ ('key_off', 'i'),
+ ('key_offmask', 'i'))
+
+ def encode(self):
+ '''
+ Key sample::
+
+ 'keys': ['0x0006/0x00ff+8',
+ '0x0000/0xffc0+2',
+ '0x5/0xf+0',
+ '0x10/0xff+33']
+
+ => 00060000/00ff0000 + 8
+ 05000000/0f00ffc0 + 0
+ 00100000/00ff0000 + 32
+ '''
+
+ def cut_field(key, separator):
+ '''
+ split a field from the end of the string
+ '''
+ field = '0'
+ pos = key.find(separator)
+ new_key = key
+ if pos > 0:
+ field = key[pos + 1:]
+ new_key = key[:pos]
+ return (new_key, field)
+
+ # 'header' array to pack keys to
+ header = [(0, 0) for i in range(256)]
+
+ keys = []
+ # iterate keys and pack them to the 'header'
+ for key in self['keys']:
+ # TODO tags: filter
+ (key, nh) = cut_field(key, '@') # FIXME: do not ignore nh
+ (key, offset) = cut_field(key, '+')
+ offset = int(offset, 0)
+ # a little trick: if you provide /00ff+8, that
+ # really means /ff+9, so we should take it into
+ # account
+ (key, mask) = cut_field(key, '/')
+ if mask[:2] == '0x':
+ mask = mask[2:]
+ while True:
+ if mask[:2] == '00':
+ offset += 1
+ mask = mask[2:]
+ else:
+ break
+ mask = '0x' + mask
+ mask = int(mask, 0)
+ value = int(key, 0)
+ bits = 24
+ if mask == 0 and value == 0:
+ key = self.u32_key(self.buf)
+ key['key_off'] = offset
+ key['key_mask'] = mask
+ key['key_val'] = value
+ keys.append(key)
+ for bmask in struct.unpack('4B', struct.pack('>I', mask)):
+ if bmask > 0:
+ bvalue = (value & (bmask << bits)) >> bits
+ header[offset] = (bvalue, bmask)
+ offset += 1
+ bits -= 8
+
+ # recalculate keys from 'header'
+ key = None
+ value = 0
+ mask = 0
+ for offset in range(256):
+ (bvalue, bmask) = header[offset]
+ if bmask > 0 and key is None:
+ key = self.u32_key(self.buf)
+ key['key_off'] = offset
+ key['key_mask'] = 0
+ key['key_val'] = 0
+ bits = 24
+ if key is not None and bits >= 0:
+ key['key_mask'] |= bmask << bits
+ key['key_val'] |= bvalue << bits
+ bits -= 8
+ if (bits < 0 or offset == 255):
+ keys.append(key)
+ key = None
+
+ assert keys
+ self['nkeys'] = len(keys)
+ # FIXME: do not hardcode flags :)
+ self['flags'] = 1
+ start = self.buf.tell()
+
+ nla.encode(self)
+ for key in keys:
+ key.encode()
+ self.update_length(start)
+
+ def decode(self):
+ nla.decode(self)
+ self['keys'] = []
+ nkeys = self['nkeys']
+ while nkeys:
+ key = self.u32_key(self.buf)
+ key.decode()
+ self['keys'].append(key)
+ nkeys -= 1
+
+ class u32_mark(nla):
+ fields = (('val', 'I'),
+ ('mask', 'I'),
+ ('success', 'I'))
+
+ class u32_pcnt(nla):
+ fields = (('rcnt', 'Q'),
+ ('rhit', 'Q'),
+ ('kcnts', 'Q'))
+
+ class options_pfifo_fast(nla):
+ fields = (('bands', 'i'),
+ ('priomap', '16B'))
+
+ class options_tbf(nla_plus_rtab):
+ nla_map = (('TCA_TBF_UNSPEC', 'none'),
+ ('TCA_TBF_PARMS', 'tbf_parms'),
+ ('TCA_TBF_RTAB', 'rtab'),
+ ('TCA_TBF_PTAB', 'ptab'))
+
+ class tbf_parms(nla_plus_rtab.parms):
+ fields = (('rate_cell_log', 'B'),
+ ('rate___reserved', 'B'),
+ ('rate_overhead', 'H'),
+ ('rate_cell_align', 'h'),
+ ('rate_mpu', 'H'),
+ ('rate', 'I'),
+ ('peak_cell_log', 'B'),
+ ('peak___reserved', 'B'),
+ ('peak_overhead', 'H'),
+ ('peak_cell_align', 'h'),
+ ('peak_mpu', 'H'),
+ ('peak', 'I'),
+ ('limit', 'I'),
+ ('buffer', 'I'),
+ ('mtu', 'I'))
+
+ class options_sfq_v0(nla):
+ fields = (('quantum', 'I'),
+ ('perturb_period', 'i'),
+ ('limit', 'I'),
+ ('divisor', 'I'),
+ ('flows', 'I'))
+
+ class options_sfq_v1(nla):
+ fields = (('quantum', 'I'),
+ ('perturb_period', 'i'),
+ ('limit_v0', 'I'),
+ ('divisor', 'I'),
+ ('flows', 'I'),
+ ('depth', 'I'),
+ ('headdrop', 'I'),
+ ('limit_v1', 'I'),
+ ('qth_min', 'I'),
+ ('qth_max', 'I'),
+ ('Wlog', 'B'),
+ ('Plog', 'B'),
+ ('Scell_log', 'B'),
+ ('flags', 'B'),
+ ('max_P', 'I'),
+ ('prob_drop', 'I'),
+ ('forced_drop', 'I'),
+ ('prob_mark', 'I'),
+ ('forced_mark', 'I'),
+ ('prob_mark_head', 'I'),
+ ('forced_mark_head', 'I'))
diff --git a/node-admin/scripts/pyroute2/netlink/taskstats/__init__.py b/node-admin/scripts/pyroute2/netlink/taskstats/__init__.py
new file mode 100644
index 00000000000..86a8d20464c
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netlink/taskstats/__init__.py
@@ -0,0 +1,167 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+TaskStats module
+================
+
+All that you should know about TaskStats, is that you should not
+use it. But if you have to, ok::
+
+ import os
+ from pyroute2 import TaskStats
+ ts = TaskStats()
+ ts.get_pid_stat(os.getpid())
+
+It is not implemented normally yet, but some methods are already
+usable.
+'''
+
+from pyroute2.netlink import NLM_F_REQUEST
+from pyroute2.netlink import nla
+from pyroute2.netlink import genlmsg
+from pyroute2.netlink.generic import GenericNetlinkSocket
+
+TASKSTATS_CMD_UNSPEC = 0 # Reserved
+TASKSTATS_CMD_GET = 1 # user->kernel request/get-response
+TASKSTATS_CMD_NEW = 2
+
+
+class tcmd(genlmsg):
+ nla_map = (('TASKSTATS_CMD_ATTR_UNSPEC', 'none'),
+ ('TASKSTATS_CMD_ATTR_PID', 'uint32'),
+ ('TASKSTATS_CMD_ATTR_TGID', 'uint32'),
+ ('TASKSTATS_CMD_ATTR_REGISTER_CPUMASK', 'asciiz'),
+ ('TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK', 'asciiz'))
+
+
+class tstats(nla):
+ pack = "struct"
+ fields = (('version', 'H'), # 2
+ ('ac_exitcode', 'I'), # 4
+ ('ac_flag', 'B'), # 1
+ ('ac_nice', 'B'), # 1 --- 10
+ ('cpu_count', 'Q'), # 8
+ ('cpu_delay_total', 'Q'), # 8
+ ('blkio_count', 'Q'), # 8
+ ('blkio_delay_total', 'Q'), # 8
+ ('swapin_count', 'Q'), # 8
+ ('swapin_delay_total', 'Q'), # 8
+ ('cpu_run_real_total', 'Q'), # 8
+ ('cpu_run_virtual_total', 'Q'), # 8
+ ('ac_comm', '32s'), # 32 +++ 112
+ ('ac_sched', 'B'), # 1
+ ('__pad', '3x'), # 1 --- 8 (!)
+ ('ac_uid', 'I'), # 4 +++ 120
+ ('ac_gid', 'I'), # 4
+ ('ac_pid', 'I'), # 4
+ ('ac_ppid', 'I'), # 4
+ ('ac_btime', 'I'), # 4 +++ 136
+ ('ac_etime', 'Q'), # 8 +++ 144
+ ('ac_utime', 'Q'), # 8
+ ('ac_stime', 'Q'), # 8
+ ('ac_minflt', 'Q'), # 8
+ ('ac_majflt', 'Q'), # 8
+ ('coremem', 'Q'), # 8
+ ('virtmem', 'Q'), # 8
+ ('hiwater_rss', 'Q'), # 8
+ ('hiwater_vm', 'Q'), # 8
+ ('read_char', 'Q'), # 8
+ ('write_char', 'Q'), # 8
+ ('read_syscalls', 'Q'), # 8
+ ('write_syscalls', 'Q'), # 8
+ ('read_bytes', 'Q'), # ...
+ ('write_bytes', 'Q'),
+ ('cancelled_write_bytes', 'Q'),
+ ('nvcsw', 'Q'),
+ ('nivcsw', 'Q'),
+ ('ac_utimescaled', 'Q'),
+ ('ac_stimescaled', 'Q'),
+ ('cpu_scaled_run_real_total', 'Q'))
+
+ def decode(self):
+ nla.decode(self)
+ self['ac_comm'] = self['ac_comm'][:self['ac_comm'].find('\0')]
+
+
+class taskstatsmsg(genlmsg):
+
+ nla_map = (('TASKSTATS_TYPE_UNSPEC', 'none'),
+ ('TASKSTATS_TYPE_PID', 'uint32'),
+ ('TASKSTATS_TYPE_TGID', 'uint32'),
+ ('TASKSTATS_TYPE_STATS', 'stats'),
+ ('TASKSTATS_TYPE_AGGR_PID', 'aggr_pid'),
+ ('TASKSTATS_TYPE_AGGR_TGID', 'aggr_tgid'))
+
+ class stats(tstats):
+ pass # FIXME: optimize me!
+
+ class aggr_id(nla):
+ nla_map = (('TASKSTATS_TYPE_UNSPEC', 'none'),
+ ('TASKSTATS_TYPE_PID', 'uint32'),
+ ('TASKSTATS_TYPE_TGID', 'uint32'),
+ ('TASKSTATS_TYPE_STATS', 'stats'))
+
+ class stats(tstats):
+ pass
+
+ class aggr_pid(aggr_id):
+ pass
+
+ class aggr_tgid(aggr_id):
+ pass
+
+
+class TaskStats(GenericNetlinkSocket):
+
+ def __init__(self):
+ GenericNetlinkSocket.__init__(self)
+
+ def bind(self):
+ GenericNetlinkSocket.bind(self, 'TASKSTATS', taskstatsmsg)
+
+ def get_pid_stat(self, pid):
+ '''
+ Get taskstats for a process. Pid should be an integer.
+ '''
+ msg = tcmd()
+ msg['cmd'] = TASKSTATS_CMD_GET
+ msg['version'] = 1
+ msg['attrs'].append(['TASKSTATS_CMD_ATTR_PID', pid])
+ return self.nlm_request(msg,
+ self.prid,
+ msg_flags=NLM_F_REQUEST)
+
+ def _register_mask(self, cmd, mask):
+ msg = tcmd()
+ msg['cmd'] = TASKSTATS_CMD_GET
+ msg['version'] = 1
+ msg['attrs'].append([cmd, mask])
+ # there is no response to this request
+ self.put(msg,
+ self.prid,
+ msg_flags=NLM_F_REQUEST)
+
+ def register_mask(self, mask):
+ '''
+ Start the accounting for a processors by a mask. Mask is
+ a string, e.g.::
+ 0,1 -- first two CPUs
+ 0-4,6-10 -- CPUs from 0 to 4 and from 6 to 10
+
+ When the accounting is turned on, on can receive messages
+ with get() routine.
+
+ Though the kernel has a procedure, that cleans up accounting,
+ when it is not used, it is recommended to run deregister_mask()
+ before process exit.
+ '''
+ self.monitor(True)
+ self._register_mask('TASKSTATS_CMD_ATTR_REGISTER_CPUMASK',
+ mask)
+
+ def deregister_mask(self, mask):
+ '''
+ Stop the accounting.
+ '''
+ self.monitor(False)
+ self._register_mask('TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK',
+ mask)
diff --git a/node-admin/scripts/pyroute2/netns/__init__.py b/node-admin/scripts/pyroute2/netns/__init__.py
new file mode 100644
index 00000000000..696ff3a14a6
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netns/__init__.py
@@ -0,0 +1,123 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+Network namespaces management
+=============================
+
+Pyroute2 provides basic namespaces management support. The
+`netns` module contains several tools for that.
+
+Please be aware, that in order to run system calls the
+library uses `ctypes` module. It can fail on platforms
+where SELinux is enforced. If the Python interpreter,
+loading this module, dumps the core, one can check the
+SELinux state with `getenforce` command.
+
+'''
+
+import os
+import errno
+import ctypes
+import sys
+
+if sys.maxsize > 2**32:
+ __NR_setns = 308
+else:
+ __NR_setns = 346
+
+CLONE_NEWNET = 0x40000000
+MNT_DETACH = 0x00000002
+MS_BIND = 4096
+MS_REC = 16384
+MS_SHARED = 1 << 20
+NETNS_RUN_DIR = '/var/run/netns'
+
+
+def listnetns():
+ '''
+ List available network namespaces.
+ '''
+ try:
+ return os.listdir(NETNS_RUN_DIR)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ return []
+ else:
+ raise
+
+
+def create(netns, libc=None):
+ '''
+ Create a network namespace.
+ '''
+ libc = libc or ctypes.CDLL('libc.so.6', use_errno=True)
+ # FIXME validate and prepare NETNS_RUN_DIR
+
+ netnspath = '%s/%s' % (NETNS_RUN_DIR, netns)
+ netnspath = netnspath.encode('ascii')
+ netnsdir = NETNS_RUN_DIR.encode('ascii')
+
+ # init netnsdir
+ try:
+ os.mkdir(netnsdir)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+ # this code is ported from iproute2
+ done = False
+ while libc.mount(b'', netnsdir, b'none', MS_SHARED | MS_REC, None) != 0:
+ if done:
+ raise OSError(ctypes.get_errno(), 'share rundir failed', netns)
+ if libc.mount(netnsdir, netnsdir, b'none', MS_BIND, None) != 0:
+ raise OSError(ctypes.get_errno(), 'mount rundir failed', netns)
+ done = True
+
+ # create mountpoint
+ os.close(os.open(netnspath, os.O_RDONLY | os.O_CREAT | os.O_EXCL, 0))
+
+ # unshare
+ if libc.unshare(CLONE_NEWNET) < 0:
+ raise OSError(ctypes.get_errno(), 'unshare failed', netns)
+
+ # bind the namespace
+ if libc.mount(b'/proc/self/ns/net', netnspath, b'none', MS_BIND, None) < 0:
+ raise OSError(ctypes.get_errno(), 'mount failed', netns)
+
+
+def remove(netns, libc=None):
+ '''
+ Remove a network namespace.
+ '''
+ libc = libc or ctypes.CDLL('libc.so.6', use_errno=True)
+ netnspath = '%s/%s' % (NETNS_RUN_DIR, netns)
+ netnspath = netnspath.encode('ascii')
+ libc.umount2(netnspath, MNT_DETACH)
+ os.unlink(netnspath)
+
+
+def setns(netns, flags=os.O_CREAT, libc=None):
+ '''
+ Set netns for the current process.
+
+ The flags semantics is the same as for the `open(2)`
+ call:
+
+ - O_CREAT -- create netns, if doesn't exist
+ - O_CREAT | O_EXCL -- create only if doesn't exist
+ '''
+ libc = libc or ctypes.CDLL('libc.so.6', use_errno=True)
+ netnspath = '%s/%s' % (NETNS_RUN_DIR, netns)
+ netnspath = netnspath.encode('ascii')
+
+ if netns in listnetns():
+ if flags & (os.O_CREAT | os.O_EXCL) == (os.O_CREAT | os.O_EXCL):
+ raise OSError(errno.EEXIST, 'netns exists', netns)
+ else:
+ if flags & os.O_CREAT:
+ create(netns, libc=libc)
+
+ nsfd = os.open(netnspath, os.O_RDONLY)
+ ret = libc.syscall(__NR_setns, nsfd, CLONE_NEWNET)
+ if ret != 0:
+ raise OSError(ctypes.get_errno(), 'failed to open netns', netns)
+ return nsfd
diff --git a/node-admin/scripts/pyroute2/netns/nslink.py b/node-admin/scripts/pyroute2/netns/nslink.py
new file mode 100644
index 00000000000..67d5fff0921
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netns/nslink.py
@@ -0,0 +1,310 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+NetNS
+=====
+
+A NetNS object is IPRoute-like. It runs in the main network
+namespace, but also creates a proxy process running in
+the required netns. All the netlink requests are done via
+that proxy process.
+
+NetNS supports standard IPRoute API, so can be used instead
+of IPRoute, e.g., in IPDB::
+
+ # start the main network settings database:
+ ipdb_main = IPDB()
+ # start the same for a netns:
+ ipdb_test = IPDB(nl=NetNS('test'))
+
+ # create VETH
+ ipdb_main.create(ifname='v0p0', kind='veth', peer='v0p1').commit()
+
+ # move peer VETH into the netns
+ with ipdb_main.interfaces.v0p1 as veth:
+ veth.net_ns_fd = 'test'
+
+ # please keep in mind, that netns move clears all the settings
+ # on a VETH interface pair, so one should run netns assignment
+ # as a separate operation only
+
+ # assign addresses
+ # please notice, that `v0p1` is already in the `test` netns,
+ # so should be accessed via `ipdb_test`
+ with ipdb_main.interfaces.v0p0 as veth:
+ veth.add_ip('172.16.200.1/24')
+ veth.up()
+ with ipdb_test.interfaces.v0p1 as veth:
+ veth.add_ip('172.16.200.2/24')
+ veth.up()
+
+Please review also the test code, under `tests/test_netns.py` for
+more examples.
+
+By default, NetNS creates requested netns, if it doesn't exist,
+or uses existing one. To control this behaviour, one can use flags
+as for `open(2)` system call::
+
+ # create a new netns or fail, if it already exists
+ netns = NetNS('test', flags=os.O_CREAT | os.O_EXIST)
+
+ # create a new netns or use existing one
+ netns = NetNS('test', flags=os.O_CREAT)
+
+ # the same as above, the default behaviour
+ netns = NetNS('test')
+
+To remove a network namespace::
+
+ from pyroute2 import NetNS
+ netns = NetNS('test')
+ netns.close()
+ netns.remove()
+
+One should stop it first with `close()`, and only after that
+run `remove()`.
+
+'''
+
+import os
+import errno
+import atexit
+import select
+import struct
+import threading
+import traceback
+from socket import SOL_SOCKET
+from socket import SO_RCVBUF
+from pyroute2.config import MpPipe
+from pyroute2.config import MpProcess
+from pyroute2.iproute import IPRoute
+from pyroute2.netlink.nlsocket import NetlinkMixin
+from pyroute2.netlink.rtnl.iprsocket import MarshalRtnl
+from pyroute2.iproute import IPRouteMixin
+from pyroute2.netns import setns
+from pyroute2.netns import remove
+
+
+def NetNServer(netns, rcvch, cmdch, flags=os.O_CREAT):
+ '''
+ The netns server supposed to be started automatically by NetNS.
+ It has two communication channels: one simplex to forward incoming
+ netlink packets, `rcvch`, and other synchronous duplex to get
+ commands and send back responses, `cmdch`.
+
+ Channels should support standard socket API, should be compatible
+ with poll/select and should be able to transparently pickle objects.
+ NetNS uses `multiprocessing.Pipe` for this purpose, but it can be
+ any other implementation with compatible API.
+
+ The first parameter, `netns`, is a netns name. Depending on the
+ `flags`, the netns can be created automatically. The `flags` semantics
+ is exactly the same as for `open(2)` system call.
+
+ ...
+
+ The server workflow is simple. The startup sequence::
+
+ 1. Create or open a netns.
+
+ 2. Start `IPRoute` instance. It will be used only on the low level,
+ the `IPRoute` will not parse any packet.
+
+ 3. Start poll/select loop on `cmdch` and `IPRoute`.
+
+ On the startup, the server sends via `cmdch` the status packet. It can be
+ `None` if all is OK, or some exception.
+
+ Further data handling, depending on the channel, server side::
+
+ 1. `IPRoute`: read an incoming netlink packet and send it unmodified
+ to the peer via `rcvch`. The peer, polling `rcvch`, can handle
+ the packet on its side.
+
+ 2. `cmdch`: read tuple (cmd, argv, kwarg). If the `cmd` starts with
+ "send", then take `argv[0]` as a packet buffer, treat it as one
+ netlink packet and substitute PID field (offset 12, uint32) with
+ its own. Strictly speaking, it is not mandatory for modern netlink
+ implementations, but it is required by the protocol standard.
+
+ '''
+ try:
+ nsfd = setns(netns, flags)
+ except OSError as e:
+ cmdch.send(e)
+ return e.errno
+ except Exception as e:
+ cmdch.send(OSError(errno.ECOMM, str(e), netns))
+ return 255
+
+ #
+ try:
+ ipr = IPRoute()
+ rcvch_lock = ipr._sproxy.lock
+ ipr._s_channel = rcvch
+ poll = select.poll()
+ poll.register(ipr, select.POLLIN | select.POLLPRI)
+ poll.register(cmdch, select.POLLIN | select.POLLPRI)
+ except Exception as e:
+ cmdch.send(e)
+ return 255
+
+ # all is OK so far
+ cmdch.send(None)
+ # 8<-------------------------------------------------------------
+ while True:
+ events = poll.poll()
+ for (fd, event) in events:
+ if fd == ipr.fileno():
+ bufsize = ipr.getsockopt(SOL_SOCKET, SO_RCVBUF) // 2
+ with rcvch_lock:
+ rcvch.send(ipr.recv(bufsize))
+ elif fd == cmdch.fileno():
+ try:
+ cmdline = cmdch.recv()
+ if cmdline is None:
+ poll.unregister(ipr)
+ poll.unregister(cmdch)
+ ipr.close()
+ os.close(nsfd)
+ return
+ (cmd, argv, kwarg) = cmdline
+ if cmd[:4] == 'send':
+ # Achtung
+ #
+ # It's a hack, but we just have to do it: one
+ # must use actual pid in netlink messages
+ #
+ # FIXME: there can be several messages in one
+ # call buffer; but right now we can ignore it
+ msg = argv[0][:12]
+ msg += struct.pack("I", os.getpid())
+ msg += argv[0][16:]
+ argv = list(argv)
+ argv[0] = msg
+ cmdch.send(getattr(ipr, cmd)(*argv, **kwarg))
+ except Exception as e:
+ e.tb = traceback.format_exc()
+ cmdch.send(e)
+
+
+class NetNSProxy(object):
+
+ netns = 'default'
+ flags = os.O_CREAT
+
+ def __init__(self, *argv, **kwarg):
+ self.cmdlock = threading.Lock()
+ self.rcvch, rcvch = MpPipe()
+ self.cmdch, cmdch = MpPipe()
+ self.server = MpProcess(target=NetNServer,
+ args=(self.netns, rcvch, cmdch, self.flags))
+ self.server.start()
+ error = self.cmdch.recv()
+ if error is not None:
+ self.server.join()
+ raise error
+ else:
+ atexit.register(self.close)
+
+ def recv(self, bufsize, flags=0):
+ return self.rcvch.recv()
+
+ def close(self):
+ self.cmdch.send(None)
+ self.server.join()
+
+ def proxy(self, cmd, *argv, **kwarg):
+ with self.cmdlock:
+ self.cmdch.send((cmd, argv, kwarg))
+ response = self.cmdch.recv()
+ if isinstance(response, Exception):
+ raise response
+ return response
+
+ def fileno(self):
+ return self.rcvch.fileno()
+
+ def bind(self, *argv, **kwarg):
+ if 'async' in kwarg:
+ kwarg['async'] = False
+ return self.proxy('bind', *argv, **kwarg)
+
+ def send(self, *argv, **kwarg):
+ return self.proxy('send', *argv, **kwarg)
+
+ def sendto(self, *argv, **kwarg):
+ return self.proxy('sendto', *argv, **kwarg)
+
+ def getsockopt(self, *argv, **kwarg):
+ return self.proxy('getsockopt', *argv, **kwarg)
+
+ def setsockopt(self, *argv, **kwarg):
+ return self.proxy('setsockopt', *argv, **kwarg)
+
+
+class NetNSocket(NetlinkMixin, NetNSProxy):
+
+ def bind(self, *argv, **kwarg):
+ return NetNSProxy.bind(self, *argv, **kwarg)
+
+ def close(self):
+ NetNSProxy.close(self)
+
+ def _sendto(self, *argv, **kwarg):
+ return NetNSProxy.sendto(self, *argv, **kwarg)
+
+ def _recv(self, *argv, **kwarg):
+ return NetNSProxy.recv(self, *argv, **kwarg)
+
+
+class NetNS(IPRouteMixin, NetNSocket):
+ '''
+ NetNS is the IPRoute API with network namespace support.
+
+ **Why not IPRoute?**
+
+ The task to run netlink commands in some network namespace, being in
+ another network namespace, requires the architecture, that differs
+ too much from a simple Netlink socket.
+
+ NetNS starts a proxy process in a network namespace and uses
+ `multiprocessing` communication channels between the main and the proxy
+ processes to route all `recv()` and `sendto()` requests/responses.
+
+ **Any specific API calls?**
+
+ Nope. `NetNS` supports all the same, that `IPRoute` does, in the same
+ way. It provides full `socket`-compatible API and can be used in
+ poll/select as well.
+
+ The only difference is the `close()` call. In the case of `NetNS` it
+ is **mandatory** to close the socket before exit.
+
+ **NetNS and IPDB**
+
+ It is possible to run IPDB with NetNS::
+
+ from pyroute2 import NetNS
+ from pyroute2 import IPDB
+
+ ip = IPDB(nl=NetNS('somenetns'))
+ ...
+ ip.release()
+
+ Do not forget to call `release()` when the work is done. It will shut
+ down `NetNS` instance as well.
+ '''
+ def __init__(self, netns, flags=os.O_CREAT):
+ self.netns = netns
+ self.flags = flags
+ super(NetNS, self).__init__()
+ self.marshal = MarshalRtnl()
+
+ def post_init(self):
+ pass
+
+ def remove(self):
+ '''
+ Try to remove this network namespace from the system.
+ '''
+ remove(self.netns)
diff --git a/node-admin/scripts/pyroute2/netns/process/__init__.py b/node-admin/scripts/pyroute2/netns/process/__init__.py
new file mode 100644
index 00000000000..a211a3993f6
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netns/process/__init__.py
@@ -0,0 +1,39 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+import types
+import subprocess
+
+
+class MetaPopen(type):
+ '''
+ API definition for NSPopen.
+
+ All this stuff is required to make `help()` function happy.
+ '''
+ def __init__(cls, *argv, **kwarg):
+ super(MetaPopen, cls).__init__(*argv, **kwarg)
+ # copy docstrings and create proxy slots
+ cls.api = {}
+ for attr_name in dir(subprocess.Popen):
+ attr = getattr(subprocess.Popen, attr_name)
+ cls.api[attr_name] = {}
+ cls.api[attr_name]['callable'] = \
+ isinstance(attr, (types.MethodType, types.FunctionType))
+ cls.api[attr_name]['doc'] = attr.__doc__ \
+ if hasattr(attr, '__doc__') else None
+
+ def __dir__(cls):
+ return list(cls.api.keys()) + ['release']
+
+ def __getattribute__(cls, key):
+ try:
+ return type.__getattribute__(cls, key)
+ except AttributeError:
+ attr = getattr(subprocess.Popen, key)
+ if isinstance(attr, (types.MethodType, types.FunctionType)):
+ def proxy(*argv, **kwarg):
+ return attr(*argv, **kwarg)
+ proxy.__doc__ = attr.__doc__
+ proxy.__objclass__ = cls
+ return proxy
+ else:
+ return attr
diff --git a/node-admin/scripts/pyroute2/netns/process/base_p2.py b/node-admin/scripts/pyroute2/netns/process/base_p2.py
new file mode 100644
index 00000000000..402329846c0
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netns/process/base_p2.py
@@ -0,0 +1,7 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+from pyroute2.netns.process import MetaPopen
+
+
+class NSPopenBase(object):
+
+ __metaclass__ = MetaPopen
diff --git a/node-admin/scripts/pyroute2/netns/process/base_p3.py b/node-admin/scripts/pyroute2/netns/process/base_p3.py
new file mode 100644
index 00000000000..1356958e24a
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netns/process/base_p3.py
@@ -0,0 +1,7 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+from pyroute2.netns.process import MetaPopen
+
+
+class NSPopenBase(object, metaclass=MetaPopen):
+
+ pass
diff --git a/node-admin/scripts/pyroute2/netns/process/proxy.py b/node-admin/scripts/pyroute2/netns/process/proxy.py
new file mode 100644
index 00000000000..11695ac3d67
--- /dev/null
+++ b/node-admin/scripts/pyroute2/netns/process/proxy.py
@@ -0,0 +1,163 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+NSPopen
+=======
+
+The `NSPopen` class has nothing to do with netlink at
+all, but it is required to have a reasonable network
+namespace support.
+
+'''
+
+
+import types
+import atexit
+import threading
+import subprocess
+from pyroute2.netns import setns
+from pyroute2.config import MpQueue
+from pyroute2.config import MpProcess
+try:
+ from pyroute2.netns.process.base_p3 import NSPopenBase
+except Exception:
+ from pyroute2.netns.process.base_p2 import NSPopenBase
+
+
+def _handle(result):
+ if result['code'] == 500:
+ raise result['data']
+ elif result['code'] == 200:
+ return result['data']
+ else:
+ raise TypeError('unsupported return code')
+
+
+def NSPopenServer(nsname, flags, channel_in, channel_out, argv, kwarg):
+ # set netns
+ try:
+ setns(nsname, flags=flags)
+ except Exception as e:
+ channel_out.put(e)
+ return
+ # create the Popen object
+ child = subprocess.Popen(*argv, **kwarg)
+ # send the API map
+ channel_out.put(None)
+
+ while True:
+ # synchronous mode
+ # 1. get the command from the API
+ call = channel_in.get()
+ # 2. stop?
+ if call['name'] == 'release':
+ break
+ # 3. run the call
+ try:
+ attr = getattr(child, call['name'])
+ if isinstance(attr, types.MethodType):
+ result = attr(*call['argv'], **call['kwarg'])
+ else:
+ result = attr
+ channel_out.put({'code': 200, 'data': result})
+ except Exception as e:
+ channel_out.put({'code': 500, 'data': e})
+ child.wait()
+
+
+class NSPopen(NSPopenBase):
+ '''
+ A proxy class to run `Popen()` object in some network namespace.
+
+ Sample to run `ip ad` command in `nsname` network namespace::
+
+ nsp = NSPopen('nsname', ['ip', 'ad'], stdout=subprocess.PIPE)
+ print(nsp.communicate())
+ nsp.wait()
+ nsp.release()
+
+ The only difference in the `release()` call. It explicitly ends
+ the proxy process and releases all the resources.
+ '''
+
+ def __init__(self, nsname, *argv, **kwarg):
+ '''
+ The only differences from the `subprocess.Popen` init are:
+ * `nsname` -- network namespace name
+ * `flags` keyword argument
+
+ All other arguments are passed directly to `subprocess.Popen`.
+
+ Flags usage samples. Create a network namespace, if it doesn't
+ exist yet::
+
+ import os
+ nsp = NSPopen('nsname', ['command'], flags=os.O_CREAT)
+
+ Create a network namespace only if it doesn't exist, otherwise
+ fail and raise an exception::
+
+ import os
+ nsp = NSPopen('nsname', ['command'], flags=os.O_CREAT | os.O_EXCL)
+ '''
+ # create a child
+ self.nsname = nsname
+ if 'flags' in kwarg:
+ self.flags = kwarg.pop('flags')
+ else:
+ self.flags = 0
+ self.channel_out = MpQueue()
+ self.channel_in = MpQueue()
+ self.lock = threading.Lock()
+ self.released = False
+ self.server = MpProcess(target=NSPopenServer,
+ args=(self.nsname,
+ self.flags,
+ self.channel_out,
+ self.channel_in,
+ argv, kwarg))
+ # start the child and check the status
+ self.server.start()
+ response = self.channel_in.get()
+ if isinstance(response, Exception):
+ self.server.join()
+ raise response
+ else:
+ atexit.register(self.release)
+
+ def release(self):
+ '''
+ Explicitly stop the proxy process and release all the
+ resources. The `NSPopen` object can not be used after
+ the `release()` call.
+ '''
+ with self.lock:
+ if self.released:
+ return
+ self.released = True
+ self.channel_out.put({'name': 'release'})
+ self.channel_out.close()
+ self.channel_in.close()
+ self.server.join()
+
+ def __dir__(self):
+ return list(self.api.keys()) + ['release']
+
+ def __getattribute__(self, key):
+ try:
+ return object.__getattribute__(self, key)
+ except AttributeError:
+ with self.lock:
+ if self.released:
+ raise RuntimeError('the object is released')
+
+ if self.api.get(key) and self.api[key]['callable']:
+ def proxy(*argv, **kwarg):
+ self.channel_out.put({'name': key,
+ 'argv': argv,
+ 'kwarg': kwarg})
+ return _handle(self.channel_in.get())
+ proxy.__doc__ = self.api[key]['doc']
+ return proxy
+ else:
+ self.channel_out.put({'name': key})
+ return _handle(self.channel_in.get())
diff --git a/node-admin/scripts/pyroute2/protocols/__init__.py b/node-admin/scripts/pyroute2/protocols/__init__.py
new file mode 100644
index 00000000000..25cf566af48
--- /dev/null
+++ b/node-admin/scripts/pyroute2/protocols/__init__.py
@@ -0,0 +1,234 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+import struct
+from socket import inet_ntop
+from socket import inet_pton
+from socket import AF_INET
+from pyroute2.common import basestring
+
+#
+# IEEE = 802.3 Ethernet magic constants. The frame sizes omit
+# the preamble and FCS/CRC (frame check sequence).
+#
+
+ETH_ALEN = 6 # Octets in one ethernet addr
+ETH_HLEN = 14 # Total octets in header.
+ETH_ZLEN = 60 # Min. octets in frame sans FCS
+ETH_DATA_LEN = 1500 # Max. octets in payload
+ETH_FRAME_LEN = 1514 # Max. octets in frame sans FCS
+ETH_FCS_LEN = 4 # Octets in the FCS
+
+#
+# These are the defined Ethernet Protocol ID's.
+#
+
+ETH_P_LOOP = 0x0060 # Ethernet Loopback packet
+ETH_P_PUP = 0x0200 # Xerox PUP packet
+ETH_P_PUPAT = 0x0201 # Xerox PUP Addr Trans packet
+ETH_P_IP = 0x0800 # Internet Protocol packet
+ETH_P_X25 = 0x0805 # CCITT X.25
+ETH_P_ARP = 0x0806 # Address Resolution packet
+ETH_P_BPQ = 0x08FF # G8BPQ AX.25 Ethernet Packet
+# ^^^ [ NOT AN OFFICIALLY REGISTERED ID ]
+ETH_P_IEEEPUP = 0x0a00 # Xerox IEEE802.3 PUP packet
+ETH_P_IEEEPUPAT = 0x0a01 # Xerox IEEE802.3 PUP Addr Trans packet
+ETH_P_DEC = 0x6000 # DEC Assigned proto
+ETH_P_DNA_DL = 0x6001 # DEC DNA Dump/Load
+ETH_P_DNA_RC = 0x6002 # DEC DNA Remote Console
+ETH_P_DNA_RT = 0x6003 # DEC DNA Routing
+ETH_P_LAT = 0x6004 # DEC LAT
+ETH_P_DIAG = 0x6005 # DEC Diagnostics
+ETH_P_CUST = 0x6006 # DEC Customer use
+ETH_P_SCA = 0x6007 # DEC Systems Comms Arch
+ETH_P_TEB = 0x6558 # Trans Ether Bridging
+ETH_P_RARP = 0x8035 # Reverse Addr Res packet
+ETH_P_ATALK = 0x809B # Appletalk DDP
+ETH_P_AARP = 0x80F3 # Appletalk AARP
+ETH_P_8021Q = 0x8100 # = 802.1Q VLAN Extended Header
+ETH_P_IPX = 0x8137 # IPX over DIX
+ETH_P_IPV6 = 0x86DD # IPv6 over bluebook
+ETH_P_PAUSE = 0x8808 # IEEE Pause frames. See = 802.3 = 31B
+ETH_P_SLOW = 0x8809 # Slow Protocol. See = 802.3ad = 43B
+ETH_P_WCCP = 0x883E # Web-cache coordination protocol
+# defined in draft-wilson-wrec-wccp-v2-00.txt
+ETH_P_PPP_DISC = 0x8863 # PPPoE discovery messages
+ETH_P_PPP_SES = 0x8864 # PPPoE session messages
+ETH_P_MPLS_UC = 0x8847 # MPLS Unicast traffic
+ETH_P_MPLS_MC = 0x8848 # MPLS Multicast traffic
+ETH_P_ATMMPOA = 0x884c # MultiProtocol Over ATM
+ETH_P_LINK_CTL = 0x886c # HPNA, wlan link local tunnel
+ETH_P_ATMFATE = 0x8884 # Frame-based ATM Transport over Ethernet
+
+ETH_P_PAE = 0x888E # Port Access Entity (IEEE = 802.1X)
+ETH_P_AOE = 0x88A2 # ATA over Ethernet
+ETH_P_8021AD = 0x88A8 # = 802.1ad Service VLAN
+ETH_P_802_EX1 = 0x88B5 # = 802.1 Local Experimental = 1.
+ETH_P_TIPC = 0x88CA # TIPC
+ETH_P_8021AH = 0x88E7 # = 802.1ah Backbone Service Tag
+ETH_P_1588 = 0x88F7 # IEEE = 1588 Timesync
+ETH_P_FCOE = 0x8906 # Fibre Channel over Ethernet
+ETH_P_TDLS = 0x890D # TDLS
+ETH_P_FIP = 0x8914 # FCoE Initialization Protocol
+ETH_P_QINQ1 = 0x9100 # deprecated QinQ VLAN
+# ^^^ [ NOT AN OFFICIALLY REGISTERED ID ]
+ETH_P_QINQ2 = 0x9200 # deprecated QinQ VLAN
+# ^^^ [ NOT AN OFFICIALLY REGISTERED ID ]
+ETH_P_QINQ3 = 0x9300 # deprecated QinQ VLAN
+# ^^^ [ NOT AN OFFICIALLY REGISTERED ID ]
+ETH_P_EDSA = 0xDADA # Ethertype DSA
+# ^^^ [ NOT AN OFFICIALLY REGISTERED ID ]
+ETH_P_AF_IUCV = 0xFBFB # IBM af_iucv
+# ^^^ [ NOT AN OFFICIALLY REGISTERED ID ]
+
+#
+# Non DIX types. Won't clash for = 1500 types.
+#
+
+ETH_P_802_3 = 0x0001 # Dummy type for = 802.3 frames
+ETH_P_AX25 = 0x0002 # Dummy protocol id for AX.25
+ETH_P_ALL = 0x0003 # Every packet (be careful!!!)
+ETH_P_802_2 = 0x0004 # = 802.2 frames
+ETH_P_SNAP = 0x0005 # Internal only
+ETH_P_DDCMP = 0x0006 # DEC DDCMP: Internal only
+ETH_P_WAN_PPP = 0x0007 # Dummy type for WAN PPP frames*/
+ETH_P_PPP_MP = 0x0008 # Dummy type for PPP MP frames
+ETH_P_LOCALTALK = 0x0009 # Localtalk pseudo type
+ETH_P_CAN = 0x000C # Controller Area Network
+ETH_P_PPPTALK = 0x0010 # Dummy type for Atalk over PPP*/
+ETH_P_TR_802_2 = 0x0011 # = 802.2 frames
+ETH_P_MOBITEX = 0x0015 # Mobitex (kaz@cafe.net)
+ETH_P_CONTROL = 0x0016 # Card specific control frames
+ETH_P_IRDA = 0x0017 # Linux-IrDA
+ETH_P_ECONET = 0x0018 # Acorn Econet
+ETH_P_HDLC = 0x0019 # HDLC frames
+ETH_P_ARCNET = 0x001A # = 1A for ArcNet :-)
+ETH_P_DSA = 0x001B # Distributed Switch Arch.
+ETH_P_TRAILER = 0x001C # Trailer switch tagging
+ETH_P_PHONET = 0x00F5 # Nokia Phonet frames
+ETH_P_IEEE802154 = 0x00F6 # IEEE802.15.4 frame
+ETH_P_CAIF = 0x00F7 # ST-Ericsson CAIF protocol
+
+
+class msg(dict):
+ buf = None
+ data_len = None
+ fields = ()
+ _fields_names = ()
+ types = {'uint8': 'B',
+ 'uint16': 'H',
+ 'uint32': 'I',
+ 'be16': '>H',
+ 'ip4addr': {'format': '4s',
+ 'decode': lambda x: inet_ntop(AF_INET, x),
+ 'encode': lambda x: [inet_pton(AF_INET, x)]},
+ 'l2addr': {'format': '6B',
+ 'decode': lambda x: ':'.join(['%x' % i for i in x]),
+ 'encode': lambda x: [int(i, 16) for i in
+ x.split(':')]},
+ 'l2paddr': {'format': '6B10s',
+ 'decode': lambda x: ':'.join(['%x' % i for i in
+ x[:6]]),
+ 'encode': lambda x: [int(i, 16) for i in
+ x.split(':')] + [10 * b'\x00']}}
+
+ def __init__(self, content=None, buf=b'', offset=0, value=None):
+ content = content or {}
+ dict.__init__(self, content)
+ self.buf = buf
+ self.offset = offset
+ self.value = value
+ self._register_fields()
+
+ def _register_fields(self):
+ self._fields_names = tuple([x[0] for x in self.fields])
+
+ def _get_routine(self, mode, fmt):
+ fmt = self.types.get(fmt, fmt)
+ if isinstance(fmt, dict):
+ return (fmt['format'],
+ fmt.get(mode, lambda x: x))
+ else:
+ return (fmt, lambda x: x)
+
+ def reset(self):
+ self.buf = b''
+
+ def decode(self):
+ self._register_fields()
+ for field in self.fields:
+ name, sfmt = field[:2]
+ fmt, routine = self._get_routine('decode', sfmt)
+ size = struct.calcsize(fmt)
+ value = struct.unpack(fmt, self.buf[self.offset:
+ self.offset + size])
+ if len(value) == 1:
+ value = value[0]
+ if isinstance(value, basestring) and sfmt[-1] == 's':
+ value = value[:value.find(b'\x00')]
+ self[name] = routine(value)
+ self.offset += size
+ return self
+
+ def encode(self):
+ self._register_fields()
+ for field in self.fields:
+ name, fmt = field[:2]
+ default = b'\x00' if len(field) <= 2 else field[2]
+ fmt, routine = self._get_routine('encode', fmt)
+ # special case: string
+ if fmt == 'string':
+ self.buf += routine(self[name])[0]
+ else:
+ size = struct.calcsize(fmt)
+ if self[name] is None:
+ if not isinstance(default, basestring):
+ self.buf += struct.pack(fmt, default)
+ else:
+ self.buf += default * (size // len(default))
+ else:
+ value = routine(self[name])
+ if not isinstance(value, (set, tuple, list)):
+ value = [value]
+ self.buf += struct.pack(fmt, *value)
+ return self
+
+ def __getitem__(self, key):
+ try:
+ return dict.__getitem__(self, key)
+ except KeyError:
+ if key in self._fields_names:
+ return None
+ raise
+
+
+class ethmsg(msg):
+ fields = (('dst', 'l2addr'),
+ ('src', 'l2addr'),
+ ('type', 'be16'))
+
+
+class ip4msg(msg):
+ fields = (('verlen', 'uint8', 0x45),
+ ('dsf', 'uint8'),
+ ('len', 'be16'),
+ ('id', 'be16'),
+ ('flags', 'uint16'),
+ ('ttl', 'uint8', 128),
+ ('proto', 'uint8'),
+ ('csum', 'be16'),
+ ('src', 'ip4addr'),
+ ('dst', 'ip4addr'))
+
+
+class udp4_pseudo_header(msg):
+ fields = (('src', 'ip4addr'),
+ ('dst', 'ip4addr'),
+ ('pad', 'uint8'),
+ ('proto', 'uint8', 17),
+ ('len', 'be16'))
+
+
+class udpmsg(msg):
+ fields = (('sport', 'be16'),
+ ('dport', 'be16'),
+ ('len', 'be16'),
+ ('csum', 'be16'))
diff --git a/node-admin/scripts/pyroute2/protocols/rawsocket.py b/node-admin/scripts/pyroute2/protocols/rawsocket.py
new file mode 100644
index 00000000000..73348a07407
--- /dev/null
+++ b/node-admin/scripts/pyroute2/protocols/rawsocket.py
@@ -0,0 +1,70 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+import struct
+from ctypes import Structure
+from ctypes import addressof
+from ctypes import string_at
+from ctypes import sizeof
+from ctypes import c_ushort
+from ctypes import c_ubyte
+from ctypes import c_uint
+from ctypes import c_void_p
+from socket import socket
+from socket import htons
+from socket import AF_PACKET
+from socket import SOCK_RAW
+from socket import SOL_SOCKET
+from pyroute2 import IPRoute
+
+ETH_P_ALL = 3
+SO_ATTACH_FILTER = 26
+
+
+class sock_filter(Structure):
+ _fields_ = [('code', c_ushort), # u16
+ ('jt', c_ubyte), # u8
+ ('jf', c_ubyte), # u8
+ ('k', c_uint)] # u32
+
+
+class sock_fprog(Structure):
+ _fields_ = [('len', c_ushort),
+ ('filter', c_void_p)]
+
+
+def compile_bpf(code):
+ ProgramType = sock_filter * len(code)
+ program = ProgramType(*[sock_filter(*line) for line in code])
+ sfp = sock_fprog(len(code), addressof(program[0]))
+ return string_at(addressof(sfp), sizeof(sfp)), program
+
+
+class RawSocket(socket):
+
+ fprog = None
+
+ def __init__(self, ifname, bpf=None):
+ self.ifname = ifname
+ # lookup the interface details
+ with IPRoute() as ip:
+ for link in ip.get_links():
+ if link.get_attr('IFLA_IFNAME') == ifname:
+ break
+ else:
+ raise IOError(2, 'Link not found')
+ self.l2addr = link.get_attr('IFLA_ADDRESS')
+ self.ifindex = link['index']
+ # bring up the socket
+ socket.__init__(self, AF_PACKET, SOCK_RAW, htons(ETH_P_ALL))
+ socket.bind(self, (self.ifname, ETH_P_ALL))
+ if bpf:
+ fstring, self.fprog = compile_bpf(bpf)
+ socket.setsockopt(self, SOL_SOCKET, SO_ATTACH_FILTER, fstring)
+
+ def csum(self, data):
+ if len(data) % 2:
+ data += b'\x00'
+ csum = sum([struct.unpack('>H', data[x*2:x*2+2])[0] for x
+ in range(len(data)//2)])
+ csum = (csum >> 16) + (csum & 0xffff)
+ csum += csum >> 16
+ return ~csum & 0xffff
diff --git a/node-admin/scripts/pyroute2/proxy.py b/node-admin/scripts/pyroute2/proxy.py
new file mode 100644
index 00000000000..bfdab39907d
--- /dev/null
+++ b/node-admin/scripts/pyroute2/proxy.py
@@ -0,0 +1,65 @@
+# By Peter V. Saveliev https://pypi.python.org/pypi/pyroute2. Dual licensed under the Apache 2 and GPLv2+ see https://github.com/svinota/pyroute2 for License details.
+'''
+Netlink proxy engine
+'''
+import errno
+import struct
+import logging
+import traceback
+import threading
+
+
+class NetlinkProxy(object):
+ '''
+ Proxy schemes::
+
+ User -> NetlinkProxy -> Kernel
+ |
+ <---------+
+
+ User <- NetlinkProxy <- Kernel
+
+ '''
+
+ def __init__(self, policy='forward', nl=None, lock=None):
+ self.nl = nl
+ self.lock = lock or threading.Lock()
+ self.pmap = {}
+ self.policy = policy
+
+ def handle(self, data):
+ #
+ # match the packet
+ #
+ ptype = struct.unpack('H', data[4:6])[0]
+ plugin = self.pmap.get(ptype, None)
+ if plugin is not None:
+ with self.lock:
+ try:
+ ret = plugin(data, self.nl)
+ if ret is None:
+ msg = struct.pack('IHH', 40, 2, 0)
+ msg += data[8:16]
+ msg += struct.pack('I', 0)
+ # nlmsgerr struct alignment
+ msg += b'\0' * 20
+ return {'verdict': self.policy,
+ 'data': msg}
+ else:
+ return ret
+
+ except Exception as e:
+ logging.error(traceback.format_exc())
+ # errmsg
+ if isinstance(e, (OSError, IOError)):
+ code = e.errno
+ else:
+ code = errno.ECOMM
+ msg = struct.pack('HH', 2, 0)
+ msg += data[8:16]
+ msg += struct.pack('I', code)
+ msg += data
+ msg = struct.pack('I', len(msg) + 4) + msg
+ return {'verdict': 'error',
+ 'data': msg}
+ return None
diff --git a/node-admin/scripts/route-osx.sh b/node-admin/scripts/route-osx.sh
new file mode 100755
index 00000000000..780d69f741e
--- /dev/null
+++ b/node-admin/scripts/route-osx.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -e
+
+source "${0%/*}/common-vm.sh"
+
+VESPA_DOCKER_MACHINE_IP=$(docker-machine ip "$DOCKER_VM_NAME")
+if [ $? -ne 0 ]; then
+ echo "Could not get the IP of the docker-machine $DOCKER_VM_NAME"
+ exit 1
+fi
+
+# Setup the route
+sudo route delete "$HOST_BRIDGE_NETWORK" &> /dev/null
+sudo route add "$HOST_BRIDGE_NETWORK" "$VESPA_DOCKER_MACHINE_IP"
diff --git a/node-admin/scripts/setup-docker.sh b/node-admin/scripts/setup-docker.sh
new file mode 100755
index 00000000000..3e4b10dbd74
--- /dev/null
+++ b/node-admin/scripts/setup-docker.sh
@@ -0,0 +1,176 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# WARNING: Please double-check with the documentation in node-admin/README*
+# whether these commands are in fact correct. If they are, this saves a bunch
+# of typing...
+#
+# See HelpAndExit below for usage.
+
+set -ex
+
+declare DAYS_VALID=3650
+
+# Note regarding the file names: Some are renamed from what you get from
+# following the recipe in the docker documentation. Here, we've used
+# underscores exclusively, never dashes. Some files have been renamed for
+# explicitness, clarity and consistency (e.g. 'key' is renamed 'client_key').
+declare CERTS_DIR=~/.docker-certs
+declare CA_FILE="$CERTS_DIR"/ca_cert.pem
+declare CA_KEY_FILE="$CERTS_DIR"/ca_key.pem
+declare CLIENT_CERT_FILE="$CERTS_DIR"/client_cert.pem
+declare CLIENT_KEY_FILE="$CERTS_DIR"/client_key.pem
+declare SERVER_CERT_FILE="$CERTS_DIR"/server_cert.pem
+declare SERVER_KEY_FILE="$CERTS_DIR"/server_key.pem
+
+declare GROUP=users
+declare YAHOO_GROUP="$GROUP"
+
+function HelpAndExit {
+ cat <<EOF
+Usage: ${0##*/} <command>...
+Setup Docker.
+
+Commands:
+ all Setup docker home and TLS certificates/keys.
+ Same as following commands: home certs
+ certs Generate and install TLS keys.
+ Same as following commands: generate-certs install-certs
+ generate-certs Generate TLS-related certificates and keys to
+ $CERTS_DIR
+ help Print this message.
+ install-certs Install TLS-related certificates and keys in
+ $CERTS_DIR
+ to /etc/dockercert_{daemon,cli,container}.
+ home Add docker user and make symbolic links from Docker dirs in
+ /var to dirs below ~docker.
+EOF
+
+ exit 0
+}
+
+function GenerateCertificates {
+ rm -rf "$CERTS_DIR"
+ mkdir -p "$CERTS_DIR"
+
+ # Generate CA private and public keys
+ echo "We're about to generate a CA key, please use a secure password."
+ echo "You will be prompted for this password many times in what follows..."
+ openssl genrsa -aes256 -out "$CA_KEY_FILE" 4096
+ openssl req -new -x509 -days "$DAYS_VALID" -key "$CA_KEY_FILE" -sha256 \
+ -out "$CA_FILE"
+
+ # Generate server key and certificate signing request (CSR)
+ openssl genrsa -out "$SERVER_KEY_FILE" 4096
+ local server_csr_file="$CERTS_DIR"/server.csr
+ openssl req -subj "/CN=$HOSTNAME" -sha256 -new -key "$SERVER_KEY_FILE" \
+ -out "$server_csr_file"
+
+ # Sign server's public key with CA
+ local server_config_file="$CERTS_DIR"/server.cnf
+ echo "subjectAltName = IP:127.0.0.1" > "$server_config_file"
+ openssl x509 -req -days "$DAYS_VALID" -sha256 -in "$server_csr_file" \
+ -CA "$CA_FILE" -CAkey "$CA_KEY_FILE" -CAcreateserial \
+ -out "$SERVER_CERT_FILE" -extfile "$server_config_file"
+
+ # Generate client key and certificate signing request (CSR)
+ openssl genrsa -out "$CLIENT_KEY_FILE" 4096
+ local client_csr_file="$CERTS_DIR"/client.csr
+ openssl req -subj '/CN=client' -new -key "$CLIENT_KEY_FILE" \
+ -out "$client_csr_file"
+
+ # Sign client's public key with CA
+ local client_config_file="$CERTS_DIR"/client.cnf
+ echo extendedKeyUsage = clientAuth > "$client_config_file"
+ openssl x509 -req -days "$DAYS_VALID" -sha256 -in "$client_csr_file" \
+ -CA "$CA_FILE" -CAkey "$CA_KEY_FILE" -CAcreateserial \
+ -out "$CLIENT_CERT_FILE" -extfile "$client_config_file"
+
+ # CSR and config files no longer needed
+ rm "$client_csr_file" "$server_csr_file"
+ rm "$server_config_file" "$client_config_file"
+
+ # Avoid accidental writes
+ chmod 0400 "$CA_KEY_FILE" "$CLIENT_KEY_FILE" "$SERVER_KEY_FILE"
+ chmod 0444 "$CA_FILE" "$SERVER_CERT_FILE" "$CLIENT_CERT_FILE"
+}
+
+function InstallCertificates {
+ # The files you end up with after GenerateKeys will be used by three
+ # parties: The docker daemon, the docker CLI, and the docker client in Node
+ # Admin. None of these parties need (nor should they have) access to all
+ # these files. Also, the three parties will run as different users. Since
+ # these files should not be world-readable, one solution is to create
+ # separate directories for the three usages, so each directory may contain
+ # only the needed files, with the correct owner and permissions.
+
+ sudo mkdir -p /etc/dockercert_daemon
+ sudo chown yahoo:users /etc/dockercert_daemon
+ sudo cp "$CA_FILE" "$SERVER_CERT_FILE" "$SERVER_KEY_FILE" /etc/dockercert_daemon
+ sudo chown root:root /etc/dockercert_daemon/*
+
+ # The docker client looks for files with certain names (you can only
+ # configure the path to the directory containing the files), so the
+ # "original" file names are used.
+ sudo mkdir -p /etc/dockercert_cli
+ sudo chown yahoo:users /etc/dockercert_cli
+ sudo cp "$CA_FILE" /etc/dockercert_cli/ca.pem
+ sudo cp "$CLIENT_CERT_FILE" /etc/dockercert_cli/cert.pem
+ sudo cp "$CLIENT_KEY_FILE" /etc/dockercert_cli/key.pem
+ sudo chown $USER:$GROUP /etc/dockercert_cli/*
+
+ sudo mkdir -p /etc/dockercert_container
+ sudo chown yahoo:$YAHOO_GROUP /etc/dockercert_container
+ # These filenames must match the config given in
+ # src/main/application/services.xml.
+ sudo cp "$CA_FILE" "$CLIENT_CERT_FILE" "$CLIENT_KEY_FILE" /etc/dockercert_container
+ sudo chown yahoo:$YAHOO_GROUP /etc/dockercert_container/*
+
+ echo "Note: Consider reloading & restarting the docker daemon to pick up"
+ echo "the new certificates and keys:"
+ echo " sudo systemctl daemon-reload"
+ echo " sudo systemctl restart docker"
+}
+
+function SetupDockerHome {
+ # Assume an error means the docker user already exists
+ sudo useradd -g docker docker || true
+
+ sudo mkdir -p ~docker/lib ~docker/run
+ sudo chmod +rx ~docker ~docker/lib ~docker/run
+ sudo systemctl stop docker
+ sudo rm -rf /var/{run,lib}/docker
+ sudo ln -s ~docker/run /var/run/docker
+ sudo ln -s ~docker/lib /var/lib/docker
+ sudo systemctl daemon-reload
+ sudo systemctl restart docker
+}
+
+function Main {
+ # Prime sudo
+ sudo true
+
+ if (($# == 0))
+ then
+ HelpAndExit
+ fi
+
+ local command
+ for command in "$@"
+ do
+ case "$command" in
+ all) Main home certs ;;
+ certs)
+ GenerateCertificates
+ InstallCertificates
+ ;;
+ generate-certs) GenerateCertificates ;;
+ help) HelpAndExit ;;
+ home) SetupDockerHome ;;
+ install-certs) InstallCertificates ;;
+ *) Fail "Unknown command '$command'" ;;
+ esac
+ done
+}
+
+Main "$@"
diff --git a/node-admin/scripts/setup-route-and-hosts-osx.sh b/node-admin/scripts/setup-route-and-hosts-osx.sh
new file mode 100755
index 00000000000..dcfcfc0f121
--- /dev/null
+++ b/node-admin/scripts/setup-route-and-hosts-osx.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -e
+
+echo "This will alter your routing table and /etc/hosts file. Continue ?"
+select yn in "Yes" "No"; do
+ case $yn in
+ Yes ) break;;
+ No ) echo "Exiting."; exit;;
+ esac
+done
+
+# Setup the route
+cd "$SCRIPT_DIR"
+./route-osx.sh
+
+# Setup the hosts file
+cd "$SCRIPT_DIR"
+./etc-hosts.sh
diff --git a/node-admin/scripts/vm.sh b/node-admin/scripts/vm.sh
new file mode 100755
index 00000000000..19542a7c392
--- /dev/null
+++ b/node-admin/scripts/vm.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+set -e
+
+source "${0%/*}/common-vm.sh"
+
+DOCKER_VM_WAS_STARTED=false
+
+if ! docker-machine status "$DOCKER_VM_NAME" &> /dev/null; then
+ # Machine does not exist and we have to create and start
+ docker-machine create -d virtualbox \
+ --virtualbox-disk-size "$DOCKER_VM_DISK_SIZE_IN_MB" \
+ --virtualbox-memory "$DOCKER_VM_MEMORY_SIZE_IN_MB" \
+ --virtualbox-cpu-count "$DOCKER_VM_CPU_COUNT" \
+ --virtualbox-hostonly-cidr "$DOCKER_VM_HOST_CIDR" \
+ "$DOCKER_VM_NAME"
+
+ eval $(docker-machine env "$DOCKER_VM_NAME")
+
+ # Node admin expects different names for the certificates. Just symlink docker has
+ # generated for us to match those in node-admin/src/main/application/services.xml.
+ (
+ cd "$DOCKER_CERT_PATH"
+ ln -s ca.pem ca_cert.pem
+ ln -s key.pem client_key.pem
+ ln -s cert.pem client_cert.pem
+ )
+ DOCKER_VM_WAS_STARTED=true
+fi
+
+
+VESPA_VM_STATUS=$(docker-machine status "$DOCKER_VM_NAME")
+if [ "$VESPA_VM_STATUS" == "Stopped" ]; then
+ docker-machine start "$DOCKER_VM_NAME"
+ DOCKER_VM_WAS_STARTED=true
+ VESPA_VM_STATUS=$(docker-machine status "$DOCKER_VM_NAME")
+fi
+
+if [ "$VESPA_VM_STATUS" != "Running" ]; then
+ echo "Unable to get Docker machine $DOCKER_VM_NAME up and running."
+ echo "You can try to manually remove the machine: docker-machine rm -y $DOCKER_VM_NAME "
+ echo " and then rerun this script."
+ echo "Exiting."
+ exit 1
+fi
+
+if $DOCKER_VM_WAS_STARTED; then
+ # Put anything that is not persisted between VM restarts in here.
+ # Set up NAT for the $HOST_BRIDGE_INTERFACE interface so that we can connect directly from OS X.
+ docker-machine ssh "$DOCKER_VM_NAME" sudo /usr/local/sbin/iptables -t nat -A POSTROUTING -s "$HOST_BRIDGE_NETWORK" ! -o "$HOST_BRIDGE_INTERFACE" -j MASQUERADE
+ docker-machine ssh "$DOCKER_VM_NAME" sudo /usr/local/sbin/iptables -A FORWARD -o "$HOST_BRIDGE_INTERFACE" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+
+ # Install dependencies used by setup scripts
+ docker-machine ssh "$DOCKER_VM_NAME" tce-load -wi python bash
+fi
+
+# Get the environment for our VM
+eval $(docker-machine env "$DOCKER_VM_NAME")
+
+if [ $# -ge 1 ]; then
+ declare -r ARG_SCRIPT=$1
+ shift
+
+ declare -r ARG_SCRIPT_BASE=$(basename "$ARG_SCRIPT")
+ declare -r ARG_SCRIPT_DIR=$(cd $(dirname "$ARG_SCRIPT") && pwd -P)
+ declare -r ARG_SCRIPT_ABS="$ARG_SCRIPT_DIR/$ARG_SCRIPT_BASE"
+
+ if ! docker-machine ssh "$DOCKER_VM_NAME" which "$ARG_SCRIPT_ABS" &> /dev/null; then
+ echo "Provided script file does not exist or is not executable in VM : $ARG_SCRIPT_ABS"
+ echo "Usage: $0 [SCRIPT] [SCRIPT_ARGS...]"
+ exit 1
+ fi
+
+ # Start the provided script. This works because the $HOME directory is mapped in the same location in the VM.
+ docker-machine ssh "$DOCKER_VM_NAME" "CONTAINER_CERT_PATH=$DOCKER_CERT_PATH NETWORK_TYPE=vm $ARG_SCRIPT_ABS $*"
+fi
+
diff --git a/node-admin/scripts/zone.sh b/node-admin/scripts/zone.sh
new file mode 100755
index 00000000000..b35f367ba59
--- /dev/null
+++ b/node-admin/scripts/zone.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+set -e
+
+source "${0%/*}/common.sh"
+
+function Usage {
+ UsageHelper "$@" <<EOF
+Usage: $SCRIPT_NAME <command> [<option>]...
+Manage Hosted Vespa zone on localhost using Docker.
+
+The Docker daemon must already be running, and the Docker image must have been
+built. The node-admin module must have been packaged.
+
+Commands:
+ start Start zone (start Config Server, Node Admin, etc)
+ stop Stop zone (take down Node Admin, Config Server, etc)
+ restart Stop, then start
+
+Options:
+ --hv-env <env>
+ Make a zone with this Hosted Vespa environment. Must be one of
+ prod, dev, test, staging, etc. Default is $DEFAULT_HOSTED_VESPA_ENVIRONMENT.
+ --hv-region <region>
+ Make a zone with this Hosted Vespa region. Default is $DEFAULT_HOSTED_VESPA_REGION.
+ --num-nodes <num-nodes>
+ Make a zone with <num-nodes> Docker nodes instead of the default $DEFAULT_NUM_APP_CONTAINERS.
+EOF
+}
+
+function Stop {
+ if (($# != 0))
+ then
+ Usage
+ fi
+
+ # Prime sudo to avoid password prompt in the middle of the script.
+ sudo true
+
+ ./node-admin.sh stop
+
+ # TODO: Stop and remove existing vespa node containers.
+
+ # There's no need to stop populate-noderepo-with-local-nodes.sh, as the
+ # whole node repo is going down when the config server is stopped.
+ #
+ # ./populate-noderepo-with-local-nodes.sh stop
+
+ ./config-server.sh stop
+ ./make-host-like-container.sh stop
+ ./network-bridge.sh stop
+ ./etc-hosts.sh stop
+}
+
+function Start {
+ if (($# != 0))
+ then
+ Usage
+ fi
+
+ # Prime sudo to avoid password prompt in the middle of the script.
+ sudo true
+
+ ./etc-hosts.sh --num-nodes "$NUM_APP_CONTAINERS"
+ ./network-bridge.sh
+ ./make-host-like-container.sh
+
+ local region="${OPTION_HV_REGION:-$DEFAULT_HOSTED_VESPA_REGION}"
+ local env="${OPTION_HV_ENV:-$DEFAULT_HOSTED_VESPA_ENVIRONMENT}"
+ ./config-server.sh --wait=true --hv-region="$region" --hv-env="$env"
+
+ ./populate-noderepo-with-local-nodes.sh --num-nodes "$NUM_APP_CONTAINERS"
+ ./node-admin.sh
+}
+
+# Makes it easier to access scripts in the same 'scripts' directory
+cd "$SCRIPT_DIR"
+
+Main "$@"
diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml
new file mode 100644
index 00000000000..f2b31b3afb9
--- /dev/null
+++ b/node-admin/src/main/application/services.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<services version="1.0">
+ <jdisc version="1.0" jetty="true">
+ <rest-api path="test" jersey2="true">
+ <components bundle="node-admin">
+ <package>com.yahoo.vespa.hosted.node.admin.testapi</package>
+ </components>
+ </rest-api>
+ <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.NodeAdminScheduler" bundle="node-admin"/>
+ <config name='nodeadmin.docker.docker'>
+ <caCertPath>/host/docker/certs/ca_cert.pem</caCertPath>
+ <clientCertPath>/host/docker/certs/client_cert.pem</clientCertPath>
+ <clientKeyPath>/host/docker/certs/client_key.pem</clientKeyPath>
+ </config>
+ </jdisc>
+</services>
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java
new file mode 100644
index 00000000000..b0c10d8d803
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/ContainerNodeSpec.java
@@ -0,0 +1,93 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.docker.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * @author stiankri
+ */
+public class ContainerNodeSpec {
+ public final HostName hostname;
+ public final Optional<DockerImage> wantedDockerImage;
+ public final ContainerName containerName;
+ public final NodeState nodeState;
+ public final Optional<Long> wantedRestartGeneration;
+ public final Optional<Long> currentRestartGeneration;
+ public final Optional<Double> minCpuCores;
+ public final Optional<Double> minMainMemoryAvailableGb;
+ public final Optional<Double> minDiskAvailableGb;
+
+ public ContainerNodeSpec(
+ final HostName hostname,
+ final Optional<DockerImage> wantedDockerImage,
+ final ContainerName containerName,
+ final NodeState nodeState,
+ final Optional<Long> wantedRestartGeneration,
+ final Optional<Long> currentRestartGeneration,
+ final Optional<Double> minCpuCores,
+ final Optional<Double> minMainMemoryAvailableGb,
+ final Optional<Double> minDiskAvailableGb) {
+ this.hostname = hostname;
+ this.wantedDockerImage = wantedDockerImage;
+ this.containerName = containerName;
+ this.nodeState = nodeState;
+ this.wantedRestartGeneration = wantedRestartGeneration;
+ this.currentRestartGeneration = currentRestartGeneration;
+ this.minCpuCores = minCpuCores;
+ this.minMainMemoryAvailableGb = minMainMemoryAvailableGb;
+ this.minDiskAvailableGb = minDiskAvailableGb;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof ContainerNodeSpec)) return false;
+
+ ContainerNodeSpec that = (ContainerNodeSpec) o;
+
+ return Objects.equals(hostname, that.hostname) &&
+ Objects.equals(wantedDockerImage, that.wantedDockerImage) &&
+ Objects.equals(containerName, that.containerName) &&
+ Objects.equals(nodeState, that.nodeState) &&
+ Objects.equals(wantedRestartGeneration, that.wantedRestartGeneration) &&
+ Objects.equals(currentRestartGeneration, that.currentRestartGeneration) &&
+ Objects.equals(minCpuCores, that.minCpuCores) &&
+ Objects.equals(minMainMemoryAvailableGb, that.minMainMemoryAvailableGb) &&
+ Objects.equals(minDiskAvailableGb, that.minDiskAvailableGb);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ hostname,
+ wantedDockerImage,
+ containerName,
+ nodeState,
+ wantedRestartGeneration,
+ currentRestartGeneration,
+ minCpuCores,
+ minMainMemoryAvailableGb,
+ minDiskAvailableGb);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + " {"
+ + " hostname=" + hostname
+ + " wantedDockerImage=" + wantedDockerImage
+ + " containerName=" + containerName
+ + " nodeState=" + nodeState
+ + " wantedRestartGeneration=" + wantedRestartGeneration
+ + " minCpuCores=" + minCpuCores
+ + " currentRestartGeneration=" + currentRestartGeneration
+ + " minMainMemoryAvailableGb=" + minMainMemoryAvailableGb
+ + " minDiskAvailableGb=" + minDiskAvailableGb
+ + " }";
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java
new file mode 100644
index 00000000000..6d4873d92bf
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdmin.java
@@ -0,0 +1,155 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.docker.Container;
+import com.yahoo.vespa.hosted.node.admin.docker.Docker;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * The "most important" class in this module, where the main business logic resides or is driven from.
+ *
+ * @author stiankri
+ */
+public class NodeAdmin {
+ private static final Logger logger = Logger.getLogger(NodeAdmin.class.getName());
+
+ private static final long MIN_AGE_IMAGE_GC_MILLIS = Duration.ofMinutes(15).toMillis();
+
+ private final Docker docker;
+ private final Function<HostName, NodeAgent> nodeAgentFactory;
+
+ private final Map<HostName, NodeAgent> nodeAgents = new HashMap<>();
+
+ private Map<DockerImage, Long> firstTimeEligibleForGC = Collections.emptyMap();
+
+ /**
+ * @param docker interface to docker daemon and docker-related tasks
+ * @param nodeAgentFactory factory for {@link NodeAgent} objects
+ */
+ public NodeAdmin(final Docker docker, final Function<HostName, NodeAgent> nodeAgentFactory) {
+ this.docker = docker;
+ this.nodeAgentFactory = nodeAgentFactory;
+ }
+
+ public void maintainWantedState(final List<ContainerNodeSpec> containersToRun) {
+ final List<Container> existingContainers = docker.getAllManagedContainers();
+
+ synchronizeLocalContainerState(containersToRun, existingContainers);
+
+ garbageCollectDockerImages(containersToRun);
+ }
+
+ private void garbageCollectDockerImages(final List<ContainerNodeSpec> containersToRun) {
+ final Set<DockerImage> deletableDockerImages = getDeletableDockerImages(
+ docker.getUnusedDockerImages(), containersToRun);
+ final long currentTime = System.currentTimeMillis();
+ // TODO: This logic should be unit tested.
+ firstTimeEligibleForGC = deletableDockerImages.stream()
+ .collect(Collectors.toMap(
+ dockerImage -> dockerImage,
+ dockerImage -> Optional.ofNullable(firstTimeEligibleForGC.get(dockerImage)).orElse(currentTime)));
+ // Delete images that have been eligible for some time.
+ firstTimeEligibleForGC.forEach((dockerImage, timestamp) -> {
+ if (currentTime - timestamp > MIN_AGE_IMAGE_GC_MILLIS) {
+ docker.deleteImage(dockerImage);
+ }
+ });
+ }
+
+ // Turns an Optional<T> into a Stream<T> of length zero or one depending upon whether a value is present.
+ // This is a workaround for Java 8 not having Stream.flatMap(Optional).
+ private static <T> Stream<T> streamOf(Optional<T> opt) {
+ return opt.map(Stream::of)
+ .orElseGet(Stream::empty);
+ }
+
+ static Set<DockerImage> getDeletableDockerImages(
+ final Set<DockerImage> currentlyUnusedDockerImages,
+ final List<ContainerNodeSpec> pendingContainers) {
+ final Set<DockerImage> imagesNeededNowOrInTheFuture = pendingContainers.stream()
+ .flatMap(nodeSpec -> streamOf(nodeSpec.wantedDockerImage))
+ .collect(Collectors.toSet());
+ return diff(currentlyUnusedDockerImages, imagesNeededNowOrInTheFuture);
+ }
+
+ // Set-difference. Returns minuend minus subtrahend.
+ private static <T> Set<T> diff(final Set<T> minuend, final Set<T> subtrahend) {
+ final HashSet<T> result = new HashSet<>(minuend);
+ result.removeAll(subtrahend);
+ return result;
+ }
+
+ // Returns a full outer join of two data sources (of types T and U) on some extractable attribute (of type V).
+ // Full outer join means that all elements of both data sources are included in the result,
+ // even when there is no corresponding element (having the same attribute) in the other data set,
+ // in which case the value from the other source will be empty.
+ static <T, U, V> Stream<Pair<Optional<T>, Optional<U>>> fullOuterJoin(
+ final Stream<T> tStream, final Function<T, V> tAttributeExtractor,
+ final Stream<U> uStream, final Function<U, V> uAttributeExtractor) {
+ final Map<V, T> tMap = tStream.collect(Collectors.toMap(tAttributeExtractor, t -> t));
+ final Map<V, U> uMap = uStream.collect(Collectors.toMap(uAttributeExtractor, u -> u));
+ return Stream.concat(tMap.keySet().stream(), uMap.keySet().stream())
+ .distinct()
+ .map(key -> new Pair<>(Optional.ofNullable(tMap.get(key)), Optional.ofNullable(uMap.get(key))));
+ }
+
+ void synchronizeLocalContainerState(
+ final List<ContainerNodeSpec> containersToRun,
+ final List<Container> existingContainers) {
+ final Stream<Pair<Optional<ContainerNodeSpec>, Optional<Container>>> nodeSpecContainerPairs = fullOuterJoin(
+ containersToRun.stream(), nodeSpec -> nodeSpec.hostname,
+ existingContainers.stream(), container -> container.hostname);
+
+ final Set<HostName> nodeHostNames = containersToRun.stream()
+ .map(spec -> spec.hostname)
+ .collect(Collectors.toSet());
+ final Set<HostName> obsoleteAgentHostNames = diff(nodeAgents.keySet(), nodeHostNames);
+ obsoleteAgentHostNames.forEach(hostName -> nodeAgents.remove(hostName).stop());
+
+ nodeSpecContainerPairs.forEach(nodeSpecContainerPair -> {
+ final Optional<ContainerNodeSpec> nodeSpec = nodeSpecContainerPair.getFirst();
+ final Optional<Container> existingContainer = nodeSpecContainerPair.getSecond();
+
+ if (!nodeSpec.isPresent()) {
+ assert existingContainer.isPresent();
+ logger.warning("Container " + existingContainer.get() + " exists, but is not in node repository runlist");
+ return;
+ }
+
+ try {
+ updateAgent(nodeSpec.get());
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed to bring container to desired state", e);
+ }
+ });
+ }
+
+ private void updateAgent(final ContainerNodeSpec nodeSpec) throws IOException {
+ final NodeAgent agent;
+ if (nodeAgents.containsKey(nodeSpec.hostname)) {
+ agent = nodeAgents.get(nodeSpec.hostname);
+ } else {
+ agent = nodeAgentFactory.apply(nodeSpec.hostname);
+ nodeAgents.put(nodeSpec.hostname, agent);
+ agent.start();
+ }
+ agent.update();
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java
new file mode 100644
index 00000000000..985e20a3ea8
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAdminScheduler.java
@@ -0,0 +1,144 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.log.LogLevel;
+import com.yahoo.nodeadmin.docker.DockerConfig;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.docker.Docker;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImpl;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl;
+import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator;
+import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl;
+
+import javax.annotation.concurrent.GuardedBy;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.function.Function;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+/**
+ * @author stiankri
+ */
+public class NodeAdminScheduler extends AbstractComponent {
+ private static final Logger log = Logger.getLogger(NodeAdminScheduler.class.getName());
+
+ private static final long INITIAL_DELAY_SECONDS = 0;
+ private static final long INTERVAL_IN_SECONDS = 60;
+
+ private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
+ private final ScheduledFuture<?> scheduledFuture;
+
+ private enum State { WAIT, WORK, STOP }
+
+ private final Object monitor = new Object();
+ @GuardedBy("monitor")
+ private State state = State.WAIT;
+ @GuardedBy("monitor")
+ private List<ContainerNodeSpec> wantedContainerState = null;
+
+ public NodeAdminScheduler(final DockerConfig dockerConfig) {
+ final Docker docker = new DockerImpl(DockerImpl.newDockerClientFromConfig(dockerConfig));
+ final NodeRepository nodeRepository = new NodeRepositoryImpl();
+ final Orchestrator orchestrator = new OrchestratorImpl(OrchestratorImpl.makeOrchestratorHostApiClient());
+ final Function<HostName, NodeAgent> nodeAgentFactory = (hostName) ->
+ new NodeAgentImpl(hostName, docker, nodeRepository, orchestrator);
+ final NodeAdmin nodeAdmin = new NodeAdmin(docker, nodeAgentFactory);
+ scheduledFuture = scheduler.scheduleWithFixedDelay(
+ throwableLoggingRunnable(fetchContainersToRunFromNodeRepository(nodeRepository)),
+ INITIAL_DELAY_SECONDS, INTERVAL_IN_SECONDS, SECONDS);
+ new Thread(maintainWantedStateRunnable(nodeAdmin), "Node Admin Scheduler main thread").start();
+ }
+
+ private void notifyWorkToDo(final Runnable codeToExecuteInCriticalSection) {
+ synchronized (monitor) {
+ if (state == State.STOP) {
+ return;
+ }
+ state = State.WORK;
+ codeToExecuteInCriticalSection.run();
+ monitor.notifyAll();
+ }
+ }
+
+ /**
+ * Prevents exceptions from leaking out and suppressing the scheduler from running the task again.
+ */
+ private static Runnable throwableLoggingRunnable(final Runnable task) {
+ return () -> {
+ try {
+ task.run();
+ } catch (Throwable throwable) {
+ log.log(LogLevel.ERROR, "Unhandled exception leaked out to scheduler.", throwable);
+ }
+ };
+ }
+
+ private Runnable fetchContainersToRunFromNodeRepository(final NodeRepository nodeRepository) {
+ return () -> {
+ // TODO: should the result from the config server contain both active and inactive?
+ final List<ContainerNodeSpec> containersToRun;
+ try {
+ containersToRun = nodeRepository.getContainersToRun();
+ } catch (IOException e) {
+ log.log(Level.WARNING, "Failed fetching container info from node repository", e);
+ return;
+ }
+ setWantedContainerState(containersToRun);
+ };
+ }
+
+ private void setWantedContainerState(final List<ContainerNodeSpec> wantedContainerState) {
+ if (wantedContainerState == null) {
+ throw new IllegalArgumentException("wantedContainerState must not be null");
+ }
+
+ final Runnable codeToExecuteInCriticalSection = () -> this.wantedContainerState = wantedContainerState;
+ notifyWorkToDo(codeToExecuteInCriticalSection);
+ }
+
+ private Runnable maintainWantedStateRunnable(final NodeAdmin nodeAdmin) {
+ return () -> {
+ while (true) {
+ final List<ContainerNodeSpec> containersToRun;
+
+ synchronized (monitor) {
+ while (state == State.WAIT) {
+ try {
+ monitor.wait();
+ } catch (InterruptedException e) {
+ // Ignore, properly handled by next loop iteration.
+ }
+ }
+ if (state == State.STOP) {
+ return;
+ }
+ assert state == State.WORK;
+ assert wantedContainerState != null;
+ containersToRun = wantedContainerState;
+ state = State.WAIT;
+ }
+
+ throwableLoggingRunnable(() -> nodeAdmin.maintainWantedState(containersToRun))
+ .run();
+ }
+ };
+ }
+
+ @Override
+ public void deconstruct() {
+ scheduledFuture.cancel(false);
+ scheduler.shutdown();
+ synchronized (monitor) {
+ state = State.STOP;
+ monitor.notifyAll();
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java
new file mode 100644
index 00000000000..54e7ac3e92f
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgent.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin;
+
+/**
+ * Responsible for management of a single node/container over its lifecycle.
+ * May own its own resources, threads etc. Runs independently, but receives signals
+ * on state changes in the environment that may trigger this agent to take actions.
+ *
+ * @author bakksjo
+ */
+public interface NodeAgent {
+ /**
+ * Signals to the agent that it should update the node specification and container state and maintain wanted state.
+ *
+ * This method is to be assumed asynchronous by the caller; i.e. any actions the agent will take may execute after this method call returns.
+ *
+ * It is an error to call this method on an instance after stop() has been called.
+ */
+ void update();
+
+ /**
+ * Starts the agent. After this method is called, the agent will asynchronously maintain the node, continuously
+ * striving to make the current state equal to the wanted state. The current and wanted state update as part of {@link #update()}.
+ */
+ void start();
+
+ /**
+ * Signals to the agent that the node is at the end of its lifecycle and no longer needs a managing agent.
+ * Cleans up any resources the agent owns, such as threads, connections etc. Cleanup is synchronous; when this
+ * method returns, no more actions will be taken by the agent.
+ */
+ void stop();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java
new file mode 100644
index 00000000000..b197b639166
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImpl.java
@@ -0,0 +1,404 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.hosted.node.admin.docker.Container;
+import com.yahoo.vespa.hosted.node.admin.docker.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.docker.Docker;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.docker.ProcessResult;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState;
+import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator;
+import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException;
+
+import javax.annotation.concurrent.GuardedBy;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author bakksjo
+ */
+public class NodeAgentImpl implements NodeAgent {
+ private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName());
+ static final String NODE_PROGRAM = Defaults.getDefaults().vespaHome() + "bin/vespa-nodectl";
+ private static final String[] RESUME_NODE_COMMAND = new String[] {NODE_PROGRAM, "resume"};
+ private static final String[] SUSPEND_NODE_COMMAND = new String[] {NODE_PROGRAM, "suspend"};
+
+ private final String logPrefix;
+ private final HostName hostname;
+
+ private final Docker docker;
+ private final NodeRepository nodeRepository;
+ private final Orchestrator orchestrator;
+
+ private final Thread thread;
+
+ private enum State { WAIT, WORK, STOP }
+
+ private final Object monitor = new Object();
+ @GuardedBy("monitor")
+ private State state = State.WAIT;
+
+ // The attributes of the last successful noderepo attribute update for this node. Used to avoid redundant calls.
+ // Only used internally by maintenance thread; no synchronization necessary.
+ private NodeAttributes lastAttributesSet = null;
+ // Whether we have successfully started the node using the node program. Used to avoid redundant start calls.
+ private boolean nodeStarted = false;
+
+
+
+ /**
+ * @param hostName the hostname of the node managed by this agent
+ * @param docker interface to docker daemon and docker-related tasks
+ * @param nodeRepository interface to (remote) node repository
+ * @param orchestrator interface to (remote) orchestrator
+ */
+ public NodeAgentImpl(
+ final HostName hostName,
+ final Docker docker,
+ final NodeRepository nodeRepository,
+ final Orchestrator orchestrator) {
+ this.logPrefix = "NodeAgent(" + hostName + "): ";
+ this.docker = docker;
+ this.nodeRepository = nodeRepository;
+ this.orchestrator = orchestrator;
+ this.thread = new Thread(this::maintainWantedState, "Node Agent maintenance thread for node " + hostName);
+ this.hostname = hostName;
+ }
+
+ @Override
+ public void update() {
+ changeStateAndNotify(() -> {
+ this.state = State.WORK;
+ });
+ }
+
+ @Override
+ public void start() {
+ logger.log(LogLevel.INFO, logPrefix + "Scheduling start of NodeAgent");
+ synchronized (monitor) {
+ if (state == State.STOP) {
+ throw new IllegalStateException("Cannot re-start a stopped node agent");
+ }
+ }
+ thread.start();
+ }
+
+ @Override
+ public void stop() {
+ logger.log(LogLevel.INFO, logPrefix + "Scheduling stop of NodeAgent");
+ changeStateAndNotify(() -> {
+ if (state == State.STOP) {
+ throw new IllegalStateException("Cannot stop an already stopped node agent");
+ }
+ state = State.STOP;
+ });
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ logger.log(Level.WARNING, logPrefix + "Unexpected interrupt", e);
+ }
+ }
+
+ void synchronizeLocalContainerState(
+ final ContainerNodeSpec nodeSpec,
+ Optional<Container> existingContainer) throws Exception {
+ logger.log(Level.INFO, logPrefix + "Container " + nodeSpec.containerName + " state:" + nodeSpec.nodeState);
+
+ if (nodeSpec.nodeState == NodeState.ACTIVE && !docker.imageIsDownloaded(nodeSpec.wantedDockerImage.get())) {
+ logger.log(LogLevel.INFO, logPrefix + "Schedule async download of Docker image " + nodeSpec.wantedDockerImage.get());
+ final CompletableFuture<DockerImage> asyncPullResult = docker.pullImageAsync(nodeSpec.wantedDockerImage.get());
+ asyncPullResult.whenComplete((dockerImage, throwable) -> {
+ if (throwable != null) {
+ logger.log(
+ Level.WARNING,
+ logPrefix + "Failed to pull docker image " + nodeSpec.wantedDockerImage,
+ throwable);
+ return;
+ }
+ assert nodeSpec.wantedDockerImage.get().equals(dockerImage);
+ scheduleWork();
+ });
+
+ return;
+ }
+
+ if (existingContainer.isPresent()) {
+ Optional<String> removeReason = Optional.empty();
+ if (nodeSpec.nodeState != NodeState.ACTIVE) {
+ removeReason = Optional.of("Node no longer active");
+ } else if (!nodeSpec.wantedDockerImage.get().equals(existingContainer.get().image)) {
+ removeReason = Optional.of("The node is supposed to run a new Docker image: "
+ + existingContainer.get() + " -> " + nodeSpec.wantedDockerImage.get());
+ } else if (nodeSpec.currentRestartGeneration.get() < nodeSpec.wantedRestartGeneration.get()) {
+ removeReason = Optional.of("Restart requested - wanted restart generation has been bumped: "
+ + nodeSpec.currentRestartGeneration.get() + " -> " + nodeSpec.wantedRestartGeneration.get());
+ } else if (!existingContainer.get().isRunning) {
+ removeReason = Optional.of("Container no longer running");
+ }
+
+ if (removeReason.isPresent()) {
+ logger.log(LogLevel.INFO, logPrefix + "Will remove container " + existingContainer.get() + ": "
+ + removeReason.get());
+ removeContainer(nodeSpec, existingContainer.get());
+ existingContainer = Optional.empty(); // Make logic below easier
+ }
+ }
+
+ switch (nodeSpec.nodeState) {
+ case DIRTY: // intentional fall-through
+ case PROVISIONED:
+ logger.log(LogLevel.INFO, logPrefix + "State is " + nodeSpec.nodeState
+ + ", will delete application storage and mark node as ready");
+ docker.deleteApplicationStorage(nodeSpec.containerName);
+ nodeRepository.markAsReady(nodeSpec.hostname);
+ break;
+ case ACTIVE:
+ if (!existingContainer.isPresent()) {
+ logger.log(Level.INFO, logPrefix + "Starting container " + nodeSpec.containerName);
+ // TODO: Properly handle absent min* values
+ docker.startContainer(
+ nodeSpec.wantedDockerImage.get(),
+ nodeSpec.hostname,
+ nodeSpec.containerName,
+ nodeSpec.minCpuCores.get(),
+ nodeSpec.minDiskAvailableGb.get(),
+ nodeSpec.minMainMemoryAvailableGb.get());
+ nodeStarted = false;
+ }
+
+ if (!nodeStarted) {
+ logger.log(Level.INFO, logPrefix + "Starting optional node program " + RESUME_NODE_COMMAND);
+ Optional<ProcessResult> result = executeOptionalProgram(docker, nodeSpec.containerName, RESUME_NODE_COMMAND);
+
+ if (result.isPresent() && !result.get().isSuccess()) {
+ throw new RuntimeException("Container " + nodeSpec.containerName.asString()
+ + ": Command " + Arrays.toString(RESUME_NODE_COMMAND) + " failed: " + result.get());
+ }
+
+ nodeStarted = true;
+ }
+
+ final String containerVespaVersion = nullOnException(() ->
+ docker.getVespaVersion(nodeSpec.containerName));
+
+ // Because it's more important to stop a bad release from rolling out in prod,
+ // we put the resume call last. So if we fail after updating the node repo attributes
+ // but before resume, the app may go through the tenant pipeline but will halt in prod.
+ //
+ // Note that this problem exists only because there are 2 different mechanisms
+ // that should really be parts of a single mechanism:
+ // - The content of node repo is used to determine whether a new Vespa+application
+ // has been successfully rolled out.
+ // - Slobrok and internal orchestrator state is used to determine whether
+ // to allow upgrade (suspend).
+
+ final NodeAttributes currentAttributes = new NodeAttributes(
+ nodeSpec.wantedRestartGeneration.get(),
+ nodeSpec.wantedDockerImage.get(),
+ containerVespaVersion);
+ if (!currentAttributes.equals(lastAttributesSet)) {
+ logger.log(Level.INFO, logPrefix + "Publishing new set of attributes to node repo: "
+ + lastAttributesSet + " -> " + currentAttributes);
+ nodeRepository.updateNodeAttributes(
+ nodeSpec.hostname,
+ currentAttributes.restartGeneration,
+ currentAttributes.dockerImage,
+ currentAttributes.vespaVersion);
+ lastAttributesSet = currentAttributes;
+ }
+
+ logger.log(Level.INFO, logPrefix + "Call resume against Orchestrator");
+ orchestrator.resume(nodeSpec.hostname);
+ break;
+ default:
+ // Nothing to do...
+ }
+ }
+
+ private void removeContainer(final ContainerNodeSpec nodeSpec, final Container existingContainer)
+ throws Exception {
+ final ContainerName containerName = existingContainer.name;
+ if (existingContainer.isRunning) {
+ // If we're stopping the node only to upgrade or restart the node or similar, we need to suspend
+ // the services.
+ if (nodeSpec.nodeState == NodeState.ACTIVE) {
+ // TODO: Also skip orchestration if we're downgrading in test/staging
+ // How to implement:
+ // - test/staging: We need to figure out whether we're in test/staging, by asking Chef!? Or,
+ // let the Orchestrator handle it - it may know what zone we're in.
+ // - downgrading: Impossible to know unless we look at the hosted version, which is
+ // not available in the docker image (nor its name). Not sure how to solve this. Should
+ // the node repo return the hosted version or a downgrade bit in addition to
+ // wanted docker image etc?
+ // Should the tenant pipeline instead use BCP tool to upgrade faster!?
+ //
+ // More generally, the node repo response should contain sufficient info on what the docker image is,
+ // to allow the node admin to make decisions that depend on the docker image. Or, each docker image
+ // needs to contain routines for drain and suspend. For many image, these can just be dummy routines.
+
+ logger.log(Level.INFO, logPrefix + "Ask Orchestrator for permission to suspend node " + nodeSpec.hostname);
+ final boolean suspendAllowed = orchestrator.suspend(nodeSpec.hostname);
+ if (!suspendAllowed) {
+ logger.log(Level.INFO, logPrefix + "Orchestrator rejected suspend of node");
+ // TODO: change suspend() to throw an exception if suspend is denied
+ throw new OrchestratorException("Failed to get permission to suspend " + nodeSpec.hostname);
+ }
+
+ trySuspendNode(containerName);
+ }
+
+ logger.log(Level.INFO, logPrefix + "Stopping container " + containerName);
+ docker.stopContainer(containerName);
+ }
+
+ logger.log(Level.INFO, logPrefix + "Deleting container " + containerName);
+ docker.deleteContainer(containerName);
+ }
+
+ static String[] programExistsCommand(String programPath) {
+ return new String[]{ "/usr/bin/env", "test", "-x", programPath };
+ }
+
+ /**
+ * Executes a program and returns its result, or if it doesn't exist, return a result
+ * as-if the program executed with exit status 0 and no output.
+ */
+ static Optional<ProcessResult> executeOptionalProgram(Docker docker, ContainerName containerName, String... args) {
+ assert args.length > 0;
+ String[] nodeProgramExistsCommand = programExistsCommand(args[0]);
+ if (!docker.executeInContainer(containerName, nodeProgramExistsCommand).isSuccess()) {
+ return Optional.empty();
+ }
+
+ return Optional.of(docker.executeInContainer(containerName, args));
+ }
+
+ /**
+ * Try to suspend node. Suspending a node means the node should be taken offline,
+ * such that maintenance can be done of the node (upgrading, rebooting, etc),
+ * and such that we will start serving again as soon as possible afterwards.
+ *
+ * Any failures are logged and ignored.
+ */
+ private void trySuspendNode(ContainerName containerName) {
+ Optional<ProcessResult> result;
+
+ try {
+ // TODO: Change to waiting w/o timeout (need separate thread that we can stop).
+ result = executeOptionalProgram(docker, containerName, SUSPEND_NODE_COMMAND);
+ } catch (RuntimeException e) {
+ // It's bad to continue as-if nothing happened, but on the other hand if we do not proceed to
+ // remove container, we will not be able to upgrade to fix any problems in the suspend logic!
+ logger.log(LogLevel.WARNING, logPrefix + "Failed trying to suspend node with "
+ + Arrays.toString(SUSPEND_NODE_COMMAND), e);
+ return;
+ }
+
+ if (result.isPresent() && !result.get().isSuccess()) {
+ logger.log(LogLevel.WARNING, logPrefix + "The suspend program " + Arrays.toString(SUSPEND_NODE_COMMAND)
+ + " failed: " + result.get().getOutput());
+ }
+ }
+
+ private static <T> T nullOnException(Supplier<T> supplier) {
+ try {
+ return supplier.get();
+ } catch (RuntimeException e) {
+ logger.log(Level.WARNING, "Ignoring failure", e);
+ return null;
+ }
+ }
+
+ // It somewhat sucks that this class almost duplicates a binding class used by NodeRepositoryImpl,
+ // but using the binding class here would be a layer violation, and would also tie this logic to
+ // serialization-related dependencies it needs not have.
+ private static class NodeAttributes {
+ private final long restartGeneration;
+ private final DockerImage dockerImage;
+ private final String vespaVersion;
+
+ private NodeAttributes(long restartGeneration, DockerImage dockerImage, String vespaVersion) {
+ this.restartGeneration = restartGeneration;
+ this.dockerImage = dockerImage;
+ this.vespaVersion = vespaVersion;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(restartGeneration, dockerImage, vespaVersion);
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (!(o instanceof NodeAttributes)) {
+ return false;
+ }
+ final NodeAttributes other = (NodeAttributes) o;
+
+ return Objects.equals(restartGeneration, other.restartGeneration)
+ && Objects.equals(dockerImage, other.dockerImage)
+ && Objects.equals(vespaVersion, other.vespaVersion);
+ }
+
+ @Override
+ public String toString() {
+ return "NodeAttributes{" +
+ "restartGeneration=" + restartGeneration +
+ ", dockerImage=" + dockerImage +
+ ", vespaVersion='" + vespaVersion + '\'' +
+ '}';
+ }
+ }
+
+ private void scheduleWork() {
+ changeStateAndNotify(() -> state = State.WORK);
+ }
+
+ private void changeStateAndNotify(final Runnable stateChanger) {
+ synchronized (monitor) {
+ if (state == State.STOP) {
+ return;
+ }
+ stateChanger.run();
+ monitor.notifyAll();
+ }
+ }
+
+ private void maintainWantedState() {
+ while (true) {
+ synchronized (monitor) {
+ while (state == State.WAIT) {
+ try {
+ monitor.wait();
+ } catch (InterruptedException e) {
+ // Ignore, properly handled by next loop iteration.
+ }
+ }
+ if (state == State.STOP) {
+ return;
+ }
+ assert state == State.WORK;
+ state = State.WAIT;
+ }
+
+ try {
+ final ContainerNodeSpec nodeSpec = nodeRepository.getContainer(hostname)
+ .orElseThrow(() ->
+ new IllegalStateException(String.format("Node '%s' missing from node repository.", hostname)));
+ final Optional<Container> existingContainer = docker.getContainer(hostname);
+ synchronizeLocalContainerState(nodeSpec, existingContainer);
+ } catch (Exception e) {
+ logger.log(LogLevel.ERROR, logPrefix + "Unhandled exception.", e);
+ }
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/Container.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/Container.java
new file mode 100644
index 00000000000..65929447b0e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/Container.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.docker;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+
+import java.util.Objects;
+
+/**
+ * @author stiankri
+ */
+public class Container {
+ public final HostName hostname;
+ public final DockerImage image;
+ public final ContainerName name;
+ public final boolean isRunning;
+
+ public Container(
+ final HostName hostname,
+ final DockerImage image,
+ final ContainerName containerName,
+ final boolean isRunning) {
+ this.hostname = hostname;
+ this.image = image;
+ this.name = containerName;
+ this.isRunning = isRunning;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (!(obj instanceof Container)) {
+ return false;
+ }
+ final Container other = (Container) obj;
+ return Objects.equals(hostname, other.hostname)
+ && Objects.equals(image, other.image)
+ && Objects.equals(name, other.name)
+ && Objects.equals(isRunning, other.isRunning);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(hostname, image, name, isRunning);
+ }
+
+ @Override
+ public String toString() {
+ return "Container {"
+ + " hostname=" + hostname
+ + " image=" + image
+ + " name=" + name
+ + " isRunning=" + isRunning
+ + "}";
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ContainerName.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ContainerName.java
new file mode 100644
index 00000000000..f9d74593dfa
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ContainerName.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.docker;
+
+import java.util.Objects;
+
+/**
+ * Type-safe value wrapper for docker container names.
+ *
+ * @author bakksjo
+ */
+public class ContainerName {
+ private final String name;
+
+ public ContainerName(final String name) {
+ this.name = Objects.requireNonNull(name);
+ }
+
+ public String asString() {
+ return name;
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (!(o instanceof ContainerName)) {
+ return false;
+ }
+
+ final ContainerName other = (ContainerName) o;
+
+ return Objects.equals(name, other.name);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + " {"
+ + " name=" + name
+ + " }";
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/Docker.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/Docker.java
new file mode 100644
index 00000000000..be24dda2aa4
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/Docker.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.docker;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * @author stiankri
+ */
+public interface Docker {
+
+ void startContainer(DockerImage dockerImage, HostName hostName, ContainerName containerName, double minCpuCores, double minDiskAvailableGb, double minMainMemoryAvailableGb);
+
+ void stopContainer(ContainerName containerName);
+
+ void deleteContainer(ContainerName containerName);
+
+ List<Container> getAllManagedContainers();
+
+ Optional<Container> getContainer(HostName hostname);
+
+ CompletableFuture<DockerImage> pullImageAsync(DockerImage image);
+
+ boolean imageIsDownloaded(DockerImage image);
+
+ void deleteApplicationStorage(ContainerName containerName) throws IOException;
+
+ String getVespaVersion(ContainerName containerName);
+
+ void deleteImage(DockerImage dockerImage);
+
+ /**
+ * Returns the local images that are currently not in use by any container.
+ */
+ Set<DockerImage> getUnusedDockerImages();
+
+ /**
+ * TODO: Make this function interruptible, see https://github.com/spotify/docker-client/issues/421
+ *
+ * @param args Program arguments. args[0] must be the program filename.
+ * @throws RuntimeException (or some subclass thereof) on failure, including docker failure, command failure
+ */
+ ProcessResult executeInContainer(ContainerName containerName, String... args);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImage.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImage.java
new file mode 100644
index 00000000000..f7cd6f76d3e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImage.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.docker;
+
+import java.util.Objects;
+
+/**
+ * Type-safe value wrapper for docker image reference.
+ *
+ * @author bakksjo
+ */
+public class DockerImage {
+ private final String imageId;
+
+ public DockerImage(final String imageId) {
+ this.imageId = Objects.requireNonNull(imageId);
+ }
+
+ public String asString() {
+ return imageId;
+ }
+
+ @Override
+ public int hashCode() {
+ return imageId.hashCode();
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (!(o instanceof DockerImage)) {
+ return false;
+ }
+
+ final DockerImage other = (DockerImage) o;
+
+ return Objects.equals(imageId, other.imageId);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + " {"
+ + " imageId=" + imageId
+ + " }";
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java
new file mode 100644
index 00000000000..ab8198bc9fa
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImpl.java
@@ -0,0 +1,589 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.docker;
+
+import com.google.common.base.Joiner;
+import com.google.common.io.CharStreams;
+import com.spotify.docker.client.ContainerNotFoundException;
+import com.spotify.docker.client.DefaultDockerClient;
+import com.spotify.docker.client.DockerCertificateException;
+import com.spotify.docker.client.DockerCertificates;
+import com.spotify.docker.client.DockerClient;
+import com.spotify.docker.client.DockerClient.ExecCreateParam;
+import com.spotify.docker.client.DockerException;
+import com.spotify.docker.client.LogStream;
+import com.spotify.docker.client.messages.ContainerConfig;
+import com.spotify.docker.client.messages.ContainerInfo;
+import com.spotify.docker.client.messages.ContainerState;
+import com.spotify.docker.client.messages.ExecState;
+import com.spotify.docker.client.messages.HostConfig;
+import com.spotify.docker.client.messages.Image;
+import com.spotify.docker.client.messages.RemovedImage;
+import com.yahoo.log.LogLevel;
+import com.yahoo.nodeadmin.docker.DockerConfig;
+import com.yahoo.vespa.applicationmodel.HostName;
+import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+import com.yahoo.vespa.hosted.node.admin.util.Environment;
+
+import javax.annotation.concurrent.GuardedBy;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+
+/**
+ * @author stiankri
+ */
+public class DockerImpl implements Docker {
+ private static final Logger log = Logger.getLogger(DockerImpl.class.getName());
+
+ private static final int SECONDS_TO_WAIT_BEFORE_KILLING = 10;
+ private static final String FRAMEWORK_CONTAINER_PREFIX = "/";
+ private static final String[] COMMAND_YINST_LS_VESPA = new String[]{"yinst", "ls", "vespa"};
+ private static final Pattern VESPA_PACKAGE_VERSION_PATTERN = Pattern.compile("^vespa-(\\S+)", Pattern.MULTILINE);
+
+ private static final String LABEL_NAME_MANAGEDBY = "com.yahoo.vespa.managedby";
+ private static final String LABEL_VALUE_MANAGEDBY = "node-admin";
+ private static final Map<String,String> CONTAINER_LABELS = new HashMap<>();
+ static {
+ CONTAINER_LABELS.put(LABEL_NAME_MANAGEDBY, LABEL_VALUE_MANAGEDBY);
+ }
+
+ private static final Path RELATIVE_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage");
+ private static final Path APPLICATION_STORAGE_PATH_FOR_NODE_ADMIN = Paths.get("/host").resolve(RELATIVE_APPLICATION_STORAGE_PATH);
+ private static final Path APPLICATION_STORAGE_PATH_FOR_HOST = Paths.get("/").resolve(RELATIVE_APPLICATION_STORAGE_PATH);
+
+ private static final List<String> DIRECTORIES_TO_MOUNT = Arrays.asList(
+ getDefaults().underVespaHome("logs"),
+ getDefaults().underVespaHome("var/cache"),
+ getDefaults().underVespaHome("var/crash"),
+ getDefaults().underVespaHome("var/db/jdisc"),
+ getDefaults().underVespaHome("var/db/vespa"),
+ getDefaults().underVespaHome("var/jdisc_container"),
+ getDefaults().underVespaHome("var/jdisc_core"),
+ getDefaults().underVespaHome("var/logstash-forwarder"),
+ getDefaults().underVespaHome("var/maven"),
+ getDefaults().underVespaHome("var/scoreboards"),
+ getDefaults().underVespaHome("var/service"),
+ getDefaults().underVespaHome("var/share"),
+ getDefaults().underVespaHome("var/spool"),
+ getDefaults().underVespaHome("var/vespa"),
+ getDefaults().underVespaHome("var/yca"),
+ getDefaults().underVespaHome("var/ycore++"),
+ getDefaults().underVespaHome("var/ymon"),
+ getDefaults().underVespaHome("var/zookeeper"));
+
+ private final DockerClient docker;
+
+ private final ExecutorService executor = Executors.newSingleThreadExecutor();
+
+ private final Object monitor = new Object();
+
+ @GuardedBy("monitor")
+ private final Map<DockerImage, CompletableFuture<DockerImage>> scheduledPulls = new HashMap<>();
+
+ public DockerImpl(final DockerClient dockerClient) {
+ this.docker = dockerClient;
+ }
+
+ public static DockerClient newDockerClientFromConfig(final DockerConfig config) {
+ return DefaultDockerClient.builder().
+ uri(config.uri()).
+ dockerCertificates(certificates(config)).
+ readTimeoutMillis(TimeUnit.MINUTES.toMillis(30)). // Some operations may take minutes.
+ build();
+ }
+
+ private static DockerCertificates certificates(DockerConfig config) {
+ try {
+ return DockerCertificates.builder()
+ .caCertPath(Paths.get(config.caCertPath()))
+ .clientCertPath(Paths.get(config.clientCertPath()))
+ .clientKeyPath(Paths.get(config.clientKeyPath()))
+ .build().get();
+ } catch (DockerCertificateException e) {
+ throw new RuntimeException("Failed configuring certificates for contacting docker daemon.", e);
+ }
+ }
+
+ @Override
+ public CompletableFuture<DockerImage> pullImageAsync(final DockerImage image) {
+ // We define the task before we create the CompletableFuture, to ensure that the local future variable cannot
+ // be accessed by the task, forcing it to always go through removeScheduledPoll() before completing the task.
+ final Runnable task = () -> {
+ try {
+ docker.pull(image.asString());
+ removeScheduledPoll(image).complete(image);
+ } catch (InterruptedException e) {
+ removeScheduledPoll(image).completeExceptionally(e);
+ } catch (DockerException e) {
+ if (imageIsDownloaded(image)) {
+ /* TODO: the docker client is not in sync with the server protocol causing it to throw
+ * "java.io.IOException: Stream closed", even if the pull succeeded; thus ignoring here
+ */
+ removeScheduledPoll(image).complete(image);
+ } else {
+ removeScheduledPoll(image).completeExceptionally(e);
+ }
+ } catch (RuntimeException e) {
+ removeScheduledPoll(image).completeExceptionally(e);
+ throw e;
+ }
+ };
+
+ final CompletableFuture<DockerImage> completionListener;
+ synchronized (monitor) {
+ if (scheduledPulls.containsKey(image)) {
+ return scheduledPulls.get(image);
+ }
+ completionListener = new CompletableFuture<>();
+ scheduledPulls.put(image, completionListener);
+ }
+ executor.submit(task);
+ return completionListener;
+ }
+
+ private CompletableFuture<DockerImage> removeScheduledPoll(final DockerImage image) {
+ synchronized (monitor) {
+ return scheduledPulls.remove(image);
+ }
+ }
+
+ /**
+ * Check if a given image is already in the local registry
+ */
+ @Override
+ public boolean imageIsDownloaded(final DockerImage dockerImage) {
+ try {
+ List<Image> images = docker.listImages(DockerClient.ListImagesParam.allImages());
+ return images.stream().
+ flatMap(image -> image.repoTags().stream()).
+ anyMatch(tag -> tag.equals(dockerImage.asString()));
+ } catch (DockerException|InterruptedException e) {
+ throw new RuntimeException("Failed to list image name: '" + dockerImage + "'", e);
+ }
+ }
+
+ @Override
+ public void deleteApplicationStorage(ContainerName containerName) throws IOException {
+ Path applicationStoragePath = applicationStoragePathForNodeAdmin(containerName.asString());
+ if (!Files.exists(applicationStoragePath)) {
+ log.log(LogLevel.INFO, "The application storage at " + applicationStoragePath + " doesn't exist");
+ return;
+ }
+
+ log.log(LogLevel.INFO, "Deleting application storage in " + applicationStoragePath);
+ Files.walkFileTree(applicationStoragePath,
+ new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
+ Files.delete(path);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+ return visitFile(dir, null);
+ }
+ });
+ }
+
+ @Override
+ public void startContainer(
+ final DockerImage dockerImage,
+ final HostName hostName,
+ final ContainerName containerName,
+ double minCpuCores,
+ double minDiskAvailableGb,
+ double minMainMemoryAvailableGb) {
+ try {
+ final double GIGA = Math.pow(2.0, 30.0);
+ // TODO: Enforce disk constraints
+ // TODO: Consider if CPU shares or quoata should be set. For now we are just assuming they are
+ // nicely controlled by docker.
+ ContainerConfig.Builder containerConfigBuilder = ContainerConfig.builder().
+ image(dockerImage.asString()).
+ labels(CONTAINER_LABELS).
+ hostConfig(
+ HostConfig.builder()
+ .networkMode("none")
+ .binds(applicationStorageToMount(containerName.asString()))
+ .build())
+ .env("CONFIG_SERVER_ADDRESS=" + Joiner.on(',').join(Environment.getConfigServerHostsFromYinstSetting())).
+ hostname(hostName.s());
+ if (minMainMemoryAvailableGb > 0.00001) {
+ containerConfigBuilder.memory((long) (GIGA * minMainMemoryAvailableGb));
+ }
+ docker.createContainer(containerConfigBuilder.build(), containerName.asString());
+ //HostConfig hostConfig = HostConfig.builder().create();
+ docker.startContainer(containerName.asString());
+
+ ContainerInfo containerInfo = docker.inspectContainer(containerName.asString());
+ ContainerState state = containerInfo.state();
+
+ if (state.running()) {
+ Integer pid = state.pid();
+ if (pid == null) {
+ throw new DockerException("PID of running container for host " + hostName + " is null");
+ }
+ setupContainerNetworking(containerName, hostName, pid);
+ }
+
+ } catch (IOException | DockerException | InterruptedException e) {
+ throw new RuntimeException("Failed to start container " + containerName.asString(), e);
+ }
+ }
+
+ @Override
+ public String getVespaVersion(final ContainerName containerName) {
+ ProcessResult result = executeInContainer(containerName, COMMAND_YINST_LS_VESPA);
+ if (!result.isSuccess()) {
+ throw new RuntimeException("Container " + containerName.asString() + ": Command "
+ + Arrays.toString(COMMAND_YINST_LS_VESPA) + " failed: " + result);
+ }
+
+ return parseVespaVersion(result.getOutput())
+ .orElseThrow(() -> new RuntimeException(
+ "Container " + containerName.asString() + ": Failed to parse vespa version from "
+ + result.getOutput()));
+ }
+
+ @Override
+ public ProcessResult executeInContainer(ContainerName containerName, String... args) {
+ assert args.length >= 1;
+ try {
+ final String execId = docker.execCreate(
+ containerName.asString(),
+ args,
+ ExecCreateParam.attachStdout(),
+ ExecCreateParam.attachStderr());
+
+ try (final LogStream stream = docker.execStart(execId)) {
+ // This will block until program exits
+ final String output = stream.readFully();
+
+ final ExecState state = docker.execInspect(execId);
+ assert !state.running();
+ Integer exitCode = state.exitCode();
+ assert exitCode != null;
+
+ return new ProcessResult(exitCode, output);
+ }
+ } catch (DockerException | InterruptedException e) {
+ throw new RuntimeException("Container " + containerName.asString()
+ + " failed to execute " + Arrays.toString(args));
+ }
+ }
+
+ // Returns empty if vespa version cannot be parsed.
+ static Optional<String> parseVespaVersion(final String outputFromYinstLsVespa) {
+ final Matcher matcher = VESPA_PACKAGE_VERSION_PATTERN.matcher(outputFromYinstLsVespa);
+ return matcher.find() ? Optional.of(matcher.group(1)) : Optional.empty();
+ }
+
+ private void setupContainerNetworking(ContainerName containerName,
+ HostName hostName,
+ int containerPid) throws UnknownHostException {
+ InetAddress inetAddress = InetAddress.getByName(hostName.s());
+ String ipAddress = inetAddress.getHostAddress();
+
+ final List<String> command = new LinkedList<>();
+ command.add("sudo");
+ command.add(getDefaults().underVespaHome("libexec/vespa/node-admin/configure-container-networking.py"));
+
+ Environment.NetworkType networkType = Environment.networkType();
+ if (networkType != Environment.NetworkType.normal) {
+ command.add("--" + networkType);
+ }
+ command.add(Integer.toString(containerPid));
+ command.add(ipAddress);
+
+ for (int retry = 0; retry < 30; ++retry) {
+ try {
+ runCommand(command);
+ log.log(LogLevel.INFO, "Container " + containerName.asString() + ": Done setting up network");
+ return;
+ } catch (Exception e) {
+ final int sleepSecs = 3;
+ log.log(LogLevel.WARNING, "Container " + containerName.asString()
+ + ": Failed to configure network with command " + command
+ + ", will retry in " + sleepSecs + " seconds", e);
+ try {
+ Thread.sleep(sleepSecs * 1000);
+ } catch (InterruptedException e1) {
+ log.log(LogLevel.WARNING, "Sleep interrupted", e1);
+ }
+ }
+ }
+ }
+
+ private void runCommand(final List<String> command) throws Exception {
+ ProcessBuilder builder = new ProcessBuilder(command);
+ builder.redirectErrorStream(true);
+ Process process = builder.start();
+
+ String output = CharStreams.toString(new InputStreamReader(process.getInputStream()));
+ int resultCode = process.waitFor();
+ if (resultCode != 0) {
+ throw new Exception("Command " + Joiner.on(' ').join(command) + " failed: " + output);
+ }
+ }
+
+ static List<String> applicationStorageToMount(String containerName) {
+ // From-paths when mapping volumes are as seen by the Docker daemon (host)
+ Path destination = applicationStoragePathForHost(containerName);
+
+ return Stream.concat(
+ Stream.of("/etc/hosts:/etc/hosts"),
+ DIRECTORIES_TO_MOUNT.stream()
+ .map(directory -> bindDirective(destination, directory)))
+ .collect(Collectors.toList());
+ }
+
+ private static Path applicationStoragePathForHost(String containerName) {
+ return APPLICATION_STORAGE_PATH_FOR_HOST.resolve(containerName);
+ }
+
+ public static Path applicationStoragePathForNodeAdmin(String containerName) {
+ return APPLICATION_STORAGE_PATH_FOR_NODE_ADMIN.resolve(containerName);
+ }
+
+ private static String bindDirective(Path applicationStorageStorage, String directory) {
+ if (!directory.startsWith("/")) {
+ throw new RuntimeException("Expected absolute path, got " + directory);
+ }
+
+ Path hostPath = applicationStorageStorage.resolve(directory.substring(1));
+ return hostPath.toFile().getAbsolutePath() + ":" + directory;
+ }
+
+ @Override
+ public void stopContainer(final ContainerName containerName) {
+ Optional<com.spotify.docker.client.messages.Container> dockerContainer = getContainerFromName(containerName, true);
+ if (dockerContainer.isPresent()) {
+ try {
+ docker.stopContainer(dockerContainer.get().id(), SECONDS_TO_WAIT_BEFORE_KILLING);
+ } catch (DockerException|InterruptedException e) {
+ throw new RuntimeException("Failed to stop container", e);
+ }
+ }
+ }
+
+ @Override
+ public void deleteContainer(ContainerName containerName) {
+ Optional<com.spotify.docker.client.messages.Container> dockerContainer = getContainerFromName(containerName, true);
+ if (dockerContainer.isPresent()) {
+ try {
+ docker.removeContainer(dockerContainer.get().id());
+ } catch (DockerException|InterruptedException e) {
+ throw new RuntimeException("Failed to delete container", e);
+ }
+ }
+ }
+
+ @Override
+ public List<Container> getAllManagedContainers() {
+ try {
+
+ return docker.listContainers(DockerClient.ListContainersParam.allContainers(true)).stream().
+ filter(this::isManaged).
+ flatMap(this::asContainer).
+ collect(Collectors.toList());
+ } catch (DockerException|InterruptedException e) {
+ throw new RuntimeException("Failed to delete container", e);
+ }
+ }
+
+ @Override
+ public Optional<Container> getContainer(HostName hostname) {
+ // TODO Don't rely on getAllManagedContainers
+ return getAllManagedContainers().stream()
+ .filter(c -> Objects.equals(hostname, c.hostname))
+ .findFirst();
+ }
+
+ private Stream<Container> asContainer(com.spotify.docker.client.messages.Container dockerClientContainer) {
+ try {
+ final ContainerInfo containerInfo = docker.inspectContainer(dockerClientContainer.id());
+ return Stream.of(new Container(
+ new HostName(containerInfo.config().hostname()),
+ new DockerImage(dockerClientContainer.image()),
+ new ContainerName(decode(containerInfo.name())),
+ containerInfo.state().running()));
+ } catch(ContainerNotFoundException e) {
+ return Stream.empty();
+ } catch (InterruptedException|DockerException e) {
+ //TODO: do proper exception handling
+ throw new RuntimeException("Failed talking to docker daemon", e);
+ }
+ }
+
+
+ private Optional<com.spotify.docker.client.messages.Container> getContainerFromName(
+ final ContainerName containerName, final boolean alsoGetStoppedContainers) {
+ try {
+ return docker.listContainers(DockerClient.ListContainersParam.allContainers(alsoGetStoppedContainers)).stream().
+ filter(this::isManaged).
+ filter(container -> matchName(container, containerName.asString())).
+ findFirst();
+ } catch (DockerException|InterruptedException e) {
+ throw new RuntimeException("Failed to get container from name", e);
+ }
+ }
+
+ private boolean isManaged(final com.spotify.docker.client.messages.Container container) {
+ final Map<String, String> labels = container.labels();
+ if (labels == null) {
+ return false;
+ }
+ return LABEL_VALUE_MANAGEDBY.equals(labels.get(LABEL_NAME_MANAGEDBY));
+ }
+
+ private boolean matchName(com.spotify.docker.client.messages.Container container, String targetName) {
+ return container.names().stream().anyMatch(encodedName -> decode(encodedName).equals(targetName));
+ }
+
+ private String decode(String encodedContainerName) {
+ return encodedContainerName.substring(FRAMEWORK_CONTAINER_PREFIX.length());
+ }
+
+ @Override
+ public void deleteImage(final DockerImage dockerImage) {
+ try {
+ log.info("Deleting docker image " + dockerImage);
+ final List<RemovedImage> removedImages = docker.removeImage(dockerImage.asString());
+ for (RemovedImage removedImage : removedImages) {
+ log.info("Result of deleting docker image " + dockerImage + ": " + removedImage);
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException("Unexpected interrupt", e);
+ } catch (DockerException e) {
+ log.log(Level.WARNING, "Could not delete docker image " + dockerImage, e);
+ }
+ }
+
+ @Override
+ public Set<DockerImage> getUnusedDockerImages() {
+ // Description of concepts and relationships:
+ // - a docker image has an id, and refers to its parent image (if any) by image id.
+ // - a docker image may, in addition to id, have multiple tags, but each tag identifies exactly one image.
+ // - a docker container refers to its image (exactly one) either by image id or by image tag.
+ // What this method does to find images considered unused, is build a tree of dependencies
+ // (e.g. container->tag->image->image) and identify image nodes whose only children (if any) are leaf tags.
+ // In other words, an image node with no children, or only tag children having no children themselves is unused.
+ // An image node with an image child is considered used.
+ // An image node with a container child is considered used.
+ // An image node with a tag child with a container child is considered used.
+ try {
+ final Map<String, DockerObject> objects = new HashMap<>();
+ final Map<String, String> dependencies = new HashMap<>();
+
+ // Populate maps with images (including tags) and their dependencies (parents).
+ for (Image image : docker.listImages(DockerClient.ListImagesParam.allImages())) {
+ objects.put(image.id(), new DockerObject(image.id(), DockerObjectType.IMAGE));
+ if (image.parentId() != null && !image.parentId().isEmpty()) {
+ dependencies.put(image.id(), image.parentId());
+ }
+ for (String tag : image.repoTags()) {
+ objects.put(tag, new DockerObject(tag, DockerObjectType.IMAGE_TAG));
+ dependencies.put(tag, image.id());
+ }
+ }
+
+ // Populate maps with containers and their dependency to the image they run on.
+ for (com.spotify.docker.client.messages.Container container : docker.listContainers(DockerClient.ListContainersParam.allContainers(true))) {
+ objects.put(container.id(), new DockerObject(container.id(), DockerObjectType.CONTAINER));
+ dependencies.put(container.id(), container.image());
+ }
+
+ // Now update every object with its dependencies.
+ dependencies.forEach((fromId, toId) -> {
+ Optional.ofNullable(objects.get(toId))
+ .ifPresent(obj -> obj.addDependee(objects.get(fromId)));
+ });
+
+ // Find images that are not in use (i.e. leafs not used by any containers).
+ return objects.values().stream()
+ .filter(dockerObject -> dockerObject.type == DockerObjectType.IMAGE)
+ .filter(dockerObject -> !dockerObject.isInUse())
+ .map(obj -> obj.id)
+ .map(DockerImage::new)
+ .collect(Collectors.toSet());
+ } catch (InterruptedException|DockerException e) {
+ throw new RuntimeException("Unexpected exception", e);
+ }
+ }
+
+ // Helper enum for calculating which images are unused.
+ private enum DockerObjectType {
+ IMAGE_TAG, IMAGE, CONTAINER
+ }
+
+ // Helper class for calculating which images are unused.
+ private static class DockerObject {
+ public final String id;
+ public final DockerObjectType type;
+ private final List<DockerObject> dependees = new LinkedList<>();
+
+ public DockerObject(final String id, final DockerObjectType type) {
+ this.id = id;
+ this.type = type;
+ }
+
+ public boolean isInUse() {
+ if (type == DockerObjectType.CONTAINER) {
+ return true;
+ }
+
+ if (dependees.isEmpty()) {
+ return false;
+ }
+
+ if (type == DockerObjectType.IMAGE) {
+ if (dependees.stream().anyMatch(obj -> obj.type == DockerObjectType.IMAGE)) {
+ return true;
+ }
+ }
+
+ return dependees.stream().anyMatch(DockerObject::isInUse);
+ }
+
+ public void addDependee(final DockerObject dockerObject) {
+ dependees.add(dockerObject);
+ }
+
+ @Override
+ public String toString() {
+ return "DockerObject {"
+ + " id=" + id
+ + " type=" + type.name().toLowerCase()
+ + " inUse=" + isInUse()
+ + " dependees=" + dependees.stream().map(obj -> obj.id).collect(Collectors.toList())
+ + " }";
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java
new file mode 100644
index 00000000000..75d6f641feb
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResult.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.docker;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ProcessResult {
+ private final int exitStatus;
+ private final String output;
+
+ public ProcessResult(int exitStatus, String output) {
+ this.exitStatus = exitStatus;
+ this.output = output;
+ }
+
+ public boolean isSuccess() { return exitStatus == 0; }
+ public int getExitStatus() { return exitStatus; }
+
+ /**
+ * @return The combined stdout and stderr output from the process.
+ */
+ public String getOutput() { return output; }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof ProcessResult)) return false;
+ ProcessResult other = (ProcessResult) o;
+ return Objects.equals(exitStatus, other.exitStatus)
+ && Objects.equals(output, other.output);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(exitStatus, output);
+ }
+
+ @Override
+ public String toString() {
+ return "ProcessResult {"
+ + " exitStatus=" + exitStatus
+ + " output=" + output
+ + " }";
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/package-info.java
new file mode 100644
index 00000000000..9ee1fe58916
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.node.admin.docker;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java
new file mode 100644
index 00000000000..04d68269144
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.noderepository;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * @author stiankri
+ */
+public interface NodeRepository {
+ List<ContainerNodeSpec> getContainersToRun() throws IOException;
+
+ Optional<ContainerNodeSpec> getContainer(HostName hostname) throws IOException;
+
+ void updateNodeAttributes(
+ HostName hostName,
+ long restartGeneration,
+ DockerImage dockerImage,
+ String containerVespaVersion)
+ throws IOException;
+
+ void markAsReady(HostName hostName) throws IOException;
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java
new file mode 100644
index 00000000000..87b5ecd3a93
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java
@@ -0,0 +1,150 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.noderepository;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
+import com.yahoo.vespa.hosted.node.admin.docker.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.GetNodesResponse;
+import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.NodeRepositoryApi;
+import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.UpdateNodeAttributesRequestBody;
+import com.yahoo.vespa.hosted.node.admin.util.Environment;
+import com.yahoo.vespa.jaxrs.client.JaxRsClientFactory;
+import com.yahoo.vespa.jaxrs.client.JaxRsStrategy;
+import com.yahoo.vespa.jaxrs.client.JaxRsStrategyFactory;
+import com.yahoo.vespa.jaxrs.client.JerseyJaxRsClientFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.logging.Logger;
+
+/**
+ * @author stiankri
+ */
+public class NodeRepositoryImpl implements NodeRepository {
+ private static final Logger logger = Logger.getLogger(NodeRepositoryImpl.class.getName());
+ private static final int HARDCODED_NODEREPOSITORY_PORT = 19071;
+ private static final String NODEREPOSITORY_PATH_PREFIX_NODES_API = "/";
+ private static final String ENV_HOSTNAME = "HOSTNAME";
+
+ private JaxRsStrategy<NodeRepositoryApi> nodeRepositoryClient;
+ private final String baseHostName;
+
+ public NodeRepositoryImpl() {
+ baseHostName = Optional.ofNullable(System.getenv(ENV_HOSTNAME))
+ .orElseThrow(() -> new IllegalStateException("Environment variable " + ENV_HOSTNAME + " unset"));
+ nodeRepositoryClient = getApi();
+ }
+
+ // For testing
+ NodeRepositoryImpl(String baseHostName, String configserver, int configport) {
+ this.baseHostName = baseHostName;
+ final Set<HostName> configServerHosts = new HashSet<>();
+ configServerHosts.add(new HostName(configserver));
+
+ final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory();
+ final JaxRsStrategyFactory jaxRsStrategyFactory = new JaxRsStrategyFactory(
+ configServerHosts, configport, jaxRsClientFactory);
+ nodeRepositoryClient = jaxRsStrategyFactory.apiWithRetries(NodeRepositoryApi.class, NODEREPOSITORY_PATH_PREFIX_NODES_API);
+ }
+
+ private static JaxRsStrategy<NodeRepositoryApi> getApi() {
+ final Set<HostName> configServerHosts = Environment.getConfigServerHostsFromYinstSetting();
+ if (configServerHosts.isEmpty()) {
+ throw new IllegalStateException("Environment setting for config servers missing or empty.");
+ }
+ final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory();
+ final JaxRsStrategyFactory jaxRsStrategyFactory = new JaxRsStrategyFactory(
+ configServerHosts, HARDCODED_NODEREPOSITORY_PORT, jaxRsClientFactory);
+ return jaxRsStrategyFactory.apiWithRetries(NodeRepositoryApi.class, NODEREPOSITORY_PATH_PREFIX_NODES_API);
+ }
+
+ @Override
+ public List<ContainerNodeSpec> getContainersToRun() throws IOException {
+ final GetNodesResponse nodesForHost = nodeRepositoryClient.apply(nodeRepositoryApi ->
+ nodeRepositoryApi.getNodesWithParentHost(baseHostName, true));
+
+ if (nodesForHost.nodes == null) {
+ throw new IOException("Response didn't contain nodes element");
+ }
+
+ List<ContainerNodeSpec> nodes = new ArrayList<>(nodesForHost.nodes.size());
+ for (GetNodesResponse.Node node : nodesForHost.nodes) {
+ ContainerNodeSpec nodeSpec;
+ try {
+ nodeSpec = createContainerNodeSpec(node);
+ } catch (IllegalArgumentException | NullPointerException e) {
+ logger.log(LogLevel.WARNING, "Bad node received from node repo when requesting children of the "
+ + baseHostName + " host: " + node, e);
+ continue;
+ }
+
+ nodes.add(nodeSpec);
+ }
+
+ return nodes;
+ }
+
+ @Override
+ public Optional<ContainerNodeSpec> getContainer(HostName hostname) throws IOException {
+ // TODO Use proper call to node repository
+ return getContainersToRun().stream()
+ .filter(cns -> Objects.equals(hostname, cns.hostname))
+ .findFirst();
+ }
+
+ private static ContainerNodeSpec createContainerNodeSpec(GetNodesResponse.Node node)
+ throws IllegalArgumentException, NullPointerException {
+ Objects.requireNonNull(node.nodeState, "Unknown node state");
+ NodeState nodeState = NodeState.valueOf(node.nodeState.toUpperCase());
+ if (nodeState == NodeState.ACTIVE) {
+ Objects.requireNonNull(node.wantedDockerImage, "Unknown docker image for active node");
+ Objects.requireNonNull(node.wantedRestartGeneration, "Unknown wantedRestartGeneration for active node");
+ Objects.requireNonNull(node.currentRestartGeneration, "Unknown currentRestartGeneration for active node");
+ }
+
+ String hostName = Objects.requireNonNull(node.hostname, "hostname is null");
+
+ return new ContainerNodeSpec(
+ new HostName(hostName),
+ Optional.ofNullable(node.wantedDockerImage).map(DockerImage::new),
+ containerNameFromHostName(hostName),
+ nodeState,
+ Optional.ofNullable(node.wantedRestartGeneration),
+ Optional.ofNullable(node.currentRestartGeneration),
+ Optional.ofNullable(node.minCpuCores),
+ Optional.ofNullable(node.minMainMemoryAvailableGb),
+ Optional.ofNullable(node.minDiskAvailableGb));
+ }
+
+ private static ContainerName containerNameFromHostName(final String hostName) {
+ return new ContainerName(hostName.split("\\.")[0]);
+ }
+
+ @Override
+ public void updateNodeAttributes(
+ final HostName hostName,
+ final long restartGeneration,
+ final DockerImage dockerImage,
+ final String currentVespaVersion)
+ throws IOException {
+ // TODO: Filter out redundant (repeated) invocations with the same values.
+ // TODO: Error handling.
+ nodeRepositoryClient.apply(nodeRepositoryApi ->
+ nodeRepositoryApi.updateNodeAttributes(
+ hostName.s(),
+ new UpdateNodeAttributesRequestBody(
+ restartGeneration, dockerImage.asString(), currentVespaVersion)));
+ }
+
+ @Override
+ public void markAsReady(final HostName hostName) throws IOException {
+ nodeRepositoryClient.apply(nodeRepositoryApi -> nodeRepositoryApi.setReady(hostName.s(), ""));
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeState.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeState.java
new file mode 100644
index 00000000000..ca2a0bb9955
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeState.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.noderepository;
+
+// TODO: Unite with com.yahoo.vespa.hosted.provision.Node.State
+public enum NodeState {
+ PROVISIONED, READY, RESERVED, ACTIVE, INACTIVE, DIRTY, FAILED
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetNodesResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetNodesResponse.java
new file mode 100644
index 00000000000..b0b52f911ac
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/GetNodesResponse.java
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.noderepository.bindings;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * This class represents a response from the /nodes/v2/node/ API. It is designed to be
+ * usable by any module, by not depending itself on any module-specific classes.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class GetNodesResponse {
+
+ public final List<Node> nodes;
+
+ @JsonCreator
+ public GetNodesResponse(@JsonProperty("nodes") List<Node> nodes) {
+ this.nodes = Collections.unmodifiableList(nodes);
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Node {
+
+ public final String hostname;
+ public final String wantedDockerImage;
+ public final String currentDockerImage;
+ public final String nodeState;
+ public final Long wantedRestartGeneration;
+ public final Long currentRestartGeneration;
+ public final Double minCpuCores;
+ public final Double minMainMemoryAvailableGb;
+ public final Double minDiskAvailableGb;
+
+ @JsonCreator
+ public Node(@JsonProperty("id") String hostname,
+ @JsonProperty("wantedDockerImage") String wantedDockerImage,
+ @JsonProperty("currentDockerImage") String currentDockerImage,
+ @JsonProperty("state") String nodeState,
+ @JsonProperty("restartGeneration") Long wantedRestartGeneration,
+ @JsonProperty("currentRestartGeneration") Long currentRestartGeneration,
+ @JsonProperty("minCpuCores") Double minCpuCores,
+ @JsonProperty("minMainMemoryAvailableGb") Double minMainMemoryAvailableGb,
+ @JsonProperty("minDiskAvailableGb") Double minDiskAvailableGb) {
+ this.hostname = hostname;
+ this.wantedDockerImage = wantedDockerImage;
+ this.currentDockerImage = currentDockerImage;
+ this.nodeState = nodeState;
+ this.wantedRestartGeneration = wantedRestartGeneration;
+ this.currentRestartGeneration = currentRestartGeneration;
+ this.minCpuCores = minCpuCores;
+ this.minMainMemoryAvailableGb = minMainMemoryAvailableGb;
+ this.minDiskAvailableGb = minDiskAvailableGb;
+ }
+
+ public String toString() {
+ return "Node {"
+ + " containerHostname = " + hostname
+ + " wantedDockerImage = " + wantedDockerImage
+ + " currentDockerImage = " + currentDockerImage
+ + " nodeState = " + nodeState
+ + " wantedRestartGeneration = " + wantedRestartGeneration
+ + " currentRestartGeneration = " + currentRestartGeneration
+ + " minCpuCores = " + minCpuCores
+ + " minMainMemoryAvailableGb = " + minMainMemoryAvailableGb
+ + " minDiskAvailableGb = " + minDiskAvailableGb
+ + " }";
+ }
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java
new file mode 100644
index 00000000000..36ab89a6718
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/NodeRepositoryApi.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.noderepository.bindings;
+
+import com.yahoo.vespa.jaxrs.annotation.PATCH;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+
+/**
+ * @author stiankri
+ */
+public interface NodeRepositoryApi {
+ @GET
+ @Path("/nodes/v2/node/")
+ GetNodesResponse getNodesWithParentHost(
+ @QueryParam("parentHost") String hostname,
+ @QueryParam("recursive") boolean recursive);
+
+ @PUT
+ @Path("/nodes/v2/state/ready/{hostname}")
+ // TODO: remove fake return String body; should be void and empty
+ String setReady(@PathParam("hostname") String hostname, String body);
+
+ @PATCH
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Path("/nodes/v2/node/{nodename}")
+ UpdateNodeAttributesResponse updateNodeAttributes(
+ @PathParam("nodename") String hostname,
+ UpdateNodeAttributesRequestBody body);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesRequestBody.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesRequestBody.java
new file mode 100644
index 00000000000..11bc553607f
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesRequestBody.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.noderepository.bindings;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+/**
+ * Automagically handles (de)serialization based on 1:1 message fields and identifier names.
+ * Instances of this class should serialize as:
+ * <pre>
+ * {
+ * "currentRestartGeneration": 42
+ * }
+ * </pre>
+ *
+ * @author bakksjo
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class UpdateNodeAttributesRequestBody {
+ public Long currentRestartGeneration;
+ public String currentDockerImage;
+ public String currentVespaVersion;
+
+ public UpdateNodeAttributesRequestBody(
+ final Long restartGeneration,
+ final String currentDockerImage,
+ String currentVespaVersion) {
+ this.currentRestartGeneration = restartGeneration;
+ this.currentDockerImage = currentDockerImage;
+ this.currentVespaVersion = currentVespaVersion;
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesResponse.java
new file mode 100644
index 00000000000..9f2c7426875
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/bindings/UpdateNodeAttributesResponse.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.noderepository.bindings;
+
+/**
+ * Automagically handles (de)serialization based on 1:1 message fields and identifier names.
+ * Deserializes JSON strings on the form:
+ * <pre>
+ * {
+ * "message": "Updated host.com"
+ * }
+ * </pre>
+ *
+ * @author bakksjo
+ */
+public class UpdateNodeAttributesResponse {
+ public String message;
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java
new file mode 100644
index 00000000000..00365b5fc3d
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/Orchestrator.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.orchestrator;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+
+/**
+ * Abstraction for communicating with Orchestrator.
+ *
+ * @author bakksjo
+ */
+public interface Orchestrator {
+ /**
+ * Invokes orchestrator suspend of a host. Returns whether suspend was granted.
+ */
+ boolean suspend(HostName hostName);
+
+ /**
+ * Invokes orchestrator resume of a host. Returns whether resume was granted.
+ */
+ boolean resume(HostName hostName);
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorException.java
new file mode 100644
index 00000000000..bc8e671a55e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorException.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.orchestrator;
+
+@SuppressWarnings("serial")
+public class OrchestratorException extends Exception {
+ public OrchestratorException(String message) {
+ super(message);
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java
new file mode 100644
index 00000000000..422962bdc58
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java
@@ -0,0 +1,101 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.orchestrator;
+
+import com.yahoo.vespa.hosted.node.admin.util.Environment;
+import com.yahoo.vespa.jaxrs.client.JaxRsClientFactory;
+import com.yahoo.vespa.jaxrs.client.JaxRsStrategy;
+import com.yahoo.vespa.jaxrs.client.JaxRsStrategyFactory;
+import com.yahoo.vespa.jaxrs.client.JerseyJaxRsClientFactory;
+import com.yahoo.vespa.orchestrator.restapi.HostApi;
+import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
+import com.yahoo.vespa.applicationmodel.HostName;
+
+import javax.ws.rs.ClientErrorException;
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author stiankri
+ * @author bakksjo
+ */
+public class OrchestratorImpl implements Orchestrator {
+ private static final Logger logger = Logger.getLogger(OrchestratorImpl.class.getName());
+ // TODO: Figure out the port dynamically.
+ private static final int HARDCODED_ORCHESTRATOR_PORT = 19071;
+ // TODO: Find a way to avoid duplicating this (present in orchestrator's services.xml also).
+ private static final String ORCHESTRATOR_PATH_PREFIX = "/orchestrator";
+ private static final String ORCHESTRATOR_PATH_PREFIX_HOST_API
+ = ORCHESTRATOR_PATH_PREFIX + HostApi.PATH_PREFIX;
+
+ // We use this to allow client code to treat resume() calls as idempotent and cheap,
+ // but we actually filter out redundant resume calls to orchestrator.
+ private final Set<HostName> resumedHosts = new HashSet<>();
+
+ private final JaxRsStrategy<HostApi> hostApiClient;
+
+ public OrchestratorImpl(JaxRsStrategy<HostApi> hostApiClient) {
+ this.hostApiClient = hostApiClient;
+ }
+
+ @Override
+ public boolean suspend(final HostName hostName) {
+ resumedHosts.remove(hostName);
+ try {
+ return hostApiClient.apply(api -> {
+ final UpdateHostResponse response = api.suspend(hostName.s());
+ return response.reason() == null;
+ });
+ } catch (ClientErrorException e) {
+ if (e instanceof NotFoundException || e.getResponse().getStatus() == Response.Status.NOT_FOUND.getStatusCode()) {
+ // Orchestrator doesn't care about this node, so don't let that stop us.
+ return true;
+ }
+ logger.log(Level.INFO, "Orchestrator rejected suspend request for host " + hostName, e);
+ return false;
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Unable to communicate with orchestrator", e);
+ return false;
+ }
+ }
+
+ @Override
+ public boolean resume(final HostName hostName) {
+ if (resumedHosts.contains(hostName)) {
+ return true;
+ }
+
+ try {
+ final boolean resumeSucceeded = hostApiClient.apply(api -> {
+ final UpdateHostResponse response = api.resume(hostName.s());
+ return response.reason() == null;
+ });
+ if (resumeSucceeded) {
+ resumedHosts.add(hostName);
+ }
+ return resumeSucceeded;
+ } catch (ClientErrorException e) {
+ logger.log(Level.INFO, "Orchestrator rejected resume request for host " + hostName, e);
+ return false;
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Unable to communicate with orchestrator", e);
+ return false;
+ }
+ }
+
+ public static JaxRsStrategy<HostApi> makeOrchestratorHostApiClient() {
+ final Set<HostName> configServerHosts = Environment.getConfigServerHostsFromYinstSetting();
+ if (configServerHosts.isEmpty()) {
+ throw new IllegalStateException("Emnvironment setting for config servers missing or empty.");
+ }
+ final JaxRsClientFactory jaxRsClientFactory = new JerseyJaxRsClientFactory();
+ final JaxRsStrategyFactory jaxRsStrategyFactory = new JaxRsStrategyFactory(
+ configServerHosts, HARDCODED_ORCHESTRATOR_PORT, jaxRsClientFactory);
+ return jaxRsStrategyFactory.apiWithRetries(HostApi.class, ORCHESTRATOR_PATH_PREFIX_HOST_API);
+ }
+
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java
new file mode 100644
index 00000000000..9c2af68d56a
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/NodeAdminRestAPI.java
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.restapi;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+
+/**
+ * @author stiankri
+ */
+@Path("")
+public interface NodeAdminRestAPI {
+ @GET
+ @Path("/update")
+ public String update();
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java
new file mode 100644
index 00000000000..65da2004854
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/testapi/PingResource.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.testapi;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+
+/**
+ * Resource for use in integration test, will be deleted soon.
+ * @author tonytv
+ */
+@Path("ping")
+public class PingResource {
+ @GET
+ @Produces(MediaType.TEXT_PLAIN)
+ public String ping() {
+ return "pong";
+ }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java
new file mode 100644
index 00000000000..d6703326796
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.util;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Various utilities for interacting with node-admin's environment.
+ *
+ * @author <a href="mailto:bakksjo@yahoo-inc.com">Oyvind Bakksjo</a>
+ */
+public class Environment {
+ private Environment() {} // Prevents instantiation.
+
+ private static final String ENV_CONFIGSERVERS = "services__addr_configserver";
+ private static final String ENV_NETWORK_TYPE = "NETWORK_TYPE";
+
+ public enum NetworkType { normal, local, vm }
+
+ public static Set<HostName> getConfigServerHostsFromYinstSetting() {
+ final String yinstSetting = System.getenv(ENV_CONFIGSERVERS);
+ if (yinstSetting == null) {
+ return Collections.emptySet();
+ }
+
+ final List<String> hostNameStrings = Arrays.asList(yinstSetting.split("[,\\s]+"));
+ return hostNameStrings.stream()
+ .map(HostName::new)
+ .collect(Collectors.toSet());
+ }
+
+ public static NetworkType networkType() throws IllegalArgumentException {
+ String networkTypeInEnvironment = System.getenv(ENV_NETWORK_TYPE);
+ if (networkTypeInEnvironment == null) {
+ return NetworkType.normal;
+ }
+ return NetworkType.valueOf(networkTypeInEnvironment);
+ }
+}
diff --git a/node-admin/src/main/resources/configdefinitions/docker.def b/node-admin/src/main/resources/configdefinitions/docker.def
new file mode 100644
index 00000000000..b16a555d764
--- /dev/null
+++ b/node-admin/src/main/resources/configdefinitions/docker.def
@@ -0,0 +1,7 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+namespace=nodeadmin.docker
+
+caCertPath string
+clientCertPath string
+clientKeyPath string
+uri string default = "https://127.0.0.1:2376"
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminTest.java
new file mode 100644
index 00000000000..a4843a0753e
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAdminTest.java
@@ -0,0 +1,194 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.docker.Container;
+import com.yahoo.vespa.hosted.node.admin.docker.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.docker.Docker;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState;
+import org.junit.Test;
+import org.mockito.InOrder;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.Arrays.asList;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author bakksjo
+ */
+public class NodeAdminTest {
+ private static final Optional<Double> MIN_CPU_CORES = Optional.of(1.0);
+ private static final Optional<Double> MIN_MAIN_MEMORY_AVAILABLE_GB = Optional.of(1.0);
+ private static final Optional<Double> MIN_DISK_AVAILABLE_GB = Optional.of(1.0);
+
+ // Trick to allow mocking of typed interface without casts/warnings.
+ private interface NodeAgentFactory extends Function<HostName, NodeAgent> {}
+
+ @Test
+ public void nodeAgentsAreProperlyLifeCycleManaged() throws Exception {
+ final Docker docker = mock(Docker.class);
+ final Function<HostName, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class);
+
+ final NodeAdmin nodeAdmin = new NodeAdmin(docker, nodeAgentFactory);
+
+ final NodeAgent nodeAgent1 = mock(NodeAgentImpl.class);
+ final NodeAgent nodeAgent2 = mock(NodeAgentImpl.class);
+ when(nodeAgentFactory.apply(any(HostName.class))).thenReturn(nodeAgent1).thenReturn(nodeAgent2);
+
+ final HostName hostName = new HostName("host");
+ final DockerImage dockerImage = new DockerImage("image");
+ final ContainerName containerName = new ContainerName("container");
+ final boolean isRunning = true;
+ final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning);
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.ACTIVE,
+ Optional.of(1L),
+ Optional.of(1L),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+
+ final InOrder inOrder = inOrder(nodeAgentFactory, nodeAgent1, nodeAgent2);
+ nodeAdmin.synchronizeLocalContainerState(Collections.emptyList(), asList(existingContainer));
+ verifyNoMoreInteractions(nodeAgentFactory);
+
+ nodeAdmin.synchronizeLocalContainerState(asList(nodeSpec), asList(existingContainer));
+ inOrder.verify(nodeAgentFactory).apply(hostName);
+ inOrder.verify(nodeAgent1).start();
+ inOrder.verify(nodeAgent1).update();
+ inOrder.verify(nodeAgent1, never()).stop();
+
+ nodeAdmin.synchronizeLocalContainerState(asList(nodeSpec), asList(existingContainer));
+ inOrder.verify(nodeAgentFactory, never()).apply(any(HostName.class));
+ inOrder.verify(nodeAgent1, never()).start();
+ inOrder.verify(nodeAgent1).update();
+ inOrder.verify(nodeAgent1, never()).stop();
+
+ nodeAdmin.synchronizeLocalContainerState(Collections.emptyList(), asList(existingContainer));
+ inOrder.verify(nodeAgentFactory, never()).apply(any(HostName.class));
+ inOrder.verify(nodeAgent1, never()).update();
+ verify(nodeAgent1).stop();
+
+ nodeAdmin.synchronizeLocalContainerState(asList(nodeSpec), asList(existingContainer));
+ inOrder.verify(nodeAgentFactory).apply(hostName);
+ inOrder.verify(nodeAgent2).start();
+ inOrder.verify(nodeAgent2).update();
+ inOrder.verify(nodeAgent2, never()).stop();
+
+ nodeAdmin.synchronizeLocalContainerState(Collections.emptyList(), Collections.emptyList());
+ inOrder.verify(nodeAgentFactory, never()).apply(any(HostName.class));
+ inOrder.verify(nodeAgent2, never()).start();
+ inOrder.verify(nodeAgent2, never()).update();
+ inOrder.verify(nodeAgent2).stop();
+
+ verifyNoMoreInteractions(nodeAgent1);
+ verifyNoMoreInteractions(nodeAgent2);
+ }
+
+ private static final DockerImage IMAGE_1 = new DockerImage("image-1");
+ private static final DockerImage IMAGE_2 = new DockerImage("image-2");
+ private static final DockerImage IMAGE_3 = new DockerImage("image-3");
+ private static final DockerImage IMAGE_4 = new DockerImage("image-4");
+
+ @Test
+ public void withNoUnusedImagesNoImagesAreConsideredDeletable() {
+ final Set<DockerImage> currentlyUnusedImages = Collections.emptySet();
+ final List<ContainerNodeSpec> pendingContainers = Collections.emptyList();
+
+ final Set<DockerImage> deletableImages = NodeAdmin.getDeletableDockerImages(currentlyUnusedImages, pendingContainers);
+
+ assertThat(deletableImages, is(Collections.emptySet()));
+ }
+
+ @Test
+ public void withNoPendingContainersAllUnusedImagesAreConsideredDeletable() {
+ final Set<DockerImage> currentlyUnusedImages = Stream.of(IMAGE_1, IMAGE_2, IMAGE_3)
+ .collect(Collectors.toSet());
+ final List<ContainerNodeSpec> pendingContainers = Collections.emptyList();
+
+ final Set<DockerImage> deletableImages = NodeAdmin.getDeletableDockerImages(currentlyUnusedImages, pendingContainers);
+
+ final Set<DockerImage> expectedDeletableImages = Stream.of(IMAGE_1, IMAGE_2, IMAGE_3)
+ .collect(Collectors.toSet());
+ assertThat(deletableImages, is(expectedDeletableImages));
+ }
+
+ @Test
+ public void imagesRequiredByPendingContainersAreNotConsideredDeletable() {
+ final Set<DockerImage> currentlyUnusedImages = Stream.of(IMAGE_1, IMAGE_2, IMAGE_3)
+ .collect(Collectors.toSet());
+ final List<ContainerNodeSpec> pendingContainers = Stream.of(IMAGE_2, IMAGE_4)
+ .map(NodeAdminTest::newNodeSpec)
+ .collect(Collectors.toList());
+
+ final Set<DockerImage> deletableImages = NodeAdmin.getDeletableDockerImages(currentlyUnusedImages, pendingContainers);
+
+ final Set<DockerImage> expectedDeletableImages = Stream.of(IMAGE_1, IMAGE_3)
+ .collect(Collectors.toSet());
+ assertThat(deletableImages, is(expectedDeletableImages));
+ }
+
+ @Test
+ public void fullOuterJoinTest() {
+ final List<String> strings = asList("3", "4", "5", "6", "7", "8", "9", "10");
+ final List<Integer> integers = asList(1, 2, 3, 5, 8, 13, 21);
+ final Set<Pair<Optional<String>, Optional<Integer>>> expectedResult = new HashSet<>(asList(
+ newPair(null, 1),
+ newPair(null, 2),
+ newPair("3", 3),
+ newPair("4", null),
+ newPair("5", 5),
+ newPair("6", null),
+ newPair("7", null),
+ newPair("8", 8),
+ newPair("9", null),
+ newPair("10", null),
+ newPair(null, 13),
+ newPair(null, 21)));
+
+ assertThat(
+ NodeAdmin.fullOuterJoin(
+ strings.stream(), string -> string,
+ integers.stream(), String::valueOf)
+ .collect(Collectors.toSet()),
+ is(expectedResult));
+ }
+
+ private static <T, U> Pair<Optional<T>, Optional<U>> newPair(T t, U u) {
+ return new Pair<>(Optional.ofNullable(t), Optional.ofNullable(u));
+ }
+
+ private static ContainerNodeSpec newNodeSpec(final DockerImage dockerImage) {
+ return new ContainerNodeSpec(
+ new HostName("host-for-" + dockerImage.asString()),
+ Optional.of(dockerImage),
+ new ContainerName("container-for-" + dockerImage.asString()),
+ NodeState.ACTIVE,
+ Optional.of(1L),
+ Optional.of(1L),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImplTest.java
new file mode 100644
index 00000000000..1965da591f4
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/NodeAgentImplTest.java
@@ -0,0 +1,1073 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.docker.Container;
+import com.yahoo.vespa.hosted.node.admin.docker.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.docker.Docker;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
+import com.yahoo.vespa.hosted.node.admin.docker.ProcessResult;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository;
+import com.yahoo.vespa.hosted.node.admin.noderepository.NodeState;
+import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator;
+import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException;
+import org.junit.Test;
+import org.mockito.InOrder;
+
+import java.io.IOException;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyDouble;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.anyVararg;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author bakksjo
+ */
+public class NodeAgentImplTest {
+ private static final Optional<Double> MIN_CPU_CORES = Optional.of(1.0);
+ private static final Optional<Double> MIN_MAIN_MEMORY_AVAILABLE_GB = Optional.of(1.0);
+ private static final Optional<Double> MIN_DISK_AVAILABLE_GB = Optional.of(1.0);
+
+ private static final Optional<Container> NO_CONTAINER = Optional.empty();
+
+ private static final ProcessResult NODE_PROGRAM_DOESNT_EXIST = new ProcessResult(1, "");
+
+ private final HostName hostName = new HostName("hostname");
+ private final Docker docker = mock(Docker.class);
+ private final NodeRepository nodeRepository = mock(NodeRepository.class);
+ private final Orchestrator orchestrator = mock(Orchestrator.class);
+
+ private final NodeAgentImpl nodeAgent = new NodeAgentImpl(hostName, docker, nodeRepository, orchestrator);
+
+ @Test
+ public void upToDateContainerIsUntouched() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.ACTIVE,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = true;
+ final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning);
+ final String vespaVersion = "7.8.9";
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+ when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST);
+ when(docker.getVespaVersion(containerName)).thenReturn(vespaVersion);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ verify(docker, never()).stopContainer(any(ContainerName.class));
+ verify(docker, never()).deleteContainer(any(ContainerName.class));
+ verify(docker, never()).startContainer(
+ any(DockerImage.class),
+ any(HostName.class),
+ any(ContainerName.class),
+ anyDouble(),
+ anyDouble(),
+ anyDouble());
+ verify(docker, times(1)).executeInContainer(any(), anyVararg());
+ final InOrder inOrder = inOrder(orchestrator, nodeRepository);
+ inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage, vespaVersion);
+ inOrder.verify(orchestrator).resume(hostName);
+ }
+
+ @Test
+ public void newRestartGenerationCausesRestart() throws Exception {
+ final long wantedRestartGeneration = 2;
+ final long currentRestartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.ACTIVE,
+ Optional.of(wantedRestartGeneration),
+ Optional.of(currentRestartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = true;
+ final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning);
+ final String vespaVersion = "7.8.9";
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+ when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST);
+ when(docker.getVespaVersion(containerName)).thenReturn(vespaVersion);
+ when(orchestrator.suspend(any(HostName.class))).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+
+ final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository);
+ inOrder.verify(orchestrator).suspend(hostName);
+ inOrder.verify(docker, times(1)).executeInContainer(any(), anyVararg());
+ inOrder.verify(docker).stopContainer(containerName);
+ inOrder.verify(docker).deleteContainer(containerName);
+ inOrder.verify(docker).startContainer(
+ nodeSpec.wantedDockerImage.get(),
+ nodeSpec.hostname,
+ nodeSpec.containerName,
+ nodeSpec.minCpuCores.get(),
+ nodeSpec.minDiskAvailableGb.get(),
+ nodeSpec.minMainMemoryAvailableGb.get());
+ inOrder.verify(docker, times(1)).executeInContainer(any(), anyVararg());
+ inOrder.verify(nodeRepository).updateNodeAttributes(hostName, wantedRestartGeneration, dockerImage, vespaVersion);
+ inOrder.verify(orchestrator).resume(hostName);
+ }
+
+ @Test
+ public void newDockerImageCausesRestart() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage currentDockerImage = new DockerImage("currentDockerImage");
+ final DockerImage wantedDockerImage = new DockerImage("wantedDockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(wantedDockerImage),
+ containerName,
+ NodeState.ACTIVE,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = true;
+ final Container existingContainer = new Container(hostName, currentDockerImage, containerName, isRunning);
+ final String vespaVersion = "7.8.9";
+
+ when(docker.imageIsDownloaded(wantedDockerImage)).thenReturn(true);
+ when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST);;
+ when(docker.getVespaVersion(containerName)).thenReturn(vespaVersion);
+ when(orchestrator.suspend(any(HostName.class))).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+
+ final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository);
+ inOrder.verify(orchestrator).suspend(hostName);
+ inOrder.verify(docker).stopContainer(containerName);
+ inOrder.verify(docker).deleteContainer(containerName);
+ inOrder.verify(docker).startContainer(
+ nodeSpec.wantedDockerImage.get(),
+ nodeSpec.hostname,
+ nodeSpec.containerName,
+ nodeSpec.minCpuCores.get(),
+ nodeSpec.minDiskAvailableGb.get(),
+ nodeSpec.minMainMemoryAvailableGb.get());
+ inOrder.verify(docker, times(1)).executeInContainer(any(), anyVararg());
+ inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, wantedDockerImage, vespaVersion);
+ inOrder.verify(orchestrator).resume(hostName);
+ }
+
+ @Test
+ public void containerIsNotStoppedIfNewImageMustBePulled() throws Exception {
+ final ContainerName containerName = new ContainerName("container");
+ final DockerImage oldDockerImage = new DockerImage("old-image");
+ final DockerImage newDockerImage = new DockerImage("new-image");
+ final long wantedRestartGeneration = 2;
+ final long currentRestartGeneration = 1;
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(newDockerImage),
+ containerName,
+ NodeState.ACTIVE,
+ Optional.of(wantedRestartGeneration),
+ Optional.of(currentRestartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final Container existingContainer = new Container(hostName, oldDockerImage, containerName, true);
+
+ when(docker.imageIsDownloaded(newDockerImage)).thenReturn(false);
+ when(docker.pullImageAsync(newDockerImage)).thenReturn(new CompletableFuture<>());
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+
+ verify(docker, never()).stopContainer(containerName);
+ verify(docker).pullImageAsync(newDockerImage);
+ }
+
+ @Test
+ public void stoppedContainerIsRestarted() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.ACTIVE,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = false;
+ final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning);
+ final String vespaVersion = "7.8.9";
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+ when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST);
+ when(docker.getVespaVersion(containerName)).thenReturn(vespaVersion);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+
+ verify(docker, never()).stopContainer(any(ContainerName.class));
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ verify(docker, times(1)).executeInContainer(any(), anyVararg());
+ final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository);
+ inOrder.verify(docker).deleteContainer(containerName);
+ inOrder.verify(docker).startContainer(
+ nodeSpec.wantedDockerImage.get(),
+ nodeSpec.hostname,
+ nodeSpec.containerName,
+ nodeSpec.minCpuCores.get(),
+ nodeSpec.minDiskAvailableGb.get(),
+ nodeSpec.minMainMemoryAvailableGb.get());
+ inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage, vespaVersion);
+ inOrder.verify(orchestrator).resume(hostName);
+ }
+
+ @Test
+ public void missingContainerIsStarted() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.ACTIVE,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final String vespaVersion = "7.8.9";
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+ when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST);
+ when(docker.getVespaVersion(containerName)).thenReturn(vespaVersion);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, NO_CONTAINER);
+
+ verify(docker, never()).stopContainer(any(ContainerName.class));
+ verify(docker, never()).deleteContainer(any(ContainerName.class));
+ verify(docker, times(1)).executeInContainer(any(), anyVararg());
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository);
+ inOrder.verify(docker).startContainer(
+ nodeSpec.wantedDockerImage.get(),
+ nodeSpec.hostname,
+ nodeSpec.containerName,
+ nodeSpec.minCpuCores.get(),
+ nodeSpec.minDiskAvailableGb.get(),
+ nodeSpec.minMainMemoryAvailableGb.get());
+ inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage, vespaVersion);
+ inOrder.verify(orchestrator).resume(hostName);
+ }
+
+ @Test
+ public void noRestartIfOrchestratorSuspendFails() throws Exception {
+ final long wantedRestartGeneration = 2;
+ final long currentRestartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.ACTIVE,
+ Optional.of(wantedRestartGeneration),
+ Optional.of(currentRestartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = true;
+ final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning);
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+ when(orchestrator.suspend(any(HostName.class))).thenReturn(false);
+
+ try {
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+ fail("permission to suspend should fail so we should never get here");
+ } catch (OrchestratorException e) {
+ // expected
+ }
+
+ verify(orchestrator).suspend(hostName);
+ verify(docker, never()).stopContainer(any(ContainerName.class));
+ verify(docker, never()).deleteContainer(any(ContainerName.class));
+ verify(docker, never()).startContainer(
+ any(DockerImage.class),
+ any(HostName.class),
+ any(ContainerName.class),
+ anyDouble(),
+ anyDouble(),
+ anyDouble());
+ verify(orchestrator, never()).resume(any(HostName.class));
+ verify(nodeRepository, never()).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void failedNodeRunningContainerIsTakenDown() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.FAILED,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = true;
+ final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning);
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ final InOrder inOrder = inOrder(orchestrator, docker);
+ inOrder.verify(docker).stopContainer(containerName);
+ inOrder.verify(docker).deleteContainer(containerName);
+ verify(docker, never()).startContainer(
+ any(DockerImage.class),
+ any(HostName.class),
+ any(ContainerName.class),
+ anyDouble(),
+ anyDouble(),
+ anyDouble());
+ verify(docker, never()).deleteApplicationStorage(any(ContainerName.class));
+ verify(orchestrator, never()).resume(any(HostName.class));
+ verify(nodeRepository, never()).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void failedNodeStoppedContainerIsTakenDown() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.FAILED,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = false;
+ final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning);
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ verify(docker, never()).stopContainer(any(ContainerName.class));
+ verify(docker).deleteContainer(containerName);
+ verify(docker, never()).startContainer(
+ any(DockerImage.class),
+ any(HostName.class),
+ any(ContainerName.class),
+ anyDouble(),
+ anyDouble(),
+ anyDouble());
+ verify(docker, never()).deleteApplicationStorage(any(ContainerName.class));
+ verify(orchestrator, never()).resume(any(HostName.class));
+ verify(nodeRepository, never()).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void failedNodeNoContainerNoActionTaken() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.FAILED,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, NO_CONTAINER);
+
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ verify(docker, never()).stopContainer(any(ContainerName.class));
+ verify(docker, never()).deleteContainer(containerName);
+ verify(docker, never()).startContainer(
+ any(DockerImage.class),
+ any(HostName.class),
+ any(ContainerName.class),
+ anyDouble(),
+ anyDouble(),
+ anyDouble());
+ verify(docker, never()).deleteApplicationStorage(any(ContainerName.class));
+ verify(orchestrator, never()).resume(any(HostName.class));
+ verify(nodeRepository, never()).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void inactiveNodeRunningContainerIsTakenDown() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.INACTIVE,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = true;
+ final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning);
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ final InOrder inOrder = inOrder(orchestrator, docker);
+ inOrder.verify(docker).stopContainer(containerName);
+ inOrder.verify(docker).deleteContainer(containerName);
+ verify(docker, never()).startContainer(
+ any(DockerImage.class),
+ any(HostName.class),
+ any(ContainerName.class),
+ anyDouble(),
+ anyDouble(),
+ anyDouble());
+ verify(docker, never()).deleteApplicationStorage(any(ContainerName.class));
+ verify(orchestrator, never()).resume(any(HostName.class));
+ verify(nodeRepository, never()).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void inactiveNodeStoppedContainerIsTakenDown() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.INACTIVE,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = false;
+ final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning);
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ verify(docker, never()).stopContainer(any(ContainerName.class));
+ verify(docker).deleteContainer(containerName);
+ verify(docker, never()).startContainer(
+ any(DockerImage.class),
+ any(HostName.class),
+ any(ContainerName.class),
+ anyDouble(),
+ anyDouble(),
+ anyDouble());
+ verify(docker, never()).deleteApplicationStorage(any(ContainerName.class));
+ verify(orchestrator, never()).resume(any(HostName.class));
+ verify(nodeRepository, never()).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void inactiveNodeNoContainerNoActionTaken() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.INACTIVE,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, NO_CONTAINER);
+
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ verify(docker, never()).stopContainer(any(ContainerName.class));
+ verify(docker, never()).deleteContainer(any(ContainerName.class));
+ verify(docker, never()).startContainer(
+ any(DockerImage.class),
+ any(HostName.class),
+ any(ContainerName.class),
+ anyDouble(),
+ anyDouble(),
+ anyDouble());
+ verify(docker, never()).deleteApplicationStorage(any(ContainerName.class));
+ verify(orchestrator, never()).resume(any(HostName.class));
+ verify(nodeRepository, never()).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void dirtyNodeRunningContainerIsTakenDownAndCleanedAndRecycled() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.DIRTY,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = true;
+ final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning);
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository);
+ inOrder.verify(docker).stopContainer(containerName);
+ inOrder.verify(docker).deleteContainer(containerName);
+ inOrder.verify(docker).deleteApplicationStorage(containerName);
+ inOrder.verify(nodeRepository).markAsReady(hostName);
+ verify(docker, never()).startContainer(
+ any(DockerImage.class),
+ any(HostName.class),
+ any(ContainerName.class),
+ anyDouble(),
+ anyDouble(),
+ anyDouble());
+ verify(orchestrator, never()).resume(any(HostName.class));
+ verify(nodeRepository, never()).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void dirtyNodeStoppedContainerIsTakenDownAndCleanedAndRecycled() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.DIRTY,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = false;
+ final Container existingContainer = new Container(hostName, dockerImage, containerName, isRunning);
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ verify(docker, never()).stopContainer(any(ContainerName.class));
+ final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository);
+ inOrder.verify(docker).deleteContainer(containerName);
+ inOrder.verify(docker).deleteApplicationStorage(containerName);
+ inOrder.verify(nodeRepository).markAsReady(hostName);
+ verify(docker, never()).startContainer(
+ any(DockerImage.class),
+ any(HostName.class),
+ any(ContainerName.class),
+ anyDouble(),
+ anyDouble(),
+ anyDouble());
+ verify(orchestrator, never()).resume(any(HostName.class));
+ verify(nodeRepository, never()).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void dirtyNodeWithNoContainerIsCleanedAndRecycled() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.DIRTY,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, NO_CONTAINER);
+
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ verify(docker, never()).stopContainer(any(ContainerName.class));
+ verify(docker, never()).deleteContainer(any(ContainerName.class));
+ final InOrder inOrder = inOrder(docker, nodeRepository);
+ inOrder.verify(docker).deleteApplicationStorage(containerName);
+ inOrder.verify(nodeRepository).markAsReady(hostName);
+ verify(docker, never()).startContainer(
+ any(DockerImage.class),
+ any(HostName.class),
+ any(ContainerName.class),
+ anyDouble(),
+ anyDouble(),
+ anyDouble());
+ verify(orchestrator, never()).resume(any(HostName.class));
+ verify(nodeRepository, never()).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void provisionedNodeWithNoContainerIsCleanedAndRecycled() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage = new DockerImage("dockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage),
+ containerName,
+ NodeState.PROVISIONED,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+
+ when(docker.imageIsDownloaded(dockerImage)).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, NO_CONTAINER);
+
+ verify(orchestrator, never()).suspend(any(HostName.class));
+ verify(docker, never()).stopContainer(any(ContainerName.class));
+ verify(docker, never()).deleteContainer(any(ContainerName.class));
+ final InOrder inOrder = inOrder(docker, nodeRepository);
+ inOrder.verify(docker).deleteApplicationStorage(containerName);
+ inOrder.verify(nodeRepository).markAsReady(hostName);
+ verify(docker, never()).startContainer(
+ any(DockerImage.class),
+ any(HostName.class),
+ any(ContainerName.class),
+ anyDouble(),
+ anyDouble(),
+ anyDouble());
+ verify(orchestrator, never()).resume(any(HostName.class));
+ verify(nodeRepository, never()).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void noRedundantNodeRepositoryCalls() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage1 = new DockerImage("dockerImage1");
+ final DockerImage dockerImage2 = new DockerImage("dockerImage2");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec1 = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage1),
+ containerName,
+ NodeState.ACTIVE,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final ContainerNodeSpec nodeSpec2 = new ContainerNodeSpec(
+ nodeSpec1.hostname,
+ Optional.of(dockerImage2),
+ nodeSpec1.containerName,
+ nodeSpec1.nodeState,
+ nodeSpec1.wantedRestartGeneration,
+ nodeSpec1.currentRestartGeneration,
+ nodeSpec1.minCpuCores,
+ nodeSpec1.minMainMemoryAvailableGb,
+ nodeSpec1.minDiskAvailableGb);
+ final boolean isRunning = true;
+ final Container existingContainer1 = new Container(hostName, dockerImage1, containerName, isRunning);
+ final Container existingContainer2 = new Container(hostName, dockerImage2, containerName, isRunning);
+ final String vespaVersion = "7.8.9";
+
+ when(docker.imageIsDownloaded(any(DockerImage.class))).thenReturn(true);
+ when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST);
+ when(docker.getVespaVersion(containerName)).thenReturn(vespaVersion);
+ when(orchestrator.suspend(any(HostName.class))).thenReturn(true);
+
+ final InOrder inOrder = inOrder(nodeRepository, docker);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec1, Optional.of(existingContainer1));
+ inOrder.verify(docker, times(1)).executeInContainer(any(), anyVararg());
+ // Should get exactly one invocation.
+ inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage1, vespaVersion);
+ verify(nodeRepository, times(1)).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec1, Optional.of(existingContainer1));
+ inOrder.verify(docker, never()).executeInContainer(any(), anyVararg());
+ // No attributes have changed; no second invocation should take place.
+ verify(nodeRepository, times(1)).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec2, Optional.of(existingContainer1));
+ inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg());
+ // One attribute has changed, should cause new invocation.
+ inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage2, vespaVersion);
+ verify(nodeRepository, times(2)).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec2, Optional.of(existingContainer2));
+ inOrder.verify(docker, never()).executeInContainer(any(), anyVararg());
+ // No attributes have changed; no new invocation should take place.
+ verify(nodeRepository, times(2)).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec1, Optional.of(existingContainer2));
+ inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg());
+ // Back to previous node spec should also count as new data and cause a new invocation.
+ inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage1, vespaVersion);
+ verify(nodeRepository, times(3)).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void failedNodeRepositoryUpdateIsRetried() throws Exception {
+ final long restartGeneration = 1;
+ final DockerImage dockerImage1 = new DockerImage("dockerImage1");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec1 = new ContainerNodeSpec(
+ hostName,
+ Optional.of(dockerImage1),
+ containerName,
+ NodeState.ACTIVE,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = true;
+ final Container existingContainer = new Container(hostName, dockerImage1, containerName, isRunning);
+ final String vespaVersion = "7.8.9";
+
+ when(docker.imageIsDownloaded(any(DockerImage.class))).thenReturn(true);
+ when(docker.executeInContainer(eq(containerName), anyVararg())).thenReturn(NODE_PROGRAM_DOESNT_EXIST);
+ when(docker.getVespaVersion(containerName)).thenReturn(vespaVersion);
+ when(orchestrator.suspend(any(HostName.class))).thenReturn(true);
+ doThrow(new IOException()).doNothing().when(nodeRepository).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+
+ final InOrder inOrder = inOrder(nodeRepository);
+
+ try {
+ nodeAgent.synchronizeLocalContainerState(nodeSpec1, Optional.of(existingContainer));
+ fail("Should throw exception");
+ } catch (IOException e) {
+ // As expected.
+ }
+ // Should get exactly one invocation.
+ inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage1, vespaVersion);
+ verify(nodeRepository, times(1)).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec1, Optional.of(existingContainer));
+ // First attribute update failed, so it should be retried now.
+ inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, dockerImage1, vespaVersion);
+ verify(nodeRepository, times(2)).updateNodeAttributes(
+ any(HostName.class), anyLong(), any(DockerImage.class), anyString());
+ }
+
+ @Test
+ public void resumeProgramRunsUntilSuccess() throws Exception {
+ final long restartGeneration = 1;
+ final HostName hostName = new HostName("hostname");
+ final DockerImage wantedDockerImage = new DockerImage("wantedDockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(wantedDockerImage),
+ containerName,
+ NodeState.ACTIVE,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final String vespaVersion = "7.8.9";
+ final boolean isRunning = true;
+ final Optional<Container> uptodateContainer = Optional.of(
+ new Container(hostName, wantedDockerImage, containerName, isRunning));
+
+ when(docker.imageIsDownloaded(wantedDockerImage)).thenReturn(true);
+ when(docker.executeInContainer(eq(containerName), anyVararg()))
+ .thenReturn(new ProcessResult(0, "node program exists"))
+ .thenReturn(new ProcessResult(1, "node program fails 1st time"))
+ .thenReturn(new ProcessResult(0, "node program exists"))
+ .thenReturn(new ProcessResult(1, "node program fails 2nd time"))
+ .thenReturn(new ProcessResult(0, "node program exists"))
+ .thenReturn(new ProcessResult(0, "node program succeeds 3rd time"));
+
+ when(docker.getVespaVersion(containerName)).thenReturn(vespaVersion);
+
+ final InOrder inOrder = inOrder(orchestrator, docker);
+
+ // 1st try
+ try {
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, NO_CONTAINER);
+ fail("Should have been a failure");
+ } catch (Exception e) {
+ // expected
+ }
+
+ inOrder.verify(docker).startContainer(
+ nodeSpec.wantedDockerImage.get(),
+ nodeSpec.hostname,
+ nodeSpec.containerName,
+ nodeSpec.minCpuCores.get(),
+ nodeSpec.minDiskAvailableGb.get(),
+ nodeSpec.minMainMemoryAvailableGb.get());
+ inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg());
+ inOrder.verifyNoMoreInteractions();
+
+ // 2nd try
+ try {
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, NO_CONTAINER);
+ fail("Should have been a failure");
+ } catch (Exception e) {
+ // expected
+ }
+
+ inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg());
+ inOrder.verifyNoMoreInteractions();
+
+ // 3rd try success
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, NO_CONTAINER);
+
+ inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg());
+ inOrder.verify(orchestrator).resume(hostName);
+ inOrder.verifyNoMoreInteractions();
+
+ // 4th and 5th times, already started, no calls to executeInContainer
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, uptodateContainer);
+
+ inOrder.verify(docker, never()).executeInContainer(any(), anyVararg());
+ inOrder.verify(orchestrator).resume(hostName);
+ inOrder.verifyNoMoreInteractions();
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, uptodateContainer);
+
+ inOrder.verify(docker, never()).executeInContainer(any(), anyVararg());
+ inOrder.verify(orchestrator).resume(hostName);
+ inOrder.verifyNoMoreInteractions();
+ }
+
+ // The suspend program can fail by returning non-zero exit code, or throw IOException.
+ private enum NodeProgramFailureScenario {
+ EXCEPTION, NODE_PROGRAM_FAILURE}
+
+ private void failSuspendProgram(NodeProgramFailureScenario scenario) throws Exception {
+ final long restartGeneration = 1;
+ final HostName hostName = new HostName("hostname");
+ final DockerImage currentDockerImage = new DockerImage("currentDockerImage");
+ final DockerImage wantedDockerImage = new DockerImage("wantedDockerImage");
+ final ContainerName containerName = new ContainerName("container-name");
+ final ContainerNodeSpec nodeSpec = new ContainerNodeSpec(
+ hostName,
+ Optional.of(wantedDockerImage),
+ containerName,
+ NodeState.ACTIVE,
+ Optional.of(restartGeneration),
+ Optional.of(restartGeneration),
+ MIN_CPU_CORES,
+ MIN_MAIN_MEMORY_AVAILABLE_GB,
+ MIN_DISK_AVAILABLE_GB);
+ final boolean isRunning = true;
+ final Container existingContainer = new Container(hostName, currentDockerImage, containerName, isRunning);
+ final String vespaVersion = "7.8.9";
+
+ when(docker.imageIsDownloaded(wantedDockerImage)).thenReturn(true);
+ switch (scenario) {
+ case EXCEPTION:
+ when(docker.executeInContainer(eq(containerName), anyVararg()))
+ .thenThrow(new RuntimeException()) // suspending node
+ .thenReturn(new ProcessResult(1, "")); // resuming node, node program doesn't exist
+ break;
+ case NODE_PROGRAM_FAILURE:
+ when(docker.executeInContainer(eq(containerName), anyVararg()))
+ .thenReturn(new ProcessResult(0, "")) // program exists
+ .thenReturn(new ProcessResult(1, "error")) // and program fails to suspend
+ .thenReturn(new ProcessResult(0, "")) // program exists
+ .thenReturn(new ProcessResult(0, "output")); // resuming succeeds
+ break;
+ }
+ when(docker.getVespaVersion(containerName)).thenReturn(vespaVersion);
+ when(orchestrator.suspend(any(HostName.class))).thenReturn(true);
+
+ nodeAgent.synchronizeLocalContainerState(nodeSpec, Optional.of(existingContainer));
+
+ final InOrder inOrder = inOrder(orchestrator, docker, nodeRepository);
+ inOrder.verify(orchestrator).suspend(hostName);
+
+ switch (scenario) {
+ case EXCEPTION:
+ inOrder.verify(docker, times(1)).executeInContainer(any(), anyVararg());
+ break;
+ case NODE_PROGRAM_FAILURE:
+ inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg());
+ break;
+ }
+
+ inOrder.verify(docker).stopContainer(containerName);
+ inOrder.verify(docker).deleteContainer(containerName);
+ inOrder.verify(docker).startContainer(
+ nodeSpec.wantedDockerImage.get(),
+ nodeSpec.hostname,
+ nodeSpec.containerName,
+ nodeSpec.minCpuCores.get(),
+ nodeSpec.minDiskAvailableGb.get(),
+ nodeSpec.minMainMemoryAvailableGb.get());
+
+ switch (scenario) {
+ case EXCEPTION:
+ inOrder.verify(docker, times(1)).executeInContainer(any(), anyVararg());
+ break;
+ case NODE_PROGRAM_FAILURE:
+ inOrder.verify(docker, times(2)).executeInContainer(any(), anyVararg());
+ break;
+ }
+
+ inOrder.verify(nodeRepository).updateNodeAttributes(hostName, restartGeneration, wantedDockerImage, vespaVersion);
+ inOrder.verify(orchestrator).resume(hostName);
+ inOrder.verifyNoMoreInteractions();
+ }
+
+ @Test
+ public void suspendExceptionIsIgnored() throws Exception {
+ failSuspendProgram(NodeProgramFailureScenario.EXCEPTION);
+ }
+
+ @Test
+ public void suspendFailureIsIgnored() throws Exception {
+ failSuspendProgram(NodeProgramFailureScenario.NODE_PROGRAM_FAILURE);
+ }
+
+ @Test
+ public void absenceOfNodeProgramIsSuccess() throws Exception {
+ final ContainerName containerName = new ContainerName("container-name");
+ final String programPath = "/bin/command";
+
+ when(docker.executeInContainer(any(), anyVararg())).thenReturn(new ProcessResult(3, "output"));
+
+ Optional<ProcessResult> result = NodeAgentImpl.executeOptionalProgram(
+ docker,
+ containerName,
+ programPath,
+ "arg1",
+ "arg2");
+
+ String[] nodeProgramExistsCommand = NodeAgentImpl.programExistsCommand(programPath);
+ assertThat(nodeProgramExistsCommand.length, is(4));
+
+ verify(docker, times(1)).executeInContainer(
+ eq(containerName),
+ // Mockito fails if we put the array here instead...
+ eq(nodeProgramExistsCommand[0]),
+ eq(nodeProgramExistsCommand[1]),
+ eq(nodeProgramExistsCommand[2]),
+ eq(nodeProgramExistsCommand[3]));
+ assertThat(result.isPresent(), is(false));
+ }
+
+ @Test
+ public void processResultFromNodeProgramWhenPresent() throws Exception {
+ final ContainerName containerName = new ContainerName("container-name");
+ final ProcessResult actualResult = new ProcessResult(3, "output");
+ final String programPath = "/bin/command";
+ final String[] command = new String[] {programPath, "arg"};
+
+ when(docker.executeInContainer(any(), anyVararg()))
+ .thenReturn(new ProcessResult(0, "")) // node program exists
+ .thenReturn(actualResult); // output from node program
+
+ Optional<ProcessResult> result = NodeAgentImpl.executeOptionalProgram(
+ docker,
+ containerName,
+ command);
+
+ String[] nodeProgramExistsCommand = NodeAgentImpl.programExistsCommand(programPath);
+ assertThat(nodeProgramExistsCommand.length, is(4));
+
+ final InOrder inOrder = inOrder(docker);
+ inOrder.verify(docker, times(1)).executeInContainer(
+ eq(containerName),
+ // Mockito fails if we put the array here instead...
+ eq(nodeProgramExistsCommand[0]),
+ eq(nodeProgramExistsCommand[1]),
+ eq(nodeProgramExistsCommand[2]),
+ eq(nodeProgramExistsCommand[3]));
+ inOrder.verify(docker, times(1)).executeInContainer(
+ eq(containerName),
+ eq(command[0]),
+ eq(command[1]));
+
+ assertThat(result.isPresent(), is(true));
+ assertThat(result.get(), is(actualResult));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java
new file mode 100644
index 00000000000..06c3a260790
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerImplTest.java
@@ -0,0 +1,322 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.docker;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.spotify.docker.client.DockerClient;
+import com.spotify.docker.client.LogStream;
+import com.spotify.docker.client.ObjectMapperProvider;
+import com.spotify.docker.client.messages.ExecState;
+import com.spotify.docker.client.messages.Image;
+import com.yahoo.vespa.defaults.Defaults;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.sameInstance;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyVararg;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author tonytv
+ */
+public class DockerImplTest {
+ @Test
+ public void data_directories_are_mounted_in_from_the_host() {
+ List<String> binds = DockerImpl.applicationStorageToMount("my-container");
+
+ String dataDirectory = Defaults.getDefaults().vespaHome() + "logs";
+ String directoryOnHost = "/home/docker/container-storage/my-container" + dataDirectory;
+ assertThat(binds, hasItem(directoryOnHost + ":" + dataDirectory));
+ }
+
+ @Test
+ public void locationOfContainerStorageInNodeAdmin() {
+ assertEquals(
+ "/host/home/docker/container-storage/docker1-1",
+ DockerImpl.applicationStoragePathForNodeAdmin("docker1-1").toString());
+ }
+
+ @Test
+ public void repeatedPollsOfSameImageAreNotScheduled() throws Exception {
+ final DockerClient dockerClient = mock(DockerClient.class);
+ final CountDownLatch latch = new CountDownLatch(1);
+ doAnswer(invocationOnMock -> {
+ latch.await();
+ return null;
+ }).when(dockerClient).pull(any(String.class));
+ final Docker docker = new DockerImpl(dockerClient);
+ final DockerImage dockerImage = new DockerImage("test-image");
+
+ final CompletableFuture<DockerImage> asyncPollFuture1 = docker.pullImageAsync(dockerImage);
+
+ assertThat(asyncPollFuture1.isDone(), is(false));
+
+ final CompletableFuture<DockerImage> asyncPollFuture2 = docker.pullImageAsync(dockerImage);
+
+ assertThat(asyncPollFuture2, is(sameInstance(asyncPollFuture1)));
+
+ latch.countDown(); // This should make the future complete shortly.
+ asyncPollFuture1.get(5, TimeUnit.SECONDS);
+
+ // Now that the previous poll is completed, a new poll may be scheduled.
+ final CompletableFuture<DockerImage> asyncPollFuture3 = docker.pullImageAsync(dockerImage);
+
+ assertThat(asyncPollFuture3, is(not(sameInstance(asyncPollFuture1))));
+ }
+
+ @Test
+ public void vespaVersionIsParsed() {
+ assertThat(DockerImpl.parseVespaVersion("vespa-5.119.53"), is(Optional.of("5.119.53")));
+ }
+
+ @Test
+ public void vespaVersionIsParsedWithTrailingNewline() {
+ assertThat(DockerImpl.parseVespaVersion("vespa-5.119.53\n"), is(Optional.of("5.119.53")));
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void vespaVersionIsNotParsedFromNull() {
+ assertThat(DockerImpl.parseVespaVersion(null), is(Optional.empty()));
+ }
+
+ @Test
+ public void vespaVersionIsNotParsedFromEmptyString() {
+ assertThat(DockerImpl.parseVespaVersion(""), is(Optional.empty()));
+ }
+
+ @Test
+ public void vespaVersionIsNotParsedFromUnexpectedContent() {
+ assertThat(DockerImpl.parseVespaVersion("honda-5.119.53"), is(Optional.empty()));
+ assertThat(DockerImpl.parseVespaVersion("vespa 5.119.53"), is(Optional.empty()));
+ assertThat(DockerImpl.parseVespaVersion("vespa- 5.119.53"), is(Optional.empty()));
+ assertThat(DockerImpl.parseVespaVersion("vespa-"), is(Optional.empty()));
+ assertThat(DockerImpl.parseVespaVersion("No such command 'yinst'"), is(Optional.empty()));
+ }
+
+ @Test
+ public void testExecuteCompletes() throws Exception {
+ final DockerClient dockerClient = mock(DockerClient.class);
+ final String containerId = "container-id";
+ final String[] command = new String[] {"/bin/ls", "-l"};
+ final String execId = "exec-id";
+ when(dockerClient.execCreate(
+ eq(containerId),
+ eq(command),
+ anyVararg(),
+ anyVararg())).thenReturn(execId);
+
+ final LogStream logStream = mock(LogStream.class);
+ when(dockerClient.execStart(execId)).thenReturn(logStream);
+
+ final ExecState execState = mock(ExecState.class);
+ when(dockerClient.execInspect(execId)).thenReturn(execState);
+
+ when(execState.running()).thenReturn(false);
+ final int exitCode = 3;
+ when(execState.exitCode()).thenReturn(exitCode);
+
+ final String commandOutput = "command output";
+ when(logStream.readFully()).thenReturn(commandOutput);
+
+ final Docker docker = new DockerImpl(dockerClient);
+ final ProcessResult result = docker.executeInContainer(new ContainerName(containerId), command);
+ assertThat(result.getExitStatus(), is(exitCode));
+ assertThat(result.getOutput(), is(commandOutput));
+ }
+
+ @Test
+ public void noImagesMeansNoUnusedImages() throws Exception {
+ ImageGcTester
+ .withExistingImages()
+ .expectUnusedImages();
+ }
+
+ @Test
+ public void singleImageWithoutContainersIsUnused() throws Exception {
+ ImageGcTester
+ .withExistingImages(new ImageBuilder("image-1"))
+ .expectUnusedImages("image-1");
+ }
+
+ @Test
+ public void singleImageWithContainerIsUsed() throws Exception {
+ ImageGcTester
+ .withExistingImages(ImageBuilder.forId("image-1"))
+ .andExistingContainers(ContainerBuilder.forId("container-1").withImage("image-1"))
+ .expectUnusedImages();
+ }
+
+ @Test
+ public void onlyLeafImageIsUnused() throws Exception {
+ ImageGcTester
+ .withExistingImages(
+ ImageBuilder.forId("parent-image"),
+ ImageBuilder.forId("leaf-image").withParentId("parent-image"))
+ .expectUnusedImages("leaf-image");
+ }
+
+ @Test
+ public void multipleUnusedImagesAreIdentified() throws Exception {
+ ImageGcTester
+ .withExistingImages(
+ ImageBuilder.forId("image-1"),
+ ImageBuilder.forId("image-2"))
+ .expectUnusedImages("image-1", "image-2");
+ }
+
+ @Test
+ public void multipleUnusedLeavesAreIdentified() throws Exception {
+ ImageGcTester
+ .withExistingImages(
+ ImageBuilder.forId("parent-image"),
+ ImageBuilder.forId("image-1").withParentId("parent-image"),
+ ImageBuilder.forId("image-2").withParentId("parent-image"))
+ .expectUnusedImages("image-1", "image-2");
+ }
+
+ @Test
+ public void unusedLeafWithUsedSiblingIsIdentified() throws Exception {
+ ImageGcTester
+ .withExistingImages(
+ ImageBuilder.forId("parent-image"),
+ ImageBuilder.forId("image-1").withParentId("parent-image"),
+ ImageBuilder.forId("image-2").withParentId("parent-image"))
+ .andExistingContainers(ContainerBuilder.forId("vespa-node-1").withImage("image-1"))
+ .expectUnusedImages("image-2");
+ }
+
+ @Test
+ public void containerCanReferToImageByTag() throws Exception {
+ ImageGcTester
+ .withExistingImages(ImageBuilder.forId("image-1").withTag("vespa-6"))
+ .andExistingContainers(ContainerBuilder.forId("vespa-node-1").withImage("vespa-6"))
+ .expectUnusedImages();
+ }
+
+ @Test
+ public void taggedImageWithNoContainersIsUnused() throws Exception {
+ ImageGcTester
+ .withExistingImages(ImageBuilder.forId("image-1").withTag("vespa-6"))
+ .expectUnusedImages("image-1");
+ }
+
+ private static class ImageGcTester {
+ private final List<Image> existingImages;
+ private List<com.spotify.docker.client.messages.Container> existingContainers = Collections.emptyList();
+
+ private ImageGcTester(final List<Image> images) {
+ this.existingImages = images;
+ }
+
+ public static ImageGcTester withExistingImages(final ImageBuilder... images) {
+ final List<Image> existingImages = Arrays.stream(images)
+ .map(ImageBuilder::toImage)
+ .collect(Collectors.toList());
+ return new ImageGcTester(existingImages);
+ }
+
+ public ImageGcTester andExistingContainers(final ContainerBuilder... containers) {
+ this.existingContainers = Arrays.stream(containers)
+ .map(ContainerBuilder::toContainer)
+ .collect(Collectors.toList());
+ return this;
+ }
+
+ public void expectUnusedImages(final String... imageIds) throws Exception {
+ final DockerClient dockerClient = mock(DockerClient.class);
+ final Docker docker = new DockerImpl(dockerClient);
+ when(dockerClient.listImages(anyVararg())).thenReturn(existingImages);
+ when(dockerClient.listContainers(anyVararg())).thenReturn(existingContainers);
+ final Set<DockerImage> expectedUnusedImages = Arrays.stream(imageIds)
+ .map(DockerImage::new)
+ .collect(Collectors.toSet());
+ assertThat(
+ docker.getUnusedDockerImages(),
+ is(expectedUnusedImages));
+
+ }
+ }
+
+ /**
+ * Serializes object to a JSON string using Jackson, then deserializes it to an instance of toClass
+ * (again using Jackson). This can be used to create Jackson classes with no public constructors.
+ * @throws IllegalArgumentException if Jackson fails to serialize or deserialize.
+ */
+ private static <T> T createFrom(Class<T> toClass, Object object) throws IllegalArgumentException {
+ final String serialized;
+ try {
+ serialized = new ObjectMapper().writeValueAsString(object);
+ } catch (JsonProcessingException e) {
+ throw new IllegalArgumentException("Failed to serialize object " + object + " to "
+ + toClass + " with Jackson: " + e, e);
+ }
+ try {
+ return new ObjectMapperProvider().getContext(null).readValue(serialized, toClass);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Failed to convert " + serialized + " to "
+ + toClass + " with Jackson: " + e, e);
+ }
+ }
+
+ // Workaround for Image class that can't be instantiated directly in Java (instantiate via Jackson instead).
+ private static class ImageBuilder {
+ // Json property names must match exactly the property names in the Image class.
+ @JsonProperty("Id")
+ private final String id;
+
+ @JsonProperty("ParentId")
+ private String parentId;
+
+ @JsonProperty("RepoTags")
+ private final List<String> repoTags = new LinkedList<>();
+
+ private ImageBuilder(String id) { this.id = id; }
+
+ public static ImageBuilder forId(String id) { return new ImageBuilder(id); }
+ public ImageBuilder withParentId(String parentId) { this.parentId = parentId; return this; }
+ public ImageBuilder withTag(String tag) { this.repoTags.add(tag); return this; }
+
+ public Image toImage() { return createFrom(Image.class, this); }
+ }
+
+ // Workaround for Container class that can't be instantiated directly in Java (instantiate via Jackson instead).
+ private static class ContainerBuilder {
+ // Json property names must match exactly the property names in the Container class.
+ @JsonProperty("Id")
+ private final String id;
+
+ @JsonProperty("Image")
+ private String image;
+
+ private ContainerBuilder(String id) { this.id = id; }
+ private static ContainerBuilder forId(final String id) { return new ContainerBuilder(id); }
+ public ContainerBuilder withImage(String image) { this.image = image; return this; }
+
+ public com.spotify.docker.client.messages.Container toContainer() {
+ return createFrom(com.spotify.docker.client.messages.Container.class, this);
+ }
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResultTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResultTest.java
new file mode 100644
index 00000000000..ea145dafb01
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/ProcessResultTest.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.docker;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+public class ProcessResultTest {
+ @Test
+ public void testBasicProperties() throws Exception {
+ ProcessResult processResult = new ProcessResult(0, "foo");
+ assertThat(processResult.getExitStatus(), is(0));
+ assertThat(processResult.getOutput(), is("foo"));
+ assertThat(processResult.isSuccess(), is(true));
+ }
+
+ @Test
+ public void testSuccessFails() throws Exception {
+ ProcessResult processResult = new ProcessResult(1, "foo");
+ assertThat(processResult.isSuccess(), is(false));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java
new file mode 100644
index 00000000000..633405cc5f7
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.noderepository;
+
+import com.yahoo.application.Networking;
+import com.yahoo.application.container.JDisc;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
+import com.yahoo.vespa.hosted.node.admin.docker.ContainerName;
+import com.yahoo.vespa.hosted.node.admin.docker.DockerImage;
+import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+
+public class NodeRepositoryImplTest {
+ private JDisc container;
+ private int port;
+
+ private int findRandomOpenPort() throws IOException {
+ try (ServerSocket socket = new ServerSocket(0)) {
+ return socket.getLocalPort();
+ }
+ }
+
+ /**
+ * Starts NodeRepository with
+ * com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavor
+ * com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository
+ * com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler
+ * These classes define some test data that is used in these tests.
+ */
+ @Before
+ public void startContainer() throws Exception {
+ port = findRandomOpenPort();
+ container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port), Networking.enable);
+ }
+
+ private void waitForJdiscContainerToServe() throws InterruptedException {
+ Instant start = Instant.now();
+ NodeRepository nodeRepositoryApi = new NodeRepositoryImpl("foobar", "127.0.0.1", port);
+ while (Instant.now().minusSeconds(120).isBefore(start)) {
+ try {
+ nodeRepositoryApi.getContainersToRun();
+ return;
+ } catch (Exception e) {
+ Thread.sleep(100);
+ }
+ }
+ throw new RuntimeException("Could not get answer from container.");
+ }
+
+ @After
+ public void stopContainer() {
+ if (container != null) {
+ container.close();
+ }
+ }
+
+
+ @Test
+ public void testGetContainersToRunAPi() throws IOException, InterruptedException {
+ waitForJdiscContainerToServe();
+ NodeRepository nodeRepositoryApi = new NodeRepositoryImpl("dockerhost4", "127.0.0.1", port);
+ final List<ContainerNodeSpec> containersToRun = nodeRepositoryApi.getContainersToRun();
+ assertThat(containersToRun.size(), is(1));
+ final ContainerNodeSpec nodeSpec = containersToRun.get(0);
+ assertThat(nodeSpec.hostname, is(new HostName("host4.yahoo.com")));
+ assertThat(nodeSpec.wantedDockerImage.get(), is(new DockerImage("image-123")));
+ assertThat(nodeSpec.containerName, is(new ContainerName("host4")));
+ assertThat(nodeSpec.nodeState, is(NodeState.RESERVED));
+ assertThat(nodeSpec.wantedRestartGeneration.get(), is(0L));
+ assertThat(nodeSpec.currentRestartGeneration.get(), is(0L));
+ assertThat(nodeSpec.minCpuCores.get(), is(2.0));
+ assertThat(nodeSpec.minMainMemoryAvailableGb.get(), is(16.0));
+ assertThat(nodeSpec.minDiskAvailableGb.get(), is(400.0));
+ }
+
+ @Test
+ public void testGetContainers() throws InterruptedException, IOException {
+ waitForJdiscContainerToServe();
+ NodeRepository nodeRepositoryApi = new NodeRepositoryImpl("dockerhost4", "127.0.0.1", port);
+ HostName hostname = new HostName("host4.yahoo.com");
+ Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainer(hostname);
+ assertThat(nodeSpec.isPresent(), is(true));
+ assertThat(nodeSpec.get().hostname, is(hostname));
+ assertThat(nodeSpec.get().containerName, is(new ContainerName("host4")));
+ }
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java
new file mode 100644
index 00000000000..f5c5601d661
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.orchestrator;
+
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.jaxrs.client.JaxRsStrategy;
+import com.yahoo.vespa.jaxrs.client.LocalPassThroughJaxRsStrategy;
+import com.yahoo.vespa.orchestrator.restapi.HostApi;
+import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
+import org.junit.Test;
+
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author bakksjo
+ */
+public class OrchestratorImplTest {
+ @Test
+ public void redundantResumesAreFilteredOut() throws Exception {
+ final HostApi hostApi = mock(HostApi.class);
+ final JaxRsStrategy<HostApi> hostApiClient = new LocalPassThroughJaxRsStrategy<>(hostApi);
+ final OrchestratorImpl orchestrator = new OrchestratorImpl(hostApiClient);
+ final String hostNameString = "host";
+ final HostName hostName = new HostName(hostNameString);
+
+ // Make resume and suspend always succeed.
+ when(hostApi.resume(hostNameString)).thenReturn(new UpdateHostResponse(hostNameString, null));
+ when(hostApi.suspend(hostNameString)).thenReturn(new UpdateHostResponse(hostNameString, null));
+
+ orchestrator.resume(hostName);
+ verify(hostApi, times(1)).resume(hostNameString);
+
+ // A subsequent resume does not cause a network trip.
+ orchestrator.resume(hostName);
+ verify(hostApi, times(1)).resume(anyString());
+
+ orchestrator.suspend(hostName);
+ verify(hostApi, times(1)).suspend(hostNameString);
+
+ orchestrator.resume(hostName);
+ verify(hostApi, times(2)).resume(hostNameString);
+
+ // A subsequent resume does not cause a network trip.
+ orchestrator.resume(hostName);
+ verify(hostApi, times(2)).resume(anyString());
+ }
+}