@@ -49,6 +49,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

     "os"                => "fedora",

     "dev_cluster"       => false,

     "dind_dev_cluster"  => ENV['OPENSHIFT_DIND_DEV_CLUSTER'] || false,

+    "network_plugin"    => ENV['OPENSHIFT_NETWORK_PLUGIN'] || ENV['OPENSHIFT_SDN'] || "",

     "insert_key"        => true,

     "num_minions"       => ENV['OPENSHIFT_NUM_MINIONS'] || 2,

     "rebuild_yum_cache" => false,

@@ -136,13 +137,17 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

     minion_ips = num_minion.times.collect { |n| minion_ip_base + "#{n+3}" }

     minion_ips_str = minion_ips.join(",")

-    fixup_net_udev = vagrant_openshift_config['fixup_net_udev']

+    fixup_net_udev = ''

+    if vagrant_openshift_config['fixup_net_udev']

+      fixup_net_udev = '-f'

+    end

+    network_plugin = vagrant_openshift_config['network_plugin']

     # OpenShift master

     config.vm.define "#{VM_NAME_PREFIX}master" do |config|

       config.vm.box = kube_box[kube_os]["name"]

       config.vm.box_url = kube_box[kube_os]["box_url"]

-      config.vm.provision "shell", inline: "/vagrant/contrib/vagrant/provision-master.sh #{master_ip} #{num_minion} #{minion_ips_str} #{instance_prefix} #{fixup_net_udev} #{ENV['OPENSHIFT_SDN']}"

+      config.vm.provision "shell", inline: "/bin/bash -x /vagrant/contrib/vagrant/provision-master.sh #{master_ip} #{num_minion} #{minion_ips_str} #{instance_prefix} -n '#{network_plugin}' #{fixup_net_udev}"

       config.vm.network "private_network", ip: "#{master_ip}"

       config.vm.hostname = "openshift-master"

       config.vm.synced_folder ".", "/vagrant", type: vagrant_openshift_config['sync_folders_type']

@@ -155,7 +160,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

         minion_ip = minion_ips[n]

         minion.vm.box = kube_box[kube_os]["name"]

         minion.vm.box_url = kube_box[kube_os]["box_url"]

-        minion.vm.provision "shell", inline: "/vagrant/contrib/vagrant/provision-minion.sh #{master_ip} #{num_minion} #{minion_ips_str} #{instance_prefix} #{minion_ip} #{minion_index} #{fixup_net_udev}"

+        minion.vm.provision "shell", inline: "/bin/bash -x /vagrant/contrib/vagrant/provision-node.sh #{master_ip} #{num_minion} #{minion_ips_str} #{instance_prefix} -n '#{network_plugin}' -i #{minion_index} #{fixup_net_udev}"

         minion.vm.network "private_network", ip: "#{minion_ip}"

         minion.vm.hostname = "openshift-minion-#{minion_index}"

         config.vm.synced_folder ".", "/vagrant", type: vagrant_openshift_config['sync_folders_type']

@@ -1,47 +1,47 @@

 #!/bin/bash

 os::util::install-sdn() {

+  local default_target="/usr"

+

   local deployed_root=$1

-  local target=$2

-  target=${target:-/usr}

+  local target=${2:-${default_target}}

+

   if [ ! -d ${target} ]; then

     mkdir -p ${target}

   fi

-  # Source scripts from an openshift-sdn repo if present to support

-  # openshift-sdn development.

-  local sdn_root="${deployed_root}/third-party/openshift-sdn"

-  if [ -d "${sdn_root}" ]; then

-    pushd "${sdn_root}" > /dev/null

-    # TODO: Enable these commands once we have a separate binary for openshift-sdn

-    # make

-    # make "install-dev"

-    popd > /dev/null

-  else

-    local osdn_base_path="${deployed_root}/Godeps/_workspace/src/github.com/openshift/openshift-sdn"

-    local osdn_controller_path="${osdn_base_path}/pkg/ovssubnet/controller"

-    pushd "${osdn_controller_path}" > /dev/null

-      # The subnet plugin is discovered via the kube network plugin path.

-      local kube_osdn_path="${target}/libexec/kubernetes/kubelet-plugins/net/exec/redhat~openshift-ovs-subnet"

-      mkdir -p "${kube_osdn_path}"

-      mkdir -p "${target}/bin/"

-      cp -f kube/bin/openshift-ovs-subnet "${kube_osdn_path}/"

-      cp -f kube/bin/openshift-sdn-kube-subnet-setup.sh "${target}/bin/"

-

-      # The multitenant plugin only needs to be in PATH because the

-      # origin multitenant plugin knows how to discover it.

-      cp -f multitenant/bin/openshift-ovs-multitenant "${target}/bin/"

-      cp -f multitenant/bin/openshift-sdn-multitenant-setup.sh "${target}/bin/"

-

-      # subnet and multitenant plugin setup writes docker network options

-      # to /run/openshift-sdn/docker-network, make this file to be exported

-      # as part of docker service start.

-      local system_docker_path="${target}/lib/systemd/system/docker.service.d/"

-      mkdir -p "${system_docker_path}"

-      cat <<EOF > "${system_docker_path}/docker-sdn-ovs.conf"

+

+  local osdn_base_path="${deployed_root}/Godeps/_workspace/src/github.com/openshift/openshift-sdn"

+  local osdn_controller_path="${osdn_base_path}/pkg/ovssubnet/controller"

+  local kube_osdn_path="${target}/libexec/kubernetes/kubelet-plugins/net/exec/redhat~openshift-ovs-subnet"

+  mkdir -p "${kube_osdn_path}"

+  mkdir -p "${target}/bin/"

+

+  pushd "${osdn_controller_path}" > /dev/null

+    # The subnet plugin is discovered via the kube network plugin path.

+    cp -f kube/bin/openshift-ovs-subnet "${kube_osdn_path}/"

+    cp -f kube/bin/openshift-sdn-kube-subnet-setup.sh "${target}/bin/"

+

+    # The multitenant plugin only needs to be in PATH because the

+    # origin multitenant plugin knows how to discover it.

+    cp -f multitenant/bin/openshift-ovs-multitenant "${target}/bin/"

+    cp -f multitenant/bin/openshift-sdn-multitenant-setup.sh "${target}/bin/"

+  popd > /dev/null

+

+  # subnet and multitenant plugin setup writes docker network options

+  # to /run/openshift-sdn/docker-network, make this file to be exported

+  # as part of docker service start.

+  local system_docker_path="${target}/lib/systemd/system/docker.service.d/"

+  mkdir -p "${system_docker_path}"

+  cat <<EOF > "${system_docker_path}/docker-sdn-ovs.conf"

 [Service]

 EnvironmentFile=-/run/openshift-sdn/docker-network

 EOF

-    popd > /dev/null

-  fi

+  # Assume a non-default target is an indication of deploying in an

+  # environment where openvswitch is managed in a separate container

+  # (e.g. atomic host).

+  if [[ "${target}" = "${default_target}" ]]; then

+    systemctl enable openvswitch

+    systemctl start openvswitch

+  fi

 }

@@ -12,11 +12,55 @@ ORIGIN_ROOT=$(

 )

 source ${ORIGIN_ROOT}/contrib/vagrant/provision-util.sh

-# Passed as arguments to provisioning from Vagrantfile

+# Passed as arguments to provisioning script

 MASTER_IP=${1:-""}

-NUM_MINIONS=${2:-""}

-MINION_IPS=${3:-""}

+NODE_COUNT=${2:-2}

+NODE_IPS=${3:-""}

 INSTANCE_PREFIX=${4:-${OS_INSTANCE_PREFIX:-openshift}}

+# Set defaults for optional arguments

+FIXUP_NET_UDEV=false

+NETWORK_PLUGIN=${OPENSHIFT_NETWORK_PLUGIN:-""}

+NODE_INDEX=0

+CONFIG_ROOT=${ORIGIN_ROOT}

+

+# Parse optional arguments

+# Skip the positional arguments

+OPTIND=5

+while getopts ":i:n:c:f" opt; do

+  case $opt in

+    f)

+      FIXUP_NET_UDEV=true

+      ;;

+    i)

+      NODE_INDEX=${OPTARG}

+      ;;

+    n)

+      NETWORK_PLUGIN=${OPTARG}

+      ;;

+    c)

+      CONFIG_ROOT=${OPTARG}

+      ;;

+    \?)

+      echo "Invalid option: -${OPTARG}" >&2

+      exit 1

+      ;;

+    :)

+      echo "Option -${OPTARG} requires an argument." >&2

+      exit 1

+      ;;

+  esac

+done

+

+LOG_LEVEL=${OPENSHIFT_LOG_LEVEL:-5}

+

+NODE_IPS=(${NODE_IPS//,/ })

+if [ "${CONFIG_ROOT}" = "/" ]; then

+  CONFIG_ROOT=""

+fi

+NETWORK_PLUGIN=$(os::util::get-network-plugin "${NETWORK_PLUGIN}" \

+  "${DIND_MANAGEMENT_SCRIPT:-false}")

 MASTER_NAME="${INSTANCE_PREFIX}-master"

-MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))

+NODE_PREFIX="${INSTANCE_PREFIX}-node-"

+NODE_NAMES=( $(eval echo ${NODE_PREFIX}{1..${NODE_COUNT}}) )

+SDN_NODE_NAME="${INSTANCE_PREFIX}-master-sdn"

@@ -1,72 +1,38 @@

 #!/bin/bash

-set -ex

 source $(dirname $0)/provision-config.sh

-FIXUP_NET_UDEV=$5

-

-NETWORK_PLUGIN=$(os::util::get-network-plugin ${6:-""})

-

-if [ "${FIXUP_NET_UDEV}" == "true" ]; then

-  NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/

-  rm -f ${NETWORK_CONF_PATH}ifcfg-enp*

-  if [[ -f "${NETWORK_CONF_PATH}ifcfg-eth1" ]]; then

-    sed -i 's/^NM_CONTROLLED=no/#NM_CONTROLLED=no/' ${NETWORK_CONF_PATH}ifcfg-eth1

-    if ! grep -q "NAME=" ${NETWORK_CONF_PATH}ifcfg-eth1; then

-      echo "NAME=openshift" >> ${NETWORK_CONF_PATH}ifcfg-eth1

-    fi

-    nmcli con reload

-    nmcli dev disconnect eth1

-    nmcli con up "openshift"

-  fi

-fi

-

-# Setup hosts file to ensure name resolution to each member of the cluster

-minion_ip_array=(${MINION_IPS//,/ })

-os::util::setup-hosts-file "${MASTER_NAME}" "${MASTER_IP}" MINION_NAMES \

-  minion_ip_array

-

-# Install the required packages

-yum install -y docker-io git golang e2fsprogs hg net-tools bridge-utils which

-

-# Build openshift

-echo "Building openshift"

-pushd "${ORIGIN_ROOT}"

-  ./hack/build-go.sh

-  os::util::install-cmds "${ORIGIN_ROOT}"

-  ./hack/install-etcd.sh

-popd

-

-os::util::init-certs "${ORIGIN_ROOT}" "${NETWORK_PLUGIN}" "${MASTER_NAME}" \

-  "${MASTER_IP}" MINION_NAMES minion_ip_array

-

-# Start docker

-systemctl enable docker.service

-systemctl start docker.service

-

-# Create systemd service

-node_list=$(os::util::join , ${MINION_NAMES[@]})

-cat <<EOF > /usr/lib/systemd/system/openshift-master.service

-[Unit]

-Description=OpenShift Master

-Requires=docker.service network.service

-After=network.service

-

-[Service]

-ExecStart=/usr/bin/openshift start master --master=https://${MASTER_IP}:8443 --nodes=${node_list} --network-plugin=${NETWORK_PLUGIN}

-WorkingDirectory=${ORIGIN_ROOT}/

-

-[Install]

-WantedBy=multi-user.target

-EOF

-

-# Start the service

-systemctl daemon-reload

-systemctl start openshift-master.service

-

-# setup SDN

-$(dirname $0)/provision-sdn.sh

-

-# Set up the KUBECONFIG environment variable for use by oc

-os::util::set-oc-env "${ORIGIN_ROOT}" "/root/.bash_profile"

-os::util::set-oc-env "${ORIGIN_ROOT}" "/home/vagrant/.bash_profile"

+os::util::base-provision

+

+echo "Building and installing openshift"

+${ORIGIN_ROOT}/hack/build-go.sh

+os::util::install-cmds "${ORIGIN_ROOT}"

+${ORIGIN_ROOT}/hack/install-etcd.sh

+os::util::install-sdn "${ORIGIN_ROOT}"

+

+# Running an openshift node on the master ensures connectivity between

+# the openshift service and pods.  This supports kube API calls that

+# query a service and require that the endpoints of the service be

+# reachable from the master.

+#

+# TODO(marun) This is required for connectivity with openshift-sdn,

+# but may not make sense for other plugins.

+NODE_NAMES+=(${SDN_NODE_NAME})

+NODE_IPS+=(127.0.0.1)

+# Force the addition of a hosts entry for the sdn node.

+os::util::add-to-hosts-file "${MASTER_IP}" "${SDN_NODE_NAME}" 1

+

+os::util::init-certs "${CONFIG_ROOT}" "${NETWORK_PLUGIN}" "${MASTER_NAME}" \

+  "${MASTER_IP}" NODE_NAMES NODE_IPS

+

+echo "Launching openshift daemons"

+NODE_LIST=$(os::util::join , ${NODE_NAMES[@]})

+cmd="/usr/bin/openshift start master --loglevel=${LOG_LEVEL} \

+ --master=https://${MASTER_IP}:8443 --nodes=${NODE_LIST} \

+ --network-plugin=${NETWORK_PLUGIN}"

+os::util::start-os-service "openshift-master" "OpenShift Master" "${cmd}"

+os::util::start-node-service "${SDN_NODE_NAME}"

+

+# TODO(marun) Need to disable scheduling on sdn daemon

+

+os::util::set-os-env "${ORIGIN_ROOT}" "${CONFIG_ROOT}"

deleted file mode 100755

@@ -1,76 +0,0 @@

-#!/bin/bash

-set -ex

-source $(dirname $0)/provision-config.sh

-

-MINION_IP=$5

-MINION_INDEX=$6

-FIXUP_NET_UDEV=$7

-

-if [ "${FIXUP_NET_UDEV}" == "true" ]; then

-  NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/

-  rm -f ${NETWORK_CONF_PATH}ifcfg-enp*

-  if [[ -f "${NETWORK_CONF_PATH}ifcfg-eth1" ]]; then

-    sed -i 's/^NM_CONTROLLED=no/#NM_CONTROLLED=no/' ${NETWORK_CONF_PATH}ifcfg-eth1

-    if ! grep -q "NAME=" ${NETWORK_CONF_PATH}ifcfg-eth1; then

-      echo "NAME=openshift" >> ${NETWORK_CONF_PATH}ifcfg-eth1

-    fi

-    nmcli con reload

-    nmcli dev disconnect eth1

-    nmcli con up "openshift"

-  fi

-fi

-

-# get the minion name, index is 1-based

-minion_name=${MINION_NAMES[$MINION_INDEX-1]}

-

-# Setup hosts file to ensure name resolution to each member of the cluster

-minion_ip_array=(${MINION_IPS//,/ })

-os::util::setup-hosts-file "${MASTER_NAME}" "${MASTER_IP}" MINION_NAMES \

-  minion_ip_array

-

-# Install the required packages

-yum install -y docker-io git golang e2fsprogs hg openvswitch net-tools bridge-utils which ethtool

-

-# Build openshift

-echo "Building openshift"

-pushd "${ORIGIN_ROOT}"

-  ./hack/build-go.sh

-  os::util::install-cmds "${ORIGIN_ROOT}"

-popd

-

-# Copy over the certificates directory

-cp -r "${ORIGIN_ROOT}/openshift.local.config" /

-chown -R vagrant.vagrant /openshift.local.config

-

-mkdir -p /openshift.local.volumes

-

-# Setup SDN

-$(dirname $0)/provision-sdn.sh

-

-# Create systemd service

-cat <<EOF > /usr/lib/systemd/system/openshift-node.service

-[Unit]

-Description=OpenShift Node

-Requires=network.service

-After=docker.service network.service

-

-[Service]

-ExecStart=/usr/bin/openshift start node --config=/openshift.local.config/node-${minion_name}/node-config.yaml

-Restart=on-failure

-RestartSec=10s

-

-[Install]

-WantedBy=multi-user.target

-EOF

-

-# Start the service

-systemctl daemon-reload

-systemctl enable openshift-node.service

-systemctl start openshift-node.service

-

-# Set up the KUBECONFIG environment variable for use by the client

-os::util::set-oc-env / "/root/.bash_profile"

-os::util::set-oc-env / "/home/vagrant/.bash_profile"

-

-# Register with the master

-#curl -X POST -H 'Accept: application/json' -d "{\"kind\":\"Minion\", \"id\":"${MINION_IP}", \"apiVersion\":\"v1beta1\", \"hostIP\":"${MINION_IP}" }" http://${MASTER_IP}:8080/api/v1beta1/minions

new file mode 100755

@@ -0,0 +1,17 @@

+#!/bin/bash

+

+source $(dirname $0)/provision-config.sh

+

+os::util::base-provision

+

+# openshift is assumed to have been built before node deployment

+os::util::install-cmds "${ORIGIN_ROOT}"

+

+os::util::install-sdn "${ORIGIN_ROOT}"

+

+echo "Launching openshift daemon"

+# Provided index is 1-based, array is 0 based

+NODE_NAME=${NODE_NAMES[${NODE_INDEX}-1]}

+os::util::start-node-service "${NODE_NAME}"

+

+os::util::set-os-env "${ORIGIN_ROOT}" "${CONFIG_ROOT}"

deleted file mode 100755

@@ -1,13 +0,0 @@

-#!/bin/bash

-set -ex

-source $(dirname $0)/provision-config.sh

-

-os::util::install-sdn "${ORIGIN_ROOT}"

-

-# Only start openvswitch if it has been installed (only minions).

-if rpm -qa | grep -q openvswitch; then

-  systemctl enable openvswitch

-  systemctl start openvswitch

-fi

-

-# no need to start openshift-sdn, as it is integrated with openshift binary

@@ -85,24 +85,41 @@ os::util::init-certs() {

   popd > /dev/null

 }

-# Set up the KUBECONFIG environment variable for use by oc

-os::util::set-oc-env() {

-  local config_root=$1

-  local target=$2

+os::util::set-os-env() {

+  local origin_root=$1

+  local config_root=$2

+

+  # Set up the KUBECONFIG environment variable for use by oc.

+  #

+  # Target .bashrc since docker exec doesn't invoke .bash_profile and

+  # .bash_profile loads .bashrc anyway.

+  local file_target=".bashrc"

-  if [ "${config_root}" = "/" ]; then

-    config_root=""

+  local vagrant_target="/home/vagrant/${file_target}"

+  if [ -d $(dirname "${vagrant_target}") ]; then

+    os::util::set-bash-env "${origin_root}" "${config_root}" \

+"${vagrant_target}"

   fi

+  os::util::set-bash-env "${origin_root}" "${config_root}" \

+"/root/${file_target}"

+}

+

+os::util::set-bash-env() {

+  local origin_root=$1

+  local config_root=$2

+  local target=$3

   local path="${config_root}/openshift.local.config/master/admin.kubeconfig"

   local config_line="export KUBECONFIG=${path}"

   if ! grep -q "${config_line}" "${target}" &> /dev/null; then

     echo "export KUBECONFIG=${path}" >> "${target}"

+    echo "cd ${origin_root}" >> "${target}"

   fi

 }

 os::util::get-network-plugin() {

   local plugin=$1

+  local dind_management_script=${2:-false}

   local subnet_plugin="redhat/openshift-ovs-subnet"

   local multitenant_plugin="redhat/openshift-ovs-multitenant"

@@ -110,15 +127,99 @@ os::util::get-network-plugin() {

   if [ "${plugin}" != "${subnet_plugin}" ] && \

      [ "${plugin}" != "${multitenant_plugin}" ]; then

-    if [ "${plugin}" != "" ]; then

+    # Disable output when being called from the dind management script

+    # since it may be doing something other than launching a cluster.

+    if [ "${dind_management_script}" = "false" ]; then

+      if [ "${plugin}" != "" ]; then

         >&2 echo "Invalid network plugin: ${plugin}"

+      fi

+      >&2 echo "Using default network plugin: ${default_plugin}"

     fi

-    >&2 echo "Using default network plugin: ${default_plugin}"

     plugin="${default_plugin}"

   fi

   echo "${plugin}"

 }

+os::util::base-provision() {

+  os::util::fixup-net-udev

+

+  os::util::setup-hosts-file "${MASTER_NAME}" "${MASTER_IP}" NODE_NAMES NODE_IPS

+

+  os::util::install-pkgs

+}

+

+os::util::fixup-net-udev() {

+  if [ "${FIXUP_NET_UDEV}" == "true" ]; then

+    NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/

+    rm -f ${NETWORK_CONF_PATH}ifcfg-enp*

+    if [[ -f "${NETWORK_CONF_PATH}ifcfg-eth1" ]]; then

+      sed -i 's/^NM_CONTROLLED=no/#NM_CONTROLLED=no/' ${NETWORK_CONF_PATH}ifcfg-eth1

+      if ! grep -q "NAME=" ${NETWORK_CONF_PATH}ifcfg-eth1; then

+        echo "NAME=openshift" >> ${NETWORK_CONF_PATH}ifcfg-eth1

+      fi

+      nmcli con reload

+      nmcli dev disconnect eth1

+      nmcli con up "openshift"

+    fi

+  fi

+}

+

+os::util::install-pkgs() {

+  # Only install packages if not deploying to a container.  A

+  # container is expected to have installed packages as part of image

+  # creation.

+  if [ ! -f /.dockerinit ]; then

+    yum update -y

+    yum install -y docker-io git golang e2fsprogs hg net-tools bridge-utils which ethtool

+

+    systemctl enable docker

+    systemctl start docker

+  fi

+}

+

+os::util::start-os-service() {

+  local unit_name=$1

+  local description=$2

+  local exec_start=$3

+  local work_dir=${4:-${CONFIG_ROOT}/}

+

+  # TODO(marun) Should the daemons be sharing a working directory?

+

+  cat <<EOF > "/usr/lib/systemd/system/${unit_name}.service"

+[Unit]

+Description=${description}

+Requires=network.target

+After=docker.target network.target

+

+[Service]

+ExecStart=${exec_start}

+WorkingDirectory=${work_dir}

+Restart=on-failure

+RestartSec=10s

+

+[Install]

+WantedBy=multi-user.target

+EOF

+

+  systemctl daemon-reload

+  systemctl enable "${unit_name}.service"

+  systemctl start "${unit_name}.service"

+}

+

+os::util::start-node-service() {

+  local node_name=$1

+

+  # Copy over the certificates directory so that each node has a copy.

+  cp -r "${CONFIG_ROOT}/openshift.local.config" /

+  if [ -d /home/vagrant ]; then

+    chown -R vagrant.vagrant /openshift.local.config

+  fi

+

+  cmd="/usr/bin/openshift start node --loglevel=${LOG_LEVEL} \

+--config=/openshift.local.config/node-${node_name}/node-config.yaml"

+  os::util::start-os-service "openshift-node" "OpenShift Node" "${cmd}" /

+}

+

 os::util::wait-for-condition() {

   local start_msg=$1

   local error_msg=$2

@@ -148,3 +249,30 @@ os::util::wait-for-condition() {

     echo -e '\nDone'

   fi

 }

+

+os::util::is-sdn-node-registered() {

+  local master_cid=$1

+  local node_name=$2

+

+  ${DOCKER_CMD} exec -t "${master_cid}" bash -ci \

+    "oc get nodes ${node_name} &> /dev/null"

+}

+

+os::util::disable-sdn-node() {

+  local master_cid=$1

+  local node_name=$2

+

+  local sdn_msg="for sdn node to register with the master"

+  local start_msg="Waiting ${sdn_msg}"

+  local error_msg="[ERROR] Timeout waiting ${sdn_msg}"

+  local condition="os::util::is-sdn-node-registered ${master_cid} ${node_name}"

+  local timeout=30

+  os::util::wait-for-condition "${start_msg}" "${error_msg}" "${condition}" \

+    "${timeout}"

+

+  echo "Disabling scheduling for the sdn node"

+  # Disable scheduling outside of the master provision script to give

+  # the node time to register itself to the master.

+  ${DOCKER_CMD} exec -t "${master_cid}" bash -ci \

+    "osadm manage-node ${node_name} --schedulable=false > /dev/null"

+}

@@ -59,44 +59,26 @@

 #

 #     hack/dind-cluster.sh test-net-e2e

 #

-# Bash Aliases

-# ------------

-#

-# The following bash aliases are available in the cluster containers:

-#

-# oc-create-hello - create the 'hello' example app

-# oc-less-log - invoke 'less' on the openshift daemon log (will target

-#               the master or node log depending on the type of node)

-# oc-tail-log - invoke tail on the openshift daemon log

-#

-# Process Management

-# ------------------

-#

-# Due to docker-in-docker conflicting with systemd when running in a

-# container, supervisord is used instead.  The 'supervisorctl' command

-# is the equivalent of 'systemctl' and logs for managed processes can

-# be found in /var/log/supervisor.

-#

-# Loopback Devices

-# ----------------

-#

-# Due to the way docker-in-docker daemons interact with loopback

-# devices, it is important to invoke 'dind-cluster.sh stop' on a

-# running cluster instead of manually stopping the containers.  This

-# ensures that the containerized docker daemons are gracefully

-# shutdown and allowed to release their loopback devices before

-# container shutdown.  If the daemons are not stopped before container

-# shutdown, the associated loopback devices will be effectively

-# unusable ('leaked') until a subsequent host reboot.  If enough

-# loopback devices are leaked, cluster boot may not be possible since

-# each openshift node running in a container depends on a docker

-# daemon requiring 2 loopback devices.

 set -o errexit

 set -o nounset

 set -o pipefail

-source $(dirname "${BASH_SOURCE}")/dind/init.sh

+DIND_MANAGEMENT_SCRIPT=true

+

+source $(dirname "${BASH_SOURCE}")/../contrib/vagrant/provision-config.sh

+

+DOCKER_CMD=${DOCKER_CMD:-"sudo docker"}

+

+# Override the default CONFIG_ROOT path with one that is

+# cluster-specific.

+CONFIG_ROOT=${OS_DIND_CONFIG_ROOT:-/tmp/openshift-dind-cluster/${INSTANCE_PREFIX}}

+

+DEPLOYED_CONFIG_ROOT="/config"

+

+DEPLOYED_ROOT="/data"

+

+SCRIPT_ROOT="${DEPLOYED_ROOT}/contrib/vagrant"

 function check-selinux() {

   if [ "$(getenforce)" = "Enforcing" ]; then

@@ -141,45 +123,15 @@ function get-docker-ip() {

   ${DOCKER_CMD} inspect --format '{{ .NetworkSettings.IPAddress }}' "${cid}"

 }

-# Ensure sufficient available loopback devices to support the

-# indicated number of dind nodes.  Since it's not possible to create

-# device nodes inside a container, this function needs to be called

-# before launching a container that will run dind.

-function ensure-loopback-for-dind() {

-  local node_count=$1

-

-  # Ensure extra loopback devices to minimize the potential for

-  # contention.  Sometimes docker restarts during deployment don't

-  # properly release the devices.

-  local extra_loopback=4

-  local loopback_per_node=2

-  local required_free_loopback=$(( ( ${node_count} * ${loopback_per_node} ) + \

-    ${extra_loopback} ))

-

-  # Find the maximum index of existing loopback devices.

-  local max_index=$(losetup | grep '/dev/loop' | tail -n 1 |

-    sed -e 's|^/dev/loop\([0-9]\{1,\}\).*|\1|')

-  if [ -z "${max_index}" ]; then

-    max_index=0

-  fi

-

-  local requested_max_index=$(( ${max_index} + ${required_free_loopback} - 1))

-  for i in $(eval echo "{${max_index}..${requested_max_index}}"); do

-    if [ ! -e "/dev/loop${i}" ]; then

-      sudo mknod "/dev/loop${i}" b 7 "${i}"

-    fi

-  done

-}

-

 function start() {

   # docker-in-docker's use of volumes is not compatible with SELinux

   check-selinux

+  # TODO(marun) - perform these operations in a container for boot2docker compat

   echo "Ensuring compatible host configuration"

   sudo modprobe openvswitch

   sudo modprobe br_netfilter || true

   sudo sysctl -w net.bridge.bridge-nf-call-iptables=0

-  ensure-loopback-for-dind "${NUM_NODES}"

   mkdir -p "${CONFIG_ROOT}"

   build-images

@@ -205,10 +157,12 @@ function start() {

   node_ips=$(os::util::join , ${node_ips[@]})

   ## Provision containers

-  local args="${master_ip} ${NUM_NODES} ${node_ips} ${INSTANCE_PREFIX}"

+  echo "Configured network plugin: ${NETWORK_PLUGIN}"

+  local args="${master_ip} ${NODE_COUNT} ${node_ips} ${INSTANCE_PREFIX} \

+-n '${NETWORK_PLUGIN}'"

   echo "Provisioning ${MASTER_NAME}"

   ${DOCKER_CMD} exec -t "${master_cid}" bash -c \

-    "${SCRIPT_ROOT}/provision-master.sh ${args} ${MASTER_NAME} ${NETWORK_PLUGIN}"

+    "${SCRIPT_ROOT}/provision-master.sh ${args} -c ${DEPLOYED_CONFIG_ROOT}"

   # Ensure that all users (e.g. outside the container) have read-write

   # access to the openshift configuration.  Security shouldn't be a

@@ -218,18 +172,21 @@ function start() {

   find "${openshift_config_path}" -type d -exec sudo chmod ga+x {} \;

   for (( i=0; i < ${#node_cids[@]}; i++ )); do

+    local node_index=$((i + 1))

     local cid="${node_cids[$i]}"

     local name="${NODE_NAMES[$i]}"

     echo "Provisioning ${name}"

     ${DOCKER_CMD} exec "${cid}" bash -c \

-      "${SCRIPT_ROOT}/provision-node.sh ${args} ${name}"

+      "${SCRIPT_ROOT}/provision-node.sh ${args} -i ${node_index} -c \

+${DEPLOYED_CONFIG_ROOT}"

   done

-  os::dind::disable-sdn-node "${master_cid}" "${SDN_NODE_NAME}"

+  os::util::disable-sdn-node "${master_cid}" "${SDN_NODE_NAME}"

 }

 function stop() {

   echo "Cleaning up docker-in-docker containers"

+

   local master_cid=$(${DOCKER_CMD} ps -qa --filter "name=${MASTER_NAME}")

   if [[ "${master_cid}" ]]; then

     ${DOCKER_CMD} rm -f "${master_cid}"

@@ -239,21 +196,11 @@ function stop() {

   if [[ "${node_cids}" ]]; then

     node_cids=(${node_cids//\n/ })

     for cid in "${node_cids[@]}"; do

-      # Ensure that the nested docker daemon is stopped before attempting

-      # container removal so associated loopback devices are properly

-      # released.

-      #

-      # See: https://github.com/jpetazzo/dind/issues/19

-      #

-      local is_running=$(${DOCKER_CMD} inspect -f {{.State.Running}} "${cid}")

-      if [ "${is_running}" = "true" ]; then

-        ${DOCKER_CMD} exec -t "${cid}" "${SCRIPT_ROOT}/kill-docker.sh"

-      fi

       ${DOCKER_CMD} rm -f "${cid}"

     done

   fi

-  echo "Clearing configuration to avoid conflict with a future cluster"

+  echo "Cleanup up configuration to avoid conflict with a future cluster"

   # The container will have created configuration as root

   sudo rm -rf ${CONFIG_ROOT}/openshift.local.*

@@ -314,8 +261,7 @@ case "${1:-""}" in

     test-net-e2e

     ;;

   config-host)

-    os::util::set-oc-env "${CONFIG_ROOT}" "/home/vagrant/.bashrc"

-    os::util::set-oc-env "${CONFIG_ROOT}" "/root/.bashrc"

+    os::util::set-os-env "${ORIGIN_ROOT}" "${CONFIG_ROOT}"

     ;;

   *)

     echo "Usage: $0 {start|stop|restart|build-images|test-net-e2e|config-host}"

deleted file mode 100644

@@ -1,95 +0,0 @@

-#!/bin/bash

-

-set -o errexit

-set -o nounset

-set -o pipefail

-

-source $(dirname "${BASH_SOURCE}")/../../contrib/vagrant/provision-config.sh

-

-NUM_NODES=${NUM_MINIONS:-2}

-NODE_IPS=(${MINION_IPS//,/ })

-HOST_NAME=${5:-""}

-NETWORK_PLUGIN=${6:-${OPENSHIFT_SDN:-""}}

-

-NODE_PREFIX="${INSTANCE_PREFIX}-node-"

-NODE_NAMES=( $(eval echo ${NODE_PREFIX}{1..${NUM_NODES}}) )

-SDN_NODE_NAME="${INSTANCE_PREFIX}-master-sdn"

-

-DOCKER_CMD=${DOCKER_CMD:-"sudo docker"}

-

-DEPLOYED_ROOT="/data"

-SCRIPT_ROOT="${DEPLOYED_ROOT}/hack/dind"

-SUPERVISORD_CONF="/etc/supervisord.conf"

-

-CONFIG_ROOT=${OS_DIND_CONFIG_ROOT:-/tmp/openshift-dind-cluster/${INSTANCE_PREFIX}}

-DEPLOYED_CONFIG_ROOT="/config"

-

-os::dind::set-dind-env() {

-  # Set up the KUBECONFIG environment variable for use by oc

-  local deployed_root=$1

-  local config_root=$2

-

-  # Target .bashrc by default instead of .bash_profile because a

-  # 'docker exec' invocation will not run .bash_profile

-  local target=${3:-"/root/.bashrc"}

-

-  local log_target='/var/log/supervisor/openshift-*-stderr-*'

-  os::util::set-oc-env "${config_root}" "${target}"

-  cat <<EOF >> "${target}"

-alias oc-less-log="less ${log_target}"

-alias oc-tail-log="tail -f ${log_target}"

-alias oc-create-hello="oc create -f ${deployed_root}/examples/hello-openshift/hello-pod.json"

-EOF

-}

-

-os::dind::reload-docker() {

-  # Ensure that openshift-sdn has written configuration for docker

-  # before triggering a docker restart.

-  echo "Waiting for openshift-sdn to update supervisord.conf with docker config"

-  local counter=0

-  local timeout=30

-  while grep -q 'DOCKER_DAEMON_ARGS=\"\"' "${SUPERVISORD_CONF}"; do

-    if [[ "${counter}" -lt "${timeout}" ]]; then

-      counter=$((counter + 1))

-      echo -n '.'

-      sleep 1

-    else

-      echo -e "\n[ERROR] Timeout waiting for openshift-sdn to update supervisord.conf"

-      exit 1

-    fi

-  done

-  echo -e '\nDone'

-

-  # Stop docker gracefully

-  ${SCRIPT_ROOT}/kill-docker.sh

-

-  # Restart docker

-  supervisorctl update

-}

-

-os::dind::is-sdn-node-registered() {

-  local master_cid=$1

-  local node_name=$2

-

-  ${DOCKER_CMD} exec -t "${master_cid}" bash -ci \

-    "oc get nodes ${node_name} &> /dev/null"

-}

-

-os::dind::disable-sdn-node() {

-  local master_cid=$1

-  local node_name=$2

-

-  local sdn_msg="for sdn node to register with the master"

-  local start_msg="Waiting ${sdn_msg}"

-  local error_msg="[ERROR] Timeout waiting ${sdn_msg}"

-  local condition="os::dind::is-sdn-node-registered ${master_cid} ${node_name}"

-  local timeout=30

-  os::util::wait-for-condition "${start_msg}" "${error_msg}" "${condition}" \

-    "${timeout}"

-

-  echo "Disabling scheduling for the sdn node"

-  # Disable scheduling outside of the master provision script to give

-  # the node time to register itself to the master.

-  ${DOCKER_CMD} exec -t "${master_cid}" bash -ci \

-    "osadm manage-node ${node_name} --schedulable=false > /dev/null"

-}

deleted file mode 100755

@@ -1,27 +0,0 @@

-#!/bin/bash

-

-# Ensure that docker is gracefully killed.

-

-set -o errexit

-set -o nounset

-set -o pipefail

-

-pid_file=/var/run/docker.pid

-if [ -f "${pid_file}" ]; then

-  pid=$(cat "${pid_file}")