@@ -1,6 +1,7 @@

 /_output

 /_tools

 /openshift.local.*

+/third-party

 /.project

 /.vagrant

 /cpu.pprof

@@ -155,7 +155,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

         minion_ip = minion_ips[n]

         minion.vm.box = kube_box[kube_os]["name"]

         minion.vm.box_url = kube_box[kube_os]["box_url"]

-        minion.vm.provision "shell", inline: "/vagrant/vagrant/provision-minion.sh #{master_ip} #{num_minion} #{minion_ips_str} #{minion_ip} #{minion_index} #{ENV['OPENSHIFT_SDN']}"

+        minion.vm.provision "shell", inline: "/vagrant/vagrant/provision-minion.sh #{master_ip} #{num_minion} #{minion_ips_str} #{minion_ip} #{minion_index}"

         minion.vm.network "private_network", ip: "#{minion_ip}"

         minion.vm.hostname = "openshift-minion-#{minion_index}"

       end

@@ -1,10 +1,21 @@

 #!/bin/bash

-set -e

+

+set -o errexit

+set -o nounset

+set -o pipefail

+

+ORIGIN_ROOT=$(

+  unset CDPATH

+  origin_root=$(dirname "${BASH_SOURCE}")/..

+  cd "${origin_root}"

+  pwd

+)

+source ${ORIGIN_ROOT}/vagrant/provision-util.sh

 # Passed as arguments to provisioning from Vagrantfile

-MASTER_IP=$1

-NUM_MINIONS=$2

-MINION_IPS=$3

+MASTER_IP=${1:-""}

+NUM_MINIONS=${2:-""}

+MINION_IPS=${3:-""}

 INSTANCE_PREFIX=openshift

 MASTER_NAME="${INSTANCE_PREFIX}-master"

@@ -3,86 +3,38 @@

 set -ex

 source $(dirname $0)/provision-config.sh

-OPENSHIFT_SDN=$4

-if [ "${OPENSHIFT_SDN}" == "redhat/openshift-ovs-multitenant" ] || [ "${OPENSHIFT_SDN}" == "redhat/openshift-ovs-subnet" ] || [ "${OPENSHIFT_SDN}" == "" ]; then

-	OPENSHIFT_SDN_PLUGIN=${OPENSHIFT_SDN}

-fi

-OPENSHIFT_SDN_PLUGIN=${OPENSHIFT_SDN_PLUGIN:-redhat/openshift-ovs-subnet}

+NETWORK_PLUGIN=$(os::util::get-network-plugin ${4:-""})

 NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/

 sed -i 's/^NM_CONTROLLED=no/#NM_CONTROLLED=no/' ${NETWORK_CONF_PATH}ifcfg-eth1

 systemctl restart network

-# Setup hosts file to support ping by hostname to each minion in the cluster from apiserver

-node_list=""

+# Setup hosts file to ensure name resolution to each member of the cluster

 minion_ip_array=(${MINION_IPS//,/ })

-for (( i=0; i<${#MINION_NAMES[@]}; i++)); do

-  minion=${MINION_NAMES[$i]}

-  node_list="${node_list},${minion}"

-  ip=${minion_ip_array[$i]}

-  if [ ! "$(cat /etc/hosts | grep $minion)" ]; then

-    echo "Adding $minion to hosts file"

-    echo "$ip $minion" >> /etc/hosts

-  fi

-done

-if ! grep ${MASTER_IP} /etc/hosts; then

-  echo "${MASTER_IP} ${MASTER_NAME}" >> /etc/hosts

-fi

-node_list=${node_list:1}

+os::util::setup-hosts-file "${MASTER_NAME}" "${MASTER_IP}" MINION_NAMES \

+  minion_ip_array

 # Install the required packages

 yum install -y docker-io git golang e2fsprogs hg net-tools bridge-utils which

 # Build openshift

 echo "Building openshift"

-pushd /vagrant

+pushd "${ORIGIN_ROOT}"

   ./hack/build-go.sh

-  cp _output/local/go/bin/openshift /usr/bin

+  os::util::install-cmds "${ORIGIN_ROOT}"

   ./hack/install-etcd.sh

 popd

-# Initialize certificates

-echo "Generating certs"

-pushd /vagrant

-  SERVER_CONFIG_DIR="`pwd`/openshift.local.config"

-  VOLUMES_DIR="/var/lib/openshift.local.volumes"

-  MASTER_CONFIG_DIR="${SERVER_CONFIG_DIR}/master"

-  CERT_DIR="${MASTER_CONFIG_DIR}"

-

-  # Master certs

-  /usr/bin/openshift admin ca create-master-certs \

-    --overwrite=false \

-    --cert-dir=${CERT_DIR} \

-    --master=https://${MASTER_IP}:8443 \

-    --hostnames=${MASTER_IP},${MASTER_NAME}

-

-  # Certs for nodes

-  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do

-    minion=${MINION_NAMES[$i]}

-    ip=${minion_ip_array[$i]}

-

-    /usr/bin/openshift admin create-node-config \

-      --node-dir="${SERVER_CONFIG_DIR}/node-${minion}" \

-      --node="${minion}" \

-      --hostnames="${minion},${ip}" \

-      --master="https://${MASTER_IP}:8443" \

-      --network-plugin="${OPENSHIFT_SDN_PLUGIN}" \

-      --node-client-certificate-authority="${CERT_DIR}/ca.crt" \

-      --certificate-authority="${CERT_DIR}/ca.crt" \

-      --signer-cert="${CERT_DIR}/ca.crt" \

-      --signer-key="${CERT_DIR}/ca.key" \

-      --signer-serial="${CERT_DIR}/ca.serial.txt" \

-      --volume-dir="${VOLUMES_DIR}"

-  done

-

-popd

+os::util::init-certs "${ORIGIN_ROOT}" "${NETWORK_PLUGIN}" "${MASTER_NAME}" \

+  "${MASTER_IP}" MINION_NAMES minion_ip_array

 # Start docker

 systemctl enable docker.service

 systemctl start docker.service

 # Create systemd service

+node_list=$(os::util::join , ${MINION_NAMES[@]})

 cat <<EOF > /usr/lib/systemd/system/openshift-master.service

 [Unit]

 Description=OpenShift Master

@@ -90,8 +42,8 @@ Requires=docker.service network.service

 After=network.service

 [Service]

-ExecStart=/usr/bin/openshift start master --master=https://${MASTER_IP}:8443 --nodes=${node_list} --network-plugin=${OPENSHIFT_SDN_PLUGIN}

-WorkingDirectory=/vagrant/

+ExecStart=/usr/bin/openshift start master --master=https://${MASTER_IP}:8443 --nodes=${node_list} --network-plugin=${NETWORK_PLUGIN}

+WorkingDirectory=${ORIGIN_ROOT}/

 [Install]

 WantedBy=multi-user.target

@@ -102,8 +54,8 @@ systemctl daemon-reload

 systemctl start openshift-master.service

 # setup SDN

-$(dirname $0)/provision-sdn.sh $@

+$(dirname $0)/provision-sdn.sh

 # Set up the KUBECONFIG environment variable for use by oc

-echo 'export KUBECONFIG=/vagrant/openshift.local.config/master/admin.kubeconfig' >> /root/.bash_profile

-echo 'export KUBECONFIG=/vagrant/openshift.local.config/master/admin.kubeconfig' >> /home/vagrant/.bash_profile

+os::util::set-oc-env "${ORIGIN_ROOT}" "/root/.bash_profile"

+os::util::set-oc-env "${ORIGIN_ROOT}" "/home/vagrant/.bash_profile"

@@ -3,7 +3,6 @@ set -ex

 source $(dirname $0)/provision-config.sh

 MINION_IP=$4

-OPENSHIFT_SDN=$6

 MINION_INDEX=$5

 NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/

@@ -14,44 +13,29 @@ systemctl restart network

 # get the minion name, index is 1-based

 minion_name=${MINION_NAMES[$MINION_INDEX-1]}

-# Setup hosts file to support ping by hostname to master

-if [ ! "$(cat /etc/hosts | grep $MASTER_NAME)" ]; then

-  echo "Adding $MASTER_NAME to hosts file"

-  echo "$MASTER_IP $MASTER_NAME" >> /etc/hosts

-fi

-

-# Setup hosts file to support ping by hostname to each minion in the cluster

+# Setup hosts file to ensure name resolution to each member of the cluster

 minion_ip_array=(${MINION_IPS//,/ })

-for (( i=0; i<${#MINION_NAMES[@]}; i++)); do

-  minion=${MINION_NAMES[$i]}

-  ip=${minion_ip_array[$i]}  

-  if [ ! "$(cat /etc/hosts | grep $minion)" ]; then

-    echo "Adding $minion to hosts file"

-    echo "$ip $minion" >> /etc/hosts

-  fi

-done

-if ! grep ${MINION_IP} /etc/hosts; then

-  echo "${MINION_IP} ${minion_name}" >> /etc/hosts

-fi

+os::util::setup-hosts-file "${MASTER_NAME}" "${MASTER_IP}" MINION_NAMES \

+  minion_ip_array

 # Install the required packages

 yum install -y docker-io git golang e2fsprogs hg openvswitch net-tools bridge-utils which ethtool

 # Build openshift

 echo "Building openshift"

-pushd /vagrant

+pushd "${ORIGIN_ROOT}"

   ./hack/build-go.sh

-  cp _output/local/go/bin/openshift /usr/bin

+  os::util::install-cmds "${ORIGIN_ROOT}"

 popd

 # Copy over the certificates directory

-cp -r /vagrant/openshift.local.config /

+cp -r "${ORIGIN_ROOT}/openshift.local.config" /

 chown -R vagrant.vagrant /openshift.local.config

 mkdir -p /openshift.local.volumes

 # Setup SDN

-$(dirname $0)/provision-sdn.sh $@

+$(dirname $0)/provision-sdn.sh

 # Create systemd service

 cat <<EOF > /usr/lib/systemd/system/openshift-node.service

@@ -75,8 +59,8 @@ systemctl enable openshift-node.service

 systemctl start openshift-node.service

 # Set up the KUBECONFIG environment variable for use by the client

-echo 'export KUBECONFIG=/openshift.local.config/master/admin.kubeconfig' >> /root/.bash_profile

-echo 'export KUBECONFIG=/openshift.local.config/master/admin.kubeconfig' >> /home/vagrant/.bash_profile

+os::util::set-oc-env / "/root/.bash_profile"

+os::util::set-oc-env / "/home/vagrant/.bash_profile"

 # Register with the master

 #curl -X POST -H 'Accept: application/json' -d "{\"kind\":\"Minion\", \"id\":"${MINION_IP}", \"apiVersion\":\"v1beta1\", \"hostIP\":"${MINION_IP}" }" http://${MASTER_IP}:8080/api/v1beta1/minions

@@ -2,24 +2,12 @@

 set -ex

 source $(dirname $0)/provision-config.sh

-pushd $HOME

-# build openshift-sdn

-if [ -d openshift-sdn ]; then

-    cd openshift-sdn

-    git fetch origin

-    git reset --hard origin/master

-    git checkout -b multitenant

-else

-    git clone https://github.com/openshift/openshift-sdn -b multitenant

-    cd openshift-sdn

-fi

-

-make clean

-make

-make install

-popd

+os::util::install-sdn "${ORIGIN_ROOT}"

-systemctl enable openvswitch

-systemctl start openvswitch

+# Only start openvswitch if it has been installed (only minions).

+if rpm -qa | grep -q openvswitch; then

+  systemctl enable openvswitch

+  systemctl start openvswitch

+fi

 # no need to start openshift-sdn, as it is integrated with openshift binary

new file mode 100644

@@ -0,0 +1,145 @@

+#!/bin/bash

+

+os::util::join() {

+  local IFS="$1"

+

+  shift

+  echo "$*"

+}

+

+os::util::install-cmds() {

+  local deployed_root=$1

+

+  cp ${deployed_root}/_output/local/go/bin/{openshift,oc} /usr/bin

+}

+

+os::util::add-to-hosts-file() {

+  local ip=$1

+  local name=$2

+

+  if ! grep -q "${ip}" /etc/hosts; then

+    local entry="${ip}\t${name}"

+    echo -e "Adding '${entry}' to hosts file"

+    echo -e "${entry}" >> /etc/hosts

+  fi

+}

+

+os::util::setup-hosts-file() {

+  local master_name=$1

+  local master_ip=$2

+  local -n node_names=$3

+  local -n node_ips=$4

+

+  # Setup hosts file to support ping by hostname to master

+  os::util::add-to-hosts-file "${master_ip}" "${master_name}"

+

+  # Setup hosts file to support ping by hostname to each node in the cluster

+  for (( i=0; i < ${#node_names[@]}; i++ )); do

+    os::util::add-to-hosts-file "${node_ips[$i]}" "${node_names[$i]}"

+  done

+}

+

+os::util::init-certs() {

+  local openshift_root=$1

+  local network_plugin=$2

+  local master_name=$3

+  local master_ip=$4

+  local -n node_names=$5

+  local -n node_ips=$6

+

+  local server_config_dir=${openshift_root}/openshift.local.config

+  local volumes_dir="/var/lib/openshift.local.volumes"

+  local cert_dir="${server_config_dir}/master"

+

+  echo "Generating certs"

+

+  pushd "${openshift_root}"

+

+  # Master certs

+  /usr/bin/openshift admin ca create-master-certs \

+    --overwrite=false \

+    --cert-dir="${cert_dir}" \

+    --master="https://${master_ip}:8443" \

+    --hostnames="${master_ip},${master_name}"

+

+  # Certs for nodes

+  for (( i=0; i < ${#node_names[@]}; i++ )); do

+    local name=${node_names[$i]}

+    local ip=${node_ips[$i]}

+    /usr/bin/openshift admin create-node-config \

+      --node-dir="${server_config_dir}/node-${name}" \

+      --node="${name}" \

+      --hostnames="${name},${ip}" \

+      --master="https://${master_ip}:8443" \

+      --network-plugin="${network_plugin}" \

+      --node-client-certificate-authority="${cert_dir}/ca.crt" \

+      --certificate-authority="${cert_dir}/ca.crt" \

+      --signer-cert="${cert_dir}/ca.crt" \

+      --signer-key="${cert_dir}/ca.key" \

+      --signer-serial="${cert_dir}/ca.serial.txt" \

+      --volume-dir="${volumes_dir}"

+  done

+

+  popd

+}

+

+# Set up the KUBECONFIG environment variable for use by oc

+os::util::set-oc-env() {

+  local deployed_root=$1

+  local target=$2

+

+  if [ "${deployed_root}" = "/" ]; then

+    deployed_root=""

+  fi

+

+  local path="${deployed_root}/openshift.local.config/master/admin.kubeconfig"

+  echo "export KUBECONFIG=${path}" >> "${target}"

+}

+

+os::util::get-network-plugin() {

+  local plugin=$1

+

+  local subnet_plugin="redhat/openshift-ovs-subnet"

+  local multitenant_plugin="redhat/openshift-ovs-multitenant"

+  local default_plugin="${subnet_plugin}"

+

+  if [ "${plugin}" != "${subnet_plugin}" ] && \

+     [ "${plugin}" != "${multitenant_plugin}" ]; then

+    if [ "${plugin}" != "" ]; then

+        >&2 echo "Invalid network plugin: ${plugin}"

+    fi

+    >&2 echo "Using default network plugin: ${default_plugin}"

+    plugin="${default_plugin}"

+  fi

+  echo "${plugin}"

+}

+

+os::util::install-sdn() {

+  local deployed_root=$1

+

+  # Source scripts from an openshift-sdn repo if present to support

+  # openshift-sdn development.

+  local sdn_root="${deployed_root}/third-party/openshift-sdn"

+  if [ -d "${sdn_root}" ]; then

+    pushd "${sdn_root}"

+      make

+      make "install-dev"

+    popd

+  else

+    local osdn_base_path="${deployed_root}/Godeps/_workspace/src/github.com/openshift/openshift-sdn"

+    local osdn_controller_path="${osdn_base_path}/ovssubnet/controller"

+    pushd "${osdn_controller_path}"

+      # The subnet plugin is discovered via the kube network plugin path.

+      local kube_osdn_path="/usr/libexec/kubernetes/kubelet-plugins/net/exec/redhat~openshift-ovs-subnet"

+      mkdir -p "${kube_osdn_path}"

+      cp -f kube/bin/openshift-ovs-subnet "${kube_osdn_path}/"

+      cp -f kube/bin/openshift-sdn-kube-subnet-setup.sh /usr/bin/

+

+      # The multitenant plugin only needs to be in PATH because the

+      # origin multitenant plugin knows how to discover it.

+      cp -f multitenant/bin/openshift-ovs-multitenant /usr/bin/

+      cp -f multitenant/bin/openshift-sdn-multitenant-setup.sh /usr/bin/

+    popd

+  fi

+

+}