#!/bin/bash set -ex source $(dirname $0)/provision-config.sh MINION_IP=$4 MINION_ID=$5 DOCKER_BRIDGE=kbr0 OVS_SWITCH=obr0 GRE_TUNNEL_BASE=gre BRIDGE_BASE=10.244 BRIDGE_ADDRESS=${BRIDGE_BASE}.${MINION_ID}.1 BRIDGE_NETWORK=${BRIDGE_ADDRESS}/24 BRIDGE_NETMASK=255.255.255.0 NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/ POST_NETWORK_SCRIPT=/vagrant/network_closure.sh # Add docker bridge ifcfg file cat <<EOF > ${NETWORK_CONF_PATH}ifcfg-${DOCKER_BRIDGE} # Generated by yours truly DEVICE=${DOCKER_BRIDGE} ONBOOT=yes TYPE=Bridge BOOTPROTO=static IPADDR=${BRIDGE_ADDRESS} NETMASK=${BRIDGE_NETMASK} STP=yes EOF # Add the ovs bridge ifcfg file cat <<EOF > ${NETWORK_CONF_PATH}ifcfg-${OVS_SWITCH} DEVICE=${OVS_SWITCH} ONBOOT=yes DEVICETYPE=ovs TYPE=OVSBridge BOOTPROTO=static HOTPLUG=no BRIDGE=${DOCKER_BRIDGE} EOF # Loop through all other minions and create persistent gre tunnels MINION_IPS=$3 MINION_IP_ARRAY=(`echo ${MINION_IPS} | tr "," "\n"`) GRE_NUM=0 for remote_ip in "${MINION_IP_ARRAY[@]}" do if [ "${remote_ip}" == "${MINION_IP}" ]; then continue fi ((GRE_NUM++)) || echo GRE_TUNNEL=${GRE_TUNNEL_BASE}${GRE_NUM} # ovs-vsctl add-port ${OVS_SWITCH} ${GRE_TUNNEL} -- set interface ${GRE_TUNNEL} type=gre options:remote_ip=${remote_ip} cat <<EOF > ${NETWORK_CONF_PATH}ifcfg-${GRE_TUNNEL} DEVICE=${GRE_TUNNEL} ONBOOT=yes DEVICETYPE=ovs TYPE=OVSTunnel OVS_BRIDGE=${OVS_SWITCH} OVS_TUNNEL_TYPE=gre OVS_TUNNEL_OPTIONS="options:remote_ip=${remote_ip}" EOF done # Add ip route rules such that all pod traffic flows through docker bridge and consequently to the gre tunnels cat <<EOF > /${NETWORK_CONF_PATH}route-${DOCKER_BRIDGE} ${BRIDGE_BASE}.0.0/16 dev ${DOCKER_BRIDGE} scope link src ${BRIDGE_ADDRESS} EOF systemctl enable openvswitch systemctl start openvswitch # NAT interface fails to revive on network restart, so OR-gate to true systemctl restart network.service || true # Set docker bridge up, and set stp on the OVS bridge ip link set dev ${DOCKER_BRIDGE} up ovs-vsctl set Bridge ${OVS_SWITCH} stp_enable=true # Modify the docker service file such that it uses the kube docker bridge and not its own sed -ie "s/ExecStart=\/usr\/bin\/docker -d/ExecStart=\/usr\/bin\/docker -d -b=${DOCKER_BRIDGE} --iptables=false/g" /usr/lib/systemd/system/docker.service systemctl daemon-reload systemctl enable docker.service systemctl restart docker.service # Setup iptables masquerade rules, so the pods can reach the internet iptables -t nat -A POSTROUTING -s ${BRIDGE_BASE}.0.0/16 ! -d ${BRIDGE_BASE}.0.0/16 -j MASQUERADE # Persist iptables rules iptables-save >& /etc/sysconfig/iptables