#!/bin/bash
# WARNING: The script modifies the host that docker is running on. It
# attempts to load the overlay and openvswitch modules. If this modification
# is undesirable consider running docker in a VM.
#
# Overview
# ========
#
# This script manages the lifecycle of an openshift dev cluster
# deployed to docker-in-docker containers. Once 'start' has been used
# to successfully create a dind cluster, 'docker exec' can be used to
# access the created containers (named
# openshift-{master,node-1,node-2}) as if they were VMs.
#
# Dependencies
# ------------
#
# This script has been tested on Fedora 24, but should work on any
# release. Docker is assumed to be installed. At this time,
# boot2docker is not supported.
#
# SELinux
# -------
#
# Docker-in-docker's use of volumes is not compatible with selinux set
# to enforcing mode. Set selinux to permissive or disable it
# entirely.
#
# OpenShift Configuration
# -----------------------
#
# By default, a dind openshift cluster stores its configuration
# (openshift.local.*) in /tmp/openshift-dind-cluster/openshift. It's
# possible to run multiple dind clusters simultaneously by overriding
# the instance prefix. The following command would ensure
# configuration was stored at /tmp/openshift-dind/cluster/my-cluster:
#
# OPENSHIFT_CLUSTER_ID=my-cluster hack/dind-cluster.sh [command]
#
# Suggested Workflow
# ------------------
#
# When making changes to the deployment of a dind cluster or making
# breaking golang changes, the 'restart' command will ensure that an
# existing cluster is cleaned up before deploying a new cluster.
#
# Running Tests
# -------------
#
# The extended tests can be run against a dind cluster as follows:
#
# OPENSHIFT_CONFIG_ROOT=dind test/extended/networking.sh
set -o errexit
set -o nounset
set -o pipefail
source "$(dirname "${BASH_SOURCE}")/lib/init.sh"
source "${OS_ROOT}/images/dind/node/openshift-dind-lib.sh"
function start() {
local origin_root=$1
local config_root=$2
local deployed_config_root=$3
local cluster_id=$4
local network_plugin=$5
local wait_for_cluster=$6
local node_count=$7
# docker-in-docker's use of volumes is not compatible with SELinux
check-selinux
echo "Starting dind cluster '${cluster_id}' with plugin '${network_plugin}'"
# Ensuring compatible host configuration
#
# Running in a container ensures that the docker host will be affected even
# if docker is running remotely. The openshift/dind-node image was chosen
# due to its having sysctl installed.
${DOCKER_CMD} run --privileged --net=host --rm -v /lib/modules:/lib/modules \
openshift/dind-node bash -e -c \
'/usr/sbin/modprobe openvswitch;
/usr/sbin/modprobe overlay 2> /dev/null || true;'
# Initialize the cluster config path
mkdir -p "${config_root}"
echo "OPENSHIFT_NETWORK_PLUGIN=${network_plugin}" > "${config_root}/network-plugin"
copy-runtime "${origin_root}" "${config_root}/"
local volumes="-v ${config_root}:${deployed_config_root}"
local run_cmd="${DOCKER_CMD} run -dt ${volumes} --privileged"
# Create containers
${run_cmd} --name="${MASTER_NAME}" --hostname="${MASTER_NAME}" "${MASTER_IMAGE}" > /dev/null
for name in "${NODE_NAMES[@]}"; do
${run_cmd} --name="${name}" --hostname="${name}" "${NODE_IMAGE}" > /dev/null
done
local rc_file="dind-${cluster_id}.rc"
local admin_config
admin_config="$(get-admin-config "${CONFIG_ROOT}")"
local bin_path
bin_path="$(os::build::get-bin-output-path "${OS_ROOT}")"
cat >"${rc_file}" <<EOF
export KUBECONFIG=${admin_config}
export PATH=\$PATH:${bin_path}
EOF
if [[ -n "${wait_for_cluster}" ]]; then
wait-for-cluster "${config_root}" "${node_count}"
fi
if [[ "${KUBECONFIG:-}" != "${admin_config}" ||
":${PATH}:" != *":${bin_path}:"* ]]; then
echo ""
echo "Before invoking the openshift cli, make sure to source the
cluster's rc file to configure the bash environment:
$ . ${rc_file}
$ oc get nodes
"
fi
}
function stop() {
local config_root=$1
local cluster_id=$2
echo "Stopping dind cluster '${cluster_id}'"
local master_cid
master_cid="$(${DOCKER_CMD} ps -qa --filter "name=${MASTER_NAME}")"
if [[ "${master_cid}" ]]; then
${DOCKER_CMD} rm -f "${master_cid}" > /dev/null
fi
local node_cids
node_cids="$(${DOCKER_CMD} ps -qa --filter "name=${NODE_PREFIX}")"
if [[ "${node_cids}" ]]; then
node_cids=(${node_cids//\n/ })
for cid in "${node_cids[@]}"; do
${DOCKER_CMD} rm -f "${cid}" > /dev/null
done
fi
# Cleaning up configuration to avoid conflict with a future cluster
# The container will have created configuration as root
sudo rm -rf "${config_root}"/openshift.local.etcd
sudo rm -rf "${config_root}"/openshift.local.config
# Cleanup orphaned volumes
#
# See: https://github.com/jpetazzo/dind#important-warning-about-disk-usage
#
for volume in $( ${DOCKER_CMD} volume ls -qf dangling=true ); do
${DOCKER_CMD} volume rm "${volume}" > /dev/null
done
}
function check-selinux() {
if [[ "$(getenforce)" = "Enforcing" ]]; then
>&2 echo "Error: This script is not compatible with SELinux enforcing mode."
exit 1
fi
}
function get-network-plugin() {
local plugin=$1
local subnet_plugin="redhat/openshift-ovs-subnet"
local multitenant_plugin="redhat/openshift-ovs-multitenant"
local default_plugin="${multitenant_plugin}"
if [[ "${plugin}" != "${subnet_plugin}" &&
"${plugin}" != "${multitenant_plugin}" &&
"${plugin}" != "cni" ]]; then
if [[ -n "${plugin}" ]]; then
>&2 echo "Invalid network plugin: ${plugin}"
fi
plugin="${default_plugin}"
fi
echo "${plugin}"
}
function get-docker-ip() {
local cid=$1
${DOCKER_CMD} inspect --format '{{ .NetworkSettings.IPAddress }}' "${cid}"
}
function get-admin-config() {
local config_root=$1
echo "${config_root}/openshift.local.config/master/admin.kubeconfig"
}
function copy-runtime() {
local origin_root=$1
local target=$2
cp "$(os::util::find::built_binary openshift)" "${target}"
cp "$(os::util::find::built_binary host-local)" "${target}"
cp "$(os::util::find::built_binary loopback)" "${target}"
cp "$(os::util::find::built_binary sdn-cni-plugin)" "${target}/openshift-sdn"
local osdn_plugin_path="${origin_root}/pkg/sdn/plugin"
cp "${osdn_plugin_path}/bin/openshift-sdn-ovs" "${target}"
cp "${osdn_plugin_path}/sdn-cni-plugin/80-openshift-sdn.conf" "${target}"
}
function wait-for-cluster() {
local config_root=$1
local expected_node_count=$2
# Increment the node count to ensure that the sdn node also reports readiness
(( expected_node_count++ ))
local kubeconfig
kubeconfig="$(get-admin-config "${config_root}")"
local oc
oc="$(os::util::find::built_binary oc)"
# wait for healthz to report ok before trying to get nodes
os::util::wait-for-condition "ok" "${oc} get --config=${kubeconfig} --raw=/healthz" "120"
local msg="${expected_node_count} nodes to report readiness"
local condition="nodes-are-ready ${kubeconfig} ${oc} ${expected_node_count}"
local timeout=120
os::util::wait-for-condition "${msg}" "${condition}" "${timeout}"
}
function nodes-are-ready() {
local kubeconfig=$1
local oc=$2
local expected_node_count=$3
# TODO - do not count any node whose name matches the master node e.g. 'node-master'
read -d '' template <<'EOF'
{{range $item := .items}}
{{range .status.conditions}}
{{if eq .type "Ready"}}
{{if eq .status "True"}}
{{printf "%s\\n" $item.metadata.name}}
{{end}}
{{end}}
{{end}}
{{end}}
EOF
# Remove formatting before use
template="$(echo "${template}" | tr -d '\n' | sed -e 's/} \+/}/g')"
local count
count="$("${oc}" --config="${kubeconfig}" get nodes \
--template "${template}" 2> /dev/null | \
wc -l)"
test "${count}" -ge "${expected_node_count}"
}
function build-images() {
local origin_root=$1
echo "Building container images"
build-image "${origin_root}/images/dind/" "${BASE_IMAGE}"
build-image "${origin_root}/images/dind/node" "${NODE_IMAGE}"
build-image "${origin_root}/images/dind/master" "${MASTER_IMAGE}"
}
function build-image() {
local build_root=$1
local image_name=$2
pushd "${build_root}" > /dev/null
${DOCKER_CMD} build -t "${image_name}" .
popd > /dev/null
}
DOCKER_CMD=${DOCKER_CMD:-"sudo docker"}
CLUSTER_ID="${OPENSHIFT_CLUSTER_ID:-openshift}"
TMPDIR="${TMPDIR:-"/tmp"}"
CONFIG_ROOT="${OPENSHIFT_CONFIG_ROOT:-${TMPDIR}/openshift-dind-cluster/${CLUSTER_ID}}"
DEPLOYED_CONFIG_ROOT="/data"
MASTER_NAME="${CLUSTER_ID}-master"
NODE_PREFIX="${CLUSTER_ID}-node-"
NODE_COUNT=2
NODE_NAMES=()
for (( i=1; i<=NODE_COUNT; i++ )); do
NODE_NAMES+=( "${NODE_PREFIX}${i}" )
done
BASE_IMAGE="openshift/dind"
NODE_IMAGE="openshift/dind-node"
MASTER_IMAGE="openshift/dind-master"
case "${1:-""}" in
start)
BUILD=
BUILD_IMAGES=
WAIT_FOR_CLUSTER=1
NETWORK_PLUGIN=
REMOVE_EXISTING_CLUSTER=
OPTIND=2
while getopts ":bin:rs" opt; do
case $opt in
b)
BUILD=1
;;
i)
BUILD_IMAGES=1
;;
n)
NETWORK_PLUGIN="${OPTARG}"
;;
r)
REMOVE_EXISTING_CLUSTER=1
;;
s)
WAIT_FOR_CLUSTER=
;;
\?)
echo "Invalid option: -${OPTARG}" >&2
exit 1
;;
:)
echo "Option -${OPTARG} requires an argument." >&2
exit 1
;;
esac
done
if [[ -n "${REMOVE_EXISTING_CLUSTER}" ]]; then
stop "${CONFIG_ROOT}" "${CLUSTER_ID}"
fi
# Build origin if requested or required
if [[ -n "${BUILD}" ]] || ! os::util::find::built_binary 'oc' >/dev/null 2>&1; then
"${OS_ROOT}/hack/build-go.sh"
fi
# Build images if requested or required
if [[ -n "${BUILD_IMAGES}" ||
-z "$(${DOCKER_CMD} images -q ${MASTER_IMAGE})" ]]; then
build-images "${OS_ROOT}"
fi
NETWORK_PLUGIN="$(get-network-plugin "${NETWORK_PLUGIN}")"
start "${OS_ROOT}" "${CONFIG_ROOT}" "${DEPLOYED_CONFIG_ROOT}" \
"${CLUSTER_ID}" "${NETWORK_PLUGIN}" "${WAIT_FOR_CLUSTER}" \
"${NODE_COUNT}" "${NODE_PREFIX}"
;;
stop)
stop "${CONFIG_ROOT}" "${CLUSTER_ID}"
;;
wait-for-cluster)
wait-for-cluster "${CONFIG_ROOT}" "${NODE_COUNT}"
;;
build-images)
build-images "${OS_ROOT}"
;;
*)
>&2 echo "Usage: $0 {start|stop|wait-for-cluster|build-images}
start accepts the following arguments:
-n [net plugin] the name of the network plugin to deploy
-b build origin before starting the cluster
-i build container images before starting the cluster
-r remove an existing cluster
-s skip waiting for nodes to become ready
"
exit 2
esac