Browse code

hack/env upgrades - use rsync for sync

Falls back to tar if the command fails. Also doc the various options and
ensure `hack/env` prints out the docker container create command as well
as performing a sync.

Clayton Coleman authored on 2016/10/23 08:39:47
Showing 6 changed files
... ...
@@ -1123,8 +1123,15 @@ function os::build::environment::create() {
1123 1123
     fi
1124 1124
   fi
1125 1125
 
1126
+  local args
1127
+  if [[ $# -eq 0 ]]; then
1128
+    args=( "echo" "docker create ${additional_context} ${release_image}" )
1129
+  else
1130
+    args=( "$@" )
1131
+  fi
1132
+
1126 1133
   # Create a new container to from the release environment
1127
-  docker create ${additional_context} "${release_image}" "$@"
1134
+  docker create ${additional_context} "${release_image}" "${args[@]}"
1128 1135
 }
1129 1136
 readonly -f os::build::environment::create
1130 1137
 
... ...
@@ -1191,19 +1198,39 @@ function os::build::environment::withsource() {
1191 1191
   local container=$1
1192 1192
   local commit=${2:-HEAD}
1193 1193
 
1194
-  if [[ -n "${OS_BUILD_ENV_LOCAL_DOCKER:-}" ]]; then
1194
+  if [[ -n "${OS_BUILD_ENV_LOCAL_DOCKER-}" ]]; then
1195 1195
     # running locally, no change necessary
1196 1196
     os::build::get_version_vars
1197 1197
     os::build::save_version_vars "${OS_ROOT}/os-version-defs"
1198 1198
   else
1199
-    # Generate version definitions. Tree state is clean because we are pulling from git directly.
1200
-    OS_GIT_TREE_STATE=clean os::build::get_version_vars
1201
-    os::build::save_version_vars "/tmp/os-version-defs"
1202
-
1203 1199
     local workingdir
1204 1200
     workingdir="$(docker inspect -f '{{ index . "Config" "WorkingDir" }}' "${container}")"
1205
-    tar -cf - -C /tmp/ os-version-defs | docker cp - "${container}:${workingdir}"
1206
-    git archive --format=tar "${commit}" | docker cp - "${container}:${workingdir}"
1201
+    if [[ -n "${OS_BUILD_ENV_REUSE_VOLUME-}" ]]; then
1202
+      local excluded=()
1203
+      local oldIFS="${IFS}"
1204
+      IFS=:
1205
+      for exclude in ${OS_BUILD_ENV_EXCLUDE:-_output}; do
1206
+        excluded+=("--exclude=${exclude}")
1207
+      done
1208
+      IFS="${oldIFS}"
1209
+      if which rsync &>/dev/null; then
1210
+        local name
1211
+        name="$( echo "${OS_BUILD_ENV_REUSE_VOLUME}" | tr '[:upper:]' '[:lower:]' )"
1212
+        if ! rsync -a --blocking-io ${excluded[@]} --omit-dir-times --numeric-ids -e "docker run --rm -i -v \"${name}:${workingdir}\" --entrypoint=/bin/bash \"${OS_BUILD_ENV_IMAGE}\" -c '\$@'" . remote:"${workingdir}"; then
1213
+          # fall back to a tar if rsync is not in container
1214
+          tar -cf - ${excluded[@]} . | docker cp - "${container}:${workingdir}"
1215
+        fi
1216
+      else
1217
+        tar -cf - ${excluded[@]} . | docker cp - "${container}:${workingdir}"
1218
+      fi
1219
+    else
1220
+      # Generate version definitions. Tree state is clean because we are pulling from git directly.
1221
+      OS_GIT_TREE_STATE=clean os::build::get_version_vars
1222
+      os::build::save_version_vars "/tmp/os-version-defs"
1223
+
1224
+      tar -cf - -C /tmp/ os-version-defs | docker cp - "${container}:${workingdir}"
1225
+      git archive --format=tar "${commit}" | docker cp - "${container}:${workingdir}"
1226
+    fi
1207 1227
   fi
1208 1228
 
1209 1229
   os::build::environment::start "${container}"
... ...
@@ -1215,21 +1242,14 @@ readonly -f os::build::environment::withsource
1215 1215
 function os::build::environment::run() {
1216 1216
   local commit="${OS_GIT_COMMIT:-HEAD}"
1217 1217
   local volume="${OS_BUILD_ENV_REUSE_VOLUME:-}"
1218
-  local exists=
1219
-  if [[ -z "${OS_BUILD_ENV_REUSE_VOLUME:-}" ]]; then
1218
+  if [[ -z "${volume}" ]]; then
1220 1219
     volume="origin-build-$( git rev-parse "${commit}" )"
1221
-  elif docker volume inspect "${OS_BUILD_ENV_REUSE_VOLUME}" &>/dev/null; then
1222
-    exists=y
1223 1220
   fi
1224 1221
 
1225 1222
   local container
1226 1223
   container="$( OS_BUILD_ENV_REUSE_VOLUME=${volume} os::build::environment::create "$@" )"
1227 1224
   trap "os::build::environment::cleanup ${container}" EXIT
1228 1225
 
1229
-  if [[ "${exists}" == "y" ]]; then
1230
-    os::build::environment::start "${container}"
1231
-  else
1232
-    os::build::environment::withsource "${container}" "${commit}"
1233
-  fi
1226
+  os::build::environment::withsource "${container}" "${commit}"
1234 1227
 }
1235 1228
 readonly -f os::build::environment::run
... ...
@@ -1,7 +1,31 @@
1 1
 #!/bin/bash
2 2
 
3
-# This script generates release zips into _output/releases. It requires the openshift/origin-release
4
-# image to be built prior to executing this command via hack/build-base-images.sh.
3
+# This starts a Docker container using the release image (openshift/origin-release:golang-1.6)
4
+# and syncs the local directory into that image. The default mode performs a 'git archive' of
5
+# the current HEAD, so you get a reproducible environment. You can also set
6
+# OS_BUILD_ENV_REUSE_VOLUME to a docker volume name to rsync (or docker cp) the contents of
7
+# the current directory into the image.
8
+#
9
+# Examples:
10
+#   # sync local dir into the volume and print the Docker create command
11
+#   $ hack/env
12
+#
13
+#   # builds the current HEAD in the container
14
+#   $ hack/env make
15
+#
16
+#   # builds the current HEAD and copy _output/releases back locally afterwards
17
+#   $ OS_BUILD_ENV_PRESERVE=_output/releases hack/env make release
18
+#
19
+#   # run all update tasks and copy the api, pkg, and docs directories back out
20
+#   $ OS_BUILD_ENV_PRESERVE=api:docs:pkg hack/env make update
21
+#
22
+#   # rsync the contents of the current directory into the 'local' docker volume
23
+#   # and iteratively build
24
+#   $ export OS_BUILD_ENV_REUSE_VOLUME=local
25
+#   $ export OS_BUILD_ENV_DOCKER_ARGS='-e OS_VERSION_FILE= '
26
+#   $ hack/env make # slow
27
+#   $ hack/env make # fast!
28
+#
5 29
 
6 30
 # NOTE:   only committed code is built.
7 31
 source "$(dirname "${BASH_SOURCE}")/lib/init.sh"
... ...
@@ -15,7 +15,7 @@ ENV VERSION=1.6 \
15 15
 ENV PATH=$PATH:$GOROOT/bin:$GOPATH/bin
16 16
 
17 17
 RUN mkdir $TMPDIR && \
18
-    INSTALL_PKGS="make gcc zip mercurial krb5-devel bsdtar bc" && \
18
+    INSTALL_PKGS="make gcc zip mercurial krb5-devel bsdtar bc rsync" && \
19 19
     yum install -y $INSTALL_PKGS && \
20 20
     rpm -V $INSTALL_PKGS && \
21 21
     yum clean all && \
... ...
@@ -15,7 +15,7 @@ ENV VERSION=1.4.3 \
15 15
 ENV PATH=$PATH:$GOROOT/bin:$GOPATH/bin
16 16
 
17 17
 RUN mkdir $TMPDIR && \
18
-    INSTALL_PKGS="make gcc zip mercurial krb5-devel bsdtar bc" && \
18
+    INSTALL_PKGS="make gcc zip mercurial krb5-devel bsdtar bc rsync" && \
19 19
     yum install -y $INSTALL_PKGS && \
20 20
     rpm -V $INSTALL_PKGS && \
21 21
     yum clean all && \
... ...
@@ -15,7 +15,7 @@ ENV VERSION=1.6.3 \
15 15
 ENV PATH=$PATH:$GOROOT/bin:$GOPATH/bin
16 16
 
17 17
 RUN mkdir $TMPDIR && \
18
-    INSTALL_PKGS="make gcc zip mercurial krb5-devel bsdtar bc" && \
18
+    INSTALL_PKGS="make gcc zip mercurial krb5-devel bsdtar bc rsync" && \
19 19
     yum install -y $INSTALL_PKGS && \
20 20
     rpm -V $INSTALL_PKGS && \
21 21
     yum clean all && \
... ...
@@ -15,7 +15,7 @@ ENV VERSION=1.7.1 \
15 15
 ENV PATH=$PATH:$GOROOT/bin:$GOPATH/bin
16 16
 
17 17
 RUN mkdir $TMPDIR && \
18
-    INSTALL_PKGS="make gcc zip mercurial krb5-devel bsdtar bc" && \
18
+    INSTALL_PKGS="make gcc zip mercurial krb5-devel bsdtar bc rsync" && \
19 19
     yum install -y $INSTALL_PKGS && \
20 20
     rpm -V $INSTALL_PKGS && \
21 21
     yum clean all && \