Browse code

Merge pull request #143 from cloudbuilders/fix_races

Fix NBD race conditions

Jesse Andrews authored on 2011/11/02 09:34:35
Showing 5 changed files
... ...
@@ -12,6 +12,27 @@ if [ ! "oneiric" = "$UBUNTU_VERSION" ]; then
12 12
     fi
13 13
 fi
14 14
 
15
+# Clean up any resources that may be in use
16
+cleanup() {
17
+    set +o errexit
18
+    unmount_images
19
+
20
+    if [ -n "$ROOTFS" ]; then
21
+        umount $ROOTFS/dev
22
+        umount $ROOTFS
23
+    fi
24
+
25
+    # Release NBD devices
26
+    if [ -n "$NBD" ]; then
27
+        qemu-nbd -d $NBD
28
+    fi
29
+
30
+    # Kill ourselves to signal any calling process
31
+    trap 2; kill -2 $$
32
+}
33
+
34
+trap cleanup SIGHUP SIGINT SIGTERM
35
+
15 36
 # Echo commands
16 37
 set -o xtrace
17 38
 
... ...
@@ -100,9 +121,6 @@ function kill_unmount() {
100 100
 # Install deps if needed
101 101
 dpkg -l kvm libvirt-bin kpartx || apt-get install -y --force-yes kvm libvirt-bin kpartx
102 102
 
103
-# Let Ctrl-c kill tail and exit
104
-trap kill_unmount SIGINT
105
-
106 103
 # Where Openstack code will live in image
107 104
 DEST=${DEST:-/opt/stack}
108 105
 
... ...
@@ -239,26 +257,35 @@ rm -f $VM_DIR/disk
239 239
 # Create our instance fs
240 240
 qemu-img create -f qcow2 -b $VM_IMAGE disk
241 241
 
242
+# Finds the next available NBD device
243
+# Exits script if error connecting or none free
244
+# map_nbd image
245
+# returns full nbd device path
246
+function map_nbd {
247
+    for i in `seq 0 15`; do
248
+        if [ ! -e /sys/block/nbd$i/pid ]; then
249
+            NBD=/dev/nbd$i
250
+            # Connect to nbd and wait till it is ready
251
+            qemu-nbd -c $NBD $1
252
+            if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then
253
+                echo "Couldn't connect $NBD"
254
+                exit 1
255
+            fi
256
+            break
257
+        fi
258
+    done
259
+    if [ -z "$NBD" ]; then
260
+        echo "No free NBD slots"
261
+        exit 1
262
+    fi
263
+    echo $NBD
264
+}
265
+
242 266
 # Make sure we have nbd-ness
243 267
 modprobe nbd max_part=63
244 268
 
245 269
 # Set up nbd
246
-for i in `seq 0 15`; do
247
-    if [ ! -e /sys/block/nbd$i/pid ]; then
248
-        NBD=/dev/nbd$i
249
-        # Connect to nbd and wait till it is ready
250
-        qemu-nbd -c $NBD disk
251
-        if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then
252
-            echo "Couldn't connect $NBD"
253
-            exit 1
254
-        fi
255
-        break
256
-    fi
257
-done
258
-if [ -z "$NBD" ]; then
259
-    echo "No free NBD slots"
260
-    exit 1
261
-fi
270
+NBD=`map_nbd disk`
262 271
 NBD_DEV=`basename $NBD`
263 272
 
264 273
 # Mount the instance
... ...
@@ -381,7 +408,9 @@ sed -e 's/^PasswordAuthentication.*$/PasswordAuthentication yes/' -i $ROOTFS/etc
381 381
 
382 382
 # Unmount
383 383
 umount $ROOTFS || echo 'ok'
384
+ROOTFS=""
384 385
 qemu-nbd -d $NBD
386
+NBD=""
385 387
 
386 388
 # Create the instance
387 389
 cd $VM_DIR && virsh create libvirt.xml
... ...
@@ -11,6 +11,22 @@ PXEDIR=${PXEDIR:-/var/cache/devstack/pxe}
11 11
 OPWD=`pwd`
12 12
 PROGDIR=`dirname $0`
13 13
 
14
+# Clean up any resources that may be in use
15
+cleanup() {
16
+    set +o errexit
17
+
18
+    # Mop up temporary files
19
+    if [ -n "$MNTDIR" -a -d "$MNTDIR" ]; then
20
+        umount $MNTDIR
21
+        rmdir $MNTDIR
22
+    fi
23
+
24
+    # Kill ourselves to signal any calling process
25
+    trap 2; kill -2 $$
26
+}
27
+
28
+trap cleanup SIGHUP SIGINT SIGTERM
29
+
14 30
 mkdir -p $DEST_DIR/pxelinux.cfg
15 31
 cd $DEST_DIR
16 32
 for i in memdisk menu.c32 pxelinux.0; do
... ...
@@ -1,55 +1,114 @@
1 1
 #!/bin/bash
2 2
 # build_ramdisk.sh - Build RAM disk images
3 3
 
4
+# exit on error to stop unexpected errors
5
+set -o errexit
6
+
4 7
 if [ ! "$#" -eq "1" ]; then
5
-    echo "$0 builds a gziped natty openstack install"
8
+    echo "$0 builds a gziped Ubuntu OpenStack install"
6 9
     echo "usage: $0 dest"
7 10
     exit 1
8 11
 fi
9 12
 
13
+# Clean up any resources that may be in use
14
+cleanup() {
15
+    set +o errexit
16
+
17
+    # Mop up temporary files
18
+    if [ -n "$MNTDIR" -a -d "$MNTDIR" ]; then
19
+        umount $MNTDIR
20
+        rmdir $MNTDIR
21
+    fi
22
+    if [ -n "$DEV_FILE_TMP" -a -e "$DEV_FILE_TMP "]; then
23
+        rm -f $DEV_FILE_TMP
24
+    fi
25
+    if [ -n "$IMG_FILE_TMP" -a -e "$IMG_FILE_TMP" ]; then
26
+        rm -f $IMG_FILE_TMP
27
+    fi
28
+
29
+    # Release NBD devices
30
+    if [ -n "$NBD" ]; then
31
+        qemu-nbd -d $NBD
32
+    fi
33
+
34
+    # Kill ourselves to signal any calling process
35
+    trap 2; kill -2 $$
36
+}
37
+
38
+trap cleanup SIGHUP SIGINT SIGTERM
39
+
40
+# Set up nbd
41
+modprobe nbd max_part=63
42
+
43
+# Echo commands
44
+set -o xtrace
45
+
10 46
 IMG_FILE=$1
11 47
 
12
-PROGDIR=`dirname $0`
13
-CHROOTCACHE=${CHROOTCACHE:-/var/cache/devstack}
48
+# Keep track of the current directory
49
+TOOLS_DIR=$(cd $(dirname "$0") && pwd)
50
+TOP_DIR=`cd $TOOLS_DIR/..; pwd`
51
+
52
+# Store cwd
53
+CWD=`pwd`
54
+
55
+cd $TOP_DIR
14 56
 
15 57
 # Source params
16 58
 source ./stackrc
17 59
 
18
-# Store cwd
19
-CWD=`pwd`
60
+CACHEDIR=${CACHEDIR:-/var/cache/devstack}
20 61
 
21 62
 DEST=${DEST:-/opt/stack}
22 63
 
64
+# Configure the root password of the vm to be the same as ``ADMIN_PASSWORD``
65
+ROOT_PASSWORD=${ADMIN_PASSWORD:-password}
66
+
67
+# Base image (natty by default)
68
+DIST_NAME=${DIST_NAME:-natty}
69
+
23 70
 # Param string to pass to stack.sh.  Like "EC2_DMZ_HOST=192.168.1.1 MYSQL_USER=nova"
24 71
 STACKSH_PARAMS=${STACKSH_PARAMS:-}
25 72
 
26 73
 # Option to use the version of devstack on which we are currently working
27 74
 USE_CURRENT_DEVSTACK=${USE_CURRENT_DEVSTACK:-1}
28 75
 
29
-# Set up nbd
30
-modprobe nbd max_part=63
31
-NBD=${NBD:-/dev/nbd9}
32
-NBD_DEV=`basename $NBD`
33
-
34
-# clean install of natty
35
-if [ ! -r $CHROOTCACHE/natty-base.img ]; then
36
-    $PROGDIR/get_uec_image.sh natty $CHROOTCACHE/natty-base.img
37
-#    # copy kernel modules...
38
-#    # NOTE(ja): is there a better way to do this?
39
-#    cp -pr /lib/modules/`uname -r` $CHROOTCACHE/natty-base/lib/modules
40
-#    # a simple password - pass
41
-#    echo root:pass | chroot $CHROOTCACHE/natty-base chpasswd
76
+# clean install
77
+if [ ! -r $CACHEDIR/$DIST_NAME-base.img ]; then
78
+    $TOOLS_DIR/get_uec_image.sh $DIST_NAME $CACHEDIR/$DIST_NAME-base.img
42 79
 fi
43 80
 
44
-# prime natty with as many apt/pips as we can
45
-if [ ! -r $CHROOTCACHE/natty-dev.img ]; then
46
-    cp -p $CHROOTCACHE/natty-base.img $CHROOTCACHE/natty-dev.img
47
-
48
-    qemu-nbd -c $NBD $CHROOTCACHE/natty-dev.img
49
-    if ! timeout 60 sh -c "while ! [ -e /sys/block/$NBD_DEV/pid ]; do sleep 1; done"; then
50
-        echo "Couldn't connect $NBD"
81
+# Finds the next available NBD device
82
+# Exits script if error connecting or none free
83
+# map_nbd image
84
+# returns full nbd device path
85
+function map_nbd {
86
+    for i in `seq 0 15`; do
87
+        if [ ! -e /sys/block/nbd$i/pid ]; then
88
+            NBD=/dev/nbd$i
89
+            # Connect to nbd and wait till it is ready
90
+            qemu-nbd -c $NBD $1
91
+            if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then
92
+                echo "Couldn't connect $NBD"
93
+                exit 1
94
+            fi
95
+            break
96
+        fi
97
+    done
98
+    if [ -z "$NBD" ]; then
99
+        echo "No free NBD slots"
51 100
         exit 1
52 101
     fi
102
+    echo $NBD
103
+}
104
+
105
+# prime image with as many apt/pips as we can
106
+DEV_FILE=$CACHEDIR/$DIST_NAME-dev.img
107
+DEV_FILE_TMP=`mktemp $DEV_FILE.XXXXXX`
108
+if [ ! -r $DEV_FILE ]; then
109
+    cp -p $CACHEDIR/$DIST_NAME-base.img $DEV_FILE_TMP
110
+
111
+    NBD=`map_nbd $DEV_FILE_TMP`
53 112
     MNTDIR=`mktemp -d --tmpdir mntXXXXXXXX`
54 113
     mount -t ext4 ${NBD}p1 $MNTDIR
55 114
     cp -p /etc/resolv.conf $MNTDIR/etc/resolv.conf
... ...
@@ -66,6 +125,7 @@ if [ ! -r $CHROOTCACHE/natty-dev.img ]; then
66 66
 
67 67
     # a simple password - pass
68 68
     echo stack:pass | chroot $MNTDIR chpasswd
69
+    echo root:$ROOT_PASSWORD | chroot $MNTDIR chpasswd
69 70
 
70 71
     # and has sudo ability (in the future this should be limited to only what
71 72
     # stack requires)
... ...
@@ -74,27 +134,31 @@ if [ ! -r $CHROOTCACHE/natty-dev.img ]; then
74 74
     umount $MNTDIR
75 75
     rmdir $MNTDIR
76 76
     qemu-nbd -d $NBD
77
+    NBD=""
78
+    mv $DEV_FILE_TMP $DEV_FILE
77 79
 fi
80
+rm -f $DEV_FILE_TMP
78 81
 
79 82
 # clone git repositories onto the system
80 83
 # ======================================
81 84
 
85
+IMG_FILE_TMP=`mktemp $IMG_FILE.XXXXXX`
86
+
82 87
 if [ ! -r $IMG_FILE ]; then
83
-    qemu-nbd -c $NBD $CHROOTCACHE/natty-dev.img
84
-    if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then
85
-        echo "Couldn't connect $NBD"
86
-        exit 1
87
-    fi
88
+    NBD=`map_nbd $DEV_FILE`
88 89
 
89 90
     # Pre-create the image file
90 91
     # FIXME(dt): This should really get the partition size to
91 92
     #            pre-create the image file
92
-    dd if=/dev/zero of=$IMG_FILE bs=1 count=1 seek=$((2*1024*1024*1024))
93
+    dd if=/dev/zero of=$IMG_FILE_TMP bs=1 count=1 seek=$((2*1024*1024*1024))
93 94
     # Create filesystem image for RAM disk
94
-    dd if=${NBD}p1 of=$IMG_FILE bs=1M
95
+    dd if=${NBD}p1 of=$IMG_FILE_TMP bs=1M
95 96
 
96 97
     qemu-nbd -d $NBD
98
+    NBD=""
99
+    mv $IMG_FILE_TMP $IMG_FILE
97 100
 fi
101
+rm -f $IMG_FILE_TMP
98 102
 
99 103
 MNTDIR=`mktemp -d --tmpdir mntXXXXXXXX`
100 104
 mount -t ext4 -o loop $IMG_FILE $MNTDIR
... ...
@@ -11,6 +11,26 @@ PXEDIR=${PXEDIR:-/var/cache/devstack/pxe}
11 11
 OPWD=`pwd`
12 12
 PROGDIR=`dirname $0`
13 13
 
14
+# Clean up any resources that may be in use
15
+cleanup() {
16
+    set +o errexit
17
+
18
+    # Mop up temporary files
19
+    if [ -n "$DEST_DEV" ]; then
20
+        umount $DEST_DIR
21
+        rmdir $DEST_DIR
22
+    fi
23
+    if [ -n "$MNTDIR" -a -d "$MNTDIR" ]; then
24
+        umount $MNTDIR
25
+        rmdir $MNTDIR
26
+    fi
27
+
28
+    # Kill ourselves to signal any calling process
29
+    trap 2; kill -2 $$
30
+}
31
+
32
+trap cleanup SIGHUP SIGINT SIGTERM
33
+
14 34
 if [ -b $DEST_DIR ]; then
15 35
     # We have a block device, install syslinux and mount it
16 36
     DEST_DEV=$DEST_DIR
... ...
@@ -26,6 +26,24 @@ usage() {
26 26
     exit 1
27 27
 }
28 28
 
29
+# Clean up any resources that may be in use
30
+cleanup() {
31
+    set +o errexit
32
+
33
+    # Mop up temporary files
34
+    if [ -n "$IMG_FILE_TMP" -a -e "$IMG_FILE_TMP" ]; then
35
+        rm -f $IMG_FILE_TMP
36
+    fi
37
+
38
+    # Release NBD devices
39
+    if [ -n "$NBD" ]; then
40
+        qemu-nbd -d $NBD
41
+    fi
42
+
43
+    # Kill ourselves to signal any calling process
44
+    trap 2; kill -2 $$
45
+}
46
+
29 47
 while getopts f:hmr: c; do
30 48
     case $c in
31 49
         f)  FORMAT=$OPTARG
... ...
@@ -89,6 +107,8 @@ case $DIST_NAME in
89 89
                 ;;
90 90
 esac
91 91
 
92
+trap cleanup SIGHUP SIGINT SIGTERM
93
+
92 94
 # Prepare the base image
93 95
 
94 96
 # Get the UEC image
... ...
@@ -111,25 +131,33 @@ if [ $ROOTSIZE -gt 2000 ]; then
111 111
     qemu-img resize $IMG_FILE_TMP +$((ROOTSIZE - 2000))M
112 112
 fi
113 113
 
114
-# Set up nbd
115
-modprobe nbd max_part=63
116
-for i in `seq 1 15`; do
117
-    if [ ! -e /sys/block/nbd$i/pid ]; then
118
-        NBD=/dev/nbd$i
119
-        # Connect to nbd and wait till it is ready
120
-        qemu-nbd -c $NBD $IMG_FILE_TMP
121
-        if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then
122
-            echo "Couldn't connect $NBD"
123
-            exit 1
114
+# Finds the next available NBD device
115
+# Exits script if error connecting or none free
116
+# map_nbd image
117
+# returns full nbd device path
118
+function map_nbd {
119
+    for i in `seq 0 15`; do
120
+        if [ ! -e /sys/block/nbd$i/pid ]; then
121
+            NBD=/dev/nbd$i
122
+            # Connect to nbd and wait till it is ready
123
+            qemu-nbd -c $NBD $1
124
+            if ! timeout 60 sh -c "while ! [ -e ${NBD}p1 ]; do sleep 1; done"; then
125
+                echo "Couldn't connect $NBD"
126
+                exit 1
127
+            fi
128
+            break
124 129
         fi
125
-        break
130
+    done
131
+    if [ -z "$NBD" ]; then
132
+        echo "No free NBD slots"
133
+        exit 1
126 134
     fi
127
-done
128
-if [ -z "$NBD" ]; then
129
-    echo "No free NBD slots"
130
-    exit 1
131
-fi
132
-NBD_DEV=`basename $NBD`
135
+    echo $NBD
136
+}
137
+
138
+# Set up nbd
139
+modprobe nbd max_part=63
140
+NBD=`map_nbd $IMG_FILE_TMP`
133 141
 
134 142
 # Resize partition 1 to full size of the disk image
135 143
 echo "d
... ...
@@ -162,5 +190,6 @@ rm -f $MNTDIR/etc/resolv.conf
162 162
 umount $MNTDIR
163 163
 rmdir $MNTDIR
164 164
 qemu-nbd -d $NBD
165
+NBD=""
165 166
 
166 167
 mv $IMG_FILE_TMP $IMG_FILE