Browse code

Adds support for LVM ephemeral storage in Nova

DevStack currently lacks support for LVM ephemeral storage in Nova.
This support is important for testing of Nova's LVM backend. The
proposed change adds a default volume group, to be shared by Cinder
and Nova. It also adds a configuration option NOVA_BACKEND, which
must be LVM if it is set, that determines whether Nova should be
configured to use LVM ephemeral storage.

Change-Id: I4eb9afff3536fbcd563939f2d325efbb845081bb

Daniel Genin authored on 2014/11/01 04:01:29
Showing 7 changed files
... ...
@@ -110,8 +110,8 @@ follows:
110 110
 * Global configuration that may be referenced in ``local.conf``, i.e. ``DEST``, ``DATA_DIR``
111 111
 * Global service configuration like ``ENABLED_SERVICES``
112 112
 * Variables used by multiple services that do not have a clear owner, i.e.
113
-  ``VOLUME_BACKING_FILE_SIZE`` (nova-volumes and cinder) or ``PUBLIC_NETWORK_NAME``
114
-  (nova-network and neutron)
113
+  ``VOLUME_BACKING_FILE_SIZE`` (nova-compute, nova-volumes and cinder) or
114
+  ``PUBLIC_NETWORK_NAME`` (nova-network and neutron)
115 115
 * Variables that can not be cleanly declared in a project file due to
116 116
   dependency ordering, i.e. the order of sourcing the project files can
117 117
   not be changed for other reasons but the earlier file needs to dereference a
... ...
@@ -76,9 +76,9 @@ CINDER_MULTI_LVM_BACKEND=$(trueorfalse False CINDER_MULTI_LVM_BACKEND)
76 76
 # configuration and for the volume type name.  Multiple backends are
77 77
 # comma-separated.
78 78
 if [[ $CINDER_MULTI_LVM_BACKEND == "False" ]]; then
79
-    CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1}
79
+    CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:${DEFAULT_VOLUME_GROUP_NAME##*-}}
80 80
 else
81
-    CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1,lvm:lvmdriver-2}
81
+    CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:${DEFAULT_VOLUME_GROUP_NAME##*-},lvm:cinder}
82 82
 fi
83 83
 
84 84
 
... ...
@@ -14,6 +14,7 @@
14 14
 
15 15
 # CINDER_CONF
16 16
 # DATA_DIR
17
+# VOLUME_GROUP_NAME
17 18
 
18 19
 # clean_cinder_backend_lvm - called from clean_cinder()
19 20
 # configure_cinder_backend_lvm - called from configure_cinder()
... ...
@@ -25,157 +26,44 @@ MY_XTRACE=$(set +o | grep xtrace)
25 25
 set +o xtrace
26 26
 
27 27
 
28
-# Defaults
29
-# --------
30
-
31
-# Name of the lvm volume groups to use/create for iscsi volumes
32
-# This monkey-motion is for compatibility with icehouse-generation Grenade
33
-# If ``VOLUME_GROUP`` is set, use it, otherwise we'll build a VG name based
34
-# on ``VOLUME_GROUP_NAME`` that includes the backend name
35
-# Grenade doesn't use ``VOLUME_GROUP2`` so it is left out
36
-VOLUME_GROUP=${VOLUME_GROUP:-}
37
-VOLUME_GROUP_NAME=${VOLUME_GROUP:-${VOLUME_GROUP_NAME:-stack-volumes}}
38
-
39 28
 # TODO: resurrect backing device...need to know how to set values
40 29
 #VOLUME_BACKING_DEVICE=${VOLUME_BACKING_DEVICE:-}
41 30
 
42
-VOLUME_NAME_PREFIX=${VOLUME_NAME_PREFIX:-volume-}
43
-
44
-
45 31
 # Entry Points
46 32
 # ------------
47 33
 
48
-# Compatibility for getting a volume group name from either ``VOLUME_GROUP``
49
-# or from ``VOLUME_GROUP_NAME`` plus the backend name
50
-function get_volume_group_name {
51
-    local be_name=$1
52
-
53
-    # Again with the icehouse-generation compatibility
54
-    local volume_group_name=$VOLUME_GROUP_NAME
55
-    if [[ -z $VOLUME_GROUP ]]; then
56
-        volume_group_name+="-$be_name"
57
-    fi
58
-    echo $volume_group_name
59
-}
60
-
34
+# cleanup_cinder_backend_lvm - Delete volume group and remove backing file
35
+# cleanup_cinder_backend_lvm $be_name
61 36
 function cleanup_cinder_backend_lvm {
62 37
     local be_name=$1
63 38
 
64
-    # Again with the icehouse-generation compatibility
65
-    local volume_group_name=$(get_volume_group_name $be_name)
66
-
67 39
     # Campsite rule: leave behind a volume group at least as clean as we found it
68
-    _clean_lvm_lv ${volume_group_name} $VOLUME_NAME_PREFIX
69
-    _clean_lvm_backing_file ${volume_group_name} $DATA_DIR/${volume_group_name}-backing-file
40
+    clean_lvm_volume_group $VOLUME_GROUP_NAME-$be_name
70 41
 }
71 42
 
72 43
 # configure_cinder_backend_lvm - Set config files, create data dirs, etc
73
-# configure_cinder_backend_lvm $name
44
+# configure_cinder_backend_lvm $be_name
74 45
 function configure_cinder_backend_lvm {
75 46
     local be_name=$1
76 47
 
77
-    # Again with the icehouse-generation compatibility
78
-    local volume_group_name=$(get_volume_group_name $be_name)
79
-
80 48
     iniset $CINDER_CONF $be_name volume_backend_name $be_name
81 49
     iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.lvm.LVMISCSIDriver"
82
-    iniset $CINDER_CONF $be_name volume_group $volume_group_name
50
+    iniset $CINDER_CONF $be_name volume_group $VOLUME_GROUP_NAME-$be_name
83 51
 
84 52
     if [[ "$CINDER_SECURE_DELETE" == "False" ]]; then
85 53
         iniset $CINDER_CONF $be_name volume_clear none
86 54
     fi
87 55
 }
88 56
 
89
-
57
+# init_cinder_backend_lvm - Initialize volume group
58
+# init_cinder_backend_lvm $be_name
90 59
 function init_cinder_backend_lvm {
91 60
     local be_name=$1
92 61
 
93
-    # Again with the icehouse-generation compatibility
94
-    local volume_group_name=$(get_volume_group_name $be_name)
95
-
96 62
     # Start with a clean volume group
97
-    _create_cinder_volume_group ${volume_group_name} $DATA_DIR/${volume_group_name}-backing-file
98
-
99
-    if is_fedora || is_suse; then
100
-        # service is not started by default
101
-        start_service tgtd
102
-    fi
103
-
104
-    # Remove iscsi targets
105
-    sudo tgtadm --op show --mode target | grep $VOLUME_NAME_PREFIX | grep Target | cut -f3 -d ' ' | sudo xargs -n1 tgt-admin --delete || true
106
-    _clean_lvm_lv ${volume_group_name} $VOLUME_NAME_PREFIX
107
-}
108
-
109
-
110
-# _clean_lvm_lv removes all cinder LVM volumes
111
-#
112
-# Usage: _clean_lvm_lv volume-group-name $VOLUME_NAME_PREFIX
113
-function _clean_lvm_lv {
114
-    local vg=$1
115
-    local lv_prefix=$2
116
-
117
-    # Clean out existing volumes
118
-    local lv
119
-    for lv in $(sudo lvs --noheadings -o lv_name $vg 2>/dev/null); do
120
-        # lv_prefix prefixes the LVs we want
121
-        if [[ "${lv#$lv_prefix}" != "$lv" ]]; then
122
-            sudo lvremove -f $vg/$lv
123
-        fi
124
-    done
125
-}
126
-
127
-# _clean_lvm_backing_file() removes the backing file of the
128
-# volume group used by cinder
129
-#
130
-# Usage: _clean_lvm_backing_file() volume-group-name backing-file-name
131
-function _clean_lvm_backing_file {
132
-    local vg=$1
133
-    local backing_file=$2
134
-
135
-    # if there is no logical volume left, it's safe to attempt a cleanup
136
-    # of the backing file
137
-    if [[ -z "$(sudo lvs --noheadings -o lv_name $vg 2>/dev/null)" ]]; then
138
-        # if the backing physical device is a loop device, it was probably setup by devstack
139
-        local vg_dev=$(sudo losetup -j $backing_file | awk -F':' '/backing-file/ { print $1}')
140
-        if [[ -n "$vg_dev" ]] && [[ -e "$vg_dev" ]]; then
141
-            sudo losetup -d $vg_dev
142
-            rm -f $backing_file
143
-        fi
144
-    fi
63
+    init_lvm_volume_group $VOLUME_GROUP_NAME-$be_name $VOLUME_BACKING_FILE_SIZE
145 64
 }
146 65
 
147
-# _create_cinder_volume_group volume-group-name backing-file-name
148
-function _create_cinder_volume_group {
149
-    # According to the ``CINDER_MULTI_LVM_BACKEND`` value, configure one or two default volumes
150
-    # group called ``stack-volumes`` (and ``stack-volumes2``) for the volume
151
-    # service if it (they) does (do) not yet exist. If you don't wish to use a
152
-    # file backed volume group, create your own volume group called ``stack-volumes``
153
-    # and ``stack-volumes2`` before invoking ``stack.sh``.
154
-    #
155
-    # The two backing files are ``VOLUME_BACKING_FILE_SIZE`` in size, and they are stored in
156
-    # the ``DATA_DIR``.
157
-
158
-    local vg_name=$1
159
-    local backing_file=$2
160
-
161
-    if ! sudo vgs $vg_name; then
162
-        # TODO: fix device handling
163
-        if [ -z "$VOLUME_BACKING_DEVICE" ]; then
164
-            # Only create if the file doesn't already exists
165
-            [[ -f $backing_file ]] || truncate -s $VOLUME_BACKING_FILE_SIZE $backing_file
166
-            local vg_dev=`sudo losetup -f --show $backing_file`
167
-
168
-            # Only create if the loopback device doesn't contain $VOLUME_GROUP
169
-            if ! sudo vgs $vg_name; then
170
-                sudo vgcreate $vg_name $vg_dev
171
-            fi
172
-        else
173
-            sudo vgcreate $vg_name $VOLUME_BACKING_DEVICE
174
-        fi
175
-    fi
176
-}
177
-
178
-
179 66
 # Restore xtrace
180 67
 $MY_XTRACE
181 68
 
182 69
new file mode 100644
... ...
@@ -0,0 +1,124 @@
0
+# lib/lvm
1
+# Configure the default LVM volume group used by Cinder and Nova
2
+
3
+# Dependencies:
4
+#
5
+# - ``functions`` file
6
+# - ``cinder`` configurations
7
+
8
+# DATA_DIR
9
+
10
+# clean_default_volume_group - called from clean()
11
+# configure_default_volume_group - called from configure()
12
+# init_default_volume_group - called from init()
13
+
14
+
15
+# Save trace setting
16
+MY_XTRACE=$(set +o | grep xtrace)
17
+set +o xtrace
18
+
19
+
20
+# Defaults
21
+# --------
22
+# Name of the lvm volume groups to use/create for iscsi volumes
23
+# This monkey-motion is for compatibility with icehouse-generation Grenade
24
+# If ``VOLUME_GROUP`` is set, use it, otherwise we'll build a VG name based
25
+# on ``VOLUME_GROUP_NAME`` that includes the backend name
26
+# Grenade doesn't use ``VOLUME_GROUP2`` so it is left out
27
+VOLUME_GROUP_NAME=${VOLUME_GROUP:-${VOLUME_GROUP_NAME:-stack-volumes}}
28
+DEFAULT_VOLUME_GROUP_NAME=$VOLUME_GROUP_NAME-default
29
+
30
+# Backing file name is of the form $VOLUME_GROUP$BACKING_FILE_SUFFIX
31
+BACKING_FILE_SUFFIX=-backing-file
32
+
33
+
34
+# Entry Points
35
+# ------------
36
+
37
+# _clean_lvm_volume_group removes all default LVM volumes
38
+#
39
+# Usage: clean_lvm_volume_group $vg
40
+function _clean_lvm_volume_group {
41
+    local vg=$1
42
+
43
+    # Clean out existing volumes
44
+    sudo lvremove -f $vg
45
+}
46
+
47
+# _clean_lvm_backing_file() removes the backing file of the
48
+# volume group
49
+#
50
+# Usage: _clean_lvm_backing_file() $backing_file
51
+function _clean_lvm_backing_file {
52
+    local backing_file=$1
53
+
54
+    # if the backing physical device is a loop device, it was probably setup by devstack
55
+    if [[ -n "$backing_file" ]] && [[ -e "$backing_file" ]]; then
56
+        local vg_dev=$(sudo losetup -j $backing_file | awk -F':' '/'$BACKING_FILE_SUFFIX'/ { print $1}')
57
+        sudo losetup -d $vg_dev
58
+        rm -f $backing_file
59
+    fi
60
+}
61
+
62
+# clean_lvm_volume_group() cleans up the volume group and removes the
63
+# backing file
64
+#
65
+# Usage: clean_lvm_volume_group $vg
66
+function clean_lvm_volume_group {
67
+    local vg=$1
68
+
69
+    _clean_lvm_volume_group $vg
70
+    # if there is no logical volume left, it's safe to attempt a cleanup
71
+    # of the backing file
72
+    if [[ -z "$(sudo lvs --noheadings -o lv_name $vg 2>/dev/null)" ]]; then
73
+        _clean_lvm_backing_file $DATA_DIR/$vg$BACKING_FILE_SUFFIX
74
+    fi
75
+}
76
+
77
+
78
+# _create_volume_group creates default volume group
79
+#
80
+# Usage: _create_lvm_volume_group() $vg $size
81
+function _create_lvm_volume_group {
82
+    local vg=$1
83
+    local size=$2
84
+
85
+    local backing_file=$DATA_DIR/$vg$BACKING_FILE_SUFFIX
86
+    if ! sudo vgs $vg; then
87
+        # Only create if the file doesn't already exists
88
+        [[ -f $DATA_DIR/$backing_file ]] || truncate -s $size $backing_file
89
+        local vg_dev=`sudo losetup -f --show $backing_file`
90
+
91
+        # Only create volume group if it doesn't already exist
92
+        if ! sudo vgs $vg; then
93
+            sudo vgcreate $vg $vg_dev
94
+        fi
95
+    fi
96
+}
97
+
98
+# init_lvm_volume_group() initializes the volume group creating the backing
99
+# file if necessary
100
+#
101
+# Usage: init_lvm_volume_group() $vg
102
+function init_lvm_volume_group {
103
+    local vg=$1
104
+    local size=$2
105
+    # Start with a clean volume group
106
+    _create_lvm_volume_group $vg $size
107
+
108
+    if is_fedora || is_suse; then
109
+        # service is not started by default
110
+        start_service tgtd
111
+    fi
112
+
113
+    # Remove iscsi targets
114
+    sudo tgtadm --op show --mode target | grep Target | cut -f3 -d ' ' | sudo xargs -n1 tgt-admin --delete || true
115
+
116
+    _clean_lvm_volume_group $vg
117
+}
118
+
119
+# Restore xtrace
120
+$MY_XTRACE
121
+
122
+# mode: shell-script
123
+# End:
... ...
@@ -547,6 +547,11 @@ function create_nova_conf {
547 547
     iniset $NOVA_CONF DEFAULT ec2_workers "$API_WORKERS"
548 548
     iniset $NOVA_CONF DEFAULT metadata_workers "$API_WORKERS"
549 549
 
550
+    if [[ "$NOVA_BACKEND" == "LVM" ]]; then
551
+        iniset $NOVA_CONF libvirt images_type "lvm"
552
+        iniset $NOVA_CONF libvirt images_volume_group $DEFAULT_VOLUME_GROUP_NAME
553
+    fi
554
+
550 555
     if is_ssl_enabled_service glance || is_service_enabled tls-proxy; then
551 556
         iniset $NOVA_CONF DEFAULT glance_protocol https
552 557
     fi
... ...
@@ -523,6 +523,7 @@ source $TOP_DIR/lib/tls
523 523
 source $TOP_DIR/lib/infra
524 524
 source $TOP_DIR/lib/oslo
525 525
 source $TOP_DIR/lib/stackforge
526
+source $TOP_DIR/lib/lvm
526 527
 source $TOP_DIR/lib/horizon
527 528
 source $TOP_DIR/lib/keystone
528 529
 source $TOP_DIR/lib/glance
... ...
@@ -939,6 +940,10 @@ init_service_check
939 939
 # A better kind of sysstat, with the top process per time slice
940 940
 start_dstat
941 941
 
942
+# Initialize default LVM volume group
943
+# -----------------------------------
944
+init_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME $VOLUME_BACKING_FILE_SIZE
945
+
942 946
 # Start Services
943 947
 # ==============
944 948
 
... ...
@@ -55,6 +55,7 @@ source $TOP_DIR/lib/tls
55 55
 source $TOP_DIR/lib/infra
56 56
 source $TOP_DIR/lib/oslo
57 57
 source $TOP_DIR/lib/stackforge
58
+source $TOP_DIR/lib/lvm
58 59
 source $TOP_DIR/lib/horizon
59 60
 source $TOP_DIR/lib/keystone
60 61
 source $TOP_DIR/lib/glance
... ...
@@ -177,3 +178,5 @@ if [[ -n "$SCREEN" ]]; then
177 177
         screen -X -S $SESSION quit
178 178
     fi
179 179
 fi
180
+
181
+clean_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME