We need this for every devstack run now, so downloading it from github
every time isn't the most awesome thing in the world.
Add an extra variable EXTRA_CACHE_URLS which will be appended to the
output of tools/image_list.sh. This way, these files will be
downloaded during the daily nodepool build, but they will not be in
the IMAGE_LIST and hence be considered as images to upload.
Add a function get_extra_file which echos the path to a file given the
URL. It will first check the cache at $FILES, and if not present
download it.
Update the documentation in image_list.sh to reflect what's happening.
Move the defaults for etcd variables into stackrc, since it is a base
service now.
Change-Id: I86104824a29d973a6288df1f24b7891feb86267c
... | ... |
@@ -45,6 +45,37 @@ function short_source { |
45 | 45 |
# export it so child shells have access to the 'short_source' function also. |
46 | 46 |
export -f short_source |
47 | 47 |
|
48 |
+# Download a file from a URL |
|
49 |
+# |
|
50 |
+# Will check cache (in $FILES) or download given URL. |
|
51 |
+# |
|
52 |
+# Argument is the URL to the remote file |
|
53 |
+# |
|
54 |
+# Will echo the local path to the file as the output. Will die on |
|
55 |
+# failure to download. |
|
56 |
+# |
|
57 |
+# Files can be pre-cached for CI environments, see EXTRA_CACHE_URLS |
|
58 |
+# and tools/image_list.sh |
|
59 |
+function get_extra_file { |
|
60 |
+ local file_url=$1 |
|
61 |
+ |
|
62 |
+ file_name=$(basename "$file_url") |
|
63 |
+ if [[ $file_url != file* ]]; then |
|
64 |
+ # If the file isn't cache, download it |
|
65 |
+ if [[ ! -f $FILES/$file_name ]]; then |
|
66 |
+ wget --progress=dot:giga -c $file_url -O $FILES/$file_name |
|
67 |
+ if [[ $? -ne 0 ]]; then |
|
68 |
+ die "$file_url could not be downloaded" |
|
69 |
+ fi |
|
70 |
+ fi |
|
71 |
+ echo "$FILES/$file_name" |
|
72 |
+ return |
|
73 |
+ else |
|
74 |
+ # just strip the file:// bit and that's the path to the file |
|
75 |
+ echo $file_url | sed 's/$file:\/\///g' |
|
76 |
+ fi |
|
77 |
+} |
|
78 |
+ |
|
48 | 79 |
|
49 | 80 |
# Retrieve an image from a URL and upload into Glance. |
50 | 81 |
# Uses the following variables: |
... | ... |
@@ -24,15 +24,9 @@ set +o xtrace |
24 | 24 |
# -------- |
25 | 25 |
|
26 | 26 |
# Set up default values for etcd |
27 |
-ETCD_DOWNLOAD_URL=${ETCD_DOWNLOAD_URL:-https://github.com/coreos/etcd/releases/download} |
|
28 |
-ETCD_VERSION=${ETCD_VERSION:-v3.1.7} |
|
29 | 27 |
ETCD_DATA_DIR="$DATA_DIR/etcd" |
30 | 28 |
ETCD_SYSTEMD_SERVICE="devstack@etcd.service" |
31 | 29 |
ETCD_BIN_DIR="$DEST/bin" |
32 |
-ETCD_SHA256_AMD64="4fde194bbcd259401e2b5c462dfa579ee7f6af539f13f130b8f5b4f52e3b3c52" |
|
33 |
-# NOTE(sdague): etcd v3.1.7 doesn't have anything for these architectures, though 3.2.0 does. |
|
34 |
-ETCD_SHA256_ARM64="" |
|
35 |
-ETCD_SHA256_PPC64="" |
|
36 | 30 |
ETCD_PORT=2379 |
37 | 31 |
|
38 | 32 |
if is_ubuntu ; then |
... | ... |
@@ -95,37 +89,19 @@ function cleanup_etcd3 { |
95 | 95 |
function install_etcd3 { |
96 | 96 |
echo "Installing etcd" |
97 | 97 |
|
98 |
- # Make sure etcd3 downloads the correct architecture |
|
99 |
- if is_arch "x86_64"; then |
|
100 |
- ETCD_ARCH="amd64" |
|
101 |
- ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_AMD64} |
|
102 |
- elif is_arch "aarch64"; then |
|
103 |
- ETCD_ARCH="arm64" |
|
104 |
- ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_ARM64} |
|
105 |
- elif is_arch "ppc64le"; then |
|
106 |
- ETCD_ARCH="ppc64le" |
|
107 |
- ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_PPC64} |
|
108 |
- else |
|
109 |
- exit_distro_not_supported "invalid hardware type - $ETCD_ARCH" |
|
110 |
- fi |
|
111 |
- |
|
112 |
- ETCD_NAME=etcd-$ETCD_VERSION-linux-$ETCD_ARCH |
|
113 |
- |
|
114 | 98 |
# Create the necessary directories |
115 | 99 |
sudo mkdir -p $ETCD_BIN_DIR |
116 | 100 |
sudo mkdir -p $ETCD_DATA_DIR |
117 | 101 |
|
118 | 102 |
# Download and cache the etcd tgz for subsequent use |
103 |
+ local etcd_file |
|
104 |
+ etcd_file="$(get_extra_file $ETCD_DOWNLOAD_LOCATION)" |
|
119 | 105 |
if [ ! -f "$FILES/etcd-$ETCD_VERSION-linux-$ETCD_ARCH/etcd" ]; then |
120 |
- ETCD_DOWNLOAD_FILE=$ETCD_NAME.tar.gz |
|
121 |
- if [ ! -f "$FILES/$ETCD_DOWNLOAD_FILE" ]; then |
|
122 |
- wget $ETCD_DOWNLOAD_URL/$ETCD_VERSION/$ETCD_DOWNLOAD_FILE -O $FILES/$ETCD_DOWNLOAD_FILE |
|
123 |
- fi |
|
124 |
- echo "${ETCD_SHA256} $FILES/${ETCD_DOWNLOAD_FILE}" > $FILES/etcd.sha256sum |
|
106 |
+ echo "${ETCD_SHA256} $etcd_file" > $FILES/etcd.sha256sum |
|
125 | 107 |
# NOTE(sdague): this should go fatal if this fails |
126 | 108 |
sha256sum -c $FILES/etcd.sha256sum |
127 | 109 |
|
128 |
- tar xzvf $FILES/$ETCD_DOWNLOAD_FILE -C $FILES |
|
110 |
+ tar xzvf $etcd_file -C $FILES |
|
129 | 111 |
sudo cp $FILES/$ETCD_NAME/etcd $ETCD_BIN_DIR/etcd |
130 | 112 |
fi |
131 | 113 |
if [ ! -f "$ETCD_BIN_DIR/etcd" ]; then |
... | ... |
@@ -732,6 +732,40 @@ if [[ "$DOWNLOAD_DEFAULT_IMAGES" == "True" ]]; then |
732 | 732 |
DOWNLOAD_DEFAULT_IMAGES=False |
733 | 733 |
fi |
734 | 734 |
|
735 |
+# This is a comma separated list of extra URLS to be listed for |
|
736 |
+# download by the tools/image_list.sh script. CI environments can |
|
737 |
+# pre-download these URLS and place them in $FILES. Later scripts can |
|
738 |
+# then use "get_extra_file <url>" which will print out the path to the |
|
739 |
+# file; it will either be downloaded on demand or acquired from the |
|
740 |
+# cache if there. |
|
741 |
+EXTRA_CACHE_URLS="" |
|
742 |
+ |
|
743 |
+# etcd3 defaults |
|
744 |
+ETCD_VERSION=${ETCD_VERSION:-v3.1.7} |
|
745 |
+ETCD_SHA256_AMD64="4fde194bbcd259401e2b5c462dfa579ee7f6af539f13f130b8f5b4f52e3b3c52" |
|
746 |
+# NOTE(sdague): etcd v3.1.7 doesn't have anything for these architectures, though 3.2.0 does. |
|
747 |
+ETCD_SHA256_ARM64="" |
|
748 |
+ETCD_SHA256_PPC64="" |
|
749 |
+# Make sure etcd3 downloads the correct architecture |
|
750 |
+if is_arch "x86_64"; then |
|
751 |
+ ETCD_ARCH="amd64" |
|
752 |
+ ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_AMD64} |
|
753 |
+elif is_arch "aarch64"; then |
|
754 |
+ ETCD_ARCH="arm64" |
|
755 |
+ ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_ARM64} |
|
756 |
+elif is_arch "ppc64le"; then |
|
757 |
+ ETCD_ARCH="ppc64le" |
|
758 |
+ ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_PPC64} |
|
759 |
+else |
|
760 |
+ exit_distro_not_supported "invalid hardware type - $ETCD_ARCH" |
|
761 |
+fi |
|
762 |
+ETCD_DOWNLOAD_URL=${ETCD_DOWNLOAD_URL:-https://github.com/coreos/etcd/releases/download} |
|
763 |
+ETCD_NAME=etcd-$ETCD_VERSION-linux-$ETCD_ARCH |
|
764 |
+ETCD_DOWNLOAD_FILE=$ETCD_NAME.tar.gz |
|
765 |
+ETCD_DOWNLOAD_LOCATION=$ETCD_DOWNLOAD_URL/$ETCD_VERSION/$ETCD_DOWNLOAD_FILE |
|
766 |
+# etcd is always required, so place it into list of pre-cached downloads |
|
767 |
+EXTRA_CACHE_URLS+=",$ETCD_DOWNLOAD_LOCATION" |
|
768 |
+ |
|
735 | 769 |
# Detect duplicate values in IMAGE_URLS |
736 | 770 |
for image_url in ${IMAGE_URLS//,/ }; do |
737 | 771 |
if [ $(echo "$IMAGE_URLS" | grep -o -F "$image_url" | wc -l) -gt 1 ]; then |
... | ... |
@@ -1,5 +1,14 @@ |
1 | 1 |
#!/bin/bash |
2 | 2 |
|
3 |
+# Print out a list of image and other files to download for caching. |
|
4 |
+# This is mostly used by the OpenStack infrasturucture during daily |
|
5 |
+# image builds to save the large images to /opt/cache/files (see [1]) |
|
6 |
+# |
|
7 |
+# The two lists of URL's downloaded are the IMAGE_URLS and |
|
8 |
+# EXTRA_CACHE_URLS, which are setup in stackrc |
|
9 |
+# |
|
10 |
+# [1] project-config:nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos |
|
11 |
+ |
|
3 | 12 |
# Keep track of the DevStack directory |
4 | 13 |
TOP_DIR=$(cd $(dirname "$0")/.. && pwd) |
5 | 14 |
|
... | ... |
@@ -31,12 +40,20 @@ for driver in $DRIVERS; do |
31 | 31 |
ALL_IMAGES+=$URLS |
32 | 32 |
done |
33 | 33 |
|
34 |
-# Make a nice list |
|
35 |
-echo $ALL_IMAGES | tr ',' '\n' | sort | uniq |
|
36 |
- |
|
37 | 34 |
# Sanity check - ensure we have a minimum number of images |
38 | 35 |
num=$(echo $ALL_IMAGES | tr ',' '\n' | sort | uniq | wc -l) |
39 | 36 |
if [[ "$num" -lt 4 ]]; then |
40 | 37 |
echo "ERROR: We only found $num images in $ALL_IMAGES, which can't be right." |
41 | 38 |
exit 1 |
42 | 39 |
fi |
40 |
+ |
|
41 |
+# This is extra non-image files that we want pre-cached. This is kept |
|
42 |
+# in a separate list because devstack loops over the IMAGE_LIST to |
|
43 |
+# upload files glance and these aren't images. (This was a bit of an |
|
44 |
+# after-thought which is why the naming around this is very |
|
45 |
+# image-centric) |
|
46 |
+URLS=$(source $TOP_DIR/stackrc && echo $EXTRA_CACHE_URLS) |
|
47 |
+ALL_IMAGES+=$URLS |
|
48 |
+ |
|
49 |
+# Make a nice combined list |
|
50 |
+echo $ALL_IMAGES | tr ',' '\n' | sort | uniq |