Browse code

Update "download-frozen-image-v2.sh" such that redirects are optional

If the registry responds directly with blob contents, use them,
otherwise follow the redirect without Authorization headers (which
likely aren't valid for the server being redirected to).

This preserves the basic structure of the previous output with up to one
additional progress bar per-layer (for the redirect request and then the
following blob request).

Signed-off-by: Tianon Gravi <admwiggin@gmail.com>

Tianon Gravi authored on 2017/06/21 02:56:13
Showing 1 changed files
... ...
@@ -44,17 +44,42 @@ if [ "$(go env GOHOSTOS)" = 'windows' ]; then
44 44
 	fi
45 45
 fi
46 46
 
47
+registryBase='https://registry-1.docker.io'
48
+authBase='https://auth.docker.io'
49
+authService='registry.docker.io'
50
+
51
+# https://github.com/moby/moby/issues/33700
47 52
 fetch_blob() {
48
-	url=$1
49
-	token=$2
50
-	dest=$3
51
-	echo "Attempting to download blob $url"
52
-	target=$(curl -sS -v -H "Authorization: Bearer $token" "$url" 2>&1 | grep "Location:" | sed 's/< Location: \(.*\)\r/\1/')
53
-	# curl blob (exclude auth token)
54
-	curl -fsS --progress "${target}" -o "$dest"
53
+	local token="$1"; shift
54
+	local image="$1"; shift
55
+	local digest="$1"; shift
56
+	local targetFile="$1"; shift
57
+	local curlArgs=( "$@" )
58
+
59
+	local curlHeaders="$(
60
+		curl -S "${curlArgs[@]}" \
61
+			-H "Authorization: Bearer $token" \
62
+			"$registryBase/v2/$image/blobs/$digest" \
63
+			-o "$targetFile" \
64
+			-D-
65
+	)"
66
+	curlHeaders="$(echo "$curlHeaders" | tr -d '\r')"
67
+	if [ "$(echo "$curlHeaders" | awk 'NR == 1 { print $2; exit }')" != '200' ]; then
68
+		rm -f "$targetFile"
69
+
70
+		local blobRedirect="$(echo "$curlHeaders" | awk -F ': ' 'tolower($1) == "location" { print $2; exit }')"
71
+		if [ -z "$blobRedirect" ]; then
72
+			echo >&2 "error: failed fetching '$image' blob '$digest'"
73
+			echo "$curlHeaders" | head -1 >&2
74
+			return 1
75
+		fi
76
+
77
+		curl -fSL "${curlArgs[@]}" \
78
+			"$blobRedirect" \
79
+			-o "$targetFile"
80
+	fi
55 81
 }
56 82
 
57
-
58 83
 while [ $# -gt 0 ]; do
59 84
 	imageTag="$1"
60 85
 	shift
... ...
@@ -70,14 +95,14 @@ while [ $# -gt 0 ]; do
70 70
 
71 71
 	imageFile="${image//\//_}" # "/" can't be in filenames :)
72 72
 
73
-	token="$(curl -fsSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output '.token')"
73
+	token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
74 74
 
75 75
 	manifestJson="$(
76 76
 		curl -fsSL \
77 77
 			-H "Authorization: Bearer $token" \
78 78
 			-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
79 79
 			-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
80
-			"https://registry-1.docker.io/v2/$image/manifests/$digest"
80
+			"$registryBase/v2/$image/manifests/$digest"
81 81
 	)"
82 82
 	if [ "${manifestJson:0:1}" != '{' ]; then
83 83
 		echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:"
... ...
@@ -98,7 +123,7 @@ while [ $# -gt 0 ]; do
98 98
 					imageId="${configDigest#*:}" # strip off "sha256:"
99 99
 
100 100
 					configFile="$imageId.json"
101
-					fetch_blob "https://registry-1.docker.io/v2/$image/blobs/$configDigest" $token "$dir/$configFile"
101
+					fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s
102 102
 
103 103
 					layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')"
104 104
 					IFS="$newlineIFS"
... ...
@@ -165,8 +190,8 @@ while [ $# -gt 0 ]; do
165 165
 									echo "skipping existing ${layerId:0:12}"
166 166
 									continue
167 167
 								fi
168
-								token="$(curl -fsSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output '.token')"
169
-								fetch_blob "https://registry-1.docker.io/v2/$image/blobs/$layerDigest" $token "$dir/$layerTar"
168
+								token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
169
+								fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress
170 170
 								;;
171 171
 
172 172
 							*)
... ...
@@ -235,9 +260,8 @@ while [ $# -gt 0 ]; do
235 235
 					echo "skipping existing ${layerId:0:12}"
236 236
 					continue
237 237
 				fi
238
-				token="$(curl -fsSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output '.token')"
239
-				# find redirect using token:
240
-				fetch_blob "https://registry-1.docker.io/v2/$image/blobs/$imageLayer" $token "$dir/$layerId/layer.tar"
238
+				token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
239
+				fetch_blob "$token" "$image" "$imageLayer" "$dir/$layerId/layer.tar" --progress
241 240
 			done
242 241
 			;;
243 242