... | ... |
@@ -18,8 +18,8 @@ class MultiPartUpload(object): |
18 | 18 |
self.s3 = s3 |
19 | 19 |
self.file = file |
20 | 20 |
self.uri = uri |
21 |
- self.upload_id = None |
|
22 | 21 |
self.parts = {} |
22 |
+ self.upload_id = self.initiate_multipart_upload() |
|
23 | 23 |
|
24 | 24 |
def initiate_multipart_upload(self): |
25 | 25 |
""" |
... | ... |
@@ -35,7 +35,7 @@ class MultiPartUpload(object): |
35 | 35 |
def upload_all_parts(self): |
36 | 36 |
""" |
37 | 37 |
Execute a full multipart upload on a file |
38 |
- Returns the id/etag dict |
|
38 |
+ Returns the seq/etag dict |
|
39 | 39 |
TODO use num_processes to thread it |
40 | 40 |
""" |
41 | 41 |
if not self.upload_id: |
... | ... |
@@ -46,38 +46,38 @@ class MultiPartUpload(object): |
46 | 46 |
nr_parts = file_size / self.chunk_size + (file_size % self.chunk_size and 1) |
47 | 47 |
debug("MultiPart: Uploading %s in %d parts" % (self.file.name, nr_parts)) |
48 | 48 |
|
49 |
- id = 1 |
|
49 |
+ seq = 1 |
|
50 | 50 |
while size_left > 0: |
51 |
- offset = self.chunk_size * (id - 1) |
|
51 |
+ offset = self.chunk_size * (seq - 1) |
|
52 | 52 |
current_chunk_size = min(file_size - offset, self.chunk_size) |
53 | 53 |
size_left -= current_chunk_size |
54 | 54 |
labels = { |
55 | 55 |
'source' : unicodise(self.file.name), |
56 | 56 |
'destination' : unicodise(self.uri.uri()), |
57 |
- 'extra' : "[part %d of %d, %s]" % (id, nr_parts, "%d%sB" % formatSize(current_chunk_size, human_readable = True)) |
|
57 |
+ 'extra' : "[part %d of %d, %s]" % (seq, nr_parts, "%d%sB" % formatSize(current_chunk_size, human_readable = True)) |
|
58 | 58 |
} |
59 | 59 |
try: |
60 |
- self.upload_part(id, offset, current_chunk_size, labels) |
|
60 |
+ self.upload_part(seq, offset, current_chunk_size, labels) |
|
61 | 61 |
except S3UploadError, e: |
62 |
- error(u"Upload of '%s' part %d failed too many times. Aborting multipart upload." % (self.file.name, id)) |
|
62 |
+ error(u"Upload of '%s' part %d failed too many times. Aborting multipart upload." % (self.file.name, seq)) |
|
63 | 63 |
self.abort_upload() |
64 | 64 |
raise e |
65 |
- id += 1 |
|
65 |
+ seq += 1 |
|
66 | 66 |
|
67 |
- debug("MultiPart: Upload finished: %d parts", id - 1) |
|
67 |
+ debug("MultiPart: Upload finished: %d parts", seq - 1) |
|
68 | 68 |
|
69 |
- def upload_part(self, id, offset, chunk_size, labels): |
|
69 |
+ def upload_part(self, seq, offset, chunk_size, labels): |
|
70 | 70 |
""" |
71 | 71 |
Upload a file chunk |
72 | 72 |
http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html |
73 | 73 |
""" |
74 | 74 |
# TODO implement Content-MD5 |
75 |
- debug("Uploading part %i of %r (%s bytes)" % (id, self.upload_id, chunk_size)) |
|
75 |
+ debug("Uploading part %i of %r (%s bytes)" % (seq, self.upload_id, chunk_size)) |
|
76 | 76 |
headers = { "content-length": chunk_size } |
77 |
- query_string = "?partNumber=%i&uploadId=%s" % (id, self.upload_id) |
|
77 |
+ query_string = "?partNumber=%i&uploadId=%s" % (seq, self.upload_id) |
|
78 | 78 |
request = self.s3.create_request("OBJECT_PUT", uri = self.uri, headers = headers, extra = query_string) |
79 | 79 |
response = self.s3.send_file(request, self.file, labels, offset = offset, chunk_size = chunk_size) |
80 |
- self.parts[id] = response["headers"]["etag"] |
|
80 |
+ self.parts[seq] = response["headers"]["etag"] |
|
81 | 81 |
return response |
82 | 82 |
|
83 | 83 |
def complete_multipart_upload(self): |
... | ... |
@@ -89,8 +89,8 @@ class MultiPartUpload(object): |
89 | 89 |
|
90 | 90 |
parts_xml = [] |
91 | 91 |
part_xml = "<Part><PartNumber>%i</PartNumber><ETag>%s</ETag></Part>" |
92 |
- for id, etag in self.parts.items(): |
|
93 |
- parts_xml.append(part_xml % (id, etag)) |
|
92 |
+ for seq, etag in self.parts.items(): |
|
93 |
+ parts_xml.append(part_xml % (seq, etag)) |
|
94 | 94 |
body = "<CompleteMultipartUpload>%s</CompleteMultipartUpload>" % ("".join(parts_xml)) |
95 | 95 |
|
96 | 96 |
headers = { "content-length": len(body) } |
... | ... |
@@ -743,7 +743,6 @@ class S3(object): |
743 | 743 |
|
744 | 744 |
def send_file_multipart(self, file, headers, uri, size): |
745 | 745 |
upload = MultiPartUpload(self, file, uri) |
746 |
- upload_id = upload.initiate_multipart_upload() |
|
747 | 746 |
|
748 | 747 |
chunk_size = self.config.multipart_chunk_size_mb * 1024 * 1024 |
749 | 748 |
|