| ... | ... |
@@ -18,8 +18,8 @@ class MultiPartUpload(object): |
| 18 | 18 |
self.s3 = s3 |
| 19 | 19 |
self.file = file |
| 20 | 20 |
self.uri = uri |
| 21 |
- self.upload_id = None |
|
| 22 | 21 |
self.parts = {}
|
| 22 |
+ self.upload_id = self.initiate_multipart_upload() |
|
| 23 | 23 |
|
| 24 | 24 |
def initiate_multipart_upload(self): |
| 25 | 25 |
""" |
| ... | ... |
@@ -35,7 +35,7 @@ class MultiPartUpload(object): |
| 35 | 35 |
def upload_all_parts(self): |
| 36 | 36 |
""" |
| 37 | 37 |
Execute a full multipart upload on a file |
| 38 |
- Returns the id/etag dict |
|
| 38 |
+ Returns the seq/etag dict |
|
| 39 | 39 |
TODO use num_processes to thread it |
| 40 | 40 |
""" |
| 41 | 41 |
if not self.upload_id: |
| ... | ... |
@@ -46,38 +46,38 @@ class MultiPartUpload(object): |
| 46 | 46 |
nr_parts = file_size / self.chunk_size + (file_size % self.chunk_size and 1) |
| 47 | 47 |
debug("MultiPart: Uploading %s in %d parts" % (self.file.name, nr_parts))
|
| 48 | 48 |
|
| 49 |
- id = 1 |
|
| 49 |
+ seq = 1 |
|
| 50 | 50 |
while size_left > 0: |
| 51 |
- offset = self.chunk_size * (id - 1) |
|
| 51 |
+ offset = self.chunk_size * (seq - 1) |
|
| 52 | 52 |
current_chunk_size = min(file_size - offset, self.chunk_size) |
| 53 | 53 |
size_left -= current_chunk_size |
| 54 | 54 |
labels = {
|
| 55 | 55 |
'source' : unicodise(self.file.name), |
| 56 | 56 |
'destination' : unicodise(self.uri.uri()), |
| 57 |
- 'extra' : "[part %d of %d, %s]" % (id, nr_parts, "%d%sB" % formatSize(current_chunk_size, human_readable = True)) |
|
| 57 |
+ 'extra' : "[part %d of %d, %s]" % (seq, nr_parts, "%d%sB" % formatSize(current_chunk_size, human_readable = True)) |
|
| 58 | 58 |
} |
| 59 | 59 |
try: |
| 60 |
- self.upload_part(id, offset, current_chunk_size, labels) |
|
| 60 |
+ self.upload_part(seq, offset, current_chunk_size, labels) |
|
| 61 | 61 |
except S3UploadError, e: |
| 62 |
- error(u"Upload of '%s' part %d failed too many times. Aborting multipart upload." % (self.file.name, id)) |
|
| 62 |
+ error(u"Upload of '%s' part %d failed too many times. Aborting multipart upload." % (self.file.name, seq)) |
|
| 63 | 63 |
self.abort_upload() |
| 64 | 64 |
raise e |
| 65 |
- id += 1 |
|
| 65 |
+ seq += 1 |
|
| 66 | 66 |
|
| 67 |
- debug("MultiPart: Upload finished: %d parts", id - 1)
|
|
| 67 |
+ debug("MultiPart: Upload finished: %d parts", seq - 1)
|
|
| 68 | 68 |
|
| 69 |
- def upload_part(self, id, offset, chunk_size, labels): |
|
| 69 |
+ def upload_part(self, seq, offset, chunk_size, labels): |
|
| 70 | 70 |
""" |
| 71 | 71 |
Upload a file chunk |
| 72 | 72 |
http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html |
| 73 | 73 |
""" |
| 74 | 74 |
# TODO implement Content-MD5 |
| 75 |
- debug("Uploading part %i of %r (%s bytes)" % (id, self.upload_id, chunk_size))
|
|
| 75 |
+ debug("Uploading part %i of %r (%s bytes)" % (seq, self.upload_id, chunk_size))
|
|
| 76 | 76 |
headers = { "content-length": chunk_size }
|
| 77 |
- query_string = "?partNumber=%i&uploadId=%s" % (id, self.upload_id) |
|
| 77 |
+ query_string = "?partNumber=%i&uploadId=%s" % (seq, self.upload_id) |
|
| 78 | 78 |
request = self.s3.create_request("OBJECT_PUT", uri = self.uri, headers = headers, extra = query_string)
|
| 79 | 79 |
response = self.s3.send_file(request, self.file, labels, offset = offset, chunk_size = chunk_size) |
| 80 |
- self.parts[id] = response["headers"]["etag"] |
|
| 80 |
+ self.parts[seq] = response["headers"]["etag"] |
|
| 81 | 81 |
return response |
| 82 | 82 |
|
| 83 | 83 |
def complete_multipart_upload(self): |
| ... | ... |
@@ -89,8 +89,8 @@ class MultiPartUpload(object): |
| 89 | 89 |
|
| 90 | 90 |
parts_xml = [] |
| 91 | 91 |
part_xml = "<Part><PartNumber>%i</PartNumber><ETag>%s</ETag></Part>" |
| 92 |
- for id, etag in self.parts.items(): |
|
| 93 |
- parts_xml.append(part_xml % (id, etag)) |
|
| 92 |
+ for seq, etag in self.parts.items(): |
|
| 93 |
+ parts_xml.append(part_xml % (seq, etag)) |
|
| 94 | 94 |
body = "<CompleteMultipartUpload>%s</CompleteMultipartUpload>" % ("".join(parts_xml))
|
| 95 | 95 |
|
| 96 | 96 |
headers = { "content-length": len(body) }
|
| ... | ... |
@@ -743,7 +743,6 @@ class S3(object): |
| 743 | 743 |
|
| 744 | 744 |
def send_file_multipart(self, file, headers, uri, size): |
| 745 | 745 |
upload = MultiPartUpload(self, file, uri) |
| 746 |
- upload_id = upload.initiate_multipart_upload() |
|
| 747 | 746 |
|
| 748 | 747 |
chunk_size = self.config.multipart_chunk_size_mb * 1024 * 1024 |
| 749 | 748 |
|