This fix makes all the downloads happen to temporary files of type
.s3cmd.XXXXX.tmp in the same folder as the target file's. Once the
download is complete, the file is renamed to the actual
destination. This renaming is atomic in nature; hence any parallel
thread or process could work on fully downloaded data (by filtering
all files matching .s3cmd.XXXXX.tmp pattern while walking the data
directory).
https://github.com/s3tools/s3cmd/pull/81
Patch manually applied by Matt Domsch because this portion of the code has
changed more than pulling or rebasing could handle.
... | ... |
@@ -24,6 +24,7 @@ import subprocess |
24 | 24 |
import htmlentitydefs |
25 | 25 |
import socket |
26 | 26 |
import shutil |
27 |
+import tempfile |
|
27 | 28 |
|
28 | 29 |
from copy import copy |
29 | 30 |
from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter |
... | ... |
@@ -783,17 +784,17 @@ def cmd_sync_remote2local(args): |
783 | 783 |
warning(u"%s: destination directory not writable: %s" % (file, dst_dir)) |
784 | 784 |
continue |
785 | 785 |
try: |
786 |
- open_flags = os.O_CREAT |
|
787 |
- open_flags |= os.O_TRUNC |
|
788 |
- # open_flags |= os.O_EXCL |
|
789 |
- |
|
790 | 786 |
debug(u"dst_file=%s" % unicodise(dst_file)) |
791 |
- # This will have failed should the file exist |
|
792 |
- os.close(os.open(dst_file, open_flags)) |
|
793 |
- # Yeah I know there is a race condition here. Sadly I don't know how to open() in exclusive mode. |
|
794 |
- dst_stream = open(dst_file, "wb") |
|
787 |
+ # create temporary files (of type .s3cmd.XXXX.tmp) in the same directory |
|
788 |
+ # for downloading and then rename once downloaded |
|
789 |
+ chkptfd, chkptfname = tempfile.mkstemp(".tmp",".s3cmd.",os.path.dirname(dst_file)) |
|
790 |
+ debug(u"created chkptfname=%s" % unicodise(chkptfname)) |
|
791 |
+ dst_stream = os.fdopen(chkptfd, "wb") |
|
795 | 792 |
response = s3.object_get(uri, dst_stream, extra_label = seq_label) |
796 | 793 |
dst_stream.close() |
794 |
+ # download completed, rename the file to destination |
|
795 |
+ os.rename(chkptfname, dst_file) |
|
796 |
+ debug(u"renamed chkptfname=%s to dst_file=%s" % (unicodise(chkptfname), unicodise(dst_file))) |
|
797 | 797 |
if response['headers'].has_key('x-amz-meta-s3cmd-attrs') and cfg.preserve_attrs: |
798 | 798 |
attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs']) |
799 | 799 |
if attrs.has_key('mode'): |