Browse code

[sync] download files to a temporary filename, then rename

This fix makes all the downloads happen to temporary files of type
.s3cmd.XXXXX.tmp in the same folder as the target file's. Once the
download is complete, the file is renamed to the actual
destination. This renaming is atomic in nature; hence any parallel
thread or process could work on fully downloaded data (by filtering
all files matching .s3cmd.XXXXX.tmp pattern while walking the data
directory).

https://github.com/s3tools/s3cmd/pull/81

Patch manually applied by Matt Domsch because this portion of the code has
changed more than pulling or rebasing could handle.

Sumit Kumar authored on 2012/12/06 15:06:58
Showing 1 changed files
... ...
@@ -24,6 +24,7 @@ import subprocess
24 24
 import htmlentitydefs
25 25
 import socket
26 26
 import shutil
27
+import tempfile
27 28
 
28 29
 from copy import copy
29 30
 from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter
... ...
@@ -783,17 +784,17 @@ def cmd_sync_remote2local(args):
783 783
                     warning(u"%s: destination directory not writable: %s" % (file, dst_dir))
784 784
                     continue
785 785
                 try:
786
-                    open_flags = os.O_CREAT
787
-                    open_flags |= os.O_TRUNC
788
-                    # open_flags |= os.O_EXCL
789
-
790 786
                     debug(u"dst_file=%s" % unicodise(dst_file))
791
-                    # This will have failed should the file exist
792
-                    os.close(os.open(dst_file, open_flags))
793
-                    # Yeah I know there is a race condition here. Sadly I don't know how to open() in exclusive mode.
794
-                    dst_stream = open(dst_file, "wb")
787
+                    # create temporary files (of type .s3cmd.XXXX.tmp) in the same directory 
788
+                    # for downloading and then rename once downloaded
789
+                    chkptfd, chkptfname = tempfile.mkstemp(".tmp",".s3cmd.",os.path.dirname(dst_file))
790
+                    debug(u"created chkptfname=%s" % unicodise(chkptfname))
791
+                    dst_stream = os.fdopen(chkptfd, "wb")  
795 792
                     response = s3.object_get(uri, dst_stream, extra_label = seq_label)
796 793
                     dst_stream.close()
794
+                    # download completed, rename the file to destination
795
+                    os.rename(chkptfname, dst_file)
796
+                    debug(u"renamed chkptfname=%s to dst_file=%s" % (unicodise(chkptfname), unicodise(dst_file)))
797 797
                     if response['headers'].has_key('x-amz-meta-s3cmd-attrs') and cfg.preserve_attrs:
798 798
                         attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
799 799
                         if attrs.has_key('mode'):