Browse code

handle empty directory on S3 sync remote2local

We have to recognize when it's an empty directory (the filename ends
in a trailing /) and treat it specially.

Matt Domsch authored on 2014/02/08 09:29:42
Showing 1 changed files
... ...
@@ -898,6 +898,8 @@ def cmd_sync_remote2local(args):
898 898
         _do_deletes(local_list)
899 899
 
900 900
     def _download(remote_list, seq, total, total_size, dir_cache):
901
+        original_umask = os.umask(0);
902
+        os.umask(original_umask);
901 903
         file_list = remote_list.keys()
902 904
         file_list.sort()
903 905
         for file in file_list:
... ...
@@ -905,6 +907,7 @@ def cmd_sync_remote2local(args):
905 905
             item = remote_list[file]
906 906
             uri = S3Uri(item['object_uri_str'])
907 907
             dst_file = item['local_filename']
908
+            is_empty_directory = dst_file.endswith('/')
908 909
             seq_label = "[%d of %d]" % (seq, total)
909 910
             try:
910 911
                 dst_dir = os.path.dirname(dst_file)
... ...
@@ -913,25 +916,45 @@ def cmd_sync_remote2local(args):
913 913
                 if dir_cache[dst_dir] == False:
914 914
                     warning(u"%s: destination directory not writable: %s" % (file, dst_dir))
915 915
                     continue
916
+
917
+                try:
918
+                    if not is_empty_directory: # ignore empty directory at S3:
919
+                        debug(u"dst_file=%s" % unicodise(dst_file))
920
+                        # create temporary files (of type .s3cmd.XXXX.tmp) in the same directory
921
+                        # for downloading and then rename once downloaded
922
+                        chkptfd, chkptfname = tempfile.mkstemp(".tmp",".s3cmd.",os.path.dirname(dst_file))
923
+                        debug(u"created chkptfname=%s" % unicodise(chkptfname))
924
+                        dst_stream = os.fdopen(chkptfd, "wb")
925
+                        response = s3.object_get(uri, dst_stream, extra_label = seq_label)
926
+                        dst_stream.close()
927
+                        # download completed, rename the file to destination
928
+                        os.rename(chkptfname, dst_file)
929
+                        debug(u"renamed chkptfname=%s to dst_file=%s" % (unicodise(chkptfname), unicodise(dst_file)))
930
+                except Exception, e:
931
+                    if e.errno == errno.EISDIR:
932
+                        warning(u"%s is a directory - skipping over" % dst_file)
933
+                        continue
934
+                    else:
935
+                        raise
936
+
937
+                try:
938
+                        # set permissions on destination file
939
+                        if not is_empty_directory: # a normal file
940
+                            mode = 0777 - original_umask;
941
+                        else: # an empty directory, make them readable/executable
942
+                            mode = 0o775
943
+                        debug(u"mode=%s" % oct(mode))
944
+                        os.chmod(dst_file, mode);
945
+                except:
946
+                    raise
947
+
948
+                # because we don't upload empty directories,
949
+                # we can continue the loop here, we won't be setting stat info.
950
+                # if we do start to upload empty directories, we'll have to reconsider this.
951
+                if is_empty_directory:
952
+                    continue
953
+
916 954
                 try:
917
-                    debug(u"dst_file=%s" % unicodise(dst_file))
918
-                    # create temporary files (of type .s3cmd.XXXX.tmp) in the same directory
919
-                    # for downloading and then rename once downloaded
920
-                    chkptfd, chkptfname = tempfile.mkstemp(".tmp",".s3cmd.",os.path.dirname(dst_file))
921
-                    debug(u"created chkptfname=%s" % unicodise(chkptfname))
922
-                    dst_stream = os.fdopen(chkptfd, "wb")
923
-                    response = s3.object_get(uri, dst_stream, extra_label = seq_label)
924
-                    dst_stream.close()
925
-                    # download completed, rename the file to destination
926
-                    os.rename(chkptfname, dst_file)
927
-
928
-                    # set permissions on destination file
929
-                    original_umask = os.umask(0);
930
-                    os.umask(original_umask);
931
-                    mode = 0777 - original_umask;
932
-                    debug(u"mode=%s" % oct(mode))
933
-                    os.chmod(dst_file, mode);
934
-                    debug(u"renamed chkptfname=%s to dst_file=%s" % (unicodise(chkptfname), unicodise(dst_file)))
935 955
                     if response.has_key('s3cmd-attrs') and cfg.preserve_attrs:
936 956
                         attrs = response['s3cmd-attrs']
937 957
                         if attrs.has_key('mode'):
... ...
@@ -955,9 +978,6 @@ def cmd_sync_remote2local(args):
955 955
                     if e.errno in (errno.EPERM, errno.EACCES):
956 956
                         warning(u"%s not writable: %s" % (dst_file, e.strerror))
957 957
                         continue
958
-                    if e.errno == errno.EISDIR:
959
-                        warning(u"%s is a directory - skipping over" % dst_file)
960
-                        continue
961 958
                     raise e
962 959
                 except KeyboardInterrupt:
963 960
                     try: