Browse code

add --files-from=FILE to allow transfer of select files only

This solves the change of behavior introduced by processing
excludes/includes during os.walk(), where previously:

s3cmd sync --exclude='*' --include='*.gpg'

would walk the whole tree and transfer only the files named *.gpg.

Since the change to os.walk(), the exclude '*' matches everything, and
nothing is transferred.

This patch introduces --files-from=FILE to match rsync behaviour,
where the list of files to transfer (local to remote) is taken not
from an os.walk(), but from the explicit list in FILE.

The equivalent for remote to local, and remote to remote, is not yet
implemented.

Matt Domsch authored on 2013/02/20 07:08:15
Showing 3 changed files
... ...
@@ -92,6 +92,7 @@ class Config(object):
92 92
     website_error = ""
93 93
     website_endpoint = "http://%(bucket)s.s3-website-%(location)s.amazonaws.com/"
94 94
     additional_destinations = []
95
+    files_from = []
95 96
     cache_file = ""
96 97
     add_headers = ""
97 98
 
... ...
@@ -140,6 +140,35 @@ def handle_exclude_include_walk(root, dirs, files):
140 140
         else:
141 141
             debug(u"PASS: %r" % (file))
142 142
 
143
+
144
+def _get_filelist_from_file(cfg, local_path):
145
+    def _append(d, key, value):
146
+        if key not in d:
147
+            d[key] = [value]
148
+        else:
149
+            d[key].append(value)
150
+
151
+    filelist = {}
152
+    for fname in cfg.files_from:
153
+        f = open(fname, 'r')
154
+        for line in f:
155
+            line = line.strip()
156
+            line = os.path.normpath(os.path.join(local_path, line))
157
+            dirname = os.path.dirname(line)
158
+            basename = os.path.basename(line)
159
+            _append(filelist, dirname, basename)
160
+        f.close()
161
+
162
+    # reformat to match os.walk()
163
+    result = []
164
+    keys = filelist.keys()
165
+    keys.sort()
166
+    for key in keys:
167
+        values = filelist[key]
168
+        values.sort()
169
+        result.append((key, [], values))
170
+    return result
171
+
143 172
 def fetch_local_list(args, recursive = None):
144 173
     def _get_filelist_local(loc_list, local_uri, cache):
145 174
         info(u"Compiling list of local files...")
... ...
@@ -156,11 +185,15 @@ def fetch_local_list(args, recursive = None):
156 156
         if local_uri.isdir():
157 157
             local_base = deunicodise(local_uri.basename())
158 158
             local_path = deunicodise(local_uri.path())
159
-            if cfg.follow_symlinks:
160
-                filelist = _fswalk_follow_symlinks(local_path)
159
+            if len(cfg.files_from):
160
+                filelist = _get_filelist_from_file(cfg, local_path)
161
+                single_file = False
161 162
             else:
162
-                filelist = _fswalk_no_symlinks(local_path)
163
-            single_file = False
163
+                if cfg.follow_symlinks:
164
+                    filelist = _fswalk_follow_symlinks(local_path)
165
+                else:
166
+                    filelist = _fswalk_no_symlinks(local_path)
167
+                single_file = False
164 168
         else:
165 169
             local_base = ""
166 170
             local_path = deunicodise(local_uri.dirname())
... ...
@@ -1738,6 +1738,7 @@ def main():
1738 1738
     optparser.add_option(      "--rinclude", dest="rinclude", action="append", metavar="REGEXP", help="Same as --include but uses REGEXP (regular expression) instead of GLOB")
1739 1739
     optparser.add_option(      "--rinclude-from", dest="rinclude_from", action="append", metavar="FILE", help="Read --rinclude REGEXPs from FILE")
1740 1740
 
1741
+    optparser.add_option(      "--files-from", dest="files_from", action="append", metavar="FILE", help="Read list of source-file names from FILE")
1741 1742
     optparser.add_option(      "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, ap-northeast-1, ap-southeast-1, sa-east-1, us-west-1 and us-west-2")
1742 1743
     optparser.add_option(      "--reduced-redundancy", "--rr", dest="reduced_redundancy", action="store_true", help="Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]")
1743 1744
 
... ...
@@ -1910,6 +1911,8 @@ def main():
1910 1910
 
1911 1911
     if options.additional_destinations:
1912 1912
         cfg.additional_destinations = options.additional_destinations
1913
+    if options.files_from:
1914
+        cfg.files_from = options.files_from
1913 1915
 
1914 1916
     ## Set output and filesystem encoding for printing out filenames.
1915 1917
     sys.stdout = codecs.getwriter(cfg.encoding)(sys.stdout, "replace")