This solves the change of behavior introduced by processing
excludes/includes during os.walk(), where previously:
s3cmd sync --exclude='*' --include='*.gpg'
would walk the whole tree and transfer only the files named *.gpg.
Since the change to os.walk(), the exclude '*' matches everything, and
nothing is transferred.
This patch introduces --files-from=FILE to match rsync behaviour,
where the list of files to transfer (local to remote) is taken not
from an os.walk(), but from the explicit list in FILE.
The equivalent for remote to local, and remote to remote, is not yet
implemented.
... | ... |
@@ -140,6 +140,35 @@ def handle_exclude_include_walk(root, dirs, files): |
140 | 140 |
else: |
141 | 141 |
debug(u"PASS: %r" % (file)) |
142 | 142 |
|
143 |
+ |
|
144 |
+def _get_filelist_from_file(cfg, local_path): |
|
145 |
+ def _append(d, key, value): |
|
146 |
+ if key not in d: |
|
147 |
+ d[key] = [value] |
|
148 |
+ else: |
|
149 |
+ d[key].append(value) |
|
150 |
+ |
|
151 |
+ filelist = {} |
|
152 |
+ for fname in cfg.files_from: |
|
153 |
+ f = open(fname, 'r') |
|
154 |
+ for line in f: |
|
155 |
+ line = line.strip() |
|
156 |
+ line = os.path.normpath(os.path.join(local_path, line)) |
|
157 |
+ dirname = os.path.dirname(line) |
|
158 |
+ basename = os.path.basename(line) |
|
159 |
+ _append(filelist, dirname, basename) |
|
160 |
+ f.close() |
|
161 |
+ |
|
162 |
+ # reformat to match os.walk() |
|
163 |
+ result = [] |
|
164 |
+ keys = filelist.keys() |
|
165 |
+ keys.sort() |
|
166 |
+ for key in keys: |
|
167 |
+ values = filelist[key] |
|
168 |
+ values.sort() |
|
169 |
+ result.append((key, [], values)) |
|
170 |
+ return result |
|
171 |
+ |
|
143 | 172 |
def fetch_local_list(args, recursive = None): |
144 | 173 |
def _get_filelist_local(loc_list, local_uri, cache): |
145 | 174 |
info(u"Compiling list of local files...") |
... | ... |
@@ -156,11 +185,15 @@ def fetch_local_list(args, recursive = None): |
156 | 156 |
if local_uri.isdir(): |
157 | 157 |
local_base = deunicodise(local_uri.basename()) |
158 | 158 |
local_path = deunicodise(local_uri.path()) |
159 |
- if cfg.follow_symlinks: |
|
160 |
- filelist = _fswalk_follow_symlinks(local_path) |
|
159 |
+ if len(cfg.files_from): |
|
160 |
+ filelist = _get_filelist_from_file(cfg, local_path) |
|
161 |
+ single_file = False |
|
161 | 162 |
else: |
162 |
- filelist = _fswalk_no_symlinks(local_path) |
|
163 |
- single_file = False |
|
163 |
+ if cfg.follow_symlinks: |
|
164 |
+ filelist = _fswalk_follow_symlinks(local_path) |
|
165 |
+ else: |
|
166 |
+ filelist = _fswalk_no_symlinks(local_path) |
|
167 |
+ single_file = False |
|
164 | 168 |
else: |
165 | 169 |
local_base = "" |
166 | 170 |
local_path = deunicodise(local_uri.dirname()) |
... | ... |
@@ -1738,6 +1738,7 @@ def main(): |
1738 | 1738 |
optparser.add_option( "--rinclude", dest="rinclude", action="append", metavar="REGEXP", help="Same as --include but uses REGEXP (regular expression) instead of GLOB") |
1739 | 1739 |
optparser.add_option( "--rinclude-from", dest="rinclude_from", action="append", metavar="FILE", help="Read --rinclude REGEXPs from FILE") |
1740 | 1740 |
|
1741 |
+ optparser.add_option( "--files-from", dest="files_from", action="append", metavar="FILE", help="Read list of source-file names from FILE") |
|
1741 | 1742 |
optparser.add_option( "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, ap-northeast-1, ap-southeast-1, sa-east-1, us-west-1 and us-west-2") |
1742 | 1743 |
optparser.add_option( "--reduced-redundancy", "--rr", dest="reduced_redundancy", action="store_true", help="Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]") |
1743 | 1744 |
|
... | ... |
@@ -1910,6 +1911,8 @@ def main(): |
1910 | 1910 |
|
1911 | 1911 |
if options.additional_destinations: |
1912 | 1912 |
cfg.additional_destinations = options.additional_destinations |
1913 |
+ if options.files_from: |
|
1914 |
+ cfg.files_from = options.files_from |
|
1913 | 1915 |
|
1914 | 1916 |
## Set output and filesystem encoding for printing out filenames. |
1915 | 1917 |
sys.stdout = codecs.getwriter(cfg.encoding)(sys.stdout, "replace") |