Browse code

accept --files-from=- to read from stdin

This allows shell syntax:

find . -name \*.gpg | s3cmd sync --files-from=- src dst

to take the list of files to transfer from stdin.

Be careful, as using with a --delete option will cause files on the
remote side not listed in stdin to be deleted too.

Matt Domsch authored on 2013/02/21 05:11:37
Showing 2 changed files
... ...
@@ -14,6 +14,7 @@ from HashCache import HashCache
14 14
 from logging import debug, info, warning, error
15 15
 
16 16
 import os
17
+import sys
17 18
 import glob
18 19
 import copy
19 20
 
... ...
@@ -150,14 +151,23 @@ def _get_filelist_from_file(cfg, local_path):
150 150
 
151 151
     filelist = {}
152 152
     for fname in cfg.files_from:
153
-        f = open(fname, 'r')
153
+        if fname == u'-':
154
+            f = sys.stdin
155
+        else:
156
+            try:
157
+                f = open(fname, 'r')
158
+            except IOError, e:
159
+                warning(u"--files-from input file %s could not be opened for reading (%s), skipping." % (fname, e.strerror))
160
+                continue
161
+                
154 162
         for line in f:
155 163
             line = line.strip()
156 164
             line = os.path.normpath(os.path.join(local_path, line))
157 165
             dirname = os.path.dirname(line)
158 166
             basename = os.path.basename(line)
159 167
             _append(filelist, dirname, basename)
160
-        f.close()
168
+        if f != sys.stdin:
169
+            f.close()
161 170
 
162 171
     # reformat to match os.walk()
163 172
     result = []
... ...
@@ -1738,7 +1738,7 @@ def main():
1738 1738
     optparser.add_option(      "--rinclude", dest="rinclude", action="append", metavar="REGEXP", help="Same as --include but uses REGEXP (regular expression) instead of GLOB")
1739 1739
     optparser.add_option(      "--rinclude-from", dest="rinclude_from", action="append", metavar="FILE", help="Read --rinclude REGEXPs from FILE")
1740 1740
 
1741
-    optparser.add_option(      "--files-from", dest="files_from", action="append", metavar="FILE", help="Read list of source-file names from FILE")
1741
+    optparser.add_option(      "--files-from", dest="files_from", action="append", metavar="FILE", help="Read list of source-file names from FILE. Use - to read from stdin.")
1742 1742
     optparser.add_option(      "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, ap-northeast-1, ap-southeast-1, sa-east-1, us-west-1 and us-west-2")
1743 1743
     optparser.add_option(      "--reduced-redundancy", "--rr", dest="reduced_redundancy", action="store_true", help="Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]")
1744 1744