Browse code

merge master

UENISHI Kota authored on 2013/03/18 12:46:35
Showing 9 changed files
... ...
@@ -1,5 +1,18 @@
1
-s3cmd 1.5.0   -   ???
2
-===========
1
+s3cmd 1.5.0-alpha3   -   2013-03-11
2
+==================
3
+* Persistent HTTP/HTTPS connections for massive speedup (Michal Ludvig)
4
+* New switch --quiet for suppressing all output (Siddarth Prakash)
5
+* Honour "umask" on file downloads (Jason Dalton)
6
+* Various bugfixes from many contributors
7
+
8
+s3cmd 1.5.0-alpha2   -   2013-03-04
9
+==================
10
+* IAM roles support (David Kohen, Eric Dowd)
11
+* Manage bucket policies (Kota Uenishi)
12
+* Various bugfixes from many contributors
13
+
14
+s3cmd 1.5.0-alpha1   -   2013-02-19
15
+==================
3 16
 * Server-side copy for hardlinks/softlinks to improve performance
4 17
   (Matt Domsch)
5 18
 * New [signurl] command (Craig Ringer)
6 19
new file mode 100644
... ...
@@ -0,0 +1,71 @@
0
+import httplib
1
+from urlparse import urlparse
2
+from threading import Semaphore
3
+from logging import debug, info, warning, error
4
+
5
+from Config import Config
6
+from Exceptions import ParameterError
7
+
8
+__all__ = [ "ConnMan" ]
9
+
10
+class http_connection(object):
11
+    def __init__(self, id, hostname, ssl, cfg):
12
+        self.hostname = hostname
13
+        self.ssl = ssl
14
+        self.id = id
15
+        self.counter = 0
16
+        if cfg.proxy_host != "":
17
+            self.c = httplib.HTTPConnection(cfg.proxy_host, cfg.proxy_port)
18
+        elif not ssl:
19
+            self.c = httplib.HTTPConnection(hostname)
20
+        else:
21
+            self.c = httplib.HTTPSConnection(hostname)
22
+
23
+class ConnMan(object):
24
+    conn_pool_sem = Semaphore()
25
+    conn_pool = {}
26
+    conn_max_counter = 800    ## AWS closes connection after some ~90 requests
27
+
28
+    @staticmethod
29
+    def get(hostname, ssl = None):
30
+        cfg = Config()
31
+        if ssl == None:
32
+            ssl = cfg.use_https
33
+        conn = None
34
+        if cfg.proxy_host != "":
35
+            if ssl:
36
+                raise ParameterError("use_ssl=True can't be used with proxy")
37
+            conn_id = "proxy://%s:%s" % (cfg.proxy_host, cfg.proxy_port)
38
+        else:
39
+            conn_id = "http%s://%s" % (ssl and "s" or "", hostname)
40
+        ConnMan.conn_pool_sem.acquire()
41
+        if not ConnMan.conn_pool.has_key(conn_id):
42
+            ConnMan.conn_pool[conn_id] = []
43
+        if len(ConnMan.conn_pool[conn_id]):
44
+            conn = ConnMan.conn_pool[conn_id].pop()
45
+            debug("ConnMan.get(): re-using connection: %s#%d" % (conn.id, conn.counter))
46
+        ConnMan.conn_pool_sem.release()
47
+        if not conn:
48
+            debug("ConnMan.get(): creating new connection: %s" % conn_id)
49
+            conn = http_connection(conn_id, hostname, ssl, cfg)
50
+            conn.c.connect()
51
+        conn.counter += 1
52
+        return conn
53
+
54
+    @staticmethod
55
+    def put(conn):
56
+        if conn.id.startswith("proxy://"):
57
+            conn.c.close()
58
+            debug("ConnMan.put(): closing proxy connection (keep-alive not yet supported)")
59
+            return
60
+
61
+        if conn.counter >= ConnMan.conn_max_counter:
62
+            conn.c.close()
63
+            debug("ConnMan.put(): closing over-used connection")
64
+            return
65
+
66
+        ConnMan.conn_pool_sem.acquire()
67
+        ConnMan.conn_pool[conn.id].append(conn)
68
+        ConnMan.conn_pool_sem.release()
69
+        debug("ConnMan.put(): connection put back to pool (%s#%d)" % (conn.id, conn.counter))
70
+
0 71
new file mode 100644
... ...
@@ -0,0 +1,53 @@
0
+## Amazon S3 manager
1
+## Author: Michal Ludvig <michal@logix.cz>
2
+##         http://www.logix.cz/michal
3
+## License: GPL Version 2
4
+
5
+from SortedDict import SortedDict
6
+import Utils
7
+
8
+class FileDict(SortedDict):
9
+    def __init__(self, mapping = {}, ignore_case = True, **kwargs):
10
+        SortedDict.__init__(self, mapping = mapping, ignore_case = ignore_case, **kwargs)
11
+        self.hardlinks = dict() # { dev: { inode : {'md5':, 'relative_files':}}}
12
+        self.by_md5 = dict() # {md5: set(relative_files)}
13
+
14
+    def record_md5(self, relative_file, md5):
15
+        if md5 not in self.by_md5:
16
+            self.by_md5[md5] = set()
17
+        self.by_md5[md5].add(relative_file)
18
+
19
+    def find_md5_one(self, md5):
20
+        try:
21
+            return list(self.by_md5.get(md5, set()))[0]
22
+        except:
23
+            return None
24
+
25
+    def get_md5(self, relative_file):
26
+        """returns md5 if it can, or raises IOError if file is unreadable"""
27
+        md5 = None
28
+        if 'md5' in self[relative_file]:
29
+            return self[relative_file]['md5']
30
+        md5 = self.get_hardlink_md5(relative_file)
31
+        if md5 is None:
32
+            md5 = Utils.hash_file_md5(self[relative_file]['full_name'])
33
+        self.record_md5(relative_file, md5)
34
+        self[relative_file]['md5'] = md5
35
+        return md5
36
+
37
+    def record_hardlink(self, relative_file, dev, inode, md5):
38
+        if dev not in self.hardlinks:
39
+            self.hardlinks[dev] = dict()
40
+        if inode not in self.hardlinks[dev]:
41
+            self.hardlinks[dev][inode] = dict(md5=md5, relative_files=set())
42
+        self.hardlinks[dev][inode]['relative_files'].add(relative_file)
43
+
44
+    def get_hardlink_md5(self, relative_file):
45
+        md5 = None
46
+        dev = self[relative_file]['dev']
47
+        inode = self[relative_file]['inode']
48
+        try:
49
+            md5 = self.hardlinks[dev][inode]['md5']
50
+        except:
51
+            pass
52
+        return md5
... ...
@@ -6,7 +6,7 @@
6 6
 from S3 import S3
7 7
 from Config import Config
8 8
 from S3Uri import S3Uri
9
-from SortedDict import SortedDict
9
+from FileDict import FileDict
10 10
 from Utils import *
11 11
 from Exceptions import ParameterError
12 12
 from HashCache import HashCache
... ...
@@ -58,7 +58,7 @@ def _fswalk_no_symlinks(path):
58 58
 def filter_exclude_include(src_list):
59 59
     info(u"Applying --exclude/--include")
60 60
     cfg = Config()
61
-    exclude_list = SortedDict(ignore_case = False)
61
+    exclude_list = FileDict(ignore_case = False)
62 62
     for file in src_list.keys():
63 63
         debug(u"CHECK: %s" % file)
64 64
         excluded = False
... ...
@@ -224,7 +224,7 @@ def fetch_local_list(args, recursive = None):
224 224
             info(u"No cache file found, creating it.")
225 225
 
226 226
     local_uris = []
227
-    local_list = SortedDict(ignore_case = False)
227
+    local_list = FileDict(ignore_case = False)
228 228
     single_file = False
229 229
 
230 230
     if type(args) not in (list, tuple):
... ...
@@ -284,7 +284,7 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
284 284
             rem_base = rem_base[:rem_base.rfind('/')+1]
285 285
             remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base))
286 286
         rem_base_len = len(rem_base)
287
-        rem_list = SortedDict(ignore_case = False)
287
+        rem_list = FileDict(ignore_case = False)
288 288
         break_now = False
289 289
         for object in response['list']:
290 290
             if object['Key'] == rem_base_original and object['Key'][-1] != os.path.sep:
... ...
@@ -292,7 +292,7 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
292 292
                 key = os.path.basename(object['Key'])
293 293
                 object_uri_str = remote_uri_original.uri()
294 294
                 break_now = True
295
-                rem_list = SortedDict(ignore_case = False)   ## Remove whatever has already been put to rem_list
295
+                rem_list = FileDict(ignore_case = False)   ## Remove whatever has already been put to rem_list
296 296
             else:
297 297
                 key = object['Key'][rem_base_len:]      ## Beware - this may be '' if object['Key']==rem_base !!
298 298
                 object_uri_str = remote_uri.uri() + key
... ...
@@ -314,7 +314,7 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
314 314
 
315 315
     cfg = Config()
316 316
     remote_uris = []
317
-    remote_list = SortedDict(ignore_case = False)
317
+    remote_list = FileDict(ignore_case = False)
318 318
 
319 319
     if type(args) not in (list, tuple):
320 320
         args = [args]
... ...
@@ -436,7 +436,7 @@ def compare_filelists(src_list, dst_list, src_remote, dst_remote, delay_updates
436 436
     ## Items left on src_list will be transferred
437 437
     ## Items left on update_list will be transferred after src_list
438 438
     ## Items left on copy_pairs will be copied from dst1 to dst2
439
-    update_list = SortedDict(ignore_case = False)
439
+    update_list = FileDict(ignore_case = False)
440 440
     ## Items left on dst_list will be deleted
441 441
     copy_pairs = []
442 442
 
... ...
@@ -1,5 +1,5 @@
1 1
 package = "s3cmd"
2
-version = "1.5.0-alpha1"
2
+version = "1.5.0-alpha3"
3 3
 url = "http://s3tools.org"
4 4
 license = "GPL version 2"
5 5
 short_description = "Command line tool for managing Amazon S3 and CloudFront services"
... ...
@@ -27,6 +27,7 @@ from Config import Config
27 27
 from Exceptions import *
28 28
 from MultiPart import MultiPartUpload
29 29
 from S3Uri import S3Uri
30
+from ConnMan import ConnMan
30 31
 
31 32
 try:
32 33
     import magic, gzip
... ...
@@ -190,15 +191,6 @@ class S3(object):
190 190
     def __init__(self, config):
191 191
         self.config = config
192 192
 
193
-    def get_connection(self, bucket):
194
-        if self.config.proxy_host != "":
195
-            return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port)
196
-        else:
197
-            if self.config.use_https:
198
-                return httplib.HTTPSConnection(self.get_hostname(bucket))
199
-            else:
200
-                return httplib.HTTPConnection(self.get_hostname(bucket))
201
-
202 193
     def get_hostname(self, bucket):
203 194
         if bucket and check_bucket_name_dns_conformity(bucket):
204 195
             if self.redir_map.has_key(bucket):
... ...
@@ -246,10 +238,9 @@ class S3(object):
246 246
         truncated = True
247 247
         list = []
248 248
         prefixes = []
249
-        conn = self.get_connection(bucket)
250 249
 
251 250
         while truncated:
252
-            response = self.bucket_list_noparse(conn, bucket, prefix, recursive, uri_params)
251
+            response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params)
253 252
             current_list = _get_contents(response["data"])
254 253
             current_prefixes = _get_common_prefixes(response["data"])
255 254
             truncated = _list_truncated(response["data"])
... ...
@@ -263,19 +254,17 @@ class S3(object):
263 263
             list += current_list
264 264
             prefixes += current_prefixes
265 265
 
266
-        conn.close()
267
-
268 266
         response['list'] = list
269 267
         response['common_prefixes'] = prefixes
270 268
         return response
271 269
 
272
-    def bucket_list_noparse(self, connection, bucket, prefix = None, recursive = None, uri_params = {}):
270
+    def bucket_list_noparse(self, bucket, prefix = None, recursive = None, uri_params = {}):
273 271
         if prefix:
274 272
             uri_params['prefix'] = self.urlencode_string(prefix)
275 273
         if not self.config.recursive and not recursive:
276 274
             uri_params['delimiter'] = "/"
277 275
         request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params)
278
-        response = self.send_request(request, conn = connection)
276
+        response = self.send_request(request)
279 277
         #debug(response)
280 278
         return response
281 279
 
... ...
@@ -679,7 +668,7 @@ class S3(object):
679 679
         # Wait a few seconds. The more it fails the more we wait.
680 680
         return (self._max_retries - retries + 1) * 3
681 681
 
682
-    def send_request(self, request, body = None, retries = _max_retries, conn = None):
682
+    def send_request(self, request, body = None, retries = _max_retries):
683 683
         method_string, resource, headers = request.get_triplet()
684 684
         debug("Processing request, please wait...")
685 685
         if not headers.has_key('content-length'):
... ...
@@ -688,25 +677,20 @@ class S3(object):
688 688
             # "Stringify" all headers
689 689
             for header in headers.keys():
690 690
                 headers[header] = str(headers[header])
691
-            if conn is None:
692
-                debug("Establishing connection")
693
-                conn = self.get_connection(resource['bucket'])
694
-                close_conn = True
695
-            else:
696
-                debug("Using existing connection")
697
-                close_conn = False
691
+            conn = ConnMan.get(self.get_hostname(resource['bucket']))
698 692
             uri = self.format_uri(resource)
699 693
             debug("Sending request method_string=%r, uri=%r, headers=%r, body=(%i bytes)" % (method_string, uri, headers, len(body or "")))
700
-            conn.request(method_string, uri, body, headers)
694
+            conn.c.request(method_string, uri, body, headers)
701 695
             response = {}
702
-            http_response = conn.getresponse()
696
+            http_response = conn.c.getresponse()
703 697
             response["status"] = http_response.status
704 698
             response["reason"] = http_response.reason
705 699
             response["headers"] = convertTupleListToDict(http_response.getheaders())
706 700
             response["data"] =  http_response.read()
707 701
             debug("Response: " + str(response))
708
-            if close_conn is True:
709
-                conn.close()
702
+            ConnMan.put(conn)
703
+        except ParameterError, e:
704
+            raise
710 705
         except Exception, e:
711 706
             if retries:
712 707
                 warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
... ...
@@ -749,12 +733,13 @@ class S3(object):
749 749
             info("Sending file '%s', please wait..." % file.name)
750 750
         timestamp_start = time.time()
751 751
         try:
752
-            conn = self.get_connection(resource['bucket'])
753
-            conn.connect()
754
-            conn.putrequest(method_string, self.format_uri(resource))
752
+            conn = ConnMan.get(self.get_hostname(resource['bucket']))
753
+            conn.c.putrequest(method_string, self.format_uri(resource))
755 754
             for header in headers.keys():
756
-                conn.putheader(header, str(headers[header]))
757
-            conn.endheaders()
755
+                conn.c.putheader(header, str(headers[header]))
756
+            conn.c.endheaders()
757
+        except ParameterError, e:
758
+            raise
758 759
         except Exception, e:
759 760
             if self.config.progress_meter:
760 761
                 progress.done("failed")
... ...
@@ -777,7 +762,7 @@ class S3(object):
777 777
                 else:
778 778
                     data = buffer
779 779
                 md5_hash.update(data)
780
-                conn.send(data)
780
+                conn.c.send(data)
781 781
                 if self.config.progress_meter:
782 782
                     progress.update(delta_position = len(data))
783 783
                 size_left -= len(data)
... ...
@@ -785,14 +770,16 @@ class S3(object):
785 785
                     time.sleep(throttle)
786 786
             md5_computed = md5_hash.hexdigest()
787 787
             response = {}
788
-            http_response = conn.getresponse()
788
+            http_response = conn.c.getresponse()
789 789
             response["status"] = http_response.status
790 790
             response["reason"] = http_response.reason
791 791
             response["headers"] = convertTupleListToDict(http_response.getheaders())
792 792
             response["data"] = http_response.read()
793 793
             response["size"] = size_total
794
-            conn.close()
794
+            ConnMan.put(conn)
795 795
             debug(u"Response: %s" % response)
796
+        except ParameterError, e:
797
+            raise
796 798
         except Exception, e:
797 799
             if self.config.progress_meter:
798 800
                 progress.done("failed")
... ...
@@ -814,7 +801,7 @@ class S3(object):
814 814
         response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1)
815 815
 
816 816
         if self.config.progress_meter:
817
-            ## The above conn.close() takes some time -> update() progress meter
817
+            ## Finalising the upload takes some time -> update() progress meter
818 818
             ## to correct the average speed. Otherwise people will complain that
819 819
             ## 'progress' and response["speed"] are inconsistent ;-)
820 820
             progress.update()
... ...
@@ -889,21 +876,22 @@ class S3(object):
889 889
             info("Receiving file '%s', please wait..." % stream.name)
890 890
         timestamp_start = time.time()
891 891
         try:
892
-            conn = self.get_connection(resource['bucket'])
893
-            conn.connect()
894
-            conn.putrequest(method_string, self.format_uri(resource))
892
+            conn = ConnMan.get(self.get_hostname(resource['bucket']))
893
+            conn.c.putrequest(method_string, self.format_uri(resource))
895 894
             for header in headers.keys():
896
-                conn.putheader(header, str(headers[header]))
895
+                conn.c.putheader(header, str(headers[header]))
897 896
             if start_position > 0:
898 897
                 debug("Requesting Range: %d .. end" % start_position)
899
-                conn.putheader("Range", "bytes=%d-" % start_position)
900
-            conn.endheaders()
898
+                conn.c.putheader("Range", "bytes=%d-" % start_position)
899
+            conn.c.endheaders()
901 900
             response = {}
902
-            http_response = conn.getresponse()
901
+            http_response = conn.c.getresponse()
903 902
             response["status"] = http_response.status
904 903
             response["reason"] = http_response.reason
905 904
             response["headers"] = convertTupleListToDict(http_response.getheaders())
906 905
             debug("Response: %s" % response)
906
+        except ParameterError, e:
907
+            raise
907 908
         except Exception, e:
908 909
             if self.config.progress_meter:
909 910
                 progress.done("failed")
... ...
@@ -955,7 +943,7 @@ class S3(object):
955 955
                 ## Call progress meter from here...
956 956
                 if self.config.progress_meter:
957 957
                     progress.update(delta_position = len(data))
958
-            conn.close()
958
+            ConnMan.put(conn)
959 959
         except Exception, e:
960 960
             if self.config.progress_meter:
961 961
                 progress.done("failed")
... ...
@@ -27,8 +27,6 @@ class SortedDict(dict):
27 27
         """
28 28
         dict.__init__(self, mapping, **kwargs)
29 29
         self.ignore_case = ignore_case
30
-        self.hardlinks = dict() # { dev: { inode : {'md5':, 'relative_files':}}}
31
-        self.by_md5 = dict() # {md5: set(relative_files)}
32 30
 
33 31
     def keys(self):
34 32
         keys = dict.keys(self)
... ...
@@ -49,45 +47,6 @@ class SortedDict(dict):
49 49
         return SortedDictIterator(self, self.keys())
50 50
 
51 51
 
52
-    def record_md5(self, relative_file, md5):
53
-        if md5 not in self.by_md5:
54
-            self.by_md5[md5] = set()
55
-        self.by_md5[md5].add(relative_file)
56
-
57
-    def find_md5_one(self, md5):
58
-        try:
59
-            return list(self.by_md5.get(md5, set()))[0]
60
-        except:
61
-            return None
62
-
63
-    def get_md5(self, relative_file):
64
-        """returns md5 if it can, or raises IOError if file is unreadable"""
65
-        md5 = None
66
-        if 'md5' in self[relative_file]:
67
-            return self[relative_file]['md5']
68
-        md5 = self.get_hardlink_md5(relative_file)
69
-        if md5 is None:
70
-            md5 = Utils.hash_file_md5(self[relative_file]['full_name'])
71
-        self.record_md5(relative_file, md5)
72
-        self[relative_file]['md5'] = md5
73
-        return md5
74
-
75
-    def record_hardlink(self, relative_file, dev, inode, md5):
76
-        if dev not in self.hardlinks:
77
-            self.hardlinks[dev] = dict()
78
-        if inode not in self.hardlinks[dev]:
79
-            self.hardlinks[dev][inode] = dict(md5=md5, relative_files=set())
80
-        self.hardlinks[dev][inode]['relative_files'].add(relative_file)
81
-
82
-    def get_hardlink_md5(self, relative_file):
83
-        md5 = None
84
-        dev = self[relative_file]['dev']
85
-        inode = self[relative_file]['inode']
86
-        try:
87
-            md5 = self.hardlinks[dev][inode]['md5']
88
-        except:
89
-            pass
90
-        return md5
91 52
 
92 53
 if __name__ == "__main__":
93 54
     d = { 'AWS' : 1, 'Action' : 2, 'america' : 3, 'Auckland' : 4, 'America' : 5 }
... ...
@@ -656,10 +656,11 @@ def cmd_sync_remote2remote(args):
656 656
 
657 657
     print(u"Summary: %d source files to copy, %d files at destination to delete" % (src_count, dst_count))
658 658
 
659
-    if src_count > 0:
660
-        ### Populate 'remote_uri' only if we've got something to sync from src to dst
661
-        for key in src_list:
662
-            src_list[key]['target_uri'] = destination_base + key
659
+    ### Populate 'target_uri' only if we've got something to sync from src to dst
660
+    for key in src_list:
661
+        src_list[key]['target_uri'] = destination_base + key
662
+    for key in update_list:
663
+        update_list[key]['target_uri'] = destination_base + key
663 664
 
664 665
     if cfg.dry_run:
665 666
         for key in exclude_list:
... ...
@@ -745,6 +746,8 @@ def cmd_sync_remote2local(args):
745 745
     info(u"Summary: %d remote files to download, %d local files to delete, %d local files to hardlink" % (remote_count + update_count, local_count, copy_pairs_count))
746 746
 
747 747
     def _set_local_filename(remote_list, destination_base):
748
+        if len(remote_list) == 0:
749
+            return
748 750
         if not os.path.isdir(destination_base):
749 751
             ## We were either given a file name (existing or not) or want STDOUT
750 752
             if len(remote_list) > 1:
... ...
@@ -811,6 +814,15 @@ def cmd_sync_remote2local(args):
811 811
                     dst_stream.close()
812 812
                     # download completed, rename the file to destination
813 813
                     os.rename(chkptfname, dst_file)
814
+
815
+                    # set permissions on destination file
816
+                    original_umask = os.umask(0);
817
+                    os.umask(original_umask);
818
+                    mode = 0777 - original_umask;
819
+                    debug(u"mode=%s" % oct(mode))
820
+                    
821
+                    os.chmod(dst_file, mode);
822
+                    
814 823
                     debug(u"renamed chkptfname=%s to dst_file=%s" % (unicodise(chkptfname), unicodise(dst_file)))
815 824
                     if response['headers'].has_key('x-amz-meta-s3cmd-attrs') and cfg.preserve_attrs:
816 825
                         attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
... ...
@@ -822,7 +834,9 @@ def cmd_sync_remote2local(args):
822 822
                             os.utime(dst_file, (atime, mtime))
823 823
                         ## FIXME: uid/gid / uname/gname handling comes here! TODO
824 824
                 except OSError, e:
825
-                    try: dst_stream.close()
825
+                    try: 
826
+                        dst_stream.close() 
827
+                        os.remove(chkptfname)
826 828
                     except: pass
827 829
                     if e.errno == errno.EEXIST:
828 830
                         warning(u"%s exists - not overwriting" % (dst_file))
... ...
@@ -835,19 +849,25 @@ def cmd_sync_remote2local(args):
835 835
                         continue
836 836
                     raise e
837 837
                 except KeyboardInterrupt:
838
-                    try: dst_stream.close()
838
+                    try: 
839
+                        dst_stream.close()
840
+                        os.remove(chkptfname)
839 841
                     except: pass
840 842
                     warning(u"Exiting after keyboard interrupt")
841 843
                     return
842 844
                 except Exception, e:
843
-                    try: dst_stream.close()
845
+                    try: 
846
+                        dst_stream.close()
847
+                        os.remove(chkptfname)
844 848
                     except: pass
845 849
                     error(u"%s: %s" % (file, e))
846 850
                     continue
847 851
                 # We have to keep repeating this call because
848 852
                 # Python 2.4 doesn't support try/except/finally
849 853
                 # construction :-(
850
-                try: dst_stream.close()
854
+                try: 
855
+                    dst_stream.close()
856
+                    os.remove(chkptfname)
851 857
                 except: pass
852 858
             except S3DownloadError, e:
853 859
                 error(u"%s: download failed too many times. Skipping that file." % file)
... ...
@@ -893,7 +913,7 @@ def local_copy(copy_pairs, destination_base):
893 893
     # Do NOT hardlink local files by default, that'd be silly
894 894
     # For instance all empty files would become hardlinked together!
895 895
 
896
-    failed_copy_list = SortedDict()
896
+    failed_copy_list = FileDict()
897 897
     for (src_obj, dst1, relative_file) in copy_pairs:
898 898
         src_file = os.path.join(destination_base, dst1)
899 899
         dst_file = os.path.join(destination_base, relative_file)
... ...
@@ -1058,7 +1078,8 @@ def cmd_sync_local2remote(args):
1058 1058
             ## Make remote_key same as local_key for comparison if we're dealing with only one file
1059 1059
             remote_list_entry = remote_list[remote_list.keys()[0]]
1060 1060
             # Flush remote_list, by the way
1061
-            remote_list = { local_list.keys()[0] : remote_list_entry }
1061
+            remote_list = FileDict()
1062
+            remote_list[local_list.keys()[0]] =  remote_list_entry
1062 1063
 
1063 1064
         local_list, remote_list, update_list, copy_pairs = compare_filelists(local_list, remote_list, src_remote = False, dst_remote = True, delay_updates = cfg.delay_updates)
1064 1065
 
... ...
@@ -1644,6 +1665,7 @@ def get_commands_list():
1644 1644
 
1645 1645
     {"cmd":"multipart", "label":"show multipart uploads", "param":"s3://BUCKET [Id]", "func":cmd_multipart, "argc":1},
1646 1646
     {"cmd":"abortmp",   "label":"abort a multipart upload", "param":"s3://BUCKET/OBJECT Id", "func":cmd_abort_multipart, "argc":2},
1647
+
1647 1648
     {"cmd":"listmp",    "label":"list parts of a multipart upload", "param":"s3://BUCKET/OBJECT Id", "func":cmd_list_multipart, "argc":2},
1648 1649
 
1649 1650
     {"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1},
... ...
@@ -1856,6 +1878,7 @@ def main():
1856 1856
     optparser.add_option(      "--version", dest="show_version", action="store_true", help="Show s3cmd version (%s) and exit." % (PkgInfo.version))
1857 1857
     optparser.add_option("-F", "--follow-symlinks", dest="follow_symlinks", action="store_true", default=False, help="Follow symbolic links as if they are regular files")
1858 1858
     optparser.add_option(      "--cache-file", dest="cache_file", action="store", default="",  metavar="FILE", help="Cache FILE containing local source MD5 values")
1859
+    optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Silence output on stdout")
1859 1860
 
1860 1861
     optparser.set_usage(optparser.usage + " COMMAND [parameters]")
1861 1862
     optparser.set_description('S3cmd is a tool for managing objects in '+
... ...
@@ -1878,6 +1901,14 @@ def main():
1878 1878
         output(u"s3cmd version %s" % PkgInfo.version)
1879 1879
         sys.exit(0)
1880 1880
 
1881
+    if options.quiet:
1882
+        try:
1883
+            f = open("/dev/null", "w")
1884
+            sys.stdout.close()
1885
+            sys.stdout = f
1886
+        except IOError:
1887
+            warning(u"Unable to open /dev/null: --quiet disabled.")
1888
+
1881 1889
     ## Now finally parse the config file
1882 1890
     if not options.config:
1883 1891
         error(u"Can't find a config file. Please use --config option.")
... ...
@@ -2099,6 +2130,7 @@ if __name__ == '__main__':
2099 2099
         from S3.S3 import S3
2100 2100
         from S3.Config import Config
2101 2101
         from S3.SortedDict import SortedDict
2102
+        from S3.FileDict import FileDict
2102 2103
         from S3.S3Uri import S3Uri
2103 2104
         from S3 import Utils
2104 2105
         from S3.Utils import *
... ...
@@ -57,8 +57,11 @@ Move object
57 57
 s3cmd \fBsetacl\fR \fIs3://BUCKET[/OBJECT]\fR
58 58
 Modify Access control list for Bucket or Files
59 59
 .TP
60
-s3cmd \fBsetpolicy\fR \fIs3://BUCKET POLICY_STRING\fR
61
-Set an access policy for a bucket
60
+s3cmd \fBsetpolicy\fR \fIFILE s3://BUCKET\fR
61
+Modify Bucket Policy
62
+.TP
63
+s3cmd \fBdelpolicy\fR \fIs3://BUCKET\fR
64
+Delete Bucket Policy
62 65
 .TP
63 66
 s3cmd \fBaccesslog\fR \fIs3://BUCKET\fR
64 67
 Enable/disable bucket access logging
... ...
@@ -318,13 +321,16 @@ Enable verbose output.
318 318
 Enable debug output.
319 319
 .TP
320 320
 \fB\-\-version\fR
321
-Show s3cmd version (1.5.0-alpha1) and exit.
321
+Show s3cmd version (1.5.0-alpha3) and exit.
322 322
 .TP
323 323
 \fB\-F\fR, \fB\-\-follow\-symlinks\fR
324 324
 Follow symbolic links as if they are regular files
325 325
 .TP
326 326
 \fB\-\-cache\-file\fR=FILE
327 327
 Cache FILE containing local source MD5 values
328
+.TP
329
+\fB\-q\fR, \fB\-\-quiet\fR
330
+Silence output on stdout
328 331
 
329 332
 
330 333
 .SH EXAMPLES