... | ... |
@@ -1,5 +1,18 @@ |
1 |
-s3cmd 1.5.0 - ??? |
|
2 |
-=========== |
|
1 |
+s3cmd 1.5.0-alpha3 - 2013-03-11 |
|
2 |
+================== |
|
3 |
+* Persistent HTTP/HTTPS connections for massive speedup (Michal Ludvig) |
|
4 |
+* New switch --quiet for suppressing all output (Siddarth Prakash) |
|
5 |
+* Honour "umask" on file downloads (Jason Dalton) |
|
6 |
+* Various bugfixes from many contributors |
|
7 |
+ |
|
8 |
+s3cmd 1.5.0-alpha2 - 2013-03-04 |
|
9 |
+================== |
|
10 |
+* IAM roles support (David Kohen, Eric Dowd) |
|
11 |
+* Manage bucket policies (Kota Uenishi) |
|
12 |
+* Various bugfixes from many contributors |
|
13 |
+ |
|
14 |
+s3cmd 1.5.0-alpha1 - 2013-02-19 |
|
15 |
+================== |
|
3 | 16 |
* Server-side copy for hardlinks/softlinks to improve performance |
4 | 17 |
(Matt Domsch) |
5 | 18 |
* New [signurl] command (Craig Ringer) |
6 | 19 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,71 @@ |
0 |
+import httplib |
|
1 |
+from urlparse import urlparse |
|
2 |
+from threading import Semaphore |
|
3 |
+from logging import debug, info, warning, error |
|
4 |
+ |
|
5 |
+from Config import Config |
|
6 |
+from Exceptions import ParameterError |
|
7 |
+ |
|
8 |
+__all__ = [ "ConnMan" ] |
|
9 |
+ |
|
10 |
+class http_connection(object): |
|
11 |
+ def __init__(self, id, hostname, ssl, cfg): |
|
12 |
+ self.hostname = hostname |
|
13 |
+ self.ssl = ssl |
|
14 |
+ self.id = id |
|
15 |
+ self.counter = 0 |
|
16 |
+ if cfg.proxy_host != "": |
|
17 |
+ self.c = httplib.HTTPConnection(cfg.proxy_host, cfg.proxy_port) |
|
18 |
+ elif not ssl: |
|
19 |
+ self.c = httplib.HTTPConnection(hostname) |
|
20 |
+ else: |
|
21 |
+ self.c = httplib.HTTPSConnection(hostname) |
|
22 |
+ |
|
23 |
+class ConnMan(object): |
|
24 |
+ conn_pool_sem = Semaphore() |
|
25 |
+ conn_pool = {} |
|
26 |
+ conn_max_counter = 800 ## AWS closes connection after some ~90 requests |
|
27 |
+ |
|
28 |
+ @staticmethod |
|
29 |
+ def get(hostname, ssl = None): |
|
30 |
+ cfg = Config() |
|
31 |
+ if ssl == None: |
|
32 |
+ ssl = cfg.use_https |
|
33 |
+ conn = None |
|
34 |
+ if cfg.proxy_host != "": |
|
35 |
+ if ssl: |
|
36 |
+ raise ParameterError("use_ssl=True can't be used with proxy") |
|
37 |
+ conn_id = "proxy://%s:%s" % (cfg.proxy_host, cfg.proxy_port) |
|
38 |
+ else: |
|
39 |
+ conn_id = "http%s://%s" % (ssl and "s" or "", hostname) |
|
40 |
+ ConnMan.conn_pool_sem.acquire() |
|
41 |
+ if not ConnMan.conn_pool.has_key(conn_id): |
|
42 |
+ ConnMan.conn_pool[conn_id] = [] |
|
43 |
+ if len(ConnMan.conn_pool[conn_id]): |
|
44 |
+ conn = ConnMan.conn_pool[conn_id].pop() |
|
45 |
+ debug("ConnMan.get(): re-using connection: %s#%d" % (conn.id, conn.counter)) |
|
46 |
+ ConnMan.conn_pool_sem.release() |
|
47 |
+ if not conn: |
|
48 |
+ debug("ConnMan.get(): creating new connection: %s" % conn_id) |
|
49 |
+ conn = http_connection(conn_id, hostname, ssl, cfg) |
|
50 |
+ conn.c.connect() |
|
51 |
+ conn.counter += 1 |
|
52 |
+ return conn |
|
53 |
+ |
|
54 |
+ @staticmethod |
|
55 |
+ def put(conn): |
|
56 |
+ if conn.id.startswith("proxy://"): |
|
57 |
+ conn.c.close() |
|
58 |
+ debug("ConnMan.put(): closing proxy connection (keep-alive not yet supported)") |
|
59 |
+ return |
|
60 |
+ |
|
61 |
+ if conn.counter >= ConnMan.conn_max_counter: |
|
62 |
+ conn.c.close() |
|
63 |
+ debug("ConnMan.put(): closing over-used connection") |
|
64 |
+ return |
|
65 |
+ |
|
66 |
+ ConnMan.conn_pool_sem.acquire() |
|
67 |
+ ConnMan.conn_pool[conn.id].append(conn) |
|
68 |
+ ConnMan.conn_pool_sem.release() |
|
69 |
+ debug("ConnMan.put(): connection put back to pool (%s#%d)" % (conn.id, conn.counter)) |
|
70 |
+ |
0 | 71 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,53 @@ |
0 |
+## Amazon S3 manager |
|
1 |
+## Author: Michal Ludvig <michal@logix.cz> |
|
2 |
+## http://www.logix.cz/michal |
|
3 |
+## License: GPL Version 2 |
|
4 |
+ |
|
5 |
+from SortedDict import SortedDict |
|
6 |
+import Utils |
|
7 |
+ |
|
8 |
+class FileDict(SortedDict): |
|
9 |
+ def __init__(self, mapping = {}, ignore_case = True, **kwargs): |
|
10 |
+ SortedDict.__init__(self, mapping = mapping, ignore_case = ignore_case, **kwargs) |
|
11 |
+ self.hardlinks = dict() # { dev: { inode : {'md5':, 'relative_files':}}} |
|
12 |
+ self.by_md5 = dict() # {md5: set(relative_files)} |
|
13 |
+ |
|
14 |
+ def record_md5(self, relative_file, md5): |
|
15 |
+ if md5 not in self.by_md5: |
|
16 |
+ self.by_md5[md5] = set() |
|
17 |
+ self.by_md5[md5].add(relative_file) |
|
18 |
+ |
|
19 |
+ def find_md5_one(self, md5): |
|
20 |
+ try: |
|
21 |
+ return list(self.by_md5.get(md5, set()))[0] |
|
22 |
+ except: |
|
23 |
+ return None |
|
24 |
+ |
|
25 |
+ def get_md5(self, relative_file): |
|
26 |
+ """returns md5 if it can, or raises IOError if file is unreadable""" |
|
27 |
+ md5 = None |
|
28 |
+ if 'md5' in self[relative_file]: |
|
29 |
+ return self[relative_file]['md5'] |
|
30 |
+ md5 = self.get_hardlink_md5(relative_file) |
|
31 |
+ if md5 is None: |
|
32 |
+ md5 = Utils.hash_file_md5(self[relative_file]['full_name']) |
|
33 |
+ self.record_md5(relative_file, md5) |
|
34 |
+ self[relative_file]['md5'] = md5 |
|
35 |
+ return md5 |
|
36 |
+ |
|
37 |
+ def record_hardlink(self, relative_file, dev, inode, md5): |
|
38 |
+ if dev not in self.hardlinks: |
|
39 |
+ self.hardlinks[dev] = dict() |
|
40 |
+ if inode not in self.hardlinks[dev]: |
|
41 |
+ self.hardlinks[dev][inode] = dict(md5=md5, relative_files=set()) |
|
42 |
+ self.hardlinks[dev][inode]['relative_files'].add(relative_file) |
|
43 |
+ |
|
44 |
+ def get_hardlink_md5(self, relative_file): |
|
45 |
+ md5 = None |
|
46 |
+ dev = self[relative_file]['dev'] |
|
47 |
+ inode = self[relative_file]['inode'] |
|
48 |
+ try: |
|
49 |
+ md5 = self.hardlinks[dev][inode]['md5'] |
|
50 |
+ except: |
|
51 |
+ pass |
|
52 |
+ return md5 |
... | ... |
@@ -6,7 +6,7 @@ |
6 | 6 |
from S3 import S3 |
7 | 7 |
from Config import Config |
8 | 8 |
from S3Uri import S3Uri |
9 |
-from SortedDict import SortedDict |
|
9 |
+from FileDict import FileDict |
|
10 | 10 |
from Utils import * |
11 | 11 |
from Exceptions import ParameterError |
12 | 12 |
from HashCache import HashCache |
... | ... |
@@ -58,7 +58,7 @@ def _fswalk_no_symlinks(path): |
58 | 58 |
def filter_exclude_include(src_list): |
59 | 59 |
info(u"Applying --exclude/--include") |
60 | 60 |
cfg = Config() |
61 |
- exclude_list = SortedDict(ignore_case = False) |
|
61 |
+ exclude_list = FileDict(ignore_case = False) |
|
62 | 62 |
for file in src_list.keys(): |
63 | 63 |
debug(u"CHECK: %s" % file) |
64 | 64 |
excluded = False |
... | ... |
@@ -224,7 +224,7 @@ def fetch_local_list(args, recursive = None): |
224 | 224 |
info(u"No cache file found, creating it.") |
225 | 225 |
|
226 | 226 |
local_uris = [] |
227 |
- local_list = SortedDict(ignore_case = False) |
|
227 |
+ local_list = FileDict(ignore_case = False) |
|
228 | 228 |
single_file = False |
229 | 229 |
|
230 | 230 |
if type(args) not in (list, tuple): |
... | ... |
@@ -284,7 +284,7 @@ def fetch_remote_list(args, require_attribs = False, recursive = None): |
284 | 284 |
rem_base = rem_base[:rem_base.rfind('/')+1] |
285 | 285 |
remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base)) |
286 | 286 |
rem_base_len = len(rem_base) |
287 |
- rem_list = SortedDict(ignore_case = False) |
|
287 |
+ rem_list = FileDict(ignore_case = False) |
|
288 | 288 |
break_now = False |
289 | 289 |
for object in response['list']: |
290 | 290 |
if object['Key'] == rem_base_original and object['Key'][-1] != os.path.sep: |
... | ... |
@@ -292,7 +292,7 @@ def fetch_remote_list(args, require_attribs = False, recursive = None): |
292 | 292 |
key = os.path.basename(object['Key']) |
293 | 293 |
object_uri_str = remote_uri_original.uri() |
294 | 294 |
break_now = True |
295 |
- rem_list = SortedDict(ignore_case = False) ## Remove whatever has already been put to rem_list |
|
295 |
+ rem_list = FileDict(ignore_case = False) ## Remove whatever has already been put to rem_list |
|
296 | 296 |
else: |
297 | 297 |
key = object['Key'][rem_base_len:] ## Beware - this may be '' if object['Key']==rem_base !! |
298 | 298 |
object_uri_str = remote_uri.uri() + key |
... | ... |
@@ -314,7 +314,7 @@ def fetch_remote_list(args, require_attribs = False, recursive = None): |
314 | 314 |
|
315 | 315 |
cfg = Config() |
316 | 316 |
remote_uris = [] |
317 |
- remote_list = SortedDict(ignore_case = False) |
|
317 |
+ remote_list = FileDict(ignore_case = False) |
|
318 | 318 |
|
319 | 319 |
if type(args) not in (list, tuple): |
320 | 320 |
args = [args] |
... | ... |
@@ -436,7 +436,7 @@ def compare_filelists(src_list, dst_list, src_remote, dst_remote, delay_updates |
436 | 436 |
## Items left on src_list will be transferred |
437 | 437 |
## Items left on update_list will be transferred after src_list |
438 | 438 |
## Items left on copy_pairs will be copied from dst1 to dst2 |
439 |
- update_list = SortedDict(ignore_case = False) |
|
439 |
+ update_list = FileDict(ignore_case = False) |
|
440 | 440 |
## Items left on dst_list will be deleted |
441 | 441 |
copy_pairs = [] |
442 | 442 |
|
... | ... |
@@ -27,6 +27,7 @@ from Config import Config |
27 | 27 |
from Exceptions import * |
28 | 28 |
from MultiPart import MultiPartUpload |
29 | 29 |
from S3Uri import S3Uri |
30 |
+from ConnMan import ConnMan |
|
30 | 31 |
|
31 | 32 |
try: |
32 | 33 |
import magic, gzip |
... | ... |
@@ -190,15 +191,6 @@ class S3(object): |
190 | 190 |
def __init__(self, config): |
191 | 191 |
self.config = config |
192 | 192 |
|
193 |
- def get_connection(self, bucket): |
|
194 |
- if self.config.proxy_host != "": |
|
195 |
- return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port) |
|
196 |
- else: |
|
197 |
- if self.config.use_https: |
|
198 |
- return httplib.HTTPSConnection(self.get_hostname(bucket)) |
|
199 |
- else: |
|
200 |
- return httplib.HTTPConnection(self.get_hostname(bucket)) |
|
201 |
- |
|
202 | 193 |
def get_hostname(self, bucket): |
203 | 194 |
if bucket and check_bucket_name_dns_conformity(bucket): |
204 | 195 |
if self.redir_map.has_key(bucket): |
... | ... |
@@ -246,10 +238,9 @@ class S3(object): |
246 | 246 |
truncated = True |
247 | 247 |
list = [] |
248 | 248 |
prefixes = [] |
249 |
- conn = self.get_connection(bucket) |
|
250 | 249 |
|
251 | 250 |
while truncated: |
252 |
- response = self.bucket_list_noparse(conn, bucket, prefix, recursive, uri_params) |
|
251 |
+ response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params) |
|
253 | 252 |
current_list = _get_contents(response["data"]) |
254 | 253 |
current_prefixes = _get_common_prefixes(response["data"]) |
255 | 254 |
truncated = _list_truncated(response["data"]) |
... | ... |
@@ -263,19 +254,17 @@ class S3(object): |
263 | 263 |
list += current_list |
264 | 264 |
prefixes += current_prefixes |
265 | 265 |
|
266 |
- conn.close() |
|
267 |
- |
|
268 | 266 |
response['list'] = list |
269 | 267 |
response['common_prefixes'] = prefixes |
270 | 268 |
return response |
271 | 269 |
|
272 |
- def bucket_list_noparse(self, connection, bucket, prefix = None, recursive = None, uri_params = {}): |
|
270 |
+ def bucket_list_noparse(self, bucket, prefix = None, recursive = None, uri_params = {}): |
|
273 | 271 |
if prefix: |
274 | 272 |
uri_params['prefix'] = self.urlencode_string(prefix) |
275 | 273 |
if not self.config.recursive and not recursive: |
276 | 274 |
uri_params['delimiter'] = "/" |
277 | 275 |
request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params) |
278 |
- response = self.send_request(request, conn = connection) |
|
276 |
+ response = self.send_request(request) |
|
279 | 277 |
#debug(response) |
280 | 278 |
return response |
281 | 279 |
|
... | ... |
@@ -679,7 +668,7 @@ class S3(object): |
679 | 679 |
# Wait a few seconds. The more it fails the more we wait. |
680 | 680 |
return (self._max_retries - retries + 1) * 3 |
681 | 681 |
|
682 |
- def send_request(self, request, body = None, retries = _max_retries, conn = None): |
|
682 |
+ def send_request(self, request, body = None, retries = _max_retries): |
|
683 | 683 |
method_string, resource, headers = request.get_triplet() |
684 | 684 |
debug("Processing request, please wait...") |
685 | 685 |
if not headers.has_key('content-length'): |
... | ... |
@@ -688,25 +677,20 @@ class S3(object): |
688 | 688 |
# "Stringify" all headers |
689 | 689 |
for header in headers.keys(): |
690 | 690 |
headers[header] = str(headers[header]) |
691 |
- if conn is None: |
|
692 |
- debug("Establishing connection") |
|
693 |
- conn = self.get_connection(resource['bucket']) |
|
694 |
- close_conn = True |
|
695 |
- else: |
|
696 |
- debug("Using existing connection") |
|
697 |
- close_conn = False |
|
691 |
+ conn = ConnMan.get(self.get_hostname(resource['bucket'])) |
|
698 | 692 |
uri = self.format_uri(resource) |
699 | 693 |
debug("Sending request method_string=%r, uri=%r, headers=%r, body=(%i bytes)" % (method_string, uri, headers, len(body or ""))) |
700 |
- conn.request(method_string, uri, body, headers) |
|
694 |
+ conn.c.request(method_string, uri, body, headers) |
|
701 | 695 |
response = {} |
702 |
- http_response = conn.getresponse() |
|
696 |
+ http_response = conn.c.getresponse() |
|
703 | 697 |
response["status"] = http_response.status |
704 | 698 |
response["reason"] = http_response.reason |
705 | 699 |
response["headers"] = convertTupleListToDict(http_response.getheaders()) |
706 | 700 |
response["data"] = http_response.read() |
707 | 701 |
debug("Response: " + str(response)) |
708 |
- if close_conn is True: |
|
709 |
- conn.close() |
|
702 |
+ ConnMan.put(conn) |
|
703 |
+ except ParameterError, e: |
|
704 |
+ raise |
|
710 | 705 |
except Exception, e: |
711 | 706 |
if retries: |
712 | 707 |
warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) |
... | ... |
@@ -749,12 +733,13 @@ class S3(object): |
749 | 749 |
info("Sending file '%s', please wait..." % file.name) |
750 | 750 |
timestamp_start = time.time() |
751 | 751 |
try: |
752 |
- conn = self.get_connection(resource['bucket']) |
|
753 |
- conn.connect() |
|
754 |
- conn.putrequest(method_string, self.format_uri(resource)) |
|
752 |
+ conn = ConnMan.get(self.get_hostname(resource['bucket'])) |
|
753 |
+ conn.c.putrequest(method_string, self.format_uri(resource)) |
|
755 | 754 |
for header in headers.keys(): |
756 |
- conn.putheader(header, str(headers[header])) |
|
757 |
- conn.endheaders() |
|
755 |
+ conn.c.putheader(header, str(headers[header])) |
|
756 |
+ conn.c.endheaders() |
|
757 |
+ except ParameterError, e: |
|
758 |
+ raise |
|
758 | 759 |
except Exception, e: |
759 | 760 |
if self.config.progress_meter: |
760 | 761 |
progress.done("failed") |
... | ... |
@@ -777,7 +762,7 @@ class S3(object): |
777 | 777 |
else: |
778 | 778 |
data = buffer |
779 | 779 |
md5_hash.update(data) |
780 |
- conn.send(data) |
|
780 |
+ conn.c.send(data) |
|
781 | 781 |
if self.config.progress_meter: |
782 | 782 |
progress.update(delta_position = len(data)) |
783 | 783 |
size_left -= len(data) |
... | ... |
@@ -785,14 +770,16 @@ class S3(object): |
785 | 785 |
time.sleep(throttle) |
786 | 786 |
md5_computed = md5_hash.hexdigest() |
787 | 787 |
response = {} |
788 |
- http_response = conn.getresponse() |
|
788 |
+ http_response = conn.c.getresponse() |
|
789 | 789 |
response["status"] = http_response.status |
790 | 790 |
response["reason"] = http_response.reason |
791 | 791 |
response["headers"] = convertTupleListToDict(http_response.getheaders()) |
792 | 792 |
response["data"] = http_response.read() |
793 | 793 |
response["size"] = size_total |
794 |
- conn.close() |
|
794 |
+ ConnMan.put(conn) |
|
795 | 795 |
debug(u"Response: %s" % response) |
796 |
+ except ParameterError, e: |
|
797 |
+ raise |
|
796 | 798 |
except Exception, e: |
797 | 799 |
if self.config.progress_meter: |
798 | 800 |
progress.done("failed") |
... | ... |
@@ -814,7 +801,7 @@ class S3(object): |
814 | 814 |
response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1) |
815 | 815 |
|
816 | 816 |
if self.config.progress_meter: |
817 |
- ## The above conn.close() takes some time -> update() progress meter |
|
817 |
+ ## Finalising the upload takes some time -> update() progress meter |
|
818 | 818 |
## to correct the average speed. Otherwise people will complain that |
819 | 819 |
## 'progress' and response["speed"] are inconsistent ;-) |
820 | 820 |
progress.update() |
... | ... |
@@ -889,21 +876,22 @@ class S3(object): |
889 | 889 |
info("Receiving file '%s', please wait..." % stream.name) |
890 | 890 |
timestamp_start = time.time() |
891 | 891 |
try: |
892 |
- conn = self.get_connection(resource['bucket']) |
|
893 |
- conn.connect() |
|
894 |
- conn.putrequest(method_string, self.format_uri(resource)) |
|
892 |
+ conn = ConnMan.get(self.get_hostname(resource['bucket'])) |
|
893 |
+ conn.c.putrequest(method_string, self.format_uri(resource)) |
|
895 | 894 |
for header in headers.keys(): |
896 |
- conn.putheader(header, str(headers[header])) |
|
895 |
+ conn.c.putheader(header, str(headers[header])) |
|
897 | 896 |
if start_position > 0: |
898 | 897 |
debug("Requesting Range: %d .. end" % start_position) |
899 |
- conn.putheader("Range", "bytes=%d-" % start_position) |
|
900 |
- conn.endheaders() |
|
898 |
+ conn.c.putheader("Range", "bytes=%d-" % start_position) |
|
899 |
+ conn.c.endheaders() |
|
901 | 900 |
response = {} |
902 |
- http_response = conn.getresponse() |
|
901 |
+ http_response = conn.c.getresponse() |
|
903 | 902 |
response["status"] = http_response.status |
904 | 903 |
response["reason"] = http_response.reason |
905 | 904 |
response["headers"] = convertTupleListToDict(http_response.getheaders()) |
906 | 905 |
debug("Response: %s" % response) |
906 |
+ except ParameterError, e: |
|
907 |
+ raise |
|
907 | 908 |
except Exception, e: |
908 | 909 |
if self.config.progress_meter: |
909 | 910 |
progress.done("failed") |
... | ... |
@@ -955,7 +943,7 @@ class S3(object): |
955 | 955 |
## Call progress meter from here... |
956 | 956 |
if self.config.progress_meter: |
957 | 957 |
progress.update(delta_position = len(data)) |
958 |
- conn.close() |
|
958 |
+ ConnMan.put(conn) |
|
959 | 959 |
except Exception, e: |
960 | 960 |
if self.config.progress_meter: |
961 | 961 |
progress.done("failed") |
... | ... |
@@ -27,8 +27,6 @@ class SortedDict(dict): |
27 | 27 |
""" |
28 | 28 |
dict.__init__(self, mapping, **kwargs) |
29 | 29 |
self.ignore_case = ignore_case |
30 |
- self.hardlinks = dict() # { dev: { inode : {'md5':, 'relative_files':}}} |
|
31 |
- self.by_md5 = dict() # {md5: set(relative_files)} |
|
32 | 30 |
|
33 | 31 |
def keys(self): |
34 | 32 |
keys = dict.keys(self) |
... | ... |
@@ -49,45 +47,6 @@ class SortedDict(dict): |
49 | 49 |
return SortedDictIterator(self, self.keys()) |
50 | 50 |
|
51 | 51 |
|
52 |
- def record_md5(self, relative_file, md5): |
|
53 |
- if md5 not in self.by_md5: |
|
54 |
- self.by_md5[md5] = set() |
|
55 |
- self.by_md5[md5].add(relative_file) |
|
56 |
- |
|
57 |
- def find_md5_one(self, md5): |
|
58 |
- try: |
|
59 |
- return list(self.by_md5.get(md5, set()))[0] |
|
60 |
- except: |
|
61 |
- return None |
|
62 |
- |
|
63 |
- def get_md5(self, relative_file): |
|
64 |
- """returns md5 if it can, or raises IOError if file is unreadable""" |
|
65 |
- md5 = None |
|
66 |
- if 'md5' in self[relative_file]: |
|
67 |
- return self[relative_file]['md5'] |
|
68 |
- md5 = self.get_hardlink_md5(relative_file) |
|
69 |
- if md5 is None: |
|
70 |
- md5 = Utils.hash_file_md5(self[relative_file]['full_name']) |
|
71 |
- self.record_md5(relative_file, md5) |
|
72 |
- self[relative_file]['md5'] = md5 |
|
73 |
- return md5 |
|
74 |
- |
|
75 |
- def record_hardlink(self, relative_file, dev, inode, md5): |
|
76 |
- if dev not in self.hardlinks: |
|
77 |
- self.hardlinks[dev] = dict() |
|
78 |
- if inode not in self.hardlinks[dev]: |
|
79 |
- self.hardlinks[dev][inode] = dict(md5=md5, relative_files=set()) |
|
80 |
- self.hardlinks[dev][inode]['relative_files'].add(relative_file) |
|
81 |
- |
|
82 |
- def get_hardlink_md5(self, relative_file): |
|
83 |
- md5 = None |
|
84 |
- dev = self[relative_file]['dev'] |
|
85 |
- inode = self[relative_file]['inode'] |
|
86 |
- try: |
|
87 |
- md5 = self.hardlinks[dev][inode]['md5'] |
|
88 |
- except: |
|
89 |
- pass |
|
90 |
- return md5 |
|
91 | 52 |
|
92 | 53 |
if __name__ == "__main__": |
93 | 54 |
d = { 'AWS' : 1, 'Action' : 2, 'america' : 3, 'Auckland' : 4, 'America' : 5 } |
... | ... |
@@ -656,10 +656,11 @@ def cmd_sync_remote2remote(args): |
656 | 656 |
|
657 | 657 |
print(u"Summary: %d source files to copy, %d files at destination to delete" % (src_count, dst_count)) |
658 | 658 |
|
659 |
- if src_count > 0: |
|
660 |
- ### Populate 'remote_uri' only if we've got something to sync from src to dst |
|
661 |
- for key in src_list: |
|
662 |
- src_list[key]['target_uri'] = destination_base + key |
|
659 |
+ ### Populate 'target_uri' only if we've got something to sync from src to dst |
|
660 |
+ for key in src_list: |
|
661 |
+ src_list[key]['target_uri'] = destination_base + key |
|
662 |
+ for key in update_list: |
|
663 |
+ update_list[key]['target_uri'] = destination_base + key |
|
663 | 664 |
|
664 | 665 |
if cfg.dry_run: |
665 | 666 |
for key in exclude_list: |
... | ... |
@@ -745,6 +746,8 @@ def cmd_sync_remote2local(args): |
745 | 745 |
info(u"Summary: %d remote files to download, %d local files to delete, %d local files to hardlink" % (remote_count + update_count, local_count, copy_pairs_count)) |
746 | 746 |
|
747 | 747 |
def _set_local_filename(remote_list, destination_base): |
748 |
+ if len(remote_list) == 0: |
|
749 |
+ return |
|
748 | 750 |
if not os.path.isdir(destination_base): |
749 | 751 |
## We were either given a file name (existing or not) or want STDOUT |
750 | 752 |
if len(remote_list) > 1: |
... | ... |
@@ -811,6 +814,15 @@ def cmd_sync_remote2local(args): |
811 | 811 |
dst_stream.close() |
812 | 812 |
# download completed, rename the file to destination |
813 | 813 |
os.rename(chkptfname, dst_file) |
814 |
+ |
|
815 |
+ # set permissions on destination file |
|
816 |
+ original_umask = os.umask(0); |
|
817 |
+ os.umask(original_umask); |
|
818 |
+ mode = 0777 - original_umask; |
|
819 |
+ debug(u"mode=%s" % oct(mode)) |
|
820 |
+ |
|
821 |
+ os.chmod(dst_file, mode); |
|
822 |
+ |
|
814 | 823 |
debug(u"renamed chkptfname=%s to dst_file=%s" % (unicodise(chkptfname), unicodise(dst_file))) |
815 | 824 |
if response['headers'].has_key('x-amz-meta-s3cmd-attrs') and cfg.preserve_attrs: |
816 | 825 |
attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs']) |
... | ... |
@@ -822,7 +834,9 @@ def cmd_sync_remote2local(args): |
822 | 822 |
os.utime(dst_file, (atime, mtime)) |
823 | 823 |
## FIXME: uid/gid / uname/gname handling comes here! TODO |
824 | 824 |
except OSError, e: |
825 |
- try: dst_stream.close() |
|
825 |
+ try: |
|
826 |
+ dst_stream.close() |
|
827 |
+ os.remove(chkptfname) |
|
826 | 828 |
except: pass |
827 | 829 |
if e.errno == errno.EEXIST: |
828 | 830 |
warning(u"%s exists - not overwriting" % (dst_file)) |
... | ... |
@@ -835,19 +849,25 @@ def cmd_sync_remote2local(args): |
835 | 835 |
continue |
836 | 836 |
raise e |
837 | 837 |
except KeyboardInterrupt: |
838 |
- try: dst_stream.close() |
|
838 |
+ try: |
|
839 |
+ dst_stream.close() |
|
840 |
+ os.remove(chkptfname) |
|
839 | 841 |
except: pass |
840 | 842 |
warning(u"Exiting after keyboard interrupt") |
841 | 843 |
return |
842 | 844 |
except Exception, e: |
843 |
- try: dst_stream.close() |
|
845 |
+ try: |
|
846 |
+ dst_stream.close() |
|
847 |
+ os.remove(chkptfname) |
|
844 | 848 |
except: pass |
845 | 849 |
error(u"%s: %s" % (file, e)) |
846 | 850 |
continue |
847 | 851 |
# We have to keep repeating this call because |
848 | 852 |
# Python 2.4 doesn't support try/except/finally |
849 | 853 |
# construction :-( |
850 |
- try: dst_stream.close() |
|
854 |
+ try: |
|
855 |
+ dst_stream.close() |
|
856 |
+ os.remove(chkptfname) |
|
851 | 857 |
except: pass |
852 | 858 |
except S3DownloadError, e: |
853 | 859 |
error(u"%s: download failed too many times. Skipping that file." % file) |
... | ... |
@@ -893,7 +913,7 @@ def local_copy(copy_pairs, destination_base): |
893 | 893 |
# Do NOT hardlink local files by default, that'd be silly |
894 | 894 |
# For instance all empty files would become hardlinked together! |
895 | 895 |
|
896 |
- failed_copy_list = SortedDict() |
|
896 |
+ failed_copy_list = FileDict() |
|
897 | 897 |
for (src_obj, dst1, relative_file) in copy_pairs: |
898 | 898 |
src_file = os.path.join(destination_base, dst1) |
899 | 899 |
dst_file = os.path.join(destination_base, relative_file) |
... | ... |
@@ -1058,7 +1078,8 @@ def cmd_sync_local2remote(args): |
1058 | 1058 |
## Make remote_key same as local_key for comparison if we're dealing with only one file |
1059 | 1059 |
remote_list_entry = remote_list[remote_list.keys()[0]] |
1060 | 1060 |
# Flush remote_list, by the way |
1061 |
- remote_list = { local_list.keys()[0] : remote_list_entry } |
|
1061 |
+ remote_list = FileDict() |
|
1062 |
+ remote_list[local_list.keys()[0]] = remote_list_entry |
|
1062 | 1063 |
|
1063 | 1064 |
local_list, remote_list, update_list, copy_pairs = compare_filelists(local_list, remote_list, src_remote = False, dst_remote = True, delay_updates = cfg.delay_updates) |
1064 | 1065 |
|
... | ... |
@@ -1644,6 +1665,7 @@ def get_commands_list(): |
1644 | 1644 |
|
1645 | 1645 |
{"cmd":"multipart", "label":"show multipart uploads", "param":"s3://BUCKET [Id]", "func":cmd_multipart, "argc":1}, |
1646 | 1646 |
{"cmd":"abortmp", "label":"abort a multipart upload", "param":"s3://BUCKET/OBJECT Id", "func":cmd_abort_multipart, "argc":2}, |
1647 |
+ |
|
1647 | 1648 |
{"cmd":"listmp", "label":"list parts of a multipart upload", "param":"s3://BUCKET/OBJECT Id", "func":cmd_list_multipart, "argc":2}, |
1648 | 1649 |
|
1649 | 1650 |
{"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1}, |
... | ... |
@@ -1856,6 +1878,7 @@ def main(): |
1856 | 1856 |
optparser.add_option( "--version", dest="show_version", action="store_true", help="Show s3cmd version (%s) and exit." % (PkgInfo.version)) |
1857 | 1857 |
optparser.add_option("-F", "--follow-symlinks", dest="follow_symlinks", action="store_true", default=False, help="Follow symbolic links as if they are regular files") |
1858 | 1858 |
optparser.add_option( "--cache-file", dest="cache_file", action="store", default="", metavar="FILE", help="Cache FILE containing local source MD5 values") |
1859 |
+ optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Silence output on stdout") |
|
1859 | 1860 |
|
1860 | 1861 |
optparser.set_usage(optparser.usage + " COMMAND [parameters]") |
1861 | 1862 |
optparser.set_description('S3cmd is a tool for managing objects in '+ |
... | ... |
@@ -1878,6 +1901,14 @@ def main(): |
1878 | 1878 |
output(u"s3cmd version %s" % PkgInfo.version) |
1879 | 1879 |
sys.exit(0) |
1880 | 1880 |
|
1881 |
+ if options.quiet: |
|
1882 |
+ try: |
|
1883 |
+ f = open("/dev/null", "w") |
|
1884 |
+ sys.stdout.close() |
|
1885 |
+ sys.stdout = f |
|
1886 |
+ except IOError: |
|
1887 |
+ warning(u"Unable to open /dev/null: --quiet disabled.") |
|
1888 |
+ |
|
1881 | 1889 |
## Now finally parse the config file |
1882 | 1890 |
if not options.config: |
1883 | 1891 |
error(u"Can't find a config file. Please use --config option.") |
... | ... |
@@ -2099,6 +2130,7 @@ if __name__ == '__main__': |
2099 | 2099 |
from S3.S3 import S3 |
2100 | 2100 |
from S3.Config import Config |
2101 | 2101 |
from S3.SortedDict import SortedDict |
2102 |
+ from S3.FileDict import FileDict |
|
2102 | 2103 |
from S3.S3Uri import S3Uri |
2103 | 2104 |
from S3 import Utils |
2104 | 2105 |
from S3.Utils import * |
... | ... |
@@ -57,8 +57,11 @@ Move object |
57 | 57 |
s3cmd \fBsetacl\fR \fIs3://BUCKET[/OBJECT]\fR |
58 | 58 |
Modify Access control list for Bucket or Files |
59 | 59 |
.TP |
60 |
-s3cmd \fBsetpolicy\fR \fIs3://BUCKET POLICY_STRING\fR |
|
61 |
-Set an access policy for a bucket |
|
60 |
+s3cmd \fBsetpolicy\fR \fIFILE s3://BUCKET\fR |
|
61 |
+Modify Bucket Policy |
|
62 |
+.TP |
|
63 |
+s3cmd \fBdelpolicy\fR \fIs3://BUCKET\fR |
|
64 |
+Delete Bucket Policy |
|
62 | 65 |
.TP |
63 | 66 |
s3cmd \fBaccesslog\fR \fIs3://BUCKET\fR |
64 | 67 |
Enable/disable bucket access logging |
... | ... |
@@ -318,13 +321,16 @@ Enable verbose output. |
318 | 318 |
Enable debug output. |
319 | 319 |
.TP |
320 | 320 |
\fB\-\-version\fR |
321 |
-Show s3cmd version (1.5.0-alpha1) and exit. |
|
321 |
+Show s3cmd version (1.5.0-alpha3) and exit. |
|
322 | 322 |
.TP |
323 | 323 |
\fB\-F\fR, \fB\-\-follow\-symlinks\fR |
324 | 324 |
Follow symbolic links as if they are regular files |
325 | 325 |
.TP |
326 | 326 |
\fB\-\-cache\-file\fR=FILE |
327 | 327 |
Cache FILE containing local source MD5 values |
328 |
+.TP |
|
329 |
+\fB\-q\fR, \fB\-\-quiet\fR |
|
330 |
+Silence output on stdout |
|
328 | 331 |
|
329 | 332 |
|
330 | 333 |
.SH EXAMPLES |