Batch misc fixes 423 424 448
Matt Domsch authored on 2015/01/06 13:33:59... | ... |
@@ -12,6 +12,7 @@ import sys |
12 | 12 |
import Progress |
13 | 13 |
from SortedDict import SortedDict |
14 | 14 |
import httplib |
15 |
+import locale |
|
15 | 16 |
try: |
16 | 17 |
import json |
17 | 18 |
except ImportError, e: |
... | ... |
@@ -94,7 +95,7 @@ class Config(object): |
94 | 94 |
# Dict mapping compiled REGEXPs back to their textual form |
95 | 95 |
debug_exclude = {} |
96 | 96 |
debug_include = {} |
97 |
- encoding = "utf-8" |
|
97 |
+ encoding = locale.getpreferredencoding() or "UTF-8" |
|
98 | 98 |
urlencoding_mode = "normal" |
99 | 99 |
log_target_prefix = "" |
100 | 100 |
reduced_redundancy = False |
... | ... |
@@ -11,8 +11,15 @@ import ExitCodes |
11 | 11 |
try: |
12 | 12 |
import xml.etree.ElementTree as ET |
13 | 13 |
except ImportError: |
14 |
+ # xml.etree.ElementTree was only added in python 2.5 |
|
14 | 15 |
import elementtree.ElementTree as ET |
15 | 16 |
|
17 |
+try: |
|
18 |
+ from xml.etree.ElementTree import ParseError as XmlParseError |
|
19 |
+except ImportError: |
|
20 |
+ # ParseError was only added in python2.7, before ET was raising ExpatError |
|
21 |
+ from xml.parsers.expat import ExpatError as XmlParseError |
|
22 |
+ |
|
16 | 23 |
class S3Exception(Exception): |
17 | 24 |
def __init__(self, message = ""): |
18 | 25 |
self.message = unicodise(message) |
... | ... |
@@ -49,7 +56,7 @@ class S3Error (S3Exception): |
49 | 49 |
if response.has_key("data") and response["data"]: |
50 | 50 |
try: |
51 | 51 |
tree = getTreeFromXml(response["data"]) |
52 |
- except ET.ParseError: |
|
52 |
+ except XmlParseError: |
|
53 | 53 |
debug("Not an XML response") |
54 | 54 |
else: |
55 | 55 |
self.info.update(self.parse_error_xml(tree)) |
... | ... |
@@ -53,7 +53,7 @@ class S3Uri(object): |
53 | 53 |
|
54 | 54 |
class S3UriS3(S3Uri): |
55 | 55 |
type = "s3" |
56 |
- _re = re.compile("^s3://([^/]+)/?(.*)", re.IGNORECASE) |
|
56 |
+ _re = re.compile("^s3://([^/]*)/?(.*)", re.IGNORECASE) |
|
57 | 57 |
def __init__(self, string): |
58 | 58 |
match = self._re.match(string) |
59 | 59 |
if not match: |
... | ... |
@@ -47,8 +47,8 @@ else: |
47 | 47 |
try: |
48 | 48 |
import xml.etree.ElementTree as ET |
49 | 49 |
except ImportError: |
50 |
+ # xml.etree.ElementTree was only added in python 2.5 |
|
50 | 51 |
import elementtree.ElementTree as ET |
51 |
-from xml.parsers.expat import ExpatError |
|
52 | 52 |
|
53 | 53 |
__all__ = [] |
54 | 54 |
def parseNodes(nodes): |
... | ... |
@@ -88,11 +88,8 @@ def getTreeFromXml(xml): |
88 | 88 |
if xmlns: |
89 | 89 |
tree.attrib['xmlns'] = xmlns |
90 | 90 |
return tree |
91 |
- except ExpatError, e: |
|
92 |
- error(e) |
|
93 |
- raise Exceptions.ParameterError("Bucket contains invalid filenames. Please run: s3cmd fixbucket s3://your-bucket/") |
|
94 | 91 |
except Exception, e: |
95 |
- error(e) |
|
92 |
+ error("Error parsing xml: %s", e) |
|
96 | 93 |
error(xml) |
97 | 94 |
raise |
98 | 95 |
|
... | ... |
@@ -124,10 +124,20 @@ def cmd_ls(args): |
124 | 124 |
if uri.type == "s3" and uri.has_bucket(): |
125 | 125 |
subcmd_bucket_list(s3, uri) |
126 | 126 |
return EX_OK |
127 |
- subcmd_buckets_list_all(s3) |
|
127 |
+ |
|
128 |
+ # If not a s3 type uri or no bucket was provided, list all the buckets |
|
129 |
+ subcmd_all_buckets_list(s3) |
|
128 | 130 |
return EX_OK |
129 | 131 |
|
130 |
-def cmd_buckets_list_all_all(args): |
|
132 |
+def subcmd_all_buckets_list(s3): |
|
133 |
+ |
|
134 |
+ response = s3.list_all_buckets() |
|
135 |
+ |
|
136 |
+ for bucket in response["list"]: |
|
137 |
+ output(u"%s s3://%s" % (formatDateTime(bucket["CreationDate"]), |
|
138 |
+ bucket["Name"])) |
|
139 |
+ |
|
140 |
+def cmd_all_buckets_list_all_content(args): |
|
131 | 141 |
s3 = S3(Config()) |
132 | 142 |
|
133 | 143 |
response = s3.list_all_buckets() |
... | ... |
@@ -137,14 +147,6 @@ def cmd_buckets_list_all_all(args): |
137 | 137 |
output(u"") |
138 | 138 |
return EX_OK |
139 | 139 |
|
140 |
-def subcmd_buckets_list_all(s3): |
|
141 |
- response = s3.list_all_buckets() |
|
142 |
- for bucket in response["list"]: |
|
143 |
- output(u"%s s3://%s" % ( |
|
144 |
- formatDateTime(bucket["CreationDate"]), |
|
145 |
- bucket["Name"], |
|
146 |
- )) |
|
147 |
- |
|
148 | 140 |
def subcmd_bucket_list(s3, uri): |
149 | 141 |
bucket = uri.bucket() |
150 | 142 |
prefix = uri.object() |
... | ... |
@@ -1969,7 +1971,7 @@ def get_commands_list(): |
1969 | 1969 |
{"cmd":"mb", "label":"Make bucket", "param":"s3://BUCKET", "func":cmd_bucket_create, "argc":1}, |
1970 | 1970 |
{"cmd":"rb", "label":"Remove bucket", "param":"s3://BUCKET", "func":cmd_bucket_delete, "argc":1}, |
1971 | 1971 |
{"cmd":"ls", "label":"List objects or buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_ls, "argc":0}, |
1972 |
- {"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_buckets_list_all_all, "argc":0}, |
|
1972 |
+ {"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_all_buckets_list_all_content, "argc":0}, |
|
1973 | 1973 |
{"cmd":"put", "label":"Put file into bucket", "param":"FILE [FILE...] s3://BUCKET[/PREFIX]", "func":cmd_object_put, "argc":2}, |
1974 | 1974 |
{"cmd":"get", "label":"Get file from bucket", "param":"s3://BUCKET/OBJECT LOCAL_FILE", "func":cmd_object_get, "argc":1}, |
1975 | 1975 |
{"cmd":"del", "label":"Delete file from bucket", "param":"s3://BUCKET/OBJECT", "func":cmd_object_del, "argc":1}, |
... | ... |
@@ -2121,9 +2123,8 @@ def main(): |
2121 | 2121 |
from os.path import expanduser |
2122 | 2122 |
config_file = os.path.join(expanduser("~"), ".s3cfg") |
2123 | 2123 |
|
2124 |
- preferred_encoding = locale.getpreferredencoding() or "UTF-8" |
|
2124 |
+ autodetected_encoding = locale.getpreferredencoding() or "UTF-8" |
|
2125 | 2125 |
|
2126 |
- optparser.set_defaults(encoding = preferred_encoding) |
|
2127 | 2126 |
optparser.set_defaults(config = config_file) |
2128 | 2127 |
|
2129 | 2128 |
optparser.add_option( "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool. Optionally use as '--configure s3://some-bucket' to test access to a specific bucket instead of attempting to list them all.") |
... | ... |
@@ -2191,7 +2192,7 @@ def main(): |
2191 | 2191 |
|
2192 | 2192 |
optparser.add_option( "--server-side-encryption", dest="server_side_encryption", action="store_true", help="Specifies that server-side encryption will be used when putting objects. [put, sync, cp, modify]") |
2193 | 2193 |
|
2194 |
- optparser.add_option( "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % preferred_encoding) |
|
2194 |
+ optparser.add_option( "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % autodetected_encoding) |
|
2195 | 2195 |
optparser.add_option( "--add-encoding-exts", dest="add_encoding_exts", metavar="EXTENSIONs", help="Add encoding to these comma delimited extensions i.e. (css,js,html) when uploading to S3 )") |
2196 | 2196 |
optparser.add_option( "--verbatim", dest="urlencoding_mode", action="store_const", const="verbatim", help="Use the S3 name as given on the command line. No pre-processing, encoding, etc. Use with caution!") |
2197 | 2197 |
|
... | ... |
@@ -2469,13 +2470,13 @@ def report_exception(e, msg=''): |
2469 | 2469 |
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! |
2470 | 2470 |
|
2471 | 2471 |
""" % msg) |
2472 |
- if type(e) == ImportError: |
|
2473 |
- s = u' '.join([(a) for a in sys.argv]) |
|
2474 |
- else: |
|
2472 |
+ tb = traceback.format_exc(sys.exc_info()) |
|
2473 |
+ try: |
|
2475 | 2474 |
s = u' '.join([unicodise(a) for a in sys.argv]) |
2475 |
+ except NameError: |
|
2476 |
+ s = u' '.join([(a) for a in sys.argv]) |
|
2476 | 2477 |
sys.stderr.write(u"Invoked as: %s\n" % s) |
2477 | 2478 |
|
2478 |
- tb = traceback.format_exc(sys.exc_info()) |
|
2479 | 2479 |
e_class = str(e.__class__) |
2480 | 2480 |
e_class = e_class[e_class.rfind(".")+1 : -2] |
2481 | 2481 |
sys.stderr.write(u"Problem: %s: %s\n" % (e_class, e)) |
... | ... |
@@ -2529,13 +2530,18 @@ if __name__ == '__main__': |
2529 | 2529 |
from S3.CloudFront import CloudFront |
2530 | 2530 |
from S3.FileLists import * |
2531 | 2531 |
from S3.MultiPart import MultiPartUpload |
2532 |
+ except Exception as e: |
|
2533 |
+ report_exception(e, "Error loading some components of s3cmd (Import Error)")) |
|
2534 |
+ # 1 = EX_GENERAL but be safe in that situation |
|
2535 |
+ sys.exit(1) |
|
2532 | 2536 |
|
2537 |
+ try: |
|
2533 | 2538 |
rc = main() |
2534 | 2539 |
sys.exit(rc) |
2535 | 2540 |
|
2536 | 2541 |
except ImportError, e: |
2537 | 2542 |
report_exception(e) |
2538 |
- sys.exit(1) |
|
2543 |
+ sys.exit(EX_GENERAL) |
|
2539 | 2544 |
|
2540 | 2545 |
except (ParameterError, InvalidFileError), e: |
2541 | 2546 |
error(u"Parameter problem: %s" % e) |