Browse code

Merge pull request #450 from fviard/batch_misc_fixes_423_424_448

Batch misc fixes 423 424 448

Matt Domsch authored on 2015/01/06 13:33:59
Showing 5 changed files
... ...
@@ -12,6 +12,7 @@ import sys
12 12
 import Progress
13 13
 from SortedDict import SortedDict
14 14
 import httplib
15
+import locale
15 16
 try:
16 17
     import json
17 18
 except ImportError, e:
... ...
@@ -94,7 +95,7 @@ class Config(object):
94 94
     # Dict mapping compiled REGEXPs back to their textual form
95 95
     debug_exclude = {}
96 96
     debug_include = {}
97
-    encoding = "utf-8"
97
+    encoding = locale.getpreferredencoding() or "UTF-8"
98 98
     urlencoding_mode = "normal"
99 99
     log_target_prefix = ""
100 100
     reduced_redundancy = False
... ...
@@ -11,8 +11,15 @@ import ExitCodes
11 11
 try:
12 12
     import xml.etree.ElementTree as ET
13 13
 except ImportError:
14
+    # xml.etree.ElementTree was only added in python 2.5
14 15
     import elementtree.ElementTree as ET
15 16
 
17
+try:
18
+    from xml.etree.ElementTree import ParseError as XmlParseError
19
+except ImportError:
20
+    # ParseError was only added in python2.7, before ET was raising ExpatError
21
+    from xml.parsers.expat import ExpatError as XmlParseError
22
+
16 23
 class S3Exception(Exception):
17 24
     def __init__(self, message = ""):
18 25
         self.message = unicodise(message)
... ...
@@ -49,7 +56,7 @@ class S3Error (S3Exception):
49 49
         if response.has_key("data") and response["data"]:
50 50
             try:
51 51
                 tree = getTreeFromXml(response["data"])
52
-            except ET.ParseError:
52
+            except XmlParseError:
53 53
                 debug("Not an XML response")
54 54
             else:
55 55
                 self.info.update(self.parse_error_xml(tree))
... ...
@@ -53,7 +53,7 @@ class S3Uri(object):
53 53
 
54 54
 class S3UriS3(S3Uri):
55 55
     type = "s3"
56
-    _re = re.compile("^s3://([^/]+)/?(.*)", re.IGNORECASE)
56
+    _re = re.compile("^s3://([^/]*)/?(.*)", re.IGNORECASE)
57 57
     def __init__(self, string):
58 58
         match = self._re.match(string)
59 59
         if not match:
... ...
@@ -47,8 +47,8 @@ else:
47 47
 try:
48 48
     import xml.etree.ElementTree as ET
49 49
 except ImportError:
50
+    # xml.etree.ElementTree was only added in python 2.5
50 51
     import elementtree.ElementTree as ET
51
-from xml.parsers.expat import ExpatError
52 52
 
53 53
 __all__ = []
54 54
 def parseNodes(nodes):
... ...
@@ -88,11 +88,8 @@ def getTreeFromXml(xml):
88 88
         if xmlns:
89 89
             tree.attrib['xmlns'] = xmlns
90 90
         return tree
91
-    except ExpatError, e:
92
-        error(e)
93
-        raise Exceptions.ParameterError("Bucket contains invalid filenames. Please run: s3cmd fixbucket s3://your-bucket/")
94 91
     except Exception, e:
95
-        error(e)
92
+        error("Error parsing xml: %s", e)
96 93
         error(xml)
97 94
         raise
98 95
 
... ...
@@ -124,10 +124,20 @@ def cmd_ls(args):
124 124
         if uri.type == "s3" and uri.has_bucket():
125 125
             subcmd_bucket_list(s3, uri)
126 126
             return EX_OK
127
-    subcmd_buckets_list_all(s3)
127
+
128
+    # If not a s3 type uri or no bucket was provided, list all the buckets
129
+    subcmd_all_buckets_list(s3)
128 130
     return EX_OK
129 131
 
130
-def cmd_buckets_list_all_all(args):
132
+def subcmd_all_buckets_list(s3):
133
+
134
+    response = s3.list_all_buckets()
135
+
136
+    for bucket in response["list"]:
137
+        output(u"%s  s3://%s" % (formatDateTime(bucket["CreationDate"]),
138
+                                 bucket["Name"]))
139
+
140
+def cmd_all_buckets_list_all_content(args):
131 141
     s3 = S3(Config())
132 142
 
133 143
     response = s3.list_all_buckets()
... ...
@@ -137,14 +147,6 @@ def cmd_buckets_list_all_all(args):
137 137
         output(u"")
138 138
     return EX_OK
139 139
 
140
-def subcmd_buckets_list_all(s3):
141
-    response = s3.list_all_buckets()
142
-    for bucket in response["list"]:
143
-        output(u"%s  s3://%s" % (
144
-            formatDateTime(bucket["CreationDate"]),
145
-            bucket["Name"],
146
-            ))
147
-
148 140
 def subcmd_bucket_list(s3, uri):
149 141
     bucket = uri.bucket()
150 142
     prefix = uri.object()
... ...
@@ -1969,7 +1971,7 @@ def get_commands_list():
1969 1969
     {"cmd":"mb", "label":"Make bucket", "param":"s3://BUCKET", "func":cmd_bucket_create, "argc":1},
1970 1970
     {"cmd":"rb", "label":"Remove bucket", "param":"s3://BUCKET", "func":cmd_bucket_delete, "argc":1},
1971 1971
     {"cmd":"ls", "label":"List objects or buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_ls, "argc":0},
1972
-    {"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_buckets_list_all_all, "argc":0},
1972
+    {"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_all_buckets_list_all_content, "argc":0},
1973 1973
     {"cmd":"put", "label":"Put file into bucket", "param":"FILE [FILE...] s3://BUCKET[/PREFIX]", "func":cmd_object_put, "argc":2},
1974 1974
     {"cmd":"get", "label":"Get file from bucket", "param":"s3://BUCKET/OBJECT LOCAL_FILE", "func":cmd_object_get, "argc":1},
1975 1975
     {"cmd":"del", "label":"Delete file from bucket", "param":"s3://BUCKET/OBJECT", "func":cmd_object_del, "argc":1},
... ...
@@ -2121,9 +2123,8 @@ def main():
2121 2121
         from os.path import expanduser
2122 2122
         config_file = os.path.join(expanduser("~"), ".s3cfg")
2123 2123
 
2124
-    preferred_encoding = locale.getpreferredencoding() or "UTF-8"
2124
+    autodetected_encoding = locale.getpreferredencoding() or "UTF-8"
2125 2125
 
2126
-    optparser.set_defaults(encoding = preferred_encoding)
2127 2126
     optparser.set_defaults(config = config_file)
2128 2127
 
2129 2128
     optparser.add_option(      "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool. Optionally use as '--configure s3://some-bucket' to test access to a specific bucket instead of attempting to list them all.")
... ...
@@ -2191,7 +2192,7 @@ def main():
2191 2191
 
2192 2192
     optparser.add_option(      "--server-side-encryption", dest="server_side_encryption", action="store_true", help="Specifies that server-side encryption will be used when putting objects. [put, sync, cp, modify]")
2193 2193
 
2194
-    optparser.add_option(      "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % preferred_encoding)
2194
+    optparser.add_option(      "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % autodetected_encoding)
2195 2195
     optparser.add_option(      "--add-encoding-exts", dest="add_encoding_exts", metavar="EXTENSIONs", help="Add encoding to these comma delimited extensions i.e. (css,js,html) when uploading to S3 )")
2196 2196
     optparser.add_option(      "--verbatim", dest="urlencoding_mode", action="store_const", const="verbatim", help="Use the S3 name as given on the command line. No pre-processing, encoding, etc. Use with caution!")
2197 2197
 
... ...
@@ -2469,13 +2470,13 @@ def report_exception(e, msg=''):
2469 2469
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2470 2470
 
2471 2471
 """ % msg)
2472
-	if type(e) == ImportError:
2473
-	    s = u' '.join([(a) for a in sys.argv])
2474
-	else:
2472
+        tb = traceback.format_exc(sys.exc_info())
2473
+        try:
2475 2474
             s = u' '.join([unicodise(a) for a in sys.argv])
2475
+        except NameError:
2476
+            s = u' '.join([(a) for a in sys.argv])
2476 2477
         sys.stderr.write(u"Invoked as: %s\n" % s)
2477 2478
 
2478
-        tb = traceback.format_exc(sys.exc_info())
2479 2479
         e_class = str(e.__class__)
2480 2480
         e_class = e_class[e_class.rfind(".")+1 : -2]
2481 2481
         sys.stderr.write(u"Problem: %s: %s\n" % (e_class, e))
... ...
@@ -2529,13 +2530,18 @@ if __name__ == '__main__':
2529 2529
         from S3.CloudFront import CloudFront
2530 2530
         from S3.FileLists import *
2531 2531
         from S3.MultiPart import MultiPartUpload
2532
+    except Exception as e:
2533
+        report_exception(e, "Error loading some components of s3cmd (Import Error)"))
2534
+        # 1 = EX_GENERAL but be safe in that situation
2535
+        sys.exit(1)
2532 2536
 
2537
+    try:
2533 2538
         rc = main()
2534 2539
         sys.exit(rc)
2535 2540
 
2536 2541
     except ImportError, e:
2537 2542
         report_exception(e)
2538
-        sys.exit(1)
2543
+        sys.exit(EX_GENERAL)
2539 2544
 
2540 2545
     except (ParameterError, InvalidFileError), e:
2541 2546
         error(u"Parameter problem: %s" % e)