git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3py/trunk@42 830e0280-6d2a-0410-9c65-932aecc39d9d
Michal Ludvig authored on 2007/01/11 20:13:01... | ... |
@@ -31,6 +31,7 @@ class AwsConfig: |
31 | 31 |
recv_chunk = 4096 |
32 | 32 |
human_readable_sizes = False |
33 | 33 |
force = False |
34 |
+ show_uri = False |
|
34 | 35 |
|
35 | 36 |
def __init__(self, configfile = None): |
36 | 37 |
if configfile: |
... | ... |
@@ -52,14 +53,24 @@ class S3Error (Exception): |
52 | 52 |
def __init__(self, response): |
53 | 53 |
self.status = response["status"] |
54 | 54 |
self.reason = response["reason"] |
55 |
- tree = ET.fromstring(response["data"]) |
|
56 |
- for child in tree.getchildren(): |
|
57 |
- if child.text != "": |
|
58 |
- debug(child.tag + ": " + repr(child.text)) |
|
59 |
- self.__setattr__(child.tag, child.text) |
|
55 |
+ debug("S3Error: %s (%s)" % (self.status, self.reason)) |
|
56 |
+ if response.has_key("headers"): |
|
57 |
+ for header in response["headers"]: |
|
58 |
+ debug("HttpHeader: %s: %s" % (header, response["headers"][header])) |
|
59 |
+ if response.has_key("data"): |
|
60 |
+ tree = ET.fromstring(response["data"]) |
|
61 |
+ for child in tree.getchildren(): |
|
62 |
+ if child.text != "": |
|
63 |
+ debug("ErrorXML: " + child.tag + ": " + repr(child.text)) |
|
64 |
+ self.__setattr__(child.tag, child.text) |
|
60 | 65 |
|
61 | 66 |
def __str__(self): |
62 |
- return "%d (%s): %s" % (self.status, self.reason, self.Code) |
|
67 |
+ retval = "%d (%s)" % (self.status, self.reason) |
|
68 |
+ try: |
|
69 |
+ retval += (": %s" % self.Code) |
|
70 |
+ except AttributeError: |
|
71 |
+ pass |
|
72 |
+ return retval |
|
63 | 73 |
|
64 | 74 |
class ParameterError(Exception): |
65 | 75 |
pass |
... | ... |
@@ -131,7 +142,7 @@ class S3: |
131 | 131 |
file = open(filename, "r") |
132 | 132 |
size = os.stat(filename)[ST_SIZE] |
133 | 133 |
except IOError, e: |
134 |
- raise ParameterProblem("%s: %s" % (filename, e.strerror)) |
|
134 |
+ raise ParameterError("%s: %s" % (filename, e.strerror)) |
|
135 | 135 |
headers = SortedDict() |
136 | 136 |
headers["content-length"] = size |
137 | 137 |
request = self.create_request("OBJECT_PUT", bucket = bucket, object = object, headers = headers) |
... | ... |
@@ -143,12 +154,17 @@ class S3: |
143 | 143 |
try: |
144 | 144 |
file = open(filename, "w") |
145 | 145 |
except IOError, e: |
146 |
- raise ParameterProblem("%s: %s" % (filename, e.strerror)) |
|
146 |
+ raise ParameterError("%s: %s" % (filename, e.strerror)) |
|
147 | 147 |
request = self.create_request("OBJECT_GET", bucket = bucket, object = object) |
148 | 148 |
response = self.recv_file(request, file) |
149 | 149 |
response["size"] = int(response["headers"]["content-length"]) |
150 | 150 |
return response |
151 | 151 |
|
152 |
+ def object_delete(self, bucket, object): |
|
153 |
+ request = self.create_request("OBJECT_DELETE", bucket = bucket, object = object) |
|
154 |
+ response = self.send_request(request) |
|
155 |
+ return response |
|
156 |
+ |
|
152 | 157 |
def create_request(self, operation, bucket = None, object = None, headers = None): |
153 | 158 |
resource = "/" |
154 | 159 |
if bucket: |
... | ... |
@@ -181,6 +197,7 @@ class S3: |
181 | 181 |
http_response = conn.getresponse() |
182 | 182 |
response["status"] = http_response.status |
183 | 183 |
response["reason"] = http_response.reason |
184 |
+ response["headers"] = convertTupleListToDict(http_response.getheaders()) |
|
184 | 185 |
response["data"] = http_response.read() |
185 | 186 |
conn.close() |
186 | 187 |
if response["status"] < 200 or response["status"] > 299: |
... | ... |
@@ -211,6 +228,7 @@ class S3: |
211 | 211 |
http_response = conn.getresponse() |
212 | 212 |
response["status"] = http_response.status |
213 | 213 |
response["reason"] = http_response.reason |
214 |
+ response["headers"] = convertTupleListToDict(http_response.getheaders()) |
|
214 | 215 |
response["data"] = http_response.read() |
215 | 216 |
conn.close() |
216 | 217 |
if response["status"] < 200 or response["status"] > 299: |
... | ... |
@@ -230,10 +248,10 @@ class S3: |
230 | 230 |
http_response = conn.getresponse() |
231 | 231 |
response["status"] = http_response.status |
232 | 232 |
response["reason"] = http_response.reason |
233 |
+ response["headers"] = convertTupleListToDict(http_response.getheaders()) |
|
233 | 234 |
if response["status"] < 200 or response["status"] > 299: |
234 | 235 |
raise S3Error(response) |
235 | 236 |
|
236 |
- response["headers"] = convertTupleListToDict(http_response.getheaders()) |
|
237 | 237 |
md5=hashlib.new("md5") |
238 | 238 |
size_left = size_total = int(response["headers"]["content-length"]) |
239 | 239 |
while (size_left > 0): |
... | ... |
@@ -278,8 +296,17 @@ class S3: |
278 | 278 |
raise ParameterError("Bucket name '%s' is too long (max 255 characters)" % bucket) |
279 | 279 |
return True |
280 | 280 |
|
281 |
+ def compose_uri(self, bucket, object = None): |
|
282 |
+ if AwsConfig.show_uri: |
|
283 |
+ uri = "s3://" + bucket |
|
284 |
+ if object: |
|
285 |
+ uri += "/"+object |
|
286 |
+ return uri |
|
287 |
+ else: |
|
288 |
+ return object and object or bucket |
|
289 |
+ |
|
281 | 290 |
def parse_s3_uri(self, uri): |
282 |
- match = re.compile("^s3://([^/]*)/*(.*)").match(uri) |
|
291 |
+ match = re.compile("^s3://([^/]*)/?(.*)").match(uri) |
|
283 | 292 |
if match: |
284 | 293 |
return (True,) + match.groups() |
285 | 294 |
else: |
... | ... |
@@ -292,14 +319,10 @@ def cmd_buckets_list_all(args): |
292 | 292 |
s3 = S3(AwsConfig()) |
293 | 293 |
response = s3.list_all_buckets() |
294 | 294 |
|
295 |
- maxlen = 0 |
|
296 |
- for bucket in response["list"]: |
|
297 |
- if len(bucket["Name"]) > maxlen: |
|
298 |
- maxlen = len(bucket["Name"]) |
|
299 | 295 |
for bucket in response["list"]: |
300 | 296 |
output("%s %s" % ( |
301 | 297 |
formatDateTime(bucket["CreationDate"]), |
302 |
- bucket["Name"].ljust(maxlen), |
|
298 |
+ s3.compose_uri(bucket["Name"]), |
|
303 | 299 |
)) |
304 | 300 |
|
305 | 301 |
def cmd_buckets_list_all_all(args): |
... | ... |
@@ -312,9 +335,11 @@ def cmd_buckets_list_all_all(args): |
312 | 312 |
|
313 | 313 |
|
314 | 314 |
def cmd_bucket_list(args): |
315 |
- bucket = args[0] |
|
316 |
- output("Bucket '%s':" % bucket) |
|
317 | 315 |
s3 = S3(AwsConfig()) |
316 |
+ isuri, bucket, object = s3.parse_s3_uri(args[0]) |
|
317 |
+ if not isuri: |
|
318 |
+ bucket = args[0] |
|
319 |
+ output("Bucket '%s':" % bucket) |
|
318 | 320 |
try: |
319 | 321 |
response = s3.bucket_list(bucket) |
320 | 322 |
except S3Error, e: |
... | ... |
@@ -323,21 +348,19 @@ def cmd_bucket_list(args): |
323 | 323 |
return |
324 | 324 |
else: |
325 | 325 |
raise |
326 |
- maxlen = 0 |
|
327 |
- for object in response["list"]: |
|
328 |
- if len(object["Key"]) > maxlen: |
|
329 |
- maxlen = len(object["Key"]) |
|
330 | 326 |
for object in response["list"]: |
331 | 327 |
size, size_coeff = formatSize(object["Size"], AwsConfig.human_readable_sizes) |
332 | 328 |
output("%s %s%s %s" % ( |
333 | 329 |
formatDateTime(object["LastModified"]), |
334 | 330 |
str(size).rjust(8), size_coeff.ljust(1), |
335 |
- object["Key"].ljust(maxlen), |
|
331 |
+ s3.compose_uri(bucket, object["Key"]), |
|
336 | 332 |
)) |
337 | 333 |
|
338 | 334 |
def cmd_bucket_create(args): |
339 |
- bucket = args[0] |
|
340 | 335 |
s3 = S3(AwsConfig()) |
336 |
+ isuri, bucket, object = s3.parse_s3_uri(args[0]) |
|
337 |
+ if not isuri: |
|
338 |
+ bucket = args[0] |
|
341 | 339 |
try: |
342 | 340 |
response = s3.bucket_create(bucket) |
343 | 341 |
except S3Error, e: |
... | ... |
@@ -349,8 +372,10 @@ def cmd_bucket_create(args): |
349 | 349 |
output("Bucket '%s' created" % bucket) |
350 | 350 |
|
351 | 351 |
def cmd_bucket_delete(args): |
352 |
- bucket = args[0] |
|
353 | 352 |
s3 = S3(AwsConfig()) |
353 |
+ isuri, bucket, object = s3.parse_s3_uri(args[0]) |
|
354 |
+ if not isuri: |
|
355 |
+ bucket = args[0] |
|
354 | 356 |
try: |
355 | 357 |
response = s3.bucket_delete(bucket) |
356 | 358 |
except S3Error, e: |
... | ... |
@@ -387,21 +412,32 @@ def cmd_object_put(args): |
387 | 387 |
else: |
388 | 388 |
object_final = object |
389 | 389 |
response = s3.object_put(file, bucket, object_final) |
390 |
- output("File '%s' stored as s3://%s/%s (%d bytes)" % |
|
391 |
- (file, bucket, object_final, response["size"])) |
|
390 |
+ output("File '%s' stored as %s (%d bytes)" % |
|
391 |
+ (file, s3.compose_uri(bucket, object_final), response["size"])) |
|
392 | 392 |
|
393 | 393 |
def cmd_object_get(args): |
394 |
- bucket = args.pop(0) |
|
395 |
- object = args.pop(0) |
|
396 |
- destination = args.pop(0) |
|
394 |
+ s3 = S3(AwsConfig()) |
|
395 |
+ s3uri = args.pop(0) |
|
396 |
+ isuri, bucket, object = s3.parse_s3_uri(s3uri) |
|
397 |
+ if not isuri or not bucket or not object: |
|
398 |
+ raise ParameterError("Expecting S3 object URI instead of '%s'" % s3uri) |
|
399 |
+ destination = len(args) > 0 and args.pop(0) or object |
|
397 | 400 |
if os.path.isdir(destination): |
398 |
- destination.append("/" + object) |
|
401 |
+ destination += ("/" + object) |
|
399 | 402 |
if not AwsConfig.force and os.path.exists(destination): |
400 | 403 |
raise ParameterError("File %s already exists. Use --force to overwrite it" % destination) |
401 |
- s3 = S3(AwsConfig()) |
|
402 | 404 |
response = s3.object_get(destination, bucket, object) |
403 |
- output("Object s3://%s/%s saved as '%s' (%d bytes)" % |
|
404 |
- (bucket, object, destination, response["size"])) |
|
405 |
+ output("Object %s saved as '%s' (%d bytes)" % |
|
406 |
+ (s3uri, destination, response["size"])) |
|
407 |
+ |
|
408 |
+def cmd_object_del(args): |
|
409 |
+ s3 = S3(AwsConfig()) |
|
410 |
+ s3uri = args.pop(0) |
|
411 |
+ isuri, bucket, object = s3.parse_s3_uri(s3uri) |
|
412 |
+ if not isuri or not bucket or not object: |
|
413 |
+ raise ParameterError("Expecting S3 object URI instead of '%s'" % s3uri) |
|
414 |
+ response = s3.object_delete(bucket, object) |
|
415 |
+ output("Object %s deleted" % s3uri) |
|
405 | 416 |
|
406 | 417 |
commands = { |
407 | 418 |
"lb" : ("List all buckets", cmd_buckets_list_all, 0), |
... | ... |
@@ -411,9 +447,9 @@ commands = { |
411 | 411 |
"db" : ("Remove bucket", cmd_bucket_delete, 1), |
412 | 412 |
"ls" : ("List objects in bucket", cmd_bucket_list, 1), |
413 | 413 |
"la" : ("List all object in all buckets", cmd_buckets_list_all_all, 0), |
414 |
- "put": ("Put file(s) into a bucket", cmd_object_put, 2), |
|
415 |
- "get": ("Get file(s) from a bucket", cmd_object_get, 1), |
|
416 |
-# "del": ("Delete file(s) from a bucket", cmd_object_del, 1), |
|
414 |
+ "put": ("Put file into bucket", cmd_object_put, 2), |
|
415 |
+ "get": ("Get file from bucket", cmd_object_get, 1), |
|
416 |
+ "del": ("Delete file from bucket", cmd_object_del, 1), |
|
417 | 417 |
} |
418 | 418 |
|
419 | 419 |
if __name__ == '__main__': |
... | ... |
@@ -432,6 +468,9 @@ if __name__ == '__main__': |
432 | 432 |
optparser.add_option("-H", "--human-readable", dest="human_readable", action="store_true", help="Print sizes in human readable form") |
433 | 433 |
optparser.set_defaults(force = False) |
434 | 434 |
optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations") |
435 |
+ optparser.set_defaults(show_uri = False) |
|
436 |
+ optparser.add_option("-u", "--show-uri", dest="show_uri", action="store_true", help="Show complete S3 URI in listings") |
|
437 |
+ |
|
435 | 438 |
(options, args) = optparser.parse_args() |
436 | 439 |
|
437 | 440 |
## Some mucking with logging levels to enable |
... | ... |
@@ -450,6 +489,7 @@ if __name__ == '__main__': |
450 | 450 |
## Update AwsConfig with other parameters |
451 | 451 |
AwsConfig.human_readable_sizes = options.human_readable |
452 | 452 |
AwsConfig.force = options.force |
453 |
+ AwsConfig.show_uri = options.show_uri |
|
453 | 454 |
|
454 | 455 |
if len(args) < 1: |
455 | 456 |
error("Missing command. Please run with --help for more information.") |