Browse code

Add support for creating signed S3 URLs with the new signurl command

Amazon S3 supports URLs signed with an API key that permit time-limited
access to a normally private resource. Add a "signurl" command to s3cmd
that generates such URLs.

Usage: s3cmd signurl s3://bucket/object `date -d '1 year' +%s`

ie: s3cmd signurl url-to-sign expiry-in-epoch-seconds

This is a purely offline operation. Your API key and secret are not sent on
the wire. Your API key is included the generated URL, but your secret is of course
not.

No validation of the URL against S3 is performed, since this is an offline-only
operation. Use s3cmd ls or similar to test for valid objects if you need to.

The URL generated is http:// but you can simply change to https:// if you want.

For more information on signed URLs, see:

http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html

Craig Ringer authored on 2012/12/06 13:49:47
Showing 3 changed files
... ...
@@ -13,6 +13,7 @@ import rfc822
13 13
 import hmac
14 14
 import base64
15 15
 import errno
16
+import urllib
16 17
 
17 18
 from logging import debug, info, warning, error
18 19
 
... ...
@@ -319,12 +320,73 @@ def replace_nonprintables(string):
319 319
 __all__.append("replace_nonprintables")
320 320
 
321 321
 def sign_string(string_to_sign):
322
-    #debug("string_to_sign: %s" % string_to_sign)
322
+    """Sign a string with the secret key, returning base64 encoded results.
323
+    By default the configured secret key is used, but may be overridden as
324
+    an argument.
325
+
326
+    Useful for REST authentication. See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
327
+    """
323 328
     signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip()
324
-    #debug("signature: %s" % signature)
325 329
     return signature
326 330
 __all__.append("sign_string")
327 331
 
332
+def sign_url(url_to_sign, expiry):
333
+    """Sign a URL in s3://bucket/object form with the given expiry
334
+    time. The object will be accessible via the signed URL until the
335
+    AWS key and secret are revoked or the expiry time is reached, even
336
+    if the object is otherwise private.
337
+
338
+    See: http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
339
+    """
340
+    return sign_url_base(
341
+        bucket = url_to_sign.bucket(),
342
+        object = url_to_sign.object(),
343
+        expiry = expiry
344
+    )
345
+__all__.append("sign_url")
346
+
347
+def sign_url_base(**parms):
348
+    """Shared implementation of sign_url methods. Takes a hash of 'bucket', 'object' and 'expiry' as args."""
349
+    parms['expiry']=time_to_epoch(parms['expiry'])
350
+    parms['access_key']=Config.Config().access_key
351
+    debug("Expiry interpreted as epoch time %s", parms['expiry'])
352
+    signtext = 'GET\n\n\n%(expiry)d\n/%(bucket)s/%(object)s' % parms
353
+    debug("Signing plaintext: %r", signtext)
354
+    parms['sig'] = urllib.quote_plus(sign_string(signtext))
355
+    debug("Urlencoded signature: %s", parms['sig'])
356
+    return "http://%(bucket)s.s3.amazonaws.com/%(object)s?AWSAccessKeyId=%(access_key)s&Expires=%(expiry)d&Signature=%(sig)s" % parms
357
+
358
+def time_to_epoch(t):
359
+    """Convert time specified in a variety of forms into UNIX epoch time.
360
+    Accepts datetime.datetime, int, anything that has a strftime() method, and standard time 9-tuples
361
+    """
362
+    if isinstance(t, int):
363
+        # Already an int
364
+        return t
365
+    elif isinstance(t, tuple) or isinstance(t, time.struct_time):
366
+        # Assume it's a time 9-tuple
367
+        return int(time.mktime(t))
368
+    elif hasattr(t, 'timetuple'):
369
+        # Looks like a datetime object or compatible
370
+        return int(time.mktime(ex.timetuple()))
371
+    elif hasattr(t, 'strftime'):
372
+        # Looks like the object supports standard srftime()
373
+        return int(t.strftime('%s'))
374
+    elif isinstance(t, str) or isinstance(t, unicode):
375
+        # See if it's a string representation of an epoch
376
+        try:
377
+            return int(t)
378
+        except ValueError:
379
+            # Try to parse it as a timestamp string
380
+            try:
381
+                return time.strptime(t)
382
+            except ValueError as ex:
383
+                # Will fall through
384
+                debug("Failed to parse date with strptime: %s", ex)
385
+                pass
386
+    raise Exceptions.ParameterError('Unable to convert %r to an epoch time. Pass an epoch time. Try `date -d \'now + 1 year\' +%%s` (shell) or time.mktime (Python).' % t)
387
+
388
+
328 389
 def check_bucket_name(bucket, dns_strict = True):
329 390
     if dns_strict:
330 391
         invalid = re.search("([^a-z0-9\.-])", bucket)
... ...
@@ -23,6 +23,7 @@ import locale
23 23
 import subprocess
24 24
 import htmlentitydefs
25 25
 import socket
26
+import S3.Exceptions
26 27
 
27 28
 from copy import copy
28 29
 from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter
... ...
@@ -1078,6 +1079,15 @@ def cmd_sign(args):
1078 1078
     signature = Utils.sign_string(string_to_sign)
1079 1079
     output("Signature: %s" % signature)
1080 1080
 
1081
+def cmd_signurl(args):
1082
+	expiry = args.pop()
1083
+	url_to_sign = S3Uri(args.pop())
1084
+	if url_to_sign.type != 's3':
1085
+		raise ParameterError("Must be S3Uri. Got: %s" % url_to_sign)
1086
+	debug("url to sign: %r" % url_to_sign)
1087
+	signed_url = Utils.sign_url(url_to_sign, expiry)
1088
+	output(signed_url)
1089
+
1081 1090
 def cmd_fixbucket(args):
1082 1091
     def _unescape(text):
1083 1092
         ##
... ...
@@ -1394,6 +1404,7 @@ def get_commands_list():
1394 1394
     {"cmd":"setacl", "label":"Modify Access control list for Bucket or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1},
1395 1395
     {"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1},
1396 1396
     {"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1},
1397
+    {"cmd":"signurl", "label":"Sign an S3 URL to provide limited public access with expiry", "param":"s3://BUCKET/OBJECT expiry_epoch", "func":cmd_signurl, "argc":2},
1397 1398
     {"cmd":"fixbucket", "label":"Fix invalid file names in a bucket", "param":"s3://BUCKET[/PREFIX]", "func":cmd_fixbucket, "argc":1},
1398 1399
 
1399 1400
     ## Website commands
... ...
@@ -63,6 +63,23 @@ Enable/disable bucket access logging
63 63
 s3cmd \fBsign\fR \fISTRING-TO-SIGN\fR
64 64
 Sign arbitrary string using the secret key
65 65
 .TP
66
+s3cmd \fBsignurl\fR \fIs3://BUCKET[/OBJECT]\fR \fIexpiry-in-epoch-seconds\fR
67
+Sign an S3 URL with the secret key, producing a URL that allows access to
68
+the named object using the credentials used to sign the URL until the date of
69
+expiry specified in epoch-seconds has passed. This is most useful for publishing
70
+time- or distribution-limited URLs to otherwise-private S3 objects.
71
+.br
72
+This is a purely offline operation. Your API key and secret are not sent on
73
+the wire, though your public API key is included in the generated URL. Because
74
+it's offline, no validation is done to ensure that the bucket and object actually
75
+exist, or that this API key has permission to access them.
76
+.br
77
+The URL generated is http:// but you can simply change to https:// if you want.
78
+.br
79
+See
80
+.B http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
81
+for more information on signed URLs, and the examples section below.
82
+.TP
66 83
 s3cmd \fBfixbucket\fR \fIs3://BUCKET[/PREFIX]\fR
67 84
 Fix invalid file names in a bucket
68 85
 
... ...
@@ -421,6 +438,12 @@ about matching file names against exclude and include rules.
421 421
 For example to exclude all files with ".jpg" extension except those beginning with a number use:
422 422
 .PP
423 423
 	\-\-exclude '*.jpg' \-\-rinclude '[0-9].*\.jpg'
424
+.PP
425
+To produce a signed HTTP URL that allows access to the normally private s3 object
426
+s3://mybucket/someobj (which you must have permission to access) to anybody
427
+with the URL for one week from today, use:
428
+.PP
429
+        s3cmd signurl s3://mybucket/someobj `date -d 'today + 1 week' +%s` 
424 430
 .SH SEE ALSO
425 431
 For the most up to date list of options run 
426 432
 .B s3cmd \-\-help