Browse code

Add support for object expiration

hrchu authored on 2014/03/05 22:31:21
Showing 3 changed files
... ...
@@ -110,6 +110,9 @@ class Config(object):
110 110
     cache_file = ""
111 111
     add_headers = ""
112 112
     ignore_failed_copy = False
113
+    expiry_days = ""
114
+    expiry_date = ""
115
+    expiry_prefix = ""
113 116
 
114 117
     ## Creating a singleton
115 118
     def __new__(self, configfile = None):
... ...
@@ -14,6 +14,7 @@ import logging
14 14
 import mimetypes
15 15
 import re
16 16
 from xml.sax import saxutils
17
+import base64
17 18
 from logging import debug, info, warning, error
18 19
 from stat import ST_SIZE
19 20
 
... ...
@@ -371,6 +372,62 @@ class S3(object):
371 371
 
372 372
         return response
373 373
 
374
+    def expiration_info(self, uri, bucket_location = None):
375
+        headers = SortedDict(ignore_case = True)
376
+        bucket = uri.bucket()
377
+        body = ""
378
+
379
+        request = self.create_request("BUCKET_LIST", bucket = bucket, extra="?lifecycle")
380
+        try:
381
+            response = self.send_request(request, body)
382
+            response['prefix'] = getTextFromXml(response['data'], ".//Rule//Prefix")
383
+            response['date'] = getTextFromXml(response['data'], ".//Rule//Expiration//Date")
384
+            response['days'] = getTextFromXml(response['data'], ".//Rule//Expiration//Days")
385
+            return response
386
+        except S3Error, e:
387
+            if e.status == 404:
388
+                debug("Could not get /?lifecycle - lifecycle probably not configured for this bucket")
389
+                return None
390
+            raise
391
+
392
+    def expiration_set(self, uri, bucket_location = None):
393
+        if self.config.expiry_date and self.config.expiry_days:
394
+             raise ParameterError("Expect either --expiry-day or --expiry-date")
395
+        if not (self.config.expiry_date or self.config.expiry_days):
396
+             if self.config.expiry_prefix:
397
+                 raise ParameterError("Expect either --expiry-day or --expiry-date")
398
+             debug("del bucket lifecycle")
399
+             bucket = uri.bucket()
400
+             body = ""
401
+             request = self.create_request("BUCKET_DELETE", bucket = bucket, extra="?lifecycle")
402
+        else:
403
+             request, body = self._expiration_set(uri)
404
+        debug("About to send request '%s' with body '%s'" % (request, body))
405
+        response = self.send_request(request, body)
406
+        debug("Received response '%s'" % (response))
407
+        return response
408
+
409
+    def _expiration_set(self, uri):
410
+        debug("put bucket lifecycle")
411
+        body = '<LifecycleConfiguration>'
412
+        body += '  <Rule>'
413
+        body += ('    <Prefix>%s</Prefix>' % self.config.expiry_prefix)
414
+        body += ('    <Status>Enabled</Status>')
415
+        body += ('    <Expiration>')
416
+        if self.config.expiry_date:
417
+            body += ('    <Date>%s</Date>' % self.config.expiry_date)
418
+        elif self.config.expiry_days:
419
+            body += ('    <Days>%s</Days>' % self.config.expiry_days)
420
+        body += ('    </Expiration>')
421
+        body += '  </Rule>'
422
+        body += '</LifecycleConfiguration>'
423
+
424
+        headers = SortedDict(ignore_case = True)
425
+        headers['content-md5'] = compute_content_md5(body)
426
+        bucket = uri.bucket()
427
+        request =  self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers, extra="?lifecycle")
428
+        return (request, body)
429
+
374 430
     def add_encoding(self, filename, content_type):
375 431
         if content_type.find("charset=") != -1:
376 432
            return False
... ...
@@ -1095,4 +1152,11 @@ def parse_attrs_header(attrs_header):
1095 1095
         key, val = attr.split(":")
1096 1096
         attrs[key] = val
1097 1097
     return attrs
1098
+
1099
+def compute_content_md5(body):
1100
+    m = md5(body)
1101
+    base64md5 = base64.encodestring(m.digest())
1102
+    if base64md5[-1] == '\n':
1103
+        base64md5 = base64md5[0:-1]
1104
+    return base64md5
1098 1105
 # vim:et:ts=4:sts=4:ai
... ...
@@ -242,6 +242,25 @@ def cmd_website_delete(args):
242 242
             else:
243 243
                 raise
244 244
 
245
+def cmd_expiration_set(args):
246
+    s3 = S3(Config())
247
+    for arg in args:
248
+        uri = S3Uri(arg)
249
+        if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
250
+            raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
251
+        try:
252
+            response = s3.expiration_set(uri, cfg.bucket_location)
253
+            if response["status"] is 200:
254
+                output(u"Bucket '%s': expiration configuration is set." % (uri.uri()))
255
+            elif response["status"] is 204:
256
+                output(u"Bucket '%s': expiration configuration is deleted." % (uri.uri()))
257
+        except S3Error, e:
258
+            if S3.codes.has_key(e.info["Code"]):
259
+                error(S3.codes[e.info["Code"]] % uri.bucket())
260
+                return
261
+            else:
262
+                raise
263
+
245 264
 def cmd_bucket_delete(args):
246 265
     def _bucket_delete_one(uri):
247 266
         try:
... ...
@@ -724,9 +743,23 @@ def cmd_info(args):
724 724
                 info = s3.bucket_info(uri)
725 725
                 output(u"%s (bucket):" % uri.uri())
726 726
                 output(u"   Location:  %s" % info['bucket-location'])
727
+                try:
728
+                    expiration = s3.expiration_info(uri, cfg.bucket_location)
729
+                    expiration_desc = "Expiration Rule: "
730
+                    if expiration['prefix'] == "":
731
+                        expiration_desc += "all objects in this bucket "
732
+                    else:
733
+                        expiration_desc += "objects with key prefix '" + expiration['prefix'] + "' "
734
+                    expiration_desc += "will expire in '"
735
+                    if expiration['days']:
736
+                        expiration_desc += expiration['days'] + "' day(s) after creation"
737
+                    elif expiration['date']:
738
+                        expiration_desc += expiration['date'] + "' "
739
+                    output(u"   %s" % expiration_desc)
740
+                except:
741
+                    output(u"   Expiration Rule: none")
727 742
             acl = s3.get_acl(uri)
728 743
             acl_grant_list = acl.getGrantList()
729
-
730 744
             try:
731 745
                 policy = s3.get_policy(uri)
732 746
                 output(u"   policy: %s" % policy)
... ...
@@ -1860,6 +1893,9 @@ def get_commands_list():
1860 1860
     {"cmd":"ws-delete", "label":"Delete Website", "param":"s3://BUCKET", "func":cmd_website_delete, "argc":1},
1861 1861
     {"cmd":"ws-info", "label":"Info about Website", "param":"s3://BUCKET", "func":cmd_website_info, "argc":1},
1862 1862
 
1863
+    ## Lifecycle commands
1864
+    {"cmd":"expire", "label":"Set or delete expiration rule for the bucket", "param":"s3://BUCKET", "func":cmd_expiration_set, "argc":1},
1865
+
1863 1866
     ## CloudFront commands
1864 1867
     {"cmd":"cflist", "label":"List CloudFront distribution points", "param":"", "func":CfCmd.info, "argc":0},
1865 1868
     {"cmd":"cfinfo", "label":"Display CloudFront distribution point parameters", "param":"[cf://DIST_ID]", "func":CfCmd.info, "argc":0},
... ...
@@ -2052,6 +2088,10 @@ def main():
2052 2052
     optparser.add_option(      "--ws-index", dest="website_index", action="store", help="Name of index-document (only for [ws-create] command)")
2053 2053
     optparser.add_option(      "--ws-error", dest="website_error", action="store", help="Name of error-document (only for [ws-create] command)")
2054 2054
 
2055
+    optparser.add_option(      "--expiry-date", dest="expiry_date", action="store", help="Indicates when the expiration rule take effect. (only for [expire] comman)")
2056
+    optparser.add_option(      "--expiry-days", dest="expiry_days", action="store", help="Indicates the number of days after object creation the expiration rule take effect. (only for [expire] command)")
2057
+    optparser.add_option(      "--expiry-prefix", dest="expiry_prefix", action="store", help="Identifying one or more objects with the prefix to which the expiration rule applies. (only for [expire] command)")
2058
+
2055 2059
     optparser.add_option(      "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default on TTY).")
2056 2060
     optparser.add_option(      "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter (default on non-TTY).")
2057 2061
     optparser.add_option(      "--enable", dest="enable", action="store_true", help="Enable given CloudFront distribution (only for [cfmodify] command)")