@@ -322,7 +322,7 @@ def fetch_local_list(args, is_src = False, recursive = None):

     return local_list, single_file

-def fetch_remote_list(args, require_attribs = False, recursive = None):

+def fetch_remote_list(args, require_attribs = False, recursive = None, batch_mode = False, uri_params = {}):

     def _get_remote_attribs(uri, remote_item):

         response = S3(cfg).object_info(uri)

         remote_item.update({

@@ -337,7 +337,7 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):

         except KeyError:

             pass

-    def _get_filelist_remote(remote_uri, recursive = True):

+    def _get_filelist_remote(remote_uri, recursive = True, batch_mode = False):

         ## If remote_uri ends with '/' then all remote files will have

         ## the remote_uri prefix removed in the relative path.

         ## If, on the other hand, the remote_uri ends with something else

@@ -357,7 +357,8 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):

         empty_fname_re = re.compile(r'\A\s*\Z')

         s3 = S3(Config())

-        response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = recursive)

+        response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(),

+                                  recursive = recursive, batch_mode = batch_mode, uri_params = uri_params)

         rem_base_original = rem_base = remote_uri.object()

         remote_uri_original = remote_uri

@@ -417,7 +418,7 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):

     if recursive:

         for uri in remote_uris:

-            objectlist = _get_filelist_remote(uri)

+            objectlist = _get_filelist_remote(uri, batch_mode = batch_mode)

             for key in objectlist:

                 remote_list[key] = objectlist[key]

                 remote_list.record_md5(key, objectlist.get_md5(key))

@@ -7,10 +7,12 @@ import sys

 import os, os.path

 import time

 import errno

+import base64

 import httplib

 import logging

 import mimetypes

 import re

+from xml.sax import saxutils

 from logging import debug, info, warning, error

 from stat import ST_SIZE

@@ -161,6 +163,7 @@ class S3(object):

         SERVICE = 0x0100,

         BUCKET = 0x0200,

         OBJECT = 0x0400,

+        BATCH = 0x0800,

         MASK = 0x0700,

     )

@@ -175,6 +178,7 @@ class S3(object):

         OBJECT_HEAD = targets["OBJECT"] | http_methods["HEAD"],

         OBJECT_DELETE = targets["OBJECT"] | http_methods["DELETE"],

         OBJECT_POST = targets["OBJECT"] | http_methods["POST"],

+        BATCH_DELETE = targets["BATCH"] | http_methods["POST"],

     )

     codes = {

@@ -223,7 +227,7 @@ class S3(object):

         response["list"] = getListFromXml(response["data"], "Bucket")

         return response

-    def bucket_list(self, bucket, prefix = None, recursive = None):

+    def bucket_list(self, bucket, prefix = None, recursive = None, batch_mode = False, uri_params = {}):

         def _list_truncated(data):

             ## <IsTruncated> can either be "true" or "false" or be missing completely

             is_truncated = getTextFromXml(data, ".//IsTruncated") or "false"

@@ -235,7 +239,6 @@ class S3(object):

         def _get_common_prefixes(data):

             return getListFromXml(data, "CommonPrefixes")

-        uri_params = {}

         truncated = True

         list = []

         prefixes = []

@@ -244,7 +247,7 @@ class S3(object):

             response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params)

             current_list = _get_contents(response["data"])

             current_prefixes = _get_common_prefixes(response["data"])

-            truncated = _list_truncated(response["data"])

+            truncated = _list_truncated(response["data"]) and not batch_mode

             if truncated:

                 if current_list:

                     uri_params['marker'] = self.urlencode_string(current_list[-1]["Key"])

@@ -484,13 +487,42 @@ class S3(object):

         response = self.recv_file(request, stream, labels, start_position)

         return response

+    def object_batch_delete(self, remote_list):

+        def compose_batch_del_xml(bucket, key_list):

+            body = u"<?xml version=\"1.0\" encoding=\"UTF-8\"?><Delete>"

+            for key in key_list:

+                uri = S3Uri(key)

+                if uri.type != "s3":

+                    raise ValueError("Excpected URI type 's3', got '%s'" % uri.type)

+                if not uri.has_object():

+                    raise ValueError("URI '%s' has no object" % key)

+                if uri.bucket() != bucket:

+                    raise ValueError("The batch should contain keys from the same bucket")

+                object = saxutils.escape(uri.object())

+                body += u"<Object><Key>%s</Key></Object>" % object

+            body += u"</Delete>"

+            body = body.encode('utf-8')

+            return body

+

+        batch = [remote_list[item]['object_uri_str'] for item in remote_list]

+        if len(batch) == 0:

+            raise ValueError("Key list is empty")

+        bucket = S3Uri(batch[0]).bucket()

+        request_body = compose_batch_del_xml(bucket, batch)

+        md5_hash = md5()

+        md5_hash.update(request_body)

+        headers = {'content-md5': base64.b64encode(md5_hash.digest())}

+        request = self.create_request("BATCH_DELETE", bucket = bucket, extra = '?delete', headers = headers)

+        response = self.send_request(request, request_body)

+        return response

+

     def object_delete(self, uri):

         if uri.type != "s3":

             raise ValueError("Expected URI type 's3', got '%s'" % uri.type)

         request = self.create_request("OBJECT_DELETE", uri = uri)

         response = self.send_request(request)

         return response

-    

+

     def object_restore(self, uri):

         if uri.type != "s3":

             raise ValueError("Expected URI type 's3', got '%s'" % uri.type)

@@ -245,11 +245,15 @@ def cmd_bucket_delete(args):

     def _bucket_delete_one(uri):

         try:

             response = s3.bucket_delete(uri.bucket())

+            output(u"Bucket '%s' removed" % uri.uri())

         except S3Error, e:

             if e.info['Code'] == 'BucketNotEmpty' and (cfg.force or cfg.recursive):

                 warning(u"Bucket is not empty. Removing all the objects from it first. This may take some time...")

-                subcmd_object_del_uri(uri.uri(), recursive = True)

-                return _bucket_delete_one(uri)

+                success = subcmd_batch_del(uri.uri())

+                if success:

+                    return _bucket_delete_one(uri)

+                else:

+                    output(u"Bucket was not removed")

             elif S3.codes.has_key(e.info["Code"]):

                 error(S3.codes[e.info["Code"]] % uri.bucket())

                 return

@@ -262,7 +266,6 @@ def cmd_bucket_delete(args):

         if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

             raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

         _bucket_delete_one(uri)

-        output(u"Bucket '%s' removed" % uri.uri())

 def cmd_object_put(args):

     cfg = Config()

@@ -508,6 +511,47 @@ def cmd_object_del(args):

                 raise ParameterError("File name required, not only the bucket name. Alternatively use --recursive")

         subcmd_object_del_uri(uri_str)

+def subcmd_batch_del(uri_str = None, bucket = None, remote_list = None):

+    s3 = S3(cfg)

+

+    if len([item for item in [uri_str, bucket, remote_list] if item]) != 1:

+        raise ValueError("One and only one of 'uri_str', 'bucket', 'remote_list' can be specified.")

+

+    batch_mode = False

+    if bucket:

+        uri_str = "s3://%s" % bucket

+    if not remote_list:

+        batch_mode = True

+        remote_list = fetch_remote_list(uri_str, require_attribs = False, batch_mode = True)

+    if len(remote_list) == 0:

+        warning(u"Remote list is empty.")

+        return False

+

+    if cfg.max_delete > 0 and len(remote_list) > cfg.max_delete:

+        warning(u"delete: maximum requested number of deletes would be exceeded, none performed.")

+        return False

+

+    while True:

+        if cfg.dry_run:

+            output('\n'.join((u"File %s deleted" % remote_list[p]['object_uri_str']) for p in remote_list))

+        else:

+            response = s3.object_batch_delete(remote_list)

+            output('\n'.join((u"File %s deleted" % remote_list[p]['object_uri_str']) for p in remote_list))

+

+        keys_count = len(remote_list) if remote_list else 0

+        if keys_count < 1000 or not batch_mode:

+            break

+        last_key = S3Uri(remote_list[remote_list.keys()[-1]]['object_uri_str']) if remote_list else None

+        if not last_key or not last_key.has_object:

+            break

+        marker = last_key.object()

+        remote_list = fetch_remote_list(uri_str, require_attribs = False, batch_mode = True,

+                                        uri_params = {"marker": marker})

+    if cfg.dry_run:

+        warning(u"Exiting now because of --dry-run")

+        return False

+    return True

+

 def subcmd_object_del_uri(uri_str, recursive = None):

     s3 = S3(cfg)

@@ -537,10 +581,10 @@ def subcmd_object_del_uri(uri_str, recursive = None):

         item = remote_list[key]

         response = s3.object_delete(S3Uri(item['object_uri_str']))

         output(u"File %s deleted" % item['object_uri_str'])

-        

+

 def cmd_object_restore(args):

     s3 = S3(cfg)

-    

+

     if cfg.restore_days < 1:

         raise ParameterError("You must restore a file for 1 or more days")

@@ -562,14 +606,14 @@ def cmd_object_restore(args):

     for key in remote_list:

         item = remote_list[key]

-        

+

         uri = S3Uri(item['object_uri_str'])

         if not item['object_uri_str'].endswith("/"):

             response = s3.object_restore(S3Uri(item['object_uri_str']))

             output(u"File %s restoration started" % item['object_uri_str'])

         else:

             debug(u"Skipping directory since only files may be restored")

-        

+

 def subcmd_cp_mv(args, process_fce, action_str, message):

     if len(args) < 2:

@@ -697,20 +741,6 @@ def filedicts_to_keys(*args):

     return keys

 def cmd_sync_remote2remote(args):

-    def _do_deletes(s3, dst_list):

-        if cfg.max_delete > 0 and len(dst_list) > cfg.max_delete:

-            warning(u"delete: maximum requested number of deletes would be exceeded, none performed.")

-            return

-        # Delete items in destination that are not in source

-        if cfg.dry_run:

-            for key in dst_list:

-                output(u"delete: %s" % dst_list[key]['object_uri_str'])

-        else:

-            for key in dst_list:

-                uri = S3Uri(dst_list[key]['object_uri_str'])

-                s3.object_delete(uri)

-                output(u"deleted: '%s'" % uri)

-

     s3 = S3(Config())

     # Normalise s3://uri (e.g. assert trailing slash)

@@ -765,7 +795,7 @@ def cmd_sync_remote2remote(args):

     # Delete items in destination that are not in source

     if cfg.delete_removed and not cfg.delete_after:

-        _do_deletes(s3, dst_list)

+        subcmd_batch_del(remote_list = dst_list)

     def _upload(src_list, seq, src_count):

         file_list = src_list.keys()

@@ -809,7 +839,7 @@ def cmd_sync_remote2remote(args):

     # Delete items in destination that are not in source

     if cfg.delete_removed and cfg.delete_after:

-        _do_deletes(s3, dst_list)

+        subcmd_batch_del(remote_list = dst_list)

 def cmd_sync_remote2local(args):

     def _do_deletes(local_list):

@@ -1115,16 +1145,6 @@ def _build_attr_header(local_list, src):

 def cmd_sync_local2remote(args):

-

-    def _do_deletes(s3, remote_list):

-        if cfg.max_delete > 0 and len(remote_list) > cfg.max_delete:

-            warning(u"delete: maximum requested number of deletes would be exceeded, none performed.")

-            return

-        for key in remote_list:

-            uri = S3Uri(remote_list[key]['object_uri_str'])

-            s3.object_delete(uri)

-            output(u"deleted: '%s'" % uri)

-

     def _single_process(local_list):

         for dest in destinations:

             ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)

@@ -1260,7 +1280,7 @@ def cmd_sync_local2remote(args):

             cfg.delete_removed = False

         if cfg.delete_removed and not cfg.delete_after:

-            _do_deletes(s3, remote_list)

+            subcmd_batch_del(remote_list = remote_list)

         total_size = 0

         total_elapsed = 0.0

@@ -1276,7 +1296,7 @@ def cmd_sync_local2remote(args):

         n, total_size = _upload(failed_copy_files, n, failed_copy_count, total_size)

         if cfg.delete_removed and cfg.delete_after:

-            _do_deletes(s3, remote_list)

+            subcmd_batch_del(remote_list = remote_list)

         total_elapsed = time.time() - timestamp_start

         total_speed = total_elapsed and total_size/total_elapsed or 0.0

         speed_fmt = formatSize(total_speed, human_readable = True, floating_point = True)