@@ -1,3264 +0,0 @@

-#!/usr/bin/env python

-# -*- coding: utf-8 -*-

-

-## --------------------------------------------------------------------

-## s3cmd - S3 client

-##

-## Authors   : Michal Ludvig and contributors

-## Copyright : TGRMN Software - http://www.tgrmn.com - and contributors

-## Website   : http://s3tools.org

-## License   : GPL Version 2

-## --------------------------------------------------------------------

-## This program is free software; you can redistribute it and/or modify

-## it under the terms of the GNU General Public License as published by

-## the Free Software Foundation; either version 2 of the License, or

-## (at your option) any later version.

-## This program is distributed in the hope that it will be useful,

-## but WITHOUT ANY WARRANTY; without even the implied warranty of

-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

-## GNU General Public License for more details.

-## --------------------------------------------------------------------

-

-from __future__ import absolute_import, print_function, division

-

-import sys

-

-if sys.version_info < (2, 6):

-    sys.stderr.write(u"ERROR: Python 2.6 or higher required, sorry.\n")

-    sys.exit(EX_OSFILE)

-

-PY3 = (sys.version_info >= (3, 0))

-

-import codecs

-import errno

-import glob

-import io

-import locale

-import logging

-import os

-import re

-import shutil

-import socket

-import subprocess

-import tempfile

-import time

-import traceback

-

-from copy import copy

-from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter

-from logging import debug, info, warning, error

-

-

-try:

-    import htmlentitydefs

-except Exception:

-    # python 3 support

-    import html.entities as htmlentitydefs

-

-try:

-    unicode

-except NameError:

-    # python 3 support

-    # In python 3, unicode -> str, and str -> bytes

-    unicode = str

-

-try:

-    from shutil import which

-except ImportError:

-    # python2 fallback code

-    from distutils.spawn import find_executable as which

-

-

-def output(message):

-    sys.stdout.write(message + "\n")

-    sys.stdout.flush()

-

-def check_args_type(args, type, verbose_type):

-    """NOTE: This function looks like to not be used."""

-    for arg in args:

-        if S3Uri(arg).type != type:

-            raise ParameterError("Expecting %s instead of '%s'" % (verbose_type, arg))

-

-def cmd_du(args):

-    s3 = S3(Config())

-    if len(args) > 0:

-        uri = S3Uri(args[0])

-        if uri.type == "s3" and uri.has_bucket():

-            subcmd_bucket_usage(s3, uri)

-            return EX_OK

-    subcmd_bucket_usage_all(s3)

-    return EX_OK

-

-def subcmd_bucket_usage_all(s3):

-    """

-    Returns: sum of bucket sizes as integer

-    Raises: S3Error

-    """

-    cfg = Config()

-    response = s3.list_all_buckets()

-

-    buckets_size = 0

-    for bucket in response["list"]:

-        size = subcmd_bucket_usage(s3, S3Uri("s3://" + bucket["Name"]))

-        if size != None:

-            buckets_size += size

-    total_size, size_coeff = formatSize(buckets_size, cfg.human_readable_sizes)

-    total_size_str = str(total_size) + size_coeff

-    output(u"".rjust(12, "-"))

-    output(u"%s Total" % (total_size_str.ljust(12)))

-    return size

-

-def subcmd_bucket_usage(s3, uri):

-    """

-    Returns: bucket size as integer

-    Raises: S3Error

-    """

-    bucket_size = 0

-    object_count = 0

-    extra_info = u''

-

-    bucket = uri.bucket()

-    prefix = uri.object()

-    try:

-        for _, _, objects in s3.bucket_list_streaming(bucket, prefix=prefix, recursive=True):

-            for obj in objects:

-                bucket_size += int(obj["Size"])

-                object_count += 1

-

-    except S3Error as e:

-        if e.info["Code"] in S3.codes:

-            error(S3.codes[e.info["Code"]] % bucket)

-        raise

-

-    except KeyboardInterrupt as e:

-        extra_info = u' [interrupted]'

-

-    total_size_str = u"%d%s" % formatSize(bucket_size,

-                                          Config().human_readable_sizes)

-    if Config().human_readable_sizes:

-        total_size_str = total_size_str.rjust(5)

-    else:

-        total_size_str = total_size_str.rjust(12)

-    output(u"%s %7s objects %s%s" % (total_size_str, object_count, uri,

-                                     extra_info))

-    return bucket_size

-

-def cmd_ls(args):

-    cfg = Config()

-    s3 = S3(cfg)

-    if len(args) > 0:

-        uri = S3Uri(args[0])

-        if uri.type == "s3" and uri.has_bucket():

-            subcmd_bucket_list(s3, uri, cfg.limit)

-            return EX_OK

-

-    # If not a s3 type uri or no bucket was provided, list all the buckets

-    subcmd_all_buckets_list(s3)

-    return EX_OK

-

-def subcmd_all_buckets_list(s3):

-

-    response = s3.list_all_buckets()

-

-    for bucket in sorted(response["list"], key=lambda b:b["Name"]):

-        output(u"%s  s3://%s" % (formatDateTime(bucket["CreationDate"]),

-                                 bucket["Name"]))

-

-def cmd_all_buckets_list_all_content(args):

-    cfg = Config()

-    s3 = S3(cfg)

-

-    response = s3.list_all_buckets()

-

-    for bucket in response["list"]:

-        subcmd_bucket_list(s3, S3Uri("s3://" + bucket["Name"]), cfg.limit)

-        output(u"")

-    return EX_OK

-

-def subcmd_bucket_list(s3, uri, limit):

-    cfg = Config()

-

-    bucket = uri.bucket()

-    prefix = uri.object()

-

-    debug(u"Bucket 's3://%s':" % bucket)

-    if prefix.endswith('*'):

-        prefix = prefix[:-1]

-    try:

-        response = s3.bucket_list(bucket, prefix = prefix, limit = limit)

-    except S3Error as e:

-        if e.info["Code"] in S3.codes:

-            error(S3.codes[e.info["Code"]] % bucket)

-        raise

-

-    # md5 are 32 char long, but for multipart there could be a suffix

-    if Config().human_readable_sizes:

-        # %(size)5s%(coeff)1s

-        format_size = u"%5d%1s"

-        dir_str = u"DIR".rjust(6)

-    else:

-        format_size = u"%12d%s"

-        dir_str = u"DIR".rjust(12)

-    if cfg.long_listing:

-        format_string = u"%(timestamp)16s %(size)s  %(md5)-35s  %(storageclass)-11s  %(uri)s"

-    elif cfg.list_md5:

-        format_string = u"%(timestamp)16s %(size)s  %(md5)-35s  %(uri)s"

-    else:

-        format_string = u"%(timestamp)16s %(size)s  %(uri)s"

-

-    for prefix in response['common_prefixes']:

-        output(format_string % {

-            "timestamp": "",

-            "size": dir_str,

-            "md5": "",

-            "storageclass": "",

-            "uri": uri.compose_uri(bucket, prefix["Prefix"])})

-

-    for object in response["list"]:

-        md5 = object.get('ETag', '').strip('"\'')

-        storageclass = object.get('StorageClass','')

-

-        if cfg.list_md5:

-            if '-' in md5: # need to get md5 from the object

-                object_uri = uri.compose_uri(bucket, object["Key"])

-                info_response = s3.object_info(S3Uri(object_uri))

-                try:

-                    md5 = info_response['s3cmd-attrs']['md5']

-                except KeyError:

-                    pass

-

-        size_and_coeff = formatSize(object["Size"],

-                                    Config().human_readable_sizes)

-        output(format_string % {

-            "timestamp": formatDateTime(object["LastModified"]),

-            "size" : format_size % size_and_coeff,

-            "md5" : md5,

-            "storageclass" : storageclass,

-            "uri": uri.compose_uri(bucket, object["Key"]),

-            })

-

-    if response["truncated"]:

-        warning(u"The list is truncated because the settings limit was reached.")

-

-def cmd_bucket_create(args):

-    cfg = Config()

-    s3 = S3(cfg)

-    for arg in args:

-        uri = S3Uri(arg)

-        if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

-            raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

-        try:

-            response = s3.bucket_create(uri.bucket(), cfg.bucket_location)

-            output(u"Bucket '%s' created" % uri.uri())

-        except S3Error as e:

-            if e.info["Code"] in S3.codes:

-                error(S3.codes[e.info["Code"]] % uri.bucket())

-            raise

-    return EX_OK

-

-def cmd_website_info(args):

-    cfg = Config()

-    s3 = S3(cfg)

-    for arg in args:

-        uri = S3Uri(arg)

-        if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

-            raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

-        try:

-            response = s3.website_info(uri, cfg.bucket_location)

-            if response:

-                output(u"Bucket %s: Website configuration" % uri.uri())

-                output(u"Website endpoint: %s" % response['website_endpoint'])

-                output(u"Index document:   %s" % response['index_document'])

-                output(u"Error document:   %s" % response['error_document'])

-            else:

-                output(u"Bucket %s: Unable to receive website configuration." % (uri.uri()))

-        except S3Error as e:

-            if e.info["Code"] in S3.codes:

-                error(S3.codes[e.info["Code"]] % uri.bucket())

-            raise

-    return EX_OK

-

-def cmd_website_create(args):

-    cfg = Config()

-    s3 = S3(cfg)

-    for arg in args:

-        uri = S3Uri(arg)

-        if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

-            raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

-        try:

-            response = s3.website_create(uri, cfg.bucket_location)

-            output(u"Bucket '%s': website configuration created." % (uri.uri()))

-        except S3Error as e:

-            if e.info["Code"] in S3.codes:

-                error(S3.codes[e.info["Code"]] % uri.bucket())

-            raise

-    return EX_OK

-

-def cmd_website_delete(args):

-    cfg = Config()

-    s3 = S3(cfg)

-    for arg in args:

-        uri = S3Uri(arg)

-        if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

-            raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

-        try:

-            response = s3.website_delete(uri, cfg.bucket_location)

-            output(u"Bucket '%s': website configuration deleted." % (uri.uri()))

-        except S3Error as e:

-            if e.info["Code"] in S3.codes:

-                error(S3.codes[e.info["Code"]] % uri.bucket())

-            raise

-    return EX_OK

-

-def cmd_expiration_set(args):

-    cfg = Config()

-    s3 = S3(cfg)

-    for arg in args:

-        uri = S3Uri(arg)

-        if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

-            raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

-        try:

-            response = s3.expiration_set(uri, cfg.bucket_location)

-            if response["status"] == 200:

-                output(u"Bucket '%s': expiration configuration is set." % (uri.uri()))

-            elif response["status"] == 204:

-                output(u"Bucket '%s': expiration configuration is deleted." % (uri.uri()))

-        except S3Error as e:

-            if e.info["Code"] in S3.codes:

-                error(S3.codes[e.info["Code"]] % uri.bucket())

-            raise

-    return EX_OK

-

-def cmd_bucket_delete(args):

-    cfg = Config()

-    s3 = S3(cfg)

-

-    def _bucket_delete_one(uri, retry=True):

-        try:

-            response = s3.bucket_delete(uri.bucket())

-            output(u"Bucket '%s' removed" % uri.uri())

-        except S3Error as e:

-            if e.info['Code'] == 'NoSuchBucket':

-                if cfg.force:

-                    return EX_OK

-                else:

-                    raise

-            if e.info['Code'] == 'BucketNotEmpty' and retry and (cfg.force or cfg.recursive):

-                warning(u"Bucket is not empty. Removing all the objects from it first. This may take some time...")

-                rc = subcmd_batch_del(uri_str = uri.uri())

-                if rc == EX_OK:

-                    return _bucket_delete_one(uri, False)

-                else:

-                    output(u"Bucket was not removed")

-            elif e.info["Code"] in S3.codes:

-                error(S3.codes[e.info["Code"]] % uri.bucket())

-            raise

-        return EX_OK

-

-    for arg in args:

-        uri = S3Uri(arg)

-        if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

-            raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)

-        rc = _bucket_delete_one(uri)

-        if rc != EX_OK:

-            return rc

-    return EX_OK

-

-def cmd_object_put(args):

-    cfg = Config()

-    s3 = S3(cfg)

-

-    if len(args) == 0:

-        raise ParameterError("Nothing to upload. Expecting a local file or directory and a S3 URI destination.")

-

-    ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)

-    destination_base_uri = S3Uri(args.pop())

-    if destination_base_uri.type != 's3':

-        raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)

-    destination_base = destination_base_uri.uri()

-

-    if len(args) == 0:

-        raise ParameterError("Nothing to upload. Expecting a local file or directory.")

-

-    local_list, single_file_local, exclude_list, total_size_local = fetch_local_list(args, is_src = True)

-

-    local_count = len(local_list)

-

-    info(u"Summary: %d local files to upload" % local_count)

-

-    if local_count == 0:

-        raise ParameterError("Nothing to upload.")

-

-    if local_count > 0:

-        if not single_file_local and '-' in local_list.keys():

-            raise ParameterError("Cannot specify multiple local files if uploading from '-' (ie stdin)")

-        elif single_file_local and local_list.keys()[0] == "-" and destination_base.endswith("/"):

-            raise ParameterError("Destination S3 URI must not end with '/' when uploading from stdin.")

-        elif not destination_base.endswith("/"):

-            if not single_file_local:

-                raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).")

-            local_list[local_list.keys()[0]]['remote_uri'] = destination_base

-        else:

-            for key in local_list:

-                local_list[key]['remote_uri'] = destination_base + key

-

-    if cfg.dry_run:

-        for key in exclude_list:

-            output(u"exclude: %s" % key)

-        for key in local_list:

-            if key != "-":

-                nicekey = local_list[key]['full_name']

-            else:

-                nicekey = "<stdin>"

-            output(u"upload: '%s' -> '%s'" % (nicekey, local_list[key]['remote_uri']))

-

-        warning(u"Exiting now because of --dry-run")

-        return EX_OK

-

-    seq = 0

-    ret = EX_OK

-    for key in local_list:

-        seq += 1

-

-        uri_final = S3Uri(local_list[key]['remote_uri'])

-        try:

-            src_md5 = local_list.get_md5(key)

-        except IOError:

-            src_md5 = None

-

-        extra_headers = copy(cfg.extra_headers)

-        full_name_orig = local_list[key]['full_name']

-        full_name = full_name_orig

-        seq_label = "[%d of %d]" % (seq, local_count)

-        if Config().encrypt:

-            gpg_exitcode, full_name, extra_headers["x-amz-meta-s3tools-gpgenc"] = gpg_encrypt(full_name_orig)

-        attr_header = _build_attr_header(local_list[key], key, src_md5)

-        debug(u"attr_header: %s" % attr_header)

-        extra_headers.update(attr_header)

-        try:

-            response = s3.object_put(full_name, uri_final, extra_headers, extra_label = seq_label)

-        except S3UploadError as exc:

-            error(u"Upload of '%s' failed too many times (Last reason: %s)" % (full_name_orig, exc))

-            if cfg.stop_on_error:

-                ret = EX_DATAERR

-                error(u"Exiting now because of --stop-on-error")

-                break

-            ret = EX_PARTIAL

-            continue

-        except InvalidFileError as exc:

-            error(u"Upload of '%s' is not possible (Reason: %s)" % (full_name_orig, exc))

-            ret = EX_PARTIAL

-            if cfg.stop_on_error:

-                ret = EX_OSFILE

-                error(u"Exiting now because of --stop-on-error")

-                break

-            continue

-        if response is not None:

-            speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)

-            if not Config().progress_meter:

-                if full_name_orig != "-":

-                    nicekey = full_name_orig

-                else:

-                    nicekey = "<stdin>"

-                output(u"upload: '%s' -> '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %

-                       (nicekey, uri_final, response["size"], response["elapsed"],

-                        speed_fmt[0], speed_fmt[1], seq_label))

-        if Config().acl_public:

-            output(u"Public URL of the object is: %s" %

-                   (uri_final.public_url()))

-        if Config().encrypt and full_name != full_name_orig:

-            debug(u"Removing temporary encrypted file: %s" % full_name)

-            os.remove(deunicodise(full_name))

-    return ret

-

-def cmd_object_get(args):

-    cfg = Config()

-    s3 = S3(cfg)

-

-    ## Check arguments:

-    ## if not --recursive:

-    ##   - first N arguments must be S3Uri

-    ##   - if the last one is S3 make current dir the destination_base

-    ##   - if the last one is a directory:

-    ##       - take all 'basenames' of the remote objects and

-    ##         make the destination name be 'destination_base'+'basename'

-    ##   - if the last one is a file or not existing:

-    ##       - if the number of sources (N, above) == 1 treat it

-    ##         as a filename and save the object there.

-    ##       - if there's more sources -> Error

-    ## if --recursive:

-    ##   - first N arguments must be S3Uri

-    ##       - for each Uri get a list of remote objects with that Uri as a prefix

-    ##       - apply exclude/include rules

-    ##       - each list item will have MD5sum, Timestamp and pointer to S3Uri

-    ##         used as a prefix.

-    ##   - the last arg may be '-' (stdout)

-    ##   - the last arg may be a local directory - destination_base

-    ##   - if the last one is S3 make current dir the destination_base

-    ##   - if the last one doesn't exist check remote list:

-    ##       - if there is only one item and its_prefix==its_name

-    ##         download that item to the name given in last arg.

-    ##       - if there are more remote items use the last arg as a destination_base

-    ##         and try to create the directory (incl. all parents).

-    ##

-    ## In both cases we end up with a list mapping remote object names (keys) to local file names.

-

-    ## Each item will be a dict with the following attributes

-    # {'remote_uri', 'local_filename'}

-    download_list = []

-

-    if len(args) == 0:

-        raise ParameterError("Nothing to download. Expecting S3 URI.")

-

-    if S3Uri(args[-1]).type == 'file':

-        destination_base = args.pop()

-    else:

-        destination_base = "."

-

-    if len(args) == 0:

-        raise ParameterError("Nothing to download. Expecting S3 URI.")

-

-    remote_list, exclude_list, remote_total_size = fetch_remote_list(args, require_attribs = False)

-

-    remote_count = len(remote_list)

-

-    info(u"Summary: %d remote files to download" % remote_count)

-

-    if remote_count > 0:

-        if destination_base == "-":

-            ## stdout is ok for multiple remote files!

-            for key in remote_list:

-                remote_list[key]['local_filename'] = "-"

-        elif not os.path.isdir(deunicodise(destination_base)):

-            ## We were either given a file name (existing or not)

-            if remote_count > 1:

-                raise ParameterError("Destination must be a directory or stdout when downloading multiple sources.")

-            remote_list[remote_list.keys()[0]]['local_filename'] = destination_base

-        else:

-            if destination_base[-1] != os.path.sep:

-                destination_base += os.path.sep

-            for key in remote_list:

-                local_filename = destination_base + key

-                if os.path.sep != "/":

-                    local_filename = os.path.sep.join(local_filename.split("/"))

-                remote_list[key]['local_filename'] = local_filename

-

-    if cfg.dry_run:

-        for key in exclude_list:

-            output(u"exclude: %s" % key)

-        for key in remote_list:

-            output(u"download: '%s' -> '%s'" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename']))

-

-        warning(u"Exiting now because of --dry-run")

-        return EX_OK

-

-    seq = 0

-    ret = EX_OK

-    for key in remote_list:

-        seq += 1

-        item = remote_list[key]

-        uri = S3Uri(item['object_uri_str'])

-        ## Encode / Decode destination with "replace" to make sure it's compatible with current encoding

-        destination = unicodise_safe(item['local_filename'])

-        seq_label = "[%d of %d]" % (seq, remote_count)

-

-        start_position = 0

-

-        if destination == "-":

-            ## stdout

-            dst_stream = io.open(sys.__stdout__.fileno(), mode='wb', closefd=False)

-            dst_stream.stream_name = u'<stdout>'

-            file_exists = True

-        else:

-            ## File

-            try:

-                file_exists = os.path.exists(deunicodise(destination))

-                try:

-                    dst_stream = io.open(deunicodise(destination), mode='ab')

-                    dst_stream.stream_name = destination

-                except IOError as e:

-                    if e.errno == errno.ENOENT:

-                        basename = destination[:destination.rindex(os.path.sep)]

-                        info(u"Creating directory: %s" % basename)

-                        os.makedirs(deunicodise(basename))

-                        dst_stream = io.open(deunicodise(destination), mode='ab')

-                        dst_stream.stream_name = destination

-                    else:

-                        raise

-                if file_exists:

-                    if Config().get_continue:

-                        start_position = dst_stream.tell()

-                    elif Config().force:

-                        start_position = 0

-                        dst_stream.seek(0)

-                        dst_stream.truncate()

-                    elif Config().skip_existing:

-                        info(u"Skipping over existing file: %s" % (destination))

-                        continue

-                    else:

-                        dst_stream.close()

-                        raise ParameterError(u"File %s already exists. Use either of --force / --continue / --skip-existing or give it a new name." % destination)

-            except IOError as e:

-                error(u"Skipping %s: %s" % (destination, e.strerror))

-                continue

-        try:

-            try:

-                response = s3.object_get(uri, dst_stream, destination, start_position = start_position, extra_label = seq_label)

-            finally:

-                dst_stream.close()

-        except S3DownloadError as e:

-            error(u"%s: Skipping that file.  This is usually a transient error, please try again later." % e)

-            if not file_exists: # Delete, only if file didn't exist before!

-                debug(u"object_get failed for '%s', deleting..." % (destination,))

-                os.unlink(deunicodise(destination))

-                ret = EX_PARTIAL

-                if cfg.stop_on_error:

-                    ret = EX_DATAERR

-                    break

-            continue

-        except S3Error as e:

-            if not file_exists: # Delete, only if file didn't exist before!

-                debug(u"object_get failed for '%s', deleting..." % (destination,))

-                os.unlink(deunicodise(destination))

-            raise

-

-        if "x-amz-meta-s3tools-gpgenc" in response["headers"]:

-            gpg_decrypt(destination, response["headers"]["x-amz-meta-s3tools-gpgenc"])

-            response["size"] = os.stat(deunicodise(destination))[6]

-        if "last-modified" in response["headers"] and destination != "-":

-            last_modified = time.mktime(time.strptime(response["headers"]["last-modified"], "%a, %d %b %Y %H:%M:%S GMT"))

-            os.utime(deunicodise(destination), (last_modified, last_modified))

-            debug("set mtime to %s" % last_modified)

-        if not Config().progress_meter and destination != "-":

-            speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)

-            output(u"download: '%s' -> '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s)" %

-                (uri, destination, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1]))

-        if Config().delete_after_fetch:

-            s3.object_delete(uri)

-            output(u"File '%s' removed after fetch" % (uri))

-    return EX_OK

-

-def cmd_object_del(args):

-    cfg = Config()

-    recursive = cfg.recursive

-    for uri_str in args:

-        uri = S3Uri(uri_str)

-        if uri.type != "s3":

-            raise ParameterError("Expecting S3 URI instead of '%s'" % uri_str)

-        if not uri.has_object():

-            if recursive and not cfg.force:

-                raise ParameterError("Please use --force to delete ALL contents of %s" % uri_str)

-            elif not recursive:

-                raise ParameterError("File name required, not only the bucket name. Alternatively use --recursive")

-

-        if not recursive:

-            rc = subcmd_object_del_uri(uri_str)

-        elif cfg.exclude or cfg.include or cfg.max_delete > 0:

-            # subcmd_batch_del_iterative does not support file exclusion and can't

-            # accurately know how many total files will be deleted, so revert to batch delete.

-            rc = subcmd_batch_del(uri_str = uri_str)

-        else:

-            rc = subcmd_batch_del_iterative(uri_str = uri_str)

-        if not rc:

-            return rc

-    return EX_OK

-

-def subcmd_batch_del_iterative(uri_str = None, bucket = None):

-    """ Streaming version of batch deletion (doesn't realize whole list in memory before deleting).

-

-    Differences from subcmd_batch_del:

-      - Does not obey --exclude directives or obey cfg.max_delete (use subcmd_batch_del in those cases)

-    """

-    if bucket and uri_str:

-        raise ValueError("Pass only one of uri_str or bucket")

-    if bucket: # bucket specified

-        uri_str = "s3://%s" % bucket

-    cfg = Config()

-    s3 = S3(cfg)

-    uri = S3Uri(uri_str)

-    bucket = uri.bucket()

-

-    deleted_bytes = deleted_count = 0

-

-    for _, _, to_delete in s3.bucket_list_streaming(bucket, prefix=uri.object(), recursive=True):

-        if not to_delete:

-            continue

-        if not cfg.dry_run:

-            response = s3.object_batch_delete_uri_strs([uri.compose_uri(bucket, item['Key']) for item in to_delete])

-        deleted_bytes += sum(int(item["Size"]) for item in to_delete)

-        deleted_count += len(to_delete)

-        output(u'\n'.join(u"delete: '%s'" % uri.compose_uri(bucket, p['Key']) for p in to_delete))

-

-    if deleted_count:

-        # display summary data of deleted files

-        if cfg.stats:

-            stats_info = StatsInfo()

-            stats_info.files_deleted = deleted_count

-            stats_info.size_deleted = deleted_bytes

-            output(stats_info.format_output())

-        else:

-            total_size, size_coeff = formatSize(deleted_bytes, Config().human_readable_sizes)

-            total_size_str = str(total_size) + size_coeff

-            info(u"Deleted %s objects (%s) from %s" % (deleted_count, total_size_str, uri))

-    else:

-        warning(u"Remote list is empty.")

-

-    return EX_OK

-

-def subcmd_batch_del(uri_str = None, bucket = None, remote_list = None):

-    """

-    Returns: EX_OK

-    Raises: ValueError

-    """

-    cfg = Config()

-    s3 = S3(cfg)

-    def _batch_del(remote_list):

-        to_delete = remote_list[:1000]

-        remote_list = remote_list[1000:]

-        while len(to_delete):

-            debug(u"Batch delete %d, remaining %d" % (len(to_delete), len(remote_list)))

-            if not cfg.dry_run:

-                response = s3.object_batch_delete(to_delete)

-            output(u'\n'.join((u"delete: '%s'" % to_delete[p]['object_uri_str']) for p in to_delete))

-            to_delete = remote_list[:1000]

-            remote_list = remote_list[1000:]

-

-    if remote_list is not None and len(remote_list) == 0:

-        return False

-

-    if len([item for item in [uri_str, bucket, remote_list] if item]) != 1:

-        raise ValueError("One and only one of 'uri_str', 'bucket', 'remote_list' can be specified.")

-

-    if bucket: # bucket specified

-        uri_str = "s3://%s" % bucket

-    if remote_list is None: # uri_str specified

-        remote_list, exclude_list, remote_total_size = fetch_remote_list(uri_str, require_attribs = False)

-

-    if len(remote_list) == 0:

-        warning(u"Remote list is empty.")

-        return EX_OK

-

-    if cfg.max_delete > 0 and len(remote_list) > cfg.max_delete:

-        warning(u"delete: maximum requested number of deletes would be exceeded, none performed.")

-        return EX_OK

-

-    _batch_del(remote_list)

-

-    if cfg.dry_run:

-        warning(u"Exiting now because of --dry-run")

-    return EX_OK

-

-def subcmd_object_del_uri(uri_str, recursive = None):

-    """

-    Returns: True if XXX, False if XXX

-    Raises: ValueError

-    """

-    cfg = Config()

-    s3 = S3(cfg)

-

-    if recursive is None:

-        recursive = cfg.recursive

-

-    remote_list, exclude_list, remote_total_size = fetch_remote_list(uri_str, require_attribs = False, recursive = recursive)

-

-    remote_count = len(remote_list)

-

-    info(u"Summary: %d remote files to delete" % remote_count)

-    if cfg.max_delete > 0 and remote_count > cfg.max_delete:

-        warning(u"delete: maximum requested number of deletes would be exceeded, none performed.")

-        return False

-

-    if cfg.dry_run:

-        for key in exclude_list:

-            output(u"exclude: %s" % key)

-        for key in remote_list:

-            output(u"delete: %s" % remote_list[key]['object_uri_str'])

-

-        warning(u"Exiting now because of --dry-run")

-        return True

-

-    for key in remote_list:

-        item = remote_list[key]

-        response = s3.object_delete(S3Uri(item['object_uri_str']))

-        output(u"delete: '%s'" % item['object_uri_str'])

-    return True

-

-def cmd_object_restore(args):

-    cfg = Config()

-    s3 = S3(cfg)

-

-    if cfg.restore_days < 1:

-        raise ParameterError("You must restore a file for 1 or more days")

-

-    # accept case-insensitive argument but fix it to match S3 API

-    if cfg.restore_priority.title() not in ['Standard', 'Expedited', 'Bulk']:

-        raise ParameterError("Valid restoration priorities: bulk, standard, expedited")

-    else:

-        cfg.restore_priority = cfg.restore_priority.title()

-

-    remote_list, exclude_list, remote_total_size = fetch_remote_list(args, require_attribs = False, recursive = cfg.recursive)

-

-    remote_count = len(remote_list)

-

-    info(u"Summary: Restoring %d remote files for %d days at %s priority" % (remote_count, cfg.restore_days, cfg.restore_priority))

-

-    if cfg.dry_run:

-        for key in exclude_list:

-            output(u"exclude: %s" % key)

-        for key in remote_list:

-            output(u"restore: '%s'" % remote_list[key]['object_uri_str'])

-

-        warning(u"Exiting now because of --dry-run")

-        return EX_OK

-

-    for key in remote_list:

-        item = remote_list[key]

-

-        uri = S3Uri(item['object_uri_str'])

-        if not item['object_uri_str'].endswith("/"):

-            try:

-                response = s3.object_restore(S3Uri(item['object_uri_str']))

-                output(u"restore: '%s'" % item['object_uri_str'])

-            except S3Error as e:

-                if e.code == "RestoreAlreadyInProgress":

-                    warning("%s: %s" % (e.message, item['object_uri_str']))

-                else:

-                    raise e

-        else:

-            debug(u"Skipping directory since only files may be restored")

-    return EX_OK

-

-

-def subcmd_cp_mv(args, process_fce, action_str, message):

-    cfg = Config()

-    if action_str == 'modify':

-        if len(args) < 1:

-            raise ParameterError("Expecting one or more S3 URIs for "

-                                 + action_str)

-        destination_base = None

-    else:

-        if len(args) < 2:

-            raise ParameterError("Expecting two or more S3 URIs for "

-                                 + action_str)

-        dst_base_uri = S3Uri(args.pop())

-        if dst_base_uri.type != "s3":

-            raise ParameterError("Destination must be S3 URI. To download a "

-                                 "file use 'get' or 'sync'.")

-        destination_base = dst_base_uri.uri()

-

-    scoreboard = ExitScoreboard()

-

-    remote_list, exclude_list, remote_total_size = \

-        fetch_remote_list(args, require_attribs=False)

-

-    remote_count = len(remote_list)

-

-    info(u"Summary: %d remote files to %s" % (remote_count, action_str))

-

-    if destination_base:

-        # Trying to mv dir1/ to dir2 will not pass a test in S3.FileLists,

-        # so we don't need to test for it here.

-        if not destination_base.endswith('/') \

-           and (len(remote_list) > 1 or cfg.recursive):

-            raise ParameterError("Destination must be a directory and end with"

-                                 " '/' when acting on a folder content or on "

-                                 "multiple sources.")

-

-        if cfg.recursive:

-            for key in remote_list:

-                remote_list[key]['dest_name'] = destination_base + key

-        else:

-            for key in remote_list:

-                if destination_base.endswith("/"):

-                    remote_list[key]['dest_name'] = destination_base + key

-                else:

-                    remote_list[key]['dest_name'] = destination_base

-    else:

-        for key in remote_list:

-            remote_list[key]['dest_name'] = remote_list[key]['object_uri_str']

-

-    if cfg.dry_run:

-        for key in exclude_list:

-            output(u"exclude: %s" % key)