@@ -179,6 +179,9 @@ Simple s3cmd HowTo

    account page. Be careful when copying them! They are 

    case sensitive and must be entered accurately or you'll 

    keep getting errors about invalid signatures or similar.

+   

+   Remember to add ListAllMyBuckets permissions to the keys

+   or you will get an AccessDenied error while testing access.

 3) Run "s3cmd ls" to list all your buckets.

    As you just started using S3 there are no buckets owned by 

@@ -146,7 +146,6 @@ class ACL(object):

         if self.hasGrant(name, permission):

             return

-        name = name.lower()

         permission = permission.upper()

         if "ALL" == permission:

@@ -159,12 +158,17 @@ class ACL(object):

         grantee.name = name

         grantee.permission = permission

-        if  name.find('@') <= -1: # ultra lame attempt to differenciate emails id from canonical ids

-            grantee.xsi_type = "CanonicalUser"

-            grantee.tag = "ID"

-        else:

+        if  name.find('@') > -1:

+            grantee.name = grantee.name.lower

             grantee.xsi_type = "AmazonCustomerByEmail"

             grantee.tag = "EmailAddress"

+        elif name.find('http://acs.amazonaws.com/groups/') > -1:

+            grantee.xsi_type = "Group"

+            grantee.tag = "URI"

+        else:

+            grantee.name = grantee.name.lower

+            grantee.xsi_type = "CanonicalUser"

+            grantee.tag = "ID"

         self.appendGrantee(grantee)

@@ -36,8 +36,11 @@ class Config(object):

     human_readable_sizes = False

     extra_headers = SortedDict(ignore_case = True)

     force = False

+    server_side_encryption = False

     enable = None

     get_continue = False

+    put_continue = False

+    upload_id = None

     skip_existing = False

     recursive = False

     acl_public = None

@@ -75,6 +78,7 @@ class Config(object):

     bucket_location = "US"

     default_mime_type = "binary/octet-stream"

     guess_mime_type = True

+    use_mime_magic = True

     mime_type = ""

     enable_multipart = True

     multipart_chunk_size_mb = 15    # MB

@@ -87,6 +91,7 @@ class Config(object):

     debug_exclude = {}

     debug_include = {}

     encoding = "utf-8"

+    add_content_encoding = True

     urlencoding_mode = "normal"

     log_target_prefix = ""

     reduced_redundancy = False

@@ -180,7 +185,7 @@ class Config(object):

                     if "key" in data:

                         Config().update_option(data["key"], data["value"])

                         if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):

-                            print_value = (data["value"][:2]+"...%d_chars..."+data["value"][-1:]) % (len(data["value"]) - 3)

+                            print_value = ("%s...%d_chars...%s") % (data["value"][:2], len(data["value"]) - 3, data["value"][-1:])

                         else:

                             print_value = data["value"]

                         debug("env_Config: %s->%s" % (data["key"], print_value))

@@ -274,7 +279,7 @@ class ConfigParser(object):

                     data["value"] = data["value"][1:-1]

                 self.__setitem__(data["key"], data["value"])

                 if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):

-                    print_value = (data["value"][:2]+"...%d_chars..."+data["value"][-1:]) % (len(data["value"]) - 3)

+                    print_value = ("%s...%d_chars...%s") % (data["value"][:2], len(data["value"]) - 3, data["value"][-1:])

                 else:

                     print_value = data["value"]

                 debug("ConfigParser: %s->%s" % (data["key"], print_value))

@@ -36,6 +36,7 @@ class FileDict(SortedDict):

         return md5

     def record_hardlink(self, relative_file, dev, inode, md5):

+        if dev == 0 or inode == 0: return # Windows

         if dev not in self.hardlinks:

             self.hardlinks[dev] = dict()

         if inode not in self.hardlinks[dev]:

@@ -17,6 +17,7 @@ import os

 import sys

 import glob

 import copy

+import re

 __all__ = ["fetch_local_list", "fetch_remote_list", "compare_filelists", "filter_exclude_include"]

@@ -250,7 +251,11 @@ def fetch_local_list(args, is_src = False, recursive = None):

         return loc_list, single_file

     def _maintain_cache(cache, local_list):

-        if cfg.cache_file:

+        # if getting the file list from files_from, it is going to be

+        # a subset of the actual tree.  We should not purge content

+        # outside of that subset as we don't know if it's valid or

+        # not.  Leave it to a non-files_from run to purge.

+        if cfg.cache_file and len(cfg.files_from) == 0:

             cache.mark_all_for_purge()

             for i in local_list.keys():

                 cache.unmark_for_purge(local_list[i]['dev'], local_list[i]['inode'], local_list[i]['mtime'], local_list[i]['size'])

@@ -363,7 +368,7 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):

                 'dev' : None,

                 'inode' : None,

             }

-            if rem_list[key]['md5'].find("-"): # always get it for multipart uploads

+            if rem_list[key]['md5'].find("-") > 0: # always get it for multipart uploads

                 _get_remote_attribs(S3Uri(object_uri_str), rem_list[key])

             md5 = rem_list[key]['md5']

             rem_list.record_md5(key, md5)

@@ -398,16 +403,12 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):

             uri_str = str(uri)

             ## Wildcards used in remote URI?

             ## If yes we'll need a bucket listing...

-            if uri_str.find('*') > -1 or uri_str.find('?') > -1:

-                first_wildcard = uri_str.find('*')

-                first_questionmark = uri_str.find('?')

-                if first_questionmark > -1 and first_questionmark < first_wildcard:

-                    first_wildcard = first_questionmark

-                prefix = uri_str[:first_wildcard]

-                rest = uri_str[first_wildcard+1:]

+            wildcard_split_result = re.split("\*|\?", uri_str, maxsplit=1)

+            if len(wildcard_split_result) == 2: # wildcards found

+                prefix, rest = wildcard_split_result

                 ## Only request recursive listing if the 'rest' of the URI,

                 ## i.e. the part after first wildcard, contains '/'

-                need_recursion = rest.find('/') > -1

+                need_recursion = '/' in rest

                 objectlist = _get_filelist_remote(S3Uri(prefix), recursive = need_recursion)

                 for key in objectlist:

                     ## Check whether the 'key' matches the requested wildcards

@@ -5,6 +5,7 @@ class HashCache(object):

         self.inodes = dict()

     def add(self, dev, inode, mtime, size, md5):

+        if dev == 0 or inode == 0: return # Windows

         if dev not in self.inodes:

             self.inodes[dev] = dict()

         if inode not in self.inodes[dev]:

@@ -27,7 +28,10 @@ class HashCache(object):

                     self.inodes[d][i][c]['purge'] = True

     def unmark_for_purge(self, dev, inode, mtime, size):

-        d = self.inodes[dev][inode][mtime]

+        try:

+            d = self.inodes[dev][inode][mtime]

+        except KeyError:

+            return

         if d['size'] == size and 'purge' in d:

             del self.inodes[dev][inode][mtime]['purge']

@@ -3,10 +3,12 @@

 ## License: GPL Version 2

 import os

+import sys

 from stat import ST_SIZE

 from logging import debug, info, warning, error

-from Utils import getTextFromXml, formatSize, unicodise

+from Utils import getTextFromXml, getTreeFromXml, formatSize, unicodise, calculateChecksum, parseNodes

 from Exceptions import S3UploadError

+from collections import defaultdict

 class MultiPartUpload(object):

@@ -22,15 +24,55 @@ class MultiPartUpload(object):

         self.headers_baseline = headers_baseline

         self.upload_id = self.initiate_multipart_upload()

+    def get_parts_information(self, uri, upload_id):

+        multipart_response = self.s3.list_multipart(uri, upload_id)

+        tree = getTreeFromXml(multipart_response['data'])

+

+        parts = defaultdict(lambda: None)

+        for elem in parseNodes(tree):

+            try:

+                parts[int(elem['PartNumber'])] = {'checksum': elem['ETag'], 'size': elem['Size']}

+            except KeyError:

+                pass

+

+        return parts

+

+    def get_unique_upload_id(self, uri):

+        upload_id = None

+        multipart_response = self.s3.get_multipart(uri)

+        tree = getTreeFromXml(multipart_response['data'])

+        for mpupload in parseNodes(tree):

+            try:

+                mp_upload_id = mpupload['UploadId']

+                mp_path = mpupload['Key']

+                info("mp_path: %s, object: %s" % (mp_path, uri.object()))

+                if mp_path == uri.object():

+                    if upload_id is not None:

+                        raise ValueError("More than one UploadId for URI %s.  Disable multipart upload, or use\n %s multipart %s\nto list the Ids, then pass a unique --upload-id into the put command." % (uri, sys.argv[0], uri))

+                    upload_id = mp_upload_id

+            except KeyError:

+                pass

+

+        return upload_id

+

     def initiate_multipart_upload(self):

         """

         Begin a multipart upload

         http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadInitiate.html

         """

-        request = self.s3.create_request("OBJECT_POST", uri = self.uri, headers = self.headers_baseline, extra = "?uploads")

-        response = self.s3.send_request(request)

-        data = response["data"]

-        self.upload_id = getTextFromXml(data, "UploadId")

+        if self.s3.config.upload_id is not None:

+            self.upload_id = self.s3.config.upload_id

+        elif self.s3.config.put_continue:

+            self.upload_id = self.get_unique_upload_id(self.uri)

+        else:

+            self.upload_id = None

+

+        if self.upload_id is None:

+            request = self.s3.create_request("OBJECT_POST", uri = self.uri, headers = self.headers_baseline, extra = "?uploads")

+            response = self.s3.send_request(request)

+            data = response["data"]

+            self.upload_id = getTextFromXml(data, "UploadId")

+

         return self.upload_id

     def upload_all_parts(self):

@@ -51,6 +93,10 @@ class MultiPartUpload(object):

         else:

             debug("MultiPart: Uploading from %s" % (self.file.name))

+        remote_statuses = defaultdict(lambda: None)

+        if self.s3.config.put_continue:

+            remote_statuses = self.get_parts_information(self.uri, self.upload_id)

+

         seq = 1

         if self.file.name != "<stdin>":

             while size_left > 0:

@@ -63,10 +109,10 @@ class MultiPartUpload(object):

                     'extra' : "[part %d of %d, %s]" % (seq, nr_parts, "%d%sB" % formatSize(current_chunk_size, human_readable = True))

                 }

                 try:

-                    self.upload_part(seq, offset, current_chunk_size, labels)

+                    self.upload_part(seq, offset, current_chunk_size, labels, remote_status = remote_statuses[seq])

                 except:

-                    error(u"Upload of '%s' part %d failed. Aborting multipart upload." % (self.file.name, seq))

-                    self.abort_upload()

+                    error(u"\nUpload of '%s' part %d failed. Use\n  %s abortmp %s %s\nto abort the upload, or\n  %s --upload-id %s put ...\nto continue the upload."

+                          % (self.file.name, seq, sys.argv[0], self.uri, self.upload_id, sys.argv[0], self.upload_id))

                     raise

                 seq += 1

         else:

@@ -82,22 +128,38 @@ class MultiPartUpload(object):

                 if len(buffer) == 0: # EOF

                     break

                 try:

-                    self.upload_part(seq, offset, current_chunk_size, labels, buffer)

+                    self.upload_part(seq, offset, current_chunk_size, labels, buffer, remote_status = remote_statuses[seq])

                 except:

-                    error(u"Upload of '%s' part %d failed. Aborting multipart upload." % (self.file.name, seq))

-                    self.abort_upload()

+                    error(u"\nUpload of '%s' part %d failed. Use\n  %s abortmp %s %s\nto abort, or\n  %s --upload-id %s put ...\nto continue the upload."

+                          % (self.file.name, seq, self.uri, sys.argv[0], self.upload_id, sys.argv[0], self.upload_id))

                     raise

                 seq += 1

         debug("MultiPart: Upload finished: %d parts", seq - 1)

-    def upload_part(self, seq, offset, chunk_size, labels, buffer = ''):

+    def upload_part(self, seq, offset, chunk_size, labels, buffer = '', remote_status = None):

         """

         Upload a file chunk

         http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html

         """

         # TODO implement Content-MD5

         debug("Uploading part %i of %r (%s bytes)" % (seq, self.upload_id, chunk_size))

+

+        if remote_status is not None:

+            if int(remote_status['size']) == chunk_size:

+                checksum = calculateChecksum(buffer, self.file, offset, chunk_size, self.s3.config.send_chunk)

+                remote_checksum = remote_status['checksum'].strip('"')

+                if remote_checksum == checksum:

+                    warning("MultiPart: size and md5sum match for %s part %d, skipping." % (self.uri, seq))

+                    self.parts[seq] = remote_status['checksum']

+                    return

+                else:

+                    warning("MultiPart: checksum (%s vs %s) does not match for %s part %d, reuploading."

+                            % (remote_checksum, checksum, self.uri, seq))

+            else:

+                warning("MultiPart: size (%d vs %d) does not match for %s part %d, reuploading."

+                        % (int(remote_status['size']), chunk_size, self.uri, seq))

+

         headers = { "content-length": chunk_size }

         query_string = "?partNumber=%i&uploadId=%s" % (seq, self.upload_id)

         request = self.s3.create_request("OBJECT_PUT", uri = self.uri, headers = headers, extra = query_string)

@@ -130,8 +192,19 @@ class MultiPartUpload(object):

         http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadAbort.html

         """

         debug("MultiPart: Aborting upload: %s" % self.upload_id)

-        request = self.s3.create_request("OBJECT_DELETE", uri = self.uri, extra = "?uploadId=%s" % (self.upload_id))

-        response = self.s3.send_request(request)

+        #request = self.s3.create_request("OBJECT_DELETE", uri = self.uri, extra = "?uploadId=%s" % (self.upload_id))

+        #response = self.s3.send_request(request)

+        response = None

         return response

 # vim:et:ts=4:sts=4:ai

+

+

+

+

+

+

+

+

+

+

@@ -79,7 +79,7 @@ class Progress(object):

             self._stdout.flush()

             return

-        rel_position = selfself.current_position * 100 / self.total_size

+        rel_position = self.current_position * 100 / self.total_size

         if rel_position >= self.last_milestone:

             self.last_milestone = (int(rel_position) / 5) * 5

             self._stdout.write("%d%% ", self.last_milestone)

@@ -403,12 +403,16 @@ class S3(object):

         headers = SortedDict(ignore_case = True)

         if extra_headers:

             headers.update(extra_headers)

-

+        

+        ## Set server side encryption

+        if self.config.server_side_encryption:

+            headers["x-amz-server-side-encryption"] = "AES256"

+    

         ## MIME-type handling

         content_type = self.config.mime_type

         content_encoding = None

         if filename != "-" and not content_type and self.config.guess_mime_type:

-            (content_type, content_encoding) = mime_magic(filename)

+            (content_type, content_encoding) = mime_magic(filename) if self.config.use_mime_magic else mimetypes.guess_type(filename)

         if not content_type:

             content_type = self.config.default_mime_type

@@ -417,7 +421,7 @@ class S3(object):

             content_type = content_type + "; charset=" + self.config.encoding.upper()

         headers["content-type"] = content_type

-        if content_encoding is not None:

+        if content_encoding is not None and self.config.add_content_encoding:

             headers["content-encoding"] = content_encoding

         ## Other Amazon S3 attributes

@@ -438,6 +442,31 @@ class S3(object):

             return self.send_file_multipart(file, headers, uri, size)

         ## Not multipart...

+        if self.config.put_continue:

+            # Note, if input was stdin, we would be performing multipart upload.

+            # So this will always work as long as the file already uploaded was

+            # not uploaded via MultiUpload, in which case its ETag will not be

+            # an md5.

+            try:

+                info = self.object_info(uri)

+            except:

+                info = None

+

+            if info is not None:

+                remote_size = int(info['headers']['content-length'])

+                remote_checksum = info['headers']['etag'].strip('"')

+                if size == remote_size:

+                    checksum = calculateChecksum('', file, 0, size, self.config.send_chunk)

+                    if remote_checksum == checksum:

+                        warning("Put: size and md5sum match for %s, skipping." % uri)

+                        return

+                    else:

+                        warning("MultiPart: checksum (%s vs %s) does not match for %s, reuploading."

+                                % (remote_checksum, checksum, uri))

+                else:

+                    warning("MultiPart: size (%d vs %d) does not match for %s, reuploading."

+                            % (remote_size, size, uri))

+

         headers["content-length"] = size

         request = self.create_request("OBJECT_PUT", uri = uri, headers = headers)

         labels = { 'source' : unicodise(filename), 'destination' : unicodise(uri.uri()), 'extra' : extra_label }

@@ -474,6 +503,11 @@ class S3(object):

             headers["x-amz-storage-class"] = "REDUCED_REDUNDANCY"

         # if extra_headers:

         #   headers.update(extra_headers)

+        

+        ## Set server side encryption

+        if self.config.server_side_encryption:

+            headers["x-amz-server-side-encryption"] = "AES256" 

+        

         request = self.create_request("OBJECT_PUT", uri = dst_uri, headers = headers)

         response = self.send_request(request)

         return response

@@ -535,6 +569,23 @@ class S3(object):

         response = self.send_request(request)

         return response

+    def get_multipart(self, uri):

+        request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?uploads")

+        response = self.send_request(request)

+        return response

+

+    def abort_multipart(self, uri, id):

+        request = self.create_request("OBJECT_DELETE", uri=uri,

+                                      extra = ("?uploadId=%s" % id))

+        response = self.send_request(request)

+        return response

+

+    def list_multipart(self, uri, id):

+        request = self.create_request("OBJECT_GET", uri=uri,

+                                      extra = ("?uploadId=%s" % id))

+        response = self.send_request(request)

+        return response

+

     def get_accesslog(self, uri):

         request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?logging")

         response = self.send_request(request)

@@ -739,6 +790,7 @@ class S3(object):

         if buffer == '':

             file.seek(offset)

         md5_hash = md5()

+

         try:

             while (size_left > 0):

                 #debug("SendFile: Reading up to %d bytes from '%s' - remaining bytes: %s" % (self.config.send_chunk, file.name, size_left))

@@ -746,6 +798,7 @@ class S3(object):

                     data = file.read(min(self.config.send_chunk, size_left))

                 else:

                     data = buffer

+

                 md5_hash.update(data)

                 conn.c.send(data)

                 if self.config.progress_meter:

@@ -754,6 +807,7 @@ class S3(object):

                 if throttle:

                     time.sleep(throttle)

             md5_computed = md5_hash.hexdigest()

+

             response = {}

             http_response = conn.c.getresponse()

             response["status"] = http_response.status

@@ -217,11 +217,11 @@ def mktmpsomething(prefix, randchars, createfunc):

     return dirname

 __all__.append("mktmpsomething")

-def mktmpdir(prefix = "/tmp/tmpdir-", randchars = 10):

+def mktmpdir(prefix = os.getenv('TMP','/tmp') + "/tmpdir-", randchars = 10):

     return mktmpsomething(prefix, randchars, os.mkdir)

 __all__.append("mktmpdir")

-def mktmpfile(prefix = "/tmp/tmpfile-", randchars = 20):

+def mktmpfile(prefix = os.getenv('TMP','/tmp') + "/tmpfile-", randchars = 20):

     createfunc = lambda filename : os.close(os.open(filename, os.O_CREAT | os.O_EXCL))

     return mktmpsomething(prefix, randchars, createfunc)

 __all__.append("mktmpfile")

@@ -383,7 +383,7 @@ def time_to_epoch(t):

         return int(time.mktime(t))

     elif hasattr(t, 'timetuple'):

         # Looks like a datetime object or compatible

-        return int(time.mktime(ex.timetuple()))

+        return int(time.mktime(t.timetuple()))

     elif hasattr(t, 'strftime'):

         # Looks like the object supports standard srftime()

         return int(t.strftime('%s'))

@@ -459,4 +459,22 @@ def getHostnameFromBucket(bucket):

     return Config.Config().host_bucket % { 'bucket' : bucket }

 __all__.append("getHostnameFromBucket")

+

+def calculateChecksum(buffer, mfile, offset, chunk_size, send_chunk):

+    md5_hash = md5()

+    size_left = chunk_size

+    if buffer == '':

+        mfile.seek(offset)

+        while size_left > 0:

+            data = mfile.read(min(send_chunk, size_left))

+            md5_hash.update(data)

+            size_left -= len(data)

+    else:

+        md5_hash.update(buffer)

+

+    return md5_hash.hexdigest()

+

+

+__all__.append("calculateChecksum")

+

 # vim:et:ts=4:sts=4:ai

@@ -339,14 +339,15 @@ def cmd_object_put(args):

         except InvalidFileError, e:

             warning(u"File can not be uploaded: %s" % e)

             continue

-        speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)

-        if not Config().progress_meter:

-            output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %

-                (unicodise(full_name_orig), uri_final, response["size"], response["elapsed"],

-                speed_fmt[0], speed_fmt[1], seq_label))

+        if response is not None:

+            speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)

+            if not Config().progress_meter:

+                output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %

+                       (unicodise(full_name_orig), uri_final, response["size"], response["elapsed"],

+                        speed_fmt[0], speed_fmt[1], seq_label))

         if Config().acl_public:

             output(u"Public URL of the object is: %s" %

-                (uri_final.public_url()))

+                   (uri_final.public_url()))

         if Config().encrypt and full_name != full_name_orig:

             debug(u"Removing temporary encrypted file: %s" % unicodise(full_name))

             os.remove(full_name)

@@ -625,6 +626,11 @@ def cmd_info(args):

                 except KeyError:

                     pass

                 output(u"   MD5 sum:   %s" % md5)

+                if 'x-amz-server-side-encryption' in info['headers']:

+                    output(u"   SSE:       %s" % info['headers']['x-amz-server-side-encryption'])

+                else:

+                    output(u"   SSE:       NONE")		

+

             else:

                 info = s3.bucket_info(uri)

                 output(u"%s (bucket):" % uri.uri())

@@ -751,7 +757,14 @@ def cmd_sync_remote2remote(args):

     seq = 0

     seq = _upload(src_list, seq, src_count + update_count)

     seq = _upload(update_list, seq, src_count + update_count)

-    n_copied, bytes_saved = remote_copy(s3, copy_pairs, destination_base)

+    n_copied, bytes_saved, failed_copy_files = remote_copy(s3, copy_pairs, destination_base)

+

+    #process files not copied

+    output("Process files that was not remote copied")

+    failed_copy_count = len (failed_copy_files) 

+    for key in failed_copy_files:

+        failed_copy_files[key]['target_uri'] = destination_base + key

+    seq = _upload(failed_copy_files, seq, failed_copy_count)

     total_elapsed = time.time() - timestamp_start

     outstr = "Done. Copied %d files in %0.1f seconds, %0.2f files/s" % (seq, total_elapsed, seq/total_elapsed)

@@ -797,6 +810,7 @@ def cmd_sync_remote2local(args):

     info(u"Summary: %d remote files to download, %d local files to delete, %d local files to hardlink" % (remote_count + update_count, local_count, copy_pairs_count))

+    empty_fname_re = re.compile(r'\A\s*\Z')

     def _set_local_filename(remote_list, destination_base):

         if len(remote_list) == 0:

             return

@@ -809,7 +823,12 @@ def cmd_sync_remote2local(args):

             if destination_base[-1] != os.path.sep:

                 destination_base += os.path.sep

             for key in remote_list:

-                local_filename = destination_base + key

+                local_basename = key

+                if empty_fname_re.match(key):

+                    # Objects may exist on S3 with empty names (''), which don't map so well to common filesystems.

+                    local_basename = '__AWS-EMPTY-OBJECT-NAME__'

+                    warning(u"Empty object name on S3 found, saving locally as %s" % (local_basename))

+                local_filename = destination_base + local_basename

                 if os.path.sep != "/":

                     local_filename = os.path.sep.join(local_filename.split("/"))

                 remote_list[key]['local_filename'] = deunicodise(local_filename)

@@ -887,7 +906,10 @@ def cmd_sync_remote2local(args):

                             mtime = attrs.has_key('mtime') and int(attrs['mtime']) or int(time.time())

                             atime = attrs.has_key('atime') and int(attrs['atime']) or int(time.time())

                             os.utime(dst_file, (atime, mtime))

-                        ## FIXME: uid/gid / uname/gname handling comes here! TODO

+                        if attrs.has_key('uid') and attrs.has_key('gid'):

+                            uid = int(attrs['uid'])

+                            gid = int(attrs['gid'])

+                            os.lchown(dst_file,uid,gid)				

                 except OSError, e:

                     try:

                         dst_stream.close()

@@ -986,6 +1008,7 @@ def local_copy(copy_pairs, destination_base):

 def remote_copy(s3, copy_pairs, destination_base):

     saved_bytes = 0

+    failed_copy_list = FileDict()

     for (src_obj, dst1, dst2) in copy_pairs:

         debug(u"Remote Copying from %s to %s" % (dst1, dst2))

         dst1_uri = S3Uri(destination_base + dst1)

@@ -997,8 +1020,9 @@ def remote_copy(s3, copy_pairs, destination_base):

             saved_bytes = saved_bytes + int(info['headers']['content-length'])

             output(u"remote copy: %s -> %s" % (dst1, dst2))

         except:

-            raise

-    return (len(copy_pairs), saved_bytes)

+            warning(u'Unable to remote copy files %s -> %s' % (dst1_uri, dst2_uri))

+            failed_copy_list[dst2] = src_obj    

+    return (len(copy_pairs), saved_bytes, failed_copy_list)

 def _build_attr_header(local_list, src):

     import pwd, grp

@@ -1188,7 +1212,14 @@ def cmd_sync_local2remote(args):

         timestamp_start = time.time()

         n, total_size = _upload(local_list, 0, local_count, total_size)

         n, total_size = _upload(update_list, n, local_count, total_size)

-        n_copies, saved_bytes = remote_copy(s3, copy_pairs, destination_base)

+        n_copies, saved_bytes, failed_copy_files  = remote_copy(s3, copy_pairs, destination_base)

+

+        #upload file that could not be copied

+        output("Process files that was not remote copied")

+        failed_copy_count = len(failed_copy_files)

+        _set_remote_uri(failed_copy_files, destination_base, single_file_local)

+        n, total_size = _upload(failed_copy_files, n, failed_copy_count, total_size)

+

         if cfg.delete_removed and cfg.delete_after:

             _do_deletes(s3, remote_list)

         total_elapsed = time.time() - timestamp_start

@@ -1331,6 +1362,50 @@ def cmd_delpolicy(args):

     output(u"%s: Policy deleted" % uri)

+def cmd_multipart(args):

+    s3 = S3(cfg)

+    uri = S3Uri(args[0])

+

+    #id = ''

+    #if(len(args) > 1): id = args[1]

+

+    response = s3.get_multipart(uri)

+    debug(u"response - %s" % response['status'])

+    output(u"%s" % uri)

+    tree = getTreeFromXml(response['data'])

+    debug(parseNodes(tree))

+    output(u"Initiated\tPath\tId")

+    for mpupload in parseNodes(tree):

+        try:

+            output("%s\t%s\t%s" % (mpupload['Initiated'], "s3://" + uri.bucket() + "/" + mpupload['Key'], mpupload['UploadId']))

+        except KeyError:

+            pass

+

+def cmd_abort_multipart(args):

+    '''{"cmd":"abortmp",   "label":"abort a multipart upload", "param":"s3://BUCKET Id", "func":cmd_abort_multipart, "argc":2},'''

+    s3 = S3(cfg)

+    uri = S3Uri(args[0])

+    id = args[1]

+    response = s3.abort_multipart(uri, id)

+    debug(u"response - %s" % response['status'])

+    output(u"%s" % uri)

+

+def cmd_list_multipart(args):

+    '''{"cmd":"abortmp",   "label":"list a multipart upload", "param":"s3://BUCKET Id", "func":cmd_list_multipart, "argc":2},'''

+    s3 = S3(cfg)

+    uri = S3Uri(args[0])

+    id = args[1]

+

+    response = s3.list_multipart(uri, id)

+    debug(u"response - %s" % response['status'])

+    tree = getTreeFromXml(response['data'])

+    output(u"LastModified\t\t\tPartNumber\tETag\tSize")

+    for mpupload in parseNodes(tree):

+        try:

+            output("%s\t%s\t%s\t%s" % (mpupload['LastModified'], mpupload['PartNumber'], mpupload['ETag'], mpupload['Size']))

+        except:

+            pass

+

 def cmd_accesslog(args):

     s3 = S3(cfg)

     bucket_uri = S3Uri(args.pop())

@@ -1587,6 +1662,8 @@ def run_configure(config_file, args):

                 except Exception, e:

                     error(u"Test failed: %s" % (e))

+                    if e.find('403') != -1:

+                        error(u"Are you sure your keys have ListAllMyBuckets permissions?")

                     val = raw_input("\nRetry configuration? [Y/n] ")

                     if val.lower().startswith("y") or val == "":

                         continue

@@ -1685,6 +1762,11 @@ def get_commands_list():

     {"cmd":"setpolicy", "label":"Modify Bucket Policy", "param":"FILE s3://BUCKET", "func":cmd_setpolicy, "argc":2},

     {"cmd":"delpolicy", "label":"Delete Bucket Policy", "param":"s3://BUCKET", "func":cmd_delpolicy, "argc":1},

+    {"cmd":"multipart", "label":"show multipart uploads", "param":"s3://BUCKET [Id]", "func":cmd_multipart, "argc":1},

+    {"cmd":"abortmp",   "label":"abort a multipart upload", "param":"s3://BUCKET/OBJECT Id", "func":cmd_abort_multipart, "argc":2},

+

+    {"cmd":"listmp",    "label":"list parts of a multipart upload", "param":"s3://BUCKET/OBJECT Id", "func":cmd_list_multipart, "argc":2},

+

     {"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1},

     {"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1},

     {"cmd":"signurl", "label":"Sign an S3 URL to provide limited public access with expiry", "param":"s3://BUCKET/OBJECT expiry_epoch", "func":cmd_signurl, "argc":2},

@@ -1813,7 +1895,7 @@ def main():

     optparser.set_defaults(config = config_file)

     optparser.set_defaults(verbosity = default_verbosity)

-    optparser.add_option(      "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool. Optionally use as '--configure s3://come-bucket' to test access to a specific bucket instead of attempting to list them all.")

+    optparser.add_option(      "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool. Optionally use as '--configure s3://some-bucket' to test access to a specific bucket instead of attempting to list them all.")

     optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default")

     optparser.add_option(      "--dump-config", dest="dump_config", action="store_true", help="Dump current configuration after parsing config files and command line options and exit.")

     optparser.add_option(      "--access_key", dest="access_key", help="AWS Access Key")

@@ -1825,6 +1907,8 @@ def main():

     optparser.add_option(      "--no-encrypt", dest="encrypt", action="store_false", help="Don't encrypt files.")

     optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.")

     optparser.add_option(      "--continue", dest="get_continue", action="store_true", help="Continue getting a partially downloaded file (only for [get] command).")

+    optparser.add_option(      "--continue-put", dest="put_continue", action="store_true", help="Continue uploading partially uploaded files or multipart upload parts.  Restarts/parts files that don't have matching size and md5.  Skips files/parts that do.  Note: md5sum checks are not always sufficient to check (part) file equality.  Enable this at your own risk.")

+    optparser.add_option(      "--upload-id", dest="upload_id", help="UploadId for Multipart Upload, in case you want continue an existing upload (equivalent to --continue-put) and there are multiple partial uploads.  Use s3cmd multipart [URI] to see what UploadIds are associated with the given URI.")

     optparser.add_option(      "--skip-existing", dest="skip_existing", action="store_true", help="Skip over files that exist at the destination (only for [get] and [sync] commands).")

     optparser.add_option("-r", "--recursive", dest="recursive", action="store_true", help="Recursive upload, download or removal.")

     optparser.add_option(      "--check-md5", dest="check_md5", action="store_true", help="Check MD5 sums when comparing files for [sync]. (default)")

@@ -1860,14 +1944,18 @@ def main():

     optparser.add_option(      "--access-logging-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI) (for [cfmodify] and [accesslog] commands)")

     optparser.add_option(      "--no-access-logging", dest="log_target_prefix", action="store_false", help="Disable access logging (for [cfmodify] and [accesslog] commands)")

-    optparser.add_option(      "--default-mime-type", dest="default_mime_type", action="store_true", help="Default MIME-type for stored objects. Application default is binary/octet-stream.")

+    optparser.add_option(      "--default-mime-type", dest="default_mime_type", type="mimetype", action="store", help="Default MIME-type for stored objects. Application default is binary/octet-stream.")

     optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension or mime magic. Fall back to default MIME-Type as specified by --default-mime-type option")

     optparser.add_option(      "--no-guess-mime-type", dest="guess_mime_type", action="store_false", help="Don't guess MIME-type and use the default type instead.")

+    optparser.add_option(      "--no-mime-magic", dest="use_mime_magic", action="store_false", help="Don't use mime magic when guessing MIME-type.")

     optparser.add_option("-m", "--mime-type", dest="mime_type", type="mimetype", metavar="MIME/TYPE", help="Force MIME-type. Override both --default-mime-type and --guess-mime-type.")

     optparser.add_option(      "--add-header", dest="add_header", action="append", metavar="NAME:VALUE", help="Add a given HTTP header to the upload request. Can be used multiple times. For instance set 'Expires' or 'Cache-Control' headers (or both) using this options if you like.")

+    optparser.add_option(      "--server-side-encryption", dest="server_side_encryption", action="store_true", help="Specifies that server-side encryption will be used when putting objects.")

+

     optparser.add_option(      "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % preferred_encoding)

+    optparser.add_option(      "--disable-content-encoding", dest="add_content_encoding", action="store_false", help="Don't include a Content-encoding header to the the uploaded objects")

     optparser.add_option(      "--add-encoding-exts", dest="add_encoding_exts", metavar="EXTENSIONs", help="Add encoding to these comma delimited extensions i.e. (css,js,html) when uploading to S3 )")

     optparser.add_option(      "--verbatim", dest="urlencoding_mode", action="store_const", const="verbatim", help="Use the S3 name as given on the command line. No pre-processing, encoding, etc. Use with caution!")

@@ -1906,7 +1994,7 @@ def main():

         '"buckets" and uploading, downloading and removing '+

         '"objects" from these buckets.')

     optparser.epilog = format_commands(optparser.get_prog_name(), commands_list)

-    optparser.epilog += ("\nFor more informations see the progect homepage:\n%s\n" % PkgInfo.url)

+    optparser.epilog += ("\nFor more information see the project homepage:\n%s\n" % PkgInfo.url)

     optparser.epilog += ("\nConsider a donation if you have found s3cmd useful:\n%s/donate\n" % PkgInfo.url)

     (options, args) = optparser.parse_args()

@@ -2017,6 +2105,14 @@ def main():

     if cfg.multipart_chunk_size_mb > MultiPartUpload.MAX_CHUNK_SIZE_MB:

         raise ParameterError("Chunk size %d MB is too large, must be <= %d MB. Please adjust --multipart-chunk-size-mb" % (cfg.multipart_chunk_size_mb, MultiPartUpload.MAX_CHUNK_SIZE_MB))

+    ## If an UploadId was provided, set put_continue True

+    if options.upload_id is not None:

+        cfg.upload_id = options.upload_id

+        cfg.put_continue = True

+

+    if cfg.upload_id and not cfg.multipart_chunk_size_mb:

+        raise ParameterError("Must have --multipart-chunk-size-mb if using --put-continue or --upload-id")

+

     ## CloudFront's cf_enable and Config's enable share the same --enable switch

     options.cf_enable = options.enable

@@ -1,4 +1,3 @@

-

 .TH s3cmd 1

 .SH NAME

 s3cmd \- tool for managing Amazon S3 storage space and Amazon CloudFront content delivery network

@@ -66,7 +65,7 @@ Delete Bucket Policy

 s3cmd \fBaccesslog\fR \fIs3://BUCKET\fR

 Enable/disable bucket access logging

 .TP

-s3cmd \fBsign\fR \fISTRING-TO-SIGN\fR

+s3cmd \fBsign\fR \fISTRING\-TO\-SIGN\fR

 Sign arbitrary string using the secret key

 .TP

 s3cmd \fBsignurl\fR \fIs3://BUCKET/OBJECT expiry_epoch\fR

@@ -79,13 +78,13 @@ Fix invalid file names in a bucket

 .PP

 Commands for static WebSites configuration

 .TP

-s3cmd \fBws-create\fR \fIs3://BUCKET\fR

+s3cmd \fBws\-create\fR \fIs3://BUCKET\fR

 Create Website from bucket

 .TP

-s3cmd \fBws-delete\fR \fIs3://BUCKET\fR

+s3cmd \fBws\-delete\fR \fIs3://BUCKET\fR

 Delete Website

 .TP

-s3cmd \fBws-info\fR \fIs3://BUCKET\fR

+s3cmd \fBws\-info\fR \fIs3://BUCKET\fR

 Info about Website

@@ -124,7 +123,7 @@ changes you like.

 show this help message and exit

 .TP

 \fB\-\-configure\fR

-Invoke interactive (re)configuration tool. Optionally use as '--configure s3://come-bucket' to test access to a specific bucket instead of attempting to list them all.

+Invoke interactive (re)configuration tool. Optionally use as '\-\-configure s3://come\-bucket' to test access to a specific bucket instead of attempting to list them all.

 .TP

 \fB\-c\fR FILE, \fB\-\-config\fR=FILE

 Config file name. Defaults to /home/mludvig/.s3cfg

@@ -208,32 +207,32 @@ Don't store FS attributes

 Filenames and paths matching GLOB will be excluded from sync

 .TP

 \fB\-\-exclude\-from\fR=FILE

-Read --exclude GLOBs from FILE

+Read \-\-exclude GLOBs from FILE

 .TP

 \fB\-\-rexclude\fR=REGEXP

 Filenames and paths matching REGEXP (regular expression) will be excluded from sync

 .TP

 \fB\-\-rexclude\-from\fR=FILE

-Read --rexclude REGEXPs from FILE

+Read \-\-rexclude REGEXPs from FILE

 .TP

 \fB\-\-include\fR=GLOB

-Filenames and paths matching GLOB will be included even if previously excluded by one of --(r)exclude(-from) patterns

+Filenames and paths matching GLOB will be included even if previously excluded by one of \-\-(r)exclude(\-from) patterns

 .TP

 \fB\-\-include\-from\fR=FILE

-Read --include GLOBs from FILE

+Read \-\-include GLOBs from FILE

 .TP

 \fB\-\-rinclude\fR=REGEXP

-Same as --include but uses REGEXP (regular expression) instead of GLOB

+Same as \-\-include but uses REGEXP (regular expression) instead of GLOB

 .TP

 \fB\-\-rinclude\-from\fR=FILE

-Read --rinclude REGEXPs from FILE

+Read \-\-rinclude REGEXPs from FILE

 .TP

 \fB\-\-files\-from\fR=FILE

-Read list of source-file names from FILE. Use - to read from stdin.

+Read list of source-file names from FILE. Use \- to read from stdin.

 May be repeated.

 .TP

 \fB\-\-bucket\-location\fR=BUCKET_LOCATION

-Datacentre to create bucket in. As of now the datacenters are: US (default), EU, ap-northeast-1, ap-southeast-1, sa-east-1, us-west-1 and us-west-2

+Datacentre to create bucket in. As of now the datacenters are: US (default), EU, ap\-northeast\-1, ap\-southeast\-1, sa\-east\-1, us\-west\-1 and us\-west\-2

 .TP

 \fB\-\-reduced\-redundancy\fR, \fB\-\-rr\fR

 Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]

@@ -245,22 +244,28 @@ Target prefix for access logs (S3 URI) (for [cfmodify] and [accesslog] commands)

 Disable access logging (for [cfmodify] and [accesslog] commands)

 .TP

 \fB\-\-default\-mime\-type\fR

-Default MIME-type for stored objects. Application default is binary/octet-stream.

+Default MIME-type for stored objects. Application default is binary/octet\-stream.

 .TP

 \fB\-M\fR, \fB\-\-guess\-mime\-type\fR

-Guess MIME-type of files by their extension or mime magic. Fall back to default MIME-Type as specified by \fB--default-mime-type\fR option

+Guess MIME-type of files by their extension or mime magic. Fall back to default MIME-type as specified by \fB\-\-default\-mime\-type\fR option

 .TP

 \fB\-\-no\-guess\-mime\-type\fR

 Don't guess MIME-type and use the default type instead.

 .TP

+\fB\-\-no\-mime\-magic\fR

+Don't use mime magic when guessing MIME-type.

+.TP

 \fB\-m\fR MIME/TYPE, \fB\-\-mime\-type\fR=MIME/TYPE

-Force MIME-type. Override both \fB--default-mime-type\fR and \fB--guess-mime-type\fR.

+Force MIME-type. Override both \fB\-\-default\-mime\-type\fR and \fB\-\-guess\-mime\-type\fR.

 .TP

 \fB\-\-add\-header\fR=NAME:VALUE

-Add a given HTTP header to the upload request. Can be used multiple times. For instance set 'Expires' or 'Cache-Control' headers (or both) using this options if you like.

+Add a given HTTP header to the upload request. Can be used multiple times. For instance set 'Expires' or 'Cache\-Control' headers (or both) using this options if you like.

 .TP

 \fB\-\-encoding\fR=ENCODING

-Override autodetected terminal and filesystem encoding (character set). Autodetected: UTF-8

+Override autodetected terminal and filesystem encoding (character set). Autodetected: UTF\-8

+.TP

+\fB\-\-disable\-content\-encoding\fR

+Don't include a Content-encoding header to the the uploaded objects. Default: Off

 .TP

 \fB\-\-add\-encoding\-exts\fR=EXTENSIONs

 Add encoding to these comma delimited extensions i.e. (css,js,html) when uploading to S3 )

@@ -269,11 +274,11 @@ Add encoding to these comma delimited extensions i.e. (css,js,html) when uploadi

 Use the S3 name as given on the command line. No pre-processing, encoding, etc. Use with caution!

 .TP

 \fB\-\-disable\-multipart\fR

-Disable multipart upload on files bigger than --multipart-chunk-size-mb

+Disable multipart upload on files bigger than \-\-multipart\-chunk\-size\-mb

 .TP

 \fB\-\-multipart\-chunk\-size\-mb\fR=SIZE

 Size of each chunk of a multipart upload. Files bigger than SIZE are automatically uploaded as multithreaded-multipart, smaller files are uploaded using the traditional method. SIZE is in Mega-Bytes,

-default chunk size is noneMB, minimum allowed chunk size is 5MB, maximum is 5GB.

+default chunk size is 15MB, minimum allowed chunk size is 5MB, maximum is 5GB.

 .TP

 \fB\-\-list\-md5\fR