@@ -1,3 +1,2 @@

 include INSTALL README NEWS

 include s3cmd.1

-include S3/*.py

new file mode 100644

@@ -0,0 +1,4 @@

+SHELL := /bin/bash

+

+release:

+	python setup.py register sdist upload

@@ -133,7 +133,7 @@ class DistributionConfig(object):

     ##  </Logging>

     ## </DistributionConfig>

-    EMPTY_CONFIG = "<DistributionConfig><Origin/><CallerReference/><Enabled>true</Enabled></DistributionConfig>"

+    EMPTY_CONFIG = "<DistributionConfig><S3Origin><DNSName/></S3Origin><CallerReference/><Enabled>true</Enabled></DistributionConfig>"

     xmlns = "http://cloudfront.amazonaws.com/doc/%(api_ver)s/" % { 'api_ver' : cloudfront_api_version }

     def __init__(self, xml = None, tree = None):

         if xml is None:

@@ -174,7 +174,8 @@ class DistributionConfig(object):

         tree.attrib['xmlns'] = DistributionConfig.xmlns

         ## Retain the order of the following calls!

-        appendXmlTextNode("Origin", self.info['Origin'], tree)

+        s3org = appendXmlTextNode("S3Origin", '', tree)

+        appendXmlTextNode("DNSName", self.info['S3Origin']['DNSName'], s3org)

         appendXmlTextNode("CallerReference", self.info['CallerReference'], tree)

         for cname in self.info['CNAME']:

             appendXmlTextNode("CNAME", cname.lower(), tree)

@@ -281,7 +282,7 @@ class InvalidationBatch(object):

         tree = ET.Element("InvalidationBatch")

         for path in self.paths:

-            if path[0] != "/":

+            if len(path) < 1 or path[0] != "/":

                 path = "/" + path

             appendXmlTextNode("Path", path, tree)

         appendXmlTextNode("CallerReference", self.reference, tree)

@@ -322,7 +323,7 @@ class CloudFront(object):

     def CreateDistribution(self, uri, cnames_add = [], comment = None, logging = None, default_root_object = None):

         dist_config = DistributionConfig()

         dist_config.info['Enabled'] = True

-        dist_config.info['Origin'] = uri.host_name()

+        dist_config.info['S3Origin']['DNSName'] = uri.host_name()

         dist_config.info['CallerReference'] = str(uri)

         dist_config.info['DefaultRootObject'] = default_root_object

         if comment == None:

@@ -423,7 +424,23 @@ class CloudFront(object):

                                      body = request_body, headers = headers)

         return response

-    def InvalidateObjects(self, uri, paths):

+    def InvalidateObjects(self, uri, paths, default_index_file, invalidate_default_index_on_cf, invalidate_default_index_root_on_cf):

+        # joseprio: if the user doesn't want to invalidate the default index

+        # path, or if the user wants to invalidate the root of the default

+        # index, we need to process those paths

+        if default_index_file is not None and (not invalidate_default_index_on_cf or invalidate_default_index_root_on_cf):

+            new_paths = []

+            default_index_suffix = '/' + default_index_file

+            for path in paths:

+                if path.endswith(default_index_suffix) or path ==  default_index_file:

+                    if invalidate_default_index_on_cf:

+                        new_paths.append(path)

+                    if invalidate_default_index_root_on_cf:

+                        new_paths.append(path[:-len(default_index_file)])

+                else:

+                    new_paths.append(path)

+            paths = new_paths

+        

         # uri could be either cf:// or s3:// uri

         cfuri = self.get_dist_name_for_bucket(uri)

         if len(paths) > 999:

@@ -671,7 +688,7 @@ class Cmd(object):

             d = response['distribution']

             dc = d.info['DistributionConfig']

             output("Distribution created:")

-            pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin']))

+            pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['S3Origin']['DNSName']))

             pretty_output("DistId", d.uri())

             pretty_output("DomainName", d.info['DomainName'])

             pretty_output("CNAMEs", ", ".join(dc.info['CNAME']))

@@ -713,7 +730,7 @@ class Cmd(object):

         response = cf.GetDistInfo(cfuri)

         d = response['distribution']

         dc = d.info['DistributionConfig']

-        pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin']))

+        pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['S3Origin']['DNSName']))

         pretty_output("DistId", d.uri())

         pretty_output("DomainName", d.info['DomainName'])

         pretty_output("Status", d.info['Status'])

@@ -40,6 +40,7 @@ class Config(object):

     proxy_port = 3128

     encrypt = False

     dry_run = False

+    add_encoding_exts = ""

     preserve_attrs = True

     preserve_attrs_list = [

         'uname',    # Verbose owner Name (e.g. 'root')

@@ -50,10 +51,14 @@ class Config(object):

         'mtime',    # Modification timestamp

         'ctime',    # Creation timestamp

         'mode',     # File mode (e.g. rwxr-xr-x = 755)

+        'md5',      # File MD5 (if known)

         #'acl',     # Full ACL (not yet supported)

     ]

     delete_removed = False

+    delete_after = False

+    delete_after_fetch = False

     _doc['delete_removed'] = "[sync] Remove remote S3 objects when local file has been deleted"

+    delay_updates = False

     gpg_passphrase = ""

     gpg_command = ""

     gpg_encrypt = "%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"

@@ -80,9 +85,15 @@ class Config(object):

     follow_symlinks = False

     socket_timeout = 300

     invalidate_on_cf = False

+    # joseprio: new flags for default index invalidation

+    invalidate_default_index_on_cf = False

+    invalidate_default_index_root_on_cf = True

     website_index = "index.html"

     website_error = ""

     website_endpoint = "http://%(bucket)s.s3-website-%(location)s.amazonaws.com/"

+    additional_destinations = []

+    cache_file = ""

+    add_headers = ""

     ## Creating a singleton

     def __new__(self, configfile = None):

@@ -112,6 +123,12 @@ class Config(object):

         cp = ConfigParser(configfile)

         for option in self.option_list():

             self.update_option(option, cp.get(option))

+

+        if cp.get('add_headers'):

+            for option in cp.get('add_headers').split(","):

+                (key, value) = option.split(':')

+                self.extra_headers[key.replace('_', '-').strip()] = value.strip()

+

         self._parsed_files.append(configfile)

     def dump_config(self, stream):

@@ -9,13 +9,15 @@ from S3Uri import S3Uri

 from SortedDict import SortedDict

 from Utils import *

 from Exceptions import ParameterError

+from HashCache import HashCache

 from logging import debug, info, warning, error

 import os

 import glob

+import copy

-__all__ = ["fetch_local_list", "fetch_remote_list", "compare_filelists", "filter_exclude_include"]

+__all__ = ["fetch_local_list", "fetch_remote_list", "compare_filelists", "filter_exclude_include", "parse_attrs_header"]

 def _fswalk_follow_symlinks(path):

         '''

@@ -26,13 +28,15 @@ def _fswalk_follow_symlinks(path):

         assert os.path.isdir(path) # only designed for directory argument

         walkdirs = [path]

         for dirpath, dirnames, filenames in os.walk(path):

+ 		handle_exclude_include_walk(dirpath, dirnames, [])

                 for dirname in dirnames:

                         current = os.path.join(dirpath, dirname)

                         if os.path.islink(current):

 				walkdirs.append(current)

         for walkdir in walkdirs:

-                for value in os.walk(walkdir):

-                        yield value

+		for dirpath, dirnames, filenames in os.walk(walkdir):

+			handle_exclude_include_walk(dirpath, dirnames, [])

+                        yield (dirpath, dirnames, filenames)

 def _fswalk(path, follow_symlinks):

         '''

@@ -43,8 +47,10 @@ def _fswalk(path, follow_symlinks):

         follow_symlinks (bool) indicates whether to descend into symbolically linked directories

         '''

         if follow_symlinks:

-                return _fswalk_follow_symlinks(path)

-        return os.walk(path)

+                yield _fswalk_follow_symlinks(path)

+	for dirpath, dirnames, filenames in os.walk(path):

+		handle_exclude_include_walk(dirpath, dirnames, filenames)

+		yield (dirpath, dirnames, filenames)

 def filter_exclude_include(src_list):

     info(u"Applying --exclude/--include")

@@ -71,13 +77,71 @@ def filter_exclude_include(src_list):

             exclude_list[file] = src_list[file]

             del(src_list[file])

             continue

-        else:

-            debug(u"PASS: %s" % (file))

+

     return src_list, exclude_list

+def handle_exclude_include_walk(root, dirs, files):

+    cfg = Config()

+    copydirs = copy.copy(dirs)

+    copyfiles = copy.copy(files)

+

+    # exclude dir matches in the current directory

+    # this prevents us from recursing down trees we know we want to ignore

+    for x in copydirs:

+        d = os.path.join(root, x, '')

+        excluded = False

+        for r in cfg.exclude:

+            if r.search(d):

+                excluded = True

+                debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r]))

+                break

+        if excluded:

+            ## No need to check for --include if not excluded

+            for r in cfg.include:

+                if r.search(d):

+                    excluded = False

+                    debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r]))

+                    break

+        if excluded:

+            ## Still excluded - ok, action it

+	    dirs.remove(x)

+            continue

+

+    # exclude file matches in the current directory

+    for x in copyfiles:

+        file = os.path.join(root, x)

+        excluded = False

+        for r in cfg.exclude:

+            if r.search(file):

+                excluded = True

+                debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r]))

+                break

+        if excluded:

+            ## No need to check for --include if not excluded

+            for r in cfg.include:

+                if r.search(file):

+                    excluded = False

+                    debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r]))

+                    break

+        if excluded:

+            ## Still excluded - ok, action it

+            debug(u"EXCLUDE: %s" % file)

+            files.remove(x)

+            continue

+

 def fetch_local_list(args, recursive = None):

-    def _get_filelist_local(local_uri):

+    def _get_filelist_local(loc_list, local_uri, cache):

         info(u"Compiling list of local files...")

+        

+        if deunicodise(local_uri.basename()) == "-":

+            loc_list = SortedDict(ignore_case = False)

+            loc_list["-"] = {

+                'full_name_unicode' : '-',

+                'full_name' : '-',

+                'size' : -1,

+                'mtime' : -1,

+            }

+            return loc_list, True

         if local_uri.isdir():

             local_base = deunicodise(local_uri.basename())

             local_path = deunicodise(local_uri.path())

@@ -88,7 +152,6 @@ def fetch_local_list(args, recursive = None):

             local_path = deunicodise(local_uri.dirname())

             filelist = [( local_path, [], [deunicodise(local_uri.basename())] )]

             single_file = True

-        loc_list = SortedDict(ignore_case = False)

         for root, dirs, files in filelist:

             rel_root = root.replace(local_path, local_base, 1)

             for f in files:

@@ -112,11 +175,41 @@ def fetch_local_list(args, recursive = None):

                     'full_name' : full_name,

                     'size' : sr.st_size,

                     'mtime' : sr.st_mtime,

+		    'dev'   : sr.st_dev,

+		    'inode' : sr.st_ino,

+		    'uid' : sr.st_uid,

+		    'gid' : sr.st_gid,

+		    'sr': sr # save it all, may need it in preserve_attrs_list

                     ## TODO: Possibly more to save here...

                 }

+		if 'md5' in cfg.sync_checks:

+                    md5 = cache.md5(sr.st_dev, sr.st_ino, sr.st_mtime, sr.st_size)

+		    if md5 is None:

+			    try:

+                                md5 = loc_list.get_md5(relative_file) # this does the file I/O

+			    except IOError:

+                                continue

+			    cache.add(sr.st_dev, sr.st_ino, sr.st_mtime, sr.st_size, md5)

+		    loc_list.record_hardlink(relative_file, sr.st_dev, sr.st_ino, md5)

         return loc_list, single_file

+    def _maintain_cache(cache, local_list):

+        if cfg.cache_file:

+            cache.mark_all_for_purge()

+	    for i in local_list.keys():

+                cache.unmark_for_purge(local_list[i]['dev'], local_list[i]['inode'], local_list[i]['mtime'], local_list[i]['size'])

+            cache.purge()

+	    cache.save(cfg.cache_file)

+

     cfg = Config()

+

+    cache = HashCache()

+    if cfg.cache_file:

+        try:

+            cache.load(cfg.cache_file)

+	except IOError:

+	    info(u"No cache file found, creating it.")

+    

     local_uris = []

     local_list = SortedDict(ignore_case = False)

     single_file = False

@@ -136,8 +229,7 @@ def fetch_local_list(args, recursive = None):

         local_uris.append(uri)

     for uri in local_uris:

-        list_for_uri, single_file = _get_filelist_local(uri)

-        local_list.update(list_for_uri)

+        list_for_uri, single_file = _get_filelist_local(local_list, uri, cache)

     ## Single file is True if and only if the user

     ## specified one local URI and that URI represents

@@ -147,6 +239,8 @@ def fetch_local_list(args, recursive = None):

     if len(local_list) > 1:

         single_file = False

+    _maintain_cache(cache, local_list)

+

     return local_list, single_file

 def fetch_remote_list(args, require_attribs = False, recursive = None):

@@ -196,7 +290,11 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):

                 'object_key' : object['Key'],

                 'object_uri_str' : object_uri_str,

                 'base_uri' : remote_uri,

+		'dev' : None,

+		'inode' : None,

             }

+	    md5 = object['ETag'][1:-1]

+	    rem_list.record_md5(key, md5)

             if break_now:

                 break

         return rem_list

@@ -222,6 +320,7 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):

             objectlist = _get_filelist_remote(uri)

             for key in objectlist:

                 remote_list[key] = objectlist[key]

+		remote_list.record_md5(key, objectlist.get_md5(key))

     else:

         for uri in remote_uris:

             uri_str = str(uri)

@@ -259,82 +358,146 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):

                     'md5': response['headers']['etag'].strip('"\''),

                     'timestamp' : dateRFC822toUnix(response['headers']['date'])

                     })

+		    # get md5 from header if it's present.  We would have set that during upload

+		    if response['headers'].has_key('x-amz-meta-s3cmd-attrs'):

+                        attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])

+			if attrs.has_key('md5'):

+                            remote_item.update({'md5': attrs['md5']})

+		    

                 remote_list[key] = remote_item

     return remote_list

-def compare_filelists(src_list, dst_list, src_remote, dst_remote):

+def parse_attrs_header(attrs_header):

+    attrs = {}

+    for attr in attrs_header.split("/"):

+        key, val = attr.split(":")

+	attrs[key] = val

+    return attrs

+

+

+def compare_filelists(src_list, dst_list, src_remote, dst_remote, delay_updates = False):

     def __direction_str(is_remote):

         return is_remote and "remote" or "local"

-    # We don't support local->local sync, use 'rsync' or something like that instead ;-)

+    def _compare(src_list, dst_lst, src_remote, dst_remote, file):

+        """Return True if src_list[file] matches dst_list[file], else False"""

+        attribs_match = True

+	if not (src_list.has_key(file) and dst_list.has_key(file)):

+            info(u"file does not exist in one side or the other: src_list=%s, dst_list=%s" % (src_list.has_key(file), dst_list.has_key(file)))

+            return False

+

+        ## check size first

+	if 'size' in cfg.sync_checks and dst_list[file]['size'] != src_list[file]['size']:

+            debug(u"xfer: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size']))

+	    attribs_match = False

+

+        ## check md5

+        compare_md5 = 'md5' in cfg.sync_checks

+        # Multipart-uploaded files don't have a valid md5 sum - it ends with "...-nn"

+        if compare_md5:

+            if (src_remote == True and src_list[file]['md5'].find("-") >= 0) or (dst_remote == True and dst_list[file]['md5'].find("-") >= 0):

+                compare_md5 = False

+                info(u"disabled md5 check for %s" % file)

+        if attribs_match and compare_md5:

+            try:

+                src_md5 = src_list.get_md5(file)

+                dst_md5 = dst_list.get_md5(file)

+	    except (IOError,OSError), e:

+                # md5 sum verification failed - ignore that file altogether

+                debug(u"IGNR: %s (disappeared)" % (file))

+                warning(u"%s: file disappeared, ignoring." % (file))

+		raise

+

+            if src_md5 != dst_md5:

+                ## checksums are different.

+                attribs_match = False

+                debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5))

+

+        return attribs_match

+

+    # we don't support local->local sync, use 'rsync' or something like that instead ;-)

     assert(not(src_remote == False and dst_remote == False))

     info(u"Verifying attributes...")

     cfg = Config()

-    exists_list = SortedDict(ignore_case = False)

+    ## Items left on src_list will be transferred

+    ## Items left on update_list will be transferred after src_list

+    ## Items left on copy_pairs will be copied from dst1 to dst2

+    update_list = SortedDict(ignore_case = False)

+    ## Items left on dst_list will be deleted

+    copy_pairs = []

+

     debug("Comparing filelists (direction: %s -> %s)" % (__direction_str(src_remote), __direction_str(dst_remote)))

-    debug("src_list.keys: %s" % src_list.keys())

-    debug("dst_list.keys: %s" % dst_list.keys())

-    for file in src_list.keys():

-        debug(u"CHECK: %s" % file)

-        if dst_list.has_key(file):

+    for relative_file in src_list.keys():

+        debug(u"CHECK: %s" % (relative_file))

+

+        if dst_list.has_key(relative_file):

             ## Was --skip-existing requested?

-            if cfg.skip_existing:

-                debug(u"IGNR: %s (used --skip-existing)" % (file))

-                exists_list[file] = src_list[file]

-                del(src_list[file])

-                ## Remove from destination-list, all that is left there will be deleted

-                del(dst_list[file])

+	    if cfg.skip_existing:

+	        debug(u"IGNR: %s (used --skip-existing)" % (relative_file))

+	        del(src_list[relative_file])

+	        del(dst_list[relative_file])

+	        continue

+

+	    try:

+                compare_result = _compare(src_list, dst_list, src_remote, dst_remote, relative_file)

+	    except (IOError,OSError), e:

+	        del(src_list[relative_file])

+		del(dst_list[relative_file])

                 continue

-            attribs_match = True

-            ## Check size first

-            if 'size' in cfg.sync_checks and dst_list[file]['size'] != src_list[file]['size']:

-                debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size']))

-                attribs_match = False

+	    if compare_result:

+	        debug(u"IGNR: %s (transfer not needed)" % relative_file)

+	        del(src_list[relative_file])

+		del(dst_list[relative_file])

-            ## Check MD5

-            compare_md5 = 'md5' in cfg.sync_checks

-            # Multipart-uploaded files don't have a valid MD5 sum - it ends with "...-NN"

-            if compare_md5:

-                if (src_remote == True and src_list[file]['md5'].find("-") >= 0) or (dst_remote == True and dst_list[file]['md5'].find("-") >= 0):

-                    compare_md5 = False

-                    info(u"Disabled MD5 check for %s" % file)

-            if attribs_match and compare_md5:

+	    else:

+                # look for matching file in src

                 try:

-                    if src_remote == False and dst_remote == True:

-                        src_md5 = hash_file_md5(src_list[file]['full_name'])

-                        dst_md5 = dst_list[file]['md5']

-                    elif src_remote == True and dst_remote == False:

-                        src_md5 = src_list[file]['md5']

-                        dst_md5 = hash_file_md5(dst_list[file]['full_name'])

-                    elif src_remote == True and dst_remote == True:

-                        src_md5 = src_list[file]['md5']

-                        dst_md5 = dst_list[file]['md5']

-                except (IOError,OSError), e:

-                    # MD5 sum verification failed - ignore that file altogether

-                    debug(u"IGNR: %s (disappeared)" % (file))

-                    warning(u"%s: file disappeared, ignoring." % (file))

-                    del(src_list[file])

-                    del(dst_list[file])

-                    continue

-

-                if src_md5 != dst_md5:

-                    ## Checksums are different.

-                    attribs_match = False

-                    debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5))

-

-            if attribs_match:

-                ## Remove from source-list, all that is left there will be transferred

-                debug(u"IGNR: %s (transfer not needed)" % file)

-                exists_list[file] = src_list[file]

-                del(src_list[file])

-

-            ## Remove from destination-list, all that is left there will be deleted

-            del(dst_list[file])

-

-    return src_list, dst_list, exists_list

+                    md5 = src_list.get_md5(relative_file)

+		except IOError:

+                    md5 = None

+		if md5 is not None and dst_list.by_md5.has_key(md5):

+                    # Found one, we want to copy

+                    dst1 = list(dst_list.by_md5[md5])[0]

+                    debug(u"REMOTE COPY src: %s -> %s" % (dst1, relative_file))

+		    copy_pairs.append((src_list[relative_file], dst1, relative_file))

+		    del(src_list[relative_file])

+		    del(dst_list[relative_file])

+                else:

+		    # record that we will get this file transferred to us (before all the copies), so if we come across it later again,

+		    # we can copy from _this_ copy (e.g. we only upload it once, and copy thereafter).

+		    dst_list.record_md5(relative_file, md5) 

+		    update_list[relative_file] = src_list[relative_file]

+		    del src_list[relative_file]

+		    del dst_list[relative_file]

+

+	else:

+            # dst doesn't have this file

+            # look for matching file elsewhere in dst

+            try:

+                md5 = src_list.get_md5(relative_file)

+            except IOError:

+               md5 = None

+	    dst1 = dst_list.find_md5_one(md5)

+	    if dst1 is not None:

+                # Found one, we want to copy

+                debug(u"REMOTE COPY dst: %s -> %s" % (dst1, relative_file))

+		copy_pairs.append((src_list[relative_file], dst1, relative_file))

+		del(src_list[relative_file])

+	    else:

+                # we don't have this file, and we don't have a copy of this file elsewhere.  Get it.

+	        # record that we will get this file transferred to us (before all the copies), so if we come across it later again,

+	        # we can copy from _this_ copy (e.g. we only upload it once, and copy thereafter).

+	        dst_list.record_md5(relative_file, md5) 

+

+    for f in dst_list.keys():

+        if not src_list.has_key(f) and not update_list.has_key(f):

+            # leave only those not on src_list + update_list

+            del dst_list[f]

+

+    return src_list, dst_list, update_list, copy_pairs

 # vim:et:ts=4:sts=4:ai

new file mode 100644

@@ -0,0 +1,53 @@

+import cPickle as pickle

+

+class HashCache(object):

+    def __init__(self):

+        self.inodes = dict()

+    

+    def add(self, dev, inode, mtime, size, md5):

+        if dev not in self.inodes:

+            self.inodes[dev] = dict()

+        if inode not in self.inodes[dev]:

+            self.inodes[dev][inode] = dict()

+        self.inodes[dev][inode][mtime] = dict(md5=md5, size=size)

+

+    def md5(self, dev, inode, mtime, size):

+        try:

+            d = self.inodes[dev][inode][mtime]

+            if d['size'] != size:

+                return None

+        except:

+            return None

+        return d['md5']

+

+    def mark_all_for_purge(self):

+        for d in self.inodes.keys():

+            for i in self.inodes[d].keys():

+                for c in self.inodes[d][i].keys():

+                    self.inodes[d][i][c]['purge'] = True

+

+    def unmark_for_purge(self, dev, inode, mtime, size):

+        d = self.inodes[dev][inode][mtime]

+        if d['size'] == size and 'purge' in d:

+            del self.inodes[dev][inode][mtime]['purge']

+

+    def purge(self):

+        for d in self.inodes.keys():

+            for i in self.inodes[d].keys():

+                for m in self.inodes[d][i].keys():

+                    if 'purge' in self.inodes[d][i][m]:

+                        del self.inodes[d][i]

+                        break

+    

+    def save(self, f):

+        d = dict(inodes=self.inodes, version=1)

+        f = open(f, 'w')

+        p = pickle.dump(d, f)

+        f.close()

+

+    def load(self, f):

+        f = open(f, 'r')

+        d = pickle.load(f)

+        f.close()

+        if d.get('version') == 1 and 'inodes' in d:

+            self.inodes = d['inodes']

@@ -42,32 +42,56 @@ class MultiPartUpload(object):

         if not self.upload_id:

             raise RuntimeError("Attempting to use a multipart upload that has not been initiated.")

-        size_left = file_size = os.stat(self.file.name)[ST_SIZE]

-        self.chunk_size = self.s3.config.multipart_chunk_size_mb * 1024 * 1024

-        nr_parts = file_size / self.chunk_size + (file_size % self.chunk_size and 1)

-        debug("MultiPart: Uploading %s in %d parts" % (self.file.name, nr_parts))

+	self.chunk_size = self.s3.config.multipart_chunk_size_mb * 1024 * 1024

+

+        if self.file.name != "<stdin>":

+                size_left = file_size = os.stat(self.file.name)[ST_SIZE]

+                nr_parts = file_size / self.chunk_size + (file_size % self.chunk_size and 1)

+                debug("MultiPart: Uploading %s in %d parts" % (self.file.name, nr_parts))

+        else:

+            debug("MultiPart: Uploading from %s" % (self.file.name))

         seq = 1

-        while size_left > 0:

-            offset = self.chunk_size * (seq - 1)

-            current_chunk_size = min(file_size - offset, self.chunk_size)

-            size_left -= current_chunk_size

-            labels = {

-                'source' : unicodise(self.file.name),

-                'destination' : unicodise(self.uri.uri()),

-                'extra' : "[part %d of %d, %s]" % (seq, nr_parts, "%d%sB" % formatSize(current_chunk_size, human_readable = True))

-            }

-            try:

-                self.upload_part(seq, offset, current_chunk_size, labels)

-            except:

-                error(u"Upload of '%s' part %d failed. Aborting multipart upload." % (self.file.name, seq))

-                self.abort_upload()

-                raise

-            seq += 1

+	if self.file.name != "<stdin>":

+            while size_left > 0:

+                offset = self.chunk_size * (seq - 1)

+                current_chunk_size = min(file_size - offset, self.chunk_size)

+                size_left -= current_chunk_size

+                labels = {

+                    'source' : unicodise(self.file.name),

+                    'destination' : unicodise(self.uri.uri()),

+                    'extra' : "[part %d of %d, %s]" % (seq, nr_parts, "%d%sB" % formatSize(current_chunk_size, human_readable = True))

+                }

+                try:

+                    self.upload_part(seq, offset, current_chunk_size, labels)

+                except:

+                    error(u"Upload of '%s' part %d failed. Aborting multipart upload." % (self.file.name, seq))

+                    self.abort_upload()

+                    raise

+                seq += 1

+        else:

+            while True:

+                buffer = self.file.read(self.chunk_size)

+                offset = self.chunk_size * (seq - 1)

+                current_chunk_size = len(buffer)

+                labels = {

+                    'source' : unicodise(self.file.name),

+                    'destination' : unicodise(self.uri.uri()),

+                    'extra' : "[part %d, %s]" % (seq, "%d%sB" % formatSize(current_chunk_size, human_readable = True))

+                }

+                if len(buffer) == 0: # EOF

+                    break

+                try:

+                    self.upload_part(seq, offset, current_chunk_size, labels, buffer)

+                except:

+                    error(u"Upload of '%s' part %d failed. Aborting multipart upload." % (self.file.name, seq))

+                    self.abort_upload()

+                    raise

+                seq += 1

         debug("MultiPart: Upload finished: %d parts", seq - 1)

-    def upload_part(self, seq, offset, chunk_size, labels):

+    def upload_part(self, seq, offset, chunk_size, labels, buffer = ''):

         """

         Upload a file chunk

         http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html

@@ -77,7 +101,7 @@ class MultiPartUpload(object):

         headers = { "content-length": chunk_size }

         query_string = "?partNumber=%i&uploadId=%s" % (seq, self.upload_id)

         request = self.s3.create_request("OBJECT_PUT", uri = self.uri, headers = headers, extra = query_string)

-        response = self.s3.send_file(request, self.file, labels, offset = offset, chunk_size = chunk_size)

+        response = self.s3.send_file(request, self.file, labels, buffer, offset = offset, chunk_size = chunk_size)

         self.parts[seq] = response["headers"]["etag"]

         return response

@@ -1,5 +1,5 @@

 package = "s3cmd"

-version = "1.1.0-beta3"

+version = "1.1.0-beta3-mdomsch-merge-20121206"

 url = "http://s3tools.org"

 license = "GPL version 2"

 short_description = "Command line tool for managing Amazon S3 and CloudFront services"

@@ -5,10 +5,12 @@

 import sys

 import datetime

+import time

 import Utils

 class Progress(object):

     _stdout = sys.stdout

+    _last_display = 0

     def __init__(self, labels, total_size):

         self._stdout = sys.stdout

@@ -48,6 +50,13 @@ class Progress(object):

         self._stdout.write(u"%(source)s -> %(destination)s  %(extra)s\n" % self.labels)

         self._stdout.flush()

+    def _display_needed(self):

+        # We only need to update the display every so often.

+        if time.time() - self._last_display > 1:

+            self._last_display = time.time()

+            return True

+        return False

+

     def display(self, new_file = False, done_message = None):

         """

         display(new_file = False[/True], done = False[/True])

@@ -98,6 +107,10 @@ class ProgressANSI(Progress):

             self._stdout.flush()

             return

+        # Only display progress every so often

+        if not (new_file or done_message) and not self._display_needed():

+            return

+

         timedelta = self.time_current - self.time_start

         sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0

         if (sec_elapsed > 0):

@@ -132,6 +145,10 @@ class ProgressCR(Progress):

             self.output_labels()

             return

+        # Only display progress every so often

+        if not (new_file or done_message) and not self._display_needed():

+            return

+

         timedelta = self.time_current - self.time_start

         sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0

         if (sec_elapsed > 0):

@@ -29,18 +29,47 @@ from MultiPart import MultiPartUpload

 from S3Uri import S3Uri

 try:

-    import magic

+    import magic, gzip

     try:

         ## https://github.com/ahupp/python-magic

         magic_ = magic.Magic(mime=True)

-        def mime_magic(file):

+        def mime_magic_file(file):

             return magic_.from_file(file)

-    except (TypeError, AttributeError):

+        def mime_magic_buffer(buffer):

+            return magic_.from_buffer(buffer)

+    except TypeError:

+        ## http://pypi.python.org/pypi/filemagic

+        try:

+            magic_ = magic.Magic(flags=magic.MAGIC_MIME)

+            def mime_magic_file(file):

+                return magic_.id_filename(file)

+            def mime_magic_buffer(buffer):

+                return magic_.id_buffer(buffer)

+        except TypeError:

+            ## file-5.11 built-in python bindings

+            magic_ = magic.open(magic.MAGIC_MIME)

+            magic_.load()

+            def mime_magic_file(file):

+                return magic_.file(file)

+            def mime_magic_buffer(buffer):

+                return magic_.buffer(buffer)

+            

+    except AttributeError:

         ## Older python-magic versions

         magic_ = magic.open(magic.MAGIC_MIME)

         magic_.load()

-        def mime_magic(file):

+        def mime_magic_file(file):

             return magic_.file(file)

+        def mime_magic_buffer(buffer):

+            return magic_.buffer(buffer)

+    

+    def mime_magic(file):

+        type = mime_magic_file(file)

+        if type != "application/x-gzip; charset=binary":

+            return (type, None)

+        else:

+            return (mime_magic_buffer(gzip.open(file).read(8192)), 'gzip')

+

 except ImportError, e:

     if str(e).find("magic") >= 0:

         magic_message = "Module python-magic is not available."

@@ -53,13 +82,16 @@ except ImportError, e:

         if (not magic_warned):

             warning(magic_message)

             magic_warned = True

-        return mimetypes.guess_type(file)[0]

+        return mimetypes.guess_type(file)

 __all__ = []

 class S3Request(object):

     def __init__(self, s3, method_string, resource, headers, params = {}):

         self.s3 = s3

         self.headers = SortedDict(headers or {}, ignore_case = True)

+        # Add in any extra headers from s3 config object

+        if self.s3.config.extra_headers:

+            self.headers.update(self.s3.config.extra_headers)

         self.resource = resource

         self.method_string = method_string

         self.params = params

@@ -211,9 +243,10 @@ class S3(object):

         truncated = True

         list = []

         prefixes = []

+        conn = self.get_connection(bucket)

         while truncated:

-            response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params)

+            response = self.bucket_list_noparse(conn, bucket, prefix, recursive, uri_params)

             current_list = _get_contents(response["data"])

             current_prefixes = _get_common_prefixes(response["data"])

             truncated = _list_truncated(response["data"])

@@ -227,17 +260,19 @@ class S3(object):

             list += current_list

             prefixes += current_prefixes

+        conn.close()

+

         response['list'] = list

         response['common_prefixes'] = prefixes

         return response

-    def bucket_list_noparse(self, bucket, prefix = None, recursive = None, uri_params = {}):

+    def bucket_list_noparse(self, connection, bucket, prefix = None, recursive = None, uri_params = {}):

         if prefix:

             uri_params['prefix'] = self.urlencode_string(prefix)

         if not self.config.recursive and not recursive:

             uri_params['delimiter'] = "/"

         request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params)

-        response = self.send_request(request)

+        response = self.send_request(request, conn = connection)

         #debug(response)

         return response

@@ -339,17 +374,36 @@ class S3(object):

         return response

+    def add_encoding(self, filename, content_type):

+        if content_type.find("charset=") != -1:

+           return False

+        exts = self.config.add_encoding_exts.split(',')

+        if exts[0]=='':

+            return False

+        parts = filename.rsplit('.',2)

+        if len(parts) < 2:

+            return False

+        ext = parts[1]

+        if ext in exts:

+            return True

+        else: