@@ -137,7 +137,7 @@ def handle_exclude_include_walk(root, dirs, files):

             debug(u"PASS: %s" % (file))

 def fetch_local_list(args, recursive = None):

-    def _get_filelist_local(local_uri):

+    def _get_filelist_local(loc_list, local_uri):

         info(u"Compiling list of local files...")

         if local_uri.isdir():

             local_base = deunicodise(local_uri.basename())

@@ -149,7 +149,6 @@ def fetch_local_list(args, recursive = None):

             local_path = deunicodise(local_uri.dirname())

             filelist = [( local_path, [], [deunicodise(local_uri.basename())] )]

             single_file = True

-        loc_list = SortedDict(ignore_case = False)

         for root, dirs, files in filelist:

             rel_root = root.replace(local_path, local_base, 1)

             for f in files:

@@ -173,7 +172,6 @@ def fetch_local_list(args, recursive = None):

                     'full_name' : full_name,

                     'size' : sr.st_size,

                     'mtime' : sr.st_mtime,

-		    'nlink' : sr.st_nlink, # record hardlink information

 		    'dev'   : sr.st_dev,

 		    'inode' : sr.st_ino,

 		    'uid' : sr.st_uid,

@@ -181,7 +179,9 @@ def fetch_local_list(args, recursive = None):

 		    'sr': sr # save it all, may need it in preserve_attrs_list

                     ## TODO: Possibly more to save here...

                 }

-		loc_list.record_hardlink(relative_file, sr.st_dev, sr.st_ino)

+		if 'md5' in cfg.sync_checks:

+		    md5 = loc_list.get_md5(relative_file)

+		    loc_list.record_hardlink(relative_file, sr.st_dev, sr.st_ino, md5)

         return loc_list, single_file

     cfg = Config()

@@ -204,8 +204,7 @@ def fetch_local_list(args, recursive = None):

         local_uris.append(uri)

     for uri in local_uris:

-        list_for_uri, single_file = _get_filelist_local(uri)

-        local_list.update(list_for_uri)

+        list_for_uri, single_file = _get_filelist_local(local_list, uri)

     ## Single file is True if and only if the user

     ## specified one local URI and that URI represents

@@ -264,7 +263,6 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):

                 'object_key' : object['Key'],

                 'object_uri_str' : object_uri_str,

                 'base_uri' : remote_uri,

-		'nlink' : 1, # S3 doesn't support hardlinks itself

 		'dev' : None,

 		'inode' : None,

             }

@@ -406,7 +404,7 @@ def compare_filelists(src_list, dst_list, src_remote, dst_remote, delay_updates

     debug("Comparing filelists (direction: %s -> %s)" % (__direction_str(src_remote), __direction_str(dst_remote)))

     for relative_file in src_list.keys():

-        debug(u"CHECK: %s: %s" % (relative_file, src_list.get_md5(relative_file)))

+        debug(u"CHECK: %s" % (relative_file))

         if dst_list.has_key(relative_file):

             ## Was --skip-existing requested?

@@ -416,7 +414,14 @@ def compare_filelists(src_list, dst_list, src_remote, dst_remote, delay_updates

 	        del(dst_list[relative_file])

 	        continue

-	    if _compare(src_list, dst_list, src_remote, dst_remote, relative_file):

+	    try:

+                compare_result = _compare(src_list, dst_list, src_remote, dst_remote, relative_file)

+	    except (IOError,OSError), e:

+	        del(src_list[relative_file])

+		del(dst_list[relative_file])

+                continue

+

+	    if compare_result:

 	        debug(u"IGNR: %s (transfer not needed)" % relative_file)

 	        del(src_list[relative_file])

 		del(dst_list[relative_file])

@@ -434,7 +439,6 @@ def compare_filelists(src_list, dst_list, src_remote, dst_remote, delay_updates

                 else:

 		    # record that we will get this file transferred to us (before all the copies), so if we come across it later again,

 		    # we can copy from _this_ copy (e.g. we only upload it once, and copy thereafter).

-                    debug(u"REMOTE COPY src before")

 		    dst_list.record_md5(relative_file, md5) 

 		    update_list[relative_file] = src_list[relative_file]

 		    del src_list[relative_file]

@@ -448,7 +452,6 @@ def compare_filelists(src_list, dst_list, src_remote, dst_remote, delay_updates

 	    if dst1 is not None:

                 # Found one, we want to copy

                 debug(u"REMOTE COPY dst: %s -> %s" % (dst1, relative_file))

-		# FIXME this blows up when dst1 is not in dst_list, because we added it below in record_md5 but it's not really in dst_list.

 		copy_pairs.append((dst1, relative_file))

 		del(src_list[relative_file])

 	    else:

@@ -456,7 +459,6 @@ def compare_filelists(src_list, dst_list, src_remote, dst_remote, delay_updates

 	        # record that we will get this file transferred to us (before all the copies), so if we come across it later again,

 	        # we can copy from _this_ copy (e.g. we only upload it once, and copy thereafter).

 	        dst_list.record_md5(relative_file, md5) 

-		debug(u"REMOTE COPY dst before")

     for f in dst_list.keys():

         if not src_list.has_key(f) and not update_list.has_key(f):

@@ -27,7 +27,7 @@ class SortedDict(dict):

         """

         dict.__init__(self, mapping, **kwargs)

         self.ignore_case = ignore_case

-        self.hardlinks = dict()

+        self.hardlinks = dict() # { dev: { inode : {'md5':, 'relative_files':}}}

         self.by_md5 = dict() # {md5: set(relative_files)}

     def keys(self):

@@ -60,39 +60,24 @@ class SortedDict(dict):

         except:

             return None

-

     def get_md5(self, relative_file):

         md5 = None

         if 'md5' in self[relative_file]:

             return self[relative_file]['md5']

-	if self.is_hardlinked(relative_file): # speedup by getting it from one of the hardlinks already processed

-            md5 = self.get_hardlink_md5(relative_file)

-	    if md5 is None:

-                md5 = Utils.hash_file_md5(self[relative_file]['full_name'])

-		self.record_md5(relative_file, md5)

-	        self.set_hardlink_md5(relative_file, md5)

-	else:

-                md5 = Utils.hash_file_md5(self[relative_file]['full_name'])

-	        self[relative_file]['md5'] = md5

-		l.record_md5(relative_file, md5)

+        md5 = self.get_hardlink_md5(relative_file)

+        if md5 is None:

+            md5 = Utils.hash_file_md5(self[relative_file]['full_name'])

+        self.record_md5(relative_file, md5)

+        self[relative_file]['md5'] = md5

 	return md5

-    def record_hardlink(self, relative_file, dev, inode):

+    def record_hardlink(self, relative_file, dev, inode, md5):

         if dev not in self.hardlinks:

             self.hardlinks[dev] = dict()

         if inode not in self.hardlinks[dev]:

-            self.hardlinks[dev][inode] = dict(md5=None, relative_files=set())

+            self.hardlinks[dev][inode] = dict(md5=md5, relative_files=set())

         self.hardlinks[dev][inode]['relative_files'].add(relative_file)

-    def set_hardlink_md5(self, relative_file, md5):

-        dev = self[relative_file]['dev']

-        inode = self[relative_file]['inode']

-        self.record_hardlink(relative_file, dev, inode)

-        self.hardlinks[dev][inode]['md5'] = md5

-

-    def is_hardlinked(self, relative_file):

-        return self[relative_file]['nlink'] > 1

-

     def get_hardlink_md5(self, relative_file):

         md5 = None

         dev = self[relative_file]['dev']

@@ -23,6 +23,7 @@ import locale

 import subprocess

 import htmlentitydefs

 import socket

+import shutil

 from copy import copy

 from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter

@@ -700,8 +701,9 @@ def cmd_sync_remote2local(args):

     local_count = len(local_list)

     remote_count = len(remote_list)

     update_count = len(update_list)

+    copy_pairs_count = len(copy_pairs)

-    info(u"Summary: %d remote files to download, %d local files to delete" % (remote_count + update_count, local_count))

+    info(u"Summary: %d remote files to download, %d local files to delete, %d local files to hardlink" % (remote_count + update_count, local_count, copy_pairs_count))

     def _set_local_filename(remote_list, destination_base):

         if not os.path.isdir(destination_base):

@@ -826,6 +828,7 @@ def cmd_sync_remote2local(args):

     seq = 0

     seq, total_size = _download(remote_list, seq, remote_count + update_count, total_size, dir_cache)

     seq, total_size = _download(update_list, seq, remote_count + update_count, total_size, dir_cache)

+    local_hardlink(copy_pairs, destination_base)

     total_elapsed = time.time() - timestamp_start

     speed_fmt = formatSize(total_size/total_elapsed, human_readable = True, floating_point = True)

@@ -841,6 +844,15 @@ def cmd_sync_remote2local(args):

     if cfg.delete_removed and cfg.delete_after:

         _do_deletes(local_list)

+def local_hardlink(copy_pairs, destination_base):

+    for (dst1, dst2) in copy_pairs:

+        try:

+            os.link(destination_base + dst1, destination_base + dst2)

+            debug(u"Hardlinking %s to %s" % (destination_base + dst1, destination_base + dst2))

+        except:

+            shutil.copy2(destination_base + dst1, destination_base + dst2)

+            debug(u"Hardlinking unavailable, copying %s to %s" % (destination_base + dst1, destination_base + dst2))

+

 def remote_copy(s3, copy_pairs, destination_base):

     saved_bytes = 0

     for (dst1, dst2) in copy_pairs: