Browse code

fix --cf-invalidate after merge

The --cf-invalidate code path in sync_local2remote would traceback
from undefined variables.

Refactor this code path into a subfunction, saving necessary variables
earlier in execution. This only invalidates the paths based on the
last uploaded destination in the list (if there are multiple
destinations). If this isn't the desired behavior, don't upload to
multiple destinations _and_ try to invalidate on CF simultaneously.

Matt Domsch authored on 2012/07/31 07:18:47
Showing 1 changed files
... ...
@@ -936,6 +936,7 @@ def cmd_sync_local2remote(args):
936 936
                 raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)
937 937
             destination_base = str(destination_base_uri)
938 938
             _child(destination_base, local_list)
939
+            return destination_base_uri
939 940
 
940 941
     def _parent():
941 942
         # Now that we've done all the disk I/O to look at the local file system and
... ...
@@ -959,7 +960,8 @@ def cmd_sync_local2remote(args):
959 959
         while len(child_pids):
960 960
             (pid, status) = os.wait()
961 961
             child_pids.remove(pid)
962
-        return
962
+
963
+        return destination_base_uri
963 964
 
964 965
     def _child(destination_base, local_list):
965 966
         def _set_remote_uri(local_list, destination_base, single_file_local):
... ...
@@ -1055,7 +1057,6 @@ def cmd_sync_local2remote(args):
1055 1055
         if cfg.delete_removed and not cfg.delete_after:
1056 1056
             _do_deletes(s3, remote_list)
1057 1057
 
1058
-        uploaded_objects_list = []
1059 1058
         total_size = 0
1060 1059
         total_elapsed = 0.0
1061 1060
         timestamp_start = time.time()
... ...
@@ -1078,8 +1079,25 @@ def cmd_sync_local2remote(args):
1078 1078
 
1079 1079
         return
1080 1080
 
1081
+    def _invalidate_on_cf(destination_base_uri):
1082
+        cf = CloudFront(cfg)
1083
+        default_index_file = None
1084
+        if cfg.invalidate_default_index_on_cf or cfg.invalidate_default_index_root_on_cf:
1085
+            info_response = s3.website_info(destination_base_uri, cfg.bucket_location)
1086
+            if info_response:
1087
+              default_index_file = info_response['index_document']
1088
+              if len(default_index_file) < 1:
1089
+                  default_index_file = None
1090
+
1091
+        result = cf.InvalidateObjects(destination_base_uri, uploaded_objects_list, default_index_file, cfg.invalidate_default_index_on_cf, cfg.invalidate_default_index_root_on_cf)
1092
+        if result['status'] == 201:
1093
+            output("Created invalidation request for %d paths" % len(uploaded_objects_list))
1094
+            output("Check progress with: s3cmd cfinvalinfo cf://%s/%s" % (result['dist_id'], result['request_id']))
1095
+
1096
+
1081 1097
     # main execution
1082 1098
     s3 = S3(cfg)
1099
+    uploaded_objects_list = []
1083 1100
 
1084 1101
     if cfg.encrypt:
1085 1102
         error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.")
... ...
@@ -1087,46 +1105,6 @@ def cmd_sync_local2remote(args):
1087 1087
         error(u"or disable encryption with --no-encrypt parameter.")
1088 1088
         sys.exit(1)
1089 1089
 
1090
-    uploaded_objects_list = []
1091
-    total_size = 0
1092
-    total_elapsed = 0.0
1093
-    timestamp_start = time.time()
1094
-    seq = 0
1095
-    file_list = local_list.keys()
1096
-    file_list.sort()
1097
-    for file in file_list:
1098
-        seq += 1
1099
-        item = local_list[file]
1100
-        src = item['full_name']
1101
-        uri = S3Uri(item['remote_uri'])
1102
-        seq_label = "[%d of %d]" % (seq, local_count)
1103
-        extra_headers = copy(cfg.extra_headers)
1104
-        try:
1105
-            if cfg.preserve_attrs:
1106
-                attr_header = _build_attr_header(src)
1107
-                debug(u"attr_header: %s" % attr_header)
1108
-                extra_headers.update(attr_header)
1109
-            response = s3.object_put(src, uri, extra_headers, extra_label = seq_label)
1110
-        except InvalidFileError, e:
1111
-            warning(u"File can not be uploaded: %s" % e)
1112
-            continue
1113
-        except S3UploadError, e:
1114
-            error(u"%s: upload failed too many times. Skipping that file." % item['full_name_unicode'])
1115
-            continue
1116
-        speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
1117
-        if not cfg.progress_meter:
1118
-            output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
1119
-                (item['full_name_unicode'], uri, response["size"], response["elapsed"],
1120
-                speed_fmt[0], speed_fmt[1], seq_label))
1121
-        total_size += response["size"]
1122
-        if cfg.acl_grants or cfg.acl_revokes:
1123
-            try:
1124
-                update_acl(s3, uri)
1125
-            except Exception, e:
1126
-                error(u"%s: Error while setting ACL: %s" % e)
1127
-        uploaded_objects_list.append(uri.object())
1128
-
1129
-
1130 1090
     local_list, single_file_local = fetch_local_list(args[:-1], recursive = True)
1131 1091
 
1132 1092
     destinations = [args[-1]]
... ...
@@ -1134,31 +1112,15 @@ def cmd_sync_local2remote(args):
1134 1134
         destinations = destinations + cfg.additional_destinations
1135 1135
 
1136 1136
     if 'fork' not in os.__all__ or len(destinations) < 2:
1137
-        _single_process(local_list)
1137
+        destination_base_uri = _single_process(local_list)
1138 1138
     else:
1139
-        _parent()
1139
+        destination_base_uri = _parent() # this only returns the last destination
1140 1140
 
1141 1141
     if cfg.invalidate_on_cf:
1142 1142
         if len(uploaded_objects_list) == 0:
1143 1143
             info("Nothing to invalidate in CloudFront")
1144 1144
         else:
1145
-            # 'uri' from the last iteration is still valid at this point
1146
-            cf = CloudFront(cfg)
1147
-            # joseprio: obtain default index file if we need it for invalidating
1148
-            # the files
1149
-            default_index_file = None
1150
-            if cfg.invalidate_default_index_on_cf or cfg.invalidate_default_index_root_on_cf:
1151
-                info_response = s3.website_info(destination_base_uri, cfg.bucket_location)
1152
-                if info_response:
1153
-                  default_index_file = info_response['index_document']
1154
-                  if len(default_index_file) < 1:
1155
-                      default_index_file = None
1156
-            # joseprio: pass configuration information for invalidating default
1157
-            # index file
1158
-            result = cf.InvalidateObjects(uri, uploaded_objects_list, default_index_file, cfg.invalidate_default_index_on_cf, cfg.invalidate_default_index_root_on_cf)
1159
-            if result['status'] == 201:
1160
-                output("Created invalidation request for %d paths" % len(uploaded_objects_list))
1161
-                output("Check progress with: s3cmd cfinvalinfo cf://%s/%s" % (result['dist_id'], result['request_id']))
1145
+            _invalidate_on_cf(destination_base_uri)
1162 1146
 
1163 1147
 def cmd_sync(args):
1164 1148
     if (len(args) < 2):