The --cf-invalidate code path in sync_local2remote would traceback
from undefined variables.
Refactor this code path into a subfunction, saving necessary variables
earlier in execution. This only invalidates the paths based on the
last uploaded destination in the list (if there are multiple
destinations). If this isn't the desired behavior, don't upload to
multiple destinations _and_ try to invalidate on CF simultaneously.
... | ... |
@@ -936,6 +936,7 @@ def cmd_sync_local2remote(args): |
936 | 936 |
raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri) |
937 | 937 |
destination_base = str(destination_base_uri) |
938 | 938 |
_child(destination_base, local_list) |
939 |
+ return destination_base_uri |
|
939 | 940 |
|
940 | 941 |
def _parent(): |
941 | 942 |
# Now that we've done all the disk I/O to look at the local file system and |
... | ... |
@@ -959,7 +960,8 @@ def cmd_sync_local2remote(args): |
959 | 959 |
while len(child_pids): |
960 | 960 |
(pid, status) = os.wait() |
961 | 961 |
child_pids.remove(pid) |
962 |
- return |
|
962 |
+ |
|
963 |
+ return destination_base_uri |
|
963 | 964 |
|
964 | 965 |
def _child(destination_base, local_list): |
965 | 966 |
def _set_remote_uri(local_list, destination_base, single_file_local): |
... | ... |
@@ -1055,7 +1057,6 @@ def cmd_sync_local2remote(args): |
1055 | 1055 |
if cfg.delete_removed and not cfg.delete_after: |
1056 | 1056 |
_do_deletes(s3, remote_list) |
1057 | 1057 |
|
1058 |
- uploaded_objects_list = [] |
|
1059 | 1058 |
total_size = 0 |
1060 | 1059 |
total_elapsed = 0.0 |
1061 | 1060 |
timestamp_start = time.time() |
... | ... |
@@ -1078,8 +1079,25 @@ def cmd_sync_local2remote(args): |
1078 | 1078 |
|
1079 | 1079 |
return |
1080 | 1080 |
|
1081 |
+ def _invalidate_on_cf(destination_base_uri): |
|
1082 |
+ cf = CloudFront(cfg) |
|
1083 |
+ default_index_file = None |
|
1084 |
+ if cfg.invalidate_default_index_on_cf or cfg.invalidate_default_index_root_on_cf: |
|
1085 |
+ info_response = s3.website_info(destination_base_uri, cfg.bucket_location) |
|
1086 |
+ if info_response: |
|
1087 |
+ default_index_file = info_response['index_document'] |
|
1088 |
+ if len(default_index_file) < 1: |
|
1089 |
+ default_index_file = None |
|
1090 |
+ |
|
1091 |
+ result = cf.InvalidateObjects(destination_base_uri, uploaded_objects_list, default_index_file, cfg.invalidate_default_index_on_cf, cfg.invalidate_default_index_root_on_cf) |
|
1092 |
+ if result['status'] == 201: |
|
1093 |
+ output("Created invalidation request for %d paths" % len(uploaded_objects_list)) |
|
1094 |
+ output("Check progress with: s3cmd cfinvalinfo cf://%s/%s" % (result['dist_id'], result['request_id'])) |
|
1095 |
+ |
|
1096 |
+ |
|
1081 | 1097 |
# main execution |
1082 | 1098 |
s3 = S3(cfg) |
1099 |
+ uploaded_objects_list = [] |
|
1083 | 1100 |
|
1084 | 1101 |
if cfg.encrypt: |
1085 | 1102 |
error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.") |
... | ... |
@@ -1087,46 +1105,6 @@ def cmd_sync_local2remote(args): |
1087 | 1087 |
error(u"or disable encryption with --no-encrypt parameter.") |
1088 | 1088 |
sys.exit(1) |
1089 | 1089 |
|
1090 |
- uploaded_objects_list = [] |
|
1091 |
- total_size = 0 |
|
1092 |
- total_elapsed = 0.0 |
|
1093 |
- timestamp_start = time.time() |
|
1094 |
- seq = 0 |
|
1095 |
- file_list = local_list.keys() |
|
1096 |
- file_list.sort() |
|
1097 |
- for file in file_list: |
|
1098 |
- seq += 1 |
|
1099 |
- item = local_list[file] |
|
1100 |
- src = item['full_name'] |
|
1101 |
- uri = S3Uri(item['remote_uri']) |
|
1102 |
- seq_label = "[%d of %d]" % (seq, local_count) |
|
1103 |
- extra_headers = copy(cfg.extra_headers) |
|
1104 |
- try: |
|
1105 |
- if cfg.preserve_attrs: |
|
1106 |
- attr_header = _build_attr_header(src) |
|
1107 |
- debug(u"attr_header: %s" % attr_header) |
|
1108 |
- extra_headers.update(attr_header) |
|
1109 |
- response = s3.object_put(src, uri, extra_headers, extra_label = seq_label) |
|
1110 |
- except InvalidFileError, e: |
|
1111 |
- warning(u"File can not be uploaded: %s" % e) |
|
1112 |
- continue |
|
1113 |
- except S3UploadError, e: |
|
1114 |
- error(u"%s: upload failed too many times. Skipping that file." % item['full_name_unicode']) |
|
1115 |
- continue |
|
1116 |
- speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) |
|
1117 |
- if not cfg.progress_meter: |
|
1118 |
- output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % |
|
1119 |
- (item['full_name_unicode'], uri, response["size"], response["elapsed"], |
|
1120 |
- speed_fmt[0], speed_fmt[1], seq_label)) |
|
1121 |
- total_size += response["size"] |
|
1122 |
- if cfg.acl_grants or cfg.acl_revokes: |
|
1123 |
- try: |
|
1124 |
- update_acl(s3, uri) |
|
1125 |
- except Exception, e: |
|
1126 |
- error(u"%s: Error while setting ACL: %s" % e) |
|
1127 |
- uploaded_objects_list.append(uri.object()) |
|
1128 |
- |
|
1129 |
- |
|
1130 | 1090 |
local_list, single_file_local = fetch_local_list(args[:-1], recursive = True) |
1131 | 1091 |
|
1132 | 1092 |
destinations = [args[-1]] |
... | ... |
@@ -1134,31 +1112,15 @@ def cmd_sync_local2remote(args): |
1134 | 1134 |
destinations = destinations + cfg.additional_destinations |
1135 | 1135 |
|
1136 | 1136 |
if 'fork' not in os.__all__ or len(destinations) < 2: |
1137 |
- _single_process(local_list) |
|
1137 |
+ destination_base_uri = _single_process(local_list) |
|
1138 | 1138 |
else: |
1139 |
- _parent() |
|
1139 |
+ destination_base_uri = _parent() # this only returns the last destination |
|
1140 | 1140 |
|
1141 | 1141 |
if cfg.invalidate_on_cf: |
1142 | 1142 |
if len(uploaded_objects_list) == 0: |
1143 | 1143 |
info("Nothing to invalidate in CloudFront") |
1144 | 1144 |
else: |
1145 |
- # 'uri' from the last iteration is still valid at this point |
|
1146 |
- cf = CloudFront(cfg) |
|
1147 |
- # joseprio: obtain default index file if we need it for invalidating |
|
1148 |
- # the files |
|
1149 |
- default_index_file = None |
|
1150 |
- if cfg.invalidate_default_index_on_cf or cfg.invalidate_default_index_root_on_cf: |
|
1151 |
- info_response = s3.website_info(destination_base_uri, cfg.bucket_location) |
|
1152 |
- if info_response: |
|
1153 |
- default_index_file = info_response['index_document'] |
|
1154 |
- if len(default_index_file) < 1: |
|
1155 |
- default_index_file = None |
|
1156 |
- # joseprio: pass configuration information for invalidating default |
|
1157 |
- # index file |
|
1158 |
- result = cf.InvalidateObjects(uri, uploaded_objects_list, default_index_file, cfg.invalidate_default_index_on_cf, cfg.invalidate_default_index_root_on_cf) |
|
1159 |
- if result['status'] == 201: |
|
1160 |
- output("Created invalidation request for %d paths" % len(uploaded_objects_list)) |
|
1161 |
- output("Check progress with: s3cmd cfinvalinfo cf://%s/%s" % (result['dist_id'], result['request_id'])) |
|
1145 |
+ _invalidate_on_cf(destination_base_uri) |
|
1162 | 1146 |
|
1163 | 1147 |
def cmd_sync(args): |
1164 | 1148 |
if (len(args) < 2): |