Browse code

* s3cmd, S3/S3.py: Ignore inaccessible (and missing) files on upload. * run-tests.py: Extended [sync] test to verify correct handling of inaccessible files. * testsuite/permission-tests: New testsuite files.

git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3cmd/trunk@441 830e0280-6d2a-0410-9c65-932aecc39d9d

Michal Ludvig authored on 2010/10/24 21:40:52
Showing 6 changed files
... ...
@@ -1,3 +1,11 @@
1
+2010-10-25  Michal Ludvig  <mludvig@logix.net.nz>
2
+
3
+	* s3cmd, S3/S3.py: Ignore inaccessible (and missing) files
4
+	  on upload.
5
+	* run-tests.py: Extended [sync] test to verify correct
6
+	  handling of inaccessible files.
7
+	* testsuite/permission-tests: New testsuite files.
8
+
1 9
 2010-10-24  Michal Ludvig  <mludvig@logix.net.nz>
2 10
 
3 11
 	* S3/S3.py: "Stringify" all headers. Httplib should do
... ...
@@ -255,7 +255,7 @@ class S3(object):
255 255
 		try:
256 256
 			file = open(filename, "rb")
257 257
 			size = os.stat(filename)[ST_SIZE]
258
-		except IOError, e:
258
+		except (IOError, OSError), e:
259 259
 			raise InvalidFileError(u"%s: %s" % (unicodise(filename), e.strerror))
260 260
 		headers = SortedDict(ignore_case = True)
261 261
 		if extra_headers:
... ...
@@ -58,6 +58,10 @@ if not os.path.isdir('testsuite/crappy-file-name'):
58 58
 	# TODO: also unpack if the tarball is newer than the directory timestamp
59 59
 	#       for instance when a new version was pulled from SVN.
60 60
 
61
+## Fix up permissions for permission-denied tests
62
+os.chmod("testsuite/permission-tests/permission-denied-dir", 0444)
63
+os.chmod("testsuite/permission-tests/permission-denied.txt", 0000)
64
+
61 65
 def test(label, cmd_args = [], retcode = 0, must_find = [], must_not_find = [], must_find_re = [], must_not_find_re = []):
62 66
 	def command_output():
63 67
 		print "----"
... ...
@@ -251,8 +255,9 @@ test_s3cmd("Buckets list", ["ls"],
251 251
 ## ====== Sync to S3
252 252
 test_s3cmd("Sync to S3", ['sync', 'testsuite/', pbucket(1) + '/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt', '--exclude-from', 'testsuite/exclude.encodings' ],
253 253
 	must_find = [ "WARNING: 32 non-printable characters replaced in: crappy-file-name/non-printables ^A^B^C^D^E^F^G^H^I^J^K^L^M^N^O^P^Q^R^S^T^U^V^W^X^Y^Z^[^\^]^^^_^? +-[\]^<>%%\"'#{}`&?.end",
254
+				  "WARNING: File can not be uploaded: testsuite/permission-tests/permission-denied.txt: Permission denied",
254 255
 	              "stored as '%s/xyz/crappy-file-name/non-printables ^A^B^C^D^E^F^G^H^I^J^K^L^M^N^O^P^Q^R^S^T^U^V^W^X^Y^Z^[^\^]^^^_^? +-[\\]^<>%%%%\"'#{}`&?.end'" % pbucket(1) ],
255
-	must_not_find_re = [ "\.svn/", "\.png$" ])
256
+	must_not_find_re = [ "\.svn/", "\.png$", "permission-denied-dir" ])
256 257
 
257 258
 if have_encoding:
258 259
 	## ====== Sync UTF-8 / GBK / ... to S3
... ...
@@ -972,7 +972,10 @@ def cmd_sync_local2remote(args):
972 972
 		import pwd, grp
973 973
 		attrs = {}
974 974
 		src = deunicodise(src)
975
-		st = os.stat_result(os.stat(src))
975
+		try:
976
+			st = os.stat_result(os.stat(src))
977
+		except OSError, e:
978
+			raise InvalidFileError(u"%s: %s" % (unicodise(src), e.strerror))
976 979
 		for attr in cfg.preserve_attrs_list:
977 980
 			if attr == 'uname':
978 981
 				try:
... ...
@@ -1073,18 +1076,18 @@ def cmd_sync_local2remote(args):
1073 1073
 		uri = S3Uri(item['remote_uri'])
1074 1074
 		seq_label = "[%d of %d]" % (seq, local_count)
1075 1075
 		extra_headers = copy(cfg.extra_headers)
1076
-		if cfg.preserve_attrs:
1077
-			attr_header = _build_attr_header(src)
1078
-			debug(u"attr_header: %s" % attr_header)
1079
-			extra_headers.update(attr_header)
1080 1076
 		try:
1077
+			if cfg.preserve_attrs:
1078
+				attr_header = _build_attr_header(src)
1079
+				debug(u"attr_header: %s" % attr_header)
1080
+				extra_headers.update(attr_header)
1081 1081
 			response = s3.object_put(src, uri, extra_headers, extra_label = seq_label)
1082
-		except S3UploadError, e:
1083
-			error(u"%s: upload failed too many times. Skipping that file." % item['full_name_unicode'])
1084
-			continue
1085 1082
 		except InvalidFileError, e:
1086 1083
 			warning(u"File can not be uploaded: %s" % e)
1087 1084
 			continue
1085
+		except S3UploadError, e:
1086
+			error(u"%s: upload failed too many times. Skipping that file." % item['full_name_unicode'])
1087
+			continue
1088 1088
 		speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
1089 1089
 		if not cfg.progress_meter:
1090 1090
 			output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
1091 1091
new file mode 100644
... ...
@@ -0,0 +1 @@
0
+inaccessible
0 1
new file mode 100644
... ...
@@ -0,0 +1 @@
0
+permission denied