@@ -1,5 +1,9 @@

 2008-12-31  Michal Ludvig  <michal@logix.cz>

+	* s3cmd, S3/Utils.py, S3/Exceptions.py, S3/Progress.py,

+	  S3/Config.py, S3/S3.py: Added --encoding switch and 

+	  Config.encoding variable. Don't assume utf-8 for filesystem

+	  and terminal output anymore.

 	* s3cmd: Avoid ZeroDivisionError on fast links.

 	* s3cmd: Unicodised all info() output.

@@ -59,6 +59,7 @@ class Config(object):

 	exclude = []

 	# Dict mapping compiled REGEXPs back to their textual form

 	debug_exclude = {}

+	encoding = "utf-8"

 	## Creating a singleton

 	def __new__(self, configfile = None):

@@ -13,11 +13,10 @@ except ImportError:

 class S3Exception(Exception):

 	def __init__(self, message = ""):

-		self.message = message

+		self.message = unicodise(message)

 	def __str__(self):

-		## Is this legal?

-		return unicode(self)

+		return deunicodise(self.message)

 	def __unicode__(self):

 		return self.message

@@ -11,6 +11,7 @@ class Progress(object):

 	_stdout = sys.stdout

 	def __init__(self, labels, total_size):

+		self._stdout = sys.stdout

 		self.new_file(labels, total_size)

 	def new_file(self, labels, total_size):

@@ -174,12 +174,12 @@ class S3(object):

 			raise ValueError("Expected URI type 's3', got '%s'" % uri.type)

 		if not os.path.isfile(filename):

-			raise InvalidFileError("%s is not a regular file" % filename)

+			raise InvalidFileError(u"%s is not a regular file" % unicodise(filename))

 		try:

 			file = open(filename, "rb")

 			size = os.stat(filename)[ST_SIZE]

 		except IOError, e:

-			raise InvalidFileError("%s: %s" % (filename, e.strerror))

+			raise InvalidFileError(u"%s: %s" % (unicodise(filename), e.strerror))

 		headers = SortedDict()

 		if extra_headers:

 			headers.update(extra_headers)

@@ -194,7 +194,7 @@ class S3(object):

 		if self.config.acl_public:

 			headers["x-amz-acl"] = "public-read"

 		request = self.create_request("OBJECT_PUT", uri = uri, headers = headers)

-		labels = { 'source' : file.name, 'destination' : uri, 'extra' : extra_label }

+		labels = { 'source' : unicodise(filename), 'destination' : unicodise(uri.uri()), 'extra' : extra_label }

 		response = self.send_file(request, file, labels)

 		return response

@@ -13,6 +13,8 @@ import errno

 from logging import debug, info, warning, error

+import Config

+

 try:

 	import xml.etree.ElementTree as ET

 except ImportError:

@@ -181,21 +183,44 @@ def mkdir_with_parents(dir_name):

 			return False

 	return True

-def unicodise(string):

+def unicodise(string, encoding = None, errors = "replace"):

 	"""

 	Convert 'string' to Unicode or raise an exception.

 	"""

-	debug("Unicodising %r" % string)

+

+	if not encoding:

+		encoding = Config.Config().encoding

+

+	debug("Unicodising %r using %s" % (string, encoding))

 	if type(string) == unicode:

 		return string

 	try:

-		return string.decode("utf-8")

+		return string.decode(encoding, errors)

 	except UnicodeDecodeError:

 		raise UnicodeDecodeError("Conversion to unicode failed: %r" % string)

-def try_unicodise(string):

+def deunicodise(string, encoding = None, errors = "replace"):

+	"""

+	Convert unicode 'string' to <type str>, by default replacing

+	all invalid characters with '?' or raise an exception.

+	"""

+

+	if not encoding:

+		encoding = Config.Config().encoding

+

+	debug("DeUnicodising %r using %s" % (string, encoding))

+	if type(string) != unicode:

+		return str(string)

 	try:

-		return unicodise(string)

-	except UnicodeDecodeError:

-		return string

+		return string.encode(encoding, errors)

+	except UnicodeEncodeError:

+		raise UnicodeEncodeError("Conversion from unicode failed: %r" % string)

+

+def unicodise_safe(string, encoding = None):

+	"""

+	Convert 'string' to Unicode according to current encoding 

+	and replace all invalid characters with '?'

+	"""

+

+	return unicodise(deunicodise(string, encoding), encoding).replace(u'\ufffd', '?')

@@ -21,13 +21,8 @@ from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatt

 from logging import debug, info, warning, error

 from distutils.spawn import find_executable

-## Output UTF-8 in all cases

-encoding = locale.getpreferredencoding() or "utf-8"

-_stdout = codecs.getwriter(encoding)(sys.stdout, "replace")

-_stderr = codecs.getwriter(encoding)(sys.stderr, "replace")

-

 def output(message):

-	_stdout.write(message + "\n")

+	sys.stdout.write(message + "\n")

 def check_args_type(args, type, verbose_type):

 	for arg in args:

@@ -353,7 +348,7 @@ def cmd_object_get(args):

 		if destination == "-":

 			## stdout

-			dst_stream = sys.stdout

+			dst_stream = sys.__stdout__

 		else:

 			## File

 			try:

@@ -1121,7 +1116,10 @@ def main():

 	elif os.name == "nt" and os.getenv("USERPROFILE"):

 		config_file = os.path.join(os.getenv("USERPROFILE"), "Application Data", "s3cmd.ini")

-	optparser.set_defaults(config=config_file)

+	preferred_encoding = locale.getpreferredencoding() or "UTF-8"

+

+	optparser.set_defaults(encoding = preferred_encoding)

+	optparser.set_defaults(config = config_file)

 	optparser.set_defaults(verbosity = default_verbosity)

 	optparser.add_option(      "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool.")

@@ -1153,6 +1151,8 @@ def main():

 	optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", help="Default MIME-type to be set for objects stored.")

 	optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option")

+	optparser.add_option(      "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % preferred_encoding)

+

 	optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", help="Print sizes in human readable form.")

 	optparser.add_option(      "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default on TTY).")

@@ -1175,7 +1175,7 @@ def main():

 	## debugging/verbose output for config file parser on request

 	logging.basicConfig(level=options.verbosity,

 	                    format='%(levelname)s: %(message)s',

-	                    stream = _stderr)

+	                    stream = sys.stderr)

 	if options.show_version:

 		output(u"s3cmd version %s" % PkgInfo.version)

@@ -1207,9 +1207,6 @@ def main():

 	## Can be overriden by actual --(no-)progress parameter

 	cfg.update_option('progress_meter', sys.stdout.isatty())

-	## We may need a way to display progress meter on STDERR or somewhere else

-	Progress._stdout = _stdout

-

 	## Unsupported features on Win32 platform

 	if os.name == "nt":

 		if cfg.preserve_attrs:

@@ -1229,6 +1226,10 @@ def main():

 			## Some Config() options are not settable from command line

 			pass

+	## Set output and filesystem encoding for printing out filenames.

+	sys.stdout = codecs.getwriter(cfg.encoding)(sys.stdout, "replace")

+	sys.stderr = codecs.getwriter(cfg.encoding)(sys.stderr, "replace")

+

 	## Process GLOB (shell wildcard style) excludes

 	if options.exclude is None:

 		options.exclude = []