@@ -31,6 +31,19 @@ except NameError:

     # In python 3, unicode -> str, and str -> bytes

     unicode = str

+def config_unicodise(string, encoding = "utf-8", errors = "replace"):

+    """

+    Convert 'string' to Unicode or raise an exception.

+    Config can't use toolbox from Utils that is itself using Config

+    """

+    if type(string) == unicode:

+        return string

+

+    try:

+        return unicode(string, encoding, errors)

+    except UnicodeDecodeError:

+        raise UnicodeDecodeError("Conversion to unicode failed: %r" % string)

+

 class Config(object):

     _instance = None

     _parsed_files = []

@@ -168,7 +181,6 @@ class Config(object):

             if access_key and secret_key:

                 self.access_key = access_key

                 self.secret_key = secret_key

-                

             if access_token:

                 self.access_token = access_token

                 # Do not refresh the IAM role when an access token is provided.

@@ -179,12 +191,13 @@ class Config(object):

                 env_secret_key = os.getenv('AWS_SECRET_KEY') or os.getenv('AWS_SECRET_ACCESS_KEY')

                 env_access_token = os.getenv('AWS_SESSION_TOKEN') or os.getenv('AWS_SECURITY_TOKEN')

                 if env_access_key:

-                    self.access_key = env_access_key

-                    self.secret_key = env_secret_key

+                    # py3 getenv returns unicode and py2 returns bytes.

+                    self.access_key = config_unicodise(env_access_key)

+                    self.secret_key = config_unicodise(env_secret_key)

                     if env_access_token:

                         # Do not refresh the IAM role when an access token is provided.

                         self._access_token_refresh = False

-                        self.access_token = env_access_token

+                        self.access_token = config_unicodise(env_access_token)

                 else:

                     self.role_config()

@@ -544,8 +544,6 @@ def calculateChecksum(buffer, mfile, offset, chunk_size, send_chunk):

         md5_hash.update(buffer)

     return md5_hash.hexdigest()

-

-

 __all__.append("calculateChecksum")

@@ -39,6 +39,33 @@ exclude_tests = []

 verbose = False

+encoding = locale.getpreferredencoding()

+if not encoding:

+    print("Guessing current system encoding failed. Consider setting $LANG variable.")

+    sys.exit(1)

+else:

+    print("System encoding: " + encoding)

+

+try:

+    unicode

+except NameError:

+    # python 3 support

+    # In python 3, unicode -> str, and str -> bytes

+    unicode = str

+

+def unicodise(string, encoding = "utf-8", errors = "replace"):

+    """

+    Convert 'string' to Unicode or raise an exception.

+    Config can't use toolbox from Utils that is itself using Config

+    """

+    if type(string) == unicode:

+        return string

+

+    try:

+        return unicode(string, encoding, errors)

+    except UnicodeDecodeError:

+        raise UnicodeDecodeError("Conversion to unicode failed: %r" % string)

+

 # https://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028

 def which(program):

     def is_exe(fpath):

@@ -64,9 +91,12 @@ else:

 config_file = None

 if os.getenv("HOME"):

-    config_file = os.path.join(os.getenv("HOME"), ".s3cfg")

+    config_file = os.path.join(unicodise(os.getenv("HOME"), encoding), ".s3cfg")

 elif os.name == "nt" and os.getenv("USERPROFILE"):

-    config_file = os.path.join(os.getenv("USERPROFILE").decode('mbcs'), os.getenv("APPDATA").decode('mbcs') or 'Application Data', "s3cmd.ini")

+    config_file = os.path.join(unicodise(os.getenv("USERPROFILE"), encoding),

+                               os.getenv("APPDATA") and unicodise(os.getenv("APPDATA"), encoding)

+                               or 'Application Data',

+                               "s3cmd.ini")

 ## Unpack testsuite/ directory

@@ -90,13 +120,6 @@ patterns = {}

 patterns['UTF-8'] = u"ŪņЇЌœđЗ/☺ unicode € rocks ™"

 patterns['GBK'] = u"12月31日/1-特色條目"

-encoding = locale.getpreferredencoding()

-if not encoding:

-    print("Guessing current system encoding failed. Consider setting $LANG variable.")

-    sys.exit(1)

-else:

-    print("System encoding: " + encoding)

-

 have_encoding = os.path.isdir('testsuite/encodings/' + encoding)

 if not have_encoding and os.path.isfile('testsuite/encodings/%s.tar.gz' % encoding):

     os.system("tar xvz -C testsuite/encodings -f testsuite/encodings/%s.tar.gz" % encoding)

@@ -39,6 +39,33 @@ exclude_tests = []

 verbose = False

+encoding = locale.getpreferredencoding()

+if not encoding:

+    print("Guessing current system encoding failed. Consider setting $LANG variable.")

+    sys.exit(1)

+else:

+    print("System encoding: " + encoding)

+

+try:

+    unicode

+except NameError:

+    # python 3 support

+    # In python 3, unicode -> str, and str -> bytes

+    unicode = str

+

+def unicodise(string, encoding = "utf-8", errors = "replace"):

+    """

+    Convert 'string' to Unicode or raise an exception.

+    Config can't use toolbox from Utils that is itself using Config

+    """

+    if type(string) == unicode:

+        return string

+

+    try:

+        return unicode(string, encoding, errors)

+    except UnicodeDecodeError:

+        raise UnicodeDecodeError("Conversion to unicode failed: %r" % string)

+

 # https://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028

 def which(program):

     def is_exe(fpath):

@@ -64,9 +91,12 @@ else:

 config_file = None

 if os.getenv("HOME"):

-    config_file = os.path.join(os.getenv("HOME"), ".s3cfg")

+    config_file = os.path.join(unicodise(os.getenv("HOME"), encoding), ".s3cfg")

 elif os.name == "nt" and os.getenv("USERPROFILE"):

-    config_file = os.path.join(os.getenv("USERPROFILE").decode('mbcs'), os.getenv("APPDATA").decode('mbcs') or 'Application Data', "s3cmd.ini")

+    config_file = os.path.join(unicodise(os.getenv("USERPROFILE"), encoding),

+                               os.getenv("APPDATA") and unicodise(os.getenv("APPDATA"), encoding)

+                               or 'Application Data',

+                               "s3cmd.ini")

 ## Unpack testsuite/ directory

@@ -90,13 +120,6 @@ patterns = {}

 patterns['UTF-8'] = u"ŪņЇЌœđЗ/☺ unicode € rocks ™"

 patterns['GBK'] = u"12月31日/1-特色條目"

-encoding = locale.getpreferredencoding()

-if not encoding:

-    print("Guessing current system encoding failed. Consider setting $LANG variable.")

-    sys.exit(1)

-else:

-    print("System encoding: " + encoding)

-

 have_encoding = os.path.isdir('testsuite/encodings/' + encoding)

 if not have_encoding and os.path.isfile('testsuite/encodings/%s.tar.gz' % encoding):

     os.system("tar xvz -C testsuite/encodings -f testsuite/encodings/%s.tar.gz" % encoding)

@@ -2270,7 +2270,9 @@ def run_configure(config_file, args):

         setattr(cfg, "gpg_command", find_executable("gpg"))

     if getattr(cfg, "proxy_host") == "" and os.getenv("http_proxy"):

-        re_match=re.match("(http://)?([^:]+):(\d+)", os.getenv("http_proxy"))

+        autodetected_encoding = locale.getpreferredencoding() or "UTF-8"

+        re_match=re.match("(http://)?([^:]+):(\d+)",

+                          unicodise_s(os.getenv("http_proxy"), autodetected_encoding))

         if re_match:

             setattr(cfg, "proxy_host", re_match.groups()[1])

             setattr(cfg, "proxy_port", re_match.groups()[2])

@@ -2605,17 +2607,20 @@ def main():

     optparser = OptionParser(option_class=OptionAll, formatter=MyHelpFormatter())

     #optparser.disable_interspersed_args()

+    autodetected_encoding = locale.getpreferredencoding() or "UTF-8"

+

     config_file = None

     if os.getenv("S3CMD_CONFIG"):

-        config_file = os.getenv("S3CMD_CONFIG")

+        config_file = unicodise_s(os.getenv("S3CMD_CONFIG"), autodetected_encoding)

     elif os.name == "nt" and os.getenv("USERPROFILE"):

-        config_file = os.path.join(os.getenv("USERPROFILE").decode('mbcs'), os.getenv("APPDATA").decode('mbcs') or 'Application Data', "s3cmd.ini")

+        config_file = os.path.join(unicodise_s(os.getenv("USERPROFILE"), autodetected_encoding),

+                                   os.getenv("APPDATA") and unicodise_s(os.getenv("APPDATA"), autodetected_encoding)

+                                   or 'Application Data',

+                                   "s3cmd.ini")

     else:

         from os.path import expanduser

         config_file = os.path.join(expanduser("~"), ".s3cfg")

-    autodetected_encoding = locale.getpreferredencoding() or "UTF-8"

-

     optparser.set_defaults(config = config_file)

     optparser.add_option(      "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool. Optionally use as '--configure s3://some-bucket' to test access to a specific bucket instead of attempting to list them all.")

@@ -3019,7 +3024,7 @@ def report_exception(e, msg=u''):

         except NameError:

             sys.stderr.write(u"S3cmd:   unknown version. Module import problem?\n")

         sys.stderr.write(u"python:   %s\n" % sys.version)

-        sys.stderr.write(u"environment LANG=%s\n" % os.getenv("LANG"))

+        sys.stderr.write(u"environment LANG=%s\n" % unicodise_s(os.getenv("LANG"), 'ascii'))

         sys.stderr.write(u"\n")

         if type(tb) == unicode:

             sys.stderr.write(tb)

@@ -3131,7 +3136,7 @@ The solutions to this are:

         sys.exit(EX_OSERR)

     except UnicodeEncodeError as e:

-        lang = os.getenv("LANG")

+        lang = unicodise_s(os.getenv("LANG"), 'ascii')

         msg = """

 You have encountered a UnicodeEncodeError.  Your environment

 variable LANG=%s may not specify a Unicode encoding (e.g. UTF-8).