## Amazon S3 manager
 ## Author: Michal Ludvig <michal@logix.cz>
 ##         http://www.logix.cz/michal
 ## License: GPL Version 2

 ## Copyright: TGRMN Software and contributors

 import logging

 from logging import debug, info, warning, error

 import re

 import os

 import sys

 import Progress

 from SortedDict import SortedDict

 import httplib

 try:
     import json
 except ImportError, e:
     pass

 class Config(object):

     _instance = None
     _parsed_files = []
     _doc = {}
     access_key = ""
     secret_key = ""

     access_token = ""

     host_base = "s3.amazonaws.com"
     host_bucket = "%(bucket)s.s3.amazonaws.com"
     simpledb_host = "sdb.amazonaws.com"
     cloudfront_host = "cloudfront.amazonaws.com"
     verbosity = logging.WARNING
     progress_meter = True
     progress_class = Progress.ProgressCR
     send_chunk = 4096
     recv_chunk = 4096
     list_md5 = False
     human_readable_sizes = False
     extra_headers = SortedDict(ignore_case = True)
     force = False

     server_side_encryption = False

     enable = None
     get_continue = False

     put_continue = False
     upload_id = None

     skip_existing = False
     recursive = False

     restore_days = 1

     acl_public = None
     acl_grants = []
     acl_revokes = []
     proxy_host = ""
     proxy_port = 3128
     encrypt = False
     dry_run = False

     add_encoding_exts = ""

     preserve_attrs = True
     preserve_attrs_list = [
         'uname',    # Verbose owner Name (e.g. 'root')
         'uid',      # Numeric user ID (e.g. 0)
         'gname',    # Group name (e.g. 'users')
         'gid',      # Numeric group ID (e.g. 100)
         'atime',    # Last access timestamp
         'mtime',    # Modification timestamp
         'ctime',    # Creation timestamp
         'mode',     # File mode (e.g. rwxr-xr-x = 755)

         'md5',      # File MD5 (if known)

         #'acl',     # Full ACL (not yet supported)
     ]
     delete_removed = False

     delete_after = False

     delete_after_fetch = False

     max_delete = -1

     _doc['delete_removed'] = "[sync] Remove remote S3 objects when local file has been deleted"

     delay_updates = False

     gpg_passphrase = ""
     gpg_command = ""
     gpg_encrypt = "%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
     gpg_decrypt = "%(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
     use_https = False
     bucket_location = "US"
     default_mime_type = "binary/octet-stream"

     guess_mime_type = True

     use_mime_magic = True

     mime_type = ""

     enable_multipart = True

     multipart_chunk_size_mb = 15    # MB

     # List of checks to be performed for 'sync'
     sync_checks = ['size', 'md5']   # 'weak-timestamp'
     # List of compiled REGEXPs
     exclude = []
     include = []
     # Dict mapping compiled REGEXPs back to their textual form
     debug_exclude = {}
     debug_include = {}
     encoding = "utf-8"
     urlencoding_mode = "normal"
     log_target_prefix = ""
     reduced_redundancy = False
     follow_symlinks = False
     socket_timeout = 300
     invalidate_on_cf = False

     # joseprio: new flags for default index invalidation
     invalidate_default_index_on_cf = False
     invalidate_default_index_root_on_cf = True

     website_index = "index.html"
     website_error = ""
     website_endpoint = "http://%(bucket)s.s3-website-%(location)s.amazonaws.com/"

     additional_destinations = []

     files_from = []

     cache_file = ""

     add_headers = ""

     ignore_failed_copy = False

 
     ## Creating a singleton
     def __new__(self, configfile = None):
         if self._instance is None:
             self._instance = object.__new__(self)
         return self._instance
 
     def __init__(self, configfile = None):
         if configfile:

             try:
                 self.read_config_file(configfile)
             except IOError, e:
                 if 'AWS_CREDENTIAL_FILE' in os.environ:
                     self.env_config()
             if len(self.access_key)==0:

                 self.role_config()

 
     def role_config(self):

         if sys.version_info[0] * 10 + sys.version_info[1] < 26:
             error("IAM authentication requires Python 2.6 or newer")
             raise
         if not 'json' in sys.modules:
             error("IAM authentication not available -- missing module json")
             raise

         try:

             conn = httplib.HTTPConnection(host='169.254.169.254', timeout = 2)

             conn.request('GET', "/latest/meta-data/iam/security-credentials/")
             resp = conn.getresponse()
             files = resp.read()

             if resp.status == 200 and len(files)>1:

                 conn.request('GET', "/latest/meta-data/iam/security-credentials/%s"%files)
                 resp=conn.getresponse()
                 if resp.status == 200:
                     creds=json.load(resp)
                     Config().update_option('access_key', creds['AccessKeyId'].encode('ascii'))
                     Config().update_option('secret_key', creds['SecretAccessKey'].encode('ascii'))
                     Config().update_option('access_token', creds['Token'].encode('ascii'))
                 else:
                     raise IOError
             else:
                 raise IOError
         except:
             raise
 
     def role_refresh(self):
         try:
             self.role_config()
         except:
             warning("Could not refresh role")
 
     def env_config(self):
         cred_content = ""
         try:
             cred_file = open(os.environ['AWS_CREDENTIAL_FILE'],'r')
             cred_content = cred_file.read()
         except IOError, e:
             debug("Error %d accessing credentials file %s" % (e.errno,os.environ['AWS_CREDENTIAL_FILE']))
         r_data = re.compile("^\s*(?P<orig_key>\w+)\s*=\s*(?P<value>.*)")
         r_quotes = re.compile("^\"(.*)\"\s*$")
         if len(cred_content)>0:
             for line in cred_content.splitlines():
                 is_data = r_data.match(line)
                 is_data = r_data.match(line)
                 if is_data:
                     data = is_data.groupdict()
                     if r_quotes.match(data["value"]):
                         data["value"] = data["value"][1:-1]
                     if data["orig_key"]=="AWSAccessKeyId":
                         data["key"] = "access_key"
                     elif data["orig_key"]=="AWSSecretKey":
                         data["key"] = "secret_key"
                     else:

                         del data["key"]

                     if "key" in data:
                         Config().update_option(data["key"], data["value"])
                         if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):

                             print_value = ("%s...%d_chars...%s") % (data["value"][:2], len(data["value"]) - 3, data["value"][-1:])

                         else:
                             print_value = data["value"]
                         debug("env_Config: %s->%s" % (data["key"], print_value))

 
     def option_list(self):
         retval = []
         for option in dir(self):
             ## Skip attributes that start with underscore or are not string, int or bool
             option_type = type(getattr(Config, option))
             if option.startswith("_") or \
                not (option_type in (
                     type("string"), # str
                         type(42),   # int
                     type(True))):   # bool
                 continue
             retval.append(option)
         return retval
 
     def read_config_file(self, configfile):
         cp = ConfigParser(configfile)
         for option in self.option_list():
             self.update_option(option, cp.get(option))

 
         if cp.get('add_headers'):
             for option in cp.get('add_headers').split(","):
                 (key, value) = option.split(':')
                 self.extra_headers[key.replace('_', '-').strip()] = value.strip()
 

         self._parsed_files.append(configfile)
 
     def dump_config(self, stream):
         ConfigDumper(stream).dump("default", self)
 
     def update_option(self, option, value):
         if value is None:
             return

         #### Handle environment reference
         if str(value).startswith("$"):
             return self.update_option(option, os.getenv(str(value)[1:]))

         #### Special treatment of some options
         ## verbosity must be known to "logging" module
         if option == "verbosity":

             # support integer verboisities

             try:

                 value = int(value)
             except ValueError, e:
                 try:
                     # otherwise it must be a key known to the logging module
                     value = logging._levelNames[value]
                 except KeyError:
                     error("Config: verbosity level '%s' is not valid" % value)
                     return
 

         ## allow yes/no, true/false, on/off and 1/0 for boolean options
         elif type(getattr(Config, option)) is type(True):   # bool
             if str(value).lower() in ("true", "yes", "on", "1"):

                 value = True

             elif str(value).lower() in ("false", "no", "off", "0"):

                 value = False

             else:
                 error("Config: value of option '%s' must be Yes or No, not '%s'" % (option, value))

                 return
 

         elif type(getattr(Config, option)) is type(42):     # int
             try:

                 value = int(value)

             except ValueError, e:
                 error("Config: value of option '%s' must be an integer, not '%s'" % (option, value))

                 return
 
         setattr(Config, option, value)

 class ConfigParser(object):

     def __init__(self, file, sections = []):
         self.cfg = {}
         self.parse_file(file, sections)
 
     def parse_file(self, file, sections = []):
         debug("ConfigParser: Reading file '%s'" % file)
         if type(sections) != type([]):
             sections = [sections]
         in_our_section = True
         f = open(file, "r")
         r_comment = re.compile("^\s*#.*")
         r_empty = re.compile("^\s*$")
         r_section = re.compile("^\[([^\]]+)\]")
         r_data = re.compile("^\s*(?P<key>\w+)\s*=\s*(?P<value>.*)")
         r_quotes = re.compile("^\"(.*)\"\s*$")
         for line in f:
             if r_comment.match(line) or r_empty.match(line):
                 continue
             is_section = r_section.match(line)
             if is_section:
                 section = is_section.groups()[0]
                 in_our_section = (section in sections) or (len(sections) == 0)
                 continue
             is_data = r_data.match(line)
             if is_data and in_our_section:
                 data = is_data.groupdict()
                 if r_quotes.match(data["value"]):
                     data["value"] = data["value"][1:-1]
                 self.__setitem__(data["key"], data["value"])
                 if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):

                     print_value = ("%s...%d_chars...%s") % (data["value"][:2], len(data["value"]) - 3, data["value"][-1:])

                 else:
                     print_value = data["value"]
                 debug("ConfigParser: %s->%s" % (data["key"], print_value))
                 continue
             warning("Ignoring invalid line in '%s': %s" % (file, line))
 
     def __getitem__(self, name):
         return self.cfg[name]
 
     def __setitem__(self, name, value):
         self.cfg[name] = value
 
     def get(self, name, default = None):
         if self.cfg.has_key(name):
             return self.cfg[name]
         return default

 
 class ConfigDumper(object):

     def __init__(self, stream):
         self.stream = stream

     def dump(self, section, config):
         self.stream.write("[%s]\n" % section)
         for option in config.option_list():

             value = getattr(config, option)
             if option == "verbosity":
                 # we turn level numbers back into strings if possible
                 if isinstance(value,int) and value in logging._levelNames:
                     value = logging._levelNames[value]
 
             self.stream.write("%s = %s\n" % (option, value))

 # vim:et:ts=4:sts=4:ai