@@ -1,3 +1,11 @@

+2010-06-12  Michal Ludvig  <mludvig@logix.net.nz>

+

+	* s3cmd, S3/CloudFront.py, S3/Config.py: Support access 

+	  logging for CloudFront distributions.

+	* S3/S3.py, S3/Utils.py: Moved some functions to Utils.py

+	  to make them available to CloudFront.py

+	* NEWS: Document the above.

+

 2010-05-27  Michal Ludvig  <mludvig@logix.net.nz>

 	* S3/S3.py: Fix bucket listing for buckets with

@@ -1,6 +1,8 @@

 s3cmd 0.9.9.92 -  ???

 ==============

 * Added [accesslog] command. (needs manpage!)

+* Added access logging for CloudFront distributions 

+  using [cfmodify --log]

 * Added --acl-grant and --acl-revoke (by Timothee Linden).

 s3cmd 0.9.9.91 -  2009-10-08

@@ -15,7 +15,7 @@ except ImportError:

 from Config import Config

 from Exceptions import *

-from Utils import getTreeFromXml, appendXmlTextNode, getDictFromTree, dateS3toPython, sign_string

+from Utils import getTreeFromXml, appendXmlTextNode, getDictFromTree, dateS3toPython, sign_string, getBucketFromHostname, getHostnameFromBucket

 from S3Uri import S3Uri, S3UriS3

 def output(message):

@@ -53,7 +53,7 @@ class DistributionSummary(object):

 class DistributionList(object):

 	## Example:

 	## 

-	## <DistributionList xmlns="http://cloudfront.amazonaws.com/doc/2008-06-30/">

+	## <DistributionList xmlns="http://cloudfront.amazonaws.com/doc/2010-06-01/">

 	##	<Marker />

 	##	<MaxItems>100</MaxItems>

 	##	<IsTruncated>false</IsTruncated>

@@ -80,7 +80,7 @@ class DistributionList(object):

 class Distribution(object):

 	## Example:

 	##

-	## <Distribution xmlns="http://cloudfront.amazonaws.com/doc/2008-06-30/">

+	## <Distribution xmlns="http://cloudfront.amazonaws.com/doc/2010-06-01/">

 	##	<Id>1234567890ABC</Id>

 	##	<Status>InProgress</Status>

 	##	<LastModifiedTime>2009-01-16T13:07:11.319Z</LastModifiedTime>

@@ -114,10 +114,14 @@ class DistributionConfig(object):

 	##	<CallerReference>s3://somebucket/</CallerReference>

 	##	<Comment>http://somebucket.s3.amazonaws.com/</Comment>

 	##	<Enabled>true</Enabled>

+	##  <Logging>

+	##    <Bucket>bu.ck.et</Bucket>

+	##    <Prefix>/cf-somebucket/</Prefix>

+	##  </Logging>

 	## </DistributionConfig>

 	EMPTY_CONFIG = "<DistributionConfig><Origin/><CallerReference/><Enabled>true</Enabled></DistributionConfig>"

-	xmlns = "http://cloudfront.amazonaws.com/doc/2008-06-30/"

+	xmlns = "http://cloudfront.amazonaws.com/doc/2010-06-01/"

 	def __init__(self, xml = None, tree = None):

 		if not xml:

 			xml = DistributionConfig.EMPTY_CONFIG

@@ -139,6 +143,16 @@ class DistributionConfig(object):

 		self.info['CNAME'] = [cname.lower() for cname in self.info['CNAME']]

 		if not self.info.has_key("Comment"):

 			self.info['Comment'] = ""

+		## Figure out logging - complex node not parsed by getDictFromTree()

+		logging_nodes = tree.findall(".//Logging")

+		if logging_nodes:

+			logging_dict = getDictFromTree(logging_nodes[0])

+			logging_dict['Bucket'], success = getBucketFromHostname(logging_dict['Bucket'])

+			if not success:

+				warning("Logging to unparsable bucket name: %s" % logging_dict['Bucket'])

+			self.info['Logging'] = S3UriS3("s3://%(Bucket)s/%(Prefix)s" % logging_dict)

+		else:

+			self.info['Logging'] = None

 	def __str__(self):

 		tree = ET.Element("DistributionConfig")

@@ -152,7 +166,11 @@ class DistributionConfig(object):

 		if self.info['Comment']:

 			appendXmlTextNode("Comment", self.info['Comment'], tree)

 		appendXmlTextNode("Enabled", str(self.info['Enabled']).lower(), tree)

-

+		if self.info['Logging']:

+			logging_el = ET.Element("Logging")

+			appendXmlTextNode("Bucket", getHostnameFromBucket(self.info['Logging'].bucket()), logging_el)

+			appendXmlTextNode("Prefix", self.info['Logging'].object(), logging_el)

+			tree.append(logging_el)

 		return ET.tostring(tree)

 class CloudFront(object):

@@ -183,7 +201,7 @@ class CloudFront(object):

 		## TODO: handle Truncated 

 		return response

-	def CreateDistribution(self, uri, cnames_add = [], comment = None):

+	def CreateDistribution(self, uri, cnames_add = [], comment = None, logging = None):

 		dist_config = DistributionConfig()

 		dist_config.info['Enabled'] = True

 		dist_config.info['Origin'] = uri.host_name()

@@ -195,6 +213,8 @@ class CloudFront(object):

 		for cname in cnames_add:

 			if dist_config.info['CNAME'].count(cname) == 0:

 				dist_config.info['CNAME'].append(cname)

+		if logging != None:

+			dist_config.info['Logging'] = S3UriS3(logging)

 		request_body = str(dist_config)

 		debug("CreateDistribution(): request_body: %s" % request_body)

 		response = self.send_request("CreateDist", body = request_body)

@@ -202,7 +222,7 @@ class CloudFront(object):

 		return response

 	def ModifyDistribution(self, cfuri, cnames_add = [], cnames_remove = [],

-	                       comment = None, enabled = None):

+	                       comment = None, enabled = None, logging = None):

 		if cfuri.type != "cf":

 			raise ValueError("Expected CFUri instead of: %s" % cfuri)

 		# Get current dist status (enabled/disabled) and Etag

@@ -219,6 +239,8 @@ class CloudFront(object):

 		for cname in cnames_remove:

 			while dc.info['CNAME'].count(cname) > 0:

 				dc.info['CNAME'].remove(cname)

+		if logging != None:

+			dc.info['Logging'] = S3UriS3(logging)

 		response = self.SetDistConfig(cfuri, dc, response['headers']['etag'])

 		return response

@@ -364,6 +386,7 @@ class Cmd(object):

 		cf_cnames_remove = []

 		cf_comment = None

 		cf_enable = None

+		cf_logging = None

 		def option_list(self):

 			return [opt for opt in dir(self) if opt.startswith("cf_")]

@@ -402,6 +425,7 @@ class Cmd(object):

 				pretty_output("CNAMEs", ", ".join(dc.info['CNAME']))

 				pretty_output("Comment", dc.info['Comment'])

 				pretty_output("Enabled", dc.info['Enabled'])

+				pretty_output("Logging", dc.info['Logging'] or "Disabled")

 				pretty_output("Etag", response['headers']['etag'])

 	@staticmethod

@@ -422,7 +446,8 @@ class Cmd(object):

 		for uri in buckets:

 			info("Creating distribution from: %s" % uri)

 			response = cf.CreateDistribution(uri, cnames_add = Cmd.options.cf_cnames_add, 

-			                                 comment = Cmd.options.cf_comment)

+			                                 comment = Cmd.options.cf_comment,

+			                                 logging = Cmd.options.cf_logging)

 			d = response['distribution']

 			dc = d.info['DistributionConfig']

 			output("Distribution created:")

@@ -462,7 +487,8 @@ class Cmd(object):

 		                                 cnames_add = Cmd.options.cf_cnames_add,

 		                                 cnames_remove = Cmd.options.cf_cnames_remove,

 		                                 comment = Cmd.options.cf_comment,

-		                                 enabled = Cmd.options.cf_enable)

+		                                 enabled = Cmd.options.cf_enable,

+		                                 logging = Cmd.options.cf_logging)

 		if response['status'] >= 400:

 			error("Distribution %s could not be modified: %s" % (cfuri, response['reason']))

 		output("Distribution modified: %s" % cfuri)

@@ -19,7 +19,7 @@ class Config(object):

 	host_bucket = "%(bucket)s.s3.amazonaws.com"

 	simpledb_host = "sdb.amazonaws.com"

 	cloudfront_host = "cloudfront.amazonaws.com"

-	cloudfront_resource = "/2008-06-30/distribution"

+	cloudfront_resource = "/2010-06-01/distribution"

 	verbosity = logging.WARNING

 	progress_meter = True

 	progress_class = Progress.ProgressCR

@@ -135,11 +135,11 @@ class S3(object):

 				return httplib.HTTPConnection(self.get_hostname(bucket))

 	def get_hostname(self, bucket):

-		if bucket and self.check_bucket_name_dns_conformity(bucket):

+		if bucket and check_bucket_name_dns_conformity(bucket):

 			if self.redir_map.has_key(bucket):

 				host = self.redir_map[bucket]

 			else:

-				host = self.config.host_bucket % { 'bucket' : bucket }

+				host = getHostnameFromBucket(bucket)

 		else:

 			host = self.config.host_base

 		debug('get_hostname(%s): %s' % (bucket, host))

@@ -149,7 +149,7 @@ class S3(object):

 		self.redir_map[bucket] = redir_hostname

 	def format_uri(self, resource):

-		if resource['bucket'] and not self.check_bucket_name_dns_conformity(resource['bucket']):

+		if resource['bucket'] and not check_bucket_name_dns_conformity(resource['bucket']):

 			uri = "/%s%s" % (resource['bucket'], resource['uri'])

 		else:

 			uri = resource['uri']

@@ -224,9 +224,9 @@ class S3(object):

 			body += bucket_location

 			body += "</LocationConstraint></CreateBucketConfiguration>"

 			debug("bucket_location: " + body)

-			self.check_bucket_name(bucket, dns_strict = True)

+			check_bucket_name(bucket, dns_strict = True)

 		else:

-			self.check_bucket_name(bucket, dns_strict = False)

+			check_bucket_name(bucket, dns_strict = False)

 		if self.config.acl_public:

 			headers["x-amz-acl"] = "public-read"

 		request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers)

@@ -753,39 +753,4 @@ class S3(object):

 			warning("MD5 signatures do not match: computed=%s, received=%s" % (

 				response["md5"], response["headers"]["etag"]))

 		return response

-

-	@staticmethod

-	def check_bucket_name(bucket, dns_strict = True):

-		if dns_strict:

-			invalid = re.search("([^a-z0-9\.-])", bucket)

-			if invalid:

-				raise ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: lowercase us-ascii letters (a-z), digits (0-9), dot (.) and hyphen (-)." % (bucket, invalid.groups()[0]))

-		else:

-			invalid = re.search("([^A-Za-z0-9\._-])", bucket)

-			if invalid:

-				raise ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: us-ascii letters (a-z, A-Z), digits (0-9), dot (.), hyphen (-) and underscore (_)." % (bucket, invalid.groups()[0]))

-

-		if len(bucket) < 3:

-			raise ParameterError("Bucket name '%s' is too short (min 3 characters)" % bucket)

-		if len(bucket) > 255:

-			raise ParameterError("Bucket name '%s' is too long (max 255 characters)" % bucket)

-		if dns_strict:

-			if len(bucket) > 63:

-				raise ParameterError("Bucket name '%s' is too long (max 63 characters)" % bucket)

-			if re.search("-\.", bucket):

-				raise ParameterError("Bucket name '%s' must not contain sequence '-.' for DNS compatibility" % bucket)

-			if re.search("\.\.", bucket):

-				raise ParameterError("Bucket name '%s' must not contain sequence '..' for DNS compatibility" % bucket)

-			if not re.search("^[0-9a-z]", bucket):

-				raise ParameterError("Bucket name '%s' must start with a letter or a digit" % bucket)

-			if not re.search("[0-9a-z]$", bucket):

-				raise ParameterError("Bucket name '%s' must end with a letter or a digit" % bucket)

-		return True

-

-	@staticmethod

-	def check_bucket_name_dns_conformity(bucket):

-		try:

-			return S3.check_bucket_name(bucket, dns_strict = True)

-		except ParameterError:

-			return False

 __all__.append("S3")

@@ -119,7 +119,9 @@ def appendXmlTextNode(tag_name, text, parent):

 	created Node to 'parent' element if given.

 	Returns the newly created Node.

 	"""

-	parent.append(xmlTextNode(tag_name, text))

+	el = xmlTextNode(tag_name, text)

+	parent.append(el)

+	return el

 __all__.append("appendXmlTextNode")

 def dateS3toPython(date):

@@ -317,3 +319,60 @@ def sign_string(string_to_sign):

 	#debug("signature: %s" % signature)

 	return signature

 __all__.append("sign_string")

+

+def check_bucket_name(bucket, dns_strict = True):

+	if dns_strict:

+		invalid = re.search("([^a-z0-9\.-])", bucket)

+		if invalid:

+			raise ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: lowercase us-ascii letters (a-z), digits (0-9), dot (.) and hyphen (-)." % (bucket, invalid.groups()[0]))

+	else:

+		invalid = re.search("([^A-Za-z0-9\._-])", bucket)

+		if invalid:

+			raise ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: us-ascii letters (a-z, A-Z), digits (0-9), dot (.), hyphen (-) and underscore (_)." % (bucket, invalid.groups()[0]))

+

+	if len(bucket) < 3:

+		raise ParameterError("Bucket name '%s' is too short (min 3 characters)" % bucket)

+	if len(bucket) > 255:

+		raise ParameterError("Bucket name '%s' is too long (max 255 characters)" % bucket)

+	if dns_strict:

+		if len(bucket) > 63:

+			raise ParameterError("Bucket name '%s' is too long (max 63 characters)" % bucket)

+		if re.search("-\.", bucket):

+			raise ParameterError("Bucket name '%s' must not contain sequence '-.' for DNS compatibility" % bucket)

+		if re.search("\.\.", bucket):

+			raise ParameterError("Bucket name '%s' must not contain sequence '..' for DNS compatibility" % bucket)

+		if not re.search("^[0-9a-z]", bucket):

+			raise ParameterError("Bucket name '%s' must start with a letter or a digit" % bucket)

+		if not re.search("[0-9a-z]$", bucket):

+			raise ParameterError("Bucket name '%s' must end with a letter or a digit" % bucket)

+	return True

+__all__.append("check_bucket_name")

+

+def check_bucket_name_dns_conformity(bucket):

+	try:

+		return check_bucket_name(bucket, dns_strict = True)

+	except ParameterError:

+		return False

+__all__.append("check_bucket_name_dns_conformity")

+

+def getBucketFromHostname(hostname):

+	"""

+	bucket, success = getBucketFromHostname(hostname)

+

+	Only works for hostnames derived from bucket names

+	using Config.host_bucket pattern.

+

+	Returns bucket name and a boolean success flag.

+	"""

+

+	# Create RE pattern from Config.host_bucket

+	pattern = Config.Config().host_bucket % { 'bucket' : '(?P<bucket>.*)' }

+	m = re.match(pattern, hostname)

+	if not m:

+		return (hostname, False)

+	return m.groups()[0], True

+__all__.append("getBucketFromHostname")

+

+def getHostnameFromBucket(bucket):

+	return Config.Config().host_bucket % { 'bucket' : bucket }

+__all__.append("getHostnameFromBucket")

@@ -1599,7 +1599,7 @@ def main():

 	optparser.add_option(      "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, us-west-1, and ap-southeast-1")

 	optparser.add_option(      "--reduced-redundancy", "--rr", dest="reduced_redundancy", action="store_true", help="Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]")

-	optparser.add_option(      "--log-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI)")

+	optparser.add_option(      "--log-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI) (for [cfmodify] and [accesslog] commands)")

 	optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", help="Default MIME-type to be set for objects stored.")

 	optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option")

@@ -1719,6 +1719,9 @@ def main():

 	## CloudFront's cf_enable and Config's enable share the same --enable switch

 	options.cf_enable = options.enable

+	## CloudFront's cf_logging and Config's log_target_prefix share the same --log-target-prefix switch

+	options.cf_logging = options.log_target_prefix

+

 	## Update CloudFront options if some were set

 	for option in CfCmd.options.option_list():

 		try: