@@ -1,3 +1,7 @@

+2010-03-19  Michal Ludvig  <mludvig@logix.net.nz>

+

+	* s3cmd, S3/AccessLog.py, ...: Added [accesslog] command.

+

 2009-12-10  Michal Ludvig  <mludvig@logix.net.nz>

 	* s3cmd: Path separator conversion on Windows hosts.

@@ -1,3 +1,7 @@

+s3cmd 0.9.9.92 -  ???

+==============

+* Added [accesslog] command. (needs manpage!)

+

 s3cmd 0.9.9.91 -  2009-10-08

 ==============

 * Fixed invalid reference to a variable in failed upload handling.

@@ -3,7 +3,7 @@

 ##         http://www.logix.cz/michal

 ## License: GPL Version 2

-from Utils import *

+from Utils import getTreeFromXml

 try:

 	import xml.etree.ElementTree as ET

@@ -12,6 +12,7 @@ except ImportError:

 class Grantee(object):

 	ALL_USERS_URI = "http://acs.amazonaws.com/groups/global/AllUsers"

+	LOG_DELIVERY_URI = "http://acs.amazonaws.com/groups/s3/LogDelivery"

 	def __init__(self):

 		self.xsi_type = None

@@ -53,6 +54,17 @@ class GranteeAnonRead(Grantee):

 		self.name = Grantee.ALL_USERS_URI

 		self.permission = "READ"

+class GranteeLogDelivery(Grantee):

+	def __init__(self, permission):

+		"""

+		permission must be either READ_ACP or WRITE

+		"""

+		Grantee.__init__(self)

+		self.xsi_type = "Group"

+		self.tag = "URI"

+		self.name = Grantee.LOG_DELIVERY_URI

+		self.permission = permission

+

 class ACL(object):

 	EMPTY_ACL = "<AccessControlPolicy><Owner><ID></ID></Owner><AccessControlList></AccessControlList></AccessControlPolicy>"

@@ -109,11 +121,14 @@ class ACL(object):

 	def grantAnonRead(self):

 		if not self.isAnonRead():

-			self.grantees.append(GranteeAnonRead())

+			self.appendGrantee(GranteeAnonRead())

 	def revokeAnonRead(self):

 		self.grantees = [g for g in self.grantees if not g.isAnonRead()]

+	def appendGrantee(self, grantee):

+		self.grantees.append(grantee)

+

 	def __str__(self):

 		tree = getTreeFromXml(ACL.EMPTY_ACL)

 		tree.attrib['xmlns'] = "http://s3.amazonaws.com/doc/2006-03-01/"

new file mode 100644

@@ -0,0 +1,90 @@

+## Amazon S3 - Access Control List representation

+## Author: Michal Ludvig <michal@logix.cz>

+##         http://www.logix.cz/michal

+## License: GPL Version 2

+

+import S3Uri

+from Exceptions import ParameterError

+from Utils import getTreeFromXml

+from ACL import GranteeAnonRead

+

+try:

+	import xml.etree.ElementTree as ET

+except ImportError:

+	import elementtree.ElementTree as ET

+

+__all__ = []

+class AccessLog(object):

+	LOG_DISABLED = "<BucketLoggingStatus></BucketLoggingStatus>"

+	LOG_TEMPLATE = "<LoggingEnabled><TargetBucket></TargetBucket><TargetPrefix></TargetPrefix></LoggingEnabled>"

+

+	def __init__(self, xml = None):

+		if not xml:

+			xml = self.LOG_DISABLED

+		self.tree = getTreeFromXml(xml)

+		self.tree.attrib['xmlns'] = "http://doc.s3.amazonaws.com/2006-03-01"

+	

+	def isLoggingEnabled(self):

+		return bool(self.tree.find(".//LoggingEnabled"))

+

+	def disableLogging(self):

+		el = self.tree.find(".//LoggingEnabled")

+		if el:

+			self.tree.remove(el)

+	

+	def enableLogging(self, target_prefix_uri):

+		el = self.tree.find(".//LoggingEnabled")

+		if not el:

+			el = getTreeFromXml(self.LOG_TEMPLATE)

+			self.tree.append(el)

+		el.find(".//TargetBucket").text = target_prefix_uri.bucket()

+		el.find(".//TargetPrefix").text = target_prefix_uri.object()

+

+	def targetPrefix(self):

+		if self.isLoggingEnabled():

+			el = self.tree.find(".//LoggingEnabled")

+			target_prefix = "s3://%s/%s" % (

+				self.tree.find(".//LoggingEnabled//TargetBucket").text, 

+				self.tree.find(".//LoggingEnabled//TargetPrefix").text)

+			return S3Uri.S3Uri(target_prefix)

+		else:

+			return ""

+

+	def setAclPublic(self, acl_public):

+		le = self.tree.find(".//LoggingEnabled")

+		if not le:

+			raise ParameterError("Logging not enabled, can't set default ACL for logs")

+		tg = le.find(".//TargetGrants")

+		if not acl_public:

+			if not tg:

+				## All good, it's not been there

+				return

+			else:

+				le.remove(tg)

+		else: # acl_public == True

+			anon_read = GranteeAnonRead().getElement()

+			if not tg:

+				tg = ET.SubElement(le, "TargetGrants")

+			## What if TargetGrants already exists? We should check if 

+			## AnonRead is there before appending a new one. Later...

+			tg.append(anon_read)

+

+	def isAclPublic(self):

+		raise NotImplementedError()

+

+	def __str__(self):

+		return ET.tostring(self.tree)

+__all__.append("AccessLog")

+

+if __name__ == "__main__":

+	from S3Uri import S3Uri

+	log = AccessLog()

+	print log

+	log.enableLogging(S3Uri("s3://targetbucket/prefix/log-"))

+	print log

+	log.setAclPublic(True)

+	print log

+	log.setAclPublic(False)

+	print log

+	log.disableLogging()

+	print log

@@ -29,6 +29,7 @@ class Config(object):

 	human_readable_sizes = False

 	extra_headers = SortedDict(ignore_case = True)

 	force = False

+	enable = None

 	get_continue = False

 	skip_existing = False

 	recursive = False

@@ -69,6 +70,7 @@ class Config(object):

 	debug_include = {}

 	encoding = "utf-8"

 	urlencoding_mode = "normal"

+	log_target_prefix = ""

 	## Creating a singleton

 	def __new__(self, configfile = None):

@@ -9,6 +9,7 @@ import time

 import httplib

 import logging

 import mimetypes

+import re

 from logging import debug, info, warning, error

 from stat import ST_SIZE

@@ -22,8 +23,11 @@ from SortedDict import SortedDict

 from BidirMap import BidirMap

 from Config import Config

 from Exceptions import *

-from ACL import ACL

+from ACL import ACL, GranteeLogDelivery

+from AccessLog import AccessLog

+from S3Uri import S3Uri

+__all__ = []

 class S3Request(object):

 	def __init__(self, s3, method_string, resource, headers, params = {}):

 		self.s3 = s3

@@ -322,6 +326,41 @@ class S3(object):

 		response = self.send_request(request, body)

 		return response

+	def get_accesslog(self, uri):

+		request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?logging")

+		response = self.send_request(request)

+		accesslog = AccessLog(response['data'])

+		return accesslog

+

+	def set_accesslog_acl(self, uri):

+		acl = self.get_acl(uri)

+		debug("Current ACL(%s): %s" % (uri.uri(), str(acl)))

+		acl.appendGrantee(GranteeLogDelivery("READ_ACP"))

+		acl.appendGrantee(GranteeLogDelivery("WRITE"))

+		debug("Updated ACL(%s): %s" % (uri.uri(), str(acl)))

+		self.set_acl(uri, acl)

+

+	def set_accesslog(self, uri, enable, log_target_prefix_uri = None, acl_public = False):

+		request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?logging")

+		accesslog = AccessLog()

+		if enable:

+			accesslog.enableLogging(log_target_prefix_uri)

+			accesslog.setAclPublic(acl_public)

+		else:

+			accesslog.disableLogging()

+		body = str(accesslog)

+		debug(u"set_accesslog(%s): accesslog-xml: %s" % (uri, body))

+		try:

+			response = self.send_request(request, body)

+		except S3Error, e:

+			if e.info['Code'] == "InvalidTargetBucketForLogging":

+				info("Setting up log-delivery ACL for target bucket.")

+				self.set_accesslog_acl(S3Uri("s3://%s" % log_target_prefix_uri.bucket()))

+				response = self.send_request(request, body)

+			else:

+				raise

+		return accesslog, response

+

 	## Low level methods

 	def urlencode_string(self, string, urlencoding_mode = None):

 		if type(string) == unicode:

@@ -720,3 +759,4 @@ class S3(object):

 			return S3.check_bucket_name(bucket, dns_strict = True)

 		except ParameterError:

 			return False

+__all__.append("S3")

@@ -8,7 +8,7 @@ import re

 import sys

 from BidirMap import BidirMap

 from logging import debug

-from S3 import S3

+import S3

 from Utils import unicodise

 class S3Uri(object):

@@ -73,7 +73,7 @@ class S3UriS3(S3Uri):

 		return "/".join(["s3:/", self._bucket, self._object])

 	def is_dns_compatible(self):

-		return S3.check_bucket_name_dns_conformity(self._bucket)

+		return S3.S3.check_bucket_name_dns_conformity(self._bucket)

 	def public_url(self):

 		if self.is_dns_compatible():

@@ -29,6 +29,7 @@ except ImportError:

 	import elementtree.ElementTree as ET

 from xml.parsers.expat import ExpatError

+__all__ = []

 def parseNodes(nodes):

 	## WARNING: Ignores text nodes from mixed xml/text.

 	## For instance <tag1>some text<tag2>other text</tag2></tag1>

@@ -44,6 +45,7 @@ def parseNodes(nodes):

 				retval_item[name] = node.findtext(".//%s" % child.tag)

 		retval.append(retval_item)

 	return retval

+__all__.append("parseNodes")

 def stripNameSpace(xml):

 	"""

@@ -56,6 +58,7 @@ def stripNameSpace(xml):

 	else:

 		xmlns = None

 	return xml, xmlns

+__all__.append("stripNameSpace")

 def getTreeFromXml(xml):

 	xml, xmlns = stripNameSpace(xml)

@@ -67,11 +70,13 @@ def getTreeFromXml(xml):

 	except ExpatError, e:

 		error(e)

 		raise Exceptions.ParameterError("Bucket contains invalid filenames. Please run: s3cmd fixbucket s3://your-bucket/")

+__all__.append("getTreeFromXml")

 def getListFromXml(xml, node):

 	tree = getTreeFromXml(xml)

 	nodes = tree.findall('.//%s' % (node))

 	return parseNodes(nodes)

+__all__.append("getListFromXml")

 def getDictFromTree(tree):

 	ret_dict = {}

@@ -86,6 +91,7 @@ def getDictFromTree(tree):

 		else:

 			ret_dict[child.tag] = child.text or ""

 	return ret_dict

+__all__.append("getDictFromTree")

 def getTextFromXml(xml, xpath):

 	tree = getTreeFromXml(xml)

@@ -93,15 +99,18 @@ def getTextFromXml(xml, xpath):

 		return tree.text

 	else:

 		return tree.findtext(xpath)

+__all__.append("getTextFromXml")

 def getRootTagName(xml):

 	tree = getTreeFromXml(xml)

 	return tree.tag

+__all__.append("getRootTagName")

 def xmlTextNode(tag_name, text):

 	el = ET.Element(tag_name)

 	el.text = unicode(text)

 	return el

+__all__.append("xmlTextNode")

 def appendXmlTextNode(tag_name, text, parent):

 	"""

@@ -111,22 +120,27 @@ def appendXmlTextNode(tag_name, text, parent):

 	Returns the newly created Node.

 	"""

 	parent.append(xmlTextNode(tag_name, text))

+__all__.append("appendXmlTextNode")

 def dateS3toPython(date):

 	date = re.compile("(\.\d*)?Z").sub(".000Z", date)

 	return time.strptime(date, "%Y-%m-%dT%H:%M:%S.000Z")

+__all__.append("dateS3toPython")

 def dateS3toUnix(date):

 	## FIXME: This should be timezone-aware.

 	## Currently the argument to strptime() is GMT but mktime() 

 	## treats it as "localtime". Anyway...

 	return time.mktime(dateS3toPython(date))

+__all__.append("dateS3toUnix")

 def dateRFC822toPython(date):

 	return rfc822.parsedate(date)

+__all__.append("dateRFC822toPython")

 def dateRFC822toUnix(date):

 	return time.mktime(dateRFC822toPython(date))

+__all__.append("dateRFC822toUnix")

 def formatSize(size, human_readable = False, floating_point = False):

 	size = floating_point and float(size) or int(size)

@@ -139,16 +153,18 @@ def formatSize(size, human_readable = False, floating_point = False):

 		return (size, coeff)

 	else:

 		return (size, "")

+__all__.append("formatSize")

 def formatDateTime(s3timestamp):

 	return time.strftime("%Y-%m-%d %H:%M", dateS3toPython(s3timestamp))

+__all__.append("formatDateTime")

 def convertTupleListToDict(list):

 	retval = {}

 	for tuple in list:

 		retval[tuple[0]] = tuple[1]

 	return retval

-

+__all__.append("convertTupleListToDict")

 _rnd_chars = string.ascii_letters+string.digits

 _rnd_chars_len = len(_rnd_chars)

@@ -158,6 +174,7 @@ def rndstr(len):

 		retval += _rnd_chars[random.randint(0, _rnd_chars_len-1)]

 		len -= 1

 	return retval

+__all__.append("rndstr")

 def mktmpsomething(prefix, randchars, createfunc):

 	old_umask = os.umask(0077)

@@ -175,13 +192,16 @@ def mktmpsomething(prefix, randchars, createfunc):

 	os.umask(old_umask)

 	return dirname

+__all__.append("mktmpsomething")

 def mktmpdir(prefix = "/tmp/tmpdir-", randchars = 10):

 	return mktmpsomething(prefix, randchars, os.mkdir)

+__all__.append("mktmpdir")

 def mktmpfile(prefix = "/tmp/tmpfile-", randchars = 20):

 	createfunc = lambda filename : os.close(os.open(filename, os.O_CREAT | os.O_EXCL))

 	return mktmpsomething(prefix, randchars, createfunc)

+__all__.append("mktmpfile")

 def hash_file_md5(filename):

 	h = md5()

@@ -194,6 +214,7 @@ def hash_file_md5(filename):

 		h.update(data)

 	f.close()

 	return h.hexdigest()

+__all__.append("hash_file_md5")

 def mkdir_with_parents(dir_name):

 	"""

@@ -220,6 +241,7 @@ def mkdir_with_parents(dir_name):

 			warning("%s: %s" % (cur_dir, e))

 			return False

 	return True

+__all__.append("mkdir_with_parents")

 def unicodise(string, encoding = None, errors = "replace"):

 	"""

@@ -236,6 +258,7 @@ def unicodise(string, encoding = None, errors = "replace"):

 		return string.decode(encoding, errors)

 	except UnicodeDecodeError:

 		raise UnicodeDecodeError("Conversion to unicode failed: %r" % string)

+__all__.append("unicodise")

 def deunicodise(string, encoding = None, errors = "replace"):

 	"""

@@ -253,6 +276,7 @@ def deunicodise(string, encoding = None, errors = "replace"):

 		return string.encode(encoding, errors)

 	except UnicodeEncodeError:

 		raise UnicodeEncodeError("Conversion from unicode failed: %r" % string)

+__all__.append("deunicodise")

 def unicodise_safe(string, encoding = None):

 	"""

@@ -261,6 +285,7 @@ def unicodise_safe(string, encoding = None):

 	"""

 	return unicodise(deunicodise(string, encoding), encoding).replace(u'\ufffd', '?')

+__all__.append("unicodise_safe")

 def replace_nonprintables(string):

 	"""

@@ -284,9 +309,11 @@ def replace_nonprintables(string):

 	if modified and Config.Config().urlencoding_mode != "fixbucket":

 		warning("%d non-printable characters replaced in: %s" % (modified, new_string))

 	return new_string

+__all__.append("replace_nonprintables")

 def sign_string(string_to_sign):

 	#debug("string_to_sign: %s" % string_to_sign)

 	signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip()

 	#debug("signature: %s" % signature)

 	return signature

+__all__.append("sign_string")

@@ -1122,6 +1122,27 @@ def cmd_setacl(args):

 		if retsponse['status'] == 200:

 			output(u"%s: ACL set to %s  %s" % (uri, set_to_acl, seq_label))

+def cmd_accesslog(args):

+	s3 = S3(cfg)

+	bucket_uri = S3Uri(args.pop())

+	if bucket_uri.object():

+		raise ParameterError("Only bucket name is required for [accesslog] command")

+	if cfg.enable == True:

+		log_target_prefix_uri = S3Uri(cfg.log_target_prefix)

+		if log_target_prefix_uri.type != "s3":

+			raise ParameterError("--log-target-prefix must be a S3 URI")

+		accesslog, response = s3.set_accesslog(bucket_uri, enable = True, log_target_prefix_uri = log_target_prefix_uri, acl_public = cfg.acl_public)

+	elif cfg.enable == False:

+		accesslog, response = s3.set_accesslog(bucket_uri, enable = False)

+	else:	# cfg.enable == None

+		accesslog = s3.get_accesslog(bucket_uri)

+

+	output(u"Access logging for: %s" % bucket_uri.uri())

+	output(u"   Logging Enabled: %s" % accesslog.isLoggingEnabled())

+	if accesslog.isLoggingEnabled():

+		output(u"     Target prefix: %s" % accesslog.targetPrefix().uri())

+		#output(u"   Public Access:   %s" % accesslog.isAclPublic())

+		

 def cmd_sign(args):

 	string_to_sign = args.pop()

 	debug("string-to-sign: %r" % string_to_sign)

@@ -1426,6 +1447,7 @@ def get_commands_list():

 	{"cmd":"cp", "label":"Copy object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_cp, "argc":2},

 	{"cmd":"mv", "label":"Move object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_mv, "argc":2},

 	{"cmd":"setacl", "label":"Modify Access control list for Bucket or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1},

+	{"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1},

 	{"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1},

 	{"cmd":"fixbucket", "label":"Fix invalid file names in a bucket", "param":"s3://BUCKET[/PREFIX]", "func":cmd_fixbucket, "argc":1},

@@ -1516,6 +1538,8 @@ def main():

 	optparser.add_option(      "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. Either EU or US (default)")

+	optparser.add_option(      "--log-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI)")

+

 	optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", help="Default MIME-type to be set for objects stored.")

 	optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option")

@@ -1529,8 +1553,8 @@ def main():

 	optparser.add_option(      "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default on TTY).")

 	optparser.add_option(      "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter (default on non-TTY).")

-	optparser.add_option(      "--enable", dest="cf_enable", action="store_true", help="Enable given CloudFront distribution (only for [cfmodify] command)")

-	optparser.add_option(      "--disable", dest="cf_enable", action="store_false", help="Enable given CloudFront distribution (only for [cfmodify] command)")

+	optparser.add_option(      "--enable", dest="enable", action="store_true", help="Enable given CloudFront distribution (for [cfmodify] command) or access logging (for [accesslog] command)")

+	optparser.add_option(      "--disable", dest="enable", action="store_false", help="Enable given CloudFront distribution (only for [cfmodify] command) or access logging (for [accesslog] command)")

 	optparser.add_option(      "--cf-add-cname", dest="cf_cnames_add", action="append", metavar="CNAME", help="Add given CNAME to a CloudFront distribution (only for [cfcreate] and [cfmodify] commands)")

 	optparser.add_option(      "--cf-remove-cname", dest="cf_cnames_remove", action="append", metavar="CNAME", help="Remove given CNAME from a CloudFront distribution (only for [cfmodify] command)")

 	optparser.add_option(      "--cf-comment", dest="cf_comment", action="store", metavar="COMMENT", help="Set COMMENT for a given CloudFront distribution (only for [cfcreate] and [cfmodify] commands)")

@@ -1617,7 +1641,13 @@ def main():

 		except AttributeError:

 			## Some Config() options are not settable from command line

 			pass

-	

+

+	## Special handling for tri-state options (True, False, None)

+	cfg.update_option("enable", options.enable)

+

+	## CloudFront's cf_enable and Config's enable share the same --enable switch

+	options.cf_enable = options.enable

+

 	## Update CloudFront options if some were set

 	for option in CfCmd.options.option_list():

 		try:

@@ -1735,9 +1765,9 @@ if __name__ == '__main__':

 		## detect any syntax errors in there

 		from S3.Exceptions import *

 		from S3 import PkgInfo

-		from S3.S3 import *

+		from S3.S3 import S3

 		from S3.Config import Config

-		from S3.S3Uri import *

+		from S3.S3Uri import S3Uri

 		from S3 import Utils

 		from S3.Utils import unicodise

 		from S3.Progress import Progress