@@ -1,3 +1,8 @@

+2007-11-13  Michal Ludvig  <michal@logix.cz>

+

+	* S3/S3.py: Support for buckets stored in Europe, access now 

+	  goes via <bucket>.s3.amazonaws.com where possible.

+

 2007-11-12  Michal Ludvig  <michal@logix.cz>

 	* s3cmd: Support for storing file attributes (like ownership, 

@@ -1,5 +1,7 @@

 s3cmd 0.9.5   -   ???

 ===========

+* Support for buckets created in Europe

+* Initial 'sync' support, for now local to s3 direction only

 * Much better handling of multiple args to put, get and del

 * Tries to use ElementTree from any available module

 * Support for buckets with over 1000 objects.

@@ -13,7 +13,8 @@ class Config(object):

 	_doc = {}

 	access_key = ""

 	secret_key = ""

-	host = "s3.amazonaws.com"

+	host_base = "s3.amazonaws.com"

+	host_bucket = "%(bucket)s.s3.amazonaws.com"

 	verbosity = logging.WARNING

 	send_chunk = 4096

 	recv_chunk = 4096

@@ -42,6 +43,7 @@ class Config(object):

 	gpg_encrypt = "%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"

 	gpg_decrypt = "%(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"

 	use_https = False

+	bucket_location = "US"

 	## Creating a singleton

 	def __new__(self, configfile = None):

@@ -39,7 +39,7 @@ class S3Error (Exception):

 		retval = "%d (%s)" % (self.status, self.reason)

 		try:

 			retval += (": %s" % self.info["Code"])

-		except AttributeError:

+		except (AttributeError, KeyError):

 			pass

 		return retval

@@ -83,19 +83,30 @@ class S3(object):

 	def __init__(self, config):

 		self.config = config

-	def get_connection(self):

-		if self.config.use_https:

-			return httplib.HTTPSConnection(self.config.host)

+	def get_connection(self, bucket):

 		if self.config.proxy_host != "":

 			return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port)

 		else:

-			return httplib.HTTPConnection(self.config.host)

+			if self.config.use_https:

+				return httplib.HTTPSConnection(self.get_hostname(bucket))

+			else:

+				return httplib.HTTPConnection(self.get_hostname(bucket))

-	def format_resource(self, resource):

-		if self.config.proxy_host != "":

-			resource = "http://%s%s" % (self.config.host, resource)

+	def get_hostname(self, bucket):

+		if bucket:

+			host = self.config.host_bucket % { 'bucket' : bucket }

+		else:

+			host = self.config.host_base

+		debug('get_hostname(): ' + host)

+		return host

-		return resource

+	def format_uri(self, resource):

+		if self.config.proxy_host != "":

+			uri = "http://%s%s" % (self.get_hostname(resource['bucket']), resource['uri'])

+		else:

+			uri = resource['uri']

+		debug('format_uri(): ' + uri)

+		return uri

 	## Commands / Actions

 	def list_all_buckets(self):

@@ -126,12 +137,18 @@ class S3(object):

 		response['list'] = list

 		return response

-	def bucket_create(self, bucket):

+	def bucket_create(self, bucket, bucket_location = None):

 		self.check_bucket_name(bucket)

 		headers = SortedDict()

-		headers["content-length"] = 0

+		body = ""

+		if bucket_location and bucket_location.strip().upper() != "US":

+			body  = "<CreateBucketConfiguration><LocationConstraint>"

+			body += bucket_location.strip().upper()

+			body += "</LocationConstraint></CreateBucketConfiguration>"

+			debug("bucket_location: " + body)

+		headers["content-length"] = len(body)

 		request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers)

-		response = self.send_request(request)

+		response = self.send_request(request, body)

 		return response

 	def bucket_delete(self, bucket):

@@ -140,9 +157,9 @@ class S3(object):

 		return response

 	def bucket_info(self, bucket):

-		request = self.create_request("BUCKET_LIST", bucket = bucket + "?location")

+		request = self.create_request("BUCKET_LIST", bucket = bucket, extra = "?location")

 		response = self.send_request(request)

-		response['bucket-location'] = getTextFromXml(response['data'], ".//LocationConstraint") or "any"

+		response['bucket-location'] = getTextFromXml(response['data'], "LocationConstraint") or "any"

 		return response

 	def object_put(self, filename, bucket, object, extra_headers = None):

@@ -237,12 +254,14 @@ class S3(object):

 		debug("String '%s' encoded to '%s'" % (string, encoded))

 		return encoded

-	def create_request(self, operation, bucket = None, object = None, headers = None, **params):

-		resource = "/"

+	def create_request(self, operation, bucket = None, object = None, headers = None, extra = None, **params):

+		resource = { 'bucket' : None, 'uri' : "/" }

 		if bucket:

-			resource += str(bucket)

+			resource['bucket'] = str(bucket)

 			if object:

-				resource += "/" + self.urlencode_string(object)

+				resource['uri'] = "/" + self.urlencode_string(object)

+		if extra:

+			resource['uri'] += extra

 		if not headers:

 			headers = SortedDict()

@@ -265,21 +284,22 @@ class S3(object):

 			else:

 				param_str += "&%s" % param

 		if param_str != "":

-			resource += "?" + param_str[1:]

-		debug("CreateRequest: resource=" + resource)

+			resource['uri'] += "?" + param_str[1:]

+		debug("CreateRequest: resource[uri]=" + resource['uri'])

 		return (method_string, resource, headers)

-	def send_request(self, request):

+	def send_request(self, request, body = None):

 		method_string, resource, headers = request

 		info("Processing request, please wait...")

- 		conn = self.get_connection()

- 		conn.request(method_string, self.format_resource(resource), {}, headers)

+ 		conn = self.get_connection(resource['bucket'])

+ 		conn.request(method_string, self.format_uri(resource), body, headers)

 		response = {}

 		http_response = conn.getresponse()

 		response["status"] = http_response.status

 		response["reason"] = http_response.reason

 		response["headers"] = convertTupleListToDict(http_response.getheaders())

 		response["data"] =  http_response.read()

+		debug("Response: " + str(response))

 		conn.close()

 		if response["status"] < 200 or response["status"] > 299:

 			raise S3Error(response)

@@ -288,9 +308,9 @@ class S3(object):

 	def send_file(self, request, file):

 		method_string, resource, headers = request

 		info("Sending file '%s', please wait..." % file.name)

-		conn = self.get_connection()

+		conn = self.get_connection(resource['bucket'])

 		conn.connect()

-		conn.putrequest(method_string, self.format_resource(resource))

+		conn.putrequest(method_string, self.format_uri(resource))

 		for header in headers.keys():

 			conn.putheader(header, str(headers[header]))

 		conn.endheaders()

@@ -319,9 +339,9 @@ class S3(object):

 	def recv_file(self, request, stream):

 		method_string, resource, headers = request

 		info("Receiving file '%s', please wait..." % stream.name)

-		conn = self.get_connection()

+		conn = self.get_connection(resource['bucket'])

 		conn.connect()

-		conn.putrequest(method_string, self.format_resource(resource))

+		conn.putrequest(method_string, self.format_uri(resource))

 		for header in headers.keys():

 			conn.putheader(header, str(headers[header]))

 		conn.endheaders()

@@ -369,7 +389,9 @@ class S3(object):

 		for header in headers.keys():

 			if header.startswith("x-amz-"):

 				h += header+":"+str(headers[header])+"\n"

-		h += resource

+		if resource['bucket']:

+			h += "/" + resource['bucket']

+		h += resource['uri']

 		debug("SignHeaders: " + repr(h))

 		return base64.encodestring(hmac.new(self.config.secret_key, h, sha).digest()).strip()

@@ -59,7 +59,10 @@ def getListFromXml(xml, node):

 def getTextFromXml(xml, xpath):

 	tree = ET.fromstring(xml)

 	xmlns = getNameSpace(tree)

-	return tree.findtext(fixupXPath(xmlns, xpath))

+	if tree.tag.endswith(xpath):

+		return tree.text

+	else:

+		return tree.findtext(fixupXPath(xmlns, xpath))

 def dateS3toPython(date):

 	date = re.compile("\.\d\d\dZ").sub(".000Z", date)

@@ -133,10 +133,9 @@ def cmd_bucket_create(args):

 	uri = S3Uri(args[0])

 	if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():

 		raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % args[0])

-

 	try:

 		s3 = S3(Config())

-		response = s3.bucket_create(uri.bucket())

+		response = s3.bucket_create(uri.bucket(), cfg.bucket_location)

 	except S3Error, e:

 		if S3.codes.has_key(e.info["Code"]):

 			error(S3.codes[e.info["Code"]] % uri.bucket())

@@ -648,6 +647,7 @@ if __name__ == '__main__':

 	optparser.add_option(      "--no-delete-removed", dest="delete_removed", action="store_false", help="Don't delete remote objects.")

 	optparser.add_option("-p", "--preserve", dest="preserve_attrs", action="store_true", help="Preserve filesystem attributes (mode, ownership, timestamps). Default for [sync] command.")

 	optparser.add_option(      "--no-preserve", dest="preserve_attrs", action="store_false", help="Don't store FS attributes")

+	optparser.add_option(      "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. Either EU or US (default)")

 	optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", help="Default MIME-type to be set for objects stored.")

 	optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option")