@@ -1,3 +1,7 @@

+2009-01-15  Michal Ludvig  <michal@logix.cz>

+

+	* s3cmd, S3/S3Uri.py, NEWS: Support for recursive 'put'.

+

 2009-01-13  Michal Ludvig  <michal@logix.cz>

 	* TODO: Updated.

@@ -1,6 +1,7 @@

 s3cmd 0.9.9-pre5

 ================

 * New command 'setacl' for setting ACL on existing objects.

+* Recursive [put] with a slightly different semantic.

 s3cmd 0.9.9-pre4 - 2008-12-30

 ================

@@ -3,6 +3,7 @@

 ##         http://www.logix.cz/michal

 ## License: GPL Version 2

+import os

 import re

 import sys

 from BidirMap import BidirMap

@@ -117,6 +118,12 @@ class S3UriFile(S3Uri):

 	def uri(self):

 		return "/".join(["file:/", self.path()])

+	def isdir(self):

+		return os.path.isdir(self.path())

+

+	def dirname(self):

+		return os.path.dirname(self.path())

+

 if __name__ == "__main__":

 	uri = S3Uri("s3://bucket/object")

 	print "type()  =", type(uri)

@@ -169,6 +169,29 @@ def cmd_bucket_delete(args):

 		_bucket_delete_one(uri)

 		output(u"Bucket '%s' removed" % uri.uri())

+def fetch_local_list(args):

+	local_uris = []

+	local_list = {}

+

+	for arg in args:

+		uri = S3Uri(arg)

+		if not uri.type == 'file':

+			raise ParameterError("Expecting filename or directory instead of: %s" % arg)

+		if uri.isdir() and not cfg.recursive:

+			raise ParameterError("Use --recursive to upload a directory: %s" % arg)

+		local_uris.append(uri)

+

+	for uri in local_uris:

+		filelist = _get_filelist_local(uri)

+		for key in filelist:

+			upload_item = {

+				'file_info' : filelist[key],

+				'relative_key' : key,

+			}

+			local_list[key] = filelist[key]

+

+	return local_list

+

 def fetch_remote_list(args):

 	remote_uris = []

 	remote_list = []

@@ -229,39 +252,48 @@ def fetch_remote_list(args):

 	return remote_list

 def cmd_object_put(args):

-	s3 = S3(Config())

+	cfg = Config()

+	s3 = S3(cfg)

+

+	## Each item will be a dict with the following attributes

+	# {'remote_uri', 'local_filename'}

+	upload_list = []

-	uri_arg = args.pop()

-	check_args_type(args, 'file', 'filename')

+	if len(args) == 0:

+		raise ParameterError("Nothing to upload. Expecting a local file or directory.")

-	uri = S3Uri(uri_arg)

-	if uri.type != "s3":

-		raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg)

+	dst_uri = S3Uri(args.pop())

+	if dst_uri.type != 's3':

+		raise ParameterError("Destination must be S3Uri. Got: %s" % args[-1])

-	if len(args) > 1 and uri.object() != "" and not Config().force:

-		error(u"When uploading multiple files the last argument must")

-		error(u"be a S3 URI specifying just the bucket name")

-		error(u"WITHOUT object name!")

-		error(u"Alternatively use --force argument and the specified")

-		error(u"object name will be prefixed to all stored filenames.")

-		sys.exit(1)

-	

+	if len(args) == 0:

+		raise ParameterError("Nothing to upload. Expecting a local file or directory.")

+

+	local_list = fetch_local_list(args)

+	local_count = len(local_list)

+

+	if local_count > 1 and not dst_uri.object() == "" and not dst_uri.object().endswith("/"):

+		raise ParameterError(u"When uploading multiple files the last argument must be a S3 URI ending with '/' or a bucket name only!")

+

+	sorted_local_keys = local_list.keys()

+	sorted_local_keys.sort()

 	seq = 0

-	total = len(args)

-	for file in args:

+	for key in sorted_local_keys:

 		seq += 1

-		uri_arg_final = str(uri)

-		if len(args) > 1 or uri.object() == "":

-			uri_arg_final += os.path.basename(file)

-		

-		uri_final = S3Uri(uri_arg_final)

+

+		if local_count > 1 or dst_uri.object() == "":

+			uri_final = S3Uri(u"%s%s" % (dst_uri, key))

+		else:

+			uri_final = dst_uri

+

 		extra_headers = {}

-		real_filename = file

-		seq_label = "[%d of %d]" % (seq, total)

+		full_name_orig = local_list[key]['full_name']

+		full_name = full_name_orig

+		seq_label = "[%d of %d]" % (seq, local_count)

 		if Config().encrypt:

-			exitcode, real_filename, extra_headers["x-amz-meta-s3tools-gpgenc"] = gpg_encrypt(file)

+			exitcode, full_name, extra_headers["x-amz-meta-s3tools-gpgenc"] = gpg_encrypt(full_name_orig)

 		try:

-			response = s3.object_put(real_filename, uri_final, extra_headers, extra_label = seq_label)

+			response = s3.object_put(full_name, uri_final, extra_headers, extra_label = seq_label)

 		except S3UploadError, e:

 			error(u"Upload of '%s' failed too many times. Skipping that file." % real_filename)

 			continue

@@ -271,14 +303,14 @@ def cmd_object_put(args):

 		speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)

 		if not Config().progress_meter:

 			output(u"File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %

-				(file, uri_final, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1],

-				seq_label))

+				(unicodise(full_name_orig), uri_final, response["size"], response["elapsed"], 

+				speed_fmt[0], speed_fmt[1], seq_label))

 		if Config().acl_public:

 			output(u"Public URL of the object is: %s" %

 				(uri_final.public_url()))

-		if Config().encrypt and real_filename != file:

-			debug(u"Removing temporary encrypted file: %s" % real_filename)

-			os.remove(real_filename)

+		if Config().encrypt and full_name != full_name_orig:

+			debug(u"Removing temporary encrypted file: %s" % unicodise(full_name))

+			os.remove(full_name)

 def cmd_object_get(args):

 	cfg = Config()

@@ -487,29 +519,31 @@ def cmd_info(args):

 def _get_filelist_local(local_uri):

 	info(u"Compiling list of local files...")

-	local_path = deunicodise(local_uri.path())

-	if os.path.isdir(local_path):

-		loc_base = os.path.join(local_path, "")

+	if local_uri.isdir():

+		local_base = local_uri.basename()

+		local_path = deunicodise(local_uri.path())

 		filelist = os.walk(local_path)

 	else:

-		loc_base = "." + os.path.sep

-		filelist = [( '.', [], [local_path] )]

-	loc_base_len = len(loc_base)

+		local_base = ""

+		local_path = deunicodise(local_uri.dirname())

+		filelist = [( local_path, [], [deunicodise(local_uri.basename())] )]

 	loc_list = {}

 	for root, dirs, files in filelist:

+		rel_root = root.replace(local_path, local_base, 1)

 		## TODO: implement explicit exclude

 		for f in files:

 			full_name = os.path.join(root, f)

+			rel_name = os.path.join(rel_root, f)

 			if not os.path.isfile(full_name):

 				continue

 			if os.path.islink(full_name):

 				## Synchronize symlinks... one day

 				## for now skip over

 				continue

-			file = unicodise(full_name[loc_base_len:])

+			relative_file = unicodise(rel_name)

 			sr = os.stat_result(os.lstat(full_name))

-			loc_list[file] = {

-				'full_name_unicoded' : unicodise(full_name),

+			loc_list[relative_file] = {

+				'full_name_unicode' : unicodise(full_name),

 				'full_name' : full_name,

 				'size' : sr.st_size, 

 				'mtime' : sr.st_mtime,