Browse code

Major rework of Config class: - Renamed from AwsConfig to Config - Converted to Singleton (see Config.__new__() and an article on Wikipedia) - No more explicit listing of options - use introspection to get them (class variables that of type str, int or bool that don't start with underscore) - Check values read from config file and verify their type.

Added OptionMimeType and -m/-M options. Not yet implemented
functionality in the rest of S3/S3.py


git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3py/trunk@49 830e0280-6d2a-0410-9c65-932aecc39d9d

Michal Ludvig authored on 2007/01/16 08:35:35
Showing 2 changed files
... ...
@@ -12,8 +12,9 @@ from SortedDict import SortedDict
12 12
 from BidirMap import BidirMap
13 13
 from ConfigParser import ConfigParser
14 14
 
15
-class AwsConfig:
16
-	parsed_files = []
15
+class Config(object):
16
+	_instance = None
17
+	_parsed_files = []
17 18
 	access_key = ""
18 19
 	secret_key = ""
19 20
 	host = "s3.amazonaws.com"
... ...
@@ -25,21 +26,61 @@ class AwsConfig:
25 25
 	show_uri = False
26 26
 	acl_public = False
27 27
 
28
+	## Creating a singleton
29
+	def __new__(self, configfile = None):
30
+		if self._instance is None:
31
+			self._instance = object.__new__(self)
32
+		return self._instance
33
+
28 34
 	def __init__(self, configfile = None):
29 35
 		if configfile:
30 36
 			self.read_config_file(configfile)
31 37
 
38
+	def option_list(self):
39
+		retval = []
40
+		for option in dir(self):
41
+			## Skip attributes that start with underscore or are not string, int or bool
42
+			option_type = type(getattr(Config, option))
43
+			if option.startswith("_") or \
44
+			   not (option_type in (
45
+			   		type("string"),	# str
46
+			        	type(42),	# int
47
+					type(True))):	# bool
48
+				continue
49
+			retval.append(option)
50
+		return retval
51
+
32 52
 	def read_config_file(self, configfile):
33 53
 		cp = ConfigParser(configfile)
34
-		AwsConfig.access_key = cp.get("access_key", AwsConfig.access_key)
35
-		AwsConfig.secret_key = cp.get("secret_key", AwsConfig.secret_key)
36
-		AwsConfig.host = cp.get("host", AwsConfig.host)
37
-		verbosity = cp.get("verbosity", "WARNING")
38
-		try:
39
-			AwsConfig.verbosity = logging._levelNames[verbosity]
40
-		except KeyError:
41
-			error("AwsConfig: verbosity level '%s' is not valid" % verbosity)
42
-		AwsConfig.parsed_files.append(configfile)
54
+		for option in self.option_list():
55
+			self.update_option(option, cp.get(option))
56
+		self._parsed_files.append(configfile)
57
+
58
+	def update_option(self, option, value):
59
+		if value is None:
60
+			return
61
+		#### Special treatment of some options
62
+		## verbosity must be known to "logging" module
63
+		if option == "verbosity":
64
+			try:
65
+				setattr(Config, "verbosity", logging._levelNames[value])
66
+			except KeyError:
67
+				error("Config: verbosity level '%s' is not valid" % value)
68
+		## allow yes/no, true/false, on/off and 1/0 for boolean options
69
+		elif type(getattr(Config, option)) is type(True):	# bool
70
+			if str(value).lower() in ("true", "yes", "on", "1"):
71
+				setattr(Config, option, True)
72
+			elif str(value).lower() in ("false", "no", "off", "0"):
73
+				setattr(Config, option, False)
74
+			else:
75
+				error("Config: value of option '%s' must be Yes or No, not '%s'" % (option, value))
76
+		elif type(getattr(Config, option)) is type(42):		# int
77
+			try:
78
+				setattr(Config, option, int(value))
79
+			except ValueError, e:
80
+				error("Config: value of option '%s' must be an integer, not '%s'" % (option, value))
81
+		else:							# string
82
+			setattr(Config, option, value)
43 83
 
44 84
 class S3Error (Exception):
45 85
 	def __init__(self, response):
... ...
@@ -139,7 +180,7 @@ class S3:
139 139
 			raise ParameterError("%s: %s" % (filename, e.strerror))
140 140
 		headers = SortedDict()
141 141
 		headers["content-length"] = size
142
-		if AwsConfig.acl_public:
142
+		if self.config.acl_public:
143 143
 			headers["x-amz-acl"] = "public-read"
144 144
 		request = self.create_request("OBJECT_PUT", bucket = bucket, object = object, headers = headers)
145 145
 		response = self.send_file(request, file)
... ...
@@ -211,8 +252,8 @@ class S3:
211 211
 		conn.endheaders()
212 212
 		size_left = size_total = headers.get("content-length")
213 213
 		while (size_left > 0):
214
-			debug("SendFile: Reading up to %d bytes from '%s'" % (AwsConfig.send_chunk, file.name))
215
-			data = file.read(AwsConfig.send_chunk)
214
+			debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name))
215
+			data = file.read(self.config.send_chunk)
216 216
 			debug("SendFile: Sending %d bytes to the server" % len(data))
217 217
 			conn.send(data)
218 218
 			size_left -= len(data)
... ...
@@ -251,7 +292,7 @@ class S3:
251 251
 		md5=hashlib.new("md5")
252 252
 		size_left = size_total = int(response["headers"]["content-length"])
253 253
 		while (size_left > 0):
254
-			this_chunk = size_left > AwsConfig.recv_chunk and AwsConfig.recv_chunk or size_left
254
+			this_chunk = size_left > self.config.recv_chunk and self.config.recv_chunk or size_left
255 255
 			debug("ReceiveFile: Receiving up to %d bytes from the server" % this_chunk)
256 256
 			data = http_response.read(this_chunk)
257 257
 			debug("ReceiveFile: Writing %d bytes to file '%s'" % (len(data), file.name))
... ...
@@ -293,7 +334,7 @@ class S3:
293 293
 		return True
294 294
 
295 295
 	def compose_uri(self, bucket, object = None, force_uri = False):
296
-		if AwsConfig.show_uri or force_uri:
296
+		if self.config.show_uri or force_uri:
297 297
 			uri = "s3://" + bucket
298 298
 			if object:
299 299
 				uri += "/"+object
... ...
@@ -9,7 +9,8 @@ import sys
9 9
 import logging
10 10
 import time
11 11
 
12
-from optparse import OptionParser
12
+from copy import copy
13
+from optparse import OptionParser, Option, OptionValueError
13 14
 from logging import debug, info, warning, error
14 15
 import elementtree.ElementTree as ET
15 16
 
... ...
@@ -20,7 +21,7 @@ def output(message):
20 20
 	print message
21 21
 
22 22
 def cmd_ls(args):
23
-	s3 = S3(AwsConfig())
23
+	s3 = S3(Config())
24 24
 	bucket = None
25 25
 	if len(args) > 0:
26 26
 		isuri, bucket, object = s3.parse_s3_uri(args[0])
... ...
@@ -32,7 +33,7 @@ def cmd_ls(args):
32 32
 		cmd_buckets_list_all(args)
33 33
 
34 34
 def cmd_buckets_list_all(args):
35
-	s3 = S3(AwsConfig())
35
+	s3 = S3(Config())
36 36
 	response = s3.list_all_buckets()
37 37
 	for bucket in response["list"]:
38 38
 		output("%s  %s" % (
... ...
@@ -41,7 +42,7 @@ def cmd_buckets_list_all(args):
41 41
 			))
42 42
 
43 43
 def cmd_buckets_list_all_all(args):
44
-	s3 = S3(AwsConfig())
44
+	s3 = S3(Config())
45 45
 	response = s3.list_all_buckets()
46 46
 
47 47
 	for bucket in response["list"]:
... ...
@@ -50,7 +51,7 @@ def cmd_buckets_list_all_all(args):
50 50
 
51 51
 
52 52
 def cmd_bucket_list(args):
53
-	s3 = S3(AwsConfig())
53
+	s3 = S3(Config())
54 54
 	isuri, bucket, object = s3.parse_s3_uri(args[0])
55 55
 	if not isuri:
56 56
 		bucket = args[0]
... ...
@@ -64,7 +65,7 @@ def cmd_bucket_list(args):
64 64
 		else:
65 65
 			raise
66 66
 	for object in response["list"]:
67
-		size, size_coeff = formatSize(object["Size"], AwsConfig.human_readable_sizes)
67
+		size, size_coeff = formatSize(object["Size"], Config().human_readable_sizes)
68 68
 		output("%s  %s%s  %s" % (
69 69
 			formatDateTime(object["LastModified"]),
70 70
 			str(size).rjust(8), size_coeff.ljust(1),
... ...
@@ -72,7 +73,7 @@ def cmd_bucket_list(args):
72 72
 			))
73 73
 
74 74
 def cmd_bucket_create(args):
75
-	s3 = S3(AwsConfig())
75
+	s3 = S3(Config())
76 76
 	isuri, bucket, object = s3.parse_s3_uri(args[0])
77 77
 	if not isuri:
78 78
 		bucket = args[0]
... ...
@@ -87,7 +88,7 @@ def cmd_bucket_create(args):
87 87
 	output("Bucket '%s' created" % bucket)
88 88
 
89 89
 def cmd_bucket_delete(args):
90
-	s3 = S3(AwsConfig())
90
+	s3 = S3(Config())
91 91
 	isuri, bucket, object = s3.parse_s3_uri(args[0])
92 92
 	if not isuri:
93 93
 		bucket = args[0]
... ...
@@ -102,7 +103,7 @@ def cmd_bucket_delete(args):
102 102
 	output("Bucket '%s' removed" % bucket)
103 103
 
104 104
 def cmd_object_put(args):
105
-	s3 = S3(AwsConfig())
105
+	s3 = S3(Config())
106 106
 
107 107
 	s3uri = args.pop()
108 108
 	files = args[:]
... ...
@@ -111,7 +112,7 @@ def cmd_object_put(args):
111 111
 	if not isuri:
112 112
 		raise ParameterError("Expecting S3 URI instead of '%s'" % s3uri)
113 113
 
114
-	if len(files) > 1 and object != "" and not AwsConfig.force:
114
+	if len(files) > 1 and object != "" and not Config().force:
115 115
 		error("When uploading multiple files the last argument must")
116 116
 		error("be a S3 URI specifying just the bucket name")
117 117
 		error("WITHOUT object name!")
... ...
@@ -131,7 +132,7 @@ def cmd_object_put(args):
131 131
 			(file, s3.compose_uri(bucket, object_final, force_uri = True), response["size"]))
132 132
 
133 133
 def cmd_object_get(args):
134
-	s3 = S3(AwsConfig())
134
+	s3 = S3(Config())
135 135
 	s3uri = args.pop(0)
136 136
 	isuri, bucket, object = s3.parse_s3_uri(s3uri)
137 137
 	if not isuri or not bucket or not object:
... ...
@@ -139,14 +140,14 @@ def cmd_object_get(args):
139 139
 	destination = len(args) > 0 and args.pop(0) or object
140 140
 	if os.path.isdir(destination):
141 141
 		destination += ("/" + object)
142
-	if not AwsConfig.force and os.path.exists(destination):
142
+	if not Config().force and os.path.exists(destination):
143 143
 		raise ParameterError("File %s already exists. Use --force to overwrite it" % destination)
144 144
 	response = s3.object_get(destination, bucket, object)
145 145
 	output("Object %s saved as '%s' (%d bytes)" %
146 146
 		(s3uri, destination, response["size"]))
147 147
 
148 148
 def cmd_object_del(args):
149
-	s3 = S3(AwsConfig())
149
+	s3 = S3(Config())
150 150
 	s3uri = args.pop(0)
151 151
 	isuri, bucket, object = s3.parse_s3_uri(s3uri)
152 152
 	if not isuri or not bucket or not object:
... ...
@@ -164,26 +165,35 @@ commands = {
164 164
 	"del": ("Delete file from bucket", cmd_object_del, 1),
165 165
 	}
166 166
 
167
+class OptionMimeType(Option):
168
+	def check_mimetype(option, opt, value):
169
+		if re.compile("^[a-z0-9]+/[a-z0-9+\.-]+$", re.IGNORECASE).match(value):
170
+			return value
171
+		raise OptionValueError("option %s: invalid MIME-Type format: %r" % (opt, value))
172
+
173
+	TYPES = Option.TYPES + ("mimetype",)
174
+	TYPE_CHECKER = copy(Option.TYPE_CHECKER)
175
+	TYPE_CHECKER["mimetype"] = check_mimetype
176
+
167 177
 if __name__ == '__main__':
168 178
 	if float("%d.%d" %(sys.version_info[0], sys.version_info[1])) < 2.5:
169 179
 		sys.stderr.write("ERROR: Python 2.5 or higher required, sorry.\n")
170 180
 		exit(1)
171 181
 
172
-	default_verbosity = AwsConfig.verbosity
173
-	optparser = OptionParser()
182
+	default_verbosity = Config().verbosity
183
+	optparser = OptionParser(option_class=OptionMimeType)
184
+	#optparser.disable_interspersed_args()
174 185
 	optparser.set_defaults(config=os.getenv("HOME")+"/.s3cfg")
175
-	optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name")
186
+	optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default")
176 187
 	optparser.set_defaults(verbosity = default_verbosity)
177
-	optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output")
178
-	optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output")
179
-	optparser.set_defaults(human_readable = False)
180
-	optparser.add_option("-H", "--human-readable", dest="human_readable", action="store_true", help="Print sizes in human readable form")
181
-	optparser.set_defaults(force = False)
182
-	optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations")
183
-	optparser.set_defaults(show_uri = False)
184
-	optparser.add_option("-u", "--show-uri", dest="show_uri", action="store_true", help="Show complete S3 URI in listings")
185
-	optparser.set_defaults(acl_public = False)
186
-	optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read by anyone")
188
+	optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.")
189
+	optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output.")
190
+	optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", help="Print sizes in human readable form.")
191
+	optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.")
192
+	optparser.add_option("-u", "--show-uri", dest="show_uri", action="store_true", help="Show complete S3 URI in listings.")
193
+	optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read by anyone.")
194
+	optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", help="Default MIME-type to be set for objects stored.")
195
+	optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option")
187 196
 
188 197
 	(options, args) = optparser.parse_args()
189 198
 
... ...
@@ -192,19 +202,23 @@ if __name__ == '__main__':
192 192
 	logging.basicConfig(level=options.verbosity, format='%(levelname)s: %(message)s')
193 193
 	
194 194
 	## Now finally parse the config file
195
-	AwsConfig(options.config)
195
+	Config(options.config)
196 196
 
197 197
 	## And again some logging level adjustments
198 198
 	## according to configfile and command line parameters
199 199
 	if options.verbosity != default_verbosity:
200
-		AwsConfig.verbosity = options.verbosity
201
-	logging.root.setLevel(AwsConfig.verbosity)
202
-
203
-	## Update AwsConfig with other parameters
204
-	AwsConfig.human_readable_sizes = options.human_readable
205
-	AwsConfig.force = options.force
206
-	AwsConfig.show_uri = options.show_uri
207
-	AwsConfig.acl_public = options.acl_public
200
+		Config().verbosity = options.verbosity
201
+	logging.root.setLevel(Config().verbosity)
202
+
203
+	## Update Config with other parameters
204
+	for parameter in (
205
+			"human_readable_sizes",
206
+			"force",
207
+			"show_uri",
208
+			"acl_public",):
209
+		if getattr(options, parameter) != None:
210
+			debug("Updating %s -> %s" % (parameter, getattr(options, parameter)))
211
+			setattr(Config, parameter, getattr(options, parameter))
208 212
 
209 213
 	if len(args) < 1:
210 214
 		error("Missing command. Please run with --help for more information.")