Added OptionMimeType and -m/-M options. Not yet implemented
functionality in the rest of S3/S3.py
git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3py/trunk@49 830e0280-6d2a-0410-9c65-932aecc39d9d
... | ... |
@@ -12,8 +12,9 @@ from SortedDict import SortedDict |
12 | 12 |
from BidirMap import BidirMap |
13 | 13 |
from ConfigParser import ConfigParser |
14 | 14 |
|
15 |
-class AwsConfig: |
|
16 |
- parsed_files = [] |
|
15 |
+class Config(object): |
|
16 |
+ _instance = None |
|
17 |
+ _parsed_files = [] |
|
17 | 18 |
access_key = "" |
18 | 19 |
secret_key = "" |
19 | 20 |
host = "s3.amazonaws.com" |
... | ... |
@@ -25,21 +26,61 @@ class AwsConfig: |
25 | 25 |
show_uri = False |
26 | 26 |
acl_public = False |
27 | 27 |
|
28 |
+ ## Creating a singleton |
|
29 |
+ def __new__(self, configfile = None): |
|
30 |
+ if self._instance is None: |
|
31 |
+ self._instance = object.__new__(self) |
|
32 |
+ return self._instance |
|
33 |
+ |
|
28 | 34 |
def __init__(self, configfile = None): |
29 | 35 |
if configfile: |
30 | 36 |
self.read_config_file(configfile) |
31 | 37 |
|
38 |
+ def option_list(self): |
|
39 |
+ retval = [] |
|
40 |
+ for option in dir(self): |
|
41 |
+ ## Skip attributes that start with underscore or are not string, int or bool |
|
42 |
+ option_type = type(getattr(Config, option)) |
|
43 |
+ if option.startswith("_") or \ |
|
44 |
+ not (option_type in ( |
|
45 |
+ type("string"), # str |
|
46 |
+ type(42), # int |
|
47 |
+ type(True))): # bool |
|
48 |
+ continue |
|
49 |
+ retval.append(option) |
|
50 |
+ return retval |
|
51 |
+ |
|
32 | 52 |
def read_config_file(self, configfile): |
33 | 53 |
cp = ConfigParser(configfile) |
34 |
- AwsConfig.access_key = cp.get("access_key", AwsConfig.access_key) |
|
35 |
- AwsConfig.secret_key = cp.get("secret_key", AwsConfig.secret_key) |
|
36 |
- AwsConfig.host = cp.get("host", AwsConfig.host) |
|
37 |
- verbosity = cp.get("verbosity", "WARNING") |
|
38 |
- try: |
|
39 |
- AwsConfig.verbosity = logging._levelNames[verbosity] |
|
40 |
- except KeyError: |
|
41 |
- error("AwsConfig: verbosity level '%s' is not valid" % verbosity) |
|
42 |
- AwsConfig.parsed_files.append(configfile) |
|
54 |
+ for option in self.option_list(): |
|
55 |
+ self.update_option(option, cp.get(option)) |
|
56 |
+ self._parsed_files.append(configfile) |
|
57 |
+ |
|
58 |
+ def update_option(self, option, value): |
|
59 |
+ if value is None: |
|
60 |
+ return |
|
61 |
+ #### Special treatment of some options |
|
62 |
+ ## verbosity must be known to "logging" module |
|
63 |
+ if option == "verbosity": |
|
64 |
+ try: |
|
65 |
+ setattr(Config, "verbosity", logging._levelNames[value]) |
|
66 |
+ except KeyError: |
|
67 |
+ error("Config: verbosity level '%s' is not valid" % value) |
|
68 |
+ ## allow yes/no, true/false, on/off and 1/0 for boolean options |
|
69 |
+ elif type(getattr(Config, option)) is type(True): # bool |
|
70 |
+ if str(value).lower() in ("true", "yes", "on", "1"): |
|
71 |
+ setattr(Config, option, True) |
|
72 |
+ elif str(value).lower() in ("false", "no", "off", "0"): |
|
73 |
+ setattr(Config, option, False) |
|
74 |
+ else: |
|
75 |
+ error("Config: value of option '%s' must be Yes or No, not '%s'" % (option, value)) |
|
76 |
+ elif type(getattr(Config, option)) is type(42): # int |
|
77 |
+ try: |
|
78 |
+ setattr(Config, option, int(value)) |
|
79 |
+ except ValueError, e: |
|
80 |
+ error("Config: value of option '%s' must be an integer, not '%s'" % (option, value)) |
|
81 |
+ else: # string |
|
82 |
+ setattr(Config, option, value) |
|
43 | 83 |
|
44 | 84 |
class S3Error (Exception): |
45 | 85 |
def __init__(self, response): |
... | ... |
@@ -139,7 +180,7 @@ class S3: |
139 | 139 |
raise ParameterError("%s: %s" % (filename, e.strerror)) |
140 | 140 |
headers = SortedDict() |
141 | 141 |
headers["content-length"] = size |
142 |
- if AwsConfig.acl_public: |
|
142 |
+ if self.config.acl_public: |
|
143 | 143 |
headers["x-amz-acl"] = "public-read" |
144 | 144 |
request = self.create_request("OBJECT_PUT", bucket = bucket, object = object, headers = headers) |
145 | 145 |
response = self.send_file(request, file) |
... | ... |
@@ -211,8 +252,8 @@ class S3: |
211 | 211 |
conn.endheaders() |
212 | 212 |
size_left = size_total = headers.get("content-length") |
213 | 213 |
while (size_left > 0): |
214 |
- debug("SendFile: Reading up to %d bytes from '%s'" % (AwsConfig.send_chunk, file.name)) |
|
215 |
- data = file.read(AwsConfig.send_chunk) |
|
214 |
+ debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name)) |
|
215 |
+ data = file.read(self.config.send_chunk) |
|
216 | 216 |
debug("SendFile: Sending %d bytes to the server" % len(data)) |
217 | 217 |
conn.send(data) |
218 | 218 |
size_left -= len(data) |
... | ... |
@@ -251,7 +292,7 @@ class S3: |
251 | 251 |
md5=hashlib.new("md5") |
252 | 252 |
size_left = size_total = int(response["headers"]["content-length"]) |
253 | 253 |
while (size_left > 0): |
254 |
- this_chunk = size_left > AwsConfig.recv_chunk and AwsConfig.recv_chunk or size_left |
|
254 |
+ this_chunk = size_left > self.config.recv_chunk and self.config.recv_chunk or size_left |
|
255 | 255 |
debug("ReceiveFile: Receiving up to %d bytes from the server" % this_chunk) |
256 | 256 |
data = http_response.read(this_chunk) |
257 | 257 |
debug("ReceiveFile: Writing %d bytes to file '%s'" % (len(data), file.name)) |
... | ... |
@@ -293,7 +334,7 @@ class S3: |
293 | 293 |
return True |
294 | 294 |
|
295 | 295 |
def compose_uri(self, bucket, object = None, force_uri = False): |
296 |
- if AwsConfig.show_uri or force_uri: |
|
296 |
+ if self.config.show_uri or force_uri: |
|
297 | 297 |
uri = "s3://" + bucket |
298 | 298 |
if object: |
299 | 299 |
uri += "/"+object |
... | ... |
@@ -9,7 +9,8 @@ import sys |
9 | 9 |
import logging |
10 | 10 |
import time |
11 | 11 |
|
12 |
-from optparse import OptionParser |
|
12 |
+from copy import copy |
|
13 |
+from optparse import OptionParser, Option, OptionValueError |
|
13 | 14 |
from logging import debug, info, warning, error |
14 | 15 |
import elementtree.ElementTree as ET |
15 | 16 |
|
... | ... |
@@ -20,7 +21,7 @@ def output(message): |
20 | 20 |
print message |
21 | 21 |
|
22 | 22 |
def cmd_ls(args): |
23 |
- s3 = S3(AwsConfig()) |
|
23 |
+ s3 = S3(Config()) |
|
24 | 24 |
bucket = None |
25 | 25 |
if len(args) > 0: |
26 | 26 |
isuri, bucket, object = s3.parse_s3_uri(args[0]) |
... | ... |
@@ -32,7 +33,7 @@ def cmd_ls(args): |
32 | 32 |
cmd_buckets_list_all(args) |
33 | 33 |
|
34 | 34 |
def cmd_buckets_list_all(args): |
35 |
- s3 = S3(AwsConfig()) |
|
35 |
+ s3 = S3(Config()) |
|
36 | 36 |
response = s3.list_all_buckets() |
37 | 37 |
for bucket in response["list"]: |
38 | 38 |
output("%s %s" % ( |
... | ... |
@@ -41,7 +42,7 @@ def cmd_buckets_list_all(args): |
41 | 41 |
)) |
42 | 42 |
|
43 | 43 |
def cmd_buckets_list_all_all(args): |
44 |
- s3 = S3(AwsConfig()) |
|
44 |
+ s3 = S3(Config()) |
|
45 | 45 |
response = s3.list_all_buckets() |
46 | 46 |
|
47 | 47 |
for bucket in response["list"]: |
... | ... |
@@ -50,7 +51,7 @@ def cmd_buckets_list_all_all(args): |
50 | 50 |
|
51 | 51 |
|
52 | 52 |
def cmd_bucket_list(args): |
53 |
- s3 = S3(AwsConfig()) |
|
53 |
+ s3 = S3(Config()) |
|
54 | 54 |
isuri, bucket, object = s3.parse_s3_uri(args[0]) |
55 | 55 |
if not isuri: |
56 | 56 |
bucket = args[0] |
... | ... |
@@ -64,7 +65,7 @@ def cmd_bucket_list(args): |
64 | 64 |
else: |
65 | 65 |
raise |
66 | 66 |
for object in response["list"]: |
67 |
- size, size_coeff = formatSize(object["Size"], AwsConfig.human_readable_sizes) |
|
67 |
+ size, size_coeff = formatSize(object["Size"], Config().human_readable_sizes) |
|
68 | 68 |
output("%s %s%s %s" % ( |
69 | 69 |
formatDateTime(object["LastModified"]), |
70 | 70 |
str(size).rjust(8), size_coeff.ljust(1), |
... | ... |
@@ -72,7 +73,7 @@ def cmd_bucket_list(args): |
72 | 72 |
)) |
73 | 73 |
|
74 | 74 |
def cmd_bucket_create(args): |
75 |
- s3 = S3(AwsConfig()) |
|
75 |
+ s3 = S3(Config()) |
|
76 | 76 |
isuri, bucket, object = s3.parse_s3_uri(args[0]) |
77 | 77 |
if not isuri: |
78 | 78 |
bucket = args[0] |
... | ... |
@@ -87,7 +88,7 @@ def cmd_bucket_create(args): |
87 | 87 |
output("Bucket '%s' created" % bucket) |
88 | 88 |
|
89 | 89 |
def cmd_bucket_delete(args): |
90 |
- s3 = S3(AwsConfig()) |
|
90 |
+ s3 = S3(Config()) |
|
91 | 91 |
isuri, bucket, object = s3.parse_s3_uri(args[0]) |
92 | 92 |
if not isuri: |
93 | 93 |
bucket = args[0] |
... | ... |
@@ -102,7 +103,7 @@ def cmd_bucket_delete(args): |
102 | 102 |
output("Bucket '%s' removed" % bucket) |
103 | 103 |
|
104 | 104 |
def cmd_object_put(args): |
105 |
- s3 = S3(AwsConfig()) |
|
105 |
+ s3 = S3(Config()) |
|
106 | 106 |
|
107 | 107 |
s3uri = args.pop() |
108 | 108 |
files = args[:] |
... | ... |
@@ -111,7 +112,7 @@ def cmd_object_put(args): |
111 | 111 |
if not isuri: |
112 | 112 |
raise ParameterError("Expecting S3 URI instead of '%s'" % s3uri) |
113 | 113 |
|
114 |
- if len(files) > 1 and object != "" and not AwsConfig.force: |
|
114 |
+ if len(files) > 1 and object != "" and not Config().force: |
|
115 | 115 |
error("When uploading multiple files the last argument must") |
116 | 116 |
error("be a S3 URI specifying just the bucket name") |
117 | 117 |
error("WITHOUT object name!") |
... | ... |
@@ -131,7 +132,7 @@ def cmd_object_put(args): |
131 | 131 |
(file, s3.compose_uri(bucket, object_final, force_uri = True), response["size"])) |
132 | 132 |
|
133 | 133 |
def cmd_object_get(args): |
134 |
- s3 = S3(AwsConfig()) |
|
134 |
+ s3 = S3(Config()) |
|
135 | 135 |
s3uri = args.pop(0) |
136 | 136 |
isuri, bucket, object = s3.parse_s3_uri(s3uri) |
137 | 137 |
if not isuri or not bucket or not object: |
... | ... |
@@ -139,14 +140,14 @@ def cmd_object_get(args): |
139 | 139 |
destination = len(args) > 0 and args.pop(0) or object |
140 | 140 |
if os.path.isdir(destination): |
141 | 141 |
destination += ("/" + object) |
142 |
- if not AwsConfig.force and os.path.exists(destination): |
|
142 |
+ if not Config().force and os.path.exists(destination): |
|
143 | 143 |
raise ParameterError("File %s already exists. Use --force to overwrite it" % destination) |
144 | 144 |
response = s3.object_get(destination, bucket, object) |
145 | 145 |
output("Object %s saved as '%s' (%d bytes)" % |
146 | 146 |
(s3uri, destination, response["size"])) |
147 | 147 |
|
148 | 148 |
def cmd_object_del(args): |
149 |
- s3 = S3(AwsConfig()) |
|
149 |
+ s3 = S3(Config()) |
|
150 | 150 |
s3uri = args.pop(0) |
151 | 151 |
isuri, bucket, object = s3.parse_s3_uri(s3uri) |
152 | 152 |
if not isuri or not bucket or not object: |
... | ... |
@@ -164,26 +165,35 @@ commands = { |
164 | 164 |
"del": ("Delete file from bucket", cmd_object_del, 1), |
165 | 165 |
} |
166 | 166 |
|
167 |
+class OptionMimeType(Option): |
|
168 |
+ def check_mimetype(option, opt, value): |
|
169 |
+ if re.compile("^[a-z0-9]+/[a-z0-9+\.-]+$", re.IGNORECASE).match(value): |
|
170 |
+ return value |
|
171 |
+ raise OptionValueError("option %s: invalid MIME-Type format: %r" % (opt, value)) |
|
172 |
+ |
|
173 |
+ TYPES = Option.TYPES + ("mimetype",) |
|
174 |
+ TYPE_CHECKER = copy(Option.TYPE_CHECKER) |
|
175 |
+ TYPE_CHECKER["mimetype"] = check_mimetype |
|
176 |
+ |
|
167 | 177 |
if __name__ == '__main__': |
168 | 178 |
if float("%d.%d" %(sys.version_info[0], sys.version_info[1])) < 2.5: |
169 | 179 |
sys.stderr.write("ERROR: Python 2.5 or higher required, sorry.\n") |
170 | 180 |
exit(1) |
171 | 181 |
|
172 |
- default_verbosity = AwsConfig.verbosity |
|
173 |
- optparser = OptionParser() |
|
182 |
+ default_verbosity = Config().verbosity |
|
183 |
+ optparser = OptionParser(option_class=OptionMimeType) |
|
184 |
+ #optparser.disable_interspersed_args() |
|
174 | 185 |
optparser.set_defaults(config=os.getenv("HOME")+"/.s3cfg") |
175 |
- optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name") |
|
186 |
+ optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default") |
|
176 | 187 |
optparser.set_defaults(verbosity = default_verbosity) |
177 |
- optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output") |
|
178 |
- optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output") |
|
179 |
- optparser.set_defaults(human_readable = False) |
|
180 |
- optparser.add_option("-H", "--human-readable", dest="human_readable", action="store_true", help="Print sizes in human readable form") |
|
181 |
- optparser.set_defaults(force = False) |
|
182 |
- optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations") |
|
183 |
- optparser.set_defaults(show_uri = False) |
|
184 |
- optparser.add_option("-u", "--show-uri", dest="show_uri", action="store_true", help="Show complete S3 URI in listings") |
|
185 |
- optparser.set_defaults(acl_public = False) |
|
186 |
- optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read by anyone") |
|
188 |
+ optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.") |
|
189 |
+ optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output.") |
|
190 |
+ optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", help="Print sizes in human readable form.") |
|
191 |
+ optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.") |
|
192 |
+ optparser.add_option("-u", "--show-uri", dest="show_uri", action="store_true", help="Show complete S3 URI in listings.") |
|
193 |
+ optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read by anyone.") |
|
194 |
+ optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", help="Default MIME-type to be set for objects stored.") |
|
195 |
+ optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option") |
|
187 | 196 |
|
188 | 197 |
(options, args) = optparser.parse_args() |
189 | 198 |
|
... | ... |
@@ -192,19 +202,23 @@ if __name__ == '__main__': |
192 | 192 |
logging.basicConfig(level=options.verbosity, format='%(levelname)s: %(message)s') |
193 | 193 |
|
194 | 194 |
## Now finally parse the config file |
195 |
- AwsConfig(options.config) |
|
195 |
+ Config(options.config) |
|
196 | 196 |
|
197 | 197 |
## And again some logging level adjustments |
198 | 198 |
## according to configfile and command line parameters |
199 | 199 |
if options.verbosity != default_verbosity: |
200 |
- AwsConfig.verbosity = options.verbosity |
|
201 |
- logging.root.setLevel(AwsConfig.verbosity) |
|
202 |
- |
|
203 |
- ## Update AwsConfig with other parameters |
|
204 |
- AwsConfig.human_readable_sizes = options.human_readable |
|
205 |
- AwsConfig.force = options.force |
|
206 |
- AwsConfig.show_uri = options.show_uri |
|
207 |
- AwsConfig.acl_public = options.acl_public |
|
200 |
+ Config().verbosity = options.verbosity |
|
201 |
+ logging.root.setLevel(Config().verbosity) |
|
202 |
+ |
|
203 |
+ ## Update Config with other parameters |
|
204 |
+ for parameter in ( |
|
205 |
+ "human_readable_sizes", |
|
206 |
+ "force", |
|
207 |
+ "show_uri", |
|
208 |
+ "acl_public",): |
|
209 |
+ if getattr(options, parameter) != None: |
|
210 |
+ debug("Updating %s -> %s" % (parameter, getattr(options, parameter))) |
|
211 |
+ setattr(Config, parameter, getattr(options, parameter)) |
|
208 | 212 |
|
209 | 213 |
if len(args) < 1: |
210 | 214 |
error("Missing command. Please run with --help for more information.") |