git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3py/trunk@62 830e0280-6d2a-0410-9c65-932aecc39d9d
Michal Ludvig authored on 2007/01/26 08:01:18... | ... |
@@ -117,11 +117,6 @@ class S3(object): |
117 | 117 |
response["size"] = size |
118 | 118 |
return response |
119 | 119 |
|
120 |
- def object_put_uri(self, filename, uri): |
|
121 |
- if uri.type != "s3": |
|
122 |
- raise ValueError("Expected URI type 's3', got '%s'" % uri.type) |
|
123 |
- return self.object_put(filename, uri.bucket(), uri.object()) |
|
124 |
- |
|
125 | 120 |
def object_get(self, filename, bucket, object): |
126 | 121 |
try: |
127 | 122 |
file = open(filename, "w") |
... | ... |
@@ -137,6 +132,21 @@ class S3(object): |
137 | 137 |
response = self.send_request(request) |
138 | 138 |
return response |
139 | 139 |
|
140 |
+ def object_put_uri(self, filename, uri): |
|
141 |
+ if uri.type != "s3": |
|
142 |
+ raise ValueError("Expected URI type 's3', got '%s'" % uri.type) |
|
143 |
+ return self.object_put(filename, uri.bucket(), uri.object()) |
|
144 |
+ |
|
145 |
+ def object_get_uri(self, filename, uri): |
|
146 |
+ if uri.type != "s3": |
|
147 |
+ raise ValueError("Expected URI type 's3', got '%s'" % uri.type) |
|
148 |
+ return self.object_get(filename, uri.bucket(), uri.object()) |
|
149 |
+ |
|
150 |
+ def object_delete_uri(self, uri): |
|
151 |
+ if uri.type != "s3": |
|
152 |
+ raise ValueError("Expected URI type 's3', got '%s'" % uri.type) |
|
153 |
+ return self.object_delete(filename, uri.bucket(), uri.object()) |
|
154 |
+ |
|
140 | 155 |
def create_request(self, operation, bucket = None, object = None, headers = None, **params): |
141 | 156 |
resource = "/" |
142 | 157 |
if bucket: |
... | ... |
@@ -285,21 +295,3 @@ class S3(object): |
285 | 285 |
else: |
286 | 286 |
return object and object or bucket |
287 | 287 |
|
288 |
- def parse_uri(self, uri): |
|
289 |
- match = re.compile("^s3://([^/]*)/?(.*)").match(uri) |
|
290 |
- if match: |
|
291 |
- return (True,) + match.groups() |
|
292 |
- else: |
|
293 |
- return (False, "", "") |
|
294 |
- |
|
295 |
- def is_uri(self, uri): |
|
296 |
- isuri, bucket, object = self.parse_uri(uri) |
|
297 |
- return isuri |
|
298 |
- |
|
299 |
- def is_uri_bucket(self, uri): |
|
300 |
- isuri, bucket, object = self.parse_uri(uri) |
|
301 |
- return isuri and bool(bucket) |
|
302 |
- |
|
303 |
- def is_uri_object(self, uri): |
|
304 |
- isuri, bucket, object = self.parse_uri(uri) |
|
305 |
- return isuri and bool(bucket) and bool(object) |
... | ... |
@@ -30,7 +30,7 @@ class S3Uri(object): |
30 | 30 |
|
31 | 31 |
class S3UriS3(S3Uri): |
32 | 32 |
type = "s3" |
33 |
- _re = re.compile("^s3://([^/]*)/?(.*)") |
|
33 |
+ _re = re.compile("^s3://([^/]+)/?(.*)") |
|
34 | 34 |
def __init__(self, string): |
35 | 35 |
match = self._re.match(string) |
36 | 36 |
if not match: |
... | ... |
@@ -45,6 +45,12 @@ class S3UriS3(S3Uri): |
45 | 45 |
def object(self): |
46 | 46 |
return self._object |
47 | 47 |
|
48 |
+ def has_bucket(self): |
|
49 |
+ return bool(self._bucket) |
|
50 |
+ |
|
51 |
+ def has_object(self): |
|
52 |
+ return bool(self._object) |
|
53 |
+ |
|
48 | 54 |
def uri(self): |
49 | 55 |
return "/".join(["s3:/", self._bucket, self._object]) |
50 | 56 |
|
... | ... |
@@ -24,39 +24,35 @@ def output(message): |
24 | 24 |
|
25 | 25 |
def cmd_ls(args): |
26 | 26 |
s3 = S3(Config()) |
27 |
- bucket = None |
|
28 | 27 |
if len(args) > 0: |
29 |
- isuri, bucket, object = s3.parse_uri(args[0]) |
|
30 |
- if not isuri: |
|
31 |
- bucket = args[0] |
|
32 |
- if bucket: |
|
33 |
- cmd_bucket_list(args) |
|
34 |
- else: |
|
35 |
- cmd_buckets_list_all(args) |
|
36 |
- |
|
37 |
-def cmd_buckets_list_all(args): |
|
38 |
- s3 = S3(Config()) |
|
39 |
- response = s3.list_all_buckets() |
|
40 |
- for bucket in response["list"]: |
|
41 |
- output("%s %s" % ( |
|
42 |
- formatDateTime(bucket["CreationDate"]), |
|
43 |
- s3.compose_uri(bucket["Name"]), |
|
44 |
- )) |
|
28 |
+ uri = S3Uri(args[0]) |
|
29 |
+ if uri.type == "s3" and uri.has_bucket(): |
|
30 |
+ subcmd_bucket_list(s3, uri) |
|
31 |
+ return |
|
32 |
+ subcmd_buckets_list_all(s3) |
|
45 | 33 |
|
46 | 34 |
def cmd_buckets_list_all_all(args): |
47 | 35 |
s3 = S3(Config()) |
36 |
+ |
|
48 | 37 |
response = s3.list_all_buckets() |
49 | 38 |
|
50 | 39 |
for bucket in response["list"]: |
51 |
- cmd_bucket_list([bucket["Name"]]) |
|
40 |
+ subcmd_bucket_list(s3, S3Uri("s3://" + bucket["Name"])) |
|
52 | 41 |
output("") |
53 | 42 |
|
54 | 43 |
|
55 |
-def cmd_bucket_list(args): |
|
56 |
- s3 = S3(Config()) |
|
57 |
- isuri, bucket, object = s3.parse_uri(args[0]) |
|
58 |
- if not isuri: |
|
59 |
- bucket = args[0] |
|
44 |
+def subcmd_buckets_list_all(s3): |
|
45 |
+ response = s3.list_all_buckets() |
|
46 |
+ for bucket in response["list"]: |
|
47 |
+ output("%s s3://%s" % ( |
|
48 |
+ formatDateTime(bucket["CreationDate"]), |
|
49 |
+ bucket["Name"], |
|
50 |
+ )) |
|
51 |
+ |
|
52 |
+def subcmd_bucket_list(s3, uri): |
|
53 |
+ bucket = uri.bucket() |
|
54 |
+ object = uri.object() |
|
55 |
+ |
|
60 | 56 |
output("Bucket '%s':" % bucket) |
61 | 57 |
if object.endswith('*'): |
62 | 58 |
object = object[:-1] |
... | ... |
@@ -77,78 +73,45 @@ def cmd_bucket_list(args): |
77 | 77 |
)) |
78 | 78 |
|
79 | 79 |
def cmd_bucket_create(args): |
80 |
- s3 = S3(Config()) |
|
81 |
- isuri, bucket, object = s3.parse_uri(args[0]) |
|
82 |
- if not isuri: |
|
83 |
- bucket = args[0] |
|
80 |
+ uri = S3Uri(args[0]) |
|
81 |
+ if not uri.type == "s3" or not uri.has_bucket() or uri.has_object(): |
|
82 |
+ raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % args[0]) |
|
83 |
+ |
|
84 | 84 |
try: |
85 |
- response = s3.bucket_create(bucket) |
|
85 |
+ s3 = S3(Config()) |
|
86 |
+ response = s3.bucket_create(uri.bucket()) |
|
86 | 87 |
except S3Error, e: |
87 | 88 |
if S3.codes.has_key(e.Code): |
88 |
- error(S3.codes[e.Code] % bucket) |
|
89 |
+ error(S3.codes[e.Code] % uri.bucket()) |
|
89 | 90 |
return |
90 | 91 |
else: |
91 | 92 |
raise |
92 |
- output("Bucket '%s' created" % bucket) |
|
93 |
+ output("Bucket '%s' created" % uri.bucket()) |
|
93 | 94 |
|
94 | 95 |
def cmd_bucket_delete(args): |
95 |
- s3 = S3(Config()) |
|
96 |
- isuri, bucket, object = s3.parse_uri(args[0]) |
|
97 |
- if not isuri: |
|
98 |
- bucket = args[0] |
|
96 |
+ uri = S3Uri(args[0]) |
|
97 |
+ if not uri.type == "s3" or not uri.has_bucket() or uri.has_object(): |
|
98 |
+ raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % args[0]) |
|
99 | 99 |
try: |
100 |
- response = s3.bucket_delete(bucket) |
|
100 |
+ s3 = S3(Config()) |
|
101 |
+ response = s3.bucket_delete(uri.bucket()) |
|
101 | 102 |
except S3Error, e: |
102 | 103 |
if S3.codes.has_key(e.Code): |
103 |
- error(S3.codes[e.Code] % bucket) |
|
104 |
+ error(S3.codes[e.Code] % uri.bucket()) |
|
104 | 105 |
return |
105 | 106 |
else: |
106 | 107 |
raise |
107 |
- output("Bucket '%s' removed" % bucket) |
|
108 |
- |
|
109 |
-def cmd_cp(args): |
|
110 |
- s3 = S3(Config()) |
|
111 |
- xnor=lambda a, b:(a and b) or (not a and not b) |
|
112 |
- cp_remote_local = s3.is_uri(args[0]) |
|
113 |
- arg_copy = args[:] |
|
114 |
- srcs = [] |
|
115 |
- dest = [] |
|
116 |
- index = 0 |
|
117 |
- try: |
|
118 |
- while(xnor(cp_remote_local, s3.is_uri(arg_copy[index]))): |
|
119 |
- if cp_remote_local: |
|
120 |
- if not s3.is_uri_object(arg_copy[index]): |
|
121 |
- raise ParameterError("%s: source URI must include the object name!" % arg_copy[index]) |
|
122 |
- else: |
|
123 |
- if not os.path.isfile(arg_copy[index]): |
|
124 |
- raise ParameterError("%s is not a regular file!" % arg_copy[index]) |
|
125 |
- srcs.append(arg_copy[index]) |
|
126 |
- index+=1 |
|
127 |
- dest = arg_copy[index] |
|
128 |
- if len(arg_copy) > index+1: |
|
129 |
- raise ParameterError("Too many arguments for 'cp': %s" % (arg_copy[index + 1:])) |
|
130 |
- except IndexError, e: |
|
131 |
- raise ParameterError("Destination %s not set. See --help for details." % |
|
132 |
- (cp_remote_local and "local directory" or "S3-URI")) |
|
133 |
- if cp_remote_local and not os.path.isdir(dest): |
|
134 |
- raise ParameterError("Destination path '%s' is not a directory." % |
|
135 |
- (dest)) |
|
136 |
- ## All checks done, now DO IT |
|
137 |
- if cp_remote_local: |
|
138 |
- for src in srcs: |
|
139 |
- cmd_object_get([src, dest]) |
|
140 |
- else: |
|
141 |
- cmd_object_put(srcs + [dest]) |
|
108 |
+ output("Bucket '%s' removed" % uri.bucket()) |
|
142 | 109 |
|
143 | 110 |
def cmd_object_put(args): |
144 | 111 |
s3 = S3(Config()) |
145 | 112 |
|
146 |
- s3uri = args.pop() |
|
113 |
+ uri_arg = args.pop() |
|
147 | 114 |
files = args[:] |
148 | 115 |
|
149 |
- uri = S3Uri(s3uri) |
|
116 |
+ uri = S3Uri(uri_arg) |
|
150 | 117 |
if uri.type != "s3": |
151 |
- raise ParameterError("Expecting S3 URI instead of '%s'" % s3uri) |
|
118 |
+ raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg) |
|
152 | 119 |
|
153 | 120 |
if len(files) > 1 and uri.object != "" and not Config().force: |
154 | 121 |
error("When uploading multiple files the last argument must") |
... | ... |
@@ -159,38 +122,42 @@ def cmd_object_put(args): |
159 | 159 |
exit(1) |
160 | 160 |
|
161 | 161 |
for file in files: |
162 |
- s3uri_final = str(uri) |
|
162 |
+ uri_arg_final = str(uri) |
|
163 | 163 |
if len(files) > 1 or uri.object() == "": |
164 |
- s3uri_final += os.path.basename(file) |
|
164 |
+ uri_arg_final += os.path.basename(file) |
|
165 | 165 |
|
166 |
- uri_final = S3Uri(s3uri_final) |
|
166 |
+ uri_final = S3Uri(uri_arg_final) |
|
167 | 167 |
response = s3.object_put_uri(file, uri_final) |
168 | 168 |
output("File '%s' stored as %s (%d bytes)" % |
169 | 169 |
(file, uri_final, response["size"])) |
170 | 170 |
|
171 | 171 |
def cmd_object_get(args): |
172 | 172 |
s3 = S3(Config()) |
173 |
- s3uri = args.pop(0) |
|
174 |
- isuri, bucket, object = s3.parse_uri(s3uri) |
|
175 |
- if not isuri or not bucket or not object: |
|
176 |
- raise ParameterError("Expecting S3 object URI instead of '%s'" % s3uri) |
|
177 |
- destination = len(args) > 0 and args.pop(0) or object |
|
173 |
+ |
|
174 |
+ uri_arg = args.pop(0) |
|
175 |
+ uri = S3Uri(uri_arg) |
|
176 |
+ if uri.type != "s3" or not uri.has_object(): |
|
177 |
+ raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg) |
|
178 |
+ |
|
179 |
+ destination = len(args) > 0 and args.pop(0) or uri.object() |
|
178 | 180 |
if os.path.isdir(destination): |
179 |
- destination += ("/" + object) |
|
181 |
+ destination += ("/" + uri.object()) |
|
180 | 182 |
if not Config().force and os.path.exists(destination): |
181 | 183 |
raise ParameterError("File %s already exists. Use --force to overwrite it" % destination) |
182 |
- response = s3.object_get(destination, bucket, object) |
|
184 |
+ response = s3.object_get_uri(destination, uri) |
|
183 | 185 |
output("Object %s saved as '%s' (%d bytes)" % |
184 |
- (s3uri, destination, response["size"])) |
|
186 |
+ (uri, destination, response["size"])) |
|
185 | 187 |
|
186 | 188 |
def cmd_object_del(args): |
187 | 189 |
s3 = S3(Config()) |
188 |
- s3uri = args.pop(0) |
|
189 |
- isuri, bucket, object = s3.parse_uri(s3uri) |
|
190 |
- if not isuri or not bucket or not object: |
|
191 |
- raise ParameterError("Expecting S3 object URI instead of '%s'" % s3uri) |
|
192 |
- response = s3.object_delete(bucket, object) |
|
193 |
- output("Object %s deleted" % s3uri) |
|
190 |
+ |
|
191 |
+ uri_arg = args.pop(0) |
|
192 |
+ uri = S3Uri(uri_arg) |
|
193 |
+ if uri.type != "s3" or not uri.has_object(): |
|
194 |
+ raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg) |
|
195 |
+ |
|
196 |
+ response = s3.object_delete_uri(uri) |
|
197 |
+ output("Object %s deleted" % uri) |
|
194 | 198 |
|
195 | 199 |
def run_configure(config_file): |
196 | 200 |
cfg = Config() |
... | ... |
@@ -242,7 +209,6 @@ commands_list = [ |
242 | 242 |
{"cmd":"rb", "label":"Remove bucket", "param":"s3://BUCKET", "func":cmd_bucket_delete, "argc":1}, |
243 | 243 |
{"cmd":"ls", "label":"List objects or buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_ls, "argc":0}, |
244 | 244 |
{"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_buckets_list_all_all, "argc":0}, |
245 |
- {"cmd":"cp", "label":"Copy files to / from S3 bucket", "param":"SRC DST", "func":cmd_cp, "argc":2}, |
|
246 | 245 |
{"cmd":"put", "label":"Put file into bucket", "param":"FILE [FILE...] s3://BUCKET[/PREFIX]", "func":cmd_object_put, "argc":2}, |
247 | 246 |
{"cmd":"get", "label":"Get file from bucket", "param":"s3://BUCKET/OBJECT LOCAL_FILE", "func":cmd_object_get, "argc":1}, |
248 | 247 |
{"cmd":"del", "label":"Delete file from bucket", "param":"s3://BUCKET/OBJECT", "func":cmd_object_del, "argc":1}, |