git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3cmd/trunk@76 830e0280-6d2a-0410-9c65-932aecc39d9d
Michal Ludvig authored on 2007/02/08 09:35:55... | ... |
@@ -7,6 +7,8 @@ import os, os.path |
7 | 7 |
import errno |
8 | 8 |
import random |
9 | 9 |
import pickle |
10 |
+import sqlite3 |
|
11 |
+import string |
|
10 | 12 |
|
11 | 13 |
class S3fs(object): |
12 | 14 |
_sync_attrs = [ "tree" ] |
... | ... |
@@ -27,10 +29,12 @@ class S3fs(object): |
27 | 27 |
self._object_name = self.n.fs(fsname) |
28 | 28 |
if self.object_exists(self._object_name): |
29 | 29 |
raise S3fsError("Filesystem '%s' already exists" % fsname, errno.EEXIST) |
30 |
+ tree = S3fsTree(self.object_create(self._object_name)) |
|
30 | 31 |
root_inode = S3fsInode(self) |
31 | 32 |
S3fsSync.store(self, root_inode) |
32 |
- self.tree = { "/" : root_inode.inode_id } |
|
33 |
- S3fsSync.store(self, self) |
|
33 |
+ tree.mkrootdir(root_inode.inode_id) |
|
34 |
+ self.store() |
|
35 |
+ |
|
34 | 36 |
self.openfs(fsname) |
35 | 37 |
|
36 | 38 |
def openfs(self, fsname): |
... | ... |
@@ -38,17 +42,82 @@ class S3fs(object): |
38 | 38 |
self._object_name = self.n.fs(fsname) |
39 | 39 |
if not self.object_exists(self._object_name): |
40 | 40 |
raise S3fsError("Filesystem '%s' does not exist" % fsname, errno.ENOENT) |
41 |
- S3fsSync.load(self, self) |
|
41 |
+ self.tree = S3fsTree(self.object_fetch(self._object_name)) |
|
42 | 42 |
print self.tree |
43 | 43 |
|
44 |
- def syncfs(self): |
|
45 |
- S3fsSync.store(self, self) |
|
46 |
- |
|
47 | 44 |
def get_inode(self, path): |
48 |
- inode_id = self.tree[path] |
|
49 |
- inode = S3fsInode(self, inode_id) |
|
45 |
+ (inode_num, id) = self.tree.get_inode(path) |
|
46 |
+ inode = S3fsInode(self, id) |
|
50 | 47 |
return inode |
48 |
+ |
|
49 |
+ def store(self): |
|
50 |
+ self.object_store(self.fsname) |
|
51 |
+ |
|
52 |
+ |
|
53 |
+class S3fsTree(object): |
|
54 |
+ def __init__(self, fsfilename): |
|
55 |
+ print "S3fsTree(%s) opening database" % fsfilename |
|
56 |
+ self._cache = {} |
|
57 |
+ self.conn = sqlite3.connect(fsfilename) |
|
58 |
+ self.conn.isolation_level = None ## Auto-Commit mode |
|
59 |
+ self.c = self.conn.cursor() |
|
60 |
+ try: |
|
61 |
+ self.c.execute(""" |
|
62 |
+ CREATE TABLE tree ( |
|
63 |
+ inode INTEGER PRIMARY KEY AUTOINCREMENT, |
|
64 |
+ parent INTEGER, |
|
65 |
+ name TEXT, |
|
66 |
+ id TEXT, |
|
67 |
+ UNIQUE (parent, name) |
|
68 |
+ ) |
|
69 |
+ """) |
|
70 |
+ print "Table 'tree' created" |
|
71 |
+ except sqlite3.OperationalError, e: |
|
72 |
+ if e.message != "table tree already exists": |
|
73 |
+ raise |
|
74 |
+ print "Dumping filesystem:" |
|
75 |
+ r = self.c.execute("SELECT * FROM tree") |
|
76 |
+ for row in r.fetchall(): |
|
77 |
+ print row |
|
78 |
+ print "Done." |
|
79 |
+ |
|
80 |
+ def mkrootdir(self, id): |
|
81 |
+ r = self.c.execute(""" |
|
82 |
+ INSERT INTO tree (parent, name, id) |
|
83 |
+ VALUES (-1, "/", ?) |
|
84 |
+ """, (id,)) |
|
85 |
+ self._cache["/"] = (r.lastrowid, id) |
|
86 |
+ print "Stored '/': %s" % str(self._cache["/"]) |
|
87 |
+ |
|
88 |
+ def get_inode(self, path): |
|
89 |
+ print "get_inode(%s)" % path |
|
90 |
+ print "_cache = %s" % str(self._cache) |
|
91 |
+ if self._cache.has_key(path): |
|
92 |
+ return self._cache[path] |
|
93 |
+ if not path.startswith("/"): |
|
94 |
+ raise ValueError("get_inode() requires path beginning with '/'") |
|
95 |
+ path = path[1:] |
|
96 |
+ pathparts = path.split("/")[1:] |
|
97 |
+ query_from = "tree as t0" |
|
98 |
+ query_where = "t0.parent == -1 AND t0.name == '/'" |
|
99 |
+ join_index = 0 |
|
100 |
+ for p in pathparts: |
|
101 |
+ join_index += 1 |
|
102 |
+ query_from += " LEFT JOIN tree as t%d" % join_index |
|
103 |
+ query_where += " AND t%d.parent == t%d.inode AND t%d.name == ?" % \ |
|
104 |
+ (join_index, join_index-1, join_index) |
|
105 |
+ |
|
106 |
+ query = "SELECT t%d.inode, t%d.id FROM %s WHERE %s" % \ |
|
107 |
+ (join_index, join_index, query_from, query_where) |
|
108 |
+ |
|
109 |
+ print query |
|
110 |
+ retval = self.c.execute(query, pathparts).fetchone() |
|
111 |
+ print retval |
|
112 |
+ return retval |
|
51 | 113 |
|
114 |
+#class S3fsDb(object): |
|
115 |
+ |
|
116 |
+ |
|
52 | 117 |
class S3fsInode(object): |
53 | 118 |
_fs = None |
54 | 119 |
|
... | ... |
@@ -116,9 +185,36 @@ class S3fsLocalDir(S3fs): |
116 | 116 |
contents = f.read() |
117 | 117 |
f.close() |
118 | 118 |
return contents |
119 |
+ |
|
120 |
+ def object_create(self, object_name): |
|
121 |
+ """ Create object in a temporary directory |
|
122 |
+ """ |
|
123 |
+ real_path = os.path.join(self._dir, object_name) |
|
124 |
+ # Load object from S3 to a temporary directory |
|
125 |
+ return real_path |
|
126 |
+ |
|
127 |
+ def object_fetch(self, object_name): |
|
128 |
+ """ Load object from S3 to a local directory. |
|
129 |
+ |
|
130 |
+ Returns: real file name on the local filesystem. |
|
131 |
+ """ |
|
132 |
+ real_path = os.path.join(self._dir, object_name) |
|
133 |
+ return real_path |
|
134 |
+ |
|
135 |
+ def object_store(self, object_name): |
|
136 |
+ """ Store object from a local directory to S3. |
|
137 |
+ |
|
138 |
+ Returns: real file name on the local filesystem. |
|
139 |
+ """ |
|
140 |
+ real_path = os.path.join(self._dir, object_name) |
|
141 |
+ # Store file from temporary directory to S3 |
|
142 |
+ return real_path |
|
143 |
+ |
|
144 |
+ def object_real_path(self, object_name): |
|
145 |
+ return os.path.join(self._dir, object_name) |
|
119 | 146 |
|
120 | 147 |
class S3fsObjectName(object): |
121 |
- _rnd_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" |
|
148 |
+ _rnd_chars = string.ascii_letters+string.digits |
|
122 | 149 |
_rnd_chars_len = len(_rnd_chars) |
123 | 150 |
|
124 | 151 |
def __init__(self): |