1
2
3 """Utilities for clue storage.
4
5 Provides functionality needed to store clues on disk.
6 """
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25 import os
26 import csv
27 import types
28 import shutil
29
30 import Halberd.util
31 from Halberd.clues.Clue import Clue
32
33
35 """The loaded file is not a valid clue file.
36 """
39
42
43
44 -def save(filename, clues):
45 """Save a clues to a file.
46
47 @param filename: Name of the file where the clues will be written to.
48 @type filename: C{str}
49
50 @param clues: Sequence of clues to write.
51 @type clues: C{list}
52 """
53
54 cluefp = open(filename, 'w+')
55 writer = csv.writer(cluefp)
56
57 for clue in clues:
58
59 writer.writerow((clue.getCount(), clue._local, clue.headers))
60
61 cluefp.close()
62
63
65 """Load clues from file.
66
67 @param filename: Name of the files where the clues are stored.
68 @type filename: C{str}
69
70 @return: Clues extracted from the file.
71 @rtype: C{list}
72
73 @raise InvalidFile: In case there's a problem while reinterpreting the
74 clues.
75 """
76 cluefp = open(filename, 'r')
77 reader = csv.reader(cluefp)
78
79 clues = []
80 for tup in reader:
81 try:
82 count, localtime, headers = tup
83 except ValueError:
84 raise InvalidFile, 'Cannot unpack fields'
85
86
87 clue = Clue()
88 try:
89 clue._count = int(count)
90 clue._local = float(localtime)
91 except ValueError:
92 raise InvalidFile, 'Could not convert fields'
93
94
95 clue.headers = eval(headers, {}, {})
96 if not (isinstance(clue.headers, types.ListType) or
97 isinstance(clue.headers, types.TupleType)):
98 raise InvalidFile, 'Wrong clue header field'
99 clue.parse(clue.headers)
100
101 clues.append(clue)
102
103 cluefp.close()
104 return clues
105
106
108 """Stores clues hierarchically using the underlying filesystem.
109
110 ClueDir tries to be as portable as possible but requires the host operating
111 system to be able to create long filenames (and directories, of course).
112
113 This is an example layout::
114
115 http___www_microsoft_com/
116 http___www_microsoft_com/207_46_134_221.clu
117 http___www_microsoft_com/207_46_156_220.clu
118 http___www_microsoft_com/207_46_156_252.clu
119 .
120 .
121 .
122 """
124 """Initializes ClueDir object.
125
126 @param root: Root folder where to start creating sub-folders.
127 @type root: C{str}
128 """
129 self.ext = 'clu'
130 if not root:
131 self.root = os.getcwd()
132 else:
133 self.root = root
134 self._mkdir(self.root)
135
137 """Filter out potentially dangerous chars.
138 """
139 return url.translate(Halberd.util.table)
140
142 """Creates a directory to store clues.
143
144 If the directory already exists it won't complain about that.
145 """
146 try:
147 st = os.stat(dest)
148 except OSError:
149 os.mkdir(dest)
150 else:
151 if not shutil.stat.S_ISDIR(st.st_mode):
152 raise InvalidFile, \
153 '%s already exist and is not a directory' % dest
154
155 return dest
156
157 - def save(self, url, addr, clues):
158 """Hierarchically write clues.
159
160 @param url: URL scanned (will be used as a directory name).
161 @type url: C{url}
162
163 @param addr: Address of the target.
164 @type addr: C{str}
165
166 @param clues: Clues to be stored.
167 @type clues: C{list}
168
169 @raise OSError: If the directories can't be created.
170 @raise IOError: If the file can't be stored successfully.
171 """
172 assert url and addr
173
174 urldir = self._mkdir(os.path.join(self.root, self._sanitize(url)))
175 filename = self._sanitize(addr) + os.extsep + self.ext
176 cluefile = os.path.join(urldir, filename)
177
178 Halberd.clues.file.save(cluefile, clues)
179
180
181
182