1
2
3 """\
4 Work crew pattern of parallel scanners
5 ======================================
6
7 Overview
8 --------
9
10 A work crew is instantiated passing a ScanTask object as a parameter, thus
11 defining the target and the way the scanning should be done. After the
12 initialization of the work crew it can be used to scan the target and get the
13 obtained clues back.
14
15 >>> crew = WorkCrew(scantask)
16 >>> clues = crew.scan()
17
18 Requirements
19 ------------
20
21 These are the features that the WorkCrew must provide:
22
23 1. There are 3 different types of consumers:
24 - Controller thread (Performs timing + error-checking).
25 - Local scanning thread.
26 - Remote scanning thread.
27
28 2. We need a way to signal:
29 - When a fatal error has happened.
30 - When the user has pressed Control-C
31
32 Types of scanning threads
33 -------------------------
34
35 The WorkCrew object spawns different kinds of threads. Here's a brief summary
36 of what they do:
37
38 - Manager: Detects when the time for performing the scan has expired
39 and notifies the rest of the threads. This code is executed in the main
40 thread in order to be able to appropriately catch signals, etc.
41
42 - Scanner: Performs a load-balancer scan from the current machine.
43
44 The following is a diagram showing the way it works::
45
46 .--> Manager --.
47 | |
48 +--> Scanner --+
49 .----------. .----------. | | .-------.
50 IN --> | ScanTask |->-| WorkCrew |--+--> Scanner --+->-| Clues |--> OUT
51 `----------' `----------' | | `-------'
52 +--> Scanner --+
53 | |
54 `--> Scanner --'
55 """
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74 import sys
75 import time
76 import math
77 import copy
78 import signal
79 import threading
80
81 import Halberd.logger
82 import Halberd.clues.Clue
83 import Halberd.clientlib as clientlib
84
85
86 __all__ = ['WorkCrew']
87
88
90 """Shared state among scanner threads.
91
92 @ivar shouldstop: Signals when the threads should stop scanning.
93 @type shouldstop: C{threading.Event}
94
95 caught with an exception).
96 """
98 """Initializes shared state among scanning threads.
99 """
100 self.__mutex = threading.Lock()
101 self.shouldstop = threading.Event()
102 self.__error = None
103 self.__clues = []
104
105 self.__missed = 0
106 self.__replies = 0
107
109 """Provides statistics about the scanning process.
110
111 @return: Number of clues gathered so far, number of successful requests
112 and number of unsuccessful ones (missed replies).
113 @rtype: C{tuple}
114 """
115
116 self.__mutex.acquire()
117 nclues = len(self.__clues)
118 replies = self.__replies
119 missed = self.__missed
120 self.__mutex.release()
121
122 return (nclues, replies, missed)
123
125 """Inserts a clue in the list if it is new.
126 """
127 self.__mutex.acquire()
128
129 count = clue.getCount()
130 self.__replies += count
131 try:
132 idx = self.__clues.index(clue)
133 self.__clues[idx].incCount(count)
134 except ValueError:
135 self.__clues.append(clue)
136
137 self.__mutex.release()
138
140 """Clue accessor.
141
142 @return: A copy of all obtained clues.
143 @rtype: C{list}
144 """
145 self.__mutex.acquire()
146 clues = self.__clues[:]
147 self.__mutex.release()
148
149 return clues
150
152 """Increase the counter of missed replies.
153 """
154 self.__mutex.acquire()
155 self.__missed += 1
156 self.__mutex.release()
157
159 """Signal an error condition.
160 """
161 self.__mutex.acquire()
162 if self.__error is not None:
163
164 self.__mutex.release()
165 return
166 self.__error = err
167 self.shouldstop.set()
168 self.__mutex.release()
169
171 """Returns the reason of the error condition.
172 """
173 self.__mutex.acquire()
174
175
176
177 err = copy.deepcopy(self.__error)
178 self.__mutex.release()
179
180 return err
181
182
184 """Pool of scanners working in parallel.
185
186 @ivar task: A reference to scantask.
187 @type task: L{ScanTask}
188
189 @ivar working: Indicates whether the crew is working or idle.
190 @type working: C{bool}
191
192 @ivar prev: Previous SIGINT handler.
193 """
195 self.workers = []
196 self.task = scantask
197
198 self.state = ScanState()
199
200 self.working = False
201
202 self.prev = None
203
205 """Performs what's needed to catch SIGINT.
206 """
207 def interrupt(signum, frame):
208 """SIGINT handler
209 """
210 self.state.setError('received SIGINT')
211
212 self.prev = signal.signal(signal.SIGINT, interrupt)
213
215 """Restore previous SIGINT handler.
216 """
217 signal.signal(signal.SIGINT, self.prev)
218
220 """Initializes conventional (local) scanner threads.
221 """
222 for i in xrange(self.task.parallelism):
223 worker = Scanner(self.state, self.task)
224 self.workers.append(worker)
225
227 """Perform a parallel load-balancer scan.
228 """
229 self.working = True
230 self._setupSigHandler()
231
232 self._initLocal()
233
234 for worker in self.workers:
235 worker.start()
236
237
238
239 manager = Manager(self.state, self.task)
240 manager.run()
241
242 for worker in self.workers:
243 worker.join()
244
245
246 manager.showStats()
247 sys.stdout.write('\n\n')
248
249 self._restoreSigHandler()
250 self.working = False
251
252 err = self.state.getError()
253 if err is not None:
254 sys.stderr.write('*** finished (%s) ***\n\n' % err)
255
256 return self._getClues()
257
259 """Returns a sequence of clues obtained during the scan.
260 """
261 assert not self.working
262
263 return self.state.getClues()
264
265
267 """Base class for load balancer scanning threads.
268
269 @ivar timeout: Time (in seconds since the UNIX Epoch) when the scan will be
270 stopped.
271 @type timeout: C{float}
272 """
274 """Initializes the scanning thread.
275
276 @param state: Container to store the results of the scan (shared among
277 scanning threads).
278 @type state: C{instanceof(ScanState)}
279
280 @param scantask: Object providing information needed to perform the
281 scan.
282 @type scantask: C{instanceof(ScanTask)}
283 """
284 threading.Thread.__init__(self)
285 self.state = state
286 self.task = scantask
287 self.timeout = 0
288 self.logger = Halberd.logger.getLogger()
289
290 - def remaining(self, end=None):
291 """Seconds left until a given point in time.
292
293 @param end: Ending time.
294 @type end: C{float}
295
296 @return: Remaining time until L{self.timeout}
297 @rtype: C{int}
298 """
299 if not end:
300 end = self.timeout
301 return int(end - time.time())
302
304 """Expiration predicate.
305
306 @return: True if the timeout has expired, False otherwise.
307 @rtype: C{bool}
308 """
309 return (self.remaining() <= 0)
310
312 """Compute an expiration time.
313
314 @param secs: Amount of seconds to spend scanning the target.
315 @type secs: C{int}
316
317 @return: The moment in time when the task expires.
318 @rtype: C{float}
319 """
320 self.timeout = time.time() + secs
321
323 """Perform the scan.
324 """
325 self.setTimeout(self.task.scantime)
326
327 while not self.state.shouldstop.isSet():
328 self.process()
329
331 """Perform a scanning task.
332
333 This method should be overriden to do actual work.
334 """
335 pass
336
338 """Scans the target host from the local machine.
339 """
359
360 - def makeClue(self, timestamp, headers):
361 """Compose a clue object.
362
363 @param timestamp: Time when the reply was received.
364 @type timestamp: C{float}
365
366 @param headers: MIME headers coming from an HTTP response.
367 @type headers: C{str}
368
369 @return: A valid clue
370 @rtype: C{Clue}
371 """
372 clue = Halberd.clues.Clue.Clue()
373 clue.setTimestamp(timestamp)
374 clue.parse(headers)
375
376 return clue
377
378
380 """Performs management tasks during the scan.
381 """
382
383 refresh_interval = 0.25
384
386 """Controls the whole scanning process.
387
388 This method checks when the timeout has expired and notifies the rest
389 of the scanning threads that they should stop. It also displays (in
390 case the user asked for it) detailed information regarding the process.
391 """
392 self.showStats()
393
394 if self.hasExpired():
395 self.state.shouldstop.set()
396 try:
397 time.sleep(self.refresh_interval)
398 except IOError:
399
400
401 self.state.shouldstop.set()
402
404 """Displays certain statistics while the scan is happening.
405 """
406 if not self.task.verbose:
407 return
408
409 def statbar(elapsed, total):
410 """Compose a status bar string showing progress.
411 """
412 done = int(math.floor(float(total - elapsed)/total * 10))
413 notdone = int(math.ceil(float(elapsed)/total * 10))
414 return '[' + '#' * done + ' ' * notdone + ']'
415
416 nclues, replies, missed = self.state.getStats()
417
418
419 if self.remaining() < 0:
420 remaining = 0
421 else:
422 remaining = self.remaining()
423
424 statusline = '\r' + self.task.addr.ljust(15) + \
425 ' %s clues: %3d | replies: %3d | missed: %3d' \
426 % (statbar(remaining, self.task.scantime),
427 nclues, replies, missed)
428 sys.stdout.write(statusline)
429 sys.stdout.flush()
430
431
432
433