Package Halberd :: Module crew
[hide private]
[frames] | no frames]

Source Code for Module Halberd.crew

  1  # -*- coding: iso-8859-1 -*- 
  2   
  3  """\ 
  4  Work crew pattern of parallel scanners 
  5  ====================================== 
  6   
  7  Overview 
  8  -------- 
  9   
 10  A work crew is instantiated passing a ScanTask object as a parameter, thus 
 11  defining the target and the way the scanning should be done. After the 
 12  initialization of the work crew it can be used to scan the target and get the 
 13  obtained clues back. 
 14   
 15      >>> crew = WorkCrew(scantask) 
 16      >>> clues = crew.scan() 
 17   
 18  Requirements 
 19  ------------ 
 20   
 21  These are the features that the WorkCrew must provide: 
 22   
 23      1. There are 3 different types of consumers: 
 24          - Controller thread (Performs timing + error-checking). 
 25          - Local scanning thread. 
 26          - Remote scanning thread. 
 27   
 28      2. We need a way to signal: 
 29          - When a fatal error has happened. 
 30          - When the user has pressed Control-C 
 31   
 32  Types of scanning threads 
 33  ------------------------- 
 34   
 35  The WorkCrew object spawns different kinds of threads. Here's a brief summary 
 36  of what they do: 
 37   
 38      - Manager: Detects when the time for performing the scan has expired 
 39      and notifies the rest of the threads. This code is executed in the main 
 40      thread in order to be able to appropriately catch signals, etc. 
 41   
 42      - Scanner: Performs a load-balancer scan from the current machine. 
 43   
 44  The following is a diagram showing the way it works:: 
 45   
 46                                       .--> Manager --. 
 47                                       |              | 
 48                                       +--> Scanner --+ 
 49          .----------.   .----------.  |              |   .-------. 
 50   IN --> | ScanTask |->-| WorkCrew |--+--> Scanner --+->-| Clues |--> OUT 
 51          `----------'   `----------'  |              |   `-------' 
 52                                       +--> Scanner --+ 
 53                                       |              | 
 54                                       `--> Scanner --' 
 55  """ 
 56   
 57  # Copyright (C) 2004, 2005, 2006, 2010  Juan M. Bello Rivas <jmbr@superadditive.com> 
 58  # 
 59  # This program is free software; you can redistribute it and/or modify 
 60  # it under the terms of the GNU General Public License as published by 
 61  # the Free Software Foundation; either version 2 of the License, or 
 62  # (at your option) any later version. 
 63  # 
 64  # This program is distributed in the hope that it will be useful, 
 65  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 66  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 67  # GNU General Public License for more details. 
 68  # 
 69  # You should have received a copy of the GNU General Public License 
 70  # along with this program; if not, write to the Free Software 
 71  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 72   
 73   
 74  import sys 
 75  import time 
 76  import math 
 77  import copy 
 78  import signal 
 79  import threading 
 80   
 81  import Halberd.logger 
 82  import Halberd.clues.Clue 
 83  import Halberd.clientlib as clientlib 
 84   
 85   
 86  __all__ = ['WorkCrew'] 
 87   
 88   
89 -class ScanState:
90 """Shared state among scanner threads. 91 92 @ivar shouldstop: Signals when the threads should stop scanning. 93 @type shouldstop: C{threading.Event} 94 95 caught with an exception). 96 """
97 - def __init__(self):
98 """Initializes shared state among scanning threads. 99 """ 100 self.__mutex = threading.Lock() 101 self.shouldstop = threading.Event() 102 self.__error = None 103 self.__clues = [] 104 105 self.__missed = 0 106 self.__replies = 0
107
108 - def getStats(self):
109 """Provides statistics about the scanning process. 110 111 @return: Number of clues gathered so far, number of successful requests 112 and number of unsuccessful ones (missed replies). 113 @rtype: C{tuple} 114 """ 115 # xxx - I badly need read/write locks. 116 self.__mutex.acquire() 117 nclues = len(self.__clues) 118 replies = self.__replies 119 missed = self.__missed 120 self.__mutex.release() 121 122 return (nclues, replies, missed)
123
124 - def insertClue(self, clue):
125 """Inserts a clue in the list if it is new. 126 """ 127 self.__mutex.acquire() 128 129 count = clue.getCount() 130 self.__replies += count 131 try: 132 idx = self.__clues.index(clue) 133 self.__clues[idx].incCount(count) 134 except ValueError: 135 self.__clues.append(clue) 136 137 self.__mutex.release()
138
139 - def getClues(self):
140 """Clue accessor. 141 142 @return: A copy of all obtained clues. 143 @rtype: C{list} 144 """ 145 self.__mutex.acquire() 146 clues = self.__clues[:] 147 self.__mutex.release() 148 149 return clues
150
151 - def incMissed(self):
152 """Increase the counter of missed replies. 153 """ 154 self.__mutex.acquire() 155 self.__missed += 1 156 self.__mutex.release()
157
158 - def setError(self, err):
159 """Signal an error condition. 160 """ 161 self.__mutex.acquire() 162 if self.__error is not None: 163 # An error has already been signalled. 164 self.__mutex.release() 165 return 166 self.__error = err 167 self.shouldstop.set() 168 self.__mutex.release()
169
170 - def getError(self):
171 """Returns the reason of the error condition. 172 """ 173 self.__mutex.acquire() 174 # Since we don't know what the nature of __error will be, we need to 175 # provide a clean copy of it to the caller so that no possible 176 # references or changes to __error can affect the object we return. 177 err = copy.deepcopy(self.__error) 178 self.__mutex.release() 179 180 return err
181 182
183 -class WorkCrew:
184 """Pool of scanners working in parallel. 185 186 @ivar task: A reference to scantask. 187 @type task: L{ScanTask} 188 189 @ivar working: Indicates whether the crew is working or idle. 190 @type working: C{bool} 191 192 @ivar prev: Previous SIGINT handler. 193 """
194 - def __init__(self, scantask):
195 self.workers = [] 196 self.task = scantask 197 198 self.state = ScanState() 199 200 self.working = False 201 202 self.prev = None
203
204 - def _setupSigHandler(self):
205 """Performs what's needed to catch SIGINT. 206 """ 207 def interrupt(signum, frame): 208 """SIGINT handler 209 """ 210 self.state.setError('received SIGINT')
211 212 self.prev = signal.signal(signal.SIGINT, interrupt)
213
214 - def _restoreSigHandler(self):
215 """Restore previous SIGINT handler. 216 """ 217 signal.signal(signal.SIGINT, self.prev)
218
219 - def _initLocal(self):
220 """Initializes conventional (local) scanner threads. 221 """ 222 for i in xrange(self.task.parallelism): 223 worker = Scanner(self.state, self.task) 224 self.workers.append(worker)
225
226 - def scan(self):
227 """Perform a parallel load-balancer scan. 228 """ 229 self.working = True 230 self._setupSigHandler() 231 232 self._initLocal() 233 234 for worker in self.workers: 235 worker.start() 236 237 # The Manager executes in the main thread WHILE the others are working 238 # so that signals are correctly caught. 239 manager = Manager(self.state, self.task) 240 manager.run() 241 242 for worker in self.workers: 243 worker.join() 244 245 # Display status information for the last time. 246 manager.showStats() 247 sys.stdout.write('\n\n') 248 249 self._restoreSigHandler() 250 self.working = False 251 252 err = self.state.getError() 253 if err is not None: 254 sys.stderr.write('*** finished (%s) ***\n\n' % err) 255 256 return self._getClues()
257
258 - def _getClues(self):
259 """Returns a sequence of clues obtained during the scan. 260 """ 261 assert not self.working 262 263 return self.state.getClues()
264 265
266 -class BaseScanner(threading.Thread):
267 """Base class for load balancer scanning threads. 268 269 @ivar timeout: Time (in seconds since the UNIX Epoch) when the scan will be 270 stopped. 271 @type timeout: C{float} 272 """
273 - def __init__(self, state, scantask):
274 """Initializes the scanning thread. 275 276 @param state: Container to store the results of the scan (shared among 277 scanning threads). 278 @type state: C{instanceof(ScanState)} 279 280 @param scantask: Object providing information needed to perform the 281 scan. 282 @type scantask: C{instanceof(ScanTask)} 283 """ 284 threading.Thread.__init__(self) 285 self.state = state 286 self.task = scantask 287 self.timeout = 0 288 self.logger = Halberd.logger.getLogger()
289
290 - def remaining(self, end=None):
291 """Seconds left until a given point in time. 292 293 @param end: Ending time. 294 @type end: C{float} 295 296 @return: Remaining time until L{self.timeout} 297 @rtype: C{int} 298 """ 299 if not end: 300 end = self.timeout 301 return int(end - time.time())
302
303 - def hasExpired(self):
304 """Expiration predicate. 305 306 @return: True if the timeout has expired, False otherwise. 307 @rtype: C{bool} 308 """ 309 return (self.remaining() <= 0)
310
311 - def setTimeout(self, secs):
312 """Compute an expiration time. 313 314 @param secs: Amount of seconds to spend scanning the target. 315 @type secs: C{int} 316 317 @return: The moment in time when the task expires. 318 @rtype: C{float} 319 """ 320 self.timeout = time.time() + secs
321
322 - def run(self):
323 """Perform the scan. 324 """ 325 self.setTimeout(self.task.scantime) 326 327 while not self.state.shouldstop.isSet(): 328 self.process()
329
330 - def process(self):
331 """Perform a scanning task. 332 333 This method should be overriden to do actual work. 334 """ 335 pass
336
337 -class Scanner(BaseScanner):
338 """Scans the target host from the local machine. 339 """
340 - def process(self):
341 """Gathers clues connecting directly to the target web server. 342 """ 343 client = clientlib.clientFactory(self.task) 344 345 fatal_exceptions = ( 346 clientlib.ConnectionRefused, 347 clientlib.UnknownReply, 348 clientlib.HTTPSError, 349 ) 350 351 try: 352 ts, hdrs = client.getHeaders(self.task.addr, self.task.url) 353 except fatal_exceptions, msg: 354 self.state.setError(msg) 355 except clientlib.TimedOut, msg: 356 self.state.incMissed() 357 else: 358 self.state.insertClue(self.makeClue(ts, hdrs))
359
360 - def makeClue(self, timestamp, headers):
361 """Compose a clue object. 362 363 @param timestamp: Time when the reply was received. 364 @type timestamp: C{float} 365 366 @param headers: MIME headers coming from an HTTP response. 367 @type headers: C{str} 368 369 @return: A valid clue 370 @rtype: C{Clue} 371 """ 372 clue = Halberd.clues.Clue.Clue() 373 clue.setTimestamp(timestamp) 374 clue.parse(headers) 375 376 return clue
377 378
379 -class Manager(BaseScanner):
380 """Performs management tasks during the scan. 381 """ 382 # Indicates how often the state must be refreshed (in seconds). 383 refresh_interval = 0.25 384
385 - def process(self):
386 """Controls the whole scanning process. 387 388 This method checks when the timeout has expired and notifies the rest 389 of the scanning threads that they should stop. It also displays (in 390 case the user asked for it) detailed information regarding the process. 391 """ 392 self.showStats() 393 394 if self.hasExpired(): 395 self.state.shouldstop.set() 396 try: 397 time.sleep(self.refresh_interval) 398 except IOError: 399 # Catch interrupted system call exception (it happens when 400 # CONTROL-C is pressed on win32 systems). 401 self.state.shouldstop.set()
402
403 - def showStats(self):
404 """Displays certain statistics while the scan is happening. 405 """ 406 if not self.task.verbose: 407 return 408 409 def statbar(elapsed, total): 410 """Compose a status bar string showing progress. 411 """ 412 done = int(math.floor(float(total - elapsed)/total * 10)) 413 notdone = int(math.ceil(float(elapsed)/total * 10)) 414 return '[' + '#' * done + ' ' * notdone + ']'
415 416 nclues, replies, missed = self.state.getStats() 417 418 # We put a lower bound on the remaining time. 419 if self.remaining() < 0: 420 remaining = 0 421 else: 422 remaining = self.remaining() 423 424 statusline = '\r' + self.task.addr.ljust(15) + \ 425 ' %s clues: %3d | replies: %3d | missed: %3d' \ 426 % (statbar(remaining, self.task.scantime), 427 nclues, replies, missed) 428 sys.stdout.write(statusline) 429 sys.stdout.flush()
430 431 432 # vim: ts=4 sw=4 et 433