third_party/waf/waflib/extras/parallel_debug.py

   1 #! /usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy, 2007-2010 (ita)
   4
   5 """
   6 Debugging helper for parallel compilation, outputs
   7 a file named pdebug.svg in the source directory::
   8
   9         def options(opt):
  10                 opt.load('parallel_debug')
  11         def build(bld):
  12                 ...
  13 """
  14
  15 import re, sys, threading, time, traceback
  16 try:
  17         from Queue import Queue
  18 except:
  19         from queue import Queue
  20 from waflib import Runner, Options, Task, Logs, Errors
  21
  22 SVG_TEMPLATE = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
  23 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
  24 <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.0"
  25    x="${project.x}" y="${project.y}" width="${project.width}" height="${project.height}" id="svg602" xml:space="preserve">
  26
  27 <style type='text/css' media='screen'>
  28         g.over rect { stroke:#FF0000; fill-opacity:0.4 }
  29 </style>
  30
  31 <script type='text/javascript'><![CDATA[
  32 var svg  = document.getElementsByTagName('svg')[0];
  33
  34 svg.addEventListener('mouseover', function(e) {
  35         var g = e.target.parentNode;
  36         var x = document.getElementById('r_' + g.id);
  37         if (x) {
  38                 g.setAttribute('class', g.getAttribute('class') + ' over');
  39                 x.setAttribute('class', x.getAttribute('class') + ' over');
  40                 showInfo(e, g.id, e.target.attributes.tooltip.value);
  41         }
  42 }, false);
  43
  44 svg.addEventListener('mouseout', function(e) {
  45                 var g = e.target.parentNode;
  46                 var x = document.getElementById('r_' + g.id);
  47                 if (x) {
  48                         g.setAttribute('class', g.getAttribute('class').replace(' over', ''));
  49                         x.setAttribute('class', x.getAttribute('class').replace(' over', ''));
  50                         hideInfo(e);
  51                 }
  52 }, false);
  53
  54 function showInfo(evt, txt, details) {
  55 ${if project.tooltip}
  56         tooltip = document.getElementById('tooltip');
  57
  58         var t = document.getElementById('tooltiptext');
  59         t.firstChild.data = txt + " " + details;
  60
  61         var x = evt.clientX + 9;
  62         if (x > 250) { x -= t.getComputedTextLength() + 16; }
  63         var y = evt.clientY + 20;
  64         tooltip.setAttribute("transform", "translate(" + x + "," + y + ")");
  65         tooltip.setAttributeNS(null, "visibility", "visible");
  66
  67         var r = document.getElementById('tooltiprect');
  68         r.setAttribute('width', t.getComputedTextLength() + 6);
  69 ${endif}
  70 }
  71
  72 function hideInfo(evt) {
  73         var tooltip = document.getElementById('tooltip');
  74         tooltip.setAttributeNS(null,"visibility","hidden");
  75 }
  76 ]]></script>
  77
  78 <!-- inkscape requires a big rectangle or it will not export the pictures properly -->
  79 <rect
  80    x='${project.x}' y='${project.y}' width='${project.width}' height='${project.height}'
  81    style="font-size:10;fill:#ffffff;fill-opacity:0.01;fill-rule:evenodd;stroke:#ffffff;"></rect>
  82
  83 ${if project.title}
  84   <text x="${project.title_x}" y="${project.title_y}"
  85     style="font-size:15px; text-anchor:middle; font-style:normal;font-weight:normal;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;font-family:Bitstream Vera Sans">${project.title}</text>
  86 ${endif}
  87
  88
  89 ${for cls in project.groups}
  90   <g id='${cls.classname}'>
  91     ${for rect in cls.rects}
  92     <rect x='${rect.x}' y='${rect.y}' width='${rect.width}' height='${rect.height}' tooltip='${rect.name}' style="font-size:10;fill:${rect.color};fill-rule:evenodd;stroke:#000000;stroke-width:0.4;" />
  93     ${endfor}
  94   </g>
  95 ${endfor}
  96
  97 ${for info in project.infos}
  98   <g id='r_${info.classname}'>
  99    <rect x='${info.x}' y='${info.y}' width='${info.width}' height='${info.height}' style="font-size:10;fill:${info.color};fill-rule:evenodd;stroke:#000000;stroke-width:0.4;" />
 100    <text x="${info.text_x}" y="${info.text_y}"
 101        style="font-size:12px;font-style:normal;font-weight:normal;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;font-family:Bitstream Vera Sans"
 102    >${info.text}</text>
 103   </g>
 104 ${endfor}
 105
 106 ${if project.tooltip}
 107   <g transform="translate(0,0)" visibility="hidden" id="tooltip">
 108        <rect id="tooltiprect" y="-15" x="-3" width="1" height="20" style="stroke:black;fill:#edefc2;stroke-width:1"/>
 109        <text id="tooltiptext" style="font-family:Arial; font-size:12;fill:black;"> </text>
 110   </g>
 111 ${endif}
 112
 113 </svg>
 114 """
 115
 116 COMPILE_TEMPLATE = '''def f(project):
 117         lst = []
 118         def xml_escape(value):
 119                 return value.replace("&", "&amp;").replace('"', "&quot;").replace("'", "&apos;").replace("<", "&lt;").replace(">", "&gt;")
 120
 121         %s
 122         return ''.join(lst)
 123 '''
 124 reg_act = re.compile(r"(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<code>[^}]*?)\})", re.M)
 125 def compile_template(line):
 126
 127         extr = []
 128         def repl(match):
 129                 g = match.group
 130                 if g('dollar'):
 131                         return "$"
 132                 elif g('backslash'):
 133                         return "\\"
 134                 elif g('subst'):
 135                         extr.append(g('code'))
 136                         return "<<|@|>>"
 137                 return None
 138
 139         line2 = reg_act.sub(repl, line)
 140         params = line2.split('<<|@|>>')
 141         assert(extr)
 142
 143
 144         indent = 0
 145         buf = []
 146         app = buf.append
 147
 148         def app(txt):
 149                 buf.append(indent * '\t' + txt)
 150
 151         for x in range(len(extr)):
 152                 if params[x]:
 153                         app("lst.append(%r)" % params[x])
 154
 155                 f = extr[x]
 156                 if f.startswith(('if', 'for')):
 157                         app(f + ':')
 158                         indent += 1
 159                 elif f.startswith('py:'):
 160                         app(f[3:])
 161                 elif f.startswith(('endif', 'endfor')):
 162                         indent -= 1
 163                 elif f.startswith(('else', 'elif')):
 164                         indent -= 1
 165                         app(f + ':')
 166                         indent += 1
 167                 elif f.startswith('xml:'):
 168                         app('lst.append(xml_escape(%s))' % f[4:])
 169                 else:
 170                         #app('lst.append((%s) or "cannot find %s")' % (f, f))
 171                         app('lst.append(str(%s))' % f)
 172
 173         if extr:
 174                 if params[-1]:
 175                         app("lst.append(%r)" % params[-1])
 176
 177         fun = COMPILE_TEMPLATE % "\n\t".join(buf)
 178         # uncomment the following to debug the template
 179         #for i, x in enumerate(fun.splitlines()):
 180         #       print i, x
 181         return Task.funex(fun)
 182
 183 # red   #ff4d4d
 184 # green #4da74d
 185 # lila  #a751ff
 186
 187 color2code = {
 188         'GREEN'  : '#4da74d',
 189         'YELLOW' : '#fefe44',
 190         'PINK'   : '#a751ff',
 191         'RED'    : '#cc1d1d',
 192         'BLUE'   : '#6687bb',
 193         'CYAN'   : '#34e2e2',
 194 }
 195
 196 mp = {}
 197 info = [] # list of (text,color)
 198
 199 def map_to_color(name):
 200         if name in mp:
 201                 return mp[name]
 202         try:
 203                 cls = Task.classes[name]
 204         except KeyError:
 205                 return color2code['RED']
 206         if cls.color in mp:
 207                 return mp[cls.color]
 208         if cls.color in color2code:
 209                 return color2code[cls.color]
 210         return color2code['RED']
 211
 212 def process(self):
 213         m = self.generator.bld.producer
 214         try:
 215                 # TODO another place for this?
 216                 del self.generator.bld.task_sigs[self.uid()]
 217         except KeyError:
 218                 pass
 219
 220         self.generator.bld.producer.set_running(1, self)
 221
 222         try:
 223                 ret = self.run()
 224         except Exception:
 225                 self.err_msg = traceback.format_exc()
 226                 self.hasrun = Task.EXCEPTION
 227
 228                 # TODO cleanup
 229                 m.error_handler(self)
 230                 return
 231
 232         if ret:
 233                 self.err_code = ret
 234                 self.hasrun = Task.CRASHED
 235         else:
 236                 try:
 237                         self.post_run()
 238                 except Errors.WafError:
 239                         pass
 240                 except Exception:
 241                         self.err_msg = traceback.format_exc()
 242                         self.hasrun = Task.EXCEPTION
 243                 else:
 244                         self.hasrun = Task.SUCCESS
 245         if self.hasrun != Task.SUCCESS:
 246                 m.error_handler(self)
 247
 248         self.generator.bld.producer.set_running(-1, self)
 249
 250 Task.Task.process_back = Task.Task.process
 251 Task.Task.process = process
 252
 253 old_start = Runner.Parallel.start
 254 def do_start(self):
 255         try:
 256                 Options.options.dband
 257         except AttributeError:
 258                 self.bld.fatal('use def options(opt): opt.load("parallel_debug")!')
 259
 260         self.taskinfo = Queue()
 261         old_start(self)
 262         if self.dirty:
 263                 make_picture(self)
 264 Runner.Parallel.start = do_start
 265
 266 lock_running = threading.Lock()
 267 def set_running(self, by, tsk):
 268         with lock_running:
 269                 try:
 270                         cache = self.lock_cache
 271                 except AttributeError:
 272                         cache = self.lock_cache = {}
 273
 274                 i = 0
 275                 if by > 0:
 276                         vals = cache.values()
 277                         for i in range(self.numjobs):
 278                                 if i not in vals:
 279                                         cache[tsk] = i
 280                                         break
 281                 else:
 282                         i = cache[tsk]
 283                         del cache[tsk]
 284
 285                 self.taskinfo.put( (i, id(tsk), time.time(), tsk.__class__.__name__, self.processed, self.count, by, ",".join(map(str, tsk.outputs)))  )
 286 Runner.Parallel.set_running = set_running
 287
 288 def name2class(name):
 289         return name.replace(' ', '_').replace('.', '_')
 290
 291 def make_picture(producer):
 292         # first, cast the parameters
 293         if not hasattr(producer.bld, 'path'):
 294                 return
 295
 296         tmp = []
 297         try:
 298                 while True:
 299                         tup = producer.taskinfo.get(False)
 300                         tmp.append(list(tup))
 301         except:
 302                 pass
 303
 304         try:
 305                 ini = float(tmp[0][2])
 306         except:
 307                 return
 308
 309         if not info:
 310                 seen = []
 311                 for x in tmp:
 312                         name = x[3]
 313                         if not name in seen:
 314                                 seen.append(name)
 315                         else:
 316                                 continue
 317
 318                         info.append((name, map_to_color(name)))
 319                 info.sort(key=lambda x: x[0])
 320
 321         thread_count = 0
 322         acc = []
 323         for x in tmp:
 324                 thread_count += x[6]
 325                 acc.append("%d %d %f %r %d %d %d %s" % (x[0], x[1], x[2] - ini, x[3], x[4], x[5], thread_count, x[7]))
 326
 327         data_node = producer.bld.path.make_node('pdebug.dat')
 328         data_node.write('\n'.join(acc))
 329
 330         tmp = [lst[:2] + [float(lst[2]) - ini] + lst[3:] for lst in tmp]
 331
 332         st = {}
 333         for l in tmp:
 334                 if not l[0] in st:
 335                         st[l[0]] = len(st.keys())
 336         tmp = [  [st[lst[0]]] + lst[1:] for lst in tmp ]
 337         THREAD_AMOUNT = len(st.keys())
 338
 339         st = {}
 340         for l in tmp:
 341                 if not l[1] in st:
 342                         st[l[1]] = len(st.keys())
 343         tmp = [  [lst[0]] + [st[lst[1]]] + lst[2:] for lst in tmp ]
 344
 345
 346         BAND = Options.options.dband
 347
 348         seen = {}
 349         acc = []
 350         for x in range(len(tmp)):
 351                 line = tmp[x]
 352                 id = line[1]
 353
 354                 if id in seen:
 355                         continue
 356                 seen[id] = True
 357
 358                 begin = line[2]
 359                 thread_id = line[0]
 360                 for y in range(x + 1, len(tmp)):
 361                         line = tmp[y]
 362                         if line[1] == id:
 363                                 end = line[2]
 364                                 #print id, thread_id, begin, end
 365                                 #acc.append(  ( 10*thread_id, 10*(thread_id+1), 10*begin, 10*end ) )
 366                                 acc.append( (BAND * begin, BAND*thread_id, BAND*end - BAND*begin, BAND, line[3], line[7]) )
 367                                 break
 368
 369         if Options.options.dmaxtime < 0.1:
 370                 gwidth = 1
 371                 for x in tmp:
 372                         m = BAND * x[2]
 373                         if m > gwidth:
 374                                 gwidth = m
 375         else:
 376                 gwidth = BAND * Options.options.dmaxtime
 377
 378         ratio = float(Options.options.dwidth) / gwidth
 379         gwidth = Options.options.dwidth
 380         gheight = BAND * (THREAD_AMOUNT + len(info) + 1.5)
 381
 382
 383         # simple data model for our template
 384         class tobject(object):
 385                 pass
 386
 387         model = tobject()
 388         model.x = 0
 389         model.y = 0
 390         model.width = gwidth + 4
 391         model.height = gheight + 4
 392
 393         model.tooltip = not Options.options.dnotooltip
 394
 395         model.title = Options.options.dtitle
 396         model.title_x = gwidth / 2
 397         model.title_y = gheight + - 5
 398
 399         groups = {}
 400         for (x, y, w, h, clsname, name) in acc:
 401                 try:
 402                         groups[clsname].append((x, y, w, h, name))
 403                 except:
 404                         groups[clsname] = [(x, y, w, h, name)]
 405
 406         # groups of rectangles (else js highlighting is slow)
 407         model.groups = []
 408         for cls in groups:
 409                 g = tobject()
 410                 model.groups.append(g)
 411                 g.classname = name2class(cls)
 412                 g.rects = []
 413                 for (x, y, w, h, name) in groups[cls]:
 414                         r = tobject()
 415                         g.rects.append(r)
 416                         r.x = 2 + x * ratio
 417                         r.y = 2 + y
 418                         r.width = w * ratio
 419                         r.height = h
 420                         r.name = name
 421                         r.color = map_to_color(cls)
 422
 423         cnt = THREAD_AMOUNT
 424
 425         # caption
 426         model.infos = []
 427         for (text, color) in info:
 428                 inf = tobject()
 429                 model.infos.append(inf)
 430                 inf.classname = name2class(text)
 431                 inf.x = 2 + BAND
 432                 inf.y = 5 + (cnt + 0.5) * BAND
 433                 inf.width = BAND/2
 434                 inf.height = BAND/2
 435                 inf.color = color
 436
 437                 inf.text = text
 438                 inf.text_x = 2 + 2 * BAND
 439                 inf.text_y = 5 + (cnt + 0.5) * BAND + 10
 440
 441                 cnt += 1
 442
 443         # write the file...
 444         template1 = compile_template(SVG_TEMPLATE)
 445         txt = template1(model)
 446
 447         node = producer.bld.path.make_node('pdebug.svg')
 448         node.write(txt)
 449         Logs.warn('Created the diagram %r', node)
 450
 451 def options(opt):
 452         opt.add_option('--dtitle', action='store', default='Parallel build representation for %r' % ' '.join(sys.argv),
 453                 help='title for the svg diagram', dest='dtitle')
 454         opt.add_option('--dwidth', action='store', type='int', help='diagram width', default=800, dest='dwidth')
 455         opt.add_option('--dtime', action='store', type='float', help='recording interval in seconds', default=0.009, dest='dtime')
 456         opt.add_option('--dband', action='store', type='int', help='band width', default=22, dest='dband')
 457         opt.add_option('--dmaxtime', action='store', type='float', help='maximum time, for drawing fair comparisons', default=0, dest='dmaxtime')
 458         opt.add_option('--dnotooltip', action='store_true', help='disable tooltips', default=False, dest='dnotooltip')
 459