third_party/waf/waflib/extras/parallel_debug.py

   1 #! /usr/bin/env python
   2 # encoding: utf-8
   3 # WARNING! Do not edit! https://waf.io/book/index.html#_obtaining_the_waf_file
   4
   5 #! /usr/bin/env python
   6 # encoding: utf-8
   7 # Thomas Nagy, 2007-2010 (ita)
   8
   9 """
  10 Debugging helper for parallel compilation, outputs
  11 a file named pdebug.svg in the source directory::
  12
  13         def options(opt):
  14                 opt.load('parallel_debug')
  15         def build(bld):
  16                 ...
  17 """
  18
  19 import re, sys, threading, time, traceback
  20 try:
  21         from Queue import Queue
  22 except:
  23         from queue import Queue
  24 from waflib import Runner, Options, Task, Logs, Errors
  25
  26 SVG_TEMPLATE = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
  27 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
  28 <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.0"
  29    x="${project.x}" y="${project.y}" width="${project.width}" height="${project.height}" id="svg602" xml:space="preserve">
  30
  31 <style type='text/css' media='screen'>
  32         g.over rect { stroke:#FF0000; fill-opacity:0.4 }
  33 </style>
  34
  35 <script type='text/javascript'><![CDATA[
  36 var svg  = document.getElementsByTagName('svg')[0];
  37
  38 svg.addEventListener('mouseover', function(e) {
  39         var g = e.target.parentNode;
  40         var x = document.getElementById('r_' + g.id);
  41         if (x) {
  42                 g.setAttribute('class', g.getAttribute('class') + ' over');
  43                 x.setAttribute('class', x.getAttribute('class') + ' over');
  44                 showInfo(e, g.id, e.target.attributes.tooltip.value);
  45         }
  46 }, false);
  47
  48 svg.addEventListener('mouseout', function(e) {
  49                 var g = e.target.parentNode;
  50                 var x = document.getElementById('r_' + g.id);
  51                 if (x) {
  52                         g.setAttribute('class', g.getAttribute('class').replace(' over', ''));
  53                         x.setAttribute('class', x.getAttribute('class').replace(' over', ''));
  54                         hideInfo(e);
  55                 }
  56 }, false);
  57
  58 function showInfo(evt, txt, details) {
  59 ${if project.tooltip}
  60         tooltip = document.getElementById('tooltip');
  61
  62         var t = document.getElementById('tooltiptext');
  63         t.firstChild.data = txt + " " + details;
  64
  65         var x = evt.clientX + 9;
  66         if (x > 250) { x -= t.getComputedTextLength() + 16; }
  67         var y = evt.clientY + 20;
  68         tooltip.setAttribute("transform", "translate(" + x + "," + y + ")");
  69         tooltip.setAttributeNS(null, "visibility", "visible");
  70
  71         var r = document.getElementById('tooltiprect');
  72         r.setAttribute('width', t.getComputedTextLength() + 6);
  73 ${endif}
  74 }
  75
  76 function hideInfo(evt) {
  77         var tooltip = document.getElementById('tooltip');
  78         tooltip.setAttributeNS(null,"visibility","hidden");
  79 }
  80 ]]></script>
  81
  82 <!-- inkscape requires a big rectangle or it will not export the pictures properly -->
  83 <rect
  84    x='${project.x}' y='${project.y}' width='${project.width}' height='${project.height}'
  85    style="font-size:10;fill:#ffffff;fill-opacity:0.01;fill-rule:evenodd;stroke:#ffffff;"></rect>
  86
  87 ${if project.title}
  88   <text x="${project.title_x}" y="${project.title_y}"
  89     style="font-size:15px; text-anchor:middle; font-style:normal;font-weight:normal;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;font-family:Bitstream Vera Sans">${project.title}</text>
  90 ${endif}
  91
  92
  93 ${for cls in project.groups}
  94   <g id='${cls.classname}'>
  95     ${for rect in cls.rects}
  96     <rect x='${rect.x}' y='${rect.y}' width='${rect.width}' height='${rect.height}' tooltip='${rect.name}' style="font-size:10;fill:${rect.color};fill-rule:evenodd;stroke:#000000;stroke-width:0.4;" />
  97     ${endfor}
  98   </g>
  99 ${endfor}
 100
 101 ${for info in project.infos}
 102   <g id='r_${info.classname}'>
 103    <rect x='${info.x}' y='${info.y}' width='${info.width}' height='${info.height}' style="font-size:10;fill:${info.color};fill-rule:evenodd;stroke:#000000;stroke-width:0.4;" />
 104    <text x="${info.text_x}" y="${info.text_y}"
 105        style="font-size:12px;font-style:normal;font-weight:normal;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;font-family:Bitstream Vera Sans"
 106    >${info.text}</text>
 107   </g>
 108 ${endfor}
 109
 110 ${if project.tooltip}
 111   <g transform="translate(0,0)" visibility="hidden" id="tooltip">
 112        <rect id="tooltiprect" y="-15" x="-3" width="1" height="20" style="stroke:black;fill:#edefc2;stroke-width:1"/>
 113        <text id="tooltiptext" style="font-family:Arial; font-size:12;fill:black;"> </text>
 114   </g>
 115 ${endif}
 116
 117 </svg>
 118 """
 119
 120 COMPILE_TEMPLATE = '''def f(project):
 121         lst = []
 122         def xml_escape(value):
 123                 return value.replace("&", "&amp;").replace('"', "&quot;").replace("'", "&apos;").replace("<", "&lt;").replace(">", "&gt;")
 124
 125         %s
 126         return ''.join(lst)
 127 '''
 128 reg_act = re.compile(r"(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<code>[^}]*?)\})", re.M)
 129 def compile_template(line):
 130
 131         extr = []
 132         def repl(match):
 133                 g = match.group
 134                 if g('dollar'):
 135                         return "$"
 136                 elif g('backslash'):
 137                         return "\\"
 138                 elif g('subst'):
 139                         extr.append(g('code'))
 140                         return "<<|@|>>"
 141                 return None
 142
 143         line2 = reg_act.sub(repl, line)
 144         params = line2.split('<<|@|>>')
 145         assert(extr)
 146
 147
 148         indent = 0
 149         buf = []
 150         app = buf.append
 151
 152         def app(txt):
 153                 buf.append(indent * '\t' + txt)
 154
 155         for x in range(len(extr)):
 156                 if params[x]:
 157                         app("lst.append(%r)" % params[x])
 158
 159                 f = extr[x]
 160                 if f.startswith(('if', 'for')):
 161                         app(f + ':')
 162                         indent += 1
 163                 elif f.startswith('py:'):
 164                         app(f[3:])
 165                 elif f.startswith(('endif', 'endfor')):
 166                         indent -= 1
 167                 elif f.startswith(('else', 'elif')):
 168                         indent -= 1
 169                         app(f + ':')
 170                         indent += 1
 171                 elif f.startswith('xml:'):
 172                         app('lst.append(xml_escape(%s))' % f[4:])
 173                 else:
 174                         #app('lst.append((%s) or "cannot find %s")' % (f, f))
 175                         app('lst.append(str(%s))' % f)
 176
 177         if extr:
 178                 if params[-1]:
 179                         app("lst.append(%r)" % params[-1])
 180
 181         fun = COMPILE_TEMPLATE % "\n\t".join(buf)
 182         # uncomment the following to debug the template
 183         #for i, x in enumerate(fun.splitlines()):
 184         #       print i, x
 185         return Task.funex(fun)
 186
 187 # red   #ff4d4d
 188 # green #4da74d
 189 # lila  #a751ff
 190
 191 color2code = {
 192         'GREEN'  : '#4da74d',
 193         'YELLOW' : '#fefe44',
 194         'PINK'   : '#a751ff',
 195         'RED'    : '#cc1d1d',
 196         'BLUE'   : '#6687bb',
 197         'CYAN'   : '#34e2e2',
 198 }
 199
 200 mp = {}
 201 info = [] # list of (text,color)
 202
 203 def map_to_color(name):
 204         if name in mp:
 205                 return mp[name]
 206         try:
 207                 cls = Task.classes[name]
 208         except KeyError:
 209                 return color2code['RED']
 210         if cls.color in mp:
 211                 return mp[cls.color]
 212         if cls.color in color2code:
 213                 return color2code[cls.color]
 214         return color2code['RED']
 215
 216 def process(self):
 217         m = self.generator.bld.producer
 218         try:
 219                 # TODO another place for this?
 220                 del self.generator.bld.task_sigs[self.uid()]
 221         except KeyError:
 222                 pass
 223
 224         self.generator.bld.producer.set_running(1, self)
 225
 226         try:
 227                 ret = self.run()
 228         except Exception:
 229                 self.err_msg = traceback.format_exc()
 230                 self.hasrun = Task.EXCEPTION
 231
 232                 # TODO cleanup
 233                 m.error_handler(self)
 234                 return
 235
 236         if ret:
 237                 self.err_code = ret
 238                 self.hasrun = Task.CRASHED
 239         else:
 240                 try:
 241                         self.post_run()
 242                 except Errors.WafError:
 243                         pass
 244                 except Exception:
 245                         self.err_msg = traceback.format_exc()
 246                         self.hasrun = Task.EXCEPTION
 247                 else:
 248                         self.hasrun = Task.SUCCESS
 249         if self.hasrun != Task.SUCCESS:
 250                 m.error_handler(self)
 251
 252         self.generator.bld.producer.set_running(-1, self)
 253
 254 Task.Task.process_back = Task.Task.process
 255 Task.Task.process = process
 256
 257 old_start = Runner.Parallel.start
 258 def do_start(self):
 259         try:
 260                 Options.options.dband
 261         except AttributeError:
 262                 self.bld.fatal('use def options(opt): opt.load("parallel_debug")!')
 263
 264         self.taskinfo = Queue()
 265         old_start(self)
 266         if self.dirty:
 267                 make_picture(self)
 268 Runner.Parallel.start = do_start
 269
 270 lock_running = threading.Lock()
 271 def set_running(self, by, tsk):
 272         with lock_running:
 273                 try:
 274                         cache = self.lock_cache
 275                 except AttributeError:
 276                         cache = self.lock_cache = {}
 277
 278                 i = 0
 279                 if by > 0:
 280                         vals = cache.values()
 281                         for i in range(self.numjobs):
 282                                 if i not in vals:
 283                                         cache[tsk] = i
 284                                         break
 285                 else:
 286                         i = cache[tsk]
 287                         del cache[tsk]
 288
 289                 self.taskinfo.put( (i, id(tsk), time.time(), tsk.__class__.__name__, self.processed, self.count, by, ",".join(map(str, tsk.outputs)))  )
 290 Runner.Parallel.set_running = set_running
 291
 292 def name2class(name):
 293         return name.replace(' ', '_').replace('.', '_')
 294
 295 def make_picture(producer):
 296         # first, cast the parameters
 297         if not hasattr(producer.bld, 'path'):
 298                 return
 299
 300         tmp = []
 301         try:
 302                 while True:
 303                         tup = producer.taskinfo.get(False)
 304                         tmp.append(list(tup))
 305         except:
 306                 pass
 307
 308         try:
 309                 ini = float(tmp[0][2])
 310         except:
 311                 return
 312
 313         if not info:
 314                 seen = []
 315                 for x in tmp:
 316                         name = x[3]
 317                         if not name in seen:
 318                                 seen.append(name)
 319                         else:
 320                                 continue
 321
 322                         info.append((name, map_to_color(name)))
 323                 info.sort(key=lambda x: x[0])
 324
 325         thread_count = 0
 326         acc = []
 327         for x in tmp:
 328                 thread_count += x[6]
 329                 acc.append("%d %d %f %r %d %d %d %s" % (x[0], x[1], x[2] - ini, x[3], x[4], x[5], thread_count, x[7]))
 330
 331         data_node = producer.bld.path.make_node('pdebug.dat')
 332         data_node.write('\n'.join(acc))
 333
 334         tmp = [lst[:2] + [float(lst[2]) - ini] + lst[3:] for lst in tmp]
 335
 336         st = {}
 337         for l in tmp:
 338                 if not l[0] in st:
 339                         st[l[0]] = len(st.keys())
 340         tmp = [  [st[lst[0]]] + lst[1:] for lst in tmp ]
 341         THREAD_AMOUNT = len(st.keys())
 342
 343         st = {}
 344         for l in tmp:
 345                 if not l[1] in st:
 346                         st[l[1]] = len(st.keys())
 347         tmp = [  [lst[0]] + [st[lst[1]]] + lst[2:] for lst in tmp ]
 348
 349
 350         BAND = Options.options.dband
 351
 352         seen = {}
 353         acc = []
 354         for x in range(len(tmp)):
 355                 line = tmp[x]
 356                 id = line[1]
 357
 358                 if id in seen:
 359                         continue
 360                 seen[id] = True
 361
 362                 begin = line[2]
 363                 thread_id = line[0]
 364                 for y in range(x + 1, len(tmp)):
 365                         line = tmp[y]
 366                         if line[1] == id:
 367                                 end = line[2]
 368                                 #print id, thread_id, begin, end
 369                                 #acc.append(  ( 10*thread_id, 10*(thread_id+1), 10*begin, 10*end ) )
 370                                 acc.append( (BAND * begin, BAND*thread_id, BAND*end - BAND*begin, BAND, line[3], line[7]) )
 371                                 break
 372
 373         if Options.options.dmaxtime < 0.1:
 374                 gwidth = 1
 375                 for x in tmp:
 376                         m = BAND * x[2]
 377                         if m > gwidth:
 378                                 gwidth = m
 379         else:
 380                 gwidth = BAND * Options.options.dmaxtime
 381
 382         ratio = float(Options.options.dwidth) / gwidth
 383         gwidth = Options.options.dwidth
 384         gheight = BAND * (THREAD_AMOUNT + len(info) + 1.5)
 385
 386
 387         # simple data model for our template
 388         class tobject(object):
 389                 pass
 390
 391         model = tobject()
 392         model.x = 0
 393         model.y = 0
 394         model.width = gwidth + 4
 395         model.height = gheight + 4
 396
 397         model.tooltip = not Options.options.dnotooltip
 398
 399         model.title = Options.options.dtitle
 400         model.title_x = gwidth / 2
 401         model.title_y = gheight + - 5
 402
 403         groups = {}
 404         for (x, y, w, h, clsname, name) in acc:
 405                 try:
 406                         groups[clsname].append((x, y, w, h, name))
 407                 except:
 408                         groups[clsname] = [(x, y, w, h, name)]
 409
 410         # groups of rectangles (else js highlighting is slow)
 411         model.groups = []
 412         for cls in groups:
 413                 g = tobject()
 414                 model.groups.append(g)
 415                 g.classname = name2class(cls)
 416                 g.rects = []
 417                 for (x, y, w, h, name) in groups[cls]:
 418                         r = tobject()
 419                         g.rects.append(r)
 420                         r.x = 2 + x * ratio
 421                         r.y = 2 + y
 422                         r.width = w * ratio
 423                         r.height = h
 424                         r.name = name
 425                         r.color = map_to_color(cls)
 426
 427         cnt = THREAD_AMOUNT
 428
 429         # caption
 430         model.infos = []
 431         for (text, color) in info:
 432                 inf = tobject()
 433                 model.infos.append(inf)
 434                 inf.classname = name2class(text)
 435                 inf.x = 2 + BAND
 436                 inf.y = 5 + (cnt + 0.5) * BAND
 437                 inf.width = BAND/2
 438                 inf.height = BAND/2
 439                 inf.color = color
 440
 441                 inf.text = text
 442                 inf.text_x = 2 + 2 * BAND
 443                 inf.text_y = 5 + (cnt + 0.5) * BAND + 10
 444
 445                 cnt += 1
 446
 447         # write the file...
 448         template1 = compile_template(SVG_TEMPLATE)
 449         txt = template1(model)
 450
 451         node = producer.bld.path.make_node('pdebug.svg')
 452         node.write(txt)
 453         Logs.warn('Created the diagram %r', node)
 454
 455 def options(opt):
 456         opt.add_option('--dtitle', action='store', default='Parallel build representation for %r' % ' '.join(sys.argv),
 457                 help='title for the svg diagram', dest='dtitle')
 458         opt.add_option('--dwidth', action='store', type='int', help='diagram width', default=800, dest='dwidth')
 459         opt.add_option('--dtime', action='store', type='float', help='recording interval in seconds', default=0.009, dest='dtime')
 460         opt.add_option('--dband', action='store', type='int', help='band width', default=22, dest='dband')
 461         opt.add_option('--dmaxtime', action='store', type='float', help='maximum time, for drawing fair comparisons', default=0, dest='dmaxtime')
 462         opt.add_option('--dnotooltip', action='store_true', help='disable tooltips', default=False, dest='dnotooltip')
 463