third_party/waf/waflib/extras/mem_reducer.py

   1 #! /usr/bin/env python
   2 # encoding: UTF-8
   3
   4 """
   5 This tool can help to reduce the memory usage in very large builds featuring many tasks with after/before attributes.
   6 It may also improve the overall build time by decreasing the amount of iterations over tasks.
   7
   8 Usage:
   9 def options(opt):
  10         opt.load('mem_reducer')
  11 """
  12
  13 import itertools
  14 from waflib import Utils, Task, Runner
  15
  16 class SetOfTasks(object):
  17         """Wraps a set and a task which has a list of other sets.
  18         The interface is meant to mimic the interface of set. Add missing functions as needed.
  19         """
  20         def __init__(self, owner):
  21                 self._set = owner.run_after
  22                 self._owner = owner
  23
  24         def __iter__(self):
  25                 for g in self._owner.run_after_groups:
  26                         #print len(g)
  27                         for task in g:
  28                                 yield task
  29                 for task in self._set:
  30                         yield task
  31
  32         def add(self, obj):
  33                 self._set.add(obj)
  34
  35         def update(self, obj):
  36                 self._set.update(obj)
  37
  38 def set_precedence_constraints(tasks):
  39         cstr_groups = Utils.defaultdict(list)
  40         for x in tasks:
  41                 x.run_after = SetOfTasks(x)
  42                 x.run_after_groups = []
  43                 x.waiting_sets = []
  44
  45                 h = x.hash_constraints()
  46                 cstr_groups[h].append(x)
  47
  48         # create sets which can be reused for all tasks
  49         for k in cstr_groups.keys():
  50                 cstr_groups[k] = set(cstr_groups[k])
  51
  52         # this list should be short
  53         for key1, key2 in itertools.combinations(cstr_groups.keys(), 2):
  54                 group1 = cstr_groups[key1]
  55                 group2 = cstr_groups[key2]
  56                 # get the first entry of the set
  57                 t1 = next(iter(group1))
  58                 t2 = next(iter(group2))
  59
  60                 # add the constraints based on the comparisons
  61                 if Task.is_before(t1, t2):
  62                         for x in group2:
  63                                 x.run_after_groups.append(group1)
  64                         for k in group1:
  65                                 k.waiting_sets.append(group1)
  66                 elif Task.is_before(t2, t1):
  67                         for x in group1:
  68                                 x.run_after_groups.append(group2)
  69                         for k in group2:
  70                                 k.waiting_sets.append(group2)
  71
  72 Task.set_precedence_constraints = set_precedence_constraints
  73
  74 def get_out(self):
  75         tsk = self.out.get()
  76         if not self.stop:
  77                 self.add_more_tasks(tsk)
  78         self.count -= 1
  79         self.dirty = True
  80
  81         # shrinking sets
  82         try:
  83                 ws = tsk.waiting_sets
  84         except AttributeError:
  85                 pass
  86         else:
  87                 for k in ws:
  88                         try:
  89                                 k.remove(tsk)
  90                         except KeyError:
  91                                 pass
  92
  93         return tsk
  94 Runner.Parallel.get_out = get_out
  95
  96 def skip(self, tsk):
  97         tsk.hasrun = Task.SKIPPED
  98
  99         # shrinking sets
 100         try:
 101                 ws = tsk.waiting_sets
 102         except AttributeError:
 103                 pass
 104         else:
 105                 for k in ws:
 106                         try:
 107                                 k.remove(tsk)
 108                         except KeyError:
 109                                 pass
 110 Runner.Parallel.skip = skip