Include waf as an extracted source directory, rather than as a one-in-a-file script.
[samba.git] / buildtools / wafadmin / Tools / unittestw.py
1 #!/usr/bin/env python
2 # encoding: utf-8
3 # Carlos Rafael Giani, 2006
4
5 """
6 Unit tests run in the shutdown() method, and for c/c++ programs
7
8 One should NOT have to give parameters to programs to execute
9
10 In the shutdown method, add the following code:
11
12         >>> def shutdown():
13         ...     ut = UnitTest.unit_test()
14         ...     ut.run()
15         ...     ut.print_results()
16
17
18 Each object to use as a unit test must be a program and must have X{obj.unit_test=1}
19 """
20 import os, sys
21 import Build, TaskGen, Utils, Options, Logs, Task
22 from TaskGen import before, after, feature
23 from Constants import *
24
25 class unit_test(object):
26         "Unit test representation"
27         def __init__(self):
28                 self.returncode_ok = 0          # Unit test returncode considered OK. All returncodes differing from this one
29                                                 # will cause the unit test to be marked as "FAILED".
30
31                 # The following variables are filled with data by run().
32
33                 # print_results() uses these for printing the unit test summary,
34                 # but if there is need for direct access to the results,
35                 # they can be retrieved here, after calling run().
36
37                 self.num_tests_ok = 0           # Number of successful unit tests
38                 self.num_tests_failed = 0       # Number of failed unit tests
39                 self.num_tests_err = 0          # Tests that have not even run
40                 self.total_num_tests = 0        # Total amount of unit tests
41                 self.max_label_length = 0       # Maximum label length (pretty-print the output)
42
43                 self.unit_tests = Utils.ordered_dict()          # Unit test dictionary. Key: the label (unit test filename relative
44                                                 # to the build dir), value: unit test filename with absolute path
45                 self.unit_test_results = {}     # Dictionary containing the unit test results.
46                                                 # Key: the label, value: result (true = success false = failure)
47                 self.unit_test_erroneous = {}   # Dictionary indicating erroneous unit tests.
48                                                 # Key: the label, value: true = unit test has an error  false = unit test is ok
49                 self.change_to_testfile_dir = False #True if the test file needs to be executed from the same dir
50                 self.want_to_see_test_output = False #True to see the stdout from the testfile (for example check suites)
51                 self.want_to_see_test_error = False #True to see the stderr from the testfile (for example check suites)
52                 self.run_if_waf_does = 'check' #build was the old default
53
54         def run(self):
55                 "Run the unit tests and gather results (note: no output here)"
56
57                 self.num_tests_ok = 0
58                 self.num_tests_failed = 0
59                 self.num_tests_err = 0
60                 self.total_num_tests = 0
61                 self.max_label_length = 0
62
63                 self.unit_tests = Utils.ordered_dict()
64                 self.unit_test_results = {}
65                 self.unit_test_erroneous = {}
66
67                 ld_library_path = []
68
69                 # If waf is not building, don't run anything
70                 if not Options.commands[self.run_if_waf_does]: return
71
72                 # Get the paths for the shared libraries, and obtain the unit tests to execute
73                 for obj in Build.bld.all_task_gen:
74                         try:
75                                 link_task = obj.link_task
76                         except AttributeError:
77                                 pass
78                         else:
79                                 lib_path = link_task.outputs[0].parent.abspath(obj.env)
80                                 if lib_path not in ld_library_path:
81                                         ld_library_path.append(lib_path)
82
83                         unit_test = getattr(obj, 'unit_test', '')
84                         if unit_test and 'cprogram' in obj.features:
85                                 try:
86                                         output = obj.path
87                                         filename = os.path.join(output.abspath(obj.env), obj.target)
88                                         srcdir = output.abspath()
89                                         label = os.path.join(output.bldpath(obj.env), obj.target)
90                                         self.max_label_length = max(self.max_label_length, len(label))
91                                         self.unit_tests[label] = (filename, srcdir)
92                                 except KeyError:
93                                         pass
94                 self.total_num_tests = len(self.unit_tests)
95                 # Now run the unit tests
96                 Utils.pprint('GREEN', 'Running the unit tests')
97                 count = 0
98                 result = 1
99
100                 for label in self.unit_tests.allkeys:
101                         file_and_src = self.unit_tests[label]
102                         filename = file_and_src[0]
103                         srcdir = file_and_src[1]
104                         count += 1
105                         line = Build.bld.progress_line(count, self.total_num_tests, Logs.colors.GREEN, Logs.colors.NORMAL)
106                         if Options.options.progress_bar and line:
107                                 sys.stderr.write(line)
108                                 sys.stderr.flush()
109                         try:
110                                 kwargs = {}
111                                 kwargs['env'] = os.environ.copy()
112                                 if self.change_to_testfile_dir:
113                                         kwargs['cwd'] = srcdir
114                                 if not self.want_to_see_test_output:
115                                         kwargs['stdout'] = Utils.pproc.PIPE  # PIPE for ignoring output
116                                 if not self.want_to_see_test_error:
117                                         kwargs['stderr'] = Utils.pproc.PIPE  # PIPE for ignoring output
118                                 if ld_library_path:
119                                         v = kwargs['env']
120                                         def add_path(dct, path, var):
121                                                 dct[var] = os.pathsep.join(Utils.to_list(path) + [os.environ.get(var, '')])
122                                         if sys.platform == 'win32':
123                                                 add_path(v, ld_library_path, 'PATH')
124                                         elif sys.platform == 'darwin':
125                                                 add_path(v, ld_library_path, 'DYLD_LIBRARY_PATH')
126                                                 add_path(v, ld_library_path, 'LD_LIBRARY_PATH')
127                                         else:
128                                                 add_path(v, ld_library_path, 'LD_LIBRARY_PATH')
129
130                                 pp = Utils.pproc.Popen(filename, **kwargs)
131                                 (out, err) = pp.communicate() # uh, and the output is ignored?? - fortunately this is going to disappear
132
133                                 result = int(pp.returncode == self.returncode_ok)
134
135                                 if result:
136                                         self.num_tests_ok += 1
137                                 else:
138                                         self.num_tests_failed += 1
139
140                                 self.unit_test_results[label] = result
141                                 self.unit_test_erroneous[label] = 0
142                         except OSError:
143                                 self.unit_test_erroneous[label] = 1
144                                 self.num_tests_err += 1
145                         except KeyboardInterrupt:
146                                 pass
147                 if Options.options.progress_bar: sys.stdout.write(Logs.colors.cursor_on)
148
149         def print_results(self):
150                 "Pretty-prints a summary of all unit tests, along with some statistics"
151
152                 # If waf is not building, don't output anything
153                 if not Options.commands[self.run_if_waf_does]: return
154
155                 p = Utils.pprint
156                 # Early quit if no tests were performed
157                 if self.total_num_tests == 0:
158                         p('YELLOW', 'No unit tests present')
159                         return
160
161                 for label in self.unit_tests.allkeys:
162                         filename = self.unit_tests[label]
163                         err = 0
164                         result = 0
165
166                         try: err = self.unit_test_erroneous[label]
167                         except KeyError: pass
168
169                         try: result = self.unit_test_results[label]
170                         except KeyError: pass
171
172                         n = self.max_label_length - len(label)
173                         if err: n += 4
174                         elif result: n += 7
175                         else: n += 3
176
177                         line = '%s %s' % (label, '.' * n)
178
179                         if err: p('RED', '%sERROR' % line)
180                         elif result: p('GREEN', '%sOK' % line)
181                         else: p('YELLOW', '%sFAILED' % line)
182
183                 percentage_ok = float(self.num_tests_ok) / float(self.total_num_tests) * 100.0
184                 percentage_failed = float(self.num_tests_failed) / float(self.total_num_tests) * 100.0
185                 percentage_erroneous = float(self.num_tests_err) / float(self.total_num_tests) * 100.0
186
187                 p('NORMAL', '''
188 Successful tests:      %i (%.1f%%)
189 Failed tests:          %i (%.1f%%)
190 Erroneous tests:       %i (%.1f%%)
191
192 Total number of tests: %i
193 ''' % (self.num_tests_ok, percentage_ok, self.num_tests_failed, percentage_failed,
194                 self.num_tests_err, percentage_erroneous, self.total_num_tests))
195                 p('GREEN', 'Unit tests finished')
196
197
198 ############################################################################################
199
200 """
201 New unit test system
202
203 The targets with feature 'test' are executed after they are built
204 bld(features='cprogram cc test', ...)
205
206 To display the results:
207 import UnitTest
208 bld.add_post_fun(UnitTest.summary)
209 """
210
211 import threading
212 testlock = threading.Lock()
213
214 def set_options(opt):
215         opt.add_option('--alltests', action='store_true', default=True, help='Exec all unit tests', dest='all_tests')
216
217 @feature('test')
218 @after('apply_link', 'vars_target_cprogram')
219 def make_test(self):
220         if not 'cprogram' in self.features:
221                 Logs.error('test cannot be executed %s' % self)
222                 return
223
224         self.default_install_path = None
225         self.create_task('utest', self.link_task.outputs)
226
227 def exec_test(self):
228
229         status = 0
230
231         variant = self.env.variant()
232
233         filename = self.inputs[0].abspath(self.env)
234         self.ut_exec = getattr(self, 'ut_exec', [filename])
235         if getattr(self.generator, 'ut_fun', None):
236                 self.generator.ut_fun(self)
237
238         try:
239                 fu = getattr(self.generator.bld, 'all_test_paths')
240         except AttributeError:
241                 fu = os.environ.copy()
242                 self.generator.bld.all_test_paths = fu
243
244                 lst = []
245                 for obj in self.generator.bld.all_task_gen:
246                         link_task = getattr(obj, 'link_task', None)
247                         if link_task and link_task.env.variant() == variant:
248                                 lst.append(link_task.outputs[0].parent.abspath(obj.env))
249
250                 def add_path(dct, path, var):
251                         dct[var] = os.pathsep.join(Utils.to_list(path) + [os.environ.get(var, '')])
252
253                 if sys.platform == 'win32':
254                         add_path(fu, lst, 'PATH')
255                 elif sys.platform == 'darwin':
256                         add_path(fu, lst, 'DYLD_LIBRARY_PATH')
257                         add_path(fu, lst, 'LD_LIBRARY_PATH')
258                 else:
259                         add_path(fu, lst, 'LD_LIBRARY_PATH')
260
261
262         cwd = getattr(self.generator, 'ut_cwd', '') or self.inputs[0].parent.abspath(self.env)
263         proc = Utils.pproc.Popen(self.ut_exec, cwd=cwd, env=fu, stderr=Utils.pproc.PIPE, stdout=Utils.pproc.PIPE)
264         (stdout, stderr) = proc.communicate()
265
266         tup = (filename, proc.returncode, stdout, stderr)
267         self.generator.utest_result = tup
268
269         testlock.acquire()
270         try:
271                 bld = self.generator.bld
272                 Logs.debug("ut: %r", tup)
273                 try:
274                         bld.utest_results.append(tup)
275                 except AttributeError:
276                         bld.utest_results = [tup]
277         finally:
278                 testlock.release()
279
280 cls = Task.task_type_from_func('utest', func=exec_test, color='PINK', ext_in='.bin')
281
282 old = cls.runnable_status
283 def test_status(self):
284         ret = old(self)
285         if ret == SKIP_ME and getattr(Options.options, 'all_tests', False):
286                 return RUN_ME
287         return ret
288
289 cls.runnable_status = test_status
290 cls.quiet = 1
291
292 def summary(bld):
293         lst = getattr(bld, 'utest_results', [])
294         if lst:
295                 Utils.pprint('CYAN', 'execution summary')
296
297                 total = len(lst)
298                 tfail = len([x for x in lst if x[1]])
299
300                 Utils.pprint('CYAN', '  tests that pass %d/%d' % (total-tfail, total))
301                 for (f, code, out, err) in lst:
302                         if not code:
303                                 Utils.pprint('CYAN', '    %s' % f)
304
305                 Utils.pprint('CYAN', '  tests that fail %d/%d' % (tfail, total))
306                 for (f, code, out, err) in lst:
307                         if code:
308                                 Utils.pprint('CYAN', '    %s' % f)
309
310