a76309df8a203f0aa525c506114169f9c9368117
[samba.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, check_output, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 import random
13 from optparse import OptionParser
14 import smtplib
15 import email
16 from email.mime.text import MIMEText
17 from email.mime.base import MIMEBase
18 from email.mime.application import MIMEApplication
19 from email.mime.multipart import MIMEMultipart
20 from distutils.sysconfig import get_python_lib
21 import platform
22
23 try:
24     from waflib.Build import CACHE_SUFFIX
25 except ImportError:
26     sys.path.insert(0, "./third_party/waf")
27     from waflib.Build import CACHE_SUFFIX
28
29
30 os.environ["PYTHONUNBUFFERED"] = "1"
31
32 # This speeds up testing remarkably.
33 os.environ['TDB_NO_FSYNC'] = '1'
34
35
36 def find_git_root():
37     '''get to the top of the git repo'''
38     p = os.getcwd()
39     while p != '/':
40         if os.path.exists(os.path.join(p, ".git")):
41             return p
42         p = os.path.abspath(os.path.join(p, '..'))
43     return None
44
45
46 gitroot = find_git_root()
47 if gitroot is None:
48     raise Exception("Failed to find git root")
49
50
51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
52
53 parser = OptionParser()
54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
58                   default=def_testbase)
59 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
60                   default=None)
61 parser.add_option("--passcmd", help="command to run on success", default=None)
62 parser.add_option("--verbose", help="show all commands as they are run",
63                   default=False, action="store_true")
64 parser.add_option("--rebase", help="rebase on the given tree before testing",
65                   default=None, type='str')
66 parser.add_option("--pushto", help="push to a git url on success",
67                   default=None, type='str')
68 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
69                   default=False, action="store_true")
70 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
71                   default=False, action="store_true")
72 parser.add_option("--retry", help="automatically retry if master changes",
73                   default=False, action="store_true")
74 parser.add_option("--email", help="send email to the given address on failure",
75                   type='str', default=None)
76 parser.add_option("--email-from", help="send email from the given address",
77                   type='str', default="autobuild@samba.org")
78 parser.add_option("--email-server", help="send email via the given server",
79                   type='str', default='localhost')
80 parser.add_option("--always-email", help="always send email, even on success",
81                   action="store_true")
82 parser.add_option("--daemon", help="daemonize after initial setup",
83                   action="store_true")
84 parser.add_option("--branch", help="the branch to work on (default=master)",
85                   default="master", type='str')
86 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
87                   default=gitroot, type='str')
88 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
89                   default=False, action="store_true")
90 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
91                   default='')
92 parser.add_option("--enable-coverage", dest='enable_coverage',
93                   action="store_const", const='--enable-coverage', default='',
94                   help="Add --enable-coverage option while configure")
95
96 (options, args) = parser.parse_args()
97
98 if options.retry:
99     if options.rebase is None:
100         raise Exception('You can only use --retry if you also rebase')
101
102 if options.full_testbase is not None:
103     testbase = options.full_testbase
104 else:
105     testbase = "%s/b%u" % (options.testbase, os.getpid())
106 test_master = "%s/master" % testbase
107 test_prefix = "%s/prefix" % testbase
108 test_tmpdir = "%s/tmp" % testbase
109 os.environ['TMPDIR'] = test_tmpdir
110
111 if options.enable_coverage:
112     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
113 else:
114     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
115
116 if args:
117     # If we are only running specific test,
118     # do not sleep randomly to wait for it to start
119     def random_sleep(low, high):
120         return 'sleep 1'
121 else:
122     def random_sleep(low, high):
123         return 'sleep {}'.format(random.randint(low, high))
124
125 cleanup_list = []
126
127 builddirs = {
128     "ctdb": "ctdb",
129     "ldb": "lib/ldb",
130     "tdb": "lib/tdb",
131     "talloc": "lib/talloc",
132     "replace": "lib/replace",
133     "tevent": "lib/tevent",
134     "pidl": "pidl",
135     "docs-xml": "docs-xml"
136 }
137
138 ctdb_configure_params = " --enable-developer ${PREFIX}"
139 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
140
141 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
142 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
143 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
144 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
145 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
146 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
147 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
148
149
150 def format_option(name, value=None):
151     """Format option as str list."""
152     if value is None:  # boolean option
153         return [name]
154     if not isinstance(value, list):  # single value option
155         value = [value]
156     # repeatable option
157     return ['{}={}'.format(name, item) for item in value]
158
159
160 def make_test(
161         cmd='make test',
162         FAIL_IMMEDIATELY=1,
163         TESTS='',
164         include_envs=None,
165         exclude_envs=None):
166
167     test_options = []
168     if include_envs:
169         test_options = format_option('--include-env', include_envs)
170     if exclude_envs:
171         test_options = format_option('--exclude-env', exclude_envs)
172     if test_options:
173         # join envs options to original test options
174         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
175
176     _options = []
177     if FAIL_IMMEDIATELY:
178         _options.append('FAIL_IMMEDIATELY=1')
179     if TESTS:
180         _options.append("TESTS='{}'".format(TESTS))
181
182     return ' '.join([cmd] + _options)
183
184
185 # When updating this list, also update .gitlab-ci.yml to add the job
186 # and to make it a dependency of 'page' for the coverage report.
187
188 tasks = {
189     "ctdb": [
190         ("random-sleep", random_sleep(300, 900)),
191         ("configure", "./configure " + ctdb_configure_params),
192         ("make", "make all"),
193         ("install", "make install"),
194         ("test", "make autotest"),
195         ("check-clean-tree", "../script/clean-source-tree.sh"),
196         ("clean", "make clean"),
197         ],
198
199     "docs-xml": [
200         ("random-sleep", random_sleep(300, 900)),
201         ("autoconf", "autoconf"),
202         ("configure", "./configure"),
203         ("make", "make html htmlman"),
204         ("clean", "make clean"),
205         ],
206
207     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
208     "samba": [
209         ("random-sleep", random_sleep(300, 900)),
210         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
211         ("make", "make -j"),
212         ("test", make_test(exclude_envs=[
213             "none",
214             "nt4_dc",
215             "nt4_dc_smb1",
216             "nt4_dc_smb1_done",
217             "nt4_dc_schannel",
218             "nt4_member",
219             "ad_dc",
220             "ad_dc_smb1",
221             "ad_dc_smb1_done",
222             "ad_dc_backup",
223             "ad_dc_ntvfs",
224             "ad_dc_default",
225             "ad_dc_default_smb1",
226             "ad_dc_slowtests",
227             "ad_dc_no_nss",
228             "ad_dc_no_ntlm",
229             "fl2003dc",
230             "fl2008dc",
231             "fl2008r2dc",
232             "ad_member",
233             "ad_member_idmap_rid",
234             "ad_member_idmap_ad",
235             "ad_member_rfc2307",
236             "chgdcpass",
237             "vampire_2000_dc",
238             "fl2000dc",
239             "fileserver",
240             "fileserver_smb1",
241             "fileserver_smb1_done",
242             "maptoguest",
243             "simpleserver",
244             "backupfromdc",
245             "restoredc",
246             "renamedc",
247             "offlinebackupdc",
248             "labdc",
249             "preforkrestartdc",
250             "proclimitdc",
251             "promoted_dc",
252             "vampire_dc",
253             "rodc",
254             "ad_dc_default",
255             "ad_dc_default_smb1",
256             "ad_dc_default_smb1_done",
257             "ad_dc_slowtests",
258             "schema_pair_dc",
259             "schema_dc",
260             "clusteredmember",
261             ])),
262         ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
263         ("lcov", LCOV_CMD),
264         ("install", "make install"),
265         ("check-clean-tree", "script/clean-source-tree.sh"),
266         ("clean", "make clean"),
267         ],
268
269     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
270     "samba-mitkrb5": [
271         ("random-sleep", random_sleep(300, 900)),
272         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
273         ("make", "make -j"),
274         ("test", make_test(exclude_envs=[
275             "none",
276             "nt4_dc",
277             "nt4_dc_smb1",
278             "nt4_dc_smb1_done",
279             "nt4_dc_schannel",
280             "nt4_member",
281             "ad_dc",
282             "ad_dc_smb1",
283             "ad_dc_smb1_done",
284             "ad_dc_backup",
285             "ad_dc_ntvfs",
286             "ad_dc_default",
287             "ad_dc_default_smb1",
288             "ad_dc_default_smb1_done",
289             "ad_dc_slowtests",
290             "ad_dc_no_nss",
291             "ad_dc_no_ntlm",
292             "fl2003dc",
293             "fl2008dc",
294             "fl2008r2dc",
295             "ad_member",
296             "ad_member_idmap_rid",
297             "ad_member_idmap_ad",
298             "ad_member_rfc2307",
299             "chgdcpass",
300             "vampire_2000_dc",
301             "fl2000dc",
302             "fileserver",
303             "fileserver_smb1",
304             "fileserver_smb1_done",
305             "maptoguest",
306             "simpleserver",
307             "backupfromdc",
308             "restoredc",
309             "renamedc",
310             "offlinebackupdc",
311             "labdc",
312             "preforkrestartdc",
313             "proclimitdc",
314             "promoted_dc",
315             "vampire_dc",
316             "rodc",
317             "ad_dc_default",
318             "ad_dc_default_smb1",
319             "ad_dc_default_smb1_done",
320             "ad_dc_slowtests",
321             "schema_pair_dc",
322             "schema_dc",
323             "clusteredmember",
324             ])),
325         ("lcov", LCOV_CMD),
326         ("install", "make install"),
327         ("check-clean-tree", "script/clean-source-tree.sh"),
328         ("clean", "make clean"),
329         ],
330
331     "samba-nt4": [
332         ("random-sleep", random_sleep(300, 900)),
333         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
334         ("make", "make -j"),
335         ("test", make_test(include_envs=[
336             "nt4_dc",
337             "nt4_dc_smb1",
338             "nt4_dc_smb1_done",
339             "nt4_dc_schannel",
340             "nt4_member",
341             "simpleserver",
342             ])),
343         ("lcov", LCOV_CMD),
344         ("check-clean-tree", "script/clean-source-tree.sh"),
345         ("clean", "make clean"),
346         ],
347
348     "samba-fileserver": [
349         ("random-sleep", random_sleep(300, 900)),
350         ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab" + samba_configure_params),
351         ("make", "make -j"),
352         ("test", make_test(include_envs=[
353             "fileserver",
354             "fileserver_smb1",
355             "fileserver_smb1_done",
356             "maptoguest",
357             "ktest", # ktest is also tested in samba and samba-mitkrb5
358                      # but is tested here against a system Heimdal
359             ])),
360         ("lcov", LCOV_CMD),
361         ("check-clean-tree", "script/clean-source-tree.sh"),
362         ],
363
364     "samba-admem": [
365         ("random-sleep", random_sleep(300, 900)),
366         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
367         ("make", "make -j"),
368         ("test", make_test(include_envs=[
369             "ad_member",
370             "ad_member_idmap_rid",
371             "ad_member_idmap_ad",
372             "ad_member_rfc2307",
373             ])),
374         ("lcov", LCOV_CMD),
375         ("check-clean-tree", "script/clean-source-tree.sh"),
376         ],
377
378     "samba-ad-dc-1": [
379         ("random-sleep", random_sleep(1, 1)),
380         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
381         ("make", "make -j"),
382         ("test", make_test(include_envs=[
383             "ad_dc",
384             "ad_dc_smb1",
385             "ad_dc_smb1_done",
386             "ad_dc_no_nss",
387             "ad_dc_no_ntlm",
388             ])),
389         ("lcov", LCOV_CMD),
390         ("check-clean-tree", "script/clean-source-tree.sh"),
391         ],
392
393     "samba-ad-dc-2": [
394         ("random-sleep", random_sleep(1, 1)),
395         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
396         ("make", "make -j"),
397         ("test", make_test(include_envs=[
398             "vampire_dc",
399             "vampire_2000_dc",
400             "rodc",
401             ])),
402         ("lcov", LCOV_CMD),
403         ("check-clean-tree", "script/clean-source-tree.sh"),
404         ],
405
406     "samba-ad-dc-3": [
407         ("random-sleep", random_sleep(1, 1)),
408         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
409         ("make", "make -j"),
410         ("test", make_test(include_envs=[
411             "promoted_dc",
412             "chgdcpass",
413             "preforkrestartdc",
414             "proclimitdc",
415             ])),
416         ("lcov", LCOV_CMD),
417         ("check-clean-tree", "script/clean-source-tree.sh"),
418         ],
419
420     "samba-ad-dc-4": [
421         ("random-sleep", random_sleep(1, 1)),
422         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
423         ("make", "make -j"),
424         ("test", make_test(include_envs=[
425             "fl2000dc",
426             "fl2003dc",
427             "fl2008dc",
428             "fl2008r2dc",
429             ])),
430         ("lcov", LCOV_CMD),
431         ("check-clean-tree", "script/clean-source-tree.sh"),
432         ],
433
434     "samba-ad-dc-5": [
435         ("random-sleep", random_sleep(1, 1)),
436         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
437         ("make", "make -j"),
438         ("test", make_test(include_envs=[
439             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
440         ("lcov", LCOV_CMD),
441         ("check-clean-tree", "script/clean-source-tree.sh"),
442         ],
443
444     "samba-ad-dc-6": [
445         ("random-sleep", random_sleep(1, 1)),
446         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
447         ("make", "make -j"),
448         ("test", make_test(include_envs=["ad_dc_slowtests"])),
449         ("lcov", LCOV_CMD),
450         ("check-clean-tree", "script/clean-source-tree.sh"),
451         ],
452
453     "samba-schemaupgrade": [
454         ("random-sleep", random_sleep(1, 1)),
455         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
456         ("make", "make -j"),
457         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
458         ("lcov", LCOV_CMD),
459         ("check-clean-tree", "script/clean-source-tree.sh"),
460         ],
461
462     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
463     # This is currently the longest task, so we don't randomly delay it.
464     "samba-ad-dc-ntvfs": [
465         ("random-sleep", random_sleep(1, 1)),
466         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
467         ("make", "make -j"),
468         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
469         ("lcov", LCOV_CMD),
470         ("check-clean-tree", "script/clean-source-tree.sh"),
471         ],
472
473     # Test fips compliance
474     "samba-fips": [
475         ("random-sleep", random_sleep(100, 500)),
476         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
477         ("make", "make -j"),
478         ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
479         ("lcov", LCOV_CMD),
480         ("check-clean-tree", "script/clean-source-tree.sh"),
481         ],
482
483     # run the backup/restore testenvs separately as they're fairly standalone
484     # (and CI seems to max out at ~8 different DCs running at once)
485     "samba-ad-dc-backup": [
486         ("random-sleep", random_sleep(300, 900)),
487         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
488         ("make", "make -j"),
489         ("test", make_test(include_envs=[
490             "backupfromdc",
491             "restoredc",
492             "renamedc",
493             "offlinebackupdc",
494             "labdc",
495             "ad_dc_backup",
496             ])),
497         ("lcov", LCOV_CMD),
498         ("check-clean-tree", "script/clean-source-tree.sh"),
499         ],
500
501     "samba-admem-mit": [
502         ("random-sleep", random_sleep(1, 1)),
503         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
504         ("make", "make -j"),
505         ("test", make_test(include_envs=[
506             "ad_member",
507             "ad_member_idmap_rid",
508             "ad_member_idmap_ad",
509             "ad_member_rfc2307",
510             ])),
511         ("lcov", LCOV_CMD),
512         ("check-clean-tree", "script/clean-source-tree.sh"),
513         ],
514
515     "samba-ad-dc-1-mitkrb5": [
516         ("random-sleep", random_sleep(1, 1)),
517         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
518         ("make", "make -j"),
519         ("test", make_test(include_envs=[
520             "ad_dc",
521             "ad_dc_smb1",
522             "ad_dc_smb1_done",
523             "ad_dc_no_nss",
524             "ad_dc_no_ntlm",
525             ])),
526         ("lcov", LCOV_CMD),
527         ("check-clean-tree", "script/clean-source-tree.sh"),
528         ],
529
530     "samba-ad-dc-4-mitkrb5": [
531         ("random-sleep", random_sleep(1, 1)),
532         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
533         ("make", "make -j"),
534         ("test", make_test(include_envs=[
535             "fl2000dc",
536             "fl2003dc",
537             "fl2008dc",
538             "fl2008r2dc",
539             ])),
540         ("lcov", LCOV_CMD),
541         ("check-clean-tree", "script/clean-source-tree.sh"),
542         ],
543
544     "samba-test-only": [
545         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
546         ("make", "make -j"),
547         ("test", make_test(TESTS="${TESTS}")),
548         ("lcov", LCOV_CMD),
549         ],
550
551     # Test cross-compile infrastructure
552     "samba-xc": [
553         ("random-sleep", random_sleep(900, 1500)),
554         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
555         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
556             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
557         ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
558         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
559             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
560         ("compare-results", "script/compare_cc_results.py "
561             "./bin/c4che/default{} "
562             "./bin-xe/c4che/default{} "
563             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
564         ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
565         ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
566             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
567         ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
568             " = \"'1234'\"".format(CACHE_SUFFIX)),
569         ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
570         ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
571             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
572             " ; test $? -ne 0"),
573         ],
574
575     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
576     "samba-o3": [
577         ("random-sleep", random_sleep(300, 900)),
578         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
579         ("make", "make -j"),
580         ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
581         ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
582         ("lcov", LCOV_CMD),
583         ("install", "make install"),
584         ("check-clean-tree", "script/clean-source-tree.sh"),
585         ("clean", "make clean"),
586         ],
587
588     "samba-ctdb": [
589         ("random-sleep", random_sleep(900, 1500)),
590
591         # make sure we have tdb around:
592         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
593         ("tdb-make", "cd lib/tdb && make"),
594         ("tdb-install", "cd lib/tdb && make install"),
595
596         # build samba with cluster support (also building ctdb):
597         ("samba-configure",
598          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
599          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
600          "./configure.developer ${PREFIX} "
601          "--with-selftest-prefix=./bin/ab "
602          "--with-cluster-support "
603          "--without-ad-dc "
604          "--bundled-libraries=!tdb"),
605         ("samba-make", "make"),
606         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
607         ("samba-install", "make install"),
608         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
609
610         ("test",
611          make_test(cmd='make test',
612                    include_envs=["clusteredmember"])
613         ),
614
615         # clean up:
616         ("check-clean-tree", "script/clean-source-tree.sh"),
617         ("clean", "make clean"),
618         ("ctdb-clean", "cd ./ctdb && make clean"),
619         ],
620
621     "samba-libs": [
622         ("random-sleep", random_sleep(300, 900)),
623         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
624         ("talloc-make", "cd lib/talloc && make"),
625         ("talloc-install", "cd lib/talloc && make install"),
626
627         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
628         ("tdb-make", "cd lib/tdb && make"),
629         ("tdb-install", "cd lib/tdb && make install"),
630
631         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
632         ("tevent-make", "cd lib/tevent && make"),
633         ("tevent-install", "cd lib/tevent && make install"),
634
635         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
636         ("ldb-make", "cd lib/ldb && make"),
637         ("ldb-install", "cd lib/ldb && make install"),
638
639         ("nondevel-configure", "./configure ${PREFIX}"),
640         ("nondevel-make", "make -j"),
641         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
642         ("nondevel-install", "make install"),
643         ("nondevel-dist", "make dist"),
644
645         # retry with all modules shared
646         ("allshared-distclean", "make distclean"),
647         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
648         ("allshared-make", "make -j"),
649         ],
650
651     "samba-fuzz": [
652         # build the fuzzers (static) via the oss-fuzz script
653         ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
654         ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl"),
655         ],
656
657     # * Test smbd and smbtorture can build semi-static
658     #
659     # * Test Samba without python still builds.
660     #
661     # When this test fails due to more use of Python, the expectations
662     # is that the newly failing part of the code should be disabled
663     # when --disable-python is set (rather than major work being done
664     # to support this environment).
665     #
666     # The target here is for vendors shipping a minimal smbd.
667     "samba-minimal-smbd": [
668         ("random-sleep", random_sleep(300, 900)),
669
670         # build with all modules static
671         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
672         ("allstatic-make", "make -j"),
673         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
674         ("lcov", LCOV_CMD),
675
676         # retry with nonshared smbd and smbtorture
677         ("nonshared-distclean", "make distclean"),
678         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
679         ("nonshared-make", "make -j"),
680
681         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
682         ("make", "make -j"),
683         ("find-python", "script/find_python.sh ${PREFIX}"),
684         ("test", "make test-nopython"),
685         ("lcov", LCOV_CMD),
686         ("check-clean-tree", "script/clean-source-tree.sh"),
687         ("clean", "make clean"),
688
689         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
690         ("talloc-make", "cd lib/talloc && make"),
691         ("talloc-install", "cd lib/talloc && make install"),
692
693         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
694         ("tdb-make", "cd lib/tdb && make"),
695         ("tdb-install", "cd lib/tdb && make install"),
696
697         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
698         ("tevent-make", "cd lib/tevent && make"),
699         ("tevent-install", "cd lib/tevent && make install"),
700
701         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
702         ("ldb-make", "cd lib/ldb && make"),
703         ("ldb-install", "cd lib/ldb && make install"),
704
705         # retry against installed library packages, but no required modules
706         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
707         ("libs-make", "make -j"),
708         ("libs-install", "make install"),
709         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
710         ("libs-clean", "make clean"),
711
712         ],
713
714     "ldb": [
715         ("random-sleep", random_sleep(60, 600)),
716         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
717         ("make", "make"),
718         ("install", "make install"),
719         ("test", "make test"),
720         ("lcov", LCOV_CMD),
721         ("clean", "make clean"),
722         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
723         ("make-no-lmdb", "make"),
724         ("test-no-lmdb", "make test"),
725         ("lcov-no-lmdb", LCOV_CMD),
726         ("install-no-lmdb", "make install"),
727         ("check-clean-tree", "../../script/clean-source-tree.sh"),
728         ("distcheck", "make distcheck"),
729         ("clean", "make clean"),
730         ],
731
732     "tdb": [
733         ("random-sleep", random_sleep(60, 600)),
734         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
735         ("make", "make"),
736         ("install", "make install"),
737         ("test", "make test"),
738         ("lcov", LCOV_CMD),
739         ("check-clean-tree", "../../script/clean-source-tree.sh"),
740         ("distcheck", "make distcheck"),
741         ("clean", "make clean"),
742         ],
743
744     "talloc": [
745         ("random-sleep", random_sleep(60, 600)),
746         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
747         ("make", "make"),
748         ("install", "make install"),
749         ("test", "make test"),
750         ("lcov", LCOV_CMD),
751         ("check-clean-tree", "../../script/clean-source-tree.sh"),
752         ("distcheck", "make distcheck"),
753         ("clean", "make clean"),
754         ],
755
756     "replace": [
757         ("random-sleep", random_sleep(60, 600)),
758         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
759         ("make", "make"),
760         ("install", "make install"),
761         ("test", "make test"),
762         ("lcov", LCOV_CMD),
763         ("check-clean-tree", "../../script/clean-source-tree.sh"),
764         ("distcheck", "make distcheck"),
765         ("clean", "make clean"),
766         ],
767
768     "tevent": [
769         ("random-sleep", random_sleep(60, 600)),
770         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
771         ("make", "make"),
772         ("install", "make install"),
773         ("test", "make test"),
774         ("lcov", LCOV_CMD),
775         ("check-clean-tree", "../../script/clean-source-tree.sh"),
776         ("distcheck", "make distcheck"),
777         ("clean", "make clean"),
778         ],
779
780     "pidl": [
781         ("random-sleep", random_sleep(60, 600)),
782         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
783         ("touch", "touch *.yp"),
784         ("make", "make"),
785         ("test", "make test"),
786         ("install", "make install"),
787         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
788         ("check-clean-tree", "../script/clean-source-tree.sh"),
789         ("clean", "make clean"),
790         ],
791
792     # these are useful for debugging autobuild
793     'pass': [("pass", 'echo passing && /bin/true')],
794     'fail': [("fail", 'echo failing && /bin/false')],
795 }
796
797 defaulttasks = list(tasks.keys())
798
799 defaulttasks.remove("pass")
800 defaulttasks.remove("fail")
801 defaulttasks.remove("samba-test-only")
802 defaulttasks.remove("samba-fuzz")
803 defaulttasks.remove("samba-fips")
804 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
805     defaulttasks.remove("samba-o3")
806
807
808 def do_print(msg):
809     print("%s" % msg)
810     sys.stdout.flush()
811     sys.stderr.flush()
812
813
814 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
815     if show is None:
816         show = options.verbose
817     if show:
818         do_print("Running: '%s' in '%s'" % (cmd, dir))
819     if output:
820         out = check_output([cmd], shell=True, cwd=dir)
821         return out.decode(encoding='utf-8', errors='backslashreplace')
822     elif checkfail:
823         return check_call(cmd, shell=True, cwd=dir)
824     else:
825         return call(cmd, shell=True, cwd=dir)
826
827
828 class builder(object):
829     '''handle build of one directory'''
830
831     def __init__(self, name, sequence, cp=True):
832         self.name = name
833         self.dir = builddirs.get(name, '.')
834         self.tag = self.name.replace('/', '_')
835         self.sequence = sequence
836         self.next = 0
837         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
838         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
839         if options.verbose:
840             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
841             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
842         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
843         self.stdout = open(self.stdout_path, 'w')
844         self.stderr = open(self.stderr_path, 'w')
845         self.stdin  = open("/dev/null", 'r')
846         self.test_source_dir = "%s/%s" % (testbase, self.tag)
847         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
848         self.prefix = "%s/%s" % (test_prefix, self.tag)
849         run_cmd("rm -rf %s" % self.test_source_dir)
850         run_cmd("rm -rf %s" % self.prefix)
851         if cp:
852             run_cmd("cp -R -a -l %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
853         else:
854             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
855         self.start_next()
856
857     def start_next(self):
858         if self.next == len(self.sequence):
859             if not options.nocleanup:
860                 run_cmd("rm -rf %s" % self.test_source_dir)
861                 run_cmd("rm -rf %s" % self.prefix)
862             do_print('%s: Completed OK' % self.name)
863             self.done = True
864             return
865         (self.stage, self.cmd) = self.sequence[self.next]
866         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
867         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
868         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
869         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
870         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
871         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
872         self.cmd = self.cmd.replace("${NAME}", self.name)
873         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
874         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
875         self.proc = Popen(self.cmd, shell=True,
876                           close_fds=True, cwd=self.cwd,
877                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
878         self.next += 1
879
880
881 class buildlist(object):
882     '''handle build of multiple directories'''
883
884     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
885         self.tail_proc = None
886         self.retry = None
887         if not tasknames:
888             if options.restrict_tests:
889                 tasknames = ["samba-test-only"]
890             else:
891                 tasknames = defaulttasks
892
893         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
894
895         if options.retry:
896             rebase_remote = "rebaseon"
897             retry_task = [("retry",
898                             '''set -e
899                             git remote add -t %s %s %s
900                             git fetch %s
901                             while :; do
902                               sleep 60
903                               git describe %s/%s > old_remote_branch.desc
904                               git fetch %s
905                               git describe %s/%s > remote_branch.desc
906                               diff old_remote_branch.desc remote_branch.desc
907                             done
908                            ''' % (
909                                rebase_branch, rebase_remote, rebase_url,
910                                rebase_remote,
911                                rebase_remote, rebase_branch,
912                                rebase_remote,
913                                rebase_remote, rebase_branch
914                             ))]
915
916             self.retry = builder('retry', retry_task, cp=False)
917             self.need_retry = False
918
919     def kill_kids(self):
920         if self.tail_proc is not None:
921             self.tail_proc.terminate()
922             self.tail_proc.wait()
923             self.tail_proc = None
924         if self.retry is not None:
925             self.retry.proc.terminate()
926             self.retry.proc.wait()
927             self.retry = None
928         for b in self.tlist:
929             if b.proc is not None:
930                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
931                 b.proc.terminate()
932                 b.proc.wait()
933                 b.proc = None
934
935     def wait_one(self):
936         while True:
937             none_running = True
938             for b in self.tlist:
939                 if b.proc is None:
940                     continue
941                 none_running = False
942                 b.status = b.proc.poll()
943                 if b.status is None:
944                     continue
945                 b.proc = None
946                 return b
947             if options.retry:
948                 ret = self.retry.proc.poll()
949                 if ret is not None:
950                     self.need_retry = True
951                     self.retry = None
952                     return None
953             if none_running:
954                 return None
955             time.sleep(0.1)
956
957     def run(self):
958         while True:
959             b = self.wait_one()
960             if options.retry and self.need_retry:
961                 self.kill_kids()
962                 do_print("retry needed")
963                 return (0, None, None, None, "retry")
964             if b is None:
965                 break
966             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
967                 self.kill_kids()
968                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
969             b.start_next()
970         self.kill_kids()
971         return (0, None, None, None, "All OK")
972
973     def write_system_info(self, filename):
974         with open(filename, 'w') as f:
975             for cmd in ['uname -a',
976                         'lsb_release -a',
977                         'free',
978                         'mount',
979                         'cat /proc/cpuinfo',
980                         'cc --version',
981                         'df -m .',
982                         'df -m %s' % testbase]:
983                 try:
984                     out = run_cmd(cmd, output=True, checkfail=False)
985                 except subprocess.CalledProcessError as e:
986                     out = "<failed: %s>" % str(e)
987                 print('### %s' % cmd, file=f)
988                 print(out, file=f)
989                 print(file=f)
990
991     def tarlogs(self, fname):
992         with tarfile.open(fname, "w:gz") as tar:
993             for b in self.tlist:
994                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
995                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
996             if os.path.exists("autobuild.log"):
997                 tar.add("autobuild.log")
998             filename = 'system-info.txt'
999             self.write_system_info(filename)
1000             tar.add(filename)
1001
1002     def remove_logs(self):
1003         for b in self.tlist:
1004             os.unlink(b.stdout_path)
1005             os.unlink(b.stderr_path)
1006
1007     def start_tail(self):
1008         cmd = ["tail", "-f"]
1009         for b in self.tlist:
1010             cmd.append(b.stdout_path)
1011             cmd.append(b.stderr_path)
1012         self.tail_proc = Popen(cmd, close_fds=True)
1013
1014
1015 def cleanup():
1016     if options.nocleanup:
1017         return
1018     run_cmd("stat %s || true" % test_tmpdir, show=True)
1019     run_cmd("stat %s" % testbase, show=True)
1020     do_print("Cleaning up %r" % cleanup_list)
1021     for d in cleanup_list:
1022         run_cmd("rm -rf %s" % d)
1023
1024
1025 def daemonize(logfile):
1026     pid = os.fork()
1027     if pid == 0:  # Parent
1028         os.setsid()
1029         pid = os.fork()
1030         if pid != 0:  # Actual daemon
1031             os._exit(0)
1032     else:  # Grandparent
1033         os._exit(0)
1034
1035     import resource      # Resource usage information.
1036     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1037     if maxfd == resource.RLIM_INFINITY:
1038         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1039     for fd in range(0, maxfd):
1040         try:
1041             os.close(fd)
1042         except OSError:
1043             pass
1044     os.open(logfile, os.O_RDWR | os.O_CREAT)
1045     os.dup2(0, 1)
1046     os.dup2(0, 2)
1047
1048
1049 def write_pidfile(fname):
1050     '''write a pid file, cleanup on exit'''
1051     with open(fname, mode='w') as f:
1052         f.write("%u\n" % os.getpid())
1053
1054
1055 def rebase_tree(rebase_url, rebase_branch="master"):
1056     rebase_remote = "rebaseon"
1057     do_print("Rebasing on %s" % rebase_url)
1058     run_cmd("git describe HEAD", show=True, dir=test_master)
1059     run_cmd("git remote add -t %s %s %s" %
1060             (rebase_branch, rebase_remote, rebase_url),
1061             show=True, dir=test_master)
1062     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1063     if options.fix_whitespace:
1064         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1065                 (rebase_remote, rebase_branch),
1066                 show=True, dir=test_master)
1067     else:
1068         run_cmd("git rebase --force-rebase %s/%s" %
1069                 (rebase_remote, rebase_branch),
1070                 show=True, dir=test_master)
1071     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1072                    (rebase_remote, rebase_branch),
1073                    dir=test_master, output=True)
1074     if diff == '':
1075         do_print("No differences between HEAD and %s/%s - exiting" %
1076                  (rebase_remote, rebase_branch))
1077         sys.exit(0)
1078     run_cmd("git describe %s/%s" %
1079             (rebase_remote, rebase_branch),
1080             show=True, dir=test_master)
1081     run_cmd("git describe HEAD", show=True, dir=test_master)
1082     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1083             (rebase_remote, rebase_branch),
1084             show=True, dir=test_master)
1085
1086
1087 def push_to(push_url, push_branch="master"):
1088     push_remote = "pushto"
1089     do_print("Pushing to %s" % push_url)
1090     if options.mark:
1091         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1092         run_cmd("git commit --amend -c HEAD", dir=test_master)
1093         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1094         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1095     run_cmd("git remote add -t %s %s %s" %
1096             (push_branch, push_remote, push_url),
1097             show=True, dir=test_master)
1098     run_cmd("git push %s +HEAD:%s" %
1099             (push_remote, push_branch),
1100             show=True, dir=test_master)
1101
1102
1103 def send_email(subject, text, log_tar):
1104     if options.email is None:
1105         do_print("not sending email because the recipient is not set")
1106         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1107                  (subject, text))
1108         return
1109     outer = MIMEMultipart()
1110     outer['Subject'] = subject
1111     outer['To'] = options.email
1112     outer['From'] = options.email_from
1113     outer['Date'] = email.utils.formatdate(localtime=True)
1114     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1115     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1116     if options.attach_logs:
1117         with open(log_tar, 'rb') as fp:
1118             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1119         # Set the filename parameter
1120         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1121         outer.attach(msg)
1122     content = outer.as_string()
1123     s = smtplib.SMTP(options.email_server)
1124     email_user = os.getenv('SMTP_USERNAME')
1125     email_password = os.getenv('SMTP_PASSWORD')
1126     if email_user is not None:
1127         s.starttls()
1128         s.login(email_user, email_password)
1129
1130     s.sendmail(options.email_from, [options.email], content)
1131     s.set_debuglevel(1)
1132     s.quit()
1133
1134
1135 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1136                   elapsed_time, log_base=None, add_log_tail=True):
1137     '''send an email to options.email about the failure'''
1138     elapsed_minutes = elapsed_time / 60.0
1139     if log_base is None:
1140         log_base = gitroot
1141     text = '''
1142 Dear Developer,
1143
1144 Your autobuild on %s failed after %.1f minutes
1145 when trying to test %s with the following error:
1146
1147    %s
1148
1149 the autobuild has been abandoned. Please fix the error and resubmit.
1150
1151 A summary of the autobuild process is here:
1152
1153   %s/autobuild.log
1154 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1155
1156     if options.restrict_tests:
1157         text += """
1158 The build was restricted to tests matching %s\n""" % options.restrict_tests
1159
1160     if failed_task != 'rebase':
1161         text += '''
1162 You can see logs of the failed task here:
1163
1164   %s/%s.stdout
1165   %s/%s.stderr
1166
1167 or you can get full logs of all tasks in this job here:
1168
1169   %s/logs.tar.gz
1170
1171 The top commit for the tree that was built was:
1172
1173 %s
1174
1175 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1176
1177     if add_log_tail:
1178         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1179         lines = f.readlines()
1180         log_tail = "".join(lines[-50:])
1181         num_lines = len(lines)
1182         if num_lines < 50:
1183             # Also include stderr (compile failures) if < 50 lines of stdout
1184             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1185             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1186
1187         text += '''
1188 The last 50 lines of log messages:
1189
1190 %s
1191     ''' % log_tail
1192         f.close()
1193
1194     logs = os.path.join(gitroot, 'logs.tar.gz')
1195     send_email('autobuild[%s] failure on %s for task %s during %s'
1196                % (options.branch, platform.node(), failed_task, failed_stage),
1197                text, logs)
1198
1199
1200 def email_success(elapsed_time, log_base=None):
1201     '''send an email to options.email about a successful build'''
1202     if log_base is None:
1203         log_base = gitroot
1204     text = '''
1205 Dear Developer,
1206
1207 Your autobuild on %s has succeeded after %.1f minutes.
1208
1209 ''' % (platform.node(), elapsed_time / 60.)
1210
1211     if options.restrict_tests:
1212         text += """
1213 The build was restricted to tests matching %s\n""" % options.restrict_tests
1214
1215     if options.keeplogs:
1216         text += '''
1217
1218 you can get full logs of all tasks in this job here:
1219
1220   %s/logs.tar.gz
1221
1222 ''' % log_base
1223
1224     text += '''
1225 The top commit for the tree that was built was:
1226
1227 %s
1228 ''' % top_commit_msg
1229
1230     logs = os.path.join(gitroot, 'logs.tar.gz')
1231     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1232                text, logs)
1233
1234
1235 # get the top commit message, for emails
1236 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1237
1238 try:
1239     os.makedirs(testbase)
1240 except Exception as reason:
1241     raise Exception("Unable to create %s : %s" % (testbase, reason))
1242 cleanup_list.append(testbase)
1243
1244 if options.daemon:
1245     logfile = os.path.join(testbase, "log")
1246     do_print("Forking into the background, writing progress to %s" % logfile)
1247     daemonize(logfile)
1248
1249 write_pidfile(gitroot + "/autobuild.pid")
1250
1251 start_time = time.time()
1252
1253 while True:
1254     try:
1255         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1256         os.makedirs(test_tmpdir)
1257         # The waf uninstall code removes empty directories all the way
1258         # up the tree.  Creating a file in test_tmpdir stops it from
1259         # being removed.
1260         run_cmd("touch %s" % os.path.join(test_tmpdir,
1261                                           ".directory-is-not-empty"), show=True)
1262         run_cmd("stat %s" % test_tmpdir, show=True)
1263         run_cmd("stat %s" % testbase, show=True)
1264         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1265     except Exception:
1266         cleanup()
1267         raise
1268
1269     try:
1270         if options.rebase is not None:
1271             rebase_tree(options.rebase, rebase_branch=options.branch)
1272     except Exception:
1273         cleanup_list.append(gitroot + "/autobuild.pid")
1274         cleanup()
1275         elapsed_time = time.time() - start_time
1276         email_failure(-1, 'rebase', 'rebase', 'rebase',
1277                       'rebase on %s failed' % options.branch,
1278                       elapsed_time, log_base=options.log_base)
1279         sys.exit(1)
1280
1281     try:
1282         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1283         if options.tail:
1284             blist.start_tail()
1285         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1286         if status != 0 or errstr != "retry":
1287             break
1288         cleanup()
1289     except Exception:
1290         cleanup()
1291         raise
1292
1293 cleanup_list.append(gitroot + "/autobuild.pid")
1294
1295 do_print(errstr)
1296
1297 blist.kill_kids()
1298 if options.tail:
1299     do_print("waiting for tail to flush")
1300     time.sleep(1)
1301
1302 elapsed_time = time.time() - start_time
1303 if status == 0:
1304     if options.passcmd is not None:
1305         do_print("Running passcmd: %s" % options.passcmd)
1306         run_cmd(options.passcmd, dir=test_master)
1307     if options.pushto is not None:
1308         push_to(options.pushto, push_branch=options.branch)
1309     if options.keeplogs or options.attach_logs:
1310         blist.tarlogs("logs.tar.gz")
1311         do_print("Logs in logs.tar.gz")
1312     if options.always_email:
1313         email_success(elapsed_time, log_base=options.log_base)
1314     blist.remove_logs()
1315     cleanup()
1316     do_print(errstr)
1317     sys.exit(0)
1318
1319 # something failed, gather a tar of the logs
1320 blist.tarlogs("logs.tar.gz")
1321
1322 if options.email is not None:
1323     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1324                   elapsed_time, log_base=options.log_base)
1325 else:
1326     elapsed_minutes = elapsed_time / 60.0
1327     print('''
1328
1329 ####################################################################
1330
1331 AUTOBUILD FAILURE
1332
1333 Your autobuild[%s] on %s failed after %.1f minutes
1334 when trying to test %s with the following error:
1335
1336    %s
1337
1338 the autobuild has been abandoned. Please fix the error and resubmit.
1339
1340 ####################################################################
1341
1342 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1343
1344 cleanup()
1345 do_print(errstr)
1346 do_print("Logs in logs.tar.gz")
1347 sys.exit(status)