ldb: Free memory when repacking database
[garming/samba-autobuild/.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, check_output, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 import random
13 from optparse import OptionParser
14 import smtplib
15 import email
16 from email.mime.text import MIMEText
17 from email.mime.base import MIMEBase
18 from email.mime.application import MIMEApplication
19 from email.mime.multipart import MIMEMultipart
20 from distutils.sysconfig import get_python_lib
21 import platform
22
23 try:
24     from waflib.Build import CACHE_SUFFIX
25 except ImportError:
26     sys.path.insert(0, "./third_party/waf")
27     from waflib.Build import CACHE_SUFFIX
28
29
30 os.environ["PYTHONUNBUFFERED"] = "1"
31
32 # This speeds up testing remarkably.
33 os.environ['TDB_NO_FSYNC'] = '1'
34
35
36 def find_git_root():
37     '''get to the top of the git repo'''
38     p = os.getcwd()
39     while p != '/':
40         if os.path.isdir(os.path.join(p, ".git")):
41             return p
42         p = os.path.abspath(os.path.join(p, '..'))
43     return None
44
45
46 gitroot = find_git_root()
47 if gitroot is None:
48     raise Exception("Failed to find git root")
49
50
51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
52
53 parser = OptionParser()
54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
58                   default=def_testbase)
59 parser.add_option("--passcmd", help="command to run on success", default=None)
60 parser.add_option("--verbose", help="show all commands as they are run",
61                   default=False, action="store_true")
62 parser.add_option("--rebase", help="rebase on the given tree before testing",
63                   default=None, type='str')
64 parser.add_option("--pushto", help="push to a git url on success",
65                   default=None, type='str')
66 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
67                   default=False, action="store_true")
68 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
69                   default=False, action="store_true")
70 parser.add_option("--retry", help="automatically retry if master changes",
71                   default=False, action="store_true")
72 parser.add_option("--email", help="send email to the given address on failure",
73                   type='str', default=None)
74 parser.add_option("--email-from", help="send email from the given address",
75                   type='str', default="autobuild@samba.org")
76 parser.add_option("--email-server", help="send email via the given server",
77                   type='str', default='localhost')
78 parser.add_option("--always-email", help="always send email, even on success",
79                   action="store_true")
80 parser.add_option("--daemon", help="daemonize after initial setup",
81                   action="store_true")
82 parser.add_option("--branch", help="the branch to work on (default=master)",
83                   default="master", type='str')
84 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
85                   default=gitroot, type='str')
86 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
87                   default=False, action="store_true")
88 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
89                   default='')
90 parser.add_option("--enable-coverage", dest='enable_coverage',
91                   action="store_const", const='--enable-coverage', default='',
92                   help="Add --enable-coverage option while configure")
93
94 (options, args) = parser.parse_args()
95
96 if options.retry:
97     if options.rebase is None:
98         raise Exception('You can only use --retry if you also rebase')
99
100 testbase = "%s/b%u" % (options.testbase, os.getpid())
101 test_master = "%s/master" % testbase
102 test_prefix = "%s/prefix" % testbase
103 test_tmpdir = "%s/tmp" % testbase
104 os.environ['TMPDIR'] = test_tmpdir
105
106 if options.enable_coverage:
107     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
108 else:
109     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
110
111 if args:
112     # If we are only running specific test,
113     # do not sleep randomly to wait for it to start
114     def random_sleep(low, high):
115         return 'sleep 1'
116 else:
117     def random_sleep(low, high):
118         return 'sleep {}'.format(random.randint(low, high))
119
120 cleanup_list = []
121
122 builddirs = {
123     "ctdb": "ctdb",
124     "samba": ".",
125     "samba-nt4": ".",
126     "samba-fileserver": ".",
127     "samba-ad-member": ".",
128     "samba-xc": ".",
129     "samba-o3": ".",
130     "samba-ctdb": ".",
131     "samba-libs": ".",
132     "samba-static": ".",
133     "samba-none-env": ".",
134     "samba-ad-dc-1": ".",
135     "samba-ad-dc-2": ".",
136     "samba-ad-dc-3": ".",
137     "samba-ad-dc-4": ".",
138     "samba-ad-dc-5": ".",
139     "samba-ad-dc-6": ".",
140     "samba-ad-dc-ntvfs": ".",
141     "samba-ad-dc-backup": ".",
142     "samba-systemkrb5": ".",
143     "samba-nopython": ".",
144     "samba-nopython-py2": ".",
145     "samba-schemaupgrade": ".",
146     "ldb": "lib/ldb",
147     "tdb": "lib/tdb",
148     "talloc": "lib/talloc",
149     "replace": "lib/replace",
150     "tevent": "lib/tevent",
151     "pidl": "pidl"
152 }
153
154 defaulttasks = list(builddirs.keys())
155
156 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
157     defaulttasks.remove("samba-o3")
158
159 ctdb_configure_params = " --enable-developer ${PREFIX}"
160 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
161
162 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
163 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
164 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
165 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
166 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
167 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
168 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
169
170
171 def format_option(name, value=None):
172     """Format option as str list."""
173     if value is None:  # boolean option
174         return [name]
175     if not isinstance(value, list):  # single value option
176         value = [value]
177     # repeatable option
178     return ['{}={}'.format(name, item) for item in value]
179
180
181 def make_test(
182         cmd='make test',
183         FAIL_IMMEDIATELY=1,
184         TESTS='',
185         include_envs=None,
186         exclude_envs=None):
187
188     test_options = []
189     if include_envs:
190         test_options = format_option('--include-env', include_envs)
191     if exclude_envs:
192         test_options = format_option('--exclude-env', exclude_envs)
193     if test_options:
194         # join envs options to original test options
195         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
196
197     _options = []
198     if FAIL_IMMEDIATELY:
199         _options.append('FAIL_IMMEDIATELY=1')
200     if TESTS:
201         _options.append("TESTS='{}'".format(TESTS))
202
203     return ' '.join([cmd] + _options)
204
205
206 tasks = {
207     "ctdb": [
208         ("random-sleep", random_sleep(300, 900)),
209         ("configure", "./configure " + ctdb_configure_params),
210         ("make", "make all"),
211         ("install", "make install"),
212         ("test", "make autotest"),
213         ("check-clean-tree", "../script/clean-source-tree.sh"),
214         ("clean", "make clean"),
215         ],
216
217     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
218     "samba": [
219         ("random-sleep", random_sleep(300, 900)),
220         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
221         ("make", "make -j"),
222         ("test", make_test(exclude_envs=[
223             "none",
224             "nt4_dc",
225             "nt4_dc_schannel",
226             "nt4_member",
227             "ad_dc",
228             "ad_dc_backup",
229             "ad_dc_ntvfs",
230             "ad_dc_default",
231             "ad_dc_slowtests",
232             "ad_dc_no_nss",
233             "ad_dc_no_ntlm",
234             "fl2003dc",
235             "fl2008dc",
236             "fl2008r2dc",
237             "ad_member",
238             "ad_member_idmap_rid",
239             "ad_member_idmap_ad",
240             "ad_member_rfc2307",
241             "chgdcpass",
242             "vampire_2000_dc",
243             "fl2000dc",
244             "fileserver",
245             "maptoguest",
246             "simpleserver",
247             "backupfromdc",
248             "restoredc",
249             "renamedc",
250             "offlinebackupdc",
251             "labdc",
252             "preforkrestartdc",
253             "proclimitdc",
254             "promoted_dc",
255             "vampire_dc",
256             "rodc",
257             "ad_dc_default",
258             "ad_dc_slowtests",
259             "schema_pair_dc",
260             "schema_dc",
261             ])),
262         ("lcov", LCOV_CMD),
263         ("install", "make install"),
264         ("check-clean-tree", "script/clean-source-tree.sh"),
265         ("clean", "make clean"),
266         ],
267
268     "samba-nt4": [
269         ("random-sleep", random_sleep(300, 900)),
270         ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params),
271         ("make", "make -j"),
272         ("test", make_test(include_envs=[
273             "nt4_dc",
274             "nt4_dc_schannel",
275             "nt4_member",
276             ])),
277         ("lcov", LCOV_CMD),
278         ("install", "make install"),
279         ("check-clean-tree", "script/clean-source-tree.sh"),
280         ("clean", "make clean"),
281         ],
282
283     "samba-fileserver": [
284         ("random-sleep", random_sleep(300, 900)),
285         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
286         ("make", "make -j"),
287         ("test", make_test(include_envs=[
288             "fileserver",
289             "maptoguest",
290             "simpleserver",
291             ])),
292         ("lcov", LCOV_CMD),
293         ("check-clean-tree", "script/clean-source-tree.sh"),
294         ],
295
296     "samba-ad-member": [
297         ("random-sleep", random_sleep(300, 900)),
298         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
299         ("make", "make -j"),
300         ("test", make_test(include_envs=[
301             "ad_member",
302             "ad_member_idmap_rid",
303             "ad_member_idmap_ad",
304             "ad_member_rfc2307",
305             ])),
306         ("lcov", LCOV_CMD),
307         ("check-clean-tree", "script/clean-source-tree.sh"),
308         ],
309
310     "samba-ad-dc-1": [
311         ("random-sleep", random_sleep(1, 1)),
312         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
313         ("make", "make -j"),
314         ("test", make_test(include_envs=[
315             "ad_dc",
316             "ad_dc_no_nss",
317             "ad_dc_no_ntlm",
318             ])),
319         ("lcov", LCOV_CMD),
320         ("check-clean-tree", "script/clean-source-tree.sh"),
321         ],
322
323     "samba-ad-dc-2": [
324         ("random-sleep", random_sleep(1, 1)),
325         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
326         ("make", "make -j"),
327         ("test", make_test(include_envs=[
328             "vampire_dc",
329             "vampire_2000_dc",
330             "rodc",
331             ])),
332         ("lcov", LCOV_CMD),
333         ("check-clean-tree", "script/clean-source-tree.sh"),
334         ],
335
336     "samba-ad-dc-3": [
337         ("random-sleep", random_sleep(1, 1)),
338         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
339         ("make", "make -j"),
340         ("test", make_test(include_envs=[
341             "promoted_dc",
342             "chgdcpass",
343             "preforkrestartdc",
344             "proclimitdc",
345             ])),
346         ("lcov", LCOV_CMD),
347         ("check-clean-tree", "script/clean-source-tree.sh"),
348         ],
349
350     "samba-ad-dc-4": [
351         ("random-sleep", random_sleep(1, 1)),
352         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
353         ("make", "make -j"),
354         ("test", make_test(include_envs=[
355             "fl2000dc",
356             "fl2003dc",
357             "fl2008dc",
358             "fl2008r2dc",
359             ])),
360         ("lcov", LCOV_CMD),
361         ("check-clean-tree", "script/clean-source-tree.sh"),
362         ],
363
364     "samba-ad-dc-5": [
365         ("random-sleep", random_sleep(1, 1)),
366         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
367         ("make", "make -j"),
368         ("test", make_test(include_envs=["ad_dc_default"])),
369         ("lcov", LCOV_CMD),
370         ("check-clean-tree", "script/clean-source-tree.sh"),
371         ],
372
373     "samba-ad-dc-6": [
374         ("random-sleep", random_sleep(1, 1)),
375         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
376         ("make", "make -j"),
377         ("test", make_test(include_envs=["ad_dc_slowtests"])),
378         ("lcov", LCOV_CMD),
379         ("check-clean-tree", "script/clean-source-tree.sh"),
380         ],
381
382     "samba-schemaupgrade": [
383         ("random-sleep", random_sleep(1, 1)),
384         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
385         ("make", "make -j"),
386         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
387         ("lcov", LCOV_CMD),
388         ("check-clean-tree", "script/clean-source-tree.sh"),
389         ],
390
391     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
392     # This is currently the longest task, so we don't randomly delay it.
393     "samba-ad-dc-ntvfs": [
394         ("random-sleep", random_sleep(1, 1)),
395         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
396         ("make", "make -j"),
397         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
398         ("lcov", LCOV_CMD),
399         ("check-clean-tree", "script/clean-source-tree.sh"),
400         ],
401
402     # run the backup/restore testenvs separately as they're fairly standalone
403     # (and CI seems to max out at ~8 different DCs running at once)
404     "samba-ad-dc-backup": [
405         ("random-sleep", random_sleep(300, 900)),
406         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
407         ("make", "make -j"),
408         ("test", make_test(include_envs=[
409             "backupfromdc",
410             "restoredc",
411             "renamedc",
412             "offlinebackupdc",
413             "labdc",
414             "ad_dc_backup",
415             ])),
416         ("lcov", LCOV_CMD),
417         ("check-clean-tree", "script/clean-source-tree.sh"),
418         ],
419
420     "samba-test-only": [
421         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
422         ("make", "make -j"),
423         ("test", make_test(TESTS="${TESTS}")),
424         ("lcov", LCOV_CMD),
425         ],
426
427     # Test cross-compile infrastructure
428     "samba-xc": [
429         ("random-sleep", random_sleep(900, 1500)),
430         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
431         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
432             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
433         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
434             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
435         ("compare-results", "script/compare_cc_results.py "
436             "./bin/c4che/default{} "
437             "./bin-xe/c4che/default{} "
438             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
439         ],
440
441     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
442     "samba-o3": [
443         ("random-sleep", random_sleep(300, 900)),
444         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
445         ("make", "make -j"),
446         ("test", make_test(cmd='make quicktest', include_envs=["ad_dc"])),
447         ("lcov", LCOV_CMD),
448         ("install", "make install"),
449         ("check-clean-tree", "script/clean-source-tree.sh"),
450         ("clean", "make clean"),
451         ],
452
453     "samba-ctdb": [
454         ("random-sleep", random_sleep(900, 1500)),
455
456         # make sure we have tdb around:
457         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
458         ("tdb-make", "cd lib/tdb && make"),
459         ("tdb-install", "cd lib/tdb && make install"),
460
461         # build samba with cluster support (also building ctdb):
462         ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb"),
463         ("samba-make", "make"),
464         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
465         ("samba-install", "make install"),
466         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
467
468         # clean up:
469         ("check-clean-tree", "script/clean-source-tree.sh"),
470         ("clean", "make clean"),
471         ("ctdb-clean", "cd ./ctdb && make clean"),
472         ],
473
474     "samba-libs": [
475         ("random-sleep", random_sleep(300, 900)),
476         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
477         ("talloc-make", "cd lib/talloc && make"),
478         ("talloc-install", "cd lib/talloc && make install"),
479
480         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
481         ("tdb-make", "cd lib/tdb && make"),
482         ("tdb-install", "cd lib/tdb && make install"),
483
484         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
485         ("tevent-make", "cd lib/tevent && make"),
486         ("tevent-install", "cd lib/tevent && make install"),
487
488         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
489         ("ldb-make", "cd lib/ldb && make"),
490         ("ldb-install", "cd lib/ldb && make install"),
491
492         ("nondevel-configure", "./configure ${PREFIX}"),
493         ("nondevel-make", "make -j"),
494         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
495         ("nondevel-install", "make install"),
496         ("nondevel-dist", "make dist"),
497
498         # retry with all modules shared
499         ("allshared-distclean", "make distclean"),
500         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
501         ("allshared-make", "make -j"),
502         ],
503
504     "samba-none-env": [
505         ("random-sleep", random_sleep(1, 1)),
506         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
507         ("make", "make -j"),
508         ("test", make_test(include_envs=["none"])),
509         ("lcov", LCOV_CMD),
510         ],
511
512     "samba-static": [
513         ("random-sleep", random_sleep(1, 1)),
514         # build with all modules static
515         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
516         ("allstatic-make", "make -j"),
517         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
518         ("lcov", LCOV_CMD),
519
520         # retry without any required modules
521         ("none-distclean", "make distclean"),
522         ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
523         ("none-make", "make -j"),
524
525         # retry with nonshared smbd and smbtorture
526         ("nonshared-distclean", "make distclean"),
527         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
528         ("nonshared-make", "make -j"),
529         ],
530
531     "samba-systemkrb5": [
532         ("random-sleep", random_sleep(900, 1500)),
533         ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --with-experimental-mit-ad-dc"),
534         ("make", "make -j"),
535         # we currently cannot run a full make test, a limited list of tests could be run
536         # via "make test TESTS=sometests"
537         ("test", make_test(include_envs=["ktest"])),
538         ("lcov", LCOV_CMD),
539         ("install", "make install"),
540         ("check-clean-tree", "script/clean-source-tree.sh"),
541         ("clean", "make clean"),
542         ],
543
544     # Test Samba without python still builds.  When this test fails
545     # due to more use of Python, the expectations is that the newly
546     # failing part of the code should be disabled when
547     # --disable-python is set (rather than major work being done to
548     # support this environment).  The target here is for vendors
549     # shipping a minimal smbd.
550     "samba-nopython": [
551         ("random-sleep", random_sleep(300, 900)),
552         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
553         ("make", "make -j"),
554         ("install", "make install"),
555         ("find-python", "script/find_python.sh ${PREFIX}"),
556         ("test", "make test-nopython"),
557         ("lcov", LCOV_CMD),
558         ("check-clean-tree", "script/clean-source-tree.sh"),
559         ("clean", "make clean"),
560
561         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
562         ("talloc-make", "cd lib/talloc && make"),
563         ("talloc-install", "cd lib/talloc && make install"),
564
565         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
566         ("tdb-make", "cd lib/tdb && make"),
567         ("tdb-install", "cd lib/tdb && make install"),
568
569         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
570         ("tevent-make", "cd lib/tevent && make"),
571         ("tevent-install", "cd lib/tevent && make install"),
572
573         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
574         ("ldb-make", "cd lib/ldb && make"),
575         ("ldb-install", "cd lib/ldb && make install"),
576
577         # retry against installed library packages
578         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
579         ("libs-make", "make -j"),
580         ("libs-install", "make install"),
581         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
582         ("libs-clean", "make clean"),
583         ],
584
585     # check we can do the same thing using python2
586     "samba-nopython-py2": [
587         ("random-sleep", random_sleep(300, 900)),
588         ("configure", "PYTHON=python2 ./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
589         ("make", "PYTHON=python2 make -j"),
590         ("install", "PYTHON=python2 make install"),
591         ("find-python", "script/find_python.sh ${PREFIX}"),
592         ("test", "make test-nopython"),
593         ("lcov", LCOV_CMD),
594         ("check-clean-tree", "script/clean-source-tree.sh"),
595         ("clean", "PYTHON=python2 make clean"),
596
597         ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
598         ("talloc-make", "cd lib/talloc && PYTHON=python2 make"),
599         ("talloc-install", "cd lib/talloc && PYTHON=python2 make install"),
600
601         ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
602         ("tdb-make", "cd lib/tdb && PYTHON=python2 make"),
603         ("tdb-install", "cd lib/tdb && PYTHON=python2 make install"),
604
605         ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
606         ("tevent-make", "cd lib/tevent && PYTHON=python2 make"),
607         ("tevent-install", "cd lib/tevent && PYTHON=python2 make install"),
608
609         ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
610         ("ldb-make", "cd lib/ldb && PYTHON=python2 make"),
611         ("ldb-install", "cd lib/ldb && PYTHON=python2 make install"),
612
613         # retry against installed library packages
614         ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
615         ("libs-make", "PYTHON=python2 make -j"),
616         ("libs-install", "PYTHON=python2 make install"),
617         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
618         ("libs-clean", "PYTHON=python2 make clean"),
619         ],
620
621     "ldb": [
622         ("random-sleep", random_sleep(60, 600)),
623         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
624         ("make", "make"),
625         ("install", "make install"),
626         ("test", "make test"),
627         ("lcov", LCOV_CMD),
628         ("clean", "make clean"),
629         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
630         ("make-no-lmdb", "make"),
631         ("test-no-lmdb", "make test"),
632         ("lcov-no-lmdb", LCOV_CMD),
633         ("install-no-lmdb", "make install"),
634         ("check-clean-tree", "../../script/clean-source-tree.sh"),
635         ("distcheck", "make distcheck"),
636         ("clean", "make clean"),
637         ],
638
639     "tdb": [
640         ("random-sleep", random_sleep(60, 600)),
641         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
642         ("make", "make"),
643         ("install", "make install"),
644         ("test", "make test"),
645         ("lcov", LCOV_CMD),
646         ("check-clean-tree", "../../script/clean-source-tree.sh"),
647         ("distcheck", "make distcheck"),
648         ("clean", "make clean"),
649         ],
650
651     "talloc": [
652         ("random-sleep", random_sleep(60, 600)),
653         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
654         ("make", "make"),
655         ("install", "make install"),
656         ("test", "make test"),
657         ("lcov", LCOV_CMD),
658         ("check-clean-tree", "../../script/clean-source-tree.sh"),
659         ("distcheck", "make distcheck"),
660         ("clean", "make clean"),
661         ],
662
663     "replace": [
664         ("random-sleep", random_sleep(60, 600)),
665         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
666         ("make", "make"),
667         ("install", "make install"),
668         ("test", "make test"),
669         ("lcov", LCOV_CMD),
670         ("check-clean-tree", "../../script/clean-source-tree.sh"),
671         ("distcheck", "make distcheck"),
672         ("clean", "make clean"),
673         ],
674
675     "tevent": [
676         ("random-sleep", random_sleep(60, 600)),
677         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
678         ("make", "make"),
679         ("install", "make install"),
680         ("test", "make test"),
681         ("lcov", LCOV_CMD),
682         ("check-clean-tree", "../../script/clean-source-tree.sh"),
683         ("distcheck", "make distcheck"),
684         ("clean", "make clean"),
685         ],
686
687     "pidl": [
688         ("random-sleep", random_sleep(60, 600)),
689         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
690         ("touch", "touch *.yp"),
691         ("make", "make"),
692         ("test", "make test"),
693         ("install", "make install"),
694         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
695         ("check-clean-tree", "../script/clean-source-tree.sh"),
696         ("clean", "make clean"),
697         ],
698
699     # these are useful for debugging autobuild
700     'pass': [("pass", 'echo passing && /bin/true')],
701     'fail': [("fail", 'echo failing && /bin/false')],
702 }
703
704
705 def do_print(msg):
706     print("%s" % msg)
707     sys.stdout.flush()
708     sys.stderr.flush()
709
710
711 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
712     if show is None:
713         show = options.verbose
714     if show:
715         do_print("Running: '%s' in '%s'" % (cmd, dir))
716     if output:
717         out = check_output([cmd], shell=True, cwd=dir)
718         return out.decode(encoding='utf-8', errors='backslashreplace')
719     elif checkfail:
720         return check_call(cmd, shell=True, cwd=dir)
721     else:
722         return call(cmd, shell=True, cwd=dir)
723
724
725 class builder(object):
726     '''handle build of one directory'''
727
728     def __init__(self, name, sequence, cp=True):
729         self.name = name
730         self.dir = builddirs.get(name, '.')
731         self.tag = self.name.replace('/', '_')
732         self.sequence = sequence
733         self.next = 0
734         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
735         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
736         if options.verbose:
737             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
738             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
739         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
740         self.stdout = open(self.stdout_path, 'w')
741         self.stderr = open(self.stderr_path, 'w')
742         self.stdin  = open("/dev/null", 'r')
743         self.test_source_dir = "%s/%s" % (testbase, self.tag)
744         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
745         self.prefix = "%s/%s" % (test_prefix, self.tag)
746         run_cmd("rm -rf %s" % self.test_source_dir)
747         run_cmd("rm -rf %s" % self.prefix)
748         if cp:
749             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
750         else:
751             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
752         self.start_next()
753
754     def start_next(self):
755         if self.next == len(self.sequence):
756             if not options.nocleanup:
757                 run_cmd("rm -rf %s" % self.test_source_dir)
758                 run_cmd("rm -rf %s" % self.prefix)
759             do_print('%s: Completed OK' % self.name)
760             self.done = True
761             return
762         (self.stage, self.cmd) = self.sequence[self.next]
763         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
764         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
765         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
766         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
767         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
768         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
769         self.cmd = self.cmd.replace("${NAME}", self.name)
770         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
771         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
772         self.proc = Popen(self.cmd, shell=True,
773                           close_fds=True, cwd=self.cwd,
774                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
775         self.next += 1
776
777
778 class buildlist(object):
779     '''handle build of multiple directories'''
780
781     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
782         self.tail_proc = None
783         self.retry = None
784         if not tasknames:
785             if options.restrict_tests:
786                 tasknames = ["samba-test-only"]
787             else:
788                 tasknames = defaulttasks
789
790         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
791
792         if options.retry:
793             rebase_remote = "rebaseon"
794             retry_task = [("retry",
795                             '''set -e
796                             git remote add -t %s %s %s
797                             git fetch %s
798                             while :; do
799                               sleep 60
800                               git describe %s/%s > old_remote_branch.desc
801                               git fetch %s
802                               git describe %s/%s > remote_branch.desc
803                               diff old_remote_branch.desc remote_branch.desc
804                             done
805                            ''' % (
806                                rebase_branch, rebase_remote, rebase_url,
807                                rebase_remote,
808                                rebase_remote, rebase_branch,
809                                rebase_remote,
810                                rebase_remote, rebase_branch
811                             ))]
812
813             self.retry = builder('retry', retry_task, cp=False)
814             self.need_retry = False
815
816     def kill_kids(self):
817         if self.tail_proc is not None:
818             self.tail_proc.terminate()
819             self.tail_proc.wait()
820             self.tail_proc = None
821         if self.retry is not None:
822             self.retry.proc.terminate()
823             self.retry.proc.wait()
824             self.retry = None
825         for b in self.tlist:
826             if b.proc is not None:
827                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
828                 b.proc.terminate()
829                 b.proc.wait()
830                 b.proc = None
831
832     def wait_one(self):
833         while True:
834             none_running = True
835             for b in self.tlist:
836                 if b.proc is None:
837                     continue
838                 none_running = False
839                 b.status = b.proc.poll()
840                 if b.status is None:
841                     continue
842                 b.proc = None
843                 return b
844             if options.retry:
845                 ret = self.retry.proc.poll()
846                 if ret is not None:
847                     self.need_retry = True
848                     self.retry = None
849                     return None
850             if none_running:
851                 return None
852             time.sleep(0.1)
853
854     def run(self):
855         while True:
856             b = self.wait_one()
857             if options.retry and self.need_retry:
858                 self.kill_kids()
859                 do_print("retry needed")
860                 return (0, None, None, None, "retry")
861             if b is None:
862                 break
863             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
864                 self.kill_kids()
865                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
866             b.start_next()
867         self.kill_kids()
868         return (0, None, None, None, "All OK")
869
870     def write_system_info(self, filename):
871         with open(filename, 'w') as f:
872             for cmd in ['uname -a',
873                         'lsb_release -a',
874                         'free',
875                         'mount',
876                         'cat /proc/cpuinfo',
877                         'cc --version',
878                         'df -m .',
879                         'df -m %s' % testbase]:
880                 out = run_cmd(cmd, output=True, checkfail=False)
881                 print('### %s' % cmd, file=f)
882                 print(out, file=f)
883                 print(file=f)
884
885     def tarlogs(self, fname):
886         with tarfile.open(fname, "w:gz") as tar:
887             for b in self.tlist:
888                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
889                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
890             if os.path.exists("autobuild.log"):
891                 tar.add("autobuild.log")
892             filename = 'system-info.txt'
893             self.write_system_info(filename)
894             tar.add(filename)
895
896     def remove_logs(self):
897         for b in self.tlist:
898             os.unlink(b.stdout_path)
899             os.unlink(b.stderr_path)
900
901     def start_tail(self):
902         cmd = ["tail", "-f"]
903         for b in self.tlist:
904             cmd.append(b.stdout_path)
905             cmd.append(b.stderr_path)
906         self.tail_proc = Popen(cmd, close_fds=True)
907
908
909 def cleanup():
910     if options.nocleanup:
911         return
912     run_cmd("stat %s || true" % test_tmpdir, show=True)
913     run_cmd("stat %s" % testbase, show=True)
914     do_print("Cleaning up %r" % cleanup_list)
915     for d in cleanup_list:
916         run_cmd("rm -rf %s" % d)
917
918
919 def daemonize(logfile):
920     pid = os.fork()
921     if pid == 0:  # Parent
922         os.setsid()
923         pid = os.fork()
924         if pid != 0:  # Actual daemon
925             os._exit(0)
926     else:  # Grandparent
927         os._exit(0)
928
929     import resource      # Resource usage information.
930     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
931     if maxfd == resource.RLIM_INFINITY:
932         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
933     for fd in range(0, maxfd):
934         try:
935             os.close(fd)
936         except OSError:
937             pass
938     os.open(logfile, os.O_RDWR | os.O_CREAT)
939     os.dup2(0, 1)
940     os.dup2(0, 2)
941
942
943 def write_pidfile(fname):
944     '''write a pid file, cleanup on exit'''
945     with open(fname, mode='w') as f:
946         f.write("%u\n" % os.getpid())
947
948
949 def rebase_tree(rebase_url, rebase_branch="master"):
950     rebase_remote = "rebaseon"
951     do_print("Rebasing on %s" % rebase_url)
952     run_cmd("git describe HEAD", show=True, dir=test_master)
953     run_cmd("git remote add -t %s %s %s" %
954             (rebase_branch, rebase_remote, rebase_url),
955             show=True, dir=test_master)
956     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
957     if options.fix_whitespace:
958         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
959                 (rebase_remote, rebase_branch),
960                 show=True, dir=test_master)
961     else:
962         run_cmd("git rebase --force-rebase %s/%s" %
963                 (rebase_remote, rebase_branch),
964                 show=True, dir=test_master)
965     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
966                    (rebase_remote, rebase_branch),
967                    dir=test_master, output=True)
968     if diff == '':
969         do_print("No differences between HEAD and %s/%s - exiting" %
970                  (rebase_remote, rebase_branch))
971         sys.exit(0)
972     run_cmd("git describe %s/%s" %
973             (rebase_remote, rebase_branch),
974             show=True, dir=test_master)
975     run_cmd("git describe HEAD", show=True, dir=test_master)
976     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
977             (rebase_remote, rebase_branch),
978             show=True, dir=test_master)
979
980
981 def push_to(push_url, push_branch="master"):
982     push_remote = "pushto"
983     do_print("Pushing to %s" % push_url)
984     if options.mark:
985         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
986         run_cmd("git commit --amend -c HEAD", dir=test_master)
987         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
988         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
989     run_cmd("git remote add -t %s %s %s" %
990             (push_branch, push_remote, push_url),
991             show=True, dir=test_master)
992     run_cmd("git push %s +HEAD:%s" %
993             (push_remote, push_branch),
994             show=True, dir=test_master)
995
996
997 def send_email(subject, text, log_tar):
998     if options.email is None:
999         do_print("not sending email because the recipient is not set")
1000         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1001                  (subject, text))
1002         return
1003     outer = MIMEMultipart()
1004     outer['Subject'] = subject
1005     outer['To'] = options.email
1006     outer['From'] = options.email_from
1007     outer['Date'] = email.utils.formatdate(localtime=True)
1008     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1009     outer.attach(MIMEText(text, 'plain'))
1010     if options.attach_logs:
1011         with open(log_tar, 'rb') as fp:
1012             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1013         # Set the filename parameter
1014         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1015         outer.attach(msg)
1016     content = outer.as_string()
1017     s = smtplib.SMTP(options.email_server)
1018     email_user = os.getenv('SMTP_USERNAME')
1019     email_password = os.getenv('SMTP_PASSWORD')
1020     if email_user is not None:
1021         s.starttls()
1022         s.login(email_user, email_password)
1023
1024     s.sendmail(options.email_from, [options.email], content)
1025     s.set_debuglevel(1)
1026     s.quit()
1027
1028
1029 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1030                   elapsed_time, log_base=None, add_log_tail=True):
1031     '''send an email to options.email about the failure'''
1032     elapsed_minutes = elapsed_time / 60.0
1033     if log_base is None:
1034         log_base = gitroot
1035     text = '''
1036 Dear Developer,
1037
1038 Your autobuild on %s failed after %.1f minutes
1039 when trying to test %s with the following error:
1040
1041    %s
1042
1043 the autobuild has been abandoned. Please fix the error and resubmit.
1044
1045 A summary of the autobuild process is here:
1046
1047   %s/autobuild.log
1048 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1049
1050     if options.restrict_tests:
1051         text += """
1052 The build was restricted to tests matching %s\n""" % options.restrict_tests
1053
1054     if failed_task != 'rebase':
1055         text += '''
1056 You can see logs of the failed task here:
1057
1058   %s/%s.stdout
1059   %s/%s.stderr
1060
1061 or you can get full logs of all tasks in this job here:
1062
1063   %s/logs.tar.gz
1064
1065 The top commit for the tree that was built was:
1066
1067 %s
1068
1069 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1070
1071     if add_log_tail:
1072         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1073         lines = f.readlines()
1074         log_tail = "".join(lines[-50:])
1075         num_lines = len(lines)
1076         if num_lines < 50:
1077             # Also include stderr (compile failures) if < 50 lines of stdout
1078             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1079             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1080
1081         text += '''
1082 The last 50 lines of log messages:
1083
1084 %s
1085     ''' % log_tail
1086         f.close()
1087
1088     logs = os.path.join(gitroot, 'logs.tar.gz')
1089     send_email('autobuild[%s] failure on %s for task %s during %s'
1090                % (options.branch, platform.node(), failed_task, failed_stage),
1091                text, logs)
1092
1093
1094 def email_success(elapsed_time, log_base=None):
1095     '''send an email to options.email about a successful build'''
1096     if log_base is None:
1097         log_base = gitroot
1098     text = '''
1099 Dear Developer,
1100
1101 Your autobuild on %s has succeeded after %.1f minutes.
1102
1103 ''' % (platform.node(), elapsed_time / 60.)
1104
1105     if options.restrict_tests:
1106         text += """
1107 The build was restricted to tests matching %s\n""" % options.restrict_tests
1108
1109     if options.keeplogs:
1110         text += '''
1111
1112 you can get full logs of all tasks in this job here:
1113
1114   %s/logs.tar.gz
1115
1116 ''' % log_base
1117
1118     text += '''
1119 The top commit for the tree that was built was:
1120
1121 %s
1122 ''' % top_commit_msg
1123
1124     logs = os.path.join(gitroot, 'logs.tar.gz')
1125     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1126                text, logs)
1127
1128
1129 # get the top commit message, for emails
1130 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1131
1132 try:
1133     os.makedirs(testbase)
1134 except Exception as reason:
1135     raise Exception("Unable to create %s : %s" % (testbase, reason))
1136 cleanup_list.append(testbase)
1137
1138 if options.daemon:
1139     logfile = os.path.join(testbase, "log")
1140     do_print("Forking into the background, writing progress to %s" % logfile)
1141     daemonize(logfile)
1142
1143 write_pidfile(gitroot + "/autobuild.pid")
1144
1145 start_time = time.time()
1146
1147 while True:
1148     try:
1149         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1150         os.makedirs(test_tmpdir)
1151         # The waf uninstall code removes empty directories all the way
1152         # up the tree.  Creating a file in test_tmpdir stops it from
1153         # being removed.
1154         run_cmd("touch %s" % os.path.join(test_tmpdir,
1155                                           ".directory-is-not-empty"), show=True)
1156         run_cmd("stat %s" % test_tmpdir, show=True)
1157         run_cmd("stat %s" % testbase, show=True)
1158         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1159     except Exception:
1160         cleanup()
1161         raise
1162
1163     try:
1164         if options.rebase is not None:
1165             rebase_tree(options.rebase, rebase_branch=options.branch)
1166     except Exception:
1167         cleanup_list.append(gitroot + "/autobuild.pid")
1168         cleanup()
1169         elapsed_time = time.time() - start_time
1170         email_failure(-1, 'rebase', 'rebase', 'rebase',
1171                       'rebase on %s failed' % options.branch,
1172                       elapsed_time, log_base=options.log_base)
1173         sys.exit(1)
1174
1175     try:
1176         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1177         if options.tail:
1178             blist.start_tail()
1179         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1180         if status != 0 or errstr != "retry":
1181             break
1182         cleanup()
1183     except Exception:
1184         cleanup()
1185         raise
1186
1187 cleanup_list.append(gitroot + "/autobuild.pid")
1188
1189 do_print(errstr)
1190
1191 blist.kill_kids()
1192 if options.tail:
1193     do_print("waiting for tail to flush")
1194     time.sleep(1)
1195
1196 elapsed_time = time.time() - start_time
1197 if status == 0:
1198     if options.passcmd is not None:
1199         do_print("Running passcmd: %s" % options.passcmd)
1200         run_cmd(options.passcmd, dir=test_master)
1201     if options.pushto is not None:
1202         push_to(options.pushto, push_branch=options.branch)
1203     if options.keeplogs or options.attach_logs:
1204         blist.tarlogs("logs.tar.gz")
1205         do_print("Logs in logs.tar.gz")
1206     if options.always_email:
1207         email_success(elapsed_time, log_base=options.log_base)
1208     blist.remove_logs()
1209     cleanup()
1210     do_print(errstr)
1211     sys.exit(0)
1212
1213 # something failed, gather a tar of the logs
1214 blist.tarlogs("logs.tar.gz")
1215
1216 if options.email is not None:
1217     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1218                   elapsed_time, log_base=options.log_base)
1219 else:
1220     elapsed_minutes = elapsed_time / 60.0
1221     print('''
1222
1223 ####################################################################
1224
1225 AUTOBUILD FAILURE
1226
1227 Your autobuild[%s] on %s failed after %.1f minutes
1228 when trying to test %s with the following error:
1229
1230    %s
1231
1232 the autobuild has been abandoned. Please fix the error and resubmit.
1233
1234 ####################################################################
1235
1236 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1237
1238 cleanup()
1239 do_print(errstr)
1240 do_print("Logs in logs.tar.gz")
1241 sys.exit(status)