85167cfa993d576d5a5fcaa5190883c334b38be7
[samba.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, check_output, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 import random
13 from optparse import OptionParser
14 import smtplib
15 import email
16 from email.mime.text import MIMEText
17 from email.mime.base import MIMEBase
18 from email.mime.application import MIMEApplication
19 from email.mime.multipart import MIMEMultipart
20 from distutils.sysconfig import get_python_lib
21 import platform
22
23 try:
24     from waflib.Build import CACHE_SUFFIX
25 except ImportError:
26     sys.path.insert(0, "./third_party/waf")
27     from waflib.Build import CACHE_SUFFIX
28
29
30 os.environ["PYTHONUNBUFFERED"] = "1"
31
32 # This speeds up testing remarkably.
33 os.environ['TDB_NO_FSYNC'] = '1'
34
35
36 def find_git_root():
37     '''get to the top of the git repo'''
38     p = os.getcwd()
39     while p != '/':
40         if os.path.isdir(os.path.join(p, ".git")):
41             return p
42         p = os.path.abspath(os.path.join(p, '..'))
43     return None
44
45
46 gitroot = find_git_root()
47 if gitroot is None:
48     raise Exception("Failed to find git root")
49
50
51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
52
53 parser = OptionParser()
54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
58                   default=def_testbase)
59 parser.add_option("--passcmd", help="command to run on success", default=None)
60 parser.add_option("--verbose", help="show all commands as they are run",
61                   default=False, action="store_true")
62 parser.add_option("--rebase", help="rebase on the given tree before testing",
63                   default=None, type='str')
64 parser.add_option("--pushto", help="push to a git url on success",
65                   default=None, type='str')
66 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
67                   default=False, action="store_true")
68 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
69                   default=False, action="store_true")
70 parser.add_option("--retry", help="automatically retry if master changes",
71                   default=False, action="store_true")
72 parser.add_option("--email", help="send email to the given address on failure",
73                   type='str', default=None)
74 parser.add_option("--email-from", help="send email from the given address",
75                   type='str', default="autobuild@samba.org")
76 parser.add_option("--email-server", help="send email via the given server",
77                   type='str', default='localhost')
78 parser.add_option("--always-email", help="always send email, even on success",
79                   action="store_true")
80 parser.add_option("--daemon", help="daemonize after initial setup",
81                   action="store_true")
82 parser.add_option("--branch", help="the branch to work on (default=master)",
83                   default="master", type='str')
84 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
85                   default=gitroot, type='str')
86 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
87                   default=False, action="store_true")
88 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
89                   default='')
90 parser.add_option("--enable-coverage", dest='enable_coverage',
91                   action="store_const", const='--enable-coverage', default='',
92                   help="Add --enable-coverage option while configure")
93
94 (options, args) = parser.parse_args()
95
96 if options.retry:
97     if options.rebase is None:
98         raise Exception('You can only use --retry if you also rebase')
99
100 testbase = "%s/b%u" % (options.testbase, os.getpid())
101 test_master = "%s/master" % testbase
102 test_prefix = "%s/prefix" % testbase
103 test_tmpdir = "%s/tmp" % testbase
104 os.environ['TMPDIR'] = test_tmpdir
105
106 if options.enable_coverage:
107     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
108 else:
109     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
110
111 if args:
112     # If we are only running specific test,
113     # do not sleep randomly to wait for it to start
114     def random_sleep(low, high):
115         return 'sleep 1'
116 else:
117     def random_sleep(low, high):
118         return 'sleep {}'.format(random.randint(low, high))
119
120 cleanup_list = []
121
122 builddirs = {
123     "ctdb": "ctdb",
124     "samba": ".",
125     "samba-mitkrb5": ".",
126     "samba-nt4": ".",
127     "samba-fileserver": ".",
128     "samba-ktest-heimdal": ".",
129     "samba-admem": ".",
130     "samba-admem-mit": ".",
131     "samba-xc": ".",
132     "samba-o3": ".",
133     "samba-ctdb": ".",
134     "samba-libs": ".",
135     "samba-static": ".",
136     "samba-none-env": ".",
137     "samba-ad-dc-1": ".",
138     "samba-ad-dc-1-mitkrb5": ".",
139     "samba-ad-dc-2": ".",
140     "samba-ad-dc-3": ".",
141     "samba-ad-dc-4": ".",
142     "samba-ad-dc-4-mitkrb5": ".",
143     "samba-ad-dc-5": ".",
144     "samba-ad-dc-6": ".",
145     "samba-ad-dc-ntvfs": ".",
146     "samba-ad-dc-backup": ".",
147     "samba-nopython": ".",
148     "samba-nopython-py2": ".",
149     "samba-schemaupgrade": ".",
150     "ldb": "lib/ldb",
151     "tdb": "lib/tdb",
152     "talloc": "lib/talloc",
153     "replace": "lib/replace",
154     "tevent": "lib/tevent",
155     "pidl": "pidl"
156 }
157
158 defaulttasks = list(builddirs.keys())
159
160 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
161     defaulttasks.remove("samba-o3")
162
163 ctdb_configure_params = " --enable-developer ${PREFIX}"
164 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
165
166 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
167 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
168 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
169 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
170 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
171 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
172 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
173
174
175 def format_option(name, value=None):
176     """Format option as str list."""
177     if value is None:  # boolean option
178         return [name]
179     if not isinstance(value, list):  # single value option
180         value = [value]
181     # repeatable option
182     return ['{}={}'.format(name, item) for item in value]
183
184
185 def make_test(
186         cmd='make test',
187         FAIL_IMMEDIATELY=1,
188         TESTS='',
189         include_envs=None,
190         exclude_envs=None):
191
192     test_options = []
193     if include_envs:
194         test_options = format_option('--include-env', include_envs)
195     if exclude_envs:
196         test_options = format_option('--exclude-env', exclude_envs)
197     if test_options:
198         # join envs options to original test options
199         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
200
201     _options = []
202     if FAIL_IMMEDIATELY:
203         _options.append('FAIL_IMMEDIATELY=1')
204     if TESTS:
205         _options.append("TESTS='{}'".format(TESTS))
206
207     return ' '.join([cmd] + _options)
208
209
210 tasks = {
211     "ctdb": [
212         ("random-sleep", random_sleep(300, 900)),
213         ("configure", "./configure " + ctdb_configure_params),
214         ("make", "make all"),
215         ("install", "make install"),
216         ("test", "make autotest"),
217         ("check-clean-tree", "../script/clean-source-tree.sh"),
218         ("clean", "make clean"),
219         ],
220
221     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
222     "samba": [
223         ("random-sleep", random_sleep(300, 900)),
224         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
225         ("make", "make -j"),
226         ("test", make_test(exclude_envs=[
227             "none",
228             "nt4_dc",
229             "nt4_dc_schannel",
230             "nt4_member",
231             "ad_dc",
232             "ad_dc_backup",
233             "ad_dc_ntvfs",
234             "ad_dc_default",
235             "ad_dc_slowtests",
236             "ad_dc_no_nss",
237             "ad_dc_no_ntlm",
238             "fl2003dc",
239             "fl2008dc",
240             "fl2008r2dc",
241             "ad_member",
242             "ad_member_idmap_rid",
243             "ad_member_idmap_ad",
244             "ad_member_rfc2307",
245             "chgdcpass",
246             "vampire_2000_dc",
247             "fl2000dc",
248             "fileserver",
249             "maptoguest",
250             "simpleserver",
251             "backupfromdc",
252             "restoredc",
253             "renamedc",
254             "offlinebackupdc",
255             "labdc",
256             "preforkrestartdc",
257             "proclimitdc",
258             "promoted_dc",
259             "vampire_dc",
260             "rodc",
261             "ad_dc_default",
262             "ad_dc_slowtests",
263             "schema_pair_dc",
264             "schema_dc",
265             ])),
266         ("lcov", LCOV_CMD),
267         ("install", "make install"),
268         ("check-clean-tree", "script/clean-source-tree.sh"),
269         ("clean", "make clean"),
270         ],
271
272     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
273     "samba-mitkrb5": [
274         ("random-sleep", random_sleep(300, 900)),
275         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
276         ("make", "make -j"),
277         ("test", make_test(exclude_envs=[
278             "none",
279             "nt4_dc",
280             "nt4_dc_schannel",
281             "nt4_member",
282             "ad_dc",
283             "ad_dc_backup",
284             "ad_dc_ntvfs",
285             "ad_dc_default",
286             "ad_dc_slowtests",
287             "ad_dc_no_nss",
288             "ad_dc_no_ntlm",
289             "fl2003dc",
290             "fl2008dc",
291             "fl2008r2dc",
292             "ad_member",
293             "ad_member_idmap_rid",
294             "ad_member_idmap_ad",
295             "ad_member_rfc2307",
296             "chgdcpass",
297             "vampire_2000_dc",
298             "fl2000dc",
299             "fileserver",
300             "maptoguest",
301             "simpleserver",
302             "backupfromdc",
303             "restoredc",
304             "renamedc",
305             "offlinebackupdc",
306             "labdc",
307             "preforkrestartdc",
308             "proclimitdc",
309             "promoted_dc",
310             "vampire_dc",
311             "rodc",
312             "ad_dc_default",
313             "ad_dc_slowtests",
314             "schema_pair_dc",
315             "schema_dc",
316             ])),
317         ("lcov", LCOV_CMD),
318         ("install", "make install"),
319         ("check-clean-tree", "script/clean-source-tree.sh"),
320         ("clean", "make clean"),
321         ],
322
323     "samba-nt4": [
324         ("random-sleep", random_sleep(300, 900)),
325         ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params),
326         ("make", "make -j"),
327         ("test", make_test(include_envs=[
328             "nt4_dc",
329             "nt4_dc_schannel",
330             "nt4_member",
331             ])),
332         ("lcov", LCOV_CMD),
333         ("install", "make install"),
334         ("check-clean-tree", "script/clean-source-tree.sh"),
335         ("clean", "make clean"),
336         ],
337
338     "samba-simpleserver": [
339         ("random-sleep", random_sleep(300, 900)),
340         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
341         ("make", "make -j"),
342         ("test", make_test(include_envs=[
343             "simpleserver",
344             ])),
345         ("lcov", LCOV_CMD),
346         ("check-clean-tree", "script/clean-source-tree.sh"),
347         ],
348
349     "samba-fileserver": [
350         ("random-sleep", random_sleep(300, 900)),
351         ("configure", "./configure.developer --without-ad-dc --with-selftest-prefix=./bin/ab" + samba_configure_params),
352         ("make", "make -j"),
353         ("test", make_test(include_envs=[
354             "fileserver",
355             "maptoguest",
356             ])),
357         ("lcov", LCOV_CMD),
358         ("check-clean-tree", "script/clean-source-tree.sh"),
359         ],
360
361     "samba-ktest-heimdal": [
362         ("random-sleep", random_sleep(300, 900)),
363         ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab" + samba_configure_params),
364         ("make", "make -j"),
365         ("test", make_test(include_envs=[
366             "ktest",
367             ])),
368         ("lcov", LCOV_CMD),
369         ("check-clean-tree", "script/clean-source-tree.sh"),
370         ],
371
372     "samba-admem": [
373         ("random-sleep", random_sleep(300, 900)),
374         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
375         ("make", "make -j"),
376         ("test", make_test(include_envs=[
377             "ad_member",
378             "ad_member_idmap_rid",
379             "ad_member_idmap_ad",
380             "ad_member_rfc2307",
381             ])),
382         ("lcov", LCOV_CMD),
383         ("check-clean-tree", "script/clean-source-tree.sh"),
384         ],
385
386     "samba-ad-dc-1": [
387         ("random-sleep", random_sleep(1, 1)),
388         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
389         ("make", "make -j"),
390         ("test", make_test(include_envs=[
391             "ad_dc",
392             "ad_dc_no_nss",
393             "ad_dc_no_ntlm",
394             ])),
395         ("lcov", LCOV_CMD),
396         ("check-clean-tree", "script/clean-source-tree.sh"),
397         ],
398
399     "samba-ad-dc-2": [
400         ("random-sleep", random_sleep(1, 1)),
401         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
402         ("make", "make -j"),
403         ("test", make_test(include_envs=[
404             "vampire_dc",
405             "vampire_2000_dc",
406             "rodc",
407             ])),
408         ("lcov", LCOV_CMD),
409         ("check-clean-tree", "script/clean-source-tree.sh"),
410         ],
411
412     "samba-ad-dc-3": [
413         ("random-sleep", random_sleep(1, 1)),
414         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
415         ("make", "make -j"),
416         ("test", make_test(include_envs=[
417             "promoted_dc",
418             "chgdcpass",
419             "preforkrestartdc",
420             "proclimitdc",
421             ])),
422         ("lcov", LCOV_CMD),
423         ("check-clean-tree", "script/clean-source-tree.sh"),
424         ],
425
426     "samba-ad-dc-4": [
427         ("random-sleep", random_sleep(1, 1)),
428         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
429         ("make", "make -j"),
430         ("test", make_test(include_envs=[
431             "fl2000dc",
432             "fl2003dc",
433             "fl2008dc",
434             "fl2008r2dc",
435             ])),
436         ("lcov", LCOV_CMD),
437         ("check-clean-tree", "script/clean-source-tree.sh"),
438         ],
439
440     "samba-ad-dc-5": [
441         ("random-sleep", random_sleep(1, 1)),
442         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
443         ("make", "make -j"),
444         ("test", make_test(include_envs=["ad_dc_default"])),
445         ("lcov", LCOV_CMD),
446         ("check-clean-tree", "script/clean-source-tree.sh"),
447         ],
448
449     "samba-ad-dc-6": [
450         ("random-sleep", random_sleep(1, 1)),
451         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
452         ("make", "make -j"),
453         ("test", make_test(include_envs=["ad_dc_slowtests"])),
454         ("lcov", LCOV_CMD),
455         ("check-clean-tree", "script/clean-source-tree.sh"),
456         ],
457
458     "samba-schemaupgrade": [
459         ("random-sleep", random_sleep(1, 1)),
460         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
461         ("make", "make -j"),
462         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
463         ("lcov", LCOV_CMD),
464         ("check-clean-tree", "script/clean-source-tree.sh"),
465         ],
466
467     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
468     # This is currently the longest task, so we don't randomly delay it.
469     "samba-ad-dc-ntvfs": [
470         ("random-sleep", random_sleep(1, 1)),
471         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
472         ("make", "make -j"),
473         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
474         ("lcov", LCOV_CMD),
475         ("check-clean-tree", "script/clean-source-tree.sh"),
476         ],
477
478     # run the backup/restore testenvs separately as they're fairly standalone
479     # (and CI seems to max out at ~8 different DCs running at once)
480     "samba-ad-dc-backup": [
481         ("random-sleep", random_sleep(300, 900)),
482         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
483         ("make", "make -j"),
484         ("test", make_test(include_envs=[
485             "backupfromdc",
486             "restoredc",
487             "renamedc",
488             "offlinebackupdc",
489             "labdc",
490             "ad_dc_backup",
491             ])),
492         ("lcov", LCOV_CMD),
493         ("check-clean-tree", "script/clean-source-tree.sh"),
494         ],
495
496     "samba-admem-mit": [
497         ("random-sleep", random_sleep(300, 900)),
498         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
499         ("make", "make -j"),
500         ("test", make_test(include_envs=[
501             "ad_member",
502             "ad_member_idmap_rid",
503             "ad_member_idmap_ad",
504             "ad_member_rfc2307",
505             ])),
506         ("lcov", LCOV_CMD),
507         ("check-clean-tree", "script/clean-source-tree.sh"),
508         ],
509
510     "samba-ad-dc-1-mitkrb5": [
511         ("random-sleep", random_sleep(1, 1)),
512         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
513         ("make", "make -j"),
514         ("test", make_test(include_envs=[
515             "ad_dc",
516             "ad_dc_no_nss",
517             "ad_dc_no_ntlm",
518             ])),
519         ("lcov", LCOV_CMD),
520         ("check-clean-tree", "script/clean-source-tree.sh"),
521         ],
522
523     "samba-ad-dc-4-mitkrb5": [
524         ("random-sleep", random_sleep(1, 1)),
525         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
526         ("make", "make -j"),
527         ("test", make_test(include_envs=[
528             "fl2000dc",
529             "fl2003dc",
530             "fl2008dc",
531             "fl2008r2dc",
532             ])),
533         ("lcov", LCOV_CMD),
534         ("check-clean-tree", "script/clean-source-tree.sh"),
535         ],
536
537     "samba-test-only": [
538         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
539         ("make", "make -j"),
540         ("test", make_test(TESTS="${TESTS}")),
541         ("lcov", LCOV_CMD),
542         ],
543
544     # Test cross-compile infrastructure
545     "samba-xc": [
546         ("random-sleep", random_sleep(900, 1500)),
547         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
548         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
549             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
550         ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
551         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
552             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
553         ("compare-results", "script/compare_cc_results.py "
554             "./bin/c4che/default{} "
555             "./bin-xe/c4che/default{} "
556             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
557         ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
558         ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
559             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
560         ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
561             " = \"'1234'\"".format(CACHE_SUFFIX)),
562         ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
563         ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
564             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
565             " ; test $? -ne 0"),
566         ],
567
568     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
569     "samba-o3": [
570         ("random-sleep", random_sleep(300, 900)),
571         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
572         ("make", "make -j"),
573         ("test", make_test(cmd='make quicktest', include_envs=["ad_dc"])),
574         ("lcov", LCOV_CMD),
575         ("install", "make install"),
576         ("check-clean-tree", "script/clean-source-tree.sh"),
577         ("clean", "make clean"),
578         ],
579
580     "samba-ctdb": [
581         ("random-sleep", random_sleep(900, 1500)),
582
583         # make sure we have tdb around:
584         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
585         ("tdb-make", "cd lib/tdb && make"),
586         ("tdb-install", "cd lib/tdb && make install"),
587
588         # build samba with cluster support (also building ctdb):
589         ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb"),
590         ("samba-make", "make"),
591         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
592         ("samba-install", "make install"),
593         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
594
595         # clean up:
596         ("check-clean-tree", "script/clean-source-tree.sh"),
597         ("clean", "make clean"),
598         ("ctdb-clean", "cd ./ctdb && make clean"),
599         ],
600
601     "samba-libs": [
602         ("random-sleep", random_sleep(300, 900)),
603         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
604         ("talloc-make", "cd lib/talloc && make"),
605         ("talloc-install", "cd lib/talloc && make install"),
606
607         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
608         ("tdb-make", "cd lib/tdb && make"),
609         ("tdb-install", "cd lib/tdb && make install"),
610
611         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
612         ("tevent-make", "cd lib/tevent && make"),
613         ("tevent-install", "cd lib/tevent && make install"),
614
615         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
616         ("ldb-make", "cd lib/ldb && make"),
617         ("ldb-install", "cd lib/ldb && make install"),
618
619         ("nondevel-configure", "./configure ${PREFIX}"),
620         ("nondevel-make", "make -j"),
621         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
622         ("nondevel-install", "make install"),
623         ("nondevel-dist", "make dist"),
624
625         # retry with all modules shared
626         ("allshared-distclean", "make distclean"),
627         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
628         ("allshared-make", "make -j"),
629         ],
630
631     "samba-none-env": [
632         ("random-sleep", random_sleep(1, 1)),
633         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
634         ("make", "make -j"),
635         ("test", make_test(include_envs=["none"])),
636         ("lcov", LCOV_CMD),
637         ],
638
639     "samba-static": [
640         ("random-sleep", random_sleep(1, 1)),
641         # build with all modules static
642         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
643         ("allstatic-make", "make -j"),
644         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
645         ("lcov", LCOV_CMD),
646
647         # retry without any required modules
648         ("none-distclean", "make distclean"),
649         ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
650         ("none-make", "make -j"),
651
652         # retry with nonshared smbd and smbtorture
653         ("nonshared-distclean", "make distclean"),
654         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
655         ("nonshared-make", "make -j"),
656         ],
657
658     # Test Samba without python still builds.  When this test fails
659     # due to more use of Python, the expectations is that the newly
660     # failing part of the code should be disabled when
661     # --disable-python is set (rather than major work being done to
662     # support this environment).  The target here is for vendors
663     # shipping a minimal smbd.
664     "samba-nopython": [
665         ("random-sleep", random_sleep(300, 900)),
666         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
667         ("make", "make -j"),
668         ("install", "make install"),
669         ("find-python", "script/find_python.sh ${PREFIX}"),
670         ("test", "make test-nopython"),
671         ("lcov", LCOV_CMD),
672         ("check-clean-tree", "script/clean-source-tree.sh"),
673         ("clean", "make clean"),
674
675         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
676         ("talloc-make", "cd lib/talloc && make"),
677         ("talloc-install", "cd lib/talloc && make install"),
678
679         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
680         ("tdb-make", "cd lib/tdb && make"),
681         ("tdb-install", "cd lib/tdb && make install"),
682
683         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
684         ("tevent-make", "cd lib/tevent && make"),
685         ("tevent-install", "cd lib/tevent && make install"),
686
687         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
688         ("ldb-make", "cd lib/ldb && make"),
689         ("ldb-install", "cd lib/ldb && make install"),
690
691         # retry against installed library packages
692         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
693         ("libs-make", "make -j"),
694         ("libs-install", "make install"),
695         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
696         ("libs-clean", "make clean"),
697         ],
698
699     # check we can do the same thing using python2
700     "samba-nopython-py2": [
701         ("random-sleep", random_sleep(300, 900)),
702         ("configure", "PYTHON=python2 ./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
703         ("make", "PYTHON=python2 make -j"),
704         ("install", "PYTHON=python2 make install"),
705         ("find-python", "script/find_python.sh ${PREFIX}"),
706         ("test", "make test-nopython"),
707         ("lcov", LCOV_CMD),
708         ("check-clean-tree", "script/clean-source-tree.sh"),
709         ("clean", "PYTHON=python2 make clean"),
710
711         ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
712         ("talloc-make", "cd lib/talloc && PYTHON=python2 make"),
713         ("talloc-install", "cd lib/talloc && PYTHON=python2 make install"),
714
715         ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
716         ("tdb-make", "cd lib/tdb && PYTHON=python2 make"),
717         ("tdb-install", "cd lib/tdb && PYTHON=python2 make install"),
718
719         ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
720         ("tevent-make", "cd lib/tevent && PYTHON=python2 make"),
721         ("tevent-install", "cd lib/tevent && PYTHON=python2 make install"),
722
723         ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
724         ("ldb-make", "cd lib/ldb && PYTHON=python2 make"),
725         ("ldb-install", "cd lib/ldb && PYTHON=python2 make install"),
726
727         # retry against installed library packages
728         ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
729         ("libs-make", "PYTHON=python2 make -j"),
730         ("libs-install", "PYTHON=python2 make install"),
731         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
732         ("libs-clean", "PYTHON=python2 make clean"),
733         ],
734
735     "ldb": [
736         ("random-sleep", random_sleep(60, 600)),
737         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
738         ("make", "make"),
739         ("install", "make install"),
740         ("test", "make test"),
741         ("lcov", LCOV_CMD),
742         ("clean", "make clean"),
743         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
744         ("make-no-lmdb", "make"),
745         ("test-no-lmdb", "make test"),
746         ("lcov-no-lmdb", LCOV_CMD),
747         ("install-no-lmdb", "make install"),
748         ("check-clean-tree", "../../script/clean-source-tree.sh"),
749         ("distcheck", "make distcheck"),
750         ("clean", "make clean"),
751         ],
752
753     "tdb": [
754         ("random-sleep", random_sleep(60, 600)),
755         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
756         ("make", "make"),
757         ("install", "make install"),
758         ("test", "make test"),
759         ("lcov", LCOV_CMD),
760         ("check-clean-tree", "../../script/clean-source-tree.sh"),
761         ("distcheck", "make distcheck"),
762         ("clean", "make clean"),
763         ],
764
765     "talloc": [
766         ("random-sleep", random_sleep(60, 600)),
767         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
768         ("make", "make"),
769         ("install", "make install"),
770         ("test", "make test"),
771         ("lcov", LCOV_CMD),
772         ("check-clean-tree", "../../script/clean-source-tree.sh"),
773         ("distcheck", "make distcheck"),
774         ("clean", "make clean"),
775         ],
776
777     "replace": [
778         ("random-sleep", random_sleep(60, 600)),
779         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
780         ("make", "make"),
781         ("install", "make install"),
782         ("test", "make test"),
783         ("lcov", LCOV_CMD),
784         ("check-clean-tree", "../../script/clean-source-tree.sh"),
785         ("distcheck", "make distcheck"),
786         ("clean", "make clean"),
787         ],
788
789     "tevent": [
790         ("random-sleep", random_sleep(60, 600)),
791         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
792         ("make", "make"),
793         ("install", "make install"),
794         ("test", "make test"),
795         ("lcov", LCOV_CMD),
796         ("check-clean-tree", "../../script/clean-source-tree.sh"),
797         ("distcheck", "make distcheck"),
798         ("clean", "make clean"),
799         ],
800
801     "pidl": [
802         ("random-sleep", random_sleep(60, 600)),
803         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
804         ("touch", "touch *.yp"),
805         ("make", "make"),
806         ("test", "make test"),
807         ("install", "make install"),
808         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
809         ("check-clean-tree", "../script/clean-source-tree.sh"),
810         ("clean", "make clean"),
811         ],
812
813     # these are useful for debugging autobuild
814     'pass': [("pass", 'echo passing && /bin/true')],
815     'fail': [("fail", 'echo failing && /bin/false')],
816 }
817
818
819 def do_print(msg):
820     print("%s" % msg)
821     sys.stdout.flush()
822     sys.stderr.flush()
823
824
825 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
826     if show is None:
827         show = options.verbose
828     if show:
829         do_print("Running: '%s' in '%s'" % (cmd, dir))
830     if output:
831         out = check_output([cmd], shell=True, cwd=dir)
832         return out.decode(encoding='utf-8', errors='backslashreplace')
833     elif checkfail:
834         return check_call(cmd, shell=True, cwd=dir)
835     else:
836         return call(cmd, shell=True, cwd=dir)
837
838
839 class builder(object):
840     '''handle build of one directory'''
841
842     def __init__(self, name, sequence, cp=True):
843         self.name = name
844         self.dir = builddirs.get(name, '.')
845         self.tag = self.name.replace('/', '_')
846         self.sequence = sequence
847         self.next = 0
848         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
849         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
850         if options.verbose:
851             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
852             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
853         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
854         self.stdout = open(self.stdout_path, 'w')
855         self.stderr = open(self.stderr_path, 'w')
856         self.stdin  = open("/dev/null", 'r')
857         self.test_source_dir = "%s/%s" % (testbase, self.tag)
858         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
859         self.prefix = "%s/%s" % (test_prefix, self.tag)
860         run_cmd("rm -rf %s" % self.test_source_dir)
861         run_cmd("rm -rf %s" % self.prefix)
862         if cp:
863             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
864         else:
865             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
866         self.start_next()
867
868     def start_next(self):
869         if self.next == len(self.sequence):
870             if not options.nocleanup:
871                 run_cmd("rm -rf %s" % self.test_source_dir)
872                 run_cmd("rm -rf %s" % self.prefix)
873             do_print('%s: Completed OK' % self.name)
874             self.done = True
875             return
876         (self.stage, self.cmd) = self.sequence[self.next]
877         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
878         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
879         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
880         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
881         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
882         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
883         self.cmd = self.cmd.replace("${NAME}", self.name)
884         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
885         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
886         self.proc = Popen(self.cmd, shell=True,
887                           close_fds=True, cwd=self.cwd,
888                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
889         self.next += 1
890
891
892 class buildlist(object):
893     '''handle build of multiple directories'''
894
895     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
896         self.tail_proc = None
897         self.retry = None
898         if not tasknames:
899             if options.restrict_tests:
900                 tasknames = ["samba-test-only"]
901             else:
902                 tasknames = defaulttasks
903
904         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
905
906         if options.retry:
907             rebase_remote = "rebaseon"
908             retry_task = [("retry",
909                             '''set -e
910                             git remote add -t %s %s %s
911                             git fetch %s
912                             while :; do
913                               sleep 60
914                               git describe %s/%s > old_remote_branch.desc
915                               git fetch %s
916                               git describe %s/%s > remote_branch.desc
917                               diff old_remote_branch.desc remote_branch.desc
918                             done
919                            ''' % (
920                                rebase_branch, rebase_remote, rebase_url,
921                                rebase_remote,
922                                rebase_remote, rebase_branch,
923                                rebase_remote,
924                                rebase_remote, rebase_branch
925                             ))]
926
927             self.retry = builder('retry', retry_task, cp=False)
928             self.need_retry = False
929
930     def kill_kids(self):
931         if self.tail_proc is not None:
932             self.tail_proc.terminate()
933             self.tail_proc.wait()
934             self.tail_proc = None
935         if self.retry is not None:
936             self.retry.proc.terminate()
937             self.retry.proc.wait()
938             self.retry = None
939         for b in self.tlist:
940             if b.proc is not None:
941                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
942                 b.proc.terminate()
943                 b.proc.wait()
944                 b.proc = None
945
946     def wait_one(self):
947         while True:
948             none_running = True
949             for b in self.tlist:
950                 if b.proc is None:
951                     continue
952                 none_running = False
953                 b.status = b.proc.poll()
954                 if b.status is None:
955                     continue
956                 b.proc = None
957                 return b
958             if options.retry:
959                 ret = self.retry.proc.poll()
960                 if ret is not None:
961                     self.need_retry = True
962                     self.retry = None
963                     return None
964             if none_running:
965                 return None
966             time.sleep(0.1)
967
968     def run(self):
969         while True:
970             b = self.wait_one()
971             if options.retry and self.need_retry:
972                 self.kill_kids()
973                 do_print("retry needed")
974                 return (0, None, None, None, "retry")
975             if b is None:
976                 break
977             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
978                 self.kill_kids()
979                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
980             b.start_next()
981         self.kill_kids()
982         return (0, None, None, None, "All OK")
983
984     def write_system_info(self, filename):
985         with open(filename, 'w') as f:
986             for cmd in ['uname -a',
987                         'lsb_release -a',
988                         'free',
989                         'mount',
990                         'cat /proc/cpuinfo',
991                         'cc --version',
992                         'df -m .',
993                         'df -m %s' % testbase]:
994                 out = run_cmd(cmd, output=True, checkfail=False)
995                 print('### %s' % cmd, file=f)
996                 print(out, file=f)
997                 print(file=f)
998
999     def tarlogs(self, fname):
1000         with tarfile.open(fname, "w:gz") as tar:
1001             for b in self.tlist:
1002                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1003                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1004             if os.path.exists("autobuild.log"):
1005                 tar.add("autobuild.log")
1006             filename = 'system-info.txt'
1007             self.write_system_info(filename)
1008             tar.add(filename)
1009
1010     def remove_logs(self):
1011         for b in self.tlist:
1012             os.unlink(b.stdout_path)
1013             os.unlink(b.stderr_path)
1014
1015     def start_tail(self):
1016         cmd = ["tail", "-f"]
1017         for b in self.tlist:
1018             cmd.append(b.stdout_path)
1019             cmd.append(b.stderr_path)
1020         self.tail_proc = Popen(cmd, close_fds=True)
1021
1022
1023 def cleanup():
1024     if options.nocleanup:
1025         return
1026     run_cmd("stat %s || true" % test_tmpdir, show=True)
1027     run_cmd("stat %s" % testbase, show=True)
1028     do_print("Cleaning up %r" % cleanup_list)
1029     for d in cleanup_list:
1030         run_cmd("rm -rf %s" % d)
1031
1032
1033 def daemonize(logfile):
1034     pid = os.fork()
1035     if pid == 0:  # Parent
1036         os.setsid()
1037         pid = os.fork()
1038         if pid != 0:  # Actual daemon
1039             os._exit(0)
1040     else:  # Grandparent
1041         os._exit(0)
1042
1043     import resource      # Resource usage information.
1044     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1045     if maxfd == resource.RLIM_INFINITY:
1046         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1047     for fd in range(0, maxfd):
1048         try:
1049             os.close(fd)
1050         except OSError:
1051             pass
1052     os.open(logfile, os.O_RDWR | os.O_CREAT)
1053     os.dup2(0, 1)
1054     os.dup2(0, 2)
1055
1056
1057 def write_pidfile(fname):
1058     '''write a pid file, cleanup on exit'''
1059     with open(fname, mode='w') as f:
1060         f.write("%u\n" % os.getpid())
1061
1062
1063 def rebase_tree(rebase_url, rebase_branch="master"):
1064     rebase_remote = "rebaseon"
1065     do_print("Rebasing on %s" % rebase_url)
1066     run_cmd("git describe HEAD", show=True, dir=test_master)
1067     run_cmd("git remote add -t %s %s %s" %
1068             (rebase_branch, rebase_remote, rebase_url),
1069             show=True, dir=test_master)
1070     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1071     if options.fix_whitespace:
1072         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1073                 (rebase_remote, rebase_branch),
1074                 show=True, dir=test_master)
1075     else:
1076         run_cmd("git rebase --force-rebase %s/%s" %
1077                 (rebase_remote, rebase_branch),
1078                 show=True, dir=test_master)
1079     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1080                    (rebase_remote, rebase_branch),
1081                    dir=test_master, output=True)
1082     if diff == '':
1083         do_print("No differences between HEAD and %s/%s - exiting" %
1084                  (rebase_remote, rebase_branch))
1085         sys.exit(0)
1086     run_cmd("git describe %s/%s" %
1087             (rebase_remote, rebase_branch),
1088             show=True, dir=test_master)
1089     run_cmd("git describe HEAD", show=True, dir=test_master)
1090     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1091             (rebase_remote, rebase_branch),
1092             show=True, dir=test_master)
1093
1094
1095 def push_to(push_url, push_branch="master"):
1096     push_remote = "pushto"
1097     do_print("Pushing to %s" % push_url)
1098     if options.mark:
1099         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1100         run_cmd("git commit --amend -c HEAD", dir=test_master)
1101         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1102         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1103     run_cmd("git remote add -t %s %s %s" %
1104             (push_branch, push_remote, push_url),
1105             show=True, dir=test_master)
1106     run_cmd("git push %s +HEAD:%s" %
1107             (push_remote, push_branch),
1108             show=True, dir=test_master)
1109
1110
1111 def send_email(subject, text, log_tar):
1112     if options.email is None:
1113         do_print("not sending email because the recipient is not set")
1114         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1115                  (subject, text))
1116         return
1117     outer = MIMEMultipart()
1118     outer['Subject'] = subject
1119     outer['To'] = options.email
1120     outer['From'] = options.email_from
1121     outer['Date'] = email.utils.formatdate(localtime=True)
1122     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1123     outer.attach(MIMEText(text, 'plain'))
1124     if options.attach_logs:
1125         with open(log_tar, 'rb') as fp:
1126             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1127         # Set the filename parameter
1128         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1129         outer.attach(msg)
1130     content = outer.as_string()
1131     s = smtplib.SMTP(options.email_server)
1132     email_user = os.getenv('SMTP_USERNAME')
1133     email_password = os.getenv('SMTP_PASSWORD')
1134     if email_user is not None:
1135         s.starttls()
1136         s.login(email_user, email_password)
1137
1138     s.sendmail(options.email_from, [options.email], content)
1139     s.set_debuglevel(1)
1140     s.quit()
1141
1142
1143 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1144                   elapsed_time, log_base=None, add_log_tail=True):
1145     '''send an email to options.email about the failure'''
1146     elapsed_minutes = elapsed_time / 60.0
1147     if log_base is None:
1148         log_base = gitroot
1149     text = '''
1150 Dear Developer,
1151
1152 Your autobuild on %s failed after %.1f minutes
1153 when trying to test %s with the following error:
1154
1155    %s
1156
1157 the autobuild has been abandoned. Please fix the error and resubmit.
1158
1159 A summary of the autobuild process is here:
1160
1161   %s/autobuild.log
1162 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1163
1164     if options.restrict_tests:
1165         text += """
1166 The build was restricted to tests matching %s\n""" % options.restrict_tests
1167
1168     if failed_task != 'rebase':
1169         text += '''
1170 You can see logs of the failed task here:
1171
1172   %s/%s.stdout
1173   %s/%s.stderr
1174
1175 or you can get full logs of all tasks in this job here:
1176
1177   %s/logs.tar.gz
1178
1179 The top commit for the tree that was built was:
1180
1181 %s
1182
1183 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1184
1185     if add_log_tail:
1186         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1187         lines = f.readlines()
1188         log_tail = "".join(lines[-50:])
1189         num_lines = len(lines)
1190         if num_lines < 50:
1191             # Also include stderr (compile failures) if < 50 lines of stdout
1192             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1193             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1194
1195         text += '''
1196 The last 50 lines of log messages:
1197
1198 %s
1199     ''' % log_tail
1200         f.close()
1201
1202     logs = os.path.join(gitroot, 'logs.tar.gz')
1203     send_email('autobuild[%s] failure on %s for task %s during %s'
1204                % (options.branch, platform.node(), failed_task, failed_stage),
1205                text, logs)
1206
1207
1208 def email_success(elapsed_time, log_base=None):
1209     '''send an email to options.email about a successful build'''
1210     if log_base is None:
1211         log_base = gitroot
1212     text = '''
1213 Dear Developer,
1214
1215 Your autobuild on %s has succeeded after %.1f minutes.
1216
1217 ''' % (platform.node(), elapsed_time / 60.)
1218
1219     if options.restrict_tests:
1220         text += """
1221 The build was restricted to tests matching %s\n""" % options.restrict_tests
1222
1223     if options.keeplogs:
1224         text += '''
1225
1226 you can get full logs of all tasks in this job here:
1227
1228   %s/logs.tar.gz
1229
1230 ''' % log_base
1231
1232     text += '''
1233 The top commit for the tree that was built was:
1234
1235 %s
1236 ''' % top_commit_msg
1237
1238     logs = os.path.join(gitroot, 'logs.tar.gz')
1239     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1240                text, logs)
1241
1242
1243 # get the top commit message, for emails
1244 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1245
1246 try:
1247     os.makedirs(testbase)
1248 except Exception as reason:
1249     raise Exception("Unable to create %s : %s" % (testbase, reason))
1250 cleanup_list.append(testbase)
1251
1252 if options.daemon:
1253     logfile = os.path.join(testbase, "log")
1254     do_print("Forking into the background, writing progress to %s" % logfile)
1255     daemonize(logfile)
1256
1257 write_pidfile(gitroot + "/autobuild.pid")
1258
1259 start_time = time.time()
1260
1261 while True:
1262     try:
1263         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1264         os.makedirs(test_tmpdir)
1265         # The waf uninstall code removes empty directories all the way
1266         # up the tree.  Creating a file in test_tmpdir stops it from
1267         # being removed.
1268         run_cmd("touch %s" % os.path.join(test_tmpdir,
1269                                           ".directory-is-not-empty"), show=True)
1270         run_cmd("stat %s" % test_tmpdir, show=True)
1271         run_cmd("stat %s" % testbase, show=True)
1272         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1273     except Exception:
1274         cleanup()
1275         raise
1276
1277     try:
1278         if options.rebase is not None:
1279             rebase_tree(options.rebase, rebase_branch=options.branch)
1280     except Exception:
1281         cleanup_list.append(gitroot + "/autobuild.pid")
1282         cleanup()
1283         elapsed_time = time.time() - start_time
1284         email_failure(-1, 'rebase', 'rebase', 'rebase',
1285                       'rebase on %s failed' % options.branch,
1286                       elapsed_time, log_base=options.log_base)
1287         sys.exit(1)
1288
1289     try:
1290         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1291         if options.tail:
1292             blist.start_tail()
1293         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1294         if status != 0 or errstr != "retry":
1295             break
1296         cleanup()
1297     except Exception:
1298         cleanup()
1299         raise
1300
1301 cleanup_list.append(gitroot + "/autobuild.pid")
1302
1303 do_print(errstr)
1304
1305 blist.kill_kids()
1306 if options.tail:
1307     do_print("waiting for tail to flush")
1308     time.sleep(1)
1309
1310 elapsed_time = time.time() - start_time
1311 if status == 0:
1312     if options.passcmd is not None:
1313         do_print("Running passcmd: %s" % options.passcmd)
1314         run_cmd(options.passcmd, dir=test_master)
1315     if options.pushto is not None:
1316         push_to(options.pushto, push_branch=options.branch)
1317     if options.keeplogs or options.attach_logs:
1318         blist.tarlogs("logs.tar.gz")
1319         do_print("Logs in logs.tar.gz")
1320     if options.always_email:
1321         email_success(elapsed_time, log_base=options.log_base)
1322     blist.remove_logs()
1323     cleanup()
1324     do_print(errstr)
1325     sys.exit(0)
1326
1327 # something failed, gather a tar of the logs
1328 blist.tarlogs("logs.tar.gz")
1329
1330 if options.email is not None:
1331     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1332                   elapsed_time, log_base=options.log_base)
1333 else:
1334     elapsed_minutes = elapsed_time / 60.0
1335     print('''
1336
1337 ####################################################################
1338
1339 AUTOBUILD FAILURE
1340
1341 Your autobuild[%s] on %s failed after %.1f minutes
1342 when trying to test %s with the following error:
1343
1344    %s
1345
1346 the autobuild has been abandoned. Please fix the error and resubmit.
1347
1348 ####################################################################
1349
1350 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1351
1352 cleanup()
1353 do_print(errstr)
1354 do_print("Logs in logs.tar.gz")
1355 sys.exit(status)