script/autobuild.py: define LCOV_CMD and run it after each make test
[bbaumbach/samba-autobuild/.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 from optparse import OptionParser
13 import smtplib
14 import email
15 from email.mime.text import MIMEText
16 from email.mime.base import MIMEBase
17 from email.mime.application import MIMEApplication
18 from email.mime.multipart import MIMEMultipart
19 from distutils.sysconfig import get_python_lib
20 import platform
21
22 try:
23     from waflib.Build import CACHE_SUFFIX
24 except ImportError:
25     sys.path.insert(0, "./third_party/waf")
26     from waflib.Build import CACHE_SUFFIX
27
28
29 os.environ["PYTHONUNBUFFERED"] = "1"
30
31 # This speeds up testing remarkably.
32 os.environ['TDB_NO_FSYNC'] = '1'
33
34
35 def find_git_root():
36     '''get to the top of the git repo'''
37     p = os.getcwd()
38     while p != '/':
39         if os.path.isdir(os.path.join(p, ".git")):
40             return p
41         p = os.path.abspath(os.path.join(p, '..'))
42     return None
43
44
45 gitroot = find_git_root()
46 if gitroot is None:
47     raise Exception("Failed to find git root")
48
49
50 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
51
52 parser = OptionParser()
53 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
54 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
55 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
56 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
57                   default=def_testbase)
58 parser.add_option("", "--passcmd", help="command to run on success", default=None)
59 parser.add_option("", "--verbose", help="show all commands as they are run",
60                   default=False, action="store_true")
61 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
62                   default=None, type='str')
63 parser.add_option("", "--pushto", help="push to a git url on success",
64                   default=None, type='str')
65 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
66                   default=False, action="store_true")
67 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
68                   default=False, action="store_true")
69 parser.add_option("", "--retry", help="automatically retry if master changes",
70                   default=False, action="store_true")
71 parser.add_option("", "--email", help="send email to the given address on failure",
72                   type='str', default=None)
73 parser.add_option("", "--email-from", help="send email from the given address",
74                   type='str', default="autobuild@samba.org")
75 parser.add_option("", "--email-server", help="send email via the given server",
76                   type='str', default='localhost')
77 parser.add_option("", "--always-email", help="always send email, even on success",
78                   action="store_true")
79 parser.add_option("", "--daemon", help="daemonize after initial setup",
80                   action="store_true")
81 parser.add_option("", "--branch", help="the branch to work on (default=master)",
82                   default="master", type='str')
83 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
84                   default=gitroot, type='str')
85 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
86                   default=False, action="store_true")
87 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
88                   default='')
89 parser.add_option("--enable-coverage", dest='enable_coverage',
90                   action="store_const", const='--enable-coverage', default='',
91                   help="Add --enable-coverage option while configure")
92
93 (options, args) = parser.parse_args()
94
95 if options.retry:
96     if options.rebase is None:
97         raise Exception('You can only use --retry if you also rebase')
98
99 testbase = "%s/b%u" % (options.testbase, os.getpid())
100 test_master = "%s/master" % testbase
101 test_prefix = "%s/prefix" % testbase
102 test_tmpdir = "%s/tmp" % testbase
103 os.environ['TMPDIR'] = test_tmpdir
104
105 if options.enable_coverage:
106     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
107 else:
108     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
109
110 cleanup_list = []
111
112 builddirs = {
113     "ctdb": "ctdb",
114     "samba": ".",
115     "samba-nt4": ".",
116     "samba-fileserver": ".",
117     "samba-ad-member": ".",
118     "samba-xc": ".",
119     "samba-o3": ".",
120     "samba-ctdb": ".",
121     "samba-libs": ".",
122     "samba-static": ".",
123     "samba-none-env": ".",
124     "samba-ad-dc-1": ".",
125     "samba-ad-dc-2": ".",
126     "samba-ad-dc-3": ".",
127     "samba-ad-dc-4": ".",
128     "samba-ad-dc-5": ".",
129     "samba-ad-dc-6": ".",
130     "samba-ad-dc-ntvfs": ".",
131     "samba-ad-dc-backup": ".",
132     "samba-systemkrb5": ".",
133     "samba-nopython": ".",
134     "samba-nopython-py2": ".",
135     "samba-schemaupgrade": ".",
136     "ldb": "lib/ldb",
137     "tdb": "lib/tdb",
138     "talloc": "lib/talloc",
139     "replace": "lib/replace",
140     "tevent": "lib/tevent",
141     "pidl": "pidl"
142 }
143
144 defaulttasks = builddirs.keys()
145
146 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
147     defaulttasks.remove("samba-o3")
148
149 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
150 samba_configure_params = " ${ENABLE_COVERAGE} --picky-developer ${PREFIX} --with-profiling-data"
151
152 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
153 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
154 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
155 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug --picky-developer -C ${PREFIX}"
156 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
157 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
158 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
159
160 tasks = {
161     "ctdb": [("random-sleep", "../script/random-sleep.sh 300 900", "text/plain"),
162                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
163                ("make", "make all", "text/plain"),
164                ("install", "make install", "text/plain"),
165                ("test", "make autotest", "text/plain"),
166                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
167                ("clean", "make clean", "text/plain")],
168
169     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
170     "samba": [
171                 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
172                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
173                 ("make", "make -j", "text/plain"),
174                 ("test", "make test FAIL_IMMEDIATELY=1 "
175                  "TESTS='--exclude-env=none "
176                  "--exclude-env=nt4_dc "
177                  "--exclude-env=nt4_dc_schannel "
178                  "--exclude-env=nt4_member "
179                  "--exclude-env=ad_dc "
180                  "--exclude-env=ad_dc_backup "
181                  "--exclude-env=ad_dc_ntvfs "
182                  "--exclude-env=ad_dc_default "
183                  "--exclude-env=ad_dc_slowtests "
184                  "--exclude-env=ad_dc_no_nss "
185                  "--exclude-env=ad_dc_no_ntlm "
186                  "--exclude-env=fl2003dc "
187                  "--exclude-env=fl2008dc "
188                  "--exclude-env=fl2008r2dc "
189                  "--exclude-env=ad_member "
190                  "--exclude-env=ad_member_idmap_rid "
191                  "--exclude-env=ad_member_idmap_ad "
192                  "--exclude-env=ad_member_rfc2307 "
193                  "--exclude-env=chgdcpass "
194                  "--exclude-env=vampire_2000_dc "
195                  "--exclude-env=fl2000dc "
196                  "--exclude-env=fileserver "
197                  "--exclude-env=maptoguest "
198                  "--exclude-env=simpleserver "
199                  "--exclude-env=backupfromdc "
200                  "--exclude-env=restoredc "
201                  "--exclude-env=renamedc "
202                  "--exclude-env=offlinebackupdc "
203                  "--exclude-env=labdc "
204                  "--exclude-env=preforkrestartdc "
205                  "--exclude-env=proclimitdc "
206                  "--exclude-env=promoted_dc "
207                  "--exclude-env=vampire_dc "
208                  "--exclude-env=rodc "
209                  "--exclude-env=ad_dc_default "
210                  "--exclude-env=ad_dc_slowtests "
211                  "--exclude-env=schema_pair_dc "
212                  "--exclude-env=schema_dc "
213                  "'",
214                  "text/plain"),
215                 ("lcov", LCOV_CMD, "text/plain"),
216                 ("install", "make install", "text/plain"),
217                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
218                 ("clean", "make clean", "text/plain")],
219
220     "samba-nt4": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
221                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
222                     ("make", "make -j", "text/plain"),
223                     ("test", "make test FAIL_IMMEDIATELY=1 "
224                      "TESTS='"
225                      "--include-env=nt4_dc "
226                      "--include-env=nt4_dc_schannel "
227                      "--include-env=nt4_member "
228                      "'", "text/plain"),
229                     ("lcov", LCOV_CMD, "text/plain"),
230                     ("install", "make install", "text/plain"),
231                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
232                     ("clean", "make clean", "text/plain")],
233
234     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
235                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
236                            ("make", "make -j", "text/plain"),
237                            ("test", "make test FAIL_IMMEDIATELY=1 "
238                             "TESTS='"
239                             "--include-env=fileserver "
240                             "--include-env=maptoguest "
241                             "--include-env=simpleserver "
242                             "'", "text/plain"),
243                            ("lcov", LCOV_CMD, "text/plain"),
244                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
245
246     "samba-ad-member": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
247                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
248                         ("make", "make -j", "text/plain"),
249                         ("test", "make test FAIL_IMMEDIATELY=1 "
250                          "TESTS='"
251                          "--include-env=ad_member "
252                          "--include-env=ad_member_idmap_rid "
253                          "--include-env=ad_member_idmap_ad "
254                          "--include-env=ad_member_rfc2307 "
255                          "'", "text/plain"),
256                         ("lcov", LCOV_CMD, "text/plain"),
257                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
258
259     "samba-ad-dc-1": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
260                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
261                       ("make", "make -j", "text/plain"),
262                       ("test", "make test FAIL_IMMEDIATELY=1 "
263                        "TESTS='--include-env=ad_dc "
264                        "--include-env=ad_dc_no_nss "
265                        "--include-env=ad_dc_no_ntlm "
266                        "'", "text/plain"),
267                       ("lcov", LCOV_CMD, "text/plain"),
268                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
269
270     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
271                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
272                         ("make", "make -j", "text/plain"),
273                         ("test", "make test FAIL_IMMEDIATELY=1 "
274                          "TESTS='"
275                          "--include-env=vampire_dc "
276                          "--include-env=vampire_2000_dc "
277                          "--include-env=rodc "
278                          "'", "text/plain"),
279                         ("lcov", LCOV_CMD, "text/plain"),
280                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
281
282     "samba-ad-dc-3": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
283                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
284                         ("make", "make -j", "text/plain"),
285                         ("test", "make test FAIL_IMMEDIATELY=1 "
286                          "TESTS='"
287                          "--include-env=promoted_dc "
288                          "--include-env=chgdcpass "
289                          "--include-env=preforkrestartdc "
290                          "--include-env=proclimitdc "
291                          "'", "text/plain"),
292                         ("lcov", LCOV_CMD, "text/plain"),
293                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
294
295     "samba-ad-dc-4": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
296                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
297                         ("make", "make -j", "text/plain"),
298                         ("test", "make test FAIL_IMMEDIATELY=1 "
299                          "TESTS='"
300                          "--include-env=fl2000dc "
301                          "--include-env=fl2003dc "
302                          "--include-env=fl2008dc "
303                          "--include-env=fl2008r2dc "
304                          "'", "text/plain"),
305                         ("lcov", LCOV_CMD, "text/plain"),
306                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
307
308     "samba-ad-dc-5": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
309                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
310                         ("make", "make -j", "text/plain"),
311                         ("test", "make test FAIL_IMMEDIATELY=1 "
312                          "TESTS='"
313                          "--include-env=ad_dc_default "
314                          "'", "text/plain"),
315                         ("lcov", LCOV_CMD, "text/plain"),
316                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
317
318     "samba-ad-dc-6": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
319                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
320                         ("make", "make -j", "text/plain"),
321                         ("test", "make test FAIL_IMMEDIATELY=1 "
322                          "TESTS='"
323                          "--include-env=ad_dc_slowtests "
324                          "'", "text/plain"),
325                         ("lcov", LCOV_CMD, "text/plain"),
326                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
327
328
329     "samba-schemaupgrade": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
330                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
331                         ("make", "make -j", "text/plain"),
332                         ("test", "make test FAIL_IMMEDIATELY=1 "
333                          "TESTS='"
334                          "--include-env=schema_dc "
335                          "--include-env=schema_pair_dc "
336                          "'", "text/plain"),
337                         ("lcov", LCOV_CMD, "text/plain"),
338                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
339
340     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
341     # This is currently the longest task, so we don't randomly delay it.
342     "samba-ad-dc-ntvfs": [
343                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
344                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
345                       ("make", "make -j", "text/plain"),
346                       ("test", "make test FAIL_IMMEDIATELY=1 "
347                        "TESTS='"
348                        "--include-env=ad_dc_ntvfs "
349                        "'", "text/plain"),
350                       ("lcov", LCOV_CMD, "text/plain"),
351                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
352
353     # run the backup/restore testenvs separately as they're fairly standalone
354     # (and CI seems to max out at ~8 different DCs running at once)
355     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
356                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
357                         ("make", "make -j", "text/plain"),
358                         ("test", "make test FAIL_IMMEDIATELY=1 "
359                          "TESTS='--include-env=backupfromdc "
360                          "--include-env=restoredc "
361                          "--include-env=renamedc "
362                          "--include-env=offlinebackupdc "
363                          "--include-env=labdc "
364                          "--include-env=ad_dc_backup "
365                          "'", "text/plain"),
366                         ("lcov", LCOV_CMD, "text/plain"),
367                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
368
369     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
370                           ("make", "make -j", "text/plain"),
371                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain"),
372                           ("lcov", LCOV_CMD, "text/plain"),
373                         ],
374
375     # Test cross-compile infrastructure
376     "samba-xc": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
377                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
378                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
379                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
380                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
381                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
382                    ("compare-results", "script/compare_cc_results.py "
383                     "./bin/c4che/default{} "
384                     "./bin-xe/c4che/default{} "
385                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
386
387     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
388     "samba-o3": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
389                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
390                    ("make", "make -j", "text/plain"),
391                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
392                     "TESTS='--include-env=ad_dc'", "text/plain"),
393                    ("lcov", LCOV_CMD, "text/plain"),
394                    ("install", "make install", "text/plain"),
395                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
396                    ("clean", "make clean", "text/plain")],
397
398     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
399
400                      # make sure we have tdb around:
401                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
402                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
403                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
404
405
406                      # build samba with cluster support (also building ctdb):
407                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
408                      ("samba-make", "make", "text/plain"),
409                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
410                      ("samba-install", "make install", "text/plain"),
411                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
412
413                      # clean up:
414                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
415                      ("clean", "make clean", "text/plain"),
416                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
417
418     "samba-libs": [
419                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
420                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
421                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
422                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
423
424                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
425                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
426                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
427
428                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
429                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
430                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
431
432                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
433                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
434                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
435
436                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
437                       ("nondevel-make", "make -j", "text/plain"),
438                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
439                       ("nondevel-install", "make install", "text/plain"),
440                       ("nondevel-dist", "make dist", "text/plain"),
441
442                       # retry with all modules shared
443                       ("allshared-distclean", "make distclean", "text/plain"),
444                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
445                       ("allshared-make", "make -j", "text/plain")],
446
447     "samba-none-env": [
448                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
449                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
450                       ("make", "make -j", "text/plain"),
451                       ("test", "make test "
452                        "FAIL_IMMEDIATELY=1 "
453                        "TESTS='--include-env=none'",
454                        "text/plain"),
455                       ("lcov", LCOV_CMD, "text/plain")],
456
457     "samba-static": [
458                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
459                       # build with all modules static
460                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
461                       ("allstatic-make", "make -j", "text/plain"),
462                       ("allstatic-test", "make test "
463                        "FAIL_IMMEDIATELY=1 "
464                        "TESTS='samba3.smb2.create.*nt4_dc'",
465                        "text/plain"),
466                       ("lcov", LCOV_CMD, "text/plain"),
467
468                       # retry without any required modules
469                       ("none-distclean", "make distclean", "text/plain"),
470                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
471                       ("none-make", "make -j", "text/plain"),
472
473                       # retry with nonshared smbd and smbtorture
474                       ("nonshared-distclean", "make distclean", "text/plain"),
475                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
476                       ("nonshared-make", "make -j", "text/plain")],
477
478     "samba-systemkrb5": [
479                       ("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
480                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --with-experimental-mit-ad-dc", "text/plain"),
481                       ("make", "make -j", "text/plain"),
482                       # we currently cannot run a full make test, a limited list of tests could be run
483                       # via "make test TESTS=sometests"
484                       ("test", "make test FAIL_IMMEDIATELY=1 "
485                        "TESTS='--include-env=ktest'", "text/plain"),
486                       ("lcov", LCOV_CMD, "text/plain"),
487                       ("install", "make install", "text/plain"),
488                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
489                       ("clean", "make clean", "text/plain")
490                       ],
491
492     # Test Samba without python still builds.  When this test fails
493     # due to more use of Python, the expectations is that the newly
494     # failing part of the code should be disabled when
495     # --disable-python is set (rather than major work being done to
496     # support this environment).  The target here is for vendors
497     # shipping a minimal smbd.
498     "samba-nopython": [
499                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
500                       ("configure", "./configure.developer ${ENABLE_COVERAGE} --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
501                       ("make", "make -j", "text/plain"),
502                       ("install", "make install", "text/plain"),
503                       ("find-python", "script/find_python.sh ${PREFIX}", "text/plain"),
504                       ("test", "make test-nopython", "text/plain"),
505                       ("lcov", LCOV_CMD, "text/plain"),
506                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
507                       ("clean", "make clean", "text/plain"),
508
509                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
510                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
511                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
512
513                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
514                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
515                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
516
517                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
518                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
519                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
520
521                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
522                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
523                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
524
525                       # retry against installed library packages
526                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
527                       ("libs-make", "make -j", "text/plain"),
528                       ("libs-install", "make install", "text/plain"),
529                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
530                       ("libs-clean", "make clean", "text/plain")
531                       ],
532
533     # check we can do the same thing using python2
534     "samba-nopython-py2": [
535                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
536                       ("configure", "PYTHON=python2 ./configure.developer ${ENABLE_COVERAGE} --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
537                       ("make", "PYTHON=python2 make -j", "text/plain"),
538                       ("install", "PYTHON=python2 make install", "text/plain"),
539                       ("find-python", "script/find_python.sh ${PREFIX}", "text/plain"),
540                       ("test", "make test-nopython", "text/plain"),
541                       ("lcov", LCOV_CMD, "text/plain"),
542                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
543                       ("clean", "PYTHON=python2 make clean", "text/plain"),
544
545                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
546                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
547                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
548
549                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
550                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
551                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
552
553                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
554                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
555                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
556
557                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
558                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
559                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
560
561                       # retry against installed library packages
562                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
563                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
564                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
565                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
566                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
567                       ],
568
569     "ldb": [
570               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
571               ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}", "text/plain"),
572               ("make", "make", "text/plain"),
573               ("install", "make install", "text/plain"),
574               ("test", "make test", "text/plain"),
575               ("lcov", LCOV_CMD, "text/plain"),
576               ("clean", "make clean", "text/plain"),
577               ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
578               ("make-no-lmdb", "make", "text/plain"),
579               ("test-no-lmdb", "make test", "text/plain"),
580               ("lcov-no-lmdb", LCOV_CMD, "text/plain"),
581               ("install-no-lmdb", "make install", "text/plain"),
582               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
583               ("distcheck", "make distcheck", "text/plain"),
584               ("clean", "make clean", "text/plain")],
585
586     "tdb": [
587               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
588               ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}", "text/plain"),
589               ("make", "make", "text/plain"),
590               ("install", "make install", "text/plain"),
591               ("test", "make test", "text/plain"),
592               ("lcov", LCOV_CMD, "text/plain"),
593               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
594               ("distcheck", "make distcheck", "text/plain"),
595               ("clean", "make clean", "text/plain")],
596
597     "talloc": [
598                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
599                  ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}", "text/plain"),
600                  ("make", "make", "text/plain"),
601                  ("install", "make install", "text/plain"),
602                  ("test", "make test", "text/plain"),
603                  ("lcov", LCOV_CMD, "text/plain"),
604                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
605                  ("distcheck", "make distcheck", "text/plain"),
606                  ("clean", "make clean", "text/plain")],
607
608     "replace": [
609                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
610                   ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}", "text/plain"),
611                   ("make", "make", "text/plain"),
612                   ("install", "make install", "text/plain"),
613                   ("test", "make test", "text/plain"),
614                   ("lcov", LCOV_CMD, "text/plain"),
615                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
616                   ("distcheck", "make distcheck", "text/plain"),
617                   ("clean", "make clean", "text/plain")],
618
619     "tevent": [
620                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
621                  ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}", "text/plain"),
622                  ("make", "make", "text/plain"),
623                  ("install", "make install", "text/plain"),
624                  ("test", "make test", "text/plain"),
625                  ("lcov", LCOV_CMD, "text/plain"),
626                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
627                  ("distcheck", "make distcheck", "text/plain"),
628                  ("clean", "make clean", "text/plain")],
629
630     "pidl": [
631         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
632         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
633         ("touch", "touch *.yp", "text/plain"),
634         ("make", "make", "text/plain"),
635         ("test", "make test", "text/plain"),
636         ("install", "make install", "text/plain"),
637         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
638         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
639         ("clean", "make clean", "text/plain")],
640
641
642     # these are useful for debugging autobuild
643     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
644     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
645
646
647 }
648
649
650 def do_print(msg):
651     print("%s" % msg)
652     sys.stdout.flush()
653     sys.stderr.flush()
654
655
656 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
657     if show is None:
658         show = options.verbose
659     if show:
660         do_print("Running: '%s' in '%s'" % (cmd, dir))
661     if output:
662         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
663     elif checkfail:
664         return check_call(cmd, shell=True, cwd=dir)
665     else:
666         return call(cmd, shell=True, cwd=dir)
667
668
669 class builder(object):
670     '''handle build of one directory'''
671
672     def __init__(self, name, sequence, cp=True):
673         self.name = name
674         if name in builddirs:
675             self.dir = builddirs[name]
676         else:
677             self.dir = "."
678
679         self.tag = self.name.replace('/', '_')
680         self.sequence = sequence
681         self.next = 0
682         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
683         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
684         if options.verbose:
685             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
686             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
687         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
688         self.stdout = open(self.stdout_path, 'w')
689         self.stderr = open(self.stderr_path, 'w')
690         self.stdin  = open("/dev/null", 'r')
691         self.test_source_dir = "%s/%s" % (testbase, self.tag)
692         self.prefix = "%s/%s" % (test_prefix, self.tag)
693         run_cmd("rm -rf %s" % self.test_source_dir)
694         run_cmd("rm -rf %s" % self.prefix)
695         if cp:
696             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
697         else:
698             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
699         self.start_next()
700
701     def start_next(self):
702         if self.next == len(self.sequence):
703             if not options.nocleanup:
704                 run_cmd("rm -rf %s" % self.test_source_dir)
705                 run_cmd("rm -rf %s" % self.prefix)
706             do_print('%s: Completed OK' % self.name)
707             self.done = True
708             return
709         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
710         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
711         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
712         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
713         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
714         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
715         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
716         self.cmd = self.cmd.replace("${NAME}", self.name)
717         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
718 #        if self.output_mime_type == "text/x-subunit":
719 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
720         cwd = os.getcwd()
721         os.chdir("%s/%s" % (self.test_source_dir, self.dir))
722         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
723         self.proc = Popen(self.cmd, shell=True,
724                           close_fds=True,
725                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
726         os.chdir(cwd)
727         self.next += 1
728
729
730 class buildlist(object):
731     '''handle build of multiple directories'''
732
733     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
734         global tasks
735         self.tlist = []
736         self.tail_proc = None
737         self.retry = None
738         if tasknames == []:
739             if options.restrict_tests:
740                 tasknames = ["samba-test-only"]
741             else:
742                 tasknames = defaulttasks
743         else:
744             # If we are only running one test,
745             # do not sleep randomly to wait for it to start
746             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
747
748         for n in tasknames:
749             b = builder(n, tasks[n], cp=n is not "pidl")
750             self.tlist.append(b)
751         if options.retry:
752             rebase_remote = "rebaseon"
753             retry_task = [("retry",
754                             '''set -e
755                             git remote add -t %s %s %s
756                             git fetch %s
757                             while :; do
758                               sleep 60
759                               git describe %s/%s > old_remote_branch.desc
760                               git fetch %s
761                               git describe %s/%s > remote_branch.desc
762                               diff old_remote_branch.desc remote_branch.desc
763                             done
764                            ''' % (
765                                 rebase_branch, rebase_remote, rebase_url,
766                                rebase_remote,
767                                rebase_remote, rebase_branch,
768                                rebase_remote,
769                                rebase_remote, rebase_branch
770                             ),
771                             "test/plain")]
772
773             self.retry = builder('retry', retry_task, cp=False)
774             self.need_retry = False
775
776     def kill_kids(self):
777         if self.tail_proc is not None:
778             self.tail_proc.terminate()
779             self.tail_proc.wait()
780             self.tail_proc = None
781         if self.retry is not None:
782             self.retry.proc.terminate()
783             self.retry.proc.wait()
784             self.retry = None
785         for b in self.tlist:
786             if b.proc is not None:
787                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
788                 b.proc.terminate()
789                 b.proc.wait()
790                 b.proc = None
791
792     def wait_one(self):
793         while True:
794             none_running = True
795             for b in self.tlist:
796                 if b.proc is None:
797                     continue
798                 none_running = False
799                 b.status = b.proc.poll()
800                 if b.status is None:
801                     continue
802                 b.proc = None
803                 return b
804             if options.retry:
805                 ret = self.retry.proc.poll()
806                 if ret is not None:
807                     self.need_retry = True
808                     self.retry = None
809                     return None
810             if none_running:
811                 return None
812             time.sleep(0.1)
813
814     def run(self):
815         while True:
816             b = self.wait_one()
817             if options.retry and self.need_retry:
818                 self.kill_kids()
819                 do_print("retry needed")
820                 return (0, None, None, None, "retry")
821             if b is None:
822                 break
823             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
824                 self.kill_kids()
825                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
826             b.start_next()
827         self.kill_kids()
828         return (0, None, None, None, "All OK")
829
830     def write_system_info(self):
831         filename = 'system-info.txt'
832         f = open(filename, 'w')
833         for cmd in ['uname -a',
834                     'lsb_release -a',
835                     'free',
836                     'mount',
837                     'cat /proc/cpuinfo',
838                     'cc --version',
839                     'df -m .',
840                     'df -m %s' % testbase]:
841             out = run_cmd(cmd, output=True, checkfail=False)
842             print('### %s' % cmd, file=f)
843             print(out.decode('utf8', 'backslashreplace'), file=f)
844             print(file=f)
845         f.close()
846         return filename
847
848     def tarlogs(self, fname):
849         tar = tarfile.open(fname, "w:gz")
850         for b in self.tlist:
851             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
852             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
853         if os.path.exists("autobuild.log"):
854             tar.add("autobuild.log")
855         sys_info = self.write_system_info()
856         tar.add(sys_info)
857         tar.close()
858
859     def remove_logs(self):
860         for b in self.tlist:
861             os.unlink(b.stdout_path)
862             os.unlink(b.stderr_path)
863
864     def start_tail(self):
865         cmd = ["tail", "-f"]
866         for b in self.tlist:
867             cmd.append(b.stdout_path)
868             cmd.append(b.stderr_path)
869         self.tail_proc = Popen(cmd, close_fds=True)
870
871
872 def cleanup():
873     if options.nocleanup:
874         return
875     run_cmd("stat %s || true" % test_tmpdir, show=True)
876     run_cmd("stat %s" % testbase, show=True)
877     do_print("Cleaning up %r" % cleanup_list)
878     for d in cleanup_list:
879         run_cmd("rm -rf %s" % d)
880
881
882 def daemonize(logfile):
883     pid = os.fork()
884     if pid == 0:  # Parent
885         os.setsid()
886         pid = os.fork()
887         if pid != 0:  # Actual daemon
888             os._exit(0)
889     else:  # Grandparent
890         os._exit(0)
891
892     import resource      # Resource usage information.
893     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
894     if maxfd == resource.RLIM_INFINITY:
895         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
896     for fd in range(0, maxfd):
897         try:
898             os.close(fd)
899         except OSError:
900             pass
901     os.open(logfile, os.O_RDWR | os.O_CREAT)
902     os.dup2(0, 1)
903     os.dup2(0, 2)
904
905
906 def write_pidfile(fname):
907     '''write a pid file, cleanup on exit'''
908     f = open(fname, mode='w')
909     f.write("%u\n" % os.getpid())
910     f.close()
911
912
913 def rebase_tree(rebase_url, rebase_branch="master"):
914     rebase_remote = "rebaseon"
915     do_print("Rebasing on %s" % rebase_url)
916     run_cmd("git describe HEAD", show=True, dir=test_master)
917     run_cmd("git remote add -t %s %s %s" %
918             (rebase_branch, rebase_remote, rebase_url),
919             show=True, dir=test_master)
920     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
921     if options.fix_whitespace:
922         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
923                 (rebase_remote, rebase_branch),
924                 show=True, dir=test_master)
925     else:
926         run_cmd("git rebase --force-rebase %s/%s" %
927                 (rebase_remote, rebase_branch),
928                 show=True, dir=test_master)
929     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
930                    (rebase_remote, rebase_branch),
931                    dir=test_master, output=True)
932     if diff == '':
933         do_print("No differences between HEAD and %s/%s - exiting" %
934                  (rebase_remote, rebase_branch))
935         sys.exit(0)
936     run_cmd("git describe %s/%s" %
937             (rebase_remote, rebase_branch),
938             show=True, dir=test_master)
939     run_cmd("git describe HEAD", show=True, dir=test_master)
940     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
941             (rebase_remote, rebase_branch),
942             show=True, dir=test_master)
943
944
945 def push_to(push_url, push_branch="master"):
946     push_remote = "pushto"
947     do_print("Pushing to %s" % push_url)
948     if options.mark:
949         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
950         run_cmd("git commit --amend -c HEAD", dir=test_master)
951         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
952         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
953     run_cmd("git remote add -t %s %s %s" %
954             (push_branch, push_remote, push_url),
955             show=True, dir=test_master)
956     run_cmd("git push %s +HEAD:%s" %
957             (push_remote, push_branch),
958             show=True, dir=test_master)
959
960
961 def send_email(subject, text, log_tar):
962     if options.email is None:
963         do_print("not sending email because the recipient is not set")
964         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
965                  (subject, text))
966         return
967     outer = MIMEMultipart()
968     outer['Subject'] = subject
969     outer['To'] = options.email
970     outer['From'] = options.email_from
971     outer['Date'] = email.utils.formatdate(localtime=True)
972     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
973     outer.attach(MIMEText(text, 'plain'))
974     if options.attach_logs:
975         fp = open(log_tar, 'rb')
976         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
977         fp.close()
978         # Set the filename parameter
979         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
980         outer.attach(msg)
981     content = outer.as_string()
982     s = smtplib.SMTP(options.email_server)
983     email_user = os.getenv('SMTP_USERNAME')
984     email_password = os.getenv('SMTP_PASSWORD')
985     if email_user is not None:
986         s.starttls()
987         s.login(email_user, email_password)
988
989     s.sendmail(options.email_from, [options.email], content)
990     s.set_debuglevel(1)
991     s.quit()
992
993
994 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
995                   elapsed_time, log_base=None, add_log_tail=True):
996     '''send an email to options.email about the failure'''
997     elapsed_minutes = elapsed_time / 60.0
998     if log_base is None:
999         log_base = gitroot
1000     text = '''
1001 Dear Developer,
1002
1003 Your autobuild on %s failed after %.1f minutes
1004 when trying to test %s with the following error:
1005
1006    %s
1007
1008 the autobuild has been abandoned. Please fix the error and resubmit.
1009
1010 A summary of the autobuild process is here:
1011
1012   %s/autobuild.log
1013 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1014
1015     if options.restrict_tests:
1016         text += """
1017 The build was restricted to tests matching %s\n""" % options.restrict_tests
1018
1019     if failed_task != 'rebase':
1020         text += '''
1021 You can see logs of the failed task here:
1022
1023   %s/%s.stdout
1024   %s/%s.stderr
1025
1026 or you can get full logs of all tasks in this job here:
1027
1028   %s/logs.tar.gz
1029
1030 The top commit for the tree that was built was:
1031
1032 %s
1033
1034 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1035
1036     if add_log_tail:
1037         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1038         lines = f.readlines()
1039         log_tail = "".join(lines[-50:])
1040         num_lines = len(lines)
1041         if num_lines < 50:
1042             # Also include stderr (compile failures) if < 50 lines of stdout
1043             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1044             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1045
1046         text += '''
1047 The last 50 lines of log messages:
1048
1049 %s
1050     ''' % log_tail
1051         f.close()
1052
1053     logs = os.path.join(gitroot, 'logs.tar.gz')
1054     send_email('autobuild[%s] failure on %s for task %s during %s'
1055                % (options.branch, platform.node(), failed_task, failed_stage),
1056                text, logs)
1057
1058
1059 def email_success(elapsed_time, log_base=None):
1060     '''send an email to options.email about a successful build'''
1061     if log_base is None:
1062         log_base = gitroot
1063     text = '''
1064 Dear Developer,
1065
1066 Your autobuild on %s has succeeded after %.1f minutes.
1067
1068 ''' % (platform.node(), elapsed_time / 60.)
1069
1070     if options.restrict_tests:
1071         text += """
1072 The build was restricted to tests matching %s\n""" % options.restrict_tests
1073
1074     if options.keeplogs:
1075         text += '''
1076
1077 you can get full logs of all tasks in this job here:
1078
1079   %s/logs.tar.gz
1080
1081 ''' % log_base
1082
1083     text += '''
1084 The top commit for the tree that was built was:
1085
1086 %s
1087 ''' % top_commit_msg
1088
1089     logs = os.path.join(gitroot, 'logs.tar.gz')
1090     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1091                text, logs)
1092
1093
1094 # get the top commit message, for emails
1095 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1096 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
1097
1098 try:
1099     os.makedirs(testbase)
1100 except Exception as reason:
1101     raise Exception("Unable to create %s : %s" % (testbase, reason))
1102 cleanup_list.append(testbase)
1103
1104 if options.daemon:
1105     logfile = os.path.join(testbase, "log")
1106     do_print("Forking into the background, writing progress to %s" % logfile)
1107     daemonize(logfile)
1108
1109 write_pidfile(gitroot + "/autobuild.pid")
1110
1111 start_time = time.time()
1112
1113 while True:
1114     try:
1115         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1116         os.makedirs(test_tmpdir)
1117         # The waf uninstall code removes empty directories all the way
1118         # up the tree.  Creating a file in test_tmpdir stops it from
1119         # being removed.
1120         run_cmd("touch %s" % os.path.join(test_tmpdir,
1121                                           ".directory-is-not-empty"), show=True)
1122         run_cmd("stat %s" % test_tmpdir, show=True)
1123         run_cmd("stat %s" % testbase, show=True)
1124         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1125     except Exception:
1126         cleanup()
1127         raise
1128
1129     try:
1130         try:
1131             if options.rebase is not None:
1132                 rebase_tree(options.rebase, rebase_branch=options.branch)
1133         except Exception:
1134             cleanup_list.append(gitroot + "/autobuild.pid")
1135             cleanup()
1136             elapsed_time = time.time() - start_time
1137             email_failure(-1, 'rebase', 'rebase', 'rebase',
1138                           'rebase on %s failed' % options.branch,
1139                           elapsed_time, log_base=options.log_base)
1140             sys.exit(1)
1141         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1142         if options.tail:
1143             blist.start_tail()
1144         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1145         if status != 0 or errstr != "retry":
1146             break
1147         cleanup()
1148     except Exception:
1149         cleanup()
1150         raise
1151
1152 cleanup_list.append(gitroot + "/autobuild.pid")
1153
1154 do_print(errstr)
1155
1156 blist.kill_kids()
1157 if options.tail:
1158     do_print("waiting for tail to flush")
1159     time.sleep(1)
1160
1161 elapsed_time = time.time() - start_time
1162 if status == 0:
1163     if options.passcmd is not None:
1164         do_print("Running passcmd: %s" % options.passcmd)
1165         run_cmd(options.passcmd, dir=test_master)
1166     if options.pushto is not None:
1167         push_to(options.pushto, push_branch=options.branch)
1168     if options.keeplogs or options.attach_logs:
1169         blist.tarlogs("logs.tar.gz")
1170         do_print("Logs in logs.tar.gz")
1171     if options.always_email:
1172         email_success(elapsed_time, log_base=options.log_base)
1173     blist.remove_logs()
1174     cleanup()
1175     do_print(errstr)
1176     sys.exit(0)
1177
1178 # something failed, gather a tar of the logs
1179 blist.tarlogs("logs.tar.gz")
1180
1181 if options.email is not None:
1182     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1183                   elapsed_time, log_base=options.log_base)
1184 else:
1185     elapsed_minutes = elapsed_time / 60.0
1186     print('''
1187
1188 ####################################################################
1189
1190 AUTOBUILD FAILURE
1191
1192 Your autobuild[%s] on %s failed after %.1f minutes
1193 when trying to test %s with the following error:
1194
1195    %s
1196
1197 the autobuild has been abandoned. Please fix the error and resubmit.
1198
1199 ####################################################################
1200
1201 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1202
1203 cleanup()
1204 do_print(errstr)
1205 do_print("Logs in logs.tar.gz")
1206 sys.exit(status)