7e7a4951df2967875c413fd8b37316281d3a6e08
[bbaumbach/samba-autobuild/.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 from optparse import OptionParser
13 import smtplib
14 import email
15 from email.mime.text import MIMEText
16 from email.mime.base import MIMEBase
17 from email.mime.application import MIMEApplication
18 from email.mime.multipart import MIMEMultipart
19 from distutils.sysconfig import get_python_lib
20 import platform
21
22 try:
23     from waflib.Build import CACHE_SUFFIX
24 except ImportError:
25     sys.path.insert(0, "./third_party/waf")
26     from waflib.Build import CACHE_SUFFIX
27
28
29 os.environ["PYTHONUNBUFFERED"] = "1"
30
31 # This speeds up testing remarkably.
32 os.environ['TDB_NO_FSYNC'] = '1'
33
34
35 def find_git_root():
36     '''get to the top of the git repo'''
37     p = os.getcwd()
38     while p != '/':
39         if os.path.isdir(os.path.join(p, ".git")):
40             return p
41         p = os.path.abspath(os.path.join(p, '..'))
42     return None
43
44
45 gitroot = find_git_root()
46 if gitroot is None:
47     raise Exception("Failed to find git root")
48
49
50 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
51
52 parser = OptionParser()
53 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
54 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
55 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
56 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
57                   default=def_testbase)
58 parser.add_option("", "--passcmd", help="command to run on success", default=None)
59 parser.add_option("", "--verbose", help="show all commands as they are run",
60                   default=False, action="store_true")
61 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
62                   default=None, type='str')
63 parser.add_option("", "--pushto", help="push to a git url on success",
64                   default=None, type='str')
65 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
66                   default=False, action="store_true")
67 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
68                   default=False, action="store_true")
69 parser.add_option("", "--retry", help="automatically retry if master changes",
70                   default=False, action="store_true")
71 parser.add_option("", "--email", help="send email to the given address on failure",
72                   type='str', default=None)
73 parser.add_option("", "--email-from", help="send email from the given address",
74                   type='str', default="autobuild@samba.org")
75 parser.add_option("", "--email-server", help="send email via the given server",
76                   type='str', default='localhost')
77 parser.add_option("", "--always-email", help="always send email, even on success",
78                   action="store_true")
79 parser.add_option("", "--daemon", help="daemonize after initial setup",
80                   action="store_true")
81 parser.add_option("", "--branch", help="the branch to work on (default=master)",
82                   default="master", type='str')
83 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
84                   default=gitroot, type='str')
85 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
86                   default=False, action="store_true")
87 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
88                   default='')
89 parser.add_option("--enable-coverage", dest='enable_coverage',
90                   action="store_const", const='--enable-coverage', default='',
91                   help="Add --enable-coverage option while configure")
92
93 (options, args) = parser.parse_args()
94
95 if options.retry:
96     if options.rebase is None:
97         raise Exception('You can only use --retry if you also rebase')
98
99 testbase = "%s/b%u" % (options.testbase, os.getpid())
100 test_master = "%s/master" % testbase
101 test_prefix = "%s/prefix" % testbase
102 test_tmpdir = "%s/tmp" % testbase
103 os.environ['TMPDIR'] = test_tmpdir
104
105
106 cleanup_list = []
107
108 builddirs = {
109     "ctdb": "ctdb",
110     "samba": ".",
111     "samba-nt4": ".",
112     "samba-fileserver": ".",
113     "samba-ad-member": ".",
114     "samba-xc": ".",
115     "samba-o3": ".",
116     "samba-ctdb": ".",
117     "samba-libs": ".",
118     "samba-static": ".",
119     "samba-none-env": ".",
120     "samba-ad-dc-1": ".",
121     "samba-ad-dc-2": ".",
122     "samba-ad-dc-3": ".",
123     "samba-ad-dc-4": ".",
124     "samba-ad-dc-5": ".",
125     "samba-ad-dc-6": ".",
126     "samba-ad-dc-ntvfs": ".",
127     "samba-ad-dc-backup": ".",
128     "samba-systemkrb5": ".",
129     "samba-nopython": ".",
130     "samba-nopython-py2": ".",
131     "samba-schemaupgrade": ".",
132     "ldb": "lib/ldb",
133     "tdb": "lib/tdb",
134     "talloc": "lib/talloc",
135     "replace": "lib/replace",
136     "tevent": "lib/tevent",
137     "pidl": "pidl"
138 }
139
140 defaulttasks = builddirs.keys()
141
142 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
143     defaulttasks.remove("samba-o3")
144
145 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
146 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
147
148 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
149 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
150 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
151 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
152 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
153 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
154 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
155
156 tasks = {
157     "ctdb": [("random-sleep", "../script/random-sleep.sh 300 900", "text/plain"),
158                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
159                ("make", "make all", "text/plain"),
160                ("install", "make install", "text/plain"),
161                ("test", "make autotest", "text/plain"),
162                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
163                ("clean", "make clean", "text/plain")],
164
165     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
166     "samba": [
167                 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
168                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
169                 ("make", "make -j", "text/plain"),
170                 ("test", "make test FAIL_IMMEDIATELY=1 "
171                  "TESTS='--exclude-env=none "
172                  "--exclude-env=nt4_dc "
173                  "--exclude-env=nt4_dc_schannel "
174                  "--exclude-env=nt4_member "
175                  "--exclude-env=ad_dc "
176                  "--exclude-env=ad_dc_backup "
177                  "--exclude-env=ad_dc_ntvfs "
178                  "--exclude-env=ad_dc_default "
179                  "--exclude-env=ad_dc_slowtests "
180                  "--exclude-env=ad_dc_no_nss "
181                  "--exclude-env=ad_dc_no_ntlm "
182                  "--exclude-env=fl2003dc "
183                  "--exclude-env=fl2008dc "
184                  "--exclude-env=fl2008r2dc "
185                  "--exclude-env=ad_member "
186                  "--exclude-env=ad_member_idmap_rid "
187                  "--exclude-env=ad_member_idmap_ad "
188                  "--exclude-env=ad_member_rfc2307 "
189                  "--exclude-env=chgdcpass "
190                  "--exclude-env=vampire_2000_dc "
191                  "--exclude-env=fl2000dc "
192                  "--exclude-env=fileserver "
193                  "--exclude-env=maptoguest "
194                  "--exclude-env=simpleserver "
195                  "--exclude-env=backupfromdc "
196                  "--exclude-env=restoredc "
197                  "--exclude-env=renamedc "
198                  "--exclude-env=offlinebackupdc "
199                  "--exclude-env=labdc "
200                  "--exclude-env=preforkrestartdc "
201                  "--exclude-env=proclimitdc "
202                  "--exclude-env=promoted_dc "
203                  "--exclude-env=vampire_dc "
204                  "--exclude-env=rodc "
205                  "--exclude-env=ad_dc_default "
206                  "--exclude-env=ad_dc_slowtests "
207                  "--exclude-env=schema_pair_dc "
208                  "--exclude-env=schema_dc "
209                  "'",
210                  "text/plain"),
211                 ("install", "make install", "text/plain"),
212                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
213                 ("clean", "make clean", "text/plain")],
214
215     "samba-nt4": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
216                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
217                     ("make", "make -j", "text/plain"),
218                     ("test", "make test FAIL_IMMEDIATELY=1 "
219                      "TESTS='"
220                      "--include-env=nt4_dc "
221                      "--include-env=nt4_dc_schannel "
222                      "--include-env=nt4_member "
223                      "'", "text/plain"),
224                     ("install", "make install", "text/plain"),
225                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
226                     ("clean", "make clean", "text/plain")],
227
228     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
229                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
230                            ("make", "make -j", "text/plain"),
231                            ("test", "make test FAIL_IMMEDIATELY=1 "
232                             "TESTS='"
233                             "--include-env=fileserver "
234                             "--include-env=maptoguest "
235                             "--include-env=simpleserver "
236                             "'", "text/plain"),
237                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
238
239     "samba-ad-member": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
240                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
241                         ("make", "make -j", "text/plain"),
242                         ("test", "make test FAIL_IMMEDIATELY=1 "
243                          "TESTS='"
244                          "--include-env=ad_member "
245                          "--include-env=ad_member_idmap_rid "
246                          "--include-env=ad_member_idmap_ad "
247                          "--include-env=ad_member_rfc2307 "
248                          "'", "text/plain"),
249                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
250
251     "samba-ad-dc-1": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
252                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
253                       ("make", "make -j", "text/plain"),
254                       ("test", "make test FAIL_IMMEDIATELY=1 "
255                        "TESTS='--include-env=ad_dc "
256                        "--include-env=ad_dc_no_nss "
257                        "--include-env=ad_dc_no_ntlm "
258                        "'", "text/plain"),
259                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
260
261     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
262                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
263                         ("make", "make -j", "text/plain"),
264                         ("test", "make test FAIL_IMMEDIATELY=1 "
265                          "TESTS='"
266                          "--include-env=vampire_dc "
267                          "--include-env=vampire_2000_dc "
268                          "--include-env=rodc "
269                          "'", "text/plain"),
270                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
271
272     "samba-ad-dc-3": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
273                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
274                         ("make", "make -j", "text/plain"),
275                         ("test", "make test FAIL_IMMEDIATELY=1 "
276                          "TESTS='"
277                          "--include-env=promoted_dc "
278                          "--include-env=chgdcpass "
279                          "--include-env=preforkrestartdc "
280                          "--include-env=proclimitdc "
281                          "'", "text/plain"),
282                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
283
284     "samba-ad-dc-4": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
285                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
286                         ("make", "make -j", "text/plain"),
287                         ("test", "make test FAIL_IMMEDIATELY=1 "
288                          "TESTS='"
289                          "--include-env=fl2000dc "
290                          "--include-env=fl2003dc "
291                          "--include-env=fl2008dc "
292                          "--include-env=fl2008r2dc "
293                          "'", "text/plain"),
294                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
295
296     "samba-ad-dc-5": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
297                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
298                         ("make", "make -j", "text/plain"),
299                         ("test", "make test FAIL_IMMEDIATELY=1 "
300                          "TESTS='"
301                          "--include-env=ad_dc_default "
302                          "'", "text/plain"),
303                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
304
305     "samba-ad-dc-6": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
306                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
307                         ("make", "make -j", "text/plain"),
308                         ("test", "make test FAIL_IMMEDIATELY=1 "
309                          "TESTS='"
310                          "--include-env=ad_dc_slowtests "
311                          "'", "text/plain"),
312                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
313
314
315     "samba-schemaupgrade": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
316                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
317                         ("make", "make -j", "text/plain"),
318                         ("test", "make test FAIL_IMMEDIATELY=1 "
319                          "TESTS='"
320                          "--include-env=schema_dc "
321                          "--include-env=schema_pair_dc "
322                          "'", "text/plain"),
323                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
324
325     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
326     # This is currently the longest task, so we don't randomly delay it.
327     "samba-ad-dc-ntvfs": [
328                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
329                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
330                       ("make", "make -j", "text/plain"),
331                       ("test", "make test FAIL_IMMEDIATELY=1 "
332                        "TESTS='"
333                        "--include-env=ad_dc_ntvfs "
334                        "'", "text/plain"),
335                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
336
337     # run the backup/restore testenvs separately as they're fairly standalone
338     # (and CI seems to max out at ~8 different DCs running at once)
339     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
340                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
341                         ("make", "make -j", "text/plain"),
342                         ("test", "make test FAIL_IMMEDIATELY=1 "
343                          "TESTS='--include-env=backupfromdc "
344                          "--include-env=restoredc "
345                          "--include-env=renamedc "
346                          "--include-env=offlinebackupdc "
347                          "--include-env=labdc "
348                          "--include-env=ad_dc_backup "
349                          "'", "text/plain"),
350                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
351
352     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
353                           ("make", "make -j", "text/plain"),
354                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
355
356     # Test cross-compile infrastructure
357     "samba-xc": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
358                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
359                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
360                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
361                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
362                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
363                    ("compare-results", "script/compare_cc_results.py "
364                     "./bin/c4che/default{} "
365                     "./bin-xe/c4che/default{} "
366                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
367
368     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
369     "samba-o3": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
370                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
371                    ("make", "make -j", "text/plain"),
372                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
373                     "TESTS='--include-env=ad_dc'", "text/plain"),
374                    ("install", "make install", "text/plain"),
375                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
376                    ("clean", "make clean", "text/plain")],
377
378     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
379
380                      # make sure we have tdb around:
381                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
382                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
383                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
384
385
386                      # build samba with cluster support (also building ctdb):
387                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
388                      ("samba-make", "make", "text/plain"),
389                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
390                      ("samba-install", "make install", "text/plain"),
391                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
392
393                      # clean up:
394                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
395                      ("clean", "make clean", "text/plain"),
396                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
397
398     "samba-libs": [
399                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
400                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
401                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
402                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
403
404                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
405                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
406                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
407
408                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
409                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
410                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
411
412                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
413                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
414                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
415
416                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
417                       ("nondevel-make", "make -j", "text/plain"),
418                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
419                       ("nondevel-install", "make install", "text/plain"),
420                       ("nondevel-dist", "make dist", "text/plain"),
421
422                       # retry with all modules shared
423                       ("allshared-distclean", "make distclean", "text/plain"),
424                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
425                       ("allshared-make", "make -j", "text/plain")],
426
427     "samba-none-env": [
428                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
429                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
430                       ("make", "make -j", "text/plain"),
431                       ("test", "make test "
432                        "FAIL_IMMEDIATELY=1 "
433                        "TESTS='--include-env=none'",
434                        "text/plain")],
435
436     "samba-static": [
437                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
438                       # build with all modules static
439                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
440                       ("allstatic-make", "make -j", "text/plain"),
441                       ("allstatic-test", "make test "
442                        "FAIL_IMMEDIATELY=1 "
443                        "TESTS='samba3.smb2.create.*nt4_dc'",
444                        "text/plain"),
445
446                       # retry without any required modules
447                       ("none-distclean", "make distclean", "text/plain"),
448                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
449                       ("none-make", "make -j", "text/plain"),
450
451                       # retry with nonshared smbd and smbtorture
452                       ("nonshared-distclean", "make distclean", "text/plain"),
453                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
454                       ("nonshared-make", "make -j", "text/plain")],
455
456     "samba-systemkrb5": [
457                       ("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
458                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --with-experimental-mit-ad-dc", "text/plain"),
459                       ("make", "make -j", "text/plain"),
460                       # we currently cannot run a full make test, a limited list of tests could be run
461                       # via "make test TESTS=sometests"
462                       ("test", "make test FAIL_IMMEDIATELY=1 "
463                        "TESTS='--include-env=ktest'", "text/plain"),
464                       ("install", "make install", "text/plain"),
465                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
466                       ("clean", "make clean", "text/plain")
467                       ],
468
469     # Test Samba without python still builds.  When this test fails
470     # due to more use of Python, the expectations is that the newly
471     # failing part of the code should be disabled when
472     # --disable-python is set (rather than major work being done to
473     # support this environment).  The target here is for vendors
474     # shipping a minimal smbd.
475     "samba-nopython": [
476                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
477                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
478                       ("make", "make -j", "text/plain"),
479                       ("install", "make install", "text/plain"),
480                       ("find-python", "script/find_python.sh ${PREFIX}", "text/plain"),
481                       ("test", "make test-nopython", "text/plain"),
482                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
483                       ("clean", "make clean", "text/plain"),
484
485                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
486                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
487                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
488
489                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
490                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
491                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
492
493                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
494                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
495                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
496
497                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
498                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
499                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
500
501                       # retry against installed library packages
502                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
503                       ("libs-make", "make -j", "text/plain"),
504                       ("libs-install", "make install", "text/plain"),
505                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
506                       ("libs-clean", "make clean", "text/plain")
507                       ],
508
509     # check we can do the same thing using python2
510     "samba-nopython-py2": [
511                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
512                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
513                       ("make", "PYTHON=python2 make -j", "text/plain"),
514                       ("install", "PYTHON=python2 make install", "text/plain"),
515                       ("find-python", "script/find_python.sh ${PREFIX}", "text/plain"),
516                       ("test", "make test-nopython", "text/plain"),
517                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
518                       ("clean", "PYTHON=python2 make clean", "text/plain"),
519
520                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
521                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
522                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
523
524                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
525                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
526                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
527
528                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
529                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
530                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
531
532                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
533                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
534                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
535
536                       # retry against installed library packages
537                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
538                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
539                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
540                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
541                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
542                       ],
543
544     "ldb": [
545               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
546               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
547               ("make", "make", "text/plain"),
548               ("install", "make install", "text/plain"),
549               ("test", "make test", "text/plain"),
550               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
551               ("make-no-lmdb", "make", "text/plain"),
552               ("install-no-lmdb", "make install", "text/plain"),
553               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
554               ("distcheck", "make distcheck", "text/plain"),
555               ("clean", "make clean", "text/plain")],
556
557     "tdb": [
558               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
559               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
560               ("make", "make", "text/plain"),
561               ("install", "make install", "text/plain"),
562               ("test", "make test", "text/plain"),
563               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
564               ("distcheck", "make distcheck", "text/plain"),
565               ("clean", "make clean", "text/plain")],
566
567     "talloc": [
568                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
569                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
570                  ("make", "make", "text/plain"),
571                  ("install", "make install", "text/plain"),
572                  ("test", "make test", "text/plain"),
573                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
574                  ("distcheck", "make distcheck", "text/plain"),
575                  ("clean", "make clean", "text/plain")],
576
577     "replace": [
578                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
579                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
580                   ("make", "make", "text/plain"),
581                   ("install", "make install", "text/plain"),
582                   ("test", "make test", "text/plain"),
583                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
584                   ("distcheck", "make distcheck", "text/plain"),
585                   ("clean", "make clean", "text/plain")],
586
587     "tevent": [
588                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
589                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
590                  ("make", "make", "text/plain"),
591                  ("install", "make install", "text/plain"),
592                  ("test", "make test", "text/plain"),
593                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
594                  ("distcheck", "make distcheck", "text/plain"),
595                  ("clean", "make clean", "text/plain")],
596
597     "pidl": [
598         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
599         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
600         ("touch", "touch *.yp", "text/plain"),
601         ("make", "make", "text/plain"),
602         ("test", "make test", "text/plain"),
603         ("install", "make install", "text/plain"),
604         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
605         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
606         ("clean", "make clean", "text/plain")],
607
608
609     # these are useful for debugging autobuild
610     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
611     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
612
613
614 }
615
616
617 def do_print(msg):
618     print("%s" % msg)
619     sys.stdout.flush()
620     sys.stderr.flush()
621
622
623 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
624     if show is None:
625         show = options.verbose
626     if show:
627         do_print("Running: '%s' in '%s'" % (cmd, dir))
628     if output:
629         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
630     elif checkfail:
631         return check_call(cmd, shell=True, cwd=dir)
632     else:
633         return call(cmd, shell=True, cwd=dir)
634
635
636 class builder(object):
637     '''handle build of one directory'''
638
639     def __init__(self, name, sequence, cp=True):
640         self.name = name
641         if name in builddirs:
642             self.dir = builddirs[name]
643         else:
644             self.dir = "."
645
646         self.tag = self.name.replace('/', '_')
647         self.sequence = sequence
648         self.next = 0
649         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
650         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
651         if options.verbose:
652             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
653             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
654         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
655         self.stdout = open(self.stdout_path, 'w')
656         self.stderr = open(self.stderr_path, 'w')
657         self.stdin  = open("/dev/null", 'r')
658         self.test_source_dir = "%s/%s" % (testbase, self.tag)
659         self.prefix = "%s/%s" % (test_prefix, self.tag)
660         run_cmd("rm -rf %s" % self.test_source_dir)
661         run_cmd("rm -rf %s" % self.prefix)
662         if cp:
663             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
664         else:
665             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
666         self.start_next()
667
668     def start_next(self):
669         if self.next == len(self.sequence):
670             if not options.nocleanup:
671                 run_cmd("rm -rf %s" % self.test_source_dir)
672                 run_cmd("rm -rf %s" % self.prefix)
673             do_print('%s: Completed OK' % self.name)
674             self.done = True
675             return
676         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
677         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
678         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
679         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
680         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
681         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
682         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
683         self.cmd = self.cmd.replace("${NAME}", self.name)
684         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
685 #        if self.output_mime_type == "text/x-subunit":
686 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
687         cwd = os.getcwd()
688         os.chdir("%s/%s" % (self.test_source_dir, self.dir))
689         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
690         self.proc = Popen(self.cmd, shell=True,
691                           close_fds=True,
692                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
693         os.chdir(cwd)
694         self.next += 1
695
696
697 class buildlist(object):
698     '''handle build of multiple directories'''
699
700     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
701         global tasks
702         self.tlist = []
703         self.tail_proc = None
704         self.retry = None
705         if tasknames == []:
706             if options.restrict_tests:
707                 tasknames = ["samba-test-only"]
708             else:
709                 tasknames = defaulttasks
710         else:
711             # If we are only running one test,
712             # do not sleep randomly to wait for it to start
713             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
714
715         for n in tasknames:
716             b = builder(n, tasks[n], cp=n is not "pidl")
717             self.tlist.append(b)
718         if options.retry:
719             rebase_remote = "rebaseon"
720             retry_task = [("retry",
721                             '''set -e
722                             git remote add -t %s %s %s
723                             git fetch %s
724                             while :; do
725                               sleep 60
726                               git describe %s/%s > old_remote_branch.desc
727                               git fetch %s
728                               git describe %s/%s > remote_branch.desc
729                               diff old_remote_branch.desc remote_branch.desc
730                             done
731                            ''' % (
732                                 rebase_branch, rebase_remote, rebase_url,
733                                rebase_remote,
734                                rebase_remote, rebase_branch,
735                                rebase_remote,
736                                rebase_remote, rebase_branch
737                             ),
738                             "test/plain")]
739
740             self.retry = builder('retry', retry_task, cp=False)
741             self.need_retry = False
742
743     def kill_kids(self):
744         if self.tail_proc is not None:
745             self.tail_proc.terminate()
746             self.tail_proc.wait()
747             self.tail_proc = None
748         if self.retry is not None:
749             self.retry.proc.terminate()
750             self.retry.proc.wait()
751             self.retry = None
752         for b in self.tlist:
753             if b.proc is not None:
754                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
755                 b.proc.terminate()
756                 b.proc.wait()
757                 b.proc = None
758
759     def wait_one(self):
760         while True:
761             none_running = True
762             for b in self.tlist:
763                 if b.proc is None:
764                     continue
765                 none_running = False
766                 b.status = b.proc.poll()
767                 if b.status is None:
768                     continue
769                 b.proc = None
770                 return b
771             if options.retry:
772                 ret = self.retry.proc.poll()
773                 if ret is not None:
774                     self.need_retry = True
775                     self.retry = None
776                     return None
777             if none_running:
778                 return None
779             time.sleep(0.1)
780
781     def run(self):
782         while True:
783             b = self.wait_one()
784             if options.retry and self.need_retry:
785                 self.kill_kids()
786                 do_print("retry needed")
787                 return (0, None, None, None, "retry")
788             if b is None:
789                 break
790             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
791                 self.kill_kids()
792                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
793             b.start_next()
794         self.kill_kids()
795         return (0, None, None, None, "All OK")
796
797     def write_system_info(self):
798         filename = 'system-info.txt'
799         f = open(filename, 'w')
800         for cmd in ['uname -a',
801                     'lsb_release -a',
802                     'free',
803                     'mount',
804                     'cat /proc/cpuinfo',
805                     'cc --version',
806                     'df -m .',
807                     'df -m %s' % testbase]:
808             out = run_cmd(cmd, output=True, checkfail=False)
809             print('### %s' % cmd, file=f)
810             print(out.decode('utf8', 'backslashreplace'), file=f)
811             print(file=f)
812         f.close()
813         return filename
814
815     def tarlogs(self, fname):
816         tar = tarfile.open(fname, "w:gz")
817         for b in self.tlist:
818             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
819             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
820         if os.path.exists("autobuild.log"):
821             tar.add("autobuild.log")
822         sys_info = self.write_system_info()
823         tar.add(sys_info)
824         tar.close()
825
826     def remove_logs(self):
827         for b in self.tlist:
828             os.unlink(b.stdout_path)
829             os.unlink(b.stderr_path)
830
831     def start_tail(self):
832         cmd = ["tail", "-f"]
833         for b in self.tlist:
834             cmd.append(b.stdout_path)
835             cmd.append(b.stderr_path)
836         self.tail_proc = Popen(cmd, close_fds=True)
837
838
839 def cleanup():
840     if options.nocleanup:
841         return
842     run_cmd("stat %s || true" % test_tmpdir, show=True)
843     run_cmd("stat %s" % testbase, show=True)
844     do_print("Cleaning up %r" % cleanup_list)
845     for d in cleanup_list:
846         run_cmd("rm -rf %s" % d)
847
848
849 def daemonize(logfile):
850     pid = os.fork()
851     if pid == 0:  # Parent
852         os.setsid()
853         pid = os.fork()
854         if pid != 0:  # Actual daemon
855             os._exit(0)
856     else:  # Grandparent
857         os._exit(0)
858
859     import resource      # Resource usage information.
860     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
861     if maxfd == resource.RLIM_INFINITY:
862         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
863     for fd in range(0, maxfd):
864         try:
865             os.close(fd)
866         except OSError:
867             pass
868     os.open(logfile, os.O_RDWR | os.O_CREAT)
869     os.dup2(0, 1)
870     os.dup2(0, 2)
871
872
873 def write_pidfile(fname):
874     '''write a pid file, cleanup on exit'''
875     f = open(fname, mode='w')
876     f.write("%u\n" % os.getpid())
877     f.close()
878
879
880 def rebase_tree(rebase_url, rebase_branch="master"):
881     rebase_remote = "rebaseon"
882     do_print("Rebasing on %s" % rebase_url)
883     run_cmd("git describe HEAD", show=True, dir=test_master)
884     run_cmd("git remote add -t %s %s %s" %
885             (rebase_branch, rebase_remote, rebase_url),
886             show=True, dir=test_master)
887     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
888     if options.fix_whitespace:
889         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
890                 (rebase_remote, rebase_branch),
891                 show=True, dir=test_master)
892     else:
893         run_cmd("git rebase --force-rebase %s/%s" %
894                 (rebase_remote, rebase_branch),
895                 show=True, dir=test_master)
896     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
897                    (rebase_remote, rebase_branch),
898                    dir=test_master, output=True)
899     if diff == '':
900         do_print("No differences between HEAD and %s/%s - exiting" %
901                  (rebase_remote, rebase_branch))
902         sys.exit(0)
903     run_cmd("git describe %s/%s" %
904             (rebase_remote, rebase_branch),
905             show=True, dir=test_master)
906     run_cmd("git describe HEAD", show=True, dir=test_master)
907     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
908             (rebase_remote, rebase_branch),
909             show=True, dir=test_master)
910
911
912 def push_to(push_url, push_branch="master"):
913     push_remote = "pushto"
914     do_print("Pushing to %s" % push_url)
915     if options.mark:
916         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
917         run_cmd("git commit --amend -c HEAD", dir=test_master)
918         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
919         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
920     run_cmd("git remote add -t %s %s %s" %
921             (push_branch, push_remote, push_url),
922             show=True, dir=test_master)
923     run_cmd("git push %s +HEAD:%s" %
924             (push_remote, push_branch),
925             show=True, dir=test_master)
926
927
928 def send_email(subject, text, log_tar):
929     if options.email is None:
930         do_print("not sending email because the recipient is not set")
931         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
932                  (subject, text))
933         return
934     outer = MIMEMultipart()
935     outer['Subject'] = subject
936     outer['To'] = options.email
937     outer['From'] = options.email_from
938     outer['Date'] = email.utils.formatdate(localtime=True)
939     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
940     outer.attach(MIMEText(text, 'plain'))
941     if options.attach_logs:
942         fp = open(log_tar, 'rb')
943         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
944         fp.close()
945         # Set the filename parameter
946         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
947         outer.attach(msg)
948     content = outer.as_string()
949     s = smtplib.SMTP(options.email_server)
950     email_user = os.getenv('SMTP_USERNAME')
951     email_password = os.getenv('SMTP_PASSWORD')
952     if email_user is not None:
953         s.starttls()
954         s.login(email_user, email_password)
955
956     s.sendmail(options.email_from, [options.email], content)
957     s.set_debuglevel(1)
958     s.quit()
959
960
961 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
962                   elapsed_time, log_base=None, add_log_tail=True):
963     '''send an email to options.email about the failure'''
964     elapsed_minutes = elapsed_time / 60.0
965     if log_base is None:
966         log_base = gitroot
967     text = '''
968 Dear Developer,
969
970 Your autobuild on %s failed after %.1f minutes
971 when trying to test %s with the following error:
972
973    %s
974
975 the autobuild has been abandoned. Please fix the error and resubmit.
976
977 A summary of the autobuild process is here:
978
979   %s/autobuild.log
980 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
981
982     if options.restrict_tests:
983         text += """
984 The build was restricted to tests matching %s\n""" % options.restrict_tests
985
986     if failed_task != 'rebase':
987         text += '''
988 You can see logs of the failed task here:
989
990   %s/%s.stdout
991   %s/%s.stderr
992
993 or you can get full logs of all tasks in this job here:
994
995   %s/logs.tar.gz
996
997 The top commit for the tree that was built was:
998
999 %s
1000
1001 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1002
1003     if add_log_tail:
1004         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1005         lines = f.readlines()
1006         log_tail = "".join(lines[-50:])
1007         num_lines = len(lines)
1008         if num_lines < 50:
1009             # Also include stderr (compile failures) if < 50 lines of stdout
1010             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1011             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1012
1013         text += '''
1014 The last 50 lines of log messages:
1015
1016 %s
1017     ''' % log_tail
1018         f.close()
1019
1020     logs = os.path.join(gitroot, 'logs.tar.gz')
1021     send_email('autobuild[%s] failure on %s for task %s during %s'
1022                % (options.branch, platform.node(), failed_task, failed_stage),
1023                text, logs)
1024
1025
1026 def email_success(elapsed_time, log_base=None):
1027     '''send an email to options.email about a successful build'''
1028     if log_base is None:
1029         log_base = gitroot
1030     text = '''
1031 Dear Developer,
1032
1033 Your autobuild on %s has succeeded after %.1f minutes.
1034
1035 ''' % (platform.node(), elapsed_time / 60.)
1036
1037     if options.restrict_tests:
1038         text += """
1039 The build was restricted to tests matching %s\n""" % options.restrict_tests
1040
1041     if options.keeplogs:
1042         text += '''
1043
1044 you can get full logs of all tasks in this job here:
1045
1046   %s/logs.tar.gz
1047
1048 ''' % log_base
1049
1050     text += '''
1051 The top commit for the tree that was built was:
1052
1053 %s
1054 ''' % top_commit_msg
1055
1056     logs = os.path.join(gitroot, 'logs.tar.gz')
1057     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1058                text, logs)
1059
1060
1061 # get the top commit message, for emails
1062 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1063 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
1064
1065 try:
1066     os.makedirs(testbase)
1067 except Exception as reason:
1068     raise Exception("Unable to create %s : %s" % (testbase, reason))
1069 cleanup_list.append(testbase)
1070
1071 if options.daemon:
1072     logfile = os.path.join(testbase, "log")
1073     do_print("Forking into the background, writing progress to %s" % logfile)
1074     daemonize(logfile)
1075
1076 write_pidfile(gitroot + "/autobuild.pid")
1077
1078 start_time = time.time()
1079
1080 while True:
1081     try:
1082         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1083         os.makedirs(test_tmpdir)
1084         # The waf uninstall code removes empty directories all the way
1085         # up the tree.  Creating a file in test_tmpdir stops it from
1086         # being removed.
1087         run_cmd("touch %s" % os.path.join(test_tmpdir,
1088                                           ".directory-is-not-empty"), show=True)
1089         run_cmd("stat %s" % test_tmpdir, show=True)
1090         run_cmd("stat %s" % testbase, show=True)
1091         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1092     except Exception:
1093         cleanup()
1094         raise
1095
1096     try:
1097         try:
1098             if options.rebase is not None:
1099                 rebase_tree(options.rebase, rebase_branch=options.branch)
1100         except Exception:
1101             cleanup_list.append(gitroot + "/autobuild.pid")
1102             cleanup()
1103             elapsed_time = time.time() - start_time
1104             email_failure(-1, 'rebase', 'rebase', 'rebase',
1105                           'rebase on %s failed' % options.branch,
1106                           elapsed_time, log_base=options.log_base)
1107             sys.exit(1)
1108         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1109         if options.tail:
1110             blist.start_tail()
1111         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1112         if status != 0 or errstr != "retry":
1113             break
1114         cleanup()
1115     except Exception:
1116         cleanup()
1117         raise
1118
1119 cleanup_list.append(gitroot + "/autobuild.pid")
1120
1121 do_print(errstr)
1122
1123 blist.kill_kids()
1124 if options.tail:
1125     do_print("waiting for tail to flush")
1126     time.sleep(1)
1127
1128 elapsed_time = time.time() - start_time
1129 if status == 0:
1130     if options.passcmd is not None:
1131         do_print("Running passcmd: %s" % options.passcmd)
1132         run_cmd(options.passcmd, dir=test_master)
1133     if options.pushto is not None:
1134         push_to(options.pushto, push_branch=options.branch)
1135     if options.keeplogs or options.attach_logs:
1136         blist.tarlogs("logs.tar.gz")
1137         do_print("Logs in logs.tar.gz")
1138     if options.always_email:
1139         email_success(elapsed_time, log_base=options.log_base)
1140     blist.remove_logs()
1141     cleanup()
1142     do_print(errstr)
1143     sys.exit(0)
1144
1145 # something failed, gather a tar of the logs
1146 blist.tarlogs("logs.tar.gz")
1147
1148 if options.email is not None:
1149     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1150                   elapsed_time, log_base=options.log_base)
1151 else:
1152     elapsed_minutes = elapsed_time / 60.0
1153     print('''
1154
1155 ####################################################################
1156
1157 AUTOBUILD FAILURE
1158
1159 Your autobuild[%s] on %s failed after %.1f minutes
1160 when trying to test %s with the following error:
1161
1162    %s
1163
1164 the autobuild has been abandoned. Please fix the error and resubmit.
1165
1166 ####################################################################
1167
1168 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1169
1170 cleanup()
1171 do_print(errstr)
1172 do_print("Logs in logs.tar.gz")
1173 sys.exit(status)