autobuild: move maptoguest and simpleserver to 'samba-fileserver'
[garming/samba-autobuild/.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 from optparse import OptionParser
13 import smtplib
14 import email
15 from email.mime.text import MIMEText
16 from email.mime.base import MIMEBase
17 from email.mime.application import MIMEApplication
18 from email.mime.multipart import MIMEMultipart
19 from distutils.sysconfig import get_python_lib
20 import platform
21
22 try:
23     from waflib.Build import CACHE_SUFFIX
24 except ImportError:
25     sys.path.insert(0, "./third_party/waf")
26     from waflib.Build import CACHE_SUFFIX
27
28
29 os.environ["PYTHONUNBUFFERED"] = "1"
30
31 # This speeds up testing remarkably.
32 os.environ['TDB_NO_FSYNC'] = '1'
33
34 cleanup_list = []
35
36 builddirs = {
37     "ctdb": "ctdb",
38     "samba": ".",
39     "samba-nt4": ".",
40     "samba-fileserver": ".",
41     "samba-xc": ".",
42     "samba-o3": ".",
43     "samba-ctdb": ".",
44     "samba-libs": ".",
45     "samba-static": ".",
46     "samba-none-env": ".",
47     "samba-ad-dc": ".",
48     "samba-ad-dc-ntvfs": ".",
49     "samba-ad-dc-2": ".",
50     "samba-ad-dc-backup": ".",
51     "samba-systemkrb5": ".",
52     "samba-nopython": ".",
53     "samba-nopython-py2": ".",
54     "ldb": "lib/ldb",
55     "tdb": "lib/tdb",
56     "talloc": "lib/talloc",
57     "replace": "lib/replace",
58     "tevent": "lib/tevent",
59     "pidl": "pidl"
60 }
61
62 defaulttasks = builddirs.keys()
63
64 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
65     defaulttasks.remove("samba-o3")
66
67 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
68 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
69
70 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
71 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
72 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
73 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
74 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
75 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
76 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
77
78 tasks = {
79     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
80                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
81                ("make", "make all", "text/plain"),
82                ("install", "make install", "text/plain"),
83                ("test", "make autotest", "text/plain"),
84                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
85                ("clean", "make clean", "text/plain")],
86
87     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
88     "samba": [
89                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
90                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
91                 ("make", "make -j", "text/plain"),
92                 ("test", "make test FAIL_IMMEDIATELY=1 "
93                  "TESTS='--exclude-env=none "
94                  "--exclude-env=nt4_dc "
95                  "--exclude-env=nt4_dc_schannel "
96                  "--exclude-env=nt4_member "
97                  "--exclude-env=ad_dc "
98                  "--exclude-env=ad_dc_backup "
99                  "--exclude-env=ad_dc_ntvfs "
100                  "--exclude-env=ad_dc_default "
101                  "--exclude-env=ad_dc_slowtests "
102                  "--exclude-env=ad_dc_no_nss "
103                  "--exclude-env=fl2003dc "
104                  "--exclude-env=fl2008dc "
105                  "--exclude-env=fl2008r2dc "
106                  "--exclude-env=ad_member "
107                  "--exclude-env=ad_member_idmap_rid "
108                  "--exclude-env=ad_member_idmap_ad "
109                  "--exclude-env=chgdcpass "
110                  "--exclude-env=vampire_2000_dc "
111                  "--exclude-env=fl2000dc "
112                  "--exclude-env=fileserver "
113                  "--exclude-env=maptoguest "
114                  "--exclude-env=simpleserver "
115                  "--exclude-env=backupfromdc "
116                  "--exclude-env=restoredc "
117                  "--exclude-env=renamedc "
118                  "--exclude-env=offlinebackupdc "
119                  "--exclude-env=labdc "
120                  "'",
121                  "text/plain"),
122                 ("install", "make install", "text/plain"),
123                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
124                 ("clean", "make clean", "text/plain")],
125
126     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
127                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
128                     ("make", "make -j", "text/plain"),
129                     ("test", "make test FAIL_IMMEDIATELY=1 "
130                      "TESTS='"
131                      "--include-env=nt4_dc "
132                      "--include-env=nt4_dc_schannel "
133                      "--include-env=nt4_member "
134                      "'", "text/plain"),
135                     ("install", "make install", "text/plain"),
136                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
137                     ("clean", "make clean", "text/plain")],
138
139     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
140                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
141                            ("make", "make -j", "text/plain"),
142                            ("test", "make test FAIL_IMMEDIATELY=1 "
143                             "TESTS='"
144                             "--include-env=fileserver "
145                             "--include-env=maptoguest "
146                             "--include-env=simpleserver "
147                             "'", "text/plain"),
148                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
149
150     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
151                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
152                       ("make", "make -j", "text/plain"),
153                       ("test", "make test FAIL_IMMEDIATELY=1 "
154                        "TESTS='--include-env=ad_dc "
155                        "--include-env=ad_dc_backup "
156                        "--include-env=fl2003dc "
157                        "--include-env=fl2008r2dc "
158                        "--include-env=ad_member "
159                        "--include-env=ad_member_idmap_rid "
160                        "--include-env=ad_member_idmap_ad'", "text/plain"),
161                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
162
163     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
164                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
165                         ("make", "make -j", "text/plain"),
166                         ("test", "make test FAIL_IMMEDIATELY=1 "
167                          "TESTS='--include-env=chgdcpass "
168                          "--include-env=vampire_2000_dc "
169                          "--include-env=fl2000dc "
170                          "--include-env=ad_dc_no_nss "
171                          "'",
172                          "text/plain"),
173                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
174
175     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
176     # This is currently the longest task, so we don't randomly delay it.
177     "samba-ad-dc-ntvfs": [
178                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
179                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
180                       ("make", "make -j", "text/plain"),
181                       ("test", "make test FAIL_IMMEDIATELY=1 "
182                        "TESTS='"
183                        "--include-env=ad_dc_ntvfs "
184                        "--include-env=fl2008dc "
185                        "--include-env=ad_dc_default "
186                        "--include-env=ad_dc_slowtests "
187                        "'", "text/plain"),
188                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
189
190     # run the backup/restore testenvs separately as they're fairly standalone
191     # (and CI seems to max out at ~8 different DCs running at once)
192     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
193                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
194                         ("make", "make -j", "text/plain"),
195                         ("test", "make test FAIL_IMMEDIATELY=1 "
196                          "TESTS='--include-env=backupfromdc "
197                          "--include-env=restoredc "
198                          "--include-env=renamedc "
199                          "--include-env=offlinebackupdc "
200                          "--include-env=labdc "
201                          "'",
202                          "text/plain"),
203                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
204
205     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
206                           ("make", "make -j", "text/plain"),
207                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
208
209     # Test cross-compile infrastructure
210     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
211                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
212                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
213                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
214                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
215                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
216                    ("compare-results", "script/compare_cc_results.py "
217                     "./bin/c4che/default{} "
218                     "./bin-xe/c4che/default{} "
219                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
220
221     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
222     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
223                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
224                    ("make", "make -j", "text/plain"),
225                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
226                     "TESTS='--include-env=ad_dc'", "text/plain"),
227                    ("install", "make install", "text/plain"),
228                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
229                    ("clean", "make clean", "text/plain")],
230
231     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
232
233                      # make sure we have tdb around:
234                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
235                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
236                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
237
238
239                      # build samba with cluster support (also building ctdb):
240                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
241                      ("samba-make", "make", "text/plain"),
242                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
243                      ("samba-install", "make install", "text/plain"),
244                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
245
246                      # clean up:
247                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
248                      ("clean", "make clean", "text/plain"),
249                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
250
251     "samba-libs": [
252                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
253                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
254                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
255                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
256
257                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
258                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
259                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
260
261                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
262                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
263                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
264
265                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
266                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
267                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
268
269                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
270                       ("nondevel-make", "make -j", "text/plain"),
271                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
272                       ("nondevel-install", "make install", "text/plain"),
273                       ("nondevel-dist", "make dist", "text/plain"),
274
275                       # retry with all modules shared
276                       ("allshared-distclean", "make distclean", "text/plain"),
277                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
278                       ("allshared-make", "make -j", "text/plain")],
279
280     "samba-none-env": [
281                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
282                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
283                       ("make", "make -j", "text/plain"),
284                       ("test", "make test "
285                        "FAIL_IMMEDIATELY=1 "
286                        "TESTS='--include-env=none'",
287                        "text/plain")],
288
289     "samba-static": [
290                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
291                       # build with all modules static
292                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
293                       ("allstatic-make", "make -j", "text/plain"),
294                       ("allstatic-test", "make test "
295                        "FAIL_IMMEDIATELY=1 "
296                        "TESTS='samba3.smb2.create.*nt4_dc'",
297                        "text/plain"),
298
299                       # retry without any required modules
300                       ("none-distclean", "make distclean", "text/plain"),
301                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
302                       ("none-make", "make -j", "text/plain"),
303
304                       # retry with nonshared smbd and smbtorture
305                       ("nonshared-distclean", "make distclean", "text/plain"),
306                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
307                       ("nonshared-make", "make -j", "text/plain")],
308
309     "samba-systemkrb5": [
310                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
311                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
312                       ("make", "make -j", "text/plain"),
313                       # we currently cannot run a full make test, a limited list of tests could be run
314                       # via "make test TESTS=sometests"
315                       ("test", "make test FAIL_IMMEDIATELY=1 "
316                        "TESTS='--include-env=ktest'", "text/plain"),
317                       ("install", "make install", "text/plain"),
318                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
319                       ("clean", "make clean", "text/plain")
320                       ],
321
322     # Test Samba without python still builds.  When this test fails
323     # due to more use of Python, the expectations is that the newly
324     # failing part of the code should be disabled when
325     # --disable-python is set (rather than major work being done to
326     # support this environment).  The target here is for vendors
327     # shipping a minimal smbd.
328     "samba-nopython": [
329                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
330                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
331                       ("make", "make -j", "text/plain"),
332                       ("install", "make install", "text/plain"),
333                       ("test", "make test-nopython", "text/plain"),
334                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
335                       ("clean", "make clean", "text/plain"),
336
337                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
338                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
339                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
340
341                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
342                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
343                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
344
345                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
346                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
347                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
348
349                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
350                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
351                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
352
353                       # retry against installed library packages
354                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
355                       ("libs-make", "make -j", "text/plain"),
356                       ("libs-install", "make install", "text/plain"),
357                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
358                       ("libs-clean", "make clean", "text/plain")
359                       ],
360
361     # check we can do the same thing using python2
362     "samba-nopython-py2": [
363                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
364                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
365                       ("make", "PYTHON=python2 make -j", "text/plain"),
366                       ("install", "PYTHON=python2 make install", "text/plain"),
367                       ("test", "make test-nopython", "text/plain"),
368                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
369                       ("clean", "PYTHON=python2 make clean", "text/plain"),
370
371                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
372                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
373                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
374
375                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
376                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
377                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
378
379                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
380                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
381                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
382
383                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
384                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
385                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
386
387                       # retry against installed library packages
388                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
389                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
390                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
391                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
392                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
393                       ],
394
395     "ldb": [
396               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
397               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
398               ("make", "make", "text/plain"),
399               ("install", "make install", "text/plain"),
400               ("test", "make test", "text/plain"),
401               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
402               ("make-no-lmdb", "make", "text/plain"),
403               ("install-no-lmdb", "make install", "text/plain"),
404               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
405               ("distcheck", "make distcheck", "text/plain"),
406               ("clean", "make clean", "text/plain")],
407
408     "tdb": [
409               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
410               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
411               ("make", "make", "text/plain"),
412               ("install", "make install", "text/plain"),
413               ("test", "make test", "text/plain"),
414               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
415               ("distcheck", "make distcheck", "text/plain"),
416               ("clean", "make clean", "text/plain")],
417
418     "talloc": [
419                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
420                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
421                  ("make", "make", "text/plain"),
422                  ("install", "make install", "text/plain"),
423                  ("test", "make test", "text/plain"),
424                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
425                  ("distcheck", "make distcheck", "text/plain"),
426                  ("clean", "make clean", "text/plain")],
427
428     "replace": [
429                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
430                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
431                   ("make", "make", "text/plain"),
432                   ("install", "make install", "text/plain"),
433                   ("test", "make test", "text/plain"),
434                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
435                   ("distcheck", "make distcheck", "text/plain"),
436                   ("clean", "make clean", "text/plain")],
437
438     "tevent": [
439                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
440                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
441                  ("make", "make", "text/plain"),
442                  ("install", "make install", "text/plain"),
443                  ("test", "make test", "text/plain"),
444                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
445                  ("distcheck", "make distcheck", "text/plain"),
446                  ("clean", "make clean", "text/plain")],
447
448     "pidl": [
449         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
450         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
451         ("touch", "touch *.yp", "text/plain"),
452         ("make", "make", "text/plain"),
453         ("test", "make test", "text/plain"),
454         ("install", "make install", "text/plain"),
455         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
456         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
457         ("clean", "make clean", "text/plain")],
458
459
460     # these are useful for debugging autobuild
461     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
462     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
463
464
465 }
466
467
468 def do_print(msg):
469     print("%s" % msg)
470     sys.stdout.flush()
471     sys.stderr.flush()
472
473
474 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
475     if show is None:
476         show = options.verbose
477     if show:
478         do_print("Running: '%s' in '%s'" % (cmd, dir))
479     if output:
480         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
481     elif checkfail:
482         return check_call(cmd, shell=True, cwd=dir)
483     else:
484         return call(cmd, shell=True, cwd=dir)
485
486
487 class builder(object):
488     '''handle build of one directory'''
489
490     def __init__(self, name, sequence, cp=True):
491         self.name = name
492         if name in builddirs:
493             self.dir = builddirs[name]
494         else:
495             self.dir = "."
496
497         self.tag = self.name.replace('/', '_')
498         self.sequence = sequence
499         self.next = 0
500         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
501         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
502         if options.verbose:
503             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
504             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
505         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
506         self.stdout = open(self.stdout_path, 'w')
507         self.stderr = open(self.stderr_path, 'w')
508         self.stdin  = open("/dev/null", 'r')
509         self.sdir = "%s/%s" % (testbase, self.tag)
510         self.prefix = "%s/%s" % (test_prefix, self.tag)
511         run_cmd("rm -rf %s" % self.sdir)
512         run_cmd("rm -rf %s" % self.prefix)
513         if cp:
514             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
515         else:
516             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
517         self.start_next()
518
519     def start_next(self):
520         if self.next == len(self.sequence):
521             if not options.nocleanup:
522                 run_cmd("rm -rf %s" % self.sdir)
523                 run_cmd("rm -rf %s" % self.prefix)
524             do_print('%s: Completed OK' % self.name)
525             self.done = True
526             return
527         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
528         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
529         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
530         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
531         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
532 #        if self.output_mime_type == "text/x-subunit":
533 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
534         cwd = os.getcwd()
535         os.chdir("%s/%s" % (self.sdir, self.dir))
536         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
537         self.proc = Popen(self.cmd, shell=True,
538                           close_fds=True,
539                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
540         os.chdir(cwd)
541         self.next += 1
542
543
544 class buildlist(object):
545     '''handle build of multiple directories'''
546
547     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
548         global tasks
549         self.tlist = []
550         self.tail_proc = None
551         self.retry = None
552         if tasknames == []:
553             if options.restrict_tests:
554                 tasknames = ["samba-test-only"]
555             else:
556                 tasknames = defaulttasks
557         else:
558             # If we are only running one test,
559             # do not sleep randomly to wait for it to start
560             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
561
562         for n in tasknames:
563             b = builder(n, tasks[n], cp=n is not "pidl")
564             self.tlist.append(b)
565         if options.retry:
566             rebase_remote = "rebaseon"
567             retry_task = [("retry",
568                             '''set -e
569                             git remote add -t %s %s %s
570                             git fetch %s
571                             while :; do
572                               sleep 60
573                               git describe %s/%s > old_remote_branch.desc
574                               git fetch %s
575                               git describe %s/%s > remote_branch.desc
576                               diff old_remote_branch.desc remote_branch.desc
577                             done
578                            ''' % (
579                                 rebase_branch, rebase_remote, rebase_url,
580                                rebase_remote,
581                                rebase_remote, rebase_branch,
582                                rebase_remote,
583                                rebase_remote, rebase_branch
584                             ),
585                             "test/plain")]
586
587             self.retry = builder('retry', retry_task, cp=False)
588             self.need_retry = False
589
590     def kill_kids(self):
591         if self.tail_proc is not None:
592             self.tail_proc.terminate()
593             self.tail_proc.wait()
594             self.tail_proc = None
595         if self.retry is not None:
596             self.retry.proc.terminate()
597             self.retry.proc.wait()
598             self.retry = None
599         for b in self.tlist:
600             if b.proc is not None:
601                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
602                 b.proc.terminate()
603                 b.proc.wait()
604                 b.proc = None
605
606     def wait_one(self):
607         while True:
608             none_running = True
609             for b in self.tlist:
610                 if b.proc is None:
611                     continue
612                 none_running = False
613                 b.status = b.proc.poll()
614                 if b.status is None:
615                     continue
616                 b.proc = None
617                 return b
618             if options.retry:
619                 ret = self.retry.proc.poll()
620                 if ret is not None:
621                     self.need_retry = True
622                     self.retry = None
623                     return None
624             if none_running:
625                 return None
626             time.sleep(0.1)
627
628     def run(self):
629         while True:
630             b = self.wait_one()
631             if options.retry and self.need_retry:
632                 self.kill_kids()
633                 do_print("retry needed")
634                 return (0, None, None, None, "retry")
635             if b is None:
636                 break
637             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
638                 self.kill_kids()
639                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
640             b.start_next()
641         self.kill_kids()
642         return (0, None, None, None, "All OK")
643
644     def write_system_info(self):
645         filename = 'system-info.txt'
646         f = open(filename, 'w')
647         for cmd in ['uname -a',
648                     'lsb_release -a',
649                     'free',
650                     'mount',
651                     'cat /proc/cpuinfo',
652                     'cc --version',
653                     'df -m .',
654                     'df -m %s' % testbase]:
655             out = run_cmd(cmd, output=True, checkfail=False)
656             print('### %s' % cmd, file=f)
657             print(out.decode('utf8', 'backslashreplace'), file=f)
658             print(file=f)
659         f.close()
660         return filename
661
662     def tarlogs(self, fname):
663         tar = tarfile.open(fname, "w:gz")
664         for b in self.tlist:
665             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
666             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
667         if os.path.exists("autobuild.log"):
668             tar.add("autobuild.log")
669         sys_info = self.write_system_info()
670         tar.add(sys_info)
671         tar.close()
672
673     def remove_logs(self):
674         for b in self.tlist:
675             os.unlink(b.stdout_path)
676             os.unlink(b.stderr_path)
677
678     def start_tail(self):
679         cmd = ["tail", "-f"]
680         for b in self.tlist:
681             cmd.append(b.stdout_path)
682             cmd.append(b.stderr_path)
683         self.tail_proc = Popen(cmd, close_fds=True)
684
685
686 def cleanup():
687     if options.nocleanup:
688         return
689     run_cmd("stat %s || true" % test_tmpdir, show=True)
690     run_cmd("stat %s" % testbase, show=True)
691     do_print("Cleaning up %r" % cleanup_list)
692     for d in cleanup_list:
693         run_cmd("rm -rf %s" % d)
694
695
696 def find_git_root():
697     '''get to the top of the git repo'''
698     p = os.getcwd()
699     while p != '/':
700         if os.path.isdir(os.path.join(p, ".git")):
701             return p
702         p = os.path.abspath(os.path.join(p, '..'))
703     return None
704
705
706 def daemonize(logfile):
707     pid = os.fork()
708     if pid == 0:  # Parent
709         os.setsid()
710         pid = os.fork()
711         if pid != 0:  # Actual daemon
712             os._exit(0)
713     else:  # Grandparent
714         os._exit(0)
715
716     import resource      # Resource usage information.
717     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
718     if maxfd == resource.RLIM_INFINITY:
719         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
720     for fd in range(0, maxfd):
721         try:
722             os.close(fd)
723         except OSError:
724             pass
725     os.open(logfile, os.O_RDWR | os.O_CREAT)
726     os.dup2(0, 1)
727     os.dup2(0, 2)
728
729
730 def write_pidfile(fname):
731     '''write a pid file, cleanup on exit'''
732     f = open(fname, mode='w')
733     f.write("%u\n" % os.getpid())
734     f.close()
735
736
737 def rebase_tree(rebase_url, rebase_branch="master"):
738     rebase_remote = "rebaseon"
739     do_print("Rebasing on %s" % rebase_url)
740     run_cmd("git describe HEAD", show=True, dir=test_master)
741     run_cmd("git remote add -t %s %s %s" %
742             (rebase_branch, rebase_remote, rebase_url),
743             show=True, dir=test_master)
744     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
745     if options.fix_whitespace:
746         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
747                 (rebase_remote, rebase_branch),
748                 show=True, dir=test_master)
749     else:
750         run_cmd("git rebase --force-rebase %s/%s" %
751                 (rebase_remote, rebase_branch),
752                 show=True, dir=test_master)
753     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
754                    (rebase_remote, rebase_branch),
755                    dir=test_master, output=True)
756     if diff == '':
757         do_print("No differences between HEAD and %s/%s - exiting" %
758                  (rebase_remote, rebase_branch))
759         sys.exit(0)
760     run_cmd("git describe %s/%s" %
761             (rebase_remote, rebase_branch),
762             show=True, dir=test_master)
763     run_cmd("git describe HEAD", show=True, dir=test_master)
764     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
765             (rebase_remote, rebase_branch),
766             show=True, dir=test_master)
767
768
769 def push_to(push_url, push_branch="master"):
770     push_remote = "pushto"
771     do_print("Pushing to %s" % push_url)
772     if options.mark:
773         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
774         run_cmd("git commit --amend -c HEAD", dir=test_master)
775         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
776         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
777     run_cmd("git remote add -t %s %s %s" %
778             (push_branch, push_remote, push_url),
779             show=True, dir=test_master)
780     run_cmd("git push %s +HEAD:%s" %
781             (push_remote, push_branch),
782             show=True, dir=test_master)
783
784
785 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
786
787 gitroot = find_git_root()
788 if gitroot is None:
789     raise Exception("Failed to find git root")
790
791 parser = OptionParser()
792 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
793 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
794 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
795 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
796                   default=def_testbase)
797 parser.add_option("", "--passcmd", help="command to run on success", default=None)
798 parser.add_option("", "--verbose", help="show all commands as they are run",
799                   default=False, action="store_true")
800 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
801                   default=None, type='str')
802 parser.add_option("", "--pushto", help="push to a git url on success",
803                   default=None, type='str')
804 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
805                   default=False, action="store_true")
806 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
807                   default=False, action="store_true")
808 parser.add_option("", "--retry", help="automatically retry if master changes",
809                   default=False, action="store_true")
810 parser.add_option("", "--email", help="send email to the given address on failure",
811                   type='str', default=None)
812 parser.add_option("", "--email-from", help="send email from the given address",
813                   type='str', default="autobuild@samba.org")
814 parser.add_option("", "--email-server", help="send email via the given server",
815                   type='str', default='localhost')
816 parser.add_option("", "--always-email", help="always send email, even on success",
817                   action="store_true")
818 parser.add_option("", "--daemon", help="daemonize after initial setup",
819                   action="store_true")
820 parser.add_option("", "--branch", help="the branch to work on (default=master)",
821                   default="master", type='str')
822 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
823                   default=gitroot, type='str')
824 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
825                   default=False, action="store_true")
826 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
827                   default='')
828
829
830 def send_email(subject, text, log_tar):
831     if options.email is None:
832         do_print("not sending email because the recipient is not set")
833         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
834                  (subject, text))
835         return
836     outer = MIMEMultipart()
837     outer['Subject'] = subject
838     outer['To'] = options.email
839     outer['From'] = options.email_from
840     outer['Date'] = email.utils.formatdate(localtime=True)
841     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
842     outer.attach(MIMEText(text, 'plain'))
843     if options.attach_logs:
844         fp = open(log_tar, 'rb')
845         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
846         fp.close()
847         # Set the filename parameter
848         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
849         outer.attach(msg)
850     content = outer.as_string()
851     s = smtplib.SMTP(options.email_server)
852     s.sendmail(options.email_from, [options.email], content)
853     s.set_debuglevel(1)
854     s.quit()
855
856
857 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
858                   elapsed_time, log_base=None, add_log_tail=True):
859     '''send an email to options.email about the failure'''
860     elapsed_minutes = elapsed_time / 60.0
861     if log_base is None:
862         log_base = gitroot
863     text = '''
864 Dear Developer,
865
866 Your autobuild on %s failed after %.1f minutes
867 when trying to test %s with the following error:
868
869    %s
870
871 the autobuild has been abandoned. Please fix the error and resubmit.
872
873 A summary of the autobuild process is here:
874
875   %s/autobuild.log
876 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
877
878     if options.restrict_tests:
879         text += """
880 The build was restricted to tests matching %s\n""" % options.restrict_tests
881
882     if failed_task != 'rebase':
883         text += '''
884 You can see logs of the failed task here:
885
886   %s/%s.stdout
887   %s/%s.stderr
888
889 or you can get full logs of all tasks in this job here:
890
891   %s/logs.tar.gz
892
893 The top commit for the tree that was built was:
894
895 %s
896
897 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
898
899     if add_log_tail:
900         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
901         lines = f.readlines()
902         log_tail = "".join(lines[-50:])
903         num_lines = len(lines)
904         if num_lines < 50:
905             # Also include stderr (compile failures) if < 50 lines of stdout
906             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
907             log_tail += "".join(f.readlines()[-(50 - num_lines):])
908
909         text += '''
910 The last 50 lines of log messages:
911
912 %s
913     ''' % log_tail
914         f.close()
915
916     logs = os.path.join(gitroot, 'logs.tar.gz')
917     send_email('autobuild[%s] failure on %s for task %s during %s'
918                % (options.branch, platform.node(), failed_task, failed_stage),
919                text, logs)
920
921
922 def email_success(elapsed_time, log_base=None):
923     '''send an email to options.email about a successful build'''
924     if log_base is None:
925         log_base = gitroot
926     text = '''
927 Dear Developer,
928
929 Your autobuild on %s has succeeded after %.1f minutes.
930
931 ''' % (platform.node(), elapsed_time / 60.)
932
933     if options.restrict_tests:
934         text += """
935 The build was restricted to tests matching %s\n""" % options.restrict_tests
936
937     if options.keeplogs:
938         text += '''
939
940 you can get full logs of all tasks in this job here:
941
942   %s/logs.tar.gz
943
944 ''' % log_base
945
946     text += '''
947 The top commit for the tree that was built was:
948
949 %s
950 ''' % top_commit_msg
951
952     logs = os.path.join(gitroot, 'logs.tar.gz')
953     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
954                text, logs)
955
956
957 (options, args) = parser.parse_args()
958
959 if options.retry:
960     if options.rebase is None:
961         raise Exception('You can only use --retry if you also rebase')
962
963 testbase = "%s/b%u" % (options.testbase, os.getpid())
964 test_master = "%s/master" % testbase
965 test_prefix = "%s/prefix" % testbase
966 test_tmpdir = "%s/tmp" % testbase
967 os.environ['TMPDIR'] = test_tmpdir
968
969 # get the top commit message, for emails
970 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
971 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
972
973 try:
974     os.makedirs(testbase)
975 except Exception as reason:
976     raise Exception("Unable to create %s : %s" % (testbase, reason))
977 cleanup_list.append(testbase)
978
979 if options.daemon:
980     logfile = os.path.join(testbase, "log")
981     do_print("Forking into the background, writing progress to %s" % logfile)
982     daemonize(logfile)
983
984 write_pidfile(gitroot + "/autobuild.pid")
985
986 start_time = time.time()
987
988 while True:
989     try:
990         run_cmd("rm -rf %s" % test_tmpdir, show=True)
991         os.makedirs(test_tmpdir)
992         # The waf uninstall code removes empty directories all the way
993         # up the tree.  Creating a file in test_tmpdir stops it from
994         # being removed.
995         run_cmd("touch %s" % os.path.join(test_tmpdir,
996                                           ".directory-is-not-empty"), show=True)
997         run_cmd("stat %s" % test_tmpdir, show=True)
998         run_cmd("stat %s" % testbase, show=True)
999         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1000     except Exception:
1001         cleanup()
1002         raise
1003
1004     try:
1005         try:
1006             if options.rebase is not None:
1007                 rebase_tree(options.rebase, rebase_branch=options.branch)
1008         except Exception:
1009             cleanup_list.append(gitroot + "/autobuild.pid")
1010             cleanup()
1011             elapsed_time = time.time() - start_time
1012             email_failure(-1, 'rebase', 'rebase', 'rebase',
1013                           'rebase on %s failed' % options.branch,
1014                           elapsed_time, log_base=options.log_base)
1015             sys.exit(1)
1016         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1017         if options.tail:
1018             blist.start_tail()
1019         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1020         if status != 0 or errstr != "retry":
1021             break
1022         cleanup()
1023     except Exception:
1024         cleanup()
1025         raise
1026
1027 cleanup_list.append(gitroot + "/autobuild.pid")
1028
1029 do_print(errstr)
1030
1031 blist.kill_kids()
1032 if options.tail:
1033     do_print("waiting for tail to flush")
1034     time.sleep(1)
1035
1036 elapsed_time = time.time() - start_time
1037 if status == 0:
1038     if options.passcmd is not None:
1039         do_print("Running passcmd: %s" % options.passcmd)
1040         run_cmd(options.passcmd, dir=test_master)
1041     if options.pushto is not None:
1042         push_to(options.pushto, push_branch=options.branch)
1043     if options.keeplogs or options.attach_logs:
1044         blist.tarlogs("logs.tar.gz")
1045         do_print("Logs in logs.tar.gz")
1046     if options.always_email:
1047         email_success(elapsed_time, log_base=options.log_base)
1048     blist.remove_logs()
1049     cleanup()
1050     do_print(errstr)
1051     sys.exit(0)
1052
1053 # something failed, gather a tar of the logs
1054 blist.tarlogs("logs.tar.gz")
1055
1056 if options.email is not None:
1057     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1058                   elapsed_time, log_base=options.log_base)
1059 else:
1060     elapsed_minutes = elapsed_time / 60.0
1061     print('''
1062
1063 ####################################################################
1064
1065 AUTOBUILD FAILURE
1066
1067 Your autobuild[%s] on %s failed after %.1f minutes
1068 when trying to test %s with the following error:
1069
1070    %s
1071
1072 the autobuild has been abandoned. Please fix the error and resubmit.
1073
1074 ####################################################################
1075
1076 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1077
1078 cleanup()
1079 do_print(errstr)
1080 do_print("Logs in logs.tar.gz")
1081 sys.exit(status)