440e177442d7fe17ce955dcb9a3515627f0fc03a
[amitay/samba.git] / script / autobuild.py
1 #!/usr/bin/env python
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 from optparse import OptionParser
13 import smtplib
14 import email
15 from email.mime.text import MIMEText
16 from email.mime.base import MIMEBase
17 from email.mime.application import MIMEApplication
18 from email.mime.multipart import MIMEMultipart
19 from distutils.sysconfig import get_python_lib
20 import platform
21
22 try:
23     from waflib.Build import CACHE_SUFFIX
24 except ImportError:
25     sys.path.insert(0, "./third_party/waf")
26     from waflib.Build import CACHE_SUFFIX
27
28
29 os.environ["PYTHONUNBUFFERED"] = "1"
30
31 # This speeds up testing remarkably.
32 os.environ['TDB_NO_FSYNC'] = '1'
33
34 cleanup_list = []
35
36 builddirs = {
37     "ctdb": "ctdb",
38     "samba": ".",
39     "samba-py3": ".",
40     "samba-nt4": ".",
41     "samba-nt4-py3": ".",
42     "samba-fileserver": ".",
43     "samba-xc": ".",
44     "samba-o3": ".",
45     "samba-ctdb": ".",
46     "samba-libs": ".",
47     "samba-libs-py3": ".",
48     "samba-static": ".",
49     "samba-none-env": ".",
50     "samba-ad-dc": ".",
51     "samba-ad-dc-py3": ".",
52     "samba-ad-dc-2": ".",
53     "samba-ad-dc-2-py3": ".",
54     "samba-systemkrb5": ".",
55     "samba-nopython": ".",
56     "samba-buildpy3-only": ".",
57     "samba-purepy3-none-env": ".",
58     "samba-purepy3-ad-dc-2": ".",
59     "samba-purepy3-ad-dc": ".",
60     "samba-purepy3": ".",
61     "samba-purepy3-nt4": ".",
62     "ldb": "lib/ldb",
63     "tdb": "lib/tdb",
64     "talloc": "lib/talloc",
65     "replace": "lib/replace",
66     "tevent": "lib/tevent",
67     "pidl": "pidl"
68 }
69
70 defaulttasks = builddirs.keys()
71
72 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
73     defaulttasks.remove("samba-o3")
74
75 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
76 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
77
78 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
79 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
80 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
81 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
82 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
83 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
84 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
85
86 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
87     extra_python = ""
88 else:
89     extra_python = "--extra-python=/usr/bin/python3"
90
91 tasks = {
92     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
93                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
94                ("make", "make all", "text/plain"),
95                ("install", "make install", "text/plain"),
96                ("test", "make autotest", "text/plain"),
97                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
98                ("clean", "make clean", "text/plain")],
99
100     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
101     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
102                 ("make", "make -j", "text/plain"),
103                 ("test", "make test FAIL_IMMEDIATELY=1 "
104                  "TESTS='${PY3_ONLY}"
105                  "--exclude-env=none "
106                  "--exclude-env=nt4_dc "
107                  "--exclude-env=nt4_member "
108                  "--exclude-env=ad_dc "
109                  "--exclude-env=ad_dc_no_nss "
110                  "--exclude-env=fl2003dc "
111                  "--exclude-env=fl2008r2dc "
112                  "--exclude-env=ad_member "
113                  "--exclude-env=ad_member_idmap_rid "
114                  "--exclude-env=ad_member_idmap_ad "
115                  "--exclude-env=chgdcpass "
116                  "--exclude-env=vampire_2000_dc "
117                  "--exclude-env=fl2000dc "
118                  "--exclude-env=fileserver "
119                  "--exclude-env=backupfromdc "
120                  "--exclude-env=restoredc "
121                  "--exclude-env=renamedc "
122                  "--exclude-env=offlinebackupdc "
123                  "--exclude-env=labdc "
124                  "'",
125                  "text/plain"),
126                 ("install", "make install", "text/plain"),
127                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
128                 ("clean", "make clean", "text/plain")],
129
130     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
131     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
132                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
133                     ("make", "make -j", "text/plain"),
134                     ("test", "make test FAIL_IMMEDIATELY=1 "
135                      "TESTS='${PY3_ONLY}"
136                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
137                     ("install", "make install", "text/plain"),
138                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
139                     ("clean", "make clean", "text/plain")],
140
141     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
142     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
143                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
144                            ("make", "make -j", "text/plain"),
145                            ("test", "make test FAIL_IMMEDIATELY=1 "
146                             "TESTS='${PY3_ONLY}"
147                             "--include-env=fileserver'", "text/plain"),
148                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
149
150     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
151     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
152                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
153                       ("make", "make -j", "text/plain"),
154                       ("test", "make test FAIL_IMMEDIATELY=1 "
155                        "TESTS='${PY3_ONLY}"
156                        "--include-env=ad_dc "
157                        "--include-env=fl2003dc "
158                        "--include-env=fl2008r2dc "
159                        "--include-env=ad_member "
160                        "--include-env=ad_member_idmap_rid "
161                        "--include-env=ad_member_idmap_ad'", "text/plain"),
162                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
163
164     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
165     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
166                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
167                         ("make", "make -j", "text/plain"),
168                         ("test", "make test FAIL_IMMEDIATELY=1 "
169                          "TESTS='${PY3_ONLY}"
170                          "--include-env=chgdcpass "
171                          "--include-env=vampire_2000_dc "
172                          "--include-env=fl2000dc "
173                          "--include-env=ad_dc_no_nss "
174                          "--include-env=backupfromdc "
175                          "--include-env=restoredc "
176                          "--include-env=renamedc "
177                          "--include-env=offlinebackupdc "
178                          "--include-env=labdc "
179                          "'",
180                          "text/plain"),
181                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
182
183     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
184                           ("make", "make -j", "text/plain"),
185                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
186
187     # Test cross-compile infrastructure
188     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
189                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
190                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
191                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
192                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
193                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
194                    ("compare-results", "script/compare_cc_results.py "
195                     "./bin/c4che/default{} "
196                     "./bin-xe/c4che/default{} "
197                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
198
199     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
200     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
201                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
202                    ("make", "make -j", "text/plain"),
203                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
204                     "TESTS='${PY3_ONLY}"
205                     "--include-env=ad_dc'", "text/plain"),
206                    ("install", "make install", "text/plain"),
207                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
208                    ("clean", "make clean", "text/plain")],
209
210     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
211
212                      # make sure we have tdb around:
213                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
214                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
215                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
216
217
218                      # build samba with cluster support (also building ctdb):
219                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
220                      ("samba-make", "make", "text/plain"),
221                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
222                      ("samba-install", "make install", "text/plain"),
223                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
224
225                      # clean up:
226                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
227                      ("clean", "make clean", "text/plain"),
228                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
229
230     "samba-libs": [
231                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
232                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
233                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
234                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
235
236                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
237                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
238                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
239
240                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
241                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
242                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
243
244                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
245                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
246                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
247
248                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
249                       ("nondevel-make", "make -j", "text/plain"),
250                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
251                       ("nondevel-install", "make install", "text/plain"),
252                       ("nondevel-dist", "make dist", "text/plain"),
253
254                       # retry with all modules shared
255                       ("allshared-distclean", "make distclean", "text/plain"),
256                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
257                       ("allshared-make", "make -j", "text/plain")],
258
259     "samba-none-env": [
260                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
261                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
262                       ("make", "make -j", "text/plain"),
263                       ("test", "make test "
264                        "FAIL_IMMEDIATELY=1 "
265                        "TESTS='${PY3_ONLY}"
266                        "--include-env=none'",
267                        "text/plain")],
268
269     "samba-static": [
270                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
271                       # build with all modules static
272                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
273                       ("allstatic-make", "make -j", "text/plain"),
274                       ("allstatic-test", "make test "
275                        "FAIL_IMMEDIATELY=1 "
276                        "TESTS='samba3.smb2.create.*nt4_dc'",
277                        "text/plain"),
278
279                       # retry without any required modules
280                       ("none-distclean", "make distclean", "text/plain"),
281                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
282                       ("none-make", "make -j", "text/plain"),
283
284                       # retry with nonshared smbd and smbtorture
285                       ("nonshared-distclean", "make distclean", "text/plain"),
286                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
287                       ("nonshared-make", "make -j", "text/plain")],
288
289     "samba-systemkrb5": [
290                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
291                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
292                       ("make", "make -j", "text/plain"),
293                       # we currently cannot run a full make test, a limited list of tests could be run
294                       # via "make test TESTS=sometests"
295                       ("test", "make test FAIL_IMMEDIATELY=1 "
296                        "TESTS='${PY3_ONLY}"
297                        "--include-env=ktest'", "text/plain"),
298                       ("install", "make install", "text/plain"),
299                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
300                       ("clean", "make clean", "text/plain")
301                       ],
302
303     # Test Samba without python still builds.  When this test fails
304     # due to more use of Python, the expectations is that the newly
305     # failing part of the code should be disabled when
306     # --disable-python is set (rather than major work being done to
307     # support this environment).  The target here is for vendors
308     # shipping a minimal smbd.
309     "samba-nopython": [
310                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
311                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
312                       ("make", "make -j", "text/plain"),
313                       ("install", "make install", "text/plain"),
314                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
315                       ("clean", "make clean", "text/plain"),
316
317                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
318                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
319                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
320
321                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
322                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
323                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
324
325                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
326                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
327                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
328
329                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
330                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
331                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
332
333                       # retry against installed library packages
334                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
335                       ("libs-make", "make -j", "text/plain"),
336                       ("libs-install", "make install", "text/plain"),
337                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
338                       ("libs-clean", "make clean", "text/plain")
339                       ],
340
341
342
343     "ldb": [
344               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
345               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
346               ("make", "make", "text/plain"),
347               ("install", "make install", "text/plain"),
348               ("test", "make test", "text/plain"),
349               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
350               ("make-no-lmdb", "make", "text/plain"),
351               ("install-no-lmdb", "make install", "text/plain"),
352               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
353               ("distcheck", "make distcheck", "text/plain"),
354               ("clean", "make clean", "text/plain")],
355
356     "tdb": [
357               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
358               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
359               ("make", "make", "text/plain"),
360               ("install", "make install", "text/plain"),
361               ("test", "make test", "text/plain"),
362               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
363               ("distcheck", "make distcheck", "text/plain"),
364               ("clean", "make clean", "text/plain")],
365
366     "talloc": [
367                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
368                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
369                  ("make", "make", "text/plain"),
370                  ("install", "make install", "text/plain"),
371                  ("test", "make test", "text/plain"),
372                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
373                  ("distcheck", "make distcheck", "text/plain"),
374                  ("clean", "make clean", "text/plain")],
375
376     "replace": [
377                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
378                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
379                   ("make", "make", "text/plain"),
380                   ("install", "make install", "text/plain"),
381                   ("test", "make test", "text/plain"),
382                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
383                   ("distcheck", "make distcheck", "text/plain"),
384                   ("clean", "make clean", "text/plain")],
385
386     "tevent": [
387                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
388                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
389                  ("make", "make", "text/plain"),
390                  ("install", "make install", "text/plain"),
391                  ("test", "make test", "text/plain"),
392                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
393                  ("distcheck", "make distcheck", "text/plain"),
394                  ("clean", "make clean", "text/plain")],
395
396     "pidl": [
397         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
398         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
399         ("touch", "touch *.yp", "text/plain"),
400         ("make", "make", "text/plain"),
401         ("test", "make test", "text/plain"),
402         ("install", "make install", "text/plain"),
403         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
404         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
405         ("clean", "make clean", "text/plain")],
406
407     "samba-buildpy3-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
408                    ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
409                    ("make", "PYTHON='python3' make -j", "text/plain"),
410                    ("install", "PYTHON='python3' make install", "text/plain"),
411                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
412                    ("clean", "PYTHON='python3' make clean", "text/plain")],
413
414     "samba-purepy3-none-env": [
415                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
416                       ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
417                       ("make", "PYTHON='python3' make -j", "text/plain"),
418                       ("test", "PYTHON='python3' make test "
419                        "FAIL_IMMEDIATELY=1 "
420                        "TESTS='${PY3_ONLY}"
421                        "--include-env=none'",
422                        "text/plain")],
423     "samba-purepy3-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
424                         ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
425                         ("make", "PYTHON='python3' make -j", "text/plain"),
426                         ("test", "PYTHON='python3' make test FAIL_IMMEDIATELY=1 "
427                          "TESTS='${PY3_ONLY}"
428                          "--include-env=chgdcpass "
429                          "--include-env=vampire_2000_dc "
430                          "--include-env=fl2000dc "
431                          "--include-env=ad_dc_no_nss "
432                          "--include-env=backupfromdc "
433                          "--include-env=restoredc "
434                          "--include-env=renamedc "
435                          "--include-env=offlinebackupdc "
436                          "--include-env=labdc "
437                          "'",
438                          "text/plain"),
439                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
440     "samba-purepy3-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
441                       ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
442                       ("make", "PYTHON='python3' make -j", "text/plain"),
443                       ("test", "PYTHON='python3' make test FAIL_IMMEDIATELY=1 "
444                        "TESTS='${PY3_ONLY}"
445                        "--include-env=ad_dc "
446                        "--include-env=fl2003dc "
447                        "--include-env=fl2008r2dc "
448                        "--include-env=ad_member "
449                        "--include-env=ad_member_idmap_rid "
450                        "--include-env=ad_member_idmap_ad'", "text/plain"),
451                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
452     "samba-purepy3" : [("configure", "PYTHON=python3 ./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
453                 ("make", "PYTHON=python3 make -j", "text/plain"),
454                 ("test", "PYTHON=python3 make test FAIL_IMMEDIATELY=1 "
455                  "TESTS='${PY3_ONLY}"
456                  "--exclude-env=none "
457                  "--exclude-env=nt4_dc "
458                  "--exclude-env=nt4_member "
459                  "--exclude-env=ad_dc "
460                  "--exclude-env=ad_dc_no_nss "
461                  "--exclude-env=fl2003dc "
462                  "--exclude-env=fl2008r2dc "
463                  "--exclude-env=ad_member "
464                  "--exclude-env=ad_member_idmap_rid "
465                  "--exclude-env=ad_member_idmap_ad "
466                  "--exclude-env=chgdcpass "
467                  "--exclude-env=vampire_2000_dc "
468                  "--exclude-env=fl2000dc "
469                  "--exclude-env=fileserver "
470                  "--exclude-env=backupfromdc "
471                  "--exclude-env=restoredc "
472                  "--exclude-env=renamedc "
473                  "--exclude-env=offlinebackupdc "
474                  "--exclude-env=labdc "
475                  "'",
476                  "text/plain"),
477                 ("install", "make install", "text/plain"),
478                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
479                 ("clean", "make clean", "text/plain")],
480 "samba-purepy3-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
481                     ("configure", "PYTHON=python3 ./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
482                     ("make", "PYTHON=python3 make -j", "text/plain"),
483                     ("test", "PYTHON=python3 make test FAIL_IMMEDIATELY=1 "
484                      "TESTS='${PY3_ONLY}"
485                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
486                     ("install", "PYTHON=python3 make install", "text/plain"),
487                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
488                     ("clean", "PYTHON=python3 make clean", "text/plain")],
489
490
491
492     # these are useful for debugging autobuild
493     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
494     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
495
496
497 }
498
499
500 def do_print(msg):
501     print("%s" % msg)
502     sys.stdout.flush()
503     sys.stderr.flush()
504
505
506 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
507     if show is None:
508         show = options.verbose
509     if show:
510         do_print("Running: '%s' in '%s'" % (cmd, dir))
511     if output:
512         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
513     elif checkfail:
514         return check_call(cmd, shell=True, cwd=dir)
515     else:
516         return call(cmd, shell=True, cwd=dir)
517
518
519 class builder(object):
520     '''handle build of one directory'''
521
522     def __init__(self, name, sequence, cp=True, py3=False):
523         self.name = name
524         self.py3 = py3
525         if name in builddirs:
526             self.dir = builddirs[name]
527         else:
528             self.dir = "."
529
530         self.tag = self.name.replace('/', '_')
531         self.sequence = sequence
532         self.next = 0
533         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
534         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
535         if options.verbose:
536             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
537             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
538         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
539         self.stdout = open(self.stdout_path, 'w')
540         self.stderr = open(self.stderr_path, 'w')
541         self.stdin  = open("/dev/null", 'r')
542         self.sdir = "%s/%s" % (testbase, self.tag)
543         self.prefix = "%s/%s" % (test_prefix, self.tag)
544         run_cmd("rm -rf %s" % self.sdir)
545         run_cmd("rm -rf %s" % self.prefix)
546         if cp:
547             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
548         else:
549             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
550         self.start_next()
551
552     def start_next(self):
553         if self.next == len(self.sequence):
554             if not options.nocleanup:
555                 run_cmd("rm -rf %s" % self.sdir)
556                 run_cmd("rm -rf %s" % self.prefix)
557             do_print('%s: Completed OK' % self.name)
558             self.done = True
559             return
560         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
561         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
562         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
563         if self.py3:
564             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
565             # The trailing space is important
566             self.cmd = self.cmd.replace("${PY3_ONLY}", "python3 ")
567         else:
568             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
569             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
570         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
571         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
572 #        if self.output_mime_type == "text/x-subunit":
573 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
574         cwd = os.getcwd()
575         os.chdir("%s/%s" % (self.sdir, self.dir))
576         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
577         self.proc = Popen(self.cmd, shell=True,
578                           close_fds=True,
579                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
580         os.chdir(cwd)
581         self.next += 1
582
583
584 class buildlist(object):
585     '''handle build of multiple directories'''
586
587     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
588         global tasks
589         self.tlist = []
590         self.tail_proc = None
591         self.retry = None
592         if tasknames == []:
593             if options.restrict_tests:
594                 tasknames = ["samba-test-only"]
595             else:
596                 tasknames = defaulttasks
597         else:
598             # If we are only running one test,
599             # do not sleep randomly to wait for it to start
600             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
601
602         for n in tasknames:
603             if n not in tasks and n.endswith("-py3"):
604                 b = builder(n,
605                             tasks[n[:-4]],
606                             cp=n is not "pidl",
607                             py3=True)
608             else:
609                 b = builder(n, tasks[n], cp=n is not "pidl")
610             self.tlist.append(b)
611         if options.retry:
612             rebase_remote = "rebaseon"
613             retry_task = [("retry",
614                             '''set -e
615                             git remote add -t %s %s %s
616                             git fetch %s
617                             while :; do
618                               sleep 60
619                               git describe %s/%s > old_remote_branch.desc
620                               git fetch %s
621                               git describe %s/%s > remote_branch.desc
622                               diff old_remote_branch.desc remote_branch.desc
623                             done
624                            ''' % (
625                                 rebase_branch, rebase_remote, rebase_url,
626                                rebase_remote,
627                                rebase_remote, rebase_branch,
628                                rebase_remote,
629                                rebase_remote, rebase_branch
630                             ),
631                             "test/plain")]
632
633             self.retry = builder('retry', retry_task, cp=False)
634             self.need_retry = False
635
636     def kill_kids(self):
637         if self.tail_proc is not None:
638             self.tail_proc.terminate()
639             self.tail_proc.wait()
640             self.tail_proc = None
641         if self.retry is not None:
642             self.retry.proc.terminate()
643             self.retry.proc.wait()
644             self.retry = None
645         for b in self.tlist:
646             if b.proc is not None:
647                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
648                 b.proc.terminate()
649                 b.proc.wait()
650                 b.proc = None
651
652     def wait_one(self):
653         while True:
654             none_running = True
655             for b in self.tlist:
656                 if b.proc is None:
657                     continue
658                 none_running = False
659                 b.status = b.proc.poll()
660                 if b.status is None:
661                     continue
662                 b.proc = None
663                 return b
664             if options.retry:
665                 ret = self.retry.proc.poll()
666                 if ret is not None:
667                     self.need_retry = True
668                     self.retry = None
669                     return None
670             if none_running:
671                 return None
672             time.sleep(0.1)
673
674     def run(self):
675         while True:
676             b = self.wait_one()
677             if options.retry and self.need_retry:
678                 self.kill_kids()
679                 do_print("retry needed")
680                 return (0, None, None, None, "retry")
681             if b is None:
682                 break
683             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
684                 self.kill_kids()
685                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
686             b.start_next()
687         self.kill_kids()
688         return (0, None, None, None, "All OK")
689
690     def write_system_info(self):
691         filename = 'system-info.txt'
692         f = open(filename, 'w')
693         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
694                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
695             print('### %s' % cmd, file=f)
696             print(run_cmd(cmd, output=True, checkfail=False), file=f)
697             print(file=f)
698         f.close()
699         return filename
700
701     def tarlogs(self, fname):
702         tar = tarfile.open(fname, "w:gz")
703         for b in self.tlist:
704             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
705             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
706         if os.path.exists("autobuild.log"):
707             tar.add("autobuild.log")
708         sys_info = self.write_system_info()
709         tar.add(sys_info)
710         tar.close()
711
712     def remove_logs(self):
713         for b in self.tlist:
714             os.unlink(b.stdout_path)
715             os.unlink(b.stderr_path)
716
717     def start_tail(self):
718         cmd = ["tail", "-f"]
719         for b in self.tlist:
720             cmd.append(b.stdout_path)
721             cmd.append(b.stderr_path)
722         self.tail_proc = Popen(cmd, close_fds=True)
723
724
725 def cleanup():
726     if options.nocleanup:
727         return
728     run_cmd("stat %s || true" % test_tmpdir, show=True)
729     run_cmd("stat %s" % testbase, show=True)
730     do_print("Cleaning up %r" % cleanup_list)
731     for d in cleanup_list:
732         run_cmd("rm -rf %s" % d)
733
734
735 def find_git_root():
736     '''get to the top of the git repo'''
737     p = os.getcwd()
738     while p != '/':
739         if os.path.isdir(os.path.join(p, ".git")):
740             return p
741         p = os.path.abspath(os.path.join(p, '..'))
742     return None
743
744
745 def daemonize(logfile):
746     pid = os.fork()
747     if pid == 0:  # Parent
748         os.setsid()
749         pid = os.fork()
750         if pid != 0:  # Actual daemon
751             os._exit(0)
752     else:  # Grandparent
753         os._exit(0)
754
755     import resource      # Resource usage information.
756     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
757     if maxfd == resource.RLIM_INFINITY:
758         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
759     for fd in range(0, maxfd):
760         try:
761             os.close(fd)
762         except OSError:
763             pass
764     os.open(logfile, os.O_RDWR | os.O_CREAT)
765     os.dup2(0, 1)
766     os.dup2(0, 2)
767
768
769 def write_pidfile(fname):
770     '''write a pid file, cleanup on exit'''
771     f = open(fname, mode='w')
772     f.write("%u\n" % os.getpid())
773     f.close()
774
775
776 def rebase_tree(rebase_url, rebase_branch="master"):
777     rebase_remote = "rebaseon"
778     do_print("Rebasing on %s" % rebase_url)
779     run_cmd("git describe HEAD", show=True, dir=test_master)
780     run_cmd("git remote add -t %s %s %s" %
781             (rebase_branch, rebase_remote, rebase_url),
782             show=True, dir=test_master)
783     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
784     if options.fix_whitespace:
785         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
786                 (rebase_remote, rebase_branch),
787                 show=True, dir=test_master)
788     else:
789         run_cmd("git rebase --force-rebase %s/%s" %
790                 (rebase_remote, rebase_branch),
791                 show=True, dir=test_master)
792     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
793                    (rebase_remote, rebase_branch),
794                    dir=test_master, output=True)
795     if diff == '':
796         do_print("No differences between HEAD and %s/%s - exiting" %
797                  (rebase_remote, rebase_branch))
798         sys.exit(0)
799     run_cmd("git describe %s/%s" %
800             (rebase_remote, rebase_branch),
801             show=True, dir=test_master)
802     run_cmd("git describe HEAD", show=True, dir=test_master)
803     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
804             (rebase_remote, rebase_branch),
805             show=True, dir=test_master)
806
807
808 def push_to(push_url, push_branch="master"):
809     push_remote = "pushto"
810     do_print("Pushing to %s" % push_url)
811     if options.mark:
812         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
813         run_cmd("git commit --amend -c HEAD", dir=test_master)
814         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
815         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
816     run_cmd("git remote add -t %s %s %s" %
817             (push_branch, push_remote, push_url),
818             show=True, dir=test_master)
819     run_cmd("git push %s +HEAD:%s" %
820             (push_remote, push_branch),
821             show=True, dir=test_master)
822
823
824 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
825
826 gitroot = find_git_root()
827 if gitroot is None:
828     raise Exception("Failed to find git root")
829
830 parser = OptionParser()
831 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
832 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
833 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
834 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
835                   default=def_testbase)
836 parser.add_option("", "--passcmd", help="command to run on success", default=None)
837 parser.add_option("", "--verbose", help="show all commands as they are run",
838                   default=False, action="store_true")
839 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
840                   default=None, type='str')
841 parser.add_option("", "--pushto", help="push to a git url on success",
842                   default=None, type='str')
843 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
844                   default=False, action="store_true")
845 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
846                   default=False, action="store_true")
847 parser.add_option("", "--retry", help="automatically retry if master changes",
848                   default=False, action="store_true")
849 parser.add_option("", "--email", help="send email to the given address on failure",
850                   type='str', default=None)
851 parser.add_option("", "--email-from", help="send email from the given address",
852                   type='str', default="autobuild@samba.org")
853 parser.add_option("", "--email-server", help="send email via the given server",
854                   type='str', default='localhost')
855 parser.add_option("", "--always-email", help="always send email, even on success",
856                   action="store_true")
857 parser.add_option("", "--daemon", help="daemonize after initial setup",
858                   action="store_true")
859 parser.add_option("", "--branch", help="the branch to work on (default=master)",
860                   default="master", type='str')
861 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
862                   default=gitroot, type='str')
863 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
864                   default=False, action="store_true")
865 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
866                   default='')
867
868
869 def send_email(subject, text, log_tar):
870     if options.email is None:
871         do_print("not sending email because the recipient is not set")
872         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
873                  (subject, text))
874         return
875     outer = MIMEMultipart()
876     outer['Subject'] = subject
877     outer['To'] = options.email
878     outer['From'] = options.email_from
879     outer['Date'] = email.utils.formatdate(localtime=True)
880     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
881     outer.attach(MIMEText(text, 'plain'))
882     if options.attach_logs:
883         fp = open(log_tar, 'rb')
884         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
885         fp.close()
886         # Set the filename parameter
887         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
888         outer.attach(msg)
889     content = outer.as_string()
890     s = smtplib.SMTP(options.email_server)
891     s.sendmail(options.email_from, [options.email], content)
892     s.set_debuglevel(1)
893     s.quit()
894
895
896 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
897                   elapsed_time, log_base=None, add_log_tail=True):
898     '''send an email to options.email about the failure'''
899     elapsed_minutes = elapsed_time / 60.0
900     if log_base is None:
901         log_base = gitroot
902     text = '''
903 Dear Developer,
904
905 Your autobuild on %s failed after %.1f minutes
906 when trying to test %s with the following error:
907
908    %s
909
910 the autobuild has been abandoned. Please fix the error and resubmit.
911
912 A summary of the autobuild process is here:
913
914   %s/autobuild.log
915 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
916
917     if options.restrict_tests:
918         text += """
919 The build was restricted to tests matching %s\n""" % options.restrict_tests
920
921     if failed_task != 'rebase':
922         text += '''
923 You can see logs of the failed task here:
924
925   %s/%s.stdout
926   %s/%s.stderr
927
928 or you can get full logs of all tasks in this job here:
929
930   %s/logs.tar.gz
931
932 The top commit for the tree that was built was:
933
934 %s
935
936 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
937
938     if add_log_tail:
939         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
940         lines = f.readlines()
941         log_tail = "".join(lines[-50:])
942         num_lines = len(lines)
943         if num_lines < 50:
944             # Also include stderr (compile failures) if < 50 lines of stdout
945             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
946             log_tail += "".join(f.readlines()[-(50 - num_lines):])
947
948         text += '''
949 The last 50 lines of log messages:
950
951 %s
952     ''' % log_tail
953         f.close()
954
955     logs = os.path.join(gitroot, 'logs.tar.gz')
956     send_email('autobuild[%s] failure on %s for task %s during %s'
957                % (options.branch, platform.node(), failed_task, failed_stage),
958                text, logs)
959
960
961 def email_success(elapsed_time, log_base=None):
962     '''send an email to options.email about a successful build'''
963     if log_base is None:
964         log_base = gitroot
965     text = '''
966 Dear Developer,
967
968 Your autobuild on %s has succeeded after %.1f minutes.
969
970 ''' % (platform.node(), elapsed_time / 60.)
971
972     if options.restrict_tests:
973         text += """
974 The build was restricted to tests matching %s\n""" % options.restrict_tests
975
976     if options.keeplogs:
977         text += '''
978
979 you can get full logs of all tasks in this job here:
980
981   %s/logs.tar.gz
982
983 ''' % log_base
984
985     text += '''
986 The top commit for the tree that was built was:
987
988 %s
989 ''' % top_commit_msg
990
991     logs = os.path.join(gitroot, 'logs.tar.gz')
992     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
993                text, logs)
994
995
996 (options, args) = parser.parse_args()
997
998 if options.retry:
999     if options.rebase is None:
1000         raise Exception('You can only use --retry if you also rebase')
1001
1002 testbase = "%s/b%u" % (options.testbase, os.getpid())
1003 test_master = "%s/master" % testbase
1004 test_prefix = "%s/prefix" % testbase
1005 test_tmpdir = "%s/tmp" % testbase
1006 os.environ['TMPDIR'] = test_tmpdir
1007
1008 # get the top commit message, for emails
1009 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1010
1011 try:
1012     os.makedirs(testbase)
1013 except Exception as reason:
1014     raise Exception("Unable to create %s : %s" % (testbase, reason))
1015 cleanup_list.append(testbase)
1016
1017 if options.daemon:
1018     logfile = os.path.join(testbase, "log")
1019     do_print("Forking into the background, writing progress to %s" % logfile)
1020     daemonize(logfile)
1021
1022 write_pidfile(gitroot + "/autobuild.pid")
1023
1024 start_time = time.time()
1025
1026 while True:
1027     try:
1028         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1029         os.makedirs(test_tmpdir)
1030         # The waf uninstall code removes empty directories all the way
1031         # up the tree.  Creating a file in test_tmpdir stops it from
1032         # being removed.
1033         run_cmd("touch %s" % os.path.join(test_tmpdir,
1034                                           ".directory-is-not-empty"), show=True)
1035         run_cmd("stat %s" % test_tmpdir, show=True)
1036         run_cmd("stat %s" % testbase, show=True)
1037         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1038     except Exception:
1039         cleanup()
1040         raise
1041
1042     try:
1043         try:
1044             if options.rebase is not None:
1045                 rebase_tree(options.rebase, rebase_branch=options.branch)
1046         except Exception:
1047             cleanup_list.append(gitroot + "/autobuild.pid")
1048             cleanup()
1049             elapsed_time = time.time() - start_time
1050             email_failure(-1, 'rebase', 'rebase', 'rebase',
1051                           'rebase on %s failed' % options.branch,
1052                           elapsed_time, log_base=options.log_base)
1053             sys.exit(1)
1054         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1055         if options.tail:
1056             blist.start_tail()
1057         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1058         if status != 0 or errstr != "retry":
1059             break
1060         cleanup()
1061     except Exception:
1062         cleanup()
1063         raise
1064
1065 cleanup_list.append(gitroot + "/autobuild.pid")
1066
1067 do_print(errstr)
1068
1069 blist.kill_kids()
1070 if options.tail:
1071     do_print("waiting for tail to flush")
1072     time.sleep(1)
1073
1074 elapsed_time = time.time() - start_time
1075 if status == 0:
1076     if options.passcmd is not None:
1077         do_print("Running passcmd: %s" % options.passcmd)
1078         run_cmd(options.passcmd, dir=test_master)
1079     if options.pushto is not None:
1080         push_to(options.pushto, push_branch=options.branch)
1081     if options.keeplogs or options.attach_logs:
1082         blist.tarlogs("logs.tar.gz")
1083         do_print("Logs in logs.tar.gz")
1084     if options.always_email:
1085         email_success(elapsed_time, log_base=options.log_base)
1086     blist.remove_logs()
1087     cleanup()
1088     do_print(errstr)
1089     sys.exit(0)
1090
1091 # something failed, gather a tar of the logs
1092 blist.tarlogs("logs.tar.gz")
1093
1094 if options.email is not None:
1095     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1096                   elapsed_time, log_base=options.log_base)
1097 else:
1098     elapsed_minutes = elapsed_time / 60.0
1099     print('''
1100
1101 ####################################################################
1102
1103 AUTOBUILD FAILURE
1104
1105 Your autobuild[%s] on %s failed after %.1f minutes
1106 when trying to test %s with the following error:
1107
1108    %s
1109
1110 the autobuild has been abandoned. Please fix the error and resubmit.
1111
1112 ####################################################################
1113
1114 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1115
1116 cleanup()
1117 do_print(errstr)
1118 do_print("Logs in logs.tar.gz")
1119 sys.exit(status)