waf: build: Respect --disable-python for third_party modules
[bbaumbach/samba-autobuild/.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 from optparse import OptionParser
13 import smtplib
14 import email
15 from email.mime.text import MIMEText
16 from email.mime.base import MIMEBase
17 from email.mime.application import MIMEApplication
18 from email.mime.multipart import MIMEMultipart
19 from distutils.sysconfig import get_python_lib
20 import platform
21
22 try:
23     from waflib.Build import CACHE_SUFFIX
24 except ImportError:
25     sys.path.insert(0, "./third_party/waf")
26     from waflib.Build import CACHE_SUFFIX
27
28
29 os.environ["PYTHONUNBUFFERED"] = "1"
30
31 # This speeds up testing remarkably.
32 os.environ['TDB_NO_FSYNC'] = '1'
33
34 cleanup_list = []
35
36 builddirs = {
37     "ctdb": "ctdb",
38     "samba": ".",
39     "samba-nt4": ".",
40     "samba-fileserver": ".",
41     "samba-ad-member": ".",
42     "samba-xc": ".",
43     "samba-o3": ".",
44     "samba-ctdb": ".",
45     "samba-libs": ".",
46     "samba-static": ".",
47     "samba-none-env": ".",
48     "samba-ad-dc-1": ".",
49     "samba-ad-dc-2": ".",
50     "samba-ad-dc-3": ".",
51     "samba-ad-dc-4": ".",
52     "samba-ad-dc-5": ".",
53     "samba-ad-dc-6": ".",
54     "samba-ad-dc-ntvfs": ".",
55     "samba-ad-dc-backup": ".",
56     "samba-systemkrb5": ".",
57     "samba-nopython": ".",
58     "samba-nopython-py2": ".",
59     "samba-schemaupgrade": ".",
60     "ldb": "lib/ldb",
61     "tdb": "lib/tdb",
62     "talloc": "lib/talloc",
63     "replace": "lib/replace",
64     "tevent": "lib/tevent",
65     "pidl": "pidl"
66 }
67
68 defaulttasks = builddirs.keys()
69
70 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
71     defaulttasks.remove("samba-o3")
72
73 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
74 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
75
76 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
77 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
78 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
79 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
80 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
81 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
82 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
83
84 tasks = {
85     "ctdb": [("random-sleep", "../script/random-sleep.sh 300 900", "text/plain"),
86                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
87                ("make", "make all", "text/plain"),
88                ("install", "make install", "text/plain"),
89                ("test", "make autotest", "text/plain"),
90                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
91                ("clean", "make clean", "text/plain")],
92
93     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
94     "samba": [
95                 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
96                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
97                 ("make", "make -j", "text/plain"),
98                 ("test", "make test FAIL_IMMEDIATELY=1 "
99                  "TESTS='--exclude-env=none "
100                  "--exclude-env=nt4_dc "
101                  "--exclude-env=nt4_dc_schannel "
102                  "--exclude-env=nt4_member "
103                  "--exclude-env=ad_dc "
104                  "--exclude-env=ad_dc_backup "
105                  "--exclude-env=ad_dc_ntvfs "
106                  "--exclude-env=ad_dc_default "
107                  "--exclude-env=ad_dc_slowtests "
108                  "--exclude-env=ad_dc_no_nss "
109                  "--exclude-env=ad_dc_no_ntlm "
110                  "--exclude-env=fl2003dc "
111                  "--exclude-env=fl2008dc "
112                  "--exclude-env=fl2008r2dc "
113                  "--exclude-env=ad_member "
114                  "--exclude-env=ad_member_idmap_rid "
115                  "--exclude-env=ad_member_idmap_ad "
116                  "--exclude-env=ad_member_rfc2307 "
117                  "--exclude-env=chgdcpass "
118                  "--exclude-env=vampire_2000_dc "
119                  "--exclude-env=fl2000dc "
120                  "--exclude-env=fileserver "
121                  "--exclude-env=maptoguest "
122                  "--exclude-env=simpleserver "
123                  "--exclude-env=backupfromdc "
124                  "--exclude-env=restoredc "
125                  "--exclude-env=renamedc "
126                  "--exclude-env=offlinebackupdc "
127                  "--exclude-env=labdc "
128                  "--exclude-env=preforkrestartdc "
129                  "--exclude-env=proclimitdc "
130                  "--exclude-env=promoted_dc "
131                  "--exclude-env=vampire_dc "
132                  "--exclude-env=rodc "
133                  "--exclude-env=ad_dc_default "
134                  "--exclude-env=ad_dc_slowtests "
135                  "--exclude-env=schema_pair_dc "
136                  "--exclude-env=schema_dc "
137                  "'",
138                  "text/plain"),
139                 ("install", "make install", "text/plain"),
140                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
141                 ("clean", "make clean", "text/plain")],
142
143     "samba-nt4": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
144                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
145                     ("make", "make -j", "text/plain"),
146                     ("test", "make test FAIL_IMMEDIATELY=1 "
147                      "TESTS='"
148                      "--include-env=nt4_dc "
149                      "--include-env=nt4_dc_schannel "
150                      "--include-env=nt4_member "
151                      "'", "text/plain"),
152                     ("install", "make install", "text/plain"),
153                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
154                     ("clean", "make clean", "text/plain")],
155
156     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
157                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
158                            ("make", "make -j", "text/plain"),
159                            ("test", "make test FAIL_IMMEDIATELY=1 "
160                             "TESTS='"
161                             "--include-env=fileserver "
162                             "--include-env=maptoguest "
163                             "--include-env=simpleserver "
164                             "'", "text/plain"),
165                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
166
167     "samba-ad-member": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
168                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
169                         ("make", "make -j", "text/plain"),
170                         ("test", "make test FAIL_IMMEDIATELY=1 "
171                          "TESTS='"
172                          "--include-env=ad_member "
173                          "--include-env=ad_member_idmap_rid "
174                          "--include-env=ad_member_idmap_ad "
175                          "--include-env=ad_member_rfc2307 "
176                          "'", "text/plain"),
177                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
178
179     "samba-ad-dc-1": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
180                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
181                       ("make", "make -j", "text/plain"),
182                       ("test", "make test FAIL_IMMEDIATELY=1 "
183                        "TESTS='--include-env=ad_dc "
184                        "--include-env=ad_dc_no_nss "
185                        "--include-env=ad_dc_no_ntlm "
186                        "'", "text/plain"),
187                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
188
189     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
190                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
191                         ("make", "make -j", "text/plain"),
192                         ("test", "make test FAIL_IMMEDIATELY=1 "
193                          "TESTS='"
194                          "--include-env=vampire_dc "
195                          "--include-env=vampire_2000_dc "
196                          "--include-env=rodc "
197                          "'", "text/plain"),
198                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
199
200     "samba-ad-dc-3": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
201                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
202                         ("make", "make -j", "text/plain"),
203                         ("test", "make test FAIL_IMMEDIATELY=1 "
204                          "TESTS='"
205                          "--include-env=promoted_dc "
206                          "--include-env=chgdcpass "
207                          "--include-env=preforkrestartdc "
208                          "--include-env=proclimitdc "
209                          "'", "text/plain"),
210                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
211
212     "samba-ad-dc-4": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
213                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
214                         ("make", "make -j", "text/plain"),
215                         ("test", "make test FAIL_IMMEDIATELY=1 "
216                          "TESTS='"
217                          "--include-env=fl2000dc "
218                          "--include-env=fl2003dc "
219                          "--include-env=fl2008dc "
220                          "--include-env=fl2008r2dc "
221                          "'", "text/plain"),
222                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
223
224     "samba-ad-dc-5": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
225                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
226                         ("make", "make -j", "text/plain"),
227                         ("test", "make test FAIL_IMMEDIATELY=1 "
228                          "TESTS='"
229                          "--include-env=ad_dc_default "
230                          "'", "text/plain"),
231                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
232
233     "samba-ad-dc-6": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
234                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
235                         ("make", "make -j", "text/plain"),
236                         ("test", "make test FAIL_IMMEDIATELY=1 "
237                          "TESTS='"
238                          "--include-env=ad_dc_slowtests "
239                          "'", "text/plain"),
240                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
241
242
243     "samba-schemaupgrade": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
244                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
245                         ("make", "make -j", "text/plain"),
246                         ("test", "make test FAIL_IMMEDIATELY=1 "
247                          "TESTS='"
248                          "--include-env=schema_dc "
249                          "--include-env=schema_pair_dc "
250                          "'", "text/plain"),
251                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
252
253     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
254     # This is currently the longest task, so we don't randomly delay it.
255     "samba-ad-dc-ntvfs": [
256                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
257                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
258                       ("make", "make -j", "text/plain"),
259                       ("test", "make test FAIL_IMMEDIATELY=1 "
260                        "TESTS='"
261                        "--include-env=ad_dc_ntvfs "
262                        "'", "text/plain"),
263                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
264
265     # run the backup/restore testenvs separately as they're fairly standalone
266     # (and CI seems to max out at ~8 different DCs running at once)
267     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
268                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
269                         ("make", "make -j", "text/plain"),
270                         ("test", "make test FAIL_IMMEDIATELY=1 "
271                          "TESTS='--include-env=backupfromdc "
272                          "--include-env=restoredc "
273                          "--include-env=renamedc "
274                          "--include-env=offlinebackupdc "
275                          "--include-env=labdc "
276                          "--include-env=ad_dc_backup "
277                          "'", "text/plain"),
278                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
279
280     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
281                           ("make", "make -j", "text/plain"),
282                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
283
284     # Test cross-compile infrastructure
285     "samba-xc": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
286                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
287                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
288                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
289                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
290                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
291                    ("compare-results", "script/compare_cc_results.py "
292                     "./bin/c4che/default{} "
293                     "./bin-xe/c4che/default{} "
294                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
295
296     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
297     "samba-o3": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
298                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
299                    ("make", "make -j", "text/plain"),
300                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
301                     "TESTS='--include-env=ad_dc'", "text/plain"),
302                    ("install", "make install", "text/plain"),
303                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
304                    ("clean", "make clean", "text/plain")],
305
306     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
307
308                      # make sure we have tdb around:
309                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
310                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
311                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
312
313
314                      # build samba with cluster support (also building ctdb):
315                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
316                      ("samba-make", "make", "text/plain"),
317                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
318                      ("samba-install", "make install", "text/plain"),
319                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
320
321                      # clean up:
322                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
323                      ("clean", "make clean", "text/plain"),
324                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
325
326     "samba-libs": [
327                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
328                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
329                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
330                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
331
332                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
333                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
334                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
335
336                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
337                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
338                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
339
340                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
341                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
342                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
343
344                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
345                       ("nondevel-make", "make -j", "text/plain"),
346                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
347                       ("nondevel-install", "make install", "text/plain"),
348                       ("nondevel-dist", "make dist", "text/plain"),
349
350                       # retry with all modules shared
351                       ("allshared-distclean", "make distclean", "text/plain"),
352                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
353                       ("allshared-make", "make -j", "text/plain")],
354
355     "samba-none-env": [
356                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
357                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
358                       ("make", "make -j", "text/plain"),
359                       ("test", "make test "
360                        "FAIL_IMMEDIATELY=1 "
361                        "TESTS='--include-env=none'",
362                        "text/plain")],
363
364     "samba-static": [
365                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
366                       # build with all modules static
367                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
368                       ("allstatic-make", "make -j", "text/plain"),
369                       ("allstatic-test", "make test "
370                        "FAIL_IMMEDIATELY=1 "
371                        "TESTS='samba3.smb2.create.*nt4_dc'",
372                        "text/plain"),
373
374                       # retry without any required modules
375                       ("none-distclean", "make distclean", "text/plain"),
376                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
377                       ("none-make", "make -j", "text/plain"),
378
379                       # retry with nonshared smbd and smbtorture
380                       ("nonshared-distclean", "make distclean", "text/plain"),
381                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
382                       ("nonshared-make", "make -j", "text/plain")],
383
384     "samba-systemkrb5": [
385                       ("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
386                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
387                       ("make", "make -j", "text/plain"),
388                       # we currently cannot run a full make test, a limited list of tests could be run
389                       # via "make test TESTS=sometests"
390                       ("test", "make test FAIL_IMMEDIATELY=1 "
391                        "TESTS='--include-env=ktest'", "text/plain"),
392                       ("install", "make install", "text/plain"),
393                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
394                       ("clean", "make clean", "text/plain")
395                       ],
396
397     # Test Samba without python still builds.  When this test fails
398     # due to more use of Python, the expectations is that the newly
399     # failing part of the code should be disabled when
400     # --disable-python is set (rather than major work being done to
401     # support this environment).  The target here is for vendors
402     # shipping a minimal smbd.
403     "samba-nopython": [
404                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
405                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
406                       ("make", "make -j", "text/plain"),
407                       ("install", "make install", "text/plain"),
408                       ("find-python", "script/find_python.sh ${PREFIX}", "text/plain"),
409                       ("test", "make test-nopython", "text/plain"),
410                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
411                       ("clean", "make clean", "text/plain"),
412
413                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
414                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
415                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
416
417                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
418                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
419                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
420
421                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
422                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
423                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
424
425                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
426                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
427                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
428
429                       # retry against installed library packages
430                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
431                       ("libs-make", "make -j", "text/plain"),
432                       ("libs-install", "make install", "text/plain"),
433                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
434                       ("libs-clean", "make clean", "text/plain")
435                       ],
436
437     # check we can do the same thing using python2
438     "samba-nopython-py2": [
439                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
440                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
441                       ("make", "PYTHON=python2 make -j", "text/plain"),
442                       ("install", "PYTHON=python2 make install", "text/plain"),
443                       ("find-python", "script/find_python.sh ${PREFIX}", "text/plain"),
444                       ("test", "make test-nopython", "text/plain"),
445                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
446                       ("clean", "PYTHON=python2 make clean", "text/plain"),
447
448                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
449                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
450                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
451
452                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
453                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
454                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
455
456                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
457                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
458                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
459
460                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
461                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
462                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
463
464                       # retry against installed library packages
465                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
466                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
467                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
468                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
469                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
470                       ],
471
472     "ldb": [
473               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
474               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
475               ("make", "make", "text/plain"),
476               ("install", "make install", "text/plain"),
477               ("test", "make test", "text/plain"),
478               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
479               ("make-no-lmdb", "make", "text/plain"),
480               ("install-no-lmdb", "make install", "text/plain"),
481               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
482               ("distcheck", "make distcheck", "text/plain"),
483               ("clean", "make clean", "text/plain")],
484
485     "tdb": [
486               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
487               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
488               ("make", "make", "text/plain"),
489               ("install", "make install", "text/plain"),
490               ("test", "make test", "text/plain"),
491               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
492               ("distcheck", "make distcheck", "text/plain"),
493               ("clean", "make clean", "text/plain")],
494
495     "talloc": [
496                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
497                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
498                  ("make", "make", "text/plain"),
499                  ("install", "make install", "text/plain"),
500                  ("test", "make test", "text/plain"),
501                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
502                  ("distcheck", "make distcheck", "text/plain"),
503                  ("clean", "make clean", "text/plain")],
504
505     "replace": [
506                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
507                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
508                   ("make", "make", "text/plain"),
509                   ("install", "make install", "text/plain"),
510                   ("test", "make test", "text/plain"),
511                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
512                   ("distcheck", "make distcheck", "text/plain"),
513                   ("clean", "make clean", "text/plain")],
514
515     "tevent": [
516                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
517                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
518                  ("make", "make", "text/plain"),
519                  ("install", "make install", "text/plain"),
520                  ("test", "make test", "text/plain"),
521                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
522                  ("distcheck", "make distcheck", "text/plain"),
523                  ("clean", "make clean", "text/plain")],
524
525     "pidl": [
526         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
527         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
528         ("touch", "touch *.yp", "text/plain"),
529         ("make", "make", "text/plain"),
530         ("test", "make test", "text/plain"),
531         ("install", "make install", "text/plain"),
532         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
533         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
534         ("clean", "make clean", "text/plain")],
535
536
537     # these are useful for debugging autobuild
538     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
539     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
540
541
542 }
543
544
545 def do_print(msg):
546     print("%s" % msg)
547     sys.stdout.flush()
548     sys.stderr.flush()
549
550
551 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
552     if show is None:
553         show = options.verbose
554     if show:
555         do_print("Running: '%s' in '%s'" % (cmd, dir))
556     if output:
557         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
558     elif checkfail:
559         return check_call(cmd, shell=True, cwd=dir)
560     else:
561         return call(cmd, shell=True, cwd=dir)
562
563
564 class builder(object):
565     '''handle build of one directory'''
566
567     def __init__(self, name, sequence, cp=True):
568         self.name = name
569         if name in builddirs:
570             self.dir = builddirs[name]
571         else:
572             self.dir = "."
573
574         self.tag = self.name.replace('/', '_')
575         self.sequence = sequence
576         self.next = 0
577         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
578         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
579         if options.verbose:
580             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
581             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
582         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
583         self.stdout = open(self.stdout_path, 'w')
584         self.stderr = open(self.stderr_path, 'w')
585         self.stdin  = open("/dev/null", 'r')
586         self.sdir = "%s/%s" % (testbase, self.tag)
587         self.prefix = "%s/%s" % (test_prefix, self.tag)
588         run_cmd("rm -rf %s" % self.sdir)
589         run_cmd("rm -rf %s" % self.prefix)
590         if cp:
591             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
592         else:
593             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
594         self.start_next()
595
596     def start_next(self):
597         if self.next == len(self.sequence):
598             if not options.nocleanup:
599                 run_cmd("rm -rf %s" % self.sdir)
600                 run_cmd("rm -rf %s" % self.prefix)
601             do_print('%s: Completed OK' % self.name)
602             self.done = True
603             return
604         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
605         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
606         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
607         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
608         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
609 #        if self.output_mime_type == "text/x-subunit":
610 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
611         cwd = os.getcwd()
612         os.chdir("%s/%s" % (self.sdir, self.dir))
613         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
614         self.proc = Popen(self.cmd, shell=True,
615                           close_fds=True,
616                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
617         os.chdir(cwd)
618         self.next += 1
619
620
621 class buildlist(object):
622     '''handle build of multiple directories'''
623
624     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
625         global tasks
626         self.tlist = []
627         self.tail_proc = None
628         self.retry = None
629         if tasknames == []:
630             if options.restrict_tests:
631                 tasknames = ["samba-test-only"]
632             else:
633                 tasknames = defaulttasks
634         else:
635             # If we are only running one test,
636             # do not sleep randomly to wait for it to start
637             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
638
639         for n in tasknames:
640             b = builder(n, tasks[n], cp=n is not "pidl")
641             self.tlist.append(b)
642         if options.retry:
643             rebase_remote = "rebaseon"
644             retry_task = [("retry",
645                             '''set -e
646                             git remote add -t %s %s %s
647                             git fetch %s
648                             while :; do
649                               sleep 60
650                               git describe %s/%s > old_remote_branch.desc
651                               git fetch %s
652                               git describe %s/%s > remote_branch.desc
653                               diff old_remote_branch.desc remote_branch.desc
654                             done
655                            ''' % (
656                                 rebase_branch, rebase_remote, rebase_url,
657                                rebase_remote,
658                                rebase_remote, rebase_branch,
659                                rebase_remote,
660                                rebase_remote, rebase_branch
661                             ),
662                             "test/plain")]
663
664             self.retry = builder('retry', retry_task, cp=False)
665             self.need_retry = False
666
667     def kill_kids(self):
668         if self.tail_proc is not None:
669             self.tail_proc.terminate()
670             self.tail_proc.wait()
671             self.tail_proc = None
672         if self.retry is not None:
673             self.retry.proc.terminate()
674             self.retry.proc.wait()
675             self.retry = None
676         for b in self.tlist:
677             if b.proc is not None:
678                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
679                 b.proc.terminate()
680                 b.proc.wait()
681                 b.proc = None
682
683     def wait_one(self):
684         while True:
685             none_running = True
686             for b in self.tlist:
687                 if b.proc is None:
688                     continue
689                 none_running = False
690                 b.status = b.proc.poll()
691                 if b.status is None:
692                     continue
693                 b.proc = None
694                 return b
695             if options.retry:
696                 ret = self.retry.proc.poll()
697                 if ret is not None:
698                     self.need_retry = True
699                     self.retry = None
700                     return None
701             if none_running:
702                 return None
703             time.sleep(0.1)
704
705     def run(self):
706         while True:
707             b = self.wait_one()
708             if options.retry and self.need_retry:
709                 self.kill_kids()
710                 do_print("retry needed")
711                 return (0, None, None, None, "retry")
712             if b is None:
713                 break
714             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
715                 self.kill_kids()
716                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
717             b.start_next()
718         self.kill_kids()
719         return (0, None, None, None, "All OK")
720
721     def write_system_info(self):
722         filename = 'system-info.txt'
723         f = open(filename, 'w')
724         for cmd in ['uname -a',
725                     'lsb_release -a',
726                     'free',
727                     'mount',
728                     'cat /proc/cpuinfo',
729                     'cc --version',
730                     'df -m .',
731                     'df -m %s' % testbase]:
732             out = run_cmd(cmd, output=True, checkfail=False)
733             print('### %s' % cmd, file=f)
734             print(out.decode('utf8', 'backslashreplace'), file=f)
735             print(file=f)
736         f.close()
737         return filename
738
739     def tarlogs(self, fname):
740         tar = tarfile.open(fname, "w:gz")
741         for b in self.tlist:
742             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
743             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
744         if os.path.exists("autobuild.log"):
745             tar.add("autobuild.log")
746         sys_info = self.write_system_info()
747         tar.add(sys_info)
748         tar.close()
749
750     def remove_logs(self):
751         for b in self.tlist:
752             os.unlink(b.stdout_path)
753             os.unlink(b.stderr_path)
754
755     def start_tail(self):
756         cmd = ["tail", "-f"]
757         for b in self.tlist:
758             cmd.append(b.stdout_path)
759             cmd.append(b.stderr_path)
760         self.tail_proc = Popen(cmd, close_fds=True)
761
762
763 def cleanup():
764     if options.nocleanup:
765         return
766     run_cmd("stat %s || true" % test_tmpdir, show=True)
767     run_cmd("stat %s" % testbase, show=True)
768     do_print("Cleaning up %r" % cleanup_list)
769     for d in cleanup_list:
770         run_cmd("rm -rf %s" % d)
771
772
773 def find_git_root():
774     '''get to the top of the git repo'''
775     p = os.getcwd()
776     while p != '/':
777         if os.path.isdir(os.path.join(p, ".git")):
778             return p
779         p = os.path.abspath(os.path.join(p, '..'))
780     return None
781
782
783 def daemonize(logfile):
784     pid = os.fork()
785     if pid == 0:  # Parent
786         os.setsid()
787         pid = os.fork()
788         if pid != 0:  # Actual daemon
789             os._exit(0)
790     else:  # Grandparent
791         os._exit(0)
792
793     import resource      # Resource usage information.
794     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
795     if maxfd == resource.RLIM_INFINITY:
796         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
797     for fd in range(0, maxfd):
798         try:
799             os.close(fd)
800         except OSError:
801             pass
802     os.open(logfile, os.O_RDWR | os.O_CREAT)
803     os.dup2(0, 1)
804     os.dup2(0, 2)
805
806
807 def write_pidfile(fname):
808     '''write a pid file, cleanup on exit'''
809     f = open(fname, mode='w')
810     f.write("%u\n" % os.getpid())
811     f.close()
812
813
814 def rebase_tree(rebase_url, rebase_branch="master"):
815     rebase_remote = "rebaseon"
816     do_print("Rebasing on %s" % rebase_url)
817     run_cmd("git describe HEAD", show=True, dir=test_master)
818     run_cmd("git remote add -t %s %s %s" %
819             (rebase_branch, rebase_remote, rebase_url),
820             show=True, dir=test_master)
821     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
822     if options.fix_whitespace:
823         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
824                 (rebase_remote, rebase_branch),
825                 show=True, dir=test_master)
826     else:
827         run_cmd("git rebase --force-rebase %s/%s" %
828                 (rebase_remote, rebase_branch),
829                 show=True, dir=test_master)
830     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
831                    (rebase_remote, rebase_branch),
832                    dir=test_master, output=True)
833     if diff == '':
834         do_print("No differences between HEAD and %s/%s - exiting" %
835                  (rebase_remote, rebase_branch))
836         sys.exit(0)
837     run_cmd("git describe %s/%s" %
838             (rebase_remote, rebase_branch),
839             show=True, dir=test_master)
840     run_cmd("git describe HEAD", show=True, dir=test_master)
841     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
842             (rebase_remote, rebase_branch),
843             show=True, dir=test_master)
844
845
846 def push_to(push_url, push_branch="master"):
847     push_remote = "pushto"
848     do_print("Pushing to %s" % push_url)
849     if options.mark:
850         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
851         run_cmd("git commit --amend -c HEAD", dir=test_master)
852         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
853         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
854     run_cmd("git remote add -t %s %s %s" %
855             (push_branch, push_remote, push_url),
856             show=True, dir=test_master)
857     run_cmd("git push %s +HEAD:%s" %
858             (push_remote, push_branch),
859             show=True, dir=test_master)
860
861
862 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
863
864 gitroot = find_git_root()
865 if gitroot is None:
866     raise Exception("Failed to find git root")
867
868 parser = OptionParser()
869 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
870 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
871 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
872 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
873                   default=def_testbase)
874 parser.add_option("", "--passcmd", help="command to run on success", default=None)
875 parser.add_option("", "--verbose", help="show all commands as they are run",
876                   default=False, action="store_true")
877 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
878                   default=None, type='str')
879 parser.add_option("", "--pushto", help="push to a git url on success",
880                   default=None, type='str')
881 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
882                   default=False, action="store_true")
883 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
884                   default=False, action="store_true")
885 parser.add_option("", "--retry", help="automatically retry if master changes",
886                   default=False, action="store_true")
887 parser.add_option("", "--email", help="send email to the given address on failure",
888                   type='str', default=None)
889 parser.add_option("", "--email-from", help="send email from the given address",
890                   type='str', default="autobuild@samba.org")
891 parser.add_option("", "--email-server", help="send email via the given server",
892                   type='str', default='localhost')
893 parser.add_option("", "--always-email", help="always send email, even on success",
894                   action="store_true")
895 parser.add_option("", "--daemon", help="daemonize after initial setup",
896                   action="store_true")
897 parser.add_option("", "--branch", help="the branch to work on (default=master)",
898                   default="master", type='str')
899 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
900                   default=gitroot, type='str')
901 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
902                   default=False, action="store_true")
903 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
904                   default='')
905
906
907 def send_email(subject, text, log_tar):
908     if options.email is None:
909         do_print("not sending email because the recipient is not set")
910         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
911                  (subject, text))
912         return
913     outer = MIMEMultipart()
914     outer['Subject'] = subject
915     outer['To'] = options.email
916     outer['From'] = options.email_from
917     outer['Date'] = email.utils.formatdate(localtime=True)
918     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
919     outer.attach(MIMEText(text, 'plain'))
920     if options.attach_logs:
921         fp = open(log_tar, 'rb')
922         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
923         fp.close()
924         # Set the filename parameter
925         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
926         outer.attach(msg)
927     content = outer.as_string()
928     s = smtplib.SMTP(options.email_server)
929     email_user = os.getenv('SMTP_USERNAME')
930     email_password = os.getenv('SMTP_PASSWORD')
931     if email_user is not None:
932         s.starttls()
933         s.login(email_user, email_password)
934
935     s.sendmail(options.email_from, [options.email], content)
936     s.set_debuglevel(1)
937     s.quit()
938
939
940 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
941                   elapsed_time, log_base=None, add_log_tail=True):
942     '''send an email to options.email about the failure'''
943     elapsed_minutes = elapsed_time / 60.0
944     if log_base is None:
945         log_base = gitroot
946     text = '''
947 Dear Developer,
948
949 Your autobuild on %s failed after %.1f minutes
950 when trying to test %s with the following error:
951
952    %s
953
954 the autobuild has been abandoned. Please fix the error and resubmit.
955
956 A summary of the autobuild process is here:
957
958   %s/autobuild.log
959 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
960
961     if options.restrict_tests:
962         text += """
963 The build was restricted to tests matching %s\n""" % options.restrict_tests
964
965     if failed_task != 'rebase':
966         text += '''
967 You can see logs of the failed task here:
968
969   %s/%s.stdout
970   %s/%s.stderr
971
972 or you can get full logs of all tasks in this job here:
973
974   %s/logs.tar.gz
975
976 The top commit for the tree that was built was:
977
978 %s
979
980 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
981
982     if add_log_tail:
983         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
984         lines = f.readlines()
985         log_tail = "".join(lines[-50:])
986         num_lines = len(lines)
987         if num_lines < 50:
988             # Also include stderr (compile failures) if < 50 lines of stdout
989             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
990             log_tail += "".join(f.readlines()[-(50 - num_lines):])
991
992         text += '''
993 The last 50 lines of log messages:
994
995 %s
996     ''' % log_tail
997         f.close()
998
999     logs = os.path.join(gitroot, 'logs.tar.gz')
1000     send_email('autobuild[%s] failure on %s for task %s during %s'
1001                % (options.branch, platform.node(), failed_task, failed_stage),
1002                text, logs)
1003
1004
1005 def email_success(elapsed_time, log_base=None):
1006     '''send an email to options.email about a successful build'''
1007     if log_base is None:
1008         log_base = gitroot
1009     text = '''
1010 Dear Developer,
1011
1012 Your autobuild on %s has succeeded after %.1f minutes.
1013
1014 ''' % (platform.node(), elapsed_time / 60.)
1015
1016     if options.restrict_tests:
1017         text += """
1018 The build was restricted to tests matching %s\n""" % options.restrict_tests
1019
1020     if options.keeplogs:
1021         text += '''
1022
1023 you can get full logs of all tasks in this job here:
1024
1025   %s/logs.tar.gz
1026
1027 ''' % log_base
1028
1029     text += '''
1030 The top commit for the tree that was built was:
1031
1032 %s
1033 ''' % top_commit_msg
1034
1035     logs = os.path.join(gitroot, 'logs.tar.gz')
1036     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1037                text, logs)
1038
1039
1040 (options, args) = parser.parse_args()
1041
1042 if options.retry:
1043     if options.rebase is None:
1044         raise Exception('You can only use --retry if you also rebase')
1045
1046 testbase = "%s/b%u" % (options.testbase, os.getpid())
1047 test_master = "%s/master" % testbase
1048 test_prefix = "%s/prefix" % testbase
1049 test_tmpdir = "%s/tmp" % testbase
1050 os.environ['TMPDIR'] = test_tmpdir
1051
1052 # get the top commit message, for emails
1053 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1054 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
1055
1056 try:
1057     os.makedirs(testbase)
1058 except Exception as reason:
1059     raise Exception("Unable to create %s : %s" % (testbase, reason))
1060 cleanup_list.append(testbase)
1061
1062 if options.daemon:
1063     logfile = os.path.join(testbase, "log")
1064     do_print("Forking into the background, writing progress to %s" % logfile)
1065     daemonize(logfile)
1066
1067 write_pidfile(gitroot + "/autobuild.pid")
1068
1069 start_time = time.time()
1070
1071 while True:
1072     try:
1073         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1074         os.makedirs(test_tmpdir)
1075         # The waf uninstall code removes empty directories all the way
1076         # up the tree.  Creating a file in test_tmpdir stops it from
1077         # being removed.
1078         run_cmd("touch %s" % os.path.join(test_tmpdir,
1079                                           ".directory-is-not-empty"), show=True)
1080         run_cmd("stat %s" % test_tmpdir, show=True)
1081         run_cmd("stat %s" % testbase, show=True)
1082         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1083     except Exception:
1084         cleanup()
1085         raise
1086
1087     try:
1088         try:
1089             if options.rebase is not None:
1090                 rebase_tree(options.rebase, rebase_branch=options.branch)
1091         except Exception:
1092             cleanup_list.append(gitroot + "/autobuild.pid")
1093             cleanup()
1094             elapsed_time = time.time() - start_time
1095             email_failure(-1, 'rebase', 'rebase', 'rebase',
1096                           'rebase on %s failed' % options.branch,
1097                           elapsed_time, log_base=options.log_base)
1098             sys.exit(1)
1099         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1100         if options.tail:
1101             blist.start_tail()
1102         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1103         if status != 0 or errstr != "retry":
1104             break
1105         cleanup()
1106     except Exception:
1107         cleanup()
1108         raise
1109
1110 cleanup_list.append(gitroot + "/autobuild.pid")
1111
1112 do_print(errstr)
1113
1114 blist.kill_kids()
1115 if options.tail:
1116     do_print("waiting for tail to flush")
1117     time.sleep(1)
1118
1119 elapsed_time = time.time() - start_time
1120 if status == 0:
1121     if options.passcmd is not None:
1122         do_print("Running passcmd: %s" % options.passcmd)
1123         run_cmd(options.passcmd, dir=test_master)
1124     if options.pushto is not None:
1125         push_to(options.pushto, push_branch=options.branch)
1126     if options.keeplogs or options.attach_logs:
1127         blist.tarlogs("logs.tar.gz")
1128         do_print("Logs in logs.tar.gz")
1129     if options.always_email:
1130         email_success(elapsed_time, log_base=options.log_base)
1131     blist.remove_logs()
1132     cleanup()
1133     do_print(errstr)
1134     sys.exit(0)
1135
1136 # something failed, gather a tar of the logs
1137 blist.tarlogs("logs.tar.gz")
1138
1139 if options.email is not None:
1140     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1141                   elapsed_time, log_base=options.log_base)
1142 else:
1143     elapsed_minutes = elapsed_time / 60.0
1144     print('''
1145
1146 ####################################################################
1147
1148 AUTOBUILD FAILURE
1149
1150 Your autobuild[%s] on %s failed after %.1f minutes
1151 when trying to test %s with the following error:
1152
1153    %s
1154
1155 the autobuild has been abandoned. Please fix the error and resubmit.
1156
1157 ####################################################################
1158
1159 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1160
1161 cleanup()
1162 do_print(errstr)
1163 do_print("Logs in logs.tar.gz")
1164 sys.exit(status)