script/autobuild.py: add 'lsb_release -a' and 'mount' to system-info.txt
[samba.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 from optparse import OptionParser
13 import smtplib
14 import email
15 from email.mime.text import MIMEText
16 from email.mime.base import MIMEBase
17 from email.mime.application import MIMEApplication
18 from email.mime.multipart import MIMEMultipart
19 from distutils.sysconfig import get_python_lib
20 import platform
21
22 try:
23     from waflib.Build import CACHE_SUFFIX
24 except ImportError:
25     sys.path.insert(0, "./third_party/waf")
26     from waflib.Build import CACHE_SUFFIX
27
28
29 os.environ["PYTHONUNBUFFERED"] = "1"
30
31 # This speeds up testing remarkably.
32 os.environ['TDB_NO_FSYNC'] = '1'
33
34 cleanup_list = []
35
36 builddirs = {
37     "ctdb": "ctdb",
38     "samba": ".",
39     "samba-py2": ".",
40     "samba-nt4": ".",
41     "samba-nt4-py2": ".",
42     "samba-fileserver": ".",
43     "samba-xc": ".",
44     "samba-o3": ".",
45     "samba-ctdb": ".",
46     "samba-libs": ".",
47     "samba-libs-py2": ".",
48     "samba-static": ".",
49     "samba-none-env": ".",
50     "samba-ad-dc": ".",
51     "samba-ad-dc-py2": ".",
52     "samba-ad-dc-ntvfs": ".",
53     "samba-ad-dc-ntvfs-py2": ".",
54     "samba-ad-dc-2": ".",
55     "samba-ad-dc-2-py2": ".",
56     "samba-ad-dc-backup": ".",
57     "samba-systemkrb5": ".",
58     "samba-nopython": ".",
59     "samba-buildpy2-only": ".",
60     "ldb": "lib/ldb",
61     "tdb": "lib/tdb",
62     "talloc": "lib/talloc",
63     "replace": "lib/replace",
64     "tevent": "lib/tevent",
65     "pidl": "pidl"
66 }
67
68 defaulttasks = builddirs.keys()
69
70 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
71     defaulttasks.remove("samba-o3")
72
73 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
74 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
75
76 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
77 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
78 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
79 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
80 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
81 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
82 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
83
84 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
85     extra_python = ""
86 else:
87     extra_python = "--extra-python=/usr/bin/python2"
88
89 tasks = {
90     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
91                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
92                ("make", "make all", "text/plain"),
93                ("install", "make install", "text/plain"),
94                ("test", "make autotest", "text/plain"),
95                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
96                ("clean", "make clean", "text/plain")],
97
98     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
99     "samba": [
100                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
101                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
102                 ("make", "make -j", "text/plain"),
103                 ("test", "make test FAIL_IMMEDIATELY=1 "
104                  "TESTS='${PY3_ONLY}"
105                  "--exclude-env=none "
106                  "--exclude-env=nt4_dc "
107                  "--exclude-env=nt4_member "
108                  "--exclude-env=ad_dc "
109                  "--exclude-env=ad_dc_ntvfs "
110                  "--exclude-env=ad_dc_no_nss "
111                  "--exclude-env=fl2003dc "
112                  "--exclude-env=fl2008r2dc "
113                  "--exclude-env=ad_member "
114                  "--exclude-env=ad_member_idmap_rid "
115                  "--exclude-env=ad_member_idmap_ad "
116                  "--exclude-env=chgdcpass "
117                  "--exclude-env=vampire_2000_dc "
118                  "--exclude-env=fl2000dc "
119                  "--exclude-env=fileserver "
120                  "--exclude-env=backupfromdc "
121                  "--exclude-env=restoredc "
122                  "--exclude-env=renamedc "
123                  "--exclude-env=offlinebackupdc "
124                  "--exclude-env=labdc "
125                  "'",
126                  "text/plain"),
127                 ("install", "make install", "text/plain"),
128                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
129                 ("clean", "make clean", "text/plain")],
130
131     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
132                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
133                     ("make", "make -j", "text/plain"),
134                     ("test", "make test FAIL_IMMEDIATELY=1 "
135                      "TESTS='${PY3_ONLY}"
136                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
137                     ("install", "make install", "text/plain"),
138                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
139                     ("clean", "make clean", "text/plain")],
140
141     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
142                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
143                            ("make", "make -j", "text/plain"),
144                            ("test", "make test FAIL_IMMEDIATELY=1 "
145                             "TESTS='${PY3_ONLY}"
146                             "--include-env=fileserver'", "text/plain"),
147                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
148
149     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
150                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
151                       ("make", "make -j", "text/plain"),
152                       ("test", "make test FAIL_IMMEDIATELY=1 "
153                        "TESTS='${PY3_ONLY}"
154                        "--include-env=ad_dc "
155                        "--include-env=fl2003dc "
156                        "--include-env=fl2008r2dc "
157                        "--include-env=ad_member "
158                        "--include-env=ad_member_idmap_rid "
159                        "--include-env=ad_member_idmap_ad'", "text/plain"),
160                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
161
162     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
163                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
164                         ("make", "make -j", "text/plain"),
165                         ("test", "make test FAIL_IMMEDIATELY=1 "
166                          "TESTS='${PY3_ONLY}"
167                          "--include-env=chgdcpass "
168                          "--include-env=vampire_2000_dc "
169                          "--include-env=fl2000dc "
170                          "--include-env=ad_dc_no_nss "
171                          "'",
172                          "text/plain"),
173                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
174
175     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
176     # This is currently the longest task, so we don't randomly delay it.
177     "samba-ad-dc-ntvfs": [
178                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
179                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
180                       ("make", "make -j", "text/plain"),
181                       ("test", "make test FAIL_IMMEDIATELY=1 "
182                        "TESTS='${PY3_ONLY}"
183                        "--include-env=ad_dc_ntvfs "
184                        "'",
185                        "text/plain"),
186                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
187
188     # run the backup/restore testenvs separately as they're fairly standalone
189     # (and CI seems to max out at ~8 different DCs running at once)
190     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
191                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
192                         ("make", "make -j", "text/plain"),
193                         ("test", "make test FAIL_IMMEDIATELY=1 "
194                          "TESTS='${PY3_ONLY}"
195                          "--include-env=backupfromdc "
196                          "--include-env=restoredc "
197                          "--include-env=renamedc "
198                          "--include-env=offlinebackupdc "
199                          "--include-env=labdc "
200                          "'",
201                          "text/plain"),
202                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
203
204     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
205                           ("make", "make -j", "text/plain"),
206                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
207
208     # Test cross-compile infrastructure
209     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
210                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
211                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
212                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
213                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
214                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
215                    ("compare-results", "script/compare_cc_results.py "
216                     "./bin/c4che/default{} "
217                     "./bin-xe/c4che/default{} "
218                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
219
220     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
221     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
222                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
223                    ("make", "make -j", "text/plain"),
224                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
225                     "TESTS='${PY3_ONLY}"
226                     "--include-env=ad_dc'", "text/plain"),
227                    ("install", "make install", "text/plain"),
228                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
229                    ("clean", "make clean", "text/plain")],
230
231     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
232
233                      # make sure we have tdb around:
234                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
235                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
236                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
237
238
239                      # build samba with cluster support (also building ctdb):
240                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
241                      ("samba-make", "make", "text/plain"),
242                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
243                      ("samba-install", "make install", "text/plain"),
244                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
245
246                      # clean up:
247                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
248                      ("clean", "make clean", "text/plain"),
249                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
250
251     "samba-libs": [
252                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
253                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
254                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
255                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
256
257                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
258                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
259                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
260
261                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
262                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
263                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
264
265                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
266                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
267                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
268
269                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
270                       ("nondevel-make", "make -j", "text/plain"),
271                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
272                       ("nondevel-install", "make install", "text/plain"),
273                       ("nondevel-dist", "make dist", "text/plain"),
274
275                       # retry with all modules shared
276                       ("allshared-distclean", "make distclean", "text/plain"),
277                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
278                       ("allshared-make", "make -j", "text/plain")],
279
280     "samba-none-env": [
281                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
282                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
283                       ("make", "make -j", "text/plain"),
284                       ("test", "make test "
285                        "FAIL_IMMEDIATELY=1 "
286                        "TESTS='${PY3_ONLY}"
287                        "--include-env=none'",
288                        "text/plain")],
289
290     "samba-static": [
291                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
292                       # build with all modules static
293                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
294                       ("allstatic-make", "make -j", "text/plain"),
295                       ("allstatic-test", "make test "
296                        "FAIL_IMMEDIATELY=1 "
297                        "TESTS='samba3.smb2.create.*nt4_dc'",
298                        "text/plain"),
299
300                       # retry without any required modules
301                       ("none-distclean", "make distclean", "text/plain"),
302                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
303                       ("none-make", "make -j", "text/plain"),
304
305                       # retry with nonshared smbd and smbtorture
306                       ("nonshared-distclean", "make distclean", "text/plain"),
307                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
308                       ("nonshared-make", "make -j", "text/plain")],
309
310     "samba-systemkrb5": [
311                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
312                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
313                       ("make", "make -j", "text/plain"),
314                       # we currently cannot run a full make test, a limited list of tests could be run
315                       # via "make test TESTS=sometests"
316                       ("test", "make test FAIL_IMMEDIATELY=1 "
317                        "TESTS='${PY3_ONLY}"
318                        "--include-env=ktest'", "text/plain"),
319                       ("install", "make install", "text/plain"),
320                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
321                       ("clean", "make clean", "text/plain")
322                       ],
323
324     # Test Samba without python still builds.  When this test fails
325     # due to more use of Python, the expectations is that the newly
326     # failing part of the code should be disabled when
327     # --disable-python is set (rather than major work being done to
328     # support this environment).  The target here is for vendors
329     # shipping a minimal smbd.
330     "samba-nopython": [
331                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
332                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
333                       ("make", "make -j", "text/plain"),
334                       ("install", "make install", "text/plain"),
335                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
336                       ("clean", "make clean", "text/plain"),
337
338                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
339                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
340                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
341
342                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
343                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
344                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
345
346                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
347                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
348                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
349
350                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
351                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
352                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
353
354                       # retry against installed library packages
355                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
356                       ("libs-make", "make -j", "text/plain"),
357                       ("libs-install", "make install", "text/plain"),
358                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
359                       ("libs-clean", "make clean", "text/plain")
360                       ],
361
362
363
364     "ldb": [
365               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
366               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
367               ("make", "make", "text/plain"),
368               ("install", "make install", "text/plain"),
369               ("test", "make test", "text/plain"),
370               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
371               ("make-no-lmdb", "make", "text/plain"),
372               ("install-no-lmdb", "make install", "text/plain"),
373               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
374               ("distcheck", "make distcheck", "text/plain"),
375               ("clean", "make clean", "text/plain")],
376
377     "tdb": [
378               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
379               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
380               ("make", "make", "text/plain"),
381               ("install", "make install", "text/plain"),
382               ("test", "make test", "text/plain"),
383               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
384               ("distcheck", "make distcheck", "text/plain"),
385               ("clean", "make clean", "text/plain")],
386
387     "talloc": [
388                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
389                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
390                  ("make", "make", "text/plain"),
391                  ("install", "make install", "text/plain"),
392                  ("test", "make test", "text/plain"),
393                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
394                  ("distcheck", "make distcheck", "text/plain"),
395                  ("clean", "make clean", "text/plain")],
396
397     "replace": [
398                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
399                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
400                   ("make", "make", "text/plain"),
401                   ("install", "make install", "text/plain"),
402                   ("test", "make test", "text/plain"),
403                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
404                   ("distcheck", "make distcheck", "text/plain"),
405                   ("clean", "make clean", "text/plain")],
406
407     "tevent": [
408                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
409                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
410                  ("make", "make", "text/plain"),
411                  ("install", "make install", "text/plain"),
412                  ("test", "make test", "text/plain"),
413                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
414                  ("distcheck", "make distcheck", "text/plain"),
415                  ("clean", "make clean", "text/plain")],
416
417     "pidl": [
418         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
419         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
420         ("touch", "touch *.yp", "text/plain"),
421         ("make", "make", "text/plain"),
422         ("test", "make test", "text/plain"),
423         ("install", "make install", "text/plain"),
424         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
425         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
426         ("clean", "make clean", "text/plain")],
427
428     "samba-buildpy2-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
429                    ("configure", "PYTHON='python' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
430                    ("make", "PYTHON='python' make -j", "text/plain"),
431                    ("install", "PYTHON='python' make install", "text/plain"),
432                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
433                    ("clean", "PYTHON='python' make clean", "text/plain")],
434
435
436     # these are useful for debugging autobuild
437     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
438     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
439
440
441 }
442
443
444 def do_print(msg):
445     print("%s" % msg)
446     sys.stdout.flush()
447     sys.stderr.flush()
448
449
450 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
451     if show is None:
452         show = options.verbose
453     if show:
454         do_print("Running: '%s' in '%s'" % (cmd, dir))
455     if output:
456         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
457     elif checkfail:
458         return check_call(cmd, shell=True, cwd=dir)
459     else:
460         return call(cmd, shell=True, cwd=dir)
461
462
463 class builder(object):
464     '''handle build of one directory'''
465
466     def __init__(self, name, sequence, cp=True, py3=False):
467         self.name = name
468         self.py3 = py3
469         if name in builddirs:
470             self.dir = builddirs[name]
471         else:
472             self.dir = "."
473
474         self.tag = self.name.replace('/', '_')
475         self.sequence = sequence
476         self.next = 0
477         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
478         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
479         if options.verbose:
480             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
481             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
482         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
483         self.stdout = open(self.stdout_path, 'w')
484         self.stderr = open(self.stderr_path, 'w')
485         self.stdin  = open("/dev/null", 'r')
486         self.sdir = "%s/%s" % (testbase, self.tag)
487         self.prefix = "%s/%s" % (test_prefix, self.tag)
488         run_cmd("rm -rf %s" % self.sdir)
489         run_cmd("rm -rf %s" % self.prefix)
490         if cp:
491             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
492         else:
493             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
494         self.start_next()
495
496     def start_next(self):
497         if self.next == len(self.sequence):
498             if not options.nocleanup:
499                 run_cmd("rm -rf %s" % self.sdir)
500                 run_cmd("rm -rf %s" % self.prefix)
501             do_print('%s: Completed OK' % self.name)
502             self.done = True
503             return
504         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
505         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
506         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
507         if self.py3:
508             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
509             # The trailing space is important
510             self.cmd = self.cmd.replace("${PY3_ONLY}", "python2 ")
511         else:
512             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
513             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
514         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
515         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
516 #        if self.output_mime_type == "text/x-subunit":
517 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
518         cwd = os.getcwd()
519         os.chdir("%s/%s" % (self.sdir, self.dir))
520         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
521         self.proc = Popen(self.cmd, shell=True,
522                           close_fds=True,
523                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
524         os.chdir(cwd)
525         self.next += 1
526
527
528 class buildlist(object):
529     '''handle build of multiple directories'''
530
531     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
532         global tasks
533         self.tlist = []
534         self.tail_proc = None
535         self.retry = None
536         if tasknames == []:
537             if options.restrict_tests:
538                 tasknames = ["samba-test-only"]
539             else:
540                 tasknames = defaulttasks
541         else:
542             # If we are only running one test,
543             # do not sleep randomly to wait for it to start
544             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
545
546         for n in tasknames:
547             if n not in tasks and n.endswith("-py2"):
548                 b = builder(n,
549                             tasks[n[:-4]],
550                             cp=n is not "pidl",
551                             py3=True)
552             else:
553                 b = builder(n, tasks[n], cp=n is not "pidl")
554             self.tlist.append(b)
555         if options.retry:
556             rebase_remote = "rebaseon"
557             retry_task = [("retry",
558                             '''set -e
559                             git remote add -t %s %s %s
560                             git fetch %s
561                             while :; do
562                               sleep 60
563                               git describe %s/%s > old_remote_branch.desc
564                               git fetch %s
565                               git describe %s/%s > remote_branch.desc
566                               diff old_remote_branch.desc remote_branch.desc
567                             done
568                            ''' % (
569                                 rebase_branch, rebase_remote, rebase_url,
570                                rebase_remote,
571                                rebase_remote, rebase_branch,
572                                rebase_remote,
573                                rebase_remote, rebase_branch
574                             ),
575                             "test/plain")]
576
577             self.retry = builder('retry', retry_task, cp=False)
578             self.need_retry = False
579
580     def kill_kids(self):
581         if self.tail_proc is not None:
582             self.tail_proc.terminate()
583             self.tail_proc.wait()
584             self.tail_proc = None
585         if self.retry is not None:
586             self.retry.proc.terminate()
587             self.retry.proc.wait()
588             self.retry = None
589         for b in self.tlist:
590             if b.proc is not None:
591                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
592                 b.proc.terminate()
593                 b.proc.wait()
594                 b.proc = None
595
596     def wait_one(self):
597         while True:
598             none_running = True
599             for b in self.tlist:
600                 if b.proc is None:
601                     continue
602                 none_running = False
603                 b.status = b.proc.poll()
604                 if b.status is None:
605                     continue
606                 b.proc = None
607                 return b
608             if options.retry:
609                 ret = self.retry.proc.poll()
610                 if ret is not None:
611                     self.need_retry = True
612                     self.retry = None
613                     return None
614             if none_running:
615                 return None
616             time.sleep(0.1)
617
618     def run(self):
619         while True:
620             b = self.wait_one()
621             if options.retry and self.need_retry:
622                 self.kill_kids()
623                 do_print("retry needed")
624                 return (0, None, None, None, "retry")
625             if b is None:
626                 break
627             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
628                 self.kill_kids()
629                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
630             b.start_next()
631         self.kill_kids()
632         return (0, None, None, None, "All OK")
633
634     def write_system_info(self):
635         filename = 'system-info.txt'
636         f = open(filename, 'w')
637         for cmd in ['uname -a',
638                     'lsb_release -a',
639                     'free',
640                     'mount',
641                     'cat /proc/cpuinfo',
642                     'cc --version',
643                     'df -m .',
644                     'df -m %s' % testbase]:
645             out = run_cmd(cmd, output=True, checkfail=False)
646             print('### %s' % cmd, file=f)
647             print(out.decode('utf8', 'backslashreplace'), file=f)
648             print(file=f)
649         f.close()
650         return filename
651
652     def tarlogs(self, fname):
653         tar = tarfile.open(fname, "w:gz")
654         for b in self.tlist:
655             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
656             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
657         if os.path.exists("autobuild.log"):
658             tar.add("autobuild.log")
659         sys_info = self.write_system_info()
660         tar.add(sys_info)
661         tar.close()
662
663     def remove_logs(self):
664         for b in self.tlist:
665             os.unlink(b.stdout_path)
666             os.unlink(b.stderr_path)
667
668     def start_tail(self):
669         cmd = ["tail", "-f"]
670         for b in self.tlist:
671             cmd.append(b.stdout_path)
672             cmd.append(b.stderr_path)
673         self.tail_proc = Popen(cmd, close_fds=True)
674
675
676 def cleanup():
677     if options.nocleanup:
678         return
679     run_cmd("stat %s || true" % test_tmpdir, show=True)
680     run_cmd("stat %s" % testbase, show=True)
681     do_print("Cleaning up %r" % cleanup_list)
682     for d in cleanup_list:
683         run_cmd("rm -rf %s" % d)
684
685
686 def find_git_root():
687     '''get to the top of the git repo'''
688     p = os.getcwd()
689     while p != '/':
690         if os.path.isdir(os.path.join(p, ".git")):
691             return p
692         p = os.path.abspath(os.path.join(p, '..'))
693     return None
694
695
696 def daemonize(logfile):
697     pid = os.fork()
698     if pid == 0:  # Parent
699         os.setsid()
700         pid = os.fork()
701         if pid != 0:  # Actual daemon
702             os._exit(0)
703     else:  # Grandparent
704         os._exit(0)
705
706     import resource      # Resource usage information.
707     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
708     if maxfd == resource.RLIM_INFINITY:
709         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
710     for fd in range(0, maxfd):
711         try:
712             os.close(fd)
713         except OSError:
714             pass
715     os.open(logfile, os.O_RDWR | os.O_CREAT)
716     os.dup2(0, 1)
717     os.dup2(0, 2)
718
719
720 def write_pidfile(fname):
721     '''write a pid file, cleanup on exit'''
722     f = open(fname, mode='w')
723     f.write("%u\n" % os.getpid())
724     f.close()
725
726
727 def rebase_tree(rebase_url, rebase_branch="master"):
728     rebase_remote = "rebaseon"
729     do_print("Rebasing on %s" % rebase_url)
730     run_cmd("git describe HEAD", show=True, dir=test_master)
731     run_cmd("git remote add -t %s %s %s" %
732             (rebase_branch, rebase_remote, rebase_url),
733             show=True, dir=test_master)
734     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
735     if options.fix_whitespace:
736         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
737                 (rebase_remote, rebase_branch),
738                 show=True, dir=test_master)
739     else:
740         run_cmd("git rebase --force-rebase %s/%s" %
741                 (rebase_remote, rebase_branch),
742                 show=True, dir=test_master)
743     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
744                    (rebase_remote, rebase_branch),
745                    dir=test_master, output=True)
746     if diff == '':
747         do_print("No differences between HEAD and %s/%s - exiting" %
748                  (rebase_remote, rebase_branch))
749         sys.exit(0)
750     run_cmd("git describe %s/%s" %
751             (rebase_remote, rebase_branch),
752             show=True, dir=test_master)
753     run_cmd("git describe HEAD", show=True, dir=test_master)
754     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
755             (rebase_remote, rebase_branch),
756             show=True, dir=test_master)
757
758
759 def push_to(push_url, push_branch="master"):
760     push_remote = "pushto"
761     do_print("Pushing to %s" % push_url)
762     if options.mark:
763         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
764         run_cmd("git commit --amend -c HEAD", dir=test_master)
765         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
766         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
767     run_cmd("git remote add -t %s %s %s" %
768             (push_branch, push_remote, push_url),
769             show=True, dir=test_master)
770     run_cmd("git push %s +HEAD:%s" %
771             (push_remote, push_branch),
772             show=True, dir=test_master)
773
774
775 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
776
777 gitroot = find_git_root()
778 if gitroot is None:
779     raise Exception("Failed to find git root")
780
781 parser = OptionParser()
782 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
783 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
784 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
785 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
786                   default=def_testbase)
787 parser.add_option("", "--passcmd", help="command to run on success", default=None)
788 parser.add_option("", "--verbose", help="show all commands as they are run",
789                   default=False, action="store_true")
790 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
791                   default=None, type='str')
792 parser.add_option("", "--pushto", help="push to a git url on success",
793                   default=None, type='str')
794 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
795                   default=False, action="store_true")
796 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
797                   default=False, action="store_true")
798 parser.add_option("", "--retry", help="automatically retry if master changes",
799                   default=False, action="store_true")
800 parser.add_option("", "--email", help="send email to the given address on failure",
801                   type='str', default=None)
802 parser.add_option("", "--email-from", help="send email from the given address",
803                   type='str', default="autobuild@samba.org")
804 parser.add_option("", "--email-server", help="send email via the given server",
805                   type='str', default='localhost')
806 parser.add_option("", "--always-email", help="always send email, even on success",
807                   action="store_true")
808 parser.add_option("", "--daemon", help="daemonize after initial setup",
809                   action="store_true")
810 parser.add_option("", "--branch", help="the branch to work on (default=master)",
811                   default="master", type='str')
812 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
813                   default=gitroot, type='str')
814 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
815                   default=False, action="store_true")
816 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
817                   default='')
818
819
820 def send_email(subject, text, log_tar):
821     if options.email is None:
822         do_print("not sending email because the recipient is not set")
823         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
824                  (subject, text))
825         return
826     outer = MIMEMultipart()
827     outer['Subject'] = subject
828     outer['To'] = options.email
829     outer['From'] = options.email_from
830     outer['Date'] = email.utils.formatdate(localtime=True)
831     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
832     outer.attach(MIMEText(text, 'plain'))
833     if options.attach_logs:
834         fp = open(log_tar, 'rb')
835         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
836         fp.close()
837         # Set the filename parameter
838         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
839         outer.attach(msg)
840     content = outer.as_string()
841     s = smtplib.SMTP(options.email_server)
842     s.sendmail(options.email_from, [options.email], content)
843     s.set_debuglevel(1)
844     s.quit()
845
846
847 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
848                   elapsed_time, log_base=None, add_log_tail=True):
849     '''send an email to options.email about the failure'''
850     elapsed_minutes = elapsed_time / 60.0
851     if log_base is None:
852         log_base = gitroot
853     text = '''
854 Dear Developer,
855
856 Your autobuild on %s failed after %.1f minutes
857 when trying to test %s with the following error:
858
859    %s
860
861 the autobuild has been abandoned. Please fix the error and resubmit.
862
863 A summary of the autobuild process is here:
864
865   %s/autobuild.log
866 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
867
868     if options.restrict_tests:
869         text += """
870 The build was restricted to tests matching %s\n""" % options.restrict_tests
871
872     if failed_task != 'rebase':
873         text += '''
874 You can see logs of the failed task here:
875
876   %s/%s.stdout
877   %s/%s.stderr
878
879 or you can get full logs of all tasks in this job here:
880
881   %s/logs.tar.gz
882
883 The top commit for the tree that was built was:
884
885 %s
886
887 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
888
889     if add_log_tail:
890         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
891         lines = f.readlines()
892         log_tail = "".join(lines[-50:])
893         num_lines = len(lines)
894         if num_lines < 50:
895             # Also include stderr (compile failures) if < 50 lines of stdout
896             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
897             log_tail += "".join(f.readlines()[-(50 - num_lines):])
898
899         text += '''
900 The last 50 lines of log messages:
901
902 %s
903     ''' % log_tail
904         f.close()
905
906     logs = os.path.join(gitroot, 'logs.tar.gz')
907     send_email('autobuild[%s] failure on %s for task %s during %s'
908                % (options.branch, platform.node(), failed_task, failed_stage),
909                text, logs)
910
911
912 def email_success(elapsed_time, log_base=None):
913     '''send an email to options.email about a successful build'''
914     if log_base is None:
915         log_base = gitroot
916     text = '''
917 Dear Developer,
918
919 Your autobuild on %s has succeeded after %.1f minutes.
920
921 ''' % (platform.node(), elapsed_time / 60.)
922
923     if options.restrict_tests:
924         text += """
925 The build was restricted to tests matching %s\n""" % options.restrict_tests
926
927     if options.keeplogs:
928         text += '''
929
930 you can get full logs of all tasks in this job here:
931
932   %s/logs.tar.gz
933
934 ''' % log_base
935
936     text += '''
937 The top commit for the tree that was built was:
938
939 %s
940 ''' % top_commit_msg
941
942     logs = os.path.join(gitroot, 'logs.tar.gz')
943     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
944                text, logs)
945
946
947 (options, args) = parser.parse_args()
948
949 if options.retry:
950     if options.rebase is None:
951         raise Exception('You can only use --retry if you also rebase')
952
953 testbase = "%s/b%u" % (options.testbase, os.getpid())
954 test_master = "%s/master" % testbase
955 test_prefix = "%s/prefix" % testbase
956 test_tmpdir = "%s/tmp" % testbase
957 os.environ['TMPDIR'] = test_tmpdir
958
959 # get the top commit message, for emails
960 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
961 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
962
963 try:
964     os.makedirs(testbase)
965 except Exception as reason:
966     raise Exception("Unable to create %s : %s" % (testbase, reason))
967 cleanup_list.append(testbase)
968
969 if options.daemon:
970     logfile = os.path.join(testbase, "log")
971     do_print("Forking into the background, writing progress to %s" % logfile)
972     daemonize(logfile)
973
974 write_pidfile(gitroot + "/autobuild.pid")
975
976 start_time = time.time()
977
978 while True:
979     try:
980         run_cmd("rm -rf %s" % test_tmpdir, show=True)
981         os.makedirs(test_tmpdir)
982         # The waf uninstall code removes empty directories all the way
983         # up the tree.  Creating a file in test_tmpdir stops it from
984         # being removed.
985         run_cmd("touch %s" % os.path.join(test_tmpdir,
986                                           ".directory-is-not-empty"), show=True)
987         run_cmd("stat %s" % test_tmpdir, show=True)
988         run_cmd("stat %s" % testbase, show=True)
989         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
990     except Exception:
991         cleanup()
992         raise
993
994     try:
995         try:
996             if options.rebase is not None:
997                 rebase_tree(options.rebase, rebase_branch=options.branch)
998         except Exception:
999             cleanup_list.append(gitroot + "/autobuild.pid")
1000             cleanup()
1001             elapsed_time = time.time() - start_time
1002             email_failure(-1, 'rebase', 'rebase', 'rebase',
1003                           'rebase on %s failed' % options.branch,
1004                           elapsed_time, log_base=options.log_base)
1005             sys.exit(1)
1006         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1007         if options.tail:
1008             blist.start_tail()
1009         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1010         if status != 0 or errstr != "retry":
1011             break
1012         cleanup()
1013     except Exception:
1014         cleanup()
1015         raise
1016
1017 cleanup_list.append(gitroot + "/autobuild.pid")
1018
1019 do_print(errstr)
1020
1021 blist.kill_kids()
1022 if options.tail:
1023     do_print("waiting for tail to flush")
1024     time.sleep(1)
1025
1026 elapsed_time = time.time() - start_time
1027 if status == 0:
1028     if options.passcmd is not None:
1029         do_print("Running passcmd: %s" % options.passcmd)
1030         run_cmd(options.passcmd, dir=test_master)
1031     if options.pushto is not None:
1032         push_to(options.pushto, push_branch=options.branch)
1033     if options.keeplogs or options.attach_logs:
1034         blist.tarlogs("logs.tar.gz")
1035         do_print("Logs in logs.tar.gz")
1036     if options.always_email:
1037         email_success(elapsed_time, log_base=options.log_base)
1038     blist.remove_logs()
1039     cleanup()
1040     do_print(errstr)
1041     sys.exit(0)
1042
1043 # something failed, gather a tar of the logs
1044 blist.tarlogs("logs.tar.gz")
1045
1046 if options.email is not None:
1047     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1048                   elapsed_time, log_base=options.log_base)
1049 else:
1050     elapsed_minutes = elapsed_time / 60.0
1051     print('''
1052
1053 ####################################################################
1054
1055 AUTOBUILD FAILURE
1056
1057 Your autobuild[%s] on %s failed after %.1f minutes
1058 when trying to test %s with the following error:
1059
1060    %s
1061
1062 the autobuild has been abandoned. Please fix the error and resubmit.
1063
1064 ####################################################################
1065
1066 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1067
1068 cleanup()
1069 do_print(errstr)
1070 do_print("Logs in logs.tar.gz")
1071 sys.exit(status)