b1bd9d1686c025d7847aa1528d977be3400950d2
[samba.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 from optparse import OptionParser
13 import smtplib
14 import email
15 from email.mime.text import MIMEText
16 from email.mime.base import MIMEBase
17 from email.mime.application import MIMEApplication
18 from email.mime.multipart import MIMEMultipart
19 from distutils.sysconfig import get_python_lib
20 import platform
21
22 try:
23     from waflib.Build import CACHE_SUFFIX
24 except ImportError:
25     sys.path.insert(0, "./third_party/waf")
26     from waflib.Build import CACHE_SUFFIX
27
28
29 os.environ["PYTHONUNBUFFERED"] = "1"
30
31 # This speeds up testing remarkably.
32 os.environ['TDB_NO_FSYNC'] = '1'
33
34 cleanup_list = []
35
36 builddirs = {
37     "ctdb": "ctdb",
38     "samba": ".",
39     "samba-py2": ".",
40     "samba-nt4": ".",
41     "samba-nt4-py2": ".",
42     "samba-fileserver": ".",
43     "samba-xc": ".",
44     "samba-o3": ".",
45     "samba-ctdb": ".",
46     "samba-libs": ".",
47     "samba-libs-py2": ".",
48     "samba-static": ".",
49     "samba-none-env": ".",
50     "samba-ad-dc": ".",
51     "samba-ad-dc-py2": ".",
52     "samba-ad-dc-ntvfs": ".",
53     "samba-ad-dc-ntvfs-py2": ".",
54     "samba-ad-dc-2": ".",
55     "samba-ad-dc-2-py2": ".",
56     "samba-ad-dc-backup": ".",
57     "samba-systemkrb5": ".",
58     "samba-nopython": ".",
59     "samba-buildpy2-only": ".",
60     "ldb": "lib/ldb",
61     "tdb": "lib/tdb",
62     "talloc": "lib/talloc",
63     "replace": "lib/replace",
64     "tevent": "lib/tevent",
65     "pidl": "pidl"
66 }
67
68 defaulttasks = builddirs.keys()
69
70 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
71     defaulttasks.remove("samba-o3")
72
73 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
74 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
75
76 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
77 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
78 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
79 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
80 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
81 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
82 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
83
84 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
85     extra_python = ""
86 else:
87     extra_python = "--extra-python=/usr/bin/python2"
88
89 tasks = {
90     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
91                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
92                ("make", "make all", "text/plain"),
93                ("install", "make install", "text/plain"),
94                ("test", "make autotest", "text/plain"),
95                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
96                ("clean", "make clean", "text/plain")],
97
98     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
99     "samba": [
100                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
101                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
102                 ("make", "make -j", "text/plain"),
103                 ("test", "make test FAIL_IMMEDIATELY=1 "
104                  "TESTS='${PY3_ONLY}"
105                  "--exclude-env=none "
106                  "--exclude-env=nt4_dc "
107                  "--exclude-env=nt4_member "
108                  "--exclude-env=ad_dc "
109                  "--exclude-env=ad_dc_ntvfs "
110                  "--exclude-env=ad_dc_no_nss "
111                  "--exclude-env=fl2003dc "
112                  "--exclude-env=fl2008r2dc "
113                  "--exclude-env=ad_member "
114                  "--exclude-env=ad_member_idmap_rid "
115                  "--exclude-env=ad_member_idmap_ad "
116                  "--exclude-env=chgdcpass "
117                  "--exclude-env=vampire_2000_dc "
118                  "--exclude-env=fl2000dc "
119                  "--exclude-env=fileserver "
120                  "--exclude-env=backupfromdc "
121                  "--exclude-env=restoredc "
122                  "--exclude-env=renamedc "
123                  "--exclude-env=offlinebackupdc "
124                  "--exclude-env=labdc "
125                  "'",
126                  "text/plain"),
127                 ("install", "make install", "text/plain"),
128                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
129                 ("clean", "make clean", "text/plain")],
130
131     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
132                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
133                     ("make", "make -j", "text/plain"),
134                     ("test", "make test FAIL_IMMEDIATELY=1 "
135                      "TESTS='${PY3_ONLY}"
136                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
137                     ("install", "make install", "text/plain"),
138                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
139                     ("clean", "make clean", "text/plain")],
140
141     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
142                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
143                            ("make", "make -j", "text/plain"),
144                            ("test", "make test FAIL_IMMEDIATELY=1 "
145                             "TESTS='${PY3_ONLY}"
146                             "--include-env=fileserver'", "text/plain"),
147                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
148
149     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
150                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
151                       ("make", "make -j", "text/plain"),
152                       ("test", "make test FAIL_IMMEDIATELY=1 "
153                        "TESTS='${PY3_ONLY}"
154                        "--include-env=ad_dc "
155                        "--include-env=fl2003dc "
156                        "--include-env=fl2008r2dc "
157                        "--include-env=ad_member "
158                        "--include-env=ad_member_idmap_rid "
159                        "--include-env=ad_member_idmap_ad'", "text/plain"),
160                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
161
162     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
163                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
164                         ("make", "make -j", "text/plain"),
165                         ("test", "make test FAIL_IMMEDIATELY=1 "
166                          "TESTS='${PY3_ONLY}"
167                          "--include-env=chgdcpass "
168                          "--include-env=vampire_2000_dc "
169                          "--include-env=fl2000dc "
170                          "--include-env=ad_dc_no_nss "
171                          "'",
172                          "text/plain"),
173                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
174
175     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
176     # This is currently the longest task, so we don't randomly delay it.
177     "samba-ad-dc-ntvfs": [
178                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
179                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
180                       ("make", "make -j", "text/plain"),
181                       ("test", "make test FAIL_IMMEDIATELY=1 "
182                        "TESTS='${PY3_ONLY}"
183                        "--include-env=ad_dc_ntvfs "
184                        "'",
185                        "text/plain"),
186                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
187
188     # run the backup/restore testenvs separately as they're fairly standalone
189     # (and CI seems to max out at ~8 different DCs running at once)
190     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
191                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
192                         ("make", "make -j", "text/plain"),
193                         ("test", "make test FAIL_IMMEDIATELY=1 "
194                          "TESTS='${PY3_ONLY}"
195                          "--include-env=backupfromdc "
196                          "--include-env=restoredc "
197                          "--include-env=renamedc "
198                          "--include-env=offlinebackupdc "
199                          "--include-env=labdc "
200                          "'",
201                          "text/plain"),
202                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
203
204     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
205                           ("make", "make -j", "text/plain"),
206                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
207
208     # Test cross-compile infrastructure
209     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
210                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
211                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
212                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
213                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
214                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
215                    ("compare-results", "script/compare_cc_results.py "
216                     "./bin/c4che/default{} "
217                     "./bin-xe/c4che/default{} "
218                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
219
220     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
221     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
222                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
223                    ("make", "make -j", "text/plain"),
224                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
225                     "TESTS='${PY3_ONLY}"
226                     "--include-env=ad_dc'", "text/plain"),
227                    ("install", "make install", "text/plain"),
228                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
229                    ("clean", "make clean", "text/plain")],
230
231     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
232
233                      # make sure we have tdb around:
234                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
235                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
236                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
237
238
239                      # build samba with cluster support (also building ctdb):
240                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
241                      ("samba-make", "make", "text/plain"),
242                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
243                      ("samba-install", "make install", "text/plain"),
244                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
245
246                      # clean up:
247                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
248                      ("clean", "make clean", "text/plain"),
249                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
250
251     "samba-libs": [
252                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
253                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
254                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
255                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
256
257                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
258                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
259                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
260
261                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
262                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
263                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
264
265                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
266                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
267                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
268
269                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
270                       ("nondevel-make", "make -j", "text/plain"),
271                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
272                       ("nondevel-install", "make install", "text/plain"),
273                       ("nondevel-dist", "make dist", "text/plain"),
274
275                       # retry with all modules shared
276                       ("allshared-distclean", "make distclean", "text/plain"),
277                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
278                       ("allshared-make", "make -j", "text/plain")],
279
280     "samba-none-env": [
281                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
282                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
283                       ("make", "make -j", "text/plain"),
284                       ("test", "make test "
285                        "FAIL_IMMEDIATELY=1 "
286                        "TESTS='${PY3_ONLY}"
287                        "--include-env=none'",
288                        "text/plain")],
289
290     "samba-static": [
291                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
292                       # build with all modules static
293                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
294                       ("allstatic-make", "make -j", "text/plain"),
295                       ("allstatic-test", "make test "
296                        "FAIL_IMMEDIATELY=1 "
297                        "TESTS='samba3.smb2.create.*nt4_dc'",
298                        "text/plain"),
299
300                       # retry without any required modules
301                       ("none-distclean", "make distclean", "text/plain"),
302                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
303                       ("none-make", "make -j", "text/plain"),
304
305                       # retry with nonshared smbd and smbtorture
306                       ("nonshared-distclean", "make distclean", "text/plain"),
307                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
308                       ("nonshared-make", "make -j", "text/plain")],
309
310     "samba-systemkrb5": [
311                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
312                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
313                       ("make", "make -j", "text/plain"),
314                       # we currently cannot run a full make test, a limited list of tests could be run
315                       # via "make test TESTS=sometests"
316                       ("test", "make test FAIL_IMMEDIATELY=1 "
317                        "TESTS='${PY3_ONLY}"
318                        "--include-env=ktest'", "text/plain"),
319                       ("install", "make install", "text/plain"),
320                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
321                       ("clean", "make clean", "text/plain")
322                       ],
323
324     # Test Samba without python still builds.  When this test fails
325     # due to more use of Python, the expectations is that the newly
326     # failing part of the code should be disabled when
327     # --disable-python is set (rather than major work being done to
328     # support this environment).  The target here is for vendors
329     # shipping a minimal smbd.
330     "samba-nopython": [
331                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
332                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
333                       ("make", "make -j", "text/plain"),
334                       ("install", "make install", "text/plain"),
335                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
336                       ("clean", "make clean", "text/plain"),
337
338                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
339                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
340                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
341
342                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
343                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
344                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
345
346                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
347                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
348                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
349
350                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
351                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
352                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
353
354                       # retry against installed library packages
355                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
356                       ("libs-make", "make -j", "text/plain"),
357                       ("libs-install", "make install", "text/plain"),
358                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
359                       ("libs-clean", "make clean", "text/plain")
360                       ],
361
362
363
364     "ldb": [
365               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
366               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
367               ("make", "make", "text/plain"),
368               ("install", "make install", "text/plain"),
369               ("test", "make test", "text/plain"),
370               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
371               ("make-no-lmdb", "make", "text/plain"),
372               ("install-no-lmdb", "make install", "text/plain"),
373               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
374               ("distcheck", "make distcheck", "text/plain"),
375               ("clean", "make clean", "text/plain")],
376
377     "tdb": [
378               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
379               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
380               ("make", "make", "text/plain"),
381               ("install", "make install", "text/plain"),
382               ("test", "make test", "text/plain"),
383               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
384               ("distcheck", "make distcheck", "text/plain"),
385               ("clean", "make clean", "text/plain")],
386
387     "talloc": [
388                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
389                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
390                  ("make", "make", "text/plain"),
391                  ("install", "make install", "text/plain"),
392                  ("test", "make test", "text/plain"),
393                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
394                  ("distcheck", "make distcheck", "text/plain"),
395                  ("clean", "make clean", "text/plain")],
396
397     "replace": [
398                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
399                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
400                   ("make", "make", "text/plain"),
401                   ("install", "make install", "text/plain"),
402                   ("test", "make test", "text/plain"),
403                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
404                   ("distcheck", "make distcheck", "text/plain"),
405                   ("clean", "make clean", "text/plain")],
406
407     "tevent": [
408                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
409                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
410                  ("make", "make", "text/plain"),
411                  ("install", "make install", "text/plain"),
412                  ("test", "make test", "text/plain"),
413                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
414                  ("distcheck", "make distcheck", "text/plain"),
415                  ("clean", "make clean", "text/plain")],
416
417     "pidl": [
418         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
419         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
420         ("touch", "touch *.yp", "text/plain"),
421         ("make", "make", "text/plain"),
422         ("test", "make test", "text/plain"),
423         ("install", "make install", "text/plain"),
424         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
425         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
426         ("clean", "make clean", "text/plain")],
427
428     "samba-buildpy2-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
429                    ("configure", "PYTHON='python' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
430                    ("make", "PYTHON='python' make -j", "text/plain"),
431                    ("install", "PYTHON='python' make install", "text/plain"),
432                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
433                    ("clean", "PYTHON='python' make clean", "text/plain")],
434
435
436     # these are useful for debugging autobuild
437     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
438     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
439
440
441 }
442
443
444 def do_print(msg):
445     print("%s" % msg)
446     sys.stdout.flush()
447     sys.stderr.flush()
448
449
450 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
451     if show is None:
452         show = options.verbose
453     if show:
454         do_print("Running: '%s' in '%s'" % (cmd, dir))
455     if output:
456         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
457     elif checkfail:
458         return check_call(cmd, shell=True, cwd=dir)
459     else:
460         return call(cmd, shell=True, cwd=dir)
461
462
463 class builder(object):
464     '''handle build of one directory'''
465
466     def __init__(self, name, sequence, cp=True, py3=False):
467         self.name = name
468         self.py3 = py3
469         if name in builddirs:
470             self.dir = builddirs[name]
471         else:
472             self.dir = "."
473
474         self.tag = self.name.replace('/', '_')
475         self.sequence = sequence
476         self.next = 0
477         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
478         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
479         if options.verbose:
480             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
481             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
482         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
483         self.stdout = open(self.stdout_path, 'w')
484         self.stderr = open(self.stderr_path, 'w')
485         self.stdin  = open("/dev/null", 'r')
486         self.sdir = "%s/%s" % (testbase, self.tag)
487         self.prefix = "%s/%s" % (test_prefix, self.tag)
488         run_cmd("rm -rf %s" % self.sdir)
489         run_cmd("rm -rf %s" % self.prefix)
490         if cp:
491             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
492         else:
493             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
494         self.start_next()
495
496     def start_next(self):
497         if self.next == len(self.sequence):
498             if not options.nocleanup:
499                 run_cmd("rm -rf %s" % self.sdir)
500                 run_cmd("rm -rf %s" % self.prefix)
501             do_print('%s: Completed OK' % self.name)
502             self.done = True
503             return
504         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
505         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
506         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
507         if self.py3:
508             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
509             # The trailing space is important
510             self.cmd = self.cmd.replace("${PY3_ONLY}", "python2 ")
511         else:
512             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
513             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
514         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
515         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
516 #        if self.output_mime_type == "text/x-subunit":
517 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
518         cwd = os.getcwd()
519         os.chdir("%s/%s" % (self.sdir, self.dir))
520         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
521         self.proc = Popen(self.cmd, shell=True,
522                           close_fds=True,
523                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
524         os.chdir(cwd)
525         self.next += 1
526
527
528 class buildlist(object):
529     '''handle build of multiple directories'''
530
531     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
532         global tasks
533         self.tlist = []
534         self.tail_proc = None
535         self.retry = None
536         if tasknames == []:
537             if options.restrict_tests:
538                 tasknames = ["samba-test-only"]
539             else:
540                 tasknames = defaulttasks
541         else:
542             # If we are only running one test,
543             # do not sleep randomly to wait for it to start
544             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
545
546         for n in tasknames:
547             if n not in tasks and n.endswith("-py2"):
548                 b = builder(n,
549                             tasks[n[:-4]],
550                             cp=n is not "pidl",
551                             py3=True)
552             else:
553                 b = builder(n, tasks[n], cp=n is not "pidl")
554             self.tlist.append(b)
555         if options.retry:
556             rebase_remote = "rebaseon"
557             retry_task = [("retry",
558                             '''set -e
559                             git remote add -t %s %s %s
560                             git fetch %s
561                             while :; do
562                               sleep 60
563                               git describe %s/%s > old_remote_branch.desc
564                               git fetch %s
565                               git describe %s/%s > remote_branch.desc
566                               diff old_remote_branch.desc remote_branch.desc
567                             done
568                            ''' % (
569                                 rebase_branch, rebase_remote, rebase_url,
570                                rebase_remote,
571                                rebase_remote, rebase_branch,
572                                rebase_remote,
573                                rebase_remote, rebase_branch
574                             ),
575                             "test/plain")]
576
577             self.retry = builder('retry', retry_task, cp=False)
578             self.need_retry = False
579
580     def kill_kids(self):
581         if self.tail_proc is not None:
582             self.tail_proc.terminate()
583             self.tail_proc.wait()
584             self.tail_proc = None
585         if self.retry is not None:
586             self.retry.proc.terminate()
587             self.retry.proc.wait()
588             self.retry = None
589         for b in self.tlist:
590             if b.proc is not None:
591                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
592                 b.proc.terminate()
593                 b.proc.wait()
594                 b.proc = None
595
596     def wait_one(self):
597         while True:
598             none_running = True
599             for b in self.tlist:
600                 if b.proc is None:
601                     continue
602                 none_running = False
603                 b.status = b.proc.poll()
604                 if b.status is None:
605                     continue
606                 b.proc = None
607                 return b
608             if options.retry:
609                 ret = self.retry.proc.poll()
610                 if ret is not None:
611                     self.need_retry = True
612                     self.retry = None
613                     return None
614             if none_running:
615                 return None
616             time.sleep(0.1)
617
618     def run(self):
619         while True:
620             b = self.wait_one()
621             if options.retry and self.need_retry:
622                 self.kill_kids()
623                 do_print("retry needed")
624                 return (0, None, None, None, "retry")
625             if b is None:
626                 break
627             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
628                 self.kill_kids()
629                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
630             b.start_next()
631         self.kill_kids()
632         return (0, None, None, None, "All OK")
633
634     def write_system_info(self):
635         filename = 'system-info.txt'
636         f = open(filename, 'w')
637         for cmd in ['uname -a',
638                     'free',
639                     'cat /proc/cpuinfo',
640                     'cc --version',
641                     'df -m .',
642                     'df -m %s' % testbase]:
643             out = run_cmd(cmd, output=True, checkfail=False)
644             print('### %s' % cmd, file=f)
645             print(out.decode('utf8', 'backslashreplace'), file=f)
646             print(file=f)
647         f.close()
648         return filename
649
650     def tarlogs(self, fname):
651         tar = tarfile.open(fname, "w:gz")
652         for b in self.tlist:
653             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
654             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
655         if os.path.exists("autobuild.log"):
656             tar.add("autobuild.log")
657         sys_info = self.write_system_info()
658         tar.add(sys_info)
659         tar.close()
660
661     def remove_logs(self):
662         for b in self.tlist:
663             os.unlink(b.stdout_path)
664             os.unlink(b.stderr_path)
665
666     def start_tail(self):
667         cmd = ["tail", "-f"]
668         for b in self.tlist:
669             cmd.append(b.stdout_path)
670             cmd.append(b.stderr_path)
671         self.tail_proc = Popen(cmd, close_fds=True)
672
673
674 def cleanup():
675     if options.nocleanup:
676         return
677     run_cmd("stat %s || true" % test_tmpdir, show=True)
678     run_cmd("stat %s" % testbase, show=True)
679     do_print("Cleaning up %r" % cleanup_list)
680     for d in cleanup_list:
681         run_cmd("rm -rf %s" % d)
682
683
684 def find_git_root():
685     '''get to the top of the git repo'''
686     p = os.getcwd()
687     while p != '/':
688         if os.path.isdir(os.path.join(p, ".git")):
689             return p
690         p = os.path.abspath(os.path.join(p, '..'))
691     return None
692
693
694 def daemonize(logfile):
695     pid = os.fork()
696     if pid == 0:  # Parent
697         os.setsid()
698         pid = os.fork()
699         if pid != 0:  # Actual daemon
700             os._exit(0)
701     else:  # Grandparent
702         os._exit(0)
703
704     import resource      # Resource usage information.
705     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
706     if maxfd == resource.RLIM_INFINITY:
707         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
708     for fd in range(0, maxfd):
709         try:
710             os.close(fd)
711         except OSError:
712             pass
713     os.open(logfile, os.O_RDWR | os.O_CREAT)
714     os.dup2(0, 1)
715     os.dup2(0, 2)
716
717
718 def write_pidfile(fname):
719     '''write a pid file, cleanup on exit'''
720     f = open(fname, mode='w')
721     f.write("%u\n" % os.getpid())
722     f.close()
723
724
725 def rebase_tree(rebase_url, rebase_branch="master"):
726     rebase_remote = "rebaseon"
727     do_print("Rebasing on %s" % rebase_url)
728     run_cmd("git describe HEAD", show=True, dir=test_master)
729     run_cmd("git remote add -t %s %s %s" %
730             (rebase_branch, rebase_remote, rebase_url),
731             show=True, dir=test_master)
732     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
733     if options.fix_whitespace:
734         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
735                 (rebase_remote, rebase_branch),
736                 show=True, dir=test_master)
737     else:
738         run_cmd("git rebase --force-rebase %s/%s" %
739                 (rebase_remote, rebase_branch),
740                 show=True, dir=test_master)
741     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
742                    (rebase_remote, rebase_branch),
743                    dir=test_master, output=True)
744     if diff == '':
745         do_print("No differences between HEAD and %s/%s - exiting" %
746                  (rebase_remote, rebase_branch))
747         sys.exit(0)
748     run_cmd("git describe %s/%s" %
749             (rebase_remote, rebase_branch),
750             show=True, dir=test_master)
751     run_cmd("git describe HEAD", show=True, dir=test_master)
752     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
753             (rebase_remote, rebase_branch),
754             show=True, dir=test_master)
755
756
757 def push_to(push_url, push_branch="master"):
758     push_remote = "pushto"
759     do_print("Pushing to %s" % push_url)
760     if options.mark:
761         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
762         run_cmd("git commit --amend -c HEAD", dir=test_master)
763         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
764         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
765     run_cmd("git remote add -t %s %s %s" %
766             (push_branch, push_remote, push_url),
767             show=True, dir=test_master)
768     run_cmd("git push %s +HEAD:%s" %
769             (push_remote, push_branch),
770             show=True, dir=test_master)
771
772
773 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
774
775 gitroot = find_git_root()
776 if gitroot is None:
777     raise Exception("Failed to find git root")
778
779 parser = OptionParser()
780 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
781 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
782 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
783 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
784                   default=def_testbase)
785 parser.add_option("", "--passcmd", help="command to run on success", default=None)
786 parser.add_option("", "--verbose", help="show all commands as they are run",
787                   default=False, action="store_true")
788 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
789                   default=None, type='str')
790 parser.add_option("", "--pushto", help="push to a git url on success",
791                   default=None, type='str')
792 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
793                   default=False, action="store_true")
794 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
795                   default=False, action="store_true")
796 parser.add_option("", "--retry", help="automatically retry if master changes",
797                   default=False, action="store_true")
798 parser.add_option("", "--email", help="send email to the given address on failure",
799                   type='str', default=None)
800 parser.add_option("", "--email-from", help="send email from the given address",
801                   type='str', default="autobuild@samba.org")
802 parser.add_option("", "--email-server", help="send email via the given server",
803                   type='str', default='localhost')
804 parser.add_option("", "--always-email", help="always send email, even on success",
805                   action="store_true")
806 parser.add_option("", "--daemon", help="daemonize after initial setup",
807                   action="store_true")
808 parser.add_option("", "--branch", help="the branch to work on (default=master)",
809                   default="master", type='str')
810 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
811                   default=gitroot, type='str')
812 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
813                   default=False, action="store_true")
814 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
815                   default='')
816
817
818 def send_email(subject, text, log_tar):
819     if options.email is None:
820         do_print("not sending email because the recipient is not set")
821         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
822                  (subject, text))
823         return
824     outer = MIMEMultipart()
825     outer['Subject'] = subject
826     outer['To'] = options.email
827     outer['From'] = options.email_from
828     outer['Date'] = email.utils.formatdate(localtime=True)
829     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
830     outer.attach(MIMEText(text, 'plain'))
831     if options.attach_logs:
832         fp = open(log_tar, 'rb')
833         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
834         fp.close()
835         # Set the filename parameter
836         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
837         outer.attach(msg)
838     content = outer.as_string()
839     s = smtplib.SMTP(options.email_server)
840     s.sendmail(options.email_from, [options.email], content)
841     s.set_debuglevel(1)
842     s.quit()
843
844
845 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
846                   elapsed_time, log_base=None, add_log_tail=True):
847     '''send an email to options.email about the failure'''
848     elapsed_minutes = elapsed_time / 60.0
849     if log_base is None:
850         log_base = gitroot
851     text = '''
852 Dear Developer,
853
854 Your autobuild on %s failed after %.1f minutes
855 when trying to test %s with the following error:
856
857    %s
858
859 the autobuild has been abandoned. Please fix the error and resubmit.
860
861 A summary of the autobuild process is here:
862
863   %s/autobuild.log
864 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
865
866     if options.restrict_tests:
867         text += """
868 The build was restricted to tests matching %s\n""" % options.restrict_tests
869
870     if failed_task != 'rebase':
871         text += '''
872 You can see logs of the failed task here:
873
874   %s/%s.stdout
875   %s/%s.stderr
876
877 or you can get full logs of all tasks in this job here:
878
879   %s/logs.tar.gz
880
881 The top commit for the tree that was built was:
882
883 %s
884
885 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
886
887     if add_log_tail:
888         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
889         lines = f.readlines()
890         log_tail = "".join(lines[-50:])
891         num_lines = len(lines)
892         if num_lines < 50:
893             # Also include stderr (compile failures) if < 50 lines of stdout
894             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
895             log_tail += "".join(f.readlines()[-(50 - num_lines):])
896
897         text += '''
898 The last 50 lines of log messages:
899
900 %s
901     ''' % log_tail
902         f.close()
903
904     logs = os.path.join(gitroot, 'logs.tar.gz')
905     send_email('autobuild[%s] failure on %s for task %s during %s'
906                % (options.branch, platform.node(), failed_task, failed_stage),
907                text, logs)
908
909
910 def email_success(elapsed_time, log_base=None):
911     '''send an email to options.email about a successful build'''
912     if log_base is None:
913         log_base = gitroot
914     text = '''
915 Dear Developer,
916
917 Your autobuild on %s has succeeded after %.1f minutes.
918
919 ''' % (platform.node(), elapsed_time / 60.)
920
921     if options.restrict_tests:
922         text += """
923 The build was restricted to tests matching %s\n""" % options.restrict_tests
924
925     if options.keeplogs:
926         text += '''
927
928 you can get full logs of all tasks in this job here:
929
930   %s/logs.tar.gz
931
932 ''' % log_base
933
934     text += '''
935 The top commit for the tree that was built was:
936
937 %s
938 ''' % top_commit_msg
939
940     logs = os.path.join(gitroot, 'logs.tar.gz')
941     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
942                text, logs)
943
944
945 (options, args) = parser.parse_args()
946
947 if options.retry:
948     if options.rebase is None:
949         raise Exception('You can only use --retry if you also rebase')
950
951 testbase = "%s/b%u" % (options.testbase, os.getpid())
952 test_master = "%s/master" % testbase
953 test_prefix = "%s/prefix" % testbase
954 test_tmpdir = "%s/tmp" % testbase
955 os.environ['TMPDIR'] = test_tmpdir
956
957 # get the top commit message, for emails
958 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
959 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
960
961 try:
962     os.makedirs(testbase)
963 except Exception as reason:
964     raise Exception("Unable to create %s : %s" % (testbase, reason))
965 cleanup_list.append(testbase)
966
967 if options.daemon:
968     logfile = os.path.join(testbase, "log")
969     do_print("Forking into the background, writing progress to %s" % logfile)
970     daemonize(logfile)
971
972 write_pidfile(gitroot + "/autobuild.pid")
973
974 start_time = time.time()
975
976 while True:
977     try:
978         run_cmd("rm -rf %s" % test_tmpdir, show=True)
979         os.makedirs(test_tmpdir)
980         # The waf uninstall code removes empty directories all the way
981         # up the tree.  Creating a file in test_tmpdir stops it from
982         # being removed.
983         run_cmd("touch %s" % os.path.join(test_tmpdir,
984                                           ".directory-is-not-empty"), show=True)
985         run_cmd("stat %s" % test_tmpdir, show=True)
986         run_cmd("stat %s" % testbase, show=True)
987         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
988     except Exception:
989         cleanup()
990         raise
991
992     try:
993         try:
994             if options.rebase is not None:
995                 rebase_tree(options.rebase, rebase_branch=options.branch)
996         except Exception:
997             cleanup_list.append(gitroot + "/autobuild.pid")
998             cleanup()
999             elapsed_time = time.time() - start_time
1000             email_failure(-1, 'rebase', 'rebase', 'rebase',
1001                           'rebase on %s failed' % options.branch,
1002                           elapsed_time, log_base=options.log_base)
1003             sys.exit(1)
1004         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1005         if options.tail:
1006             blist.start_tail()
1007         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1008         if status != 0 or errstr != "retry":
1009             break
1010         cleanup()
1011     except Exception:
1012         cleanup()
1013         raise
1014
1015 cleanup_list.append(gitroot + "/autobuild.pid")
1016
1017 do_print(errstr)
1018
1019 blist.kill_kids()
1020 if options.tail:
1021     do_print("waiting for tail to flush")
1022     time.sleep(1)
1023
1024 elapsed_time = time.time() - start_time
1025 if status == 0:
1026     if options.passcmd is not None:
1027         do_print("Running passcmd: %s" % options.passcmd)
1028         run_cmd(options.passcmd, dir=test_master)
1029     if options.pushto is not None:
1030         push_to(options.pushto, push_branch=options.branch)
1031     if options.keeplogs or options.attach_logs:
1032         blist.tarlogs("logs.tar.gz")
1033         do_print("Logs in logs.tar.gz")
1034     if options.always_email:
1035         email_success(elapsed_time, log_base=options.log_base)
1036     blist.remove_logs()
1037     cleanup()
1038     do_print(errstr)
1039     sys.exit(0)
1040
1041 # something failed, gather a tar of the logs
1042 blist.tarlogs("logs.tar.gz")
1043
1044 if options.email is not None:
1045     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1046                   elapsed_time, log_base=options.log_base)
1047 else:
1048     elapsed_minutes = elapsed_time / 60.0
1049     print('''
1050
1051 ####################################################################
1052
1053 AUTOBUILD FAILURE
1054
1055 Your autobuild[%s] on %s failed after %.1f minutes
1056 when trying to test %s with the following error:
1057
1058    %s
1059
1060 the autobuild has been abandoned. Please fix the error and resubmit.
1061
1062 ####################################################################
1063
1064 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1065
1066 cleanup()
1067 do_print(errstr)
1068 do_print("Logs in logs.tar.gz")
1069 sys.exit(status)