autobuild: Drop py2 autobuild jobs
[amitay/samba.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 from optparse import OptionParser
13 import smtplib
14 import email
15 from email.mime.text import MIMEText
16 from email.mime.base import MIMEBase
17 from email.mime.application import MIMEApplication
18 from email.mime.multipart import MIMEMultipart
19 from distutils.sysconfig import get_python_lib
20 import platform
21
22 try:
23     from waflib.Build import CACHE_SUFFIX
24 except ImportError:
25     sys.path.insert(0, "./third_party/waf")
26     from waflib.Build import CACHE_SUFFIX
27
28
29 os.environ["PYTHONUNBUFFERED"] = "1"
30
31 # This speeds up testing remarkably.
32 os.environ['TDB_NO_FSYNC'] = '1'
33
34 cleanup_list = []
35
36 builddirs = {
37     "ctdb": "ctdb",
38     "samba": ".",
39     "samba-nt4": ".",
40     "samba-fileserver": ".",
41     "samba-xc": ".",
42     "samba-o3": ".",
43     "samba-ctdb": ".",
44     "samba-libs": ".",
45     "samba-static": ".",
46     "samba-none-env": ".",
47     "samba-ad-dc": ".",
48     "samba-ad-dc-ntvfs": ".",
49     "samba-ad-dc-2": ".",
50     "samba-ad-dc-backup": ".",
51     "samba-systemkrb5": ".",
52     "samba-nopython": ".",
53     "samba-buildpy2-only": ".",
54     "ldb": "lib/ldb",
55     "tdb": "lib/tdb",
56     "talloc": "lib/talloc",
57     "replace": "lib/replace",
58     "tevent": "lib/tevent",
59     "pidl": "pidl"
60 }
61
62 defaulttasks = builddirs.keys()
63
64 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
65     defaulttasks.remove("samba-o3")
66
67 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
68 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
69
70 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
71 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
72 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
73 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
74 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
75 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
76 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
77
78 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
79     extra_python = ""
80 else:
81     extra_python = "--extra-python=/usr/bin/python2"
82
83 tasks = {
84     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
85                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
86                ("make", "make all", "text/plain"),
87                ("install", "make install", "text/plain"),
88                ("test", "make autotest", "text/plain"),
89                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
90                ("clean", "make clean", "text/plain")],
91
92     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
93     "samba": [
94                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
95                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
96                 ("make", "make -j", "text/plain"),
97                 ("test", "make test FAIL_IMMEDIATELY=1 "
98                  "TESTS='${PY3_ONLY}"
99                  "--exclude-env=none "
100                  "--exclude-env=nt4_dc "
101                  "--exclude-env=nt4_member "
102                  "--exclude-env=ad_dc "
103                  "--exclude-env=ad_dc_ntvfs "
104                  "--exclude-env=ad_dc_no_nss "
105                  "--exclude-env=fl2003dc "
106                  "--exclude-env=fl2008r2dc "
107                  "--exclude-env=ad_member "
108                  "--exclude-env=ad_member_idmap_rid "
109                  "--exclude-env=ad_member_idmap_ad "
110                  "--exclude-env=chgdcpass "
111                  "--exclude-env=vampire_2000_dc "
112                  "--exclude-env=fl2000dc "
113                  "--exclude-env=fileserver "
114                  "--exclude-env=backupfromdc "
115                  "--exclude-env=restoredc "
116                  "--exclude-env=renamedc "
117                  "--exclude-env=offlinebackupdc "
118                  "--exclude-env=labdc "
119                  "'",
120                  "text/plain"),
121                 ("install", "make install", "text/plain"),
122                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
123                 ("clean", "make clean", "text/plain")],
124
125     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
126                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
127                     ("make", "make -j", "text/plain"),
128                     ("test", "make test FAIL_IMMEDIATELY=1 "
129                      "TESTS='${PY3_ONLY}"
130                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
131                     ("install", "make install", "text/plain"),
132                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
133                     ("clean", "make clean", "text/plain")],
134
135     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
136                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
137                            ("make", "make -j", "text/plain"),
138                            ("test", "make test FAIL_IMMEDIATELY=1 "
139                             "TESTS='${PY3_ONLY}"
140                             "--include-env=fileserver'", "text/plain"),
141                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
142
143     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
144                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
145                       ("make", "make -j", "text/plain"),
146                       ("test", "make test FAIL_IMMEDIATELY=1 "
147                        "TESTS='${PY3_ONLY}"
148                        "--include-env=ad_dc "
149                        "--include-env=fl2003dc "
150                        "--include-env=fl2008r2dc "
151                        "--include-env=ad_member "
152                        "--include-env=ad_member_idmap_rid "
153                        "--include-env=ad_member_idmap_ad'", "text/plain"),
154                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
155
156     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
157                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
158                         ("make", "make -j", "text/plain"),
159                         ("test", "make test FAIL_IMMEDIATELY=1 "
160                          "TESTS='${PY3_ONLY}"
161                          "--include-env=chgdcpass "
162                          "--include-env=vampire_2000_dc "
163                          "--include-env=fl2000dc "
164                          "--include-env=ad_dc_no_nss "
165                          "'",
166                          "text/plain"),
167                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
168
169     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
170     # This is currently the longest task, so we don't randomly delay it.
171     "samba-ad-dc-ntvfs": [
172                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
173                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
174                       ("make", "make -j", "text/plain"),
175                       ("test", "make test FAIL_IMMEDIATELY=1 "
176                        "TESTS='${PY3_ONLY}"
177                        "--include-env=ad_dc_ntvfs "
178                        "'",
179                        "text/plain"),
180                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
181
182     # run the backup/restore testenvs separately as they're fairly standalone
183     # (and CI seems to max out at ~8 different DCs running at once)
184     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
185                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
186                         ("make", "make -j", "text/plain"),
187                         ("test", "make test FAIL_IMMEDIATELY=1 "
188                          "TESTS='${PY3_ONLY}"
189                          "--include-env=backupfromdc "
190                          "--include-env=restoredc "
191                          "--include-env=renamedc "
192                          "--include-env=offlinebackupdc "
193                          "--include-env=labdc "
194                          "'",
195                          "text/plain"),
196                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
197
198     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
199                           ("make", "make -j", "text/plain"),
200                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
201
202     # Test cross-compile infrastructure
203     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
204                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
205                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
206                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
207                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
208                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
209                    ("compare-results", "script/compare_cc_results.py "
210                     "./bin/c4che/default{} "
211                     "./bin-xe/c4che/default{} "
212                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
213
214     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
215     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
216                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
217                    ("make", "make -j", "text/plain"),
218                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
219                     "TESTS='${PY3_ONLY}"
220                     "--include-env=ad_dc'", "text/plain"),
221                    ("install", "make install", "text/plain"),
222                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
223                    ("clean", "make clean", "text/plain")],
224
225     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
226
227                      # make sure we have tdb around:
228                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
229                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
230                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
231
232
233                      # build samba with cluster support (also building ctdb):
234                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
235                      ("samba-make", "make", "text/plain"),
236                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
237                      ("samba-install", "make install", "text/plain"),
238                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
239
240                      # clean up:
241                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
242                      ("clean", "make clean", "text/plain"),
243                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
244
245     "samba-libs": [
246                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
247                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
248                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
249                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
250
251                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
252                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
253                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
254
255                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
256                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
257                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
258
259                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
260                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
261                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
262
263                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
264                       ("nondevel-make", "make -j", "text/plain"),
265                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
266                       ("nondevel-install", "make install", "text/plain"),
267                       ("nondevel-dist", "make dist", "text/plain"),
268
269                       # retry with all modules shared
270                       ("allshared-distclean", "make distclean", "text/plain"),
271                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
272                       ("allshared-make", "make -j", "text/plain")],
273
274     "samba-none-env": [
275                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
276                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
277                       ("make", "make -j", "text/plain"),
278                       ("test", "make test "
279                        "FAIL_IMMEDIATELY=1 "
280                        "TESTS='${PY3_ONLY}"
281                        "--include-env=none'",
282                        "text/plain")],
283
284     "samba-static": [
285                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
286                       # build with all modules static
287                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
288                       ("allstatic-make", "make -j", "text/plain"),
289                       ("allstatic-test", "make test "
290                        "FAIL_IMMEDIATELY=1 "
291                        "TESTS='samba3.smb2.create.*nt4_dc'",
292                        "text/plain"),
293
294                       # retry without any required modules
295                       ("none-distclean", "make distclean", "text/plain"),
296                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
297                       ("none-make", "make -j", "text/plain"),
298
299                       # retry with nonshared smbd and smbtorture
300                       ("nonshared-distclean", "make distclean", "text/plain"),
301                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
302                       ("nonshared-make", "make -j", "text/plain")],
303
304     "samba-systemkrb5": [
305                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
306                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
307                       ("make", "make -j", "text/plain"),
308                       # we currently cannot run a full make test, a limited list of tests could be run
309                       # via "make test TESTS=sometests"
310                       ("test", "make test FAIL_IMMEDIATELY=1 "
311                        "TESTS='${PY3_ONLY}"
312                        "--include-env=ktest'", "text/plain"),
313                       ("install", "make install", "text/plain"),
314                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
315                       ("clean", "make clean", "text/plain")
316                       ],
317
318     # Test Samba without python still builds.  When this test fails
319     # due to more use of Python, the expectations is that the newly
320     # failing part of the code should be disabled when
321     # --disable-python is set (rather than major work being done to
322     # support this environment).  The target here is for vendors
323     # shipping a minimal smbd.
324     "samba-nopython": [
325                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
326                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
327                       ("make", "make -j", "text/plain"),
328                       ("install", "make install", "text/plain"),
329                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
330                       ("clean", "make clean", "text/plain"),
331
332                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
333                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
334                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
335
336                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
337                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
338                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
339
340                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
341                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
342                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
343
344                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
345                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
346                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
347
348                       # retry against installed library packages
349                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
350                       ("libs-make", "make -j", "text/plain"),
351                       ("libs-install", "make install", "text/plain"),
352                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
353                       ("libs-clean", "make clean", "text/plain")
354                       ],
355
356
357
358     "ldb": [
359               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
360               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
361               ("make", "make", "text/plain"),
362               ("install", "make install", "text/plain"),
363               ("test", "make test", "text/plain"),
364               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
365               ("make-no-lmdb", "make", "text/plain"),
366               ("install-no-lmdb", "make install", "text/plain"),
367               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
368               ("distcheck", "make distcheck", "text/plain"),
369               ("clean", "make clean", "text/plain")],
370
371     "tdb": [
372               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
373               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
374               ("make", "make", "text/plain"),
375               ("install", "make install", "text/plain"),
376               ("test", "make test", "text/plain"),
377               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
378               ("distcheck", "make distcheck", "text/plain"),
379               ("clean", "make clean", "text/plain")],
380
381     "talloc": [
382                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
383                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
384                  ("make", "make", "text/plain"),
385                  ("install", "make install", "text/plain"),
386                  ("test", "make test", "text/plain"),
387                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
388                  ("distcheck", "make distcheck", "text/plain"),
389                  ("clean", "make clean", "text/plain")],
390
391     "replace": [
392                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
393                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
394                   ("make", "make", "text/plain"),
395                   ("install", "make install", "text/plain"),
396                   ("test", "make test", "text/plain"),
397                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
398                   ("distcheck", "make distcheck", "text/plain"),
399                   ("clean", "make clean", "text/plain")],
400
401     "tevent": [
402                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
403                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
404                  ("make", "make", "text/plain"),
405                  ("install", "make install", "text/plain"),
406                  ("test", "make test", "text/plain"),
407                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
408                  ("distcheck", "make distcheck", "text/plain"),
409                  ("clean", "make clean", "text/plain")],
410
411     "pidl": [
412         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
413         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
414         ("touch", "touch *.yp", "text/plain"),
415         ("make", "make", "text/plain"),
416         ("test", "make test", "text/plain"),
417         ("install", "make install", "text/plain"),
418         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
419         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
420         ("clean", "make clean", "text/plain")],
421
422     "samba-buildpy2-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
423                    ("configure", "PYTHON='python' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
424                    ("make", "PYTHON='python' make -j", "text/plain"),
425                    ("install", "PYTHON='python' make install", "text/plain"),
426                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
427                    ("clean", "PYTHON='python' make clean", "text/plain")],
428
429
430     # these are useful for debugging autobuild
431     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
432     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
433
434
435 }
436
437
438 def do_print(msg):
439     print("%s" % msg)
440     sys.stdout.flush()
441     sys.stderr.flush()
442
443
444 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
445     if show is None:
446         show = options.verbose
447     if show:
448         do_print("Running: '%s' in '%s'" % (cmd, dir))
449     if output:
450         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
451     elif checkfail:
452         return check_call(cmd, shell=True, cwd=dir)
453     else:
454         return call(cmd, shell=True, cwd=dir)
455
456
457 class builder(object):
458     '''handle build of one directory'''
459
460     def __init__(self, name, sequence, cp=True, py3=False):
461         self.name = name
462         self.py3 = py3
463         if name in builddirs:
464             self.dir = builddirs[name]
465         else:
466             self.dir = "."
467
468         self.tag = self.name.replace('/', '_')
469         self.sequence = sequence
470         self.next = 0
471         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
472         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
473         if options.verbose:
474             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
475             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
476         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
477         self.stdout = open(self.stdout_path, 'w')
478         self.stderr = open(self.stderr_path, 'w')
479         self.stdin  = open("/dev/null", 'r')
480         self.sdir = "%s/%s" % (testbase, self.tag)
481         self.prefix = "%s/%s" % (test_prefix, self.tag)
482         run_cmd("rm -rf %s" % self.sdir)
483         run_cmd("rm -rf %s" % self.prefix)
484         if cp:
485             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
486         else:
487             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
488         self.start_next()
489
490     def start_next(self):
491         if self.next == len(self.sequence):
492             if not options.nocleanup:
493                 run_cmd("rm -rf %s" % self.sdir)
494                 run_cmd("rm -rf %s" % self.prefix)
495             do_print('%s: Completed OK' % self.name)
496             self.done = True
497             return
498         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
499         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
500         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
501         if self.py3:
502             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
503             # The trailing space is important
504             self.cmd = self.cmd.replace("${PY3_ONLY}", "python2 ")
505         else:
506             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
507             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
508         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
509         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
510 #        if self.output_mime_type == "text/x-subunit":
511 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
512         cwd = os.getcwd()
513         os.chdir("%s/%s" % (self.sdir, self.dir))
514         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
515         self.proc = Popen(self.cmd, shell=True,
516                           close_fds=True,
517                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
518         os.chdir(cwd)
519         self.next += 1
520
521
522 class buildlist(object):
523     '''handle build of multiple directories'''
524
525     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
526         global tasks
527         self.tlist = []
528         self.tail_proc = None
529         self.retry = None
530         if tasknames == []:
531             if options.restrict_tests:
532                 tasknames = ["samba-test-only"]
533             else:
534                 tasknames = defaulttasks
535         else:
536             # If we are only running one test,
537             # do not sleep randomly to wait for it to start
538             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
539
540         for n in tasknames:
541             if n not in tasks and n.endswith("-py2"):
542                 b = builder(n,
543                             tasks[n[:-4]],
544                             cp=n is not "pidl",
545                             py3=True)
546             else:
547                 b = builder(n, tasks[n], cp=n is not "pidl")
548             self.tlist.append(b)
549         if options.retry:
550             rebase_remote = "rebaseon"
551             retry_task = [("retry",
552                             '''set -e
553                             git remote add -t %s %s %s
554                             git fetch %s
555                             while :; do
556                               sleep 60
557                               git describe %s/%s > old_remote_branch.desc
558                               git fetch %s
559                               git describe %s/%s > remote_branch.desc
560                               diff old_remote_branch.desc remote_branch.desc
561                             done
562                            ''' % (
563                                 rebase_branch, rebase_remote, rebase_url,
564                                rebase_remote,
565                                rebase_remote, rebase_branch,
566                                rebase_remote,
567                                rebase_remote, rebase_branch
568                             ),
569                             "test/plain")]
570
571             self.retry = builder('retry', retry_task, cp=False)
572             self.need_retry = False
573
574     def kill_kids(self):
575         if self.tail_proc is not None:
576             self.tail_proc.terminate()
577             self.tail_proc.wait()
578             self.tail_proc = None
579         if self.retry is not None:
580             self.retry.proc.terminate()
581             self.retry.proc.wait()
582             self.retry = None
583         for b in self.tlist:
584             if b.proc is not None:
585                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
586                 b.proc.terminate()
587                 b.proc.wait()
588                 b.proc = None
589
590     def wait_one(self):
591         while True:
592             none_running = True
593             for b in self.tlist:
594                 if b.proc is None:
595                     continue
596                 none_running = False
597                 b.status = b.proc.poll()
598                 if b.status is None:
599                     continue
600                 b.proc = None
601                 return b
602             if options.retry:
603                 ret = self.retry.proc.poll()
604                 if ret is not None:
605                     self.need_retry = True
606                     self.retry = None
607                     return None
608             if none_running:
609                 return None
610             time.sleep(0.1)
611
612     def run(self):
613         while True:
614             b = self.wait_one()
615             if options.retry and self.need_retry:
616                 self.kill_kids()
617                 do_print("retry needed")
618                 return (0, None, None, None, "retry")
619             if b is None:
620                 break
621             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
622                 self.kill_kids()
623                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
624             b.start_next()
625         self.kill_kids()
626         return (0, None, None, None, "All OK")
627
628     def write_system_info(self):
629         filename = 'system-info.txt'
630         f = open(filename, 'w')
631         for cmd in ['uname -a',
632                     'lsb_release -a',
633                     'free',
634                     'mount',
635                     'cat /proc/cpuinfo',
636                     'cc --version',
637                     'df -m .',
638                     'df -m %s' % testbase]:
639             out = run_cmd(cmd, output=True, checkfail=False)
640             print('### %s' % cmd, file=f)
641             print(out.decode('utf8', 'backslashreplace'), file=f)
642             print(file=f)
643         f.close()
644         return filename
645
646     def tarlogs(self, fname):
647         tar = tarfile.open(fname, "w:gz")
648         for b in self.tlist:
649             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
650             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
651         if os.path.exists("autobuild.log"):
652             tar.add("autobuild.log")
653         sys_info = self.write_system_info()
654         tar.add(sys_info)
655         tar.close()
656
657     def remove_logs(self):
658         for b in self.tlist:
659             os.unlink(b.stdout_path)
660             os.unlink(b.stderr_path)
661
662     def start_tail(self):
663         cmd = ["tail", "-f"]
664         for b in self.tlist:
665             cmd.append(b.stdout_path)
666             cmd.append(b.stderr_path)
667         self.tail_proc = Popen(cmd, close_fds=True)
668
669
670 def cleanup():
671     if options.nocleanup:
672         return
673     run_cmd("stat %s || true" % test_tmpdir, show=True)
674     run_cmd("stat %s" % testbase, show=True)
675     do_print("Cleaning up %r" % cleanup_list)
676     for d in cleanup_list:
677         run_cmd("rm -rf %s" % d)
678
679
680 def find_git_root():
681     '''get to the top of the git repo'''
682     p = os.getcwd()
683     while p != '/':
684         if os.path.isdir(os.path.join(p, ".git")):
685             return p
686         p = os.path.abspath(os.path.join(p, '..'))
687     return None
688
689
690 def daemonize(logfile):
691     pid = os.fork()
692     if pid == 0:  # Parent
693         os.setsid()
694         pid = os.fork()
695         if pid != 0:  # Actual daemon
696             os._exit(0)
697     else:  # Grandparent
698         os._exit(0)
699
700     import resource      # Resource usage information.
701     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
702     if maxfd == resource.RLIM_INFINITY:
703         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
704     for fd in range(0, maxfd):
705         try:
706             os.close(fd)
707         except OSError:
708             pass
709     os.open(logfile, os.O_RDWR | os.O_CREAT)
710     os.dup2(0, 1)
711     os.dup2(0, 2)
712
713
714 def write_pidfile(fname):
715     '''write a pid file, cleanup on exit'''
716     f = open(fname, mode='w')
717     f.write("%u\n" % os.getpid())
718     f.close()
719
720
721 def rebase_tree(rebase_url, rebase_branch="master"):
722     rebase_remote = "rebaseon"
723     do_print("Rebasing on %s" % rebase_url)
724     run_cmd("git describe HEAD", show=True, dir=test_master)
725     run_cmd("git remote add -t %s %s %s" %
726             (rebase_branch, rebase_remote, rebase_url),
727             show=True, dir=test_master)
728     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
729     if options.fix_whitespace:
730         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
731                 (rebase_remote, rebase_branch),
732                 show=True, dir=test_master)
733     else:
734         run_cmd("git rebase --force-rebase %s/%s" %
735                 (rebase_remote, rebase_branch),
736                 show=True, dir=test_master)
737     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
738                    (rebase_remote, rebase_branch),
739                    dir=test_master, output=True)
740     if diff == '':
741         do_print("No differences between HEAD and %s/%s - exiting" %
742                  (rebase_remote, rebase_branch))
743         sys.exit(0)
744     run_cmd("git describe %s/%s" %
745             (rebase_remote, rebase_branch),
746             show=True, dir=test_master)
747     run_cmd("git describe HEAD", show=True, dir=test_master)
748     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
749             (rebase_remote, rebase_branch),
750             show=True, dir=test_master)
751
752
753 def push_to(push_url, push_branch="master"):
754     push_remote = "pushto"
755     do_print("Pushing to %s" % push_url)
756     if options.mark:
757         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
758         run_cmd("git commit --amend -c HEAD", dir=test_master)
759         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
760         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
761     run_cmd("git remote add -t %s %s %s" %
762             (push_branch, push_remote, push_url),
763             show=True, dir=test_master)
764     run_cmd("git push %s +HEAD:%s" %
765             (push_remote, push_branch),
766             show=True, dir=test_master)
767
768
769 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
770
771 gitroot = find_git_root()
772 if gitroot is None:
773     raise Exception("Failed to find git root")
774
775 parser = OptionParser()
776 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
777 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
778 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
779 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
780                   default=def_testbase)
781 parser.add_option("", "--passcmd", help="command to run on success", default=None)
782 parser.add_option("", "--verbose", help="show all commands as they are run",
783                   default=False, action="store_true")
784 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
785                   default=None, type='str')
786 parser.add_option("", "--pushto", help="push to a git url on success",
787                   default=None, type='str')
788 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
789                   default=False, action="store_true")
790 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
791                   default=False, action="store_true")
792 parser.add_option("", "--retry", help="automatically retry if master changes",
793                   default=False, action="store_true")
794 parser.add_option("", "--email", help="send email to the given address on failure",
795                   type='str', default=None)
796 parser.add_option("", "--email-from", help="send email from the given address",
797                   type='str', default="autobuild@samba.org")
798 parser.add_option("", "--email-server", help="send email via the given server",
799                   type='str', default='localhost')
800 parser.add_option("", "--always-email", help="always send email, even on success",
801                   action="store_true")
802 parser.add_option("", "--daemon", help="daemonize after initial setup",
803                   action="store_true")
804 parser.add_option("", "--branch", help="the branch to work on (default=master)",
805                   default="master", type='str')
806 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
807                   default=gitroot, type='str')
808 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
809                   default=False, action="store_true")
810 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
811                   default='')
812
813
814 def send_email(subject, text, log_tar):
815     if options.email is None:
816         do_print("not sending email because the recipient is not set")
817         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
818                  (subject, text))
819         return
820     outer = MIMEMultipart()
821     outer['Subject'] = subject
822     outer['To'] = options.email
823     outer['From'] = options.email_from
824     outer['Date'] = email.utils.formatdate(localtime=True)
825     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
826     outer.attach(MIMEText(text, 'plain'))
827     if options.attach_logs:
828         fp = open(log_tar, 'rb')
829         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
830         fp.close()
831         # Set the filename parameter
832         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
833         outer.attach(msg)
834     content = outer.as_string()
835     s = smtplib.SMTP(options.email_server)
836     s.sendmail(options.email_from, [options.email], content)
837     s.set_debuglevel(1)
838     s.quit()
839
840
841 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
842                   elapsed_time, log_base=None, add_log_tail=True):
843     '''send an email to options.email about the failure'''
844     elapsed_minutes = elapsed_time / 60.0
845     if log_base is None:
846         log_base = gitroot
847     text = '''
848 Dear Developer,
849
850 Your autobuild on %s failed after %.1f minutes
851 when trying to test %s with the following error:
852
853    %s
854
855 the autobuild has been abandoned. Please fix the error and resubmit.
856
857 A summary of the autobuild process is here:
858
859   %s/autobuild.log
860 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
861
862     if options.restrict_tests:
863         text += """
864 The build was restricted to tests matching %s\n""" % options.restrict_tests
865
866     if failed_task != 'rebase':
867         text += '''
868 You can see logs of the failed task here:
869
870   %s/%s.stdout
871   %s/%s.stderr
872
873 or you can get full logs of all tasks in this job here:
874
875   %s/logs.tar.gz
876
877 The top commit for the tree that was built was:
878
879 %s
880
881 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
882
883     if add_log_tail:
884         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
885         lines = f.readlines()
886         log_tail = "".join(lines[-50:])
887         num_lines = len(lines)
888         if num_lines < 50:
889             # Also include stderr (compile failures) if < 50 lines of stdout
890             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
891             log_tail += "".join(f.readlines()[-(50 - num_lines):])
892
893         text += '''
894 The last 50 lines of log messages:
895
896 %s
897     ''' % log_tail
898         f.close()
899
900     logs = os.path.join(gitroot, 'logs.tar.gz')
901     send_email('autobuild[%s] failure on %s for task %s during %s'
902                % (options.branch, platform.node(), failed_task, failed_stage),
903                text, logs)
904
905
906 def email_success(elapsed_time, log_base=None):
907     '''send an email to options.email about a successful build'''
908     if log_base is None:
909         log_base = gitroot
910     text = '''
911 Dear Developer,
912
913 Your autobuild on %s has succeeded after %.1f minutes.
914
915 ''' % (platform.node(), elapsed_time / 60.)
916
917     if options.restrict_tests:
918         text += """
919 The build was restricted to tests matching %s\n""" % options.restrict_tests
920
921     if options.keeplogs:
922         text += '''
923
924 you can get full logs of all tasks in this job here:
925
926   %s/logs.tar.gz
927
928 ''' % log_base
929
930     text += '''
931 The top commit for the tree that was built was:
932
933 %s
934 ''' % top_commit_msg
935
936     logs = os.path.join(gitroot, 'logs.tar.gz')
937     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
938                text, logs)
939
940
941 (options, args) = parser.parse_args()
942
943 if options.retry:
944     if options.rebase is None:
945         raise Exception('You can only use --retry if you also rebase')
946
947 testbase = "%s/b%u" % (options.testbase, os.getpid())
948 test_master = "%s/master" % testbase
949 test_prefix = "%s/prefix" % testbase
950 test_tmpdir = "%s/tmp" % testbase
951 os.environ['TMPDIR'] = test_tmpdir
952
953 # get the top commit message, for emails
954 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
955 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
956
957 try:
958     os.makedirs(testbase)
959 except Exception as reason:
960     raise Exception("Unable to create %s : %s" % (testbase, reason))
961 cleanup_list.append(testbase)
962
963 if options.daemon:
964     logfile = os.path.join(testbase, "log")
965     do_print("Forking into the background, writing progress to %s" % logfile)
966     daemonize(logfile)
967
968 write_pidfile(gitroot + "/autobuild.pid")
969
970 start_time = time.time()
971
972 while True:
973     try:
974         run_cmd("rm -rf %s" % test_tmpdir, show=True)
975         os.makedirs(test_tmpdir)
976         # The waf uninstall code removes empty directories all the way
977         # up the tree.  Creating a file in test_tmpdir stops it from
978         # being removed.
979         run_cmd("touch %s" % os.path.join(test_tmpdir,
980                                           ".directory-is-not-empty"), show=True)
981         run_cmd("stat %s" % test_tmpdir, show=True)
982         run_cmd("stat %s" % testbase, show=True)
983         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
984     except Exception:
985         cleanup()
986         raise
987
988     try:
989         try:
990             if options.rebase is not None:
991                 rebase_tree(options.rebase, rebase_branch=options.branch)
992         except Exception:
993             cleanup_list.append(gitroot + "/autobuild.pid")
994             cleanup()
995             elapsed_time = time.time() - start_time
996             email_failure(-1, 'rebase', 'rebase', 'rebase',
997                           'rebase on %s failed' % options.branch,
998                           elapsed_time, log_base=options.log_base)
999             sys.exit(1)
1000         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1001         if options.tail:
1002             blist.start_tail()
1003         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1004         if status != 0 or errstr != "retry":
1005             break
1006         cleanup()
1007     except Exception:
1008         cleanup()
1009         raise
1010
1011 cleanup_list.append(gitroot + "/autobuild.pid")
1012
1013 do_print(errstr)
1014
1015 blist.kill_kids()
1016 if options.tail:
1017     do_print("waiting for tail to flush")
1018     time.sleep(1)
1019
1020 elapsed_time = time.time() - start_time
1021 if status == 0:
1022     if options.passcmd is not None:
1023         do_print("Running passcmd: %s" % options.passcmd)
1024         run_cmd(options.passcmd, dir=test_master)
1025     if options.pushto is not None:
1026         push_to(options.pushto, push_branch=options.branch)
1027     if options.keeplogs or options.attach_logs:
1028         blist.tarlogs("logs.tar.gz")
1029         do_print("Logs in logs.tar.gz")
1030     if options.always_email:
1031         email_success(elapsed_time, log_base=options.log_base)
1032     blist.remove_logs()
1033     cleanup()
1034     do_print(errstr)
1035     sys.exit(0)
1036
1037 # something failed, gather a tar of the logs
1038 blist.tarlogs("logs.tar.gz")
1039
1040 if options.email is not None:
1041     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1042                   elapsed_time, log_base=options.log_base)
1043 else:
1044     elapsed_minutes = elapsed_time / 60.0
1045     print('''
1046
1047 ####################################################################
1048
1049 AUTOBUILD FAILURE
1050
1051 Your autobuild[%s] on %s failed after %.1f minutes
1052 when trying to test %s with the following error:
1053
1054    %s
1055
1056 the autobuild has been abandoned. Please fix the error and resubmit.
1057
1058 ####################################################################
1059
1060 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1061
1062 cleanup()
1063 do_print(errstr)
1064 do_print("Logs in logs.tar.gz")
1065 sys.exit(status)