2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
6 from __future__ import print_function
7 from subprocess import call, check_call, Popen, PIPE
12 from optparse import OptionParser
15 from email.mime.text import MIMEText
16 from email.mime.base import MIMEBase
17 from email.mime.application import MIMEApplication
18 from email.mime.multipart import MIMEMultipart
19 from distutils.sysconfig import get_python_lib
23 from waflib.Build import CACHE_SUFFIX
25 sys.path.insert(0, "./third_party/waf")
26 from waflib.Build import CACHE_SUFFIX
29 os.environ["PYTHONUNBUFFERED"] = "1"
31 # This speeds up testing remarkably.
32 os.environ['TDB_NO_FSYNC'] = '1'
40 "samba-fileserver": ".",
41 "samba-ad-member": ".",
47 "samba-none-env": ".",
54 "samba-ad-dc-ntvfs": ".",
55 "samba-ad-dc-backup": ".",
56 "samba-systemkrb5": ".",
57 "samba-nopython": ".",
58 "samba-nopython-py2": ".",
59 "samba-schemaupgrade": ".",
62 "talloc": "lib/talloc",
63 "replace": "lib/replace",
64 "tevent": "lib/tevent",
68 defaulttasks = builddirs.keys()
70 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
71 defaulttasks.remove("samba-o3")
73 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
74 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
76 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
77 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
78 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
79 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
80 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
81 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
82 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
85 "ctdb": [("random-sleep", "../script/random-sleep.sh 300 900", "text/plain"),
86 ("configure", "./configure " + ctdb_configure_params, "text/plain"),
87 ("make", "make all", "text/plain"),
88 ("install", "make install", "text/plain"),
89 ("test", "make autotest", "text/plain"),
90 ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
91 ("clean", "make clean", "text/plain")],
93 # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
95 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
96 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
97 ("make", "make -j", "text/plain"),
98 ("test", "make test FAIL_IMMEDIATELY=1 "
99 "TESTS='--exclude-env=none "
100 "--exclude-env=nt4_dc "
101 "--exclude-env=nt4_dc_schannel "
102 "--exclude-env=nt4_member "
103 "--exclude-env=ad_dc "
104 "--exclude-env=ad_dc_backup "
105 "--exclude-env=ad_dc_ntvfs "
106 "--exclude-env=ad_dc_default "
107 "--exclude-env=ad_dc_slowtests "
108 "--exclude-env=ad_dc_no_nss "
109 "--exclude-env=ad_dc_no_ntlm "
110 "--exclude-env=fl2003dc "
111 "--exclude-env=fl2008dc "
112 "--exclude-env=fl2008r2dc "
113 "--exclude-env=ad_member "
114 "--exclude-env=ad_member_idmap_rid "
115 "--exclude-env=ad_member_idmap_ad "
116 "--exclude-env=ad_member_rfc2307 "
117 "--exclude-env=chgdcpass "
118 "--exclude-env=vampire_2000_dc "
119 "--exclude-env=fl2000dc "
120 "--exclude-env=fileserver "
121 "--exclude-env=maptoguest "
122 "--exclude-env=simpleserver "
123 "--exclude-env=backupfromdc "
124 "--exclude-env=restoredc "
125 "--exclude-env=renamedc "
126 "--exclude-env=offlinebackupdc "
127 "--exclude-env=labdc "
128 "--exclude-env=preforkrestartdc "
129 "--exclude-env=proclimitdc "
130 "--exclude-env=promoted_dc "
131 "--exclude-env=vampire_dc "
132 "--exclude-env=rodc "
133 "--exclude-env=ad_dc_default "
134 "--exclude-env=ad_dc_slowtests "
135 "--exclude-env=schema_pair_dc "
136 "--exclude-env=schema_dc "
139 ("install", "make install", "text/plain"),
140 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
141 ("clean", "make clean", "text/plain")],
143 "samba-nt4": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
144 ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
145 ("make", "make -j", "text/plain"),
146 ("test", "make test FAIL_IMMEDIATELY=1 "
148 "--include-env=nt4_dc "
149 "--include-env=nt4_dc_schannel "
150 "--include-env=nt4_member "
152 ("install", "make install", "text/plain"),
153 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
154 ("clean", "make clean", "text/plain")],
156 "samba-fileserver": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
157 ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
158 ("make", "make -j", "text/plain"),
159 ("test", "make test FAIL_IMMEDIATELY=1 "
161 "--include-env=fileserver "
162 "--include-env=maptoguest "
163 "--include-env=simpleserver "
165 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
167 "samba-ad-member": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
168 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
169 ("make", "make -j", "text/plain"),
170 ("test", "make test FAIL_IMMEDIATELY=1 "
172 "--include-env=ad_member "
173 "--include-env=ad_member_idmap_rid "
174 "--include-env=ad_member_idmap_ad "
175 "--include-env=ad_member_rfc2307 "
177 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
179 "samba-ad-dc-1": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
180 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
181 ("make", "make -j", "text/plain"),
182 ("test", "make test FAIL_IMMEDIATELY=1 "
183 "TESTS='--include-env=ad_dc "
184 "--include-env=ad_dc_no_nss "
185 "--include-env=ad_dc_no_ntlm "
187 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
189 "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
190 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
191 ("make", "make -j", "text/plain"),
192 ("test", "make test FAIL_IMMEDIATELY=1 "
194 "--include-env=vampire_dc "
195 "--include-env=vampire_2000_dc "
196 "--include-env=rodc "
198 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
200 "samba-ad-dc-3": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
201 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
202 ("make", "make -j", "text/plain"),
203 ("test", "make test FAIL_IMMEDIATELY=1 "
205 "--include-env=promoted_dc "
206 "--include-env=chgdcpass "
207 "--include-env=preforkrestartdc "
208 "--include-env=proclimitdc "
210 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
212 "samba-ad-dc-4": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
213 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
214 ("make", "make -j", "text/plain"),
215 ("test", "make test FAIL_IMMEDIATELY=1 "
217 "--include-env=fl2000dc "
218 "--include-env=fl2003dc "
219 "--include-env=fl2008dc "
220 "--include-env=fl2008r2dc "
222 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
224 "samba-ad-dc-5": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
225 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
226 ("make", "make -j", "text/plain"),
227 ("test", "make test FAIL_IMMEDIATELY=1 "
229 "--include-env=ad_dc_default "
231 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
233 "samba-ad-dc-6": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
234 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
235 ("make", "make -j", "text/plain"),
236 ("test", "make test FAIL_IMMEDIATELY=1 "
238 "--include-env=ad_dc_slowtests "
240 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
243 "samba-schemaupgrade": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
244 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
245 ("make", "make -j", "text/plain"),
246 ("test", "make test FAIL_IMMEDIATELY=1 "
248 "--include-env=schema_dc "
249 "--include-env=schema_pair_dc "
251 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
253 # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
254 # This is currently the longest task, so we don't randomly delay it.
255 "samba-ad-dc-ntvfs": [
256 ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
257 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
258 ("make", "make -j", "text/plain"),
259 ("test", "make test FAIL_IMMEDIATELY=1 "
261 "--include-env=ad_dc_ntvfs "
263 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
265 # run the backup/restore testenvs separately as they're fairly standalone
266 # (and CI seems to max out at ~8 different DCs running at once)
267 "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
268 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
269 ("make", "make -j", "text/plain"),
270 ("test", "make test FAIL_IMMEDIATELY=1 "
271 "TESTS='--include-env=backupfromdc "
272 "--include-env=restoredc "
273 "--include-env=renamedc "
274 "--include-env=offlinebackupdc "
275 "--include-env=labdc "
276 "--include-env=ad_dc_backup "
278 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
280 "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
281 ("make", "make -j", "text/plain"),
282 ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
284 # Test cross-compile infrastructure
285 "samba-xc": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
286 ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
287 ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
288 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
289 ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
290 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
291 ("compare-results", "script/compare_cc_results.py "
292 "./bin/c4che/default{} "
293 "./bin-xe/c4che/default{} "
294 "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
296 # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
297 "samba-o3": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
298 ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
299 ("make", "make -j", "text/plain"),
300 ("test", "make quicktest FAIL_IMMEDIATELY=1 "
301 "TESTS='--include-env=ad_dc'", "text/plain"),
302 ("install", "make install", "text/plain"),
303 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
304 ("clean", "make clean", "text/plain")],
306 "samba-ctdb": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
308 # make sure we have tdb around:
309 ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
310 ("tdb-make", "cd lib/tdb && make", "text/plain"),
311 ("tdb-install", "cd lib/tdb && make install", "text/plain"),
314 # build samba with cluster support (also building ctdb):
315 ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
316 ("samba-make", "make", "text/plain"),
317 ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
318 ("samba-install", "make install", "text/plain"),
319 ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
322 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
323 ("clean", "make clean", "text/plain"),
324 ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
327 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
328 ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
329 ("talloc-make", "cd lib/talloc && make", "text/plain"),
330 ("talloc-install", "cd lib/talloc && make install", "text/plain"),
332 ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
333 ("tdb-make", "cd lib/tdb && make", "text/plain"),
334 ("tdb-install", "cd lib/tdb && make install", "text/plain"),
336 ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
337 ("tevent-make", "cd lib/tevent && make", "text/plain"),
338 ("tevent-install", "cd lib/tevent && make install", "text/plain"),
340 ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
341 ("ldb-make", "cd lib/ldb && make", "text/plain"),
342 ("ldb-install", "cd lib/ldb && make install", "text/plain"),
344 ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
345 ("nondevel-make", "make -j", "text/plain"),
346 ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
347 ("nondevel-install", "make install", "text/plain"),
348 ("nondevel-dist", "make dist", "text/plain"),
350 # retry with all modules shared
351 ("allshared-distclean", "make distclean", "text/plain"),
352 ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
353 ("allshared-make", "make -j", "text/plain")],
356 ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
357 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
358 ("make", "make -j", "text/plain"),
359 ("test", "make test "
360 "FAIL_IMMEDIATELY=1 "
361 "TESTS='--include-env=none'",
365 ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
366 # build with all modules static
367 ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
368 ("allstatic-make", "make -j", "text/plain"),
369 ("allstatic-test", "make test "
370 "FAIL_IMMEDIATELY=1 "
371 "TESTS='samba3.smb2.create.*nt4_dc'",
374 # retry without any required modules
375 ("none-distclean", "make distclean", "text/plain"),
376 ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
377 ("none-make", "make -j", "text/plain"),
379 # retry with nonshared smbd and smbtorture
380 ("nonshared-distclean", "make distclean", "text/plain"),
381 ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
382 ("nonshared-make", "make -j", "text/plain")],
384 "samba-systemkrb5": [
385 ("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
386 ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
387 ("make", "make -j", "text/plain"),
388 # we currently cannot run a full make test, a limited list of tests could be run
389 # via "make test TESTS=sometests"
390 ("test", "make test FAIL_IMMEDIATELY=1 "
391 "TESTS='--include-env=ktest'", "text/plain"),
392 ("install", "make install", "text/plain"),
393 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
394 ("clean", "make clean", "text/plain")
397 # Test Samba without python still builds. When this test fails
398 # due to more use of Python, the expectations is that the newly
399 # failing part of the code should be disabled when
400 # --disable-python is set (rather than major work being done to
401 # support this environment). The target here is for vendors
402 # shipping a minimal smbd.
404 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
405 ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
406 ("make", "make -j", "text/plain"),
407 ("install", "make install", "text/plain"),
408 ("test", "make test-nopython", "text/plain"),
409 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
410 ("clean", "make clean", "text/plain"),
412 ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
413 ("talloc-make", "cd lib/talloc && make", "text/plain"),
414 ("talloc-install", "cd lib/talloc && make install", "text/plain"),
416 ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
417 ("tdb-make", "cd lib/tdb && make", "text/plain"),
418 ("tdb-install", "cd lib/tdb && make install", "text/plain"),
420 ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
421 ("tevent-make", "cd lib/tevent && make", "text/plain"),
422 ("tevent-install", "cd lib/tevent && make install", "text/plain"),
424 ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
425 ("ldb-make", "cd lib/ldb && make", "text/plain"),
426 ("ldb-install", "cd lib/ldb && make install", "text/plain"),
428 # retry against installed library packages
429 ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
430 ("libs-make", "make -j", "text/plain"),
431 ("libs-install", "make install", "text/plain"),
432 ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
433 ("libs-clean", "make clean", "text/plain")
436 # check we can do the same thing using python2
437 "samba-nopython-py2": [
438 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
439 ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
440 ("make", "PYTHON=python2 make -j", "text/plain"),
441 ("install", "PYTHON=python2 make install", "text/plain"),
442 ("test", "make test-nopython", "text/plain"),
443 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
444 ("clean", "PYTHON=python2 make clean", "text/plain"),
446 ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
447 ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
448 ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
450 ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
451 ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
452 ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
454 ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
455 ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
456 ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
458 ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
459 ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
460 ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
462 # retry against installed library packages
463 ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
464 ("libs-make", "PYTHON=python2 make -j", "text/plain"),
465 ("libs-install", "PYTHON=python2 make install", "text/plain"),
466 ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
467 ("libs-clean", "PYTHON=python2 make clean", "text/plain")
471 ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
472 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
473 ("make", "make", "text/plain"),
474 ("install", "make install", "text/plain"),
475 ("test", "make test", "text/plain"),
476 ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
477 ("make-no-lmdb", "make", "text/plain"),
478 ("install-no-lmdb", "make install", "text/plain"),
479 ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
480 ("distcheck", "make distcheck", "text/plain"),
481 ("clean", "make clean", "text/plain")],
484 ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
485 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
486 ("make", "make", "text/plain"),
487 ("install", "make install", "text/plain"),
488 ("test", "make test", "text/plain"),
489 ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
490 ("distcheck", "make distcheck", "text/plain"),
491 ("clean", "make clean", "text/plain")],
494 ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
495 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
496 ("make", "make", "text/plain"),
497 ("install", "make install", "text/plain"),
498 ("test", "make test", "text/plain"),
499 ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
500 ("distcheck", "make distcheck", "text/plain"),
501 ("clean", "make clean", "text/plain")],
504 ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
505 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
506 ("make", "make", "text/plain"),
507 ("install", "make install", "text/plain"),
508 ("test", "make test", "text/plain"),
509 ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
510 ("distcheck", "make distcheck", "text/plain"),
511 ("clean", "make clean", "text/plain")],
514 ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
515 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
516 ("make", "make", "text/plain"),
517 ("install", "make install", "text/plain"),
518 ("test", "make test", "text/plain"),
519 ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
520 ("distcheck", "make distcheck", "text/plain"),
521 ("clean", "make clean", "text/plain")],
524 ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
525 ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
526 ("touch", "touch *.yp", "text/plain"),
527 ("make", "make", "text/plain"),
528 ("test", "make test", "text/plain"),
529 ("install", "make install", "text/plain"),
530 ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
531 ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
532 ("clean", "make clean", "text/plain")],
535 # these are useful for debugging autobuild
536 'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
537 'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
549 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
551 show = options.verbose
553 do_print("Running: '%s' in '%s'" % (cmd, dir))
555 return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
557 return check_call(cmd, shell=True, cwd=dir)
559 return call(cmd, shell=True, cwd=dir)
562 class builder(object):
563 '''handle build of one directory'''
565 def __init__(self, name, sequence, cp=True):
567 if name in builddirs:
568 self.dir = builddirs[name]
572 self.tag = self.name.replace('/', '_')
573 self.sequence = sequence
575 self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
576 self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
578 do_print("stdout for %s in %s" % (self.name, self.stdout_path))
579 do_print("stderr for %s in %s" % (self.name, self.stderr_path))
580 run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
581 self.stdout = open(self.stdout_path, 'w')
582 self.stderr = open(self.stderr_path, 'w')
583 self.stdin = open("/dev/null", 'r')
584 self.sdir = "%s/%s" % (testbase, self.tag)
585 self.prefix = "%s/%s" % (test_prefix, self.tag)
586 run_cmd("rm -rf %s" % self.sdir)
587 run_cmd("rm -rf %s" % self.prefix)
589 run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
591 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
594 def start_next(self):
595 if self.next == len(self.sequence):
596 if not options.nocleanup:
597 run_cmd("rm -rf %s" % self.sdir)
598 run_cmd("rm -rf %s" % self.prefix)
599 do_print('%s: Completed OK' % self.name)
602 (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
603 self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
604 self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
605 self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
606 self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
607 # if self.output_mime_type == "text/x-subunit":
608 # self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
610 os.chdir("%s/%s" % (self.sdir, self.dir))
611 do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
612 self.proc = Popen(self.cmd, shell=True,
614 stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
619 class buildlist(object):
620 '''handle build of multiple directories'''
622 def __init__(self, tasknames, rebase_url, rebase_branch="master"):
625 self.tail_proc = None
628 if options.restrict_tests:
629 tasknames = ["samba-test-only"]
631 tasknames = defaulttasks
633 # If we are only running one test,
634 # do not sleep randomly to wait for it to start
635 os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
638 b = builder(n, tasks[n], cp=n is not "pidl")
641 rebase_remote = "rebaseon"
642 retry_task = [("retry",
644 git remote add -t %s %s %s
648 git describe %s/%s > old_remote_branch.desc
650 git describe %s/%s > remote_branch.desc
651 diff old_remote_branch.desc remote_branch.desc
654 rebase_branch, rebase_remote, rebase_url,
656 rebase_remote, rebase_branch,
658 rebase_remote, rebase_branch
662 self.retry = builder('retry', retry_task, cp=False)
663 self.need_retry = False
666 if self.tail_proc is not None:
667 self.tail_proc.terminate()
668 self.tail_proc.wait()
669 self.tail_proc = None
670 if self.retry is not None:
671 self.retry.proc.terminate()
672 self.retry.proc.wait()
675 if b.proc is not None:
676 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
688 b.status = b.proc.poll()
694 ret = self.retry.proc.poll()
696 self.need_retry = True
706 if options.retry and self.need_retry:
708 do_print("retry needed")
709 return (0, None, None, None, "retry")
712 if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
714 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
717 return (0, None, None, None, "All OK")
719 def write_system_info(self):
720 filename = 'system-info.txt'
721 f = open(filename, 'w')
722 for cmd in ['uname -a',
729 'df -m %s' % testbase]:
730 out = run_cmd(cmd, output=True, checkfail=False)
731 print('### %s' % cmd, file=f)
732 print(out.decode('utf8', 'backslashreplace'), file=f)
737 def tarlogs(self, fname):
738 tar = tarfile.open(fname, "w:gz")
740 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
741 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
742 if os.path.exists("autobuild.log"):
743 tar.add("autobuild.log")
744 sys_info = self.write_system_info()
748 def remove_logs(self):
750 os.unlink(b.stdout_path)
751 os.unlink(b.stderr_path)
753 def start_tail(self):
756 cmd.append(b.stdout_path)
757 cmd.append(b.stderr_path)
758 self.tail_proc = Popen(cmd, close_fds=True)
762 if options.nocleanup:
764 run_cmd("stat %s || true" % test_tmpdir, show=True)
765 run_cmd("stat %s" % testbase, show=True)
766 do_print("Cleaning up %r" % cleanup_list)
767 for d in cleanup_list:
768 run_cmd("rm -rf %s" % d)
772 '''get to the top of the git repo'''
775 if os.path.isdir(os.path.join(p, ".git")):
777 p = os.path.abspath(os.path.join(p, '..'))
781 def daemonize(logfile):
783 if pid == 0: # Parent
786 if pid != 0: # Actual daemon
791 import resource # Resource usage information.
792 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
793 if maxfd == resource.RLIM_INFINITY:
794 maxfd = 1024 # Rough guess at maximum number of open file descriptors.
795 for fd in range(0, maxfd):
800 os.open(logfile, os.O_RDWR | os.O_CREAT)
805 def write_pidfile(fname):
806 '''write a pid file, cleanup on exit'''
807 f = open(fname, mode='w')
808 f.write("%u\n" % os.getpid())
812 def rebase_tree(rebase_url, rebase_branch="master"):
813 rebase_remote = "rebaseon"
814 do_print("Rebasing on %s" % rebase_url)
815 run_cmd("git describe HEAD", show=True, dir=test_master)
816 run_cmd("git remote add -t %s %s %s" %
817 (rebase_branch, rebase_remote, rebase_url),
818 show=True, dir=test_master)
819 run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
820 if options.fix_whitespace:
821 run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
822 (rebase_remote, rebase_branch),
823 show=True, dir=test_master)
825 run_cmd("git rebase --force-rebase %s/%s" %
826 (rebase_remote, rebase_branch),
827 show=True, dir=test_master)
828 diff = run_cmd("git --no-pager diff HEAD %s/%s" %
829 (rebase_remote, rebase_branch),
830 dir=test_master, output=True)
832 do_print("No differences between HEAD and %s/%s - exiting" %
833 (rebase_remote, rebase_branch))
835 run_cmd("git describe %s/%s" %
836 (rebase_remote, rebase_branch),
837 show=True, dir=test_master)
838 run_cmd("git describe HEAD", show=True, dir=test_master)
839 run_cmd("git --no-pager diff --stat HEAD %s/%s" %
840 (rebase_remote, rebase_branch),
841 show=True, dir=test_master)
844 def push_to(push_url, push_branch="master"):
845 push_remote = "pushto"
846 do_print("Pushing to %s" % push_url)
848 run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
849 run_cmd("git commit --amend -c HEAD", dir=test_master)
850 # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
851 # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
852 run_cmd("git remote add -t %s %s %s" %
853 (push_branch, push_remote, push_url),
854 show=True, dir=test_master)
855 run_cmd("git push %s +HEAD:%s" %
856 (push_remote, push_branch),
857 show=True, dir=test_master)
860 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
862 gitroot = find_git_root()
864 raise Exception("Failed to find git root")
866 parser = OptionParser()
867 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
868 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
869 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
870 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
871 default=def_testbase)
872 parser.add_option("", "--passcmd", help="command to run on success", default=None)
873 parser.add_option("", "--verbose", help="show all commands as they are run",
874 default=False, action="store_true")
875 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
876 default=None, type='str')
877 parser.add_option("", "--pushto", help="push to a git url on success",
878 default=None, type='str')
879 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
880 default=False, action="store_true")
881 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
882 default=False, action="store_true")
883 parser.add_option("", "--retry", help="automatically retry if master changes",
884 default=False, action="store_true")
885 parser.add_option("", "--email", help="send email to the given address on failure",
886 type='str', default=None)
887 parser.add_option("", "--email-from", help="send email from the given address",
888 type='str', default="autobuild@samba.org")
889 parser.add_option("", "--email-server", help="send email via the given server",
890 type='str', default='localhost')
891 parser.add_option("", "--always-email", help="always send email, even on success",
893 parser.add_option("", "--daemon", help="daemonize after initial setup",
895 parser.add_option("", "--branch", help="the branch to work on (default=master)",
896 default="master", type='str')
897 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
898 default=gitroot, type='str')
899 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
900 default=False, action="store_true")
901 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
905 def send_email(subject, text, log_tar):
906 if options.email is None:
907 do_print("not sending email because the recipient is not set")
908 do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
911 outer = MIMEMultipart()
912 outer['Subject'] = subject
913 outer['To'] = options.email
914 outer['From'] = options.email_from
915 outer['Date'] = email.utils.formatdate(localtime=True)
916 outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
917 outer.attach(MIMEText(text, 'plain'))
918 if options.attach_logs:
919 fp = open(log_tar, 'rb')
920 msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
922 # Set the filename parameter
923 msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
925 content = outer.as_string()
926 s = smtplib.SMTP(options.email_server)
927 email_user = os.getenv('SMTP_USERNAME')
928 email_password = os.getenv('SMTP_PASSWORD')
929 if email_user is not None:
931 s.login(email_user, email_password)
933 s.sendmail(options.email_from, [options.email], content)
938 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
939 elapsed_time, log_base=None, add_log_tail=True):
940 '''send an email to options.email about the failure'''
941 elapsed_minutes = elapsed_time / 60.0
947 Your autobuild on %s failed after %.1f minutes
948 when trying to test %s with the following error:
952 the autobuild has been abandoned. Please fix the error and resubmit.
954 A summary of the autobuild process is here:
957 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
959 if options.restrict_tests:
961 The build was restricted to tests matching %s\n""" % options.restrict_tests
963 if failed_task != 'rebase':
965 You can see logs of the failed task here:
970 or you can get full logs of all tasks in this job here:
974 The top commit for the tree that was built was:
978 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
981 f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
982 lines = f.readlines()
983 log_tail = "".join(lines[-50:])
984 num_lines = len(lines)
986 # Also include stderr (compile failures) if < 50 lines of stdout
987 f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
988 log_tail += "".join(f.readlines()[-(50 - num_lines):])
991 The last 50 lines of log messages:
997 logs = os.path.join(gitroot, 'logs.tar.gz')
998 send_email('autobuild[%s] failure on %s for task %s during %s'
999 % (options.branch, platform.node(), failed_task, failed_stage),
1003 def email_success(elapsed_time, log_base=None):
1004 '''send an email to options.email about a successful build'''
1005 if log_base is None:
1010 Your autobuild on %s has succeeded after %.1f minutes.
1012 ''' % (platform.node(), elapsed_time / 60.)
1014 if options.restrict_tests:
1016 The build was restricted to tests matching %s\n""" % options.restrict_tests
1018 if options.keeplogs:
1021 you can get full logs of all tasks in this job here:
1028 The top commit for the tree that was built was:
1031 ''' % top_commit_msg
1033 logs = os.path.join(gitroot, 'logs.tar.gz')
1034 send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1038 (options, args) = parser.parse_args()
1041 if options.rebase is None:
1042 raise Exception('You can only use --retry if you also rebase')
1044 testbase = "%s/b%u" % (options.testbase, os.getpid())
1045 test_master = "%s/master" % testbase
1046 test_prefix = "%s/prefix" % testbase
1047 test_tmpdir = "%s/tmp" % testbase
1048 os.environ['TMPDIR'] = test_tmpdir
1050 # get the top commit message, for emails
1051 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1052 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
1055 os.makedirs(testbase)
1056 except Exception as reason:
1057 raise Exception("Unable to create %s : %s" % (testbase, reason))
1058 cleanup_list.append(testbase)
1061 logfile = os.path.join(testbase, "log")
1062 do_print("Forking into the background, writing progress to %s" % logfile)
1065 write_pidfile(gitroot + "/autobuild.pid")
1067 start_time = time.time()
1071 run_cmd("rm -rf %s" % test_tmpdir, show=True)
1072 os.makedirs(test_tmpdir)
1073 # The waf uninstall code removes empty directories all the way
1074 # up the tree. Creating a file in test_tmpdir stops it from
1076 run_cmd("touch %s" % os.path.join(test_tmpdir,
1077 ".directory-is-not-empty"), show=True)
1078 run_cmd("stat %s" % test_tmpdir, show=True)
1079 run_cmd("stat %s" % testbase, show=True)
1080 run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1087 if options.rebase is not None:
1088 rebase_tree(options.rebase, rebase_branch=options.branch)
1090 cleanup_list.append(gitroot + "/autobuild.pid")
1092 elapsed_time = time.time() - start_time
1093 email_failure(-1, 'rebase', 'rebase', 'rebase',
1094 'rebase on %s failed' % options.branch,
1095 elapsed_time, log_base=options.log_base)
1097 blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1100 (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1101 if status != 0 or errstr != "retry":
1108 cleanup_list.append(gitroot + "/autobuild.pid")
1114 do_print("waiting for tail to flush")
1117 elapsed_time = time.time() - start_time
1119 if options.passcmd is not None:
1120 do_print("Running passcmd: %s" % options.passcmd)
1121 run_cmd(options.passcmd, dir=test_master)
1122 if options.pushto is not None:
1123 push_to(options.pushto, push_branch=options.branch)
1124 if options.keeplogs or options.attach_logs:
1125 blist.tarlogs("logs.tar.gz")
1126 do_print("Logs in logs.tar.gz")
1127 if options.always_email:
1128 email_success(elapsed_time, log_base=options.log_base)
1134 # something failed, gather a tar of the logs
1135 blist.tarlogs("logs.tar.gz")
1137 if options.email is not None:
1138 email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1139 elapsed_time, log_base=options.log_base)
1141 elapsed_minutes = elapsed_time / 60.0
1144 ####################################################################
1148 Your autobuild[%s] on %s failed after %.1f minutes
1149 when trying to test %s with the following error:
1153 the autobuild has been abandoned. Please fix the error and resubmit.
1155 ####################################################################
1157 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1161 do_print("Logs in logs.tar.gz")