e1aea054e1f761bd95abb613caae767abb4c074d
[bbaumbach/samba-autobuild/.git] / script / autobuild.py
1 #!/usr/bin/env python3
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call, Popen, PIPE
8 import os
9 import tarfile
10 import sys
11 import time
12 from optparse import OptionParser
13 import smtplib
14 import email
15 from email.mime.text import MIMEText
16 from email.mime.base import MIMEBase
17 from email.mime.application import MIMEApplication
18 from email.mime.multipart import MIMEMultipart
19 from distutils.sysconfig import get_python_lib
20 import platform
21
22 try:
23     from waflib.Build import CACHE_SUFFIX
24 except ImportError:
25     sys.path.insert(0, "./third_party/waf")
26     from waflib.Build import CACHE_SUFFIX
27
28
29 os.environ["PYTHONUNBUFFERED"] = "1"
30
31 # This speeds up testing remarkably.
32 os.environ['TDB_NO_FSYNC'] = '1'
33
34 cleanup_list = []
35
36 builddirs = {
37     "ctdb": "ctdb",
38     "samba": ".",
39     "samba-nt4": ".",
40     "samba-fileserver": ".",
41     "samba-ad-member": ".",
42     "samba-xc": ".",
43     "samba-o3": ".",
44     "samba-ctdb": ".",
45     "samba-libs": ".",
46     "samba-static": ".",
47     "samba-none-env": ".",
48     "samba-ad-dc-1": ".",
49     "samba-ad-dc-2": ".",
50     "samba-ad-dc-3": ".",
51     "samba-ad-dc-4": ".",
52     "samba-ad-dc-5": ".",
53     "samba-ad-dc-6": ".",
54     "samba-ad-dc-ntvfs": ".",
55     "samba-ad-dc-backup": ".",
56     "samba-systemkrb5": ".",
57     "samba-nopython": ".",
58     "samba-nopython-py2": ".",
59     "samba-schemaupgrade": ".",
60     "ldb": "lib/ldb",
61     "tdb": "lib/tdb",
62     "talloc": "lib/talloc",
63     "replace": "lib/replace",
64     "tevent": "lib/tevent",
65     "pidl": "pidl"
66 }
67
68 defaulttasks = builddirs.keys()
69
70 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
71     defaulttasks.remove("samba-o3")
72
73 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
74 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
75
76 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
77 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
78 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
79 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
80 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
81 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
82 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
83
84 tasks = {
85     "ctdb": [("random-sleep", "../script/random-sleep.sh 300 900", "text/plain"),
86                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
87                ("make", "make all", "text/plain"),
88                ("install", "make install", "text/plain"),
89                ("test", "make autotest", "text/plain"),
90                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
91                ("clean", "make clean", "text/plain")],
92
93     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
94     "samba": [
95                 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
96                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
97                 ("make", "make -j", "text/plain"),
98                 ("test", "make test FAIL_IMMEDIATELY=1 "
99                  "TESTS='--exclude-env=none "
100                  "--exclude-env=nt4_dc "
101                  "--exclude-env=nt4_dc_schannel "
102                  "--exclude-env=nt4_member "
103                  "--exclude-env=ad_dc "
104                  "--exclude-env=ad_dc_backup "
105                  "--exclude-env=ad_dc_ntvfs "
106                  "--exclude-env=ad_dc_default "
107                  "--exclude-env=ad_dc_slowtests "
108                  "--exclude-env=ad_dc_no_nss "
109                  "--exclude-env=ad_dc_no_ntlm "
110                  "--exclude-env=fl2003dc "
111                  "--exclude-env=fl2008dc "
112                  "--exclude-env=fl2008r2dc "
113                  "--exclude-env=ad_member "
114                  "--exclude-env=ad_member_idmap_rid "
115                  "--exclude-env=ad_member_idmap_ad "
116                  "--exclude-env=ad_member_rfc2307 "
117                  "--exclude-env=chgdcpass "
118                  "--exclude-env=vampire_2000_dc "
119                  "--exclude-env=fl2000dc "
120                  "--exclude-env=fileserver "
121                  "--exclude-env=maptoguest "
122                  "--exclude-env=simpleserver "
123                  "--exclude-env=backupfromdc "
124                  "--exclude-env=restoredc "
125                  "--exclude-env=renamedc "
126                  "--exclude-env=offlinebackupdc "
127                  "--exclude-env=labdc "
128                  "--exclude-env=preforkrestartdc "
129                  "--exclude-env=proclimitdc "
130                  "--exclude-env=promoted_dc "
131                  "--exclude-env=vampire_dc "
132                  "--exclude-env=rodc "
133                  "--exclude-env=ad_dc_default "
134                  "--exclude-env=ad_dc_slowtests "
135                  "--exclude-env=schema_pair_dc "
136                  "--exclude-env=schema_dc "
137                  "'",
138                  "text/plain"),
139                 ("install", "make install", "text/plain"),
140                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
141                 ("clean", "make clean", "text/plain")],
142
143     "samba-nt4": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
144                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
145                     ("make", "make -j", "text/plain"),
146                     ("test", "make test FAIL_IMMEDIATELY=1 "
147                      "TESTS='"
148                      "--include-env=nt4_dc "
149                      "--include-env=nt4_dc_schannel "
150                      "--include-env=nt4_member "
151                      "'", "text/plain"),
152                     ("install", "make install", "text/plain"),
153                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
154                     ("clean", "make clean", "text/plain")],
155
156     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
157                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
158                            ("make", "make -j", "text/plain"),
159                            ("test", "make test FAIL_IMMEDIATELY=1 "
160                             "TESTS='"
161                             "--include-env=fileserver "
162                             "--include-env=maptoguest "
163                             "--include-env=simpleserver "
164                             "'", "text/plain"),
165                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
166
167     "samba-ad-member": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
168                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
169                         ("make", "make -j", "text/plain"),
170                         ("test", "make test FAIL_IMMEDIATELY=1 "
171                          "TESTS='"
172                          "--include-env=ad_member "
173                          "--include-env=ad_member_idmap_rid "
174                          "--include-env=ad_member_idmap_ad "
175                          "--include-env=ad_member_rfc2307 "
176                          "'", "text/plain"),
177                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
178
179     "samba-ad-dc-1": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
180                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
181                       ("make", "make -j", "text/plain"),
182                       ("test", "make test FAIL_IMMEDIATELY=1 "
183                        "TESTS='--include-env=ad_dc "
184                        "--include-env=ad_dc_no_nss "
185                        "--include-env=ad_dc_no_ntlm "
186                        "'", "text/plain"),
187                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
188
189     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
190                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
191                         ("make", "make -j", "text/plain"),
192                         ("test", "make test FAIL_IMMEDIATELY=1 "
193                          "TESTS='"
194                          "--include-env=vampire_dc "
195                          "--include-env=vampire_2000_dc "
196                          "--include-env=rodc "
197                          "'", "text/plain"),
198                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
199
200     "samba-ad-dc-3": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
201                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
202                         ("make", "make -j", "text/plain"),
203                         ("test", "make test FAIL_IMMEDIATELY=1 "
204                          "TESTS='"
205                          "--include-env=promoted_dc "
206                          "--include-env=chgdcpass "
207                          "--include-env=preforkrestartdc "
208                          "--include-env=proclimitdc "
209                          "'", "text/plain"),
210                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
211
212     "samba-ad-dc-4": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
213                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
214                         ("make", "make -j", "text/plain"),
215                         ("test", "make test FAIL_IMMEDIATELY=1 "
216                          "TESTS='"
217                          "--include-env=fl2000dc "
218                          "--include-env=fl2003dc "
219                          "--include-env=fl2008dc "
220                          "--include-env=fl2008r2dc "
221                          "'", "text/plain"),
222                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
223
224     "samba-ad-dc-5": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
225                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
226                         ("make", "make -j", "text/plain"),
227                         ("test", "make test FAIL_IMMEDIATELY=1 "
228                          "TESTS='"
229                          "--include-env=ad_dc_default "
230                          "'", "text/plain"),
231                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
232
233     "samba-ad-dc-6": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
234                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
235                         ("make", "make -j", "text/plain"),
236                         ("test", "make test FAIL_IMMEDIATELY=1 "
237                          "TESTS='"
238                          "--include-env=ad_dc_slowtests "
239                          "'", "text/plain"),
240                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
241
242
243     "samba-schemaupgrade": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
244                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
245                         ("make", "make -j", "text/plain"),
246                         ("test", "make test FAIL_IMMEDIATELY=1 "
247                          "TESTS='"
248                          "--include-env=schema_dc "
249                          "--include-env=schema_pair_dc "
250                          "'", "text/plain"),
251                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
252
253     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
254     # This is currently the longest task, so we don't randomly delay it.
255     "samba-ad-dc-ntvfs": [
256                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
257                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
258                       ("make", "make -j", "text/plain"),
259                       ("test", "make test FAIL_IMMEDIATELY=1 "
260                        "TESTS='"
261                        "--include-env=ad_dc_ntvfs "
262                        "'", "text/plain"),
263                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
264
265     # run the backup/restore testenvs separately as they're fairly standalone
266     # (and CI seems to max out at ~8 different DCs running at once)
267     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
268                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
269                         ("make", "make -j", "text/plain"),
270                         ("test", "make test FAIL_IMMEDIATELY=1 "
271                          "TESTS='--include-env=backupfromdc "
272                          "--include-env=restoredc "
273                          "--include-env=renamedc "
274                          "--include-env=offlinebackupdc "
275                          "--include-env=labdc "
276                          "--include-env=ad_dc_backup "
277                          "'", "text/plain"),
278                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
279
280     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
281                           ("make", "make -j", "text/plain"),
282                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
283
284     # Test cross-compile infrastructure
285     "samba-xc": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
286                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
287                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
288                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
289                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
290                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
291                    ("compare-results", "script/compare_cc_results.py "
292                     "./bin/c4che/default{} "
293                     "./bin-xe/c4che/default{} "
294                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
295
296     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
297     "samba-o3": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
298                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
299                    ("make", "make -j", "text/plain"),
300                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
301                     "TESTS='--include-env=ad_dc'", "text/plain"),
302                    ("install", "make install", "text/plain"),
303                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
304                    ("clean", "make clean", "text/plain")],
305
306     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
307
308                      # make sure we have tdb around:
309                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
310                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
311                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
312
313
314                      # build samba with cluster support (also building ctdb):
315                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
316                      ("samba-make", "make", "text/plain"),
317                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
318                      ("samba-install", "make install", "text/plain"),
319                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
320
321                      # clean up:
322                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
323                      ("clean", "make clean", "text/plain"),
324                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
325
326     "samba-libs": [
327                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
328                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
329                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
330                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
331
332                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
333                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
334                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
335
336                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
337                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
338                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
339
340                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
341                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
342                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
343
344                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
345                       ("nondevel-make", "make -j", "text/plain"),
346                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
347                       ("nondevel-install", "make install", "text/plain"),
348                       ("nondevel-dist", "make dist", "text/plain"),
349
350                       # retry with all modules shared
351                       ("allshared-distclean", "make distclean", "text/plain"),
352                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
353                       ("allshared-make", "make -j", "text/plain")],
354
355     "samba-none-env": [
356                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
357                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
358                       ("make", "make -j", "text/plain"),
359                       ("test", "make test "
360                        "FAIL_IMMEDIATELY=1 "
361                        "TESTS='--include-env=none'",
362                        "text/plain")],
363
364     "samba-static": [
365                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
366                       # build with all modules static
367                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
368                       ("allstatic-make", "make -j", "text/plain"),
369                       ("allstatic-test", "make test "
370                        "FAIL_IMMEDIATELY=1 "
371                        "TESTS='samba3.smb2.create.*nt4_dc'",
372                        "text/plain"),
373
374                       # retry without any required modules
375                       ("none-distclean", "make distclean", "text/plain"),
376                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
377                       ("none-make", "make -j", "text/plain"),
378
379                       # retry with nonshared smbd and smbtorture
380                       ("nonshared-distclean", "make distclean", "text/plain"),
381                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
382                       ("nonshared-make", "make -j", "text/plain")],
383
384     "samba-systemkrb5": [
385                       ("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
386                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
387                       ("make", "make -j", "text/plain"),
388                       # we currently cannot run a full make test, a limited list of tests could be run
389                       # via "make test TESTS=sometests"
390                       ("test", "make test FAIL_IMMEDIATELY=1 "
391                        "TESTS='--include-env=ktest'", "text/plain"),
392                       ("install", "make install", "text/plain"),
393                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
394                       ("clean", "make clean", "text/plain")
395                       ],
396
397     # Test Samba without python still builds.  When this test fails
398     # due to more use of Python, the expectations is that the newly
399     # failing part of the code should be disabled when
400     # --disable-python is set (rather than major work being done to
401     # support this environment).  The target here is for vendors
402     # shipping a minimal smbd.
403     "samba-nopython": [
404                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
405                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
406                       ("make", "make -j", "text/plain"),
407                       ("install", "make install", "text/plain"),
408                       ("test", "make test-nopython", "text/plain"),
409                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
410                       ("clean", "make clean", "text/plain"),
411
412                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
413                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
414                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
415
416                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
417                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
418                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
419
420                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
421                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
422                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
423
424                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
425                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
426                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
427
428                       # retry against installed library packages
429                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
430                       ("libs-make", "make -j", "text/plain"),
431                       ("libs-install", "make install", "text/plain"),
432                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
433                       ("libs-clean", "make clean", "text/plain")
434                       ],
435
436     # check we can do the same thing using python2
437     "samba-nopython-py2": [
438                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
439                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
440                       ("make", "PYTHON=python2 make -j", "text/plain"),
441                       ("install", "PYTHON=python2 make install", "text/plain"),
442                       ("test", "make test-nopython", "text/plain"),
443                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
444                       ("clean", "PYTHON=python2 make clean", "text/plain"),
445
446                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
447                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
448                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
449
450                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
451                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
452                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
453
454                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
455                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
456                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
457
458                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
459                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
460                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
461
462                       # retry against installed library packages
463                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
464                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
465                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
466                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
467                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
468                       ],
469
470     "ldb": [
471               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
472               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
473               ("make", "make", "text/plain"),
474               ("install", "make install", "text/plain"),
475               ("test", "make test", "text/plain"),
476               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
477               ("make-no-lmdb", "make", "text/plain"),
478               ("install-no-lmdb", "make install", "text/plain"),
479               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
480               ("distcheck", "make distcheck", "text/plain"),
481               ("clean", "make clean", "text/plain")],
482
483     "tdb": [
484               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
485               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
486               ("make", "make", "text/plain"),
487               ("install", "make install", "text/plain"),
488               ("test", "make test", "text/plain"),
489               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
490               ("distcheck", "make distcheck", "text/plain"),
491               ("clean", "make clean", "text/plain")],
492
493     "talloc": [
494                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
495                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
496                  ("make", "make", "text/plain"),
497                  ("install", "make install", "text/plain"),
498                  ("test", "make test", "text/plain"),
499                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
500                  ("distcheck", "make distcheck", "text/plain"),
501                  ("clean", "make clean", "text/plain")],
502
503     "replace": [
504                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
505                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
506                   ("make", "make", "text/plain"),
507                   ("install", "make install", "text/plain"),
508                   ("test", "make test", "text/plain"),
509                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
510                   ("distcheck", "make distcheck", "text/plain"),
511                   ("clean", "make clean", "text/plain")],
512
513     "tevent": [
514                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
515                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
516                  ("make", "make", "text/plain"),
517                  ("install", "make install", "text/plain"),
518                  ("test", "make test", "text/plain"),
519                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
520                  ("distcheck", "make distcheck", "text/plain"),
521                  ("clean", "make clean", "text/plain")],
522
523     "pidl": [
524         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
525         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
526         ("touch", "touch *.yp", "text/plain"),
527         ("make", "make", "text/plain"),
528         ("test", "make test", "text/plain"),
529         ("install", "make install", "text/plain"),
530         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
531         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
532         ("clean", "make clean", "text/plain")],
533
534
535     # these are useful for debugging autobuild
536     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
537     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
538
539
540 }
541
542
543 def do_print(msg):
544     print("%s" % msg)
545     sys.stdout.flush()
546     sys.stderr.flush()
547
548
549 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
550     if show is None:
551         show = options.verbose
552     if show:
553         do_print("Running: '%s' in '%s'" % (cmd, dir))
554     if output:
555         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
556     elif checkfail:
557         return check_call(cmd, shell=True, cwd=dir)
558     else:
559         return call(cmd, shell=True, cwd=dir)
560
561
562 class builder(object):
563     '''handle build of one directory'''
564
565     def __init__(self, name, sequence, cp=True):
566         self.name = name
567         if name in builddirs:
568             self.dir = builddirs[name]
569         else:
570             self.dir = "."
571
572         self.tag = self.name.replace('/', '_')
573         self.sequence = sequence
574         self.next = 0
575         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
576         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
577         if options.verbose:
578             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
579             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
580         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
581         self.stdout = open(self.stdout_path, 'w')
582         self.stderr = open(self.stderr_path, 'w')
583         self.stdin  = open("/dev/null", 'r')
584         self.sdir = "%s/%s" % (testbase, self.tag)
585         self.prefix = "%s/%s" % (test_prefix, self.tag)
586         run_cmd("rm -rf %s" % self.sdir)
587         run_cmd("rm -rf %s" % self.prefix)
588         if cp:
589             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
590         else:
591             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
592         self.start_next()
593
594     def start_next(self):
595         if self.next == len(self.sequence):
596             if not options.nocleanup:
597                 run_cmd("rm -rf %s" % self.sdir)
598                 run_cmd("rm -rf %s" % self.prefix)
599             do_print('%s: Completed OK' % self.name)
600             self.done = True
601             return
602         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
603         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
604         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
605         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
606         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
607 #        if self.output_mime_type == "text/x-subunit":
608 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
609         cwd = os.getcwd()
610         os.chdir("%s/%s" % (self.sdir, self.dir))
611         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
612         self.proc = Popen(self.cmd, shell=True,
613                           close_fds=True,
614                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
615         os.chdir(cwd)
616         self.next += 1
617
618
619 class buildlist(object):
620     '''handle build of multiple directories'''
621
622     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
623         global tasks
624         self.tlist = []
625         self.tail_proc = None
626         self.retry = None
627         if tasknames == []:
628             if options.restrict_tests:
629                 tasknames = ["samba-test-only"]
630             else:
631                 tasknames = defaulttasks
632         else:
633             # If we are only running one test,
634             # do not sleep randomly to wait for it to start
635             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
636
637         for n in tasknames:
638             b = builder(n, tasks[n], cp=n is not "pidl")
639             self.tlist.append(b)
640         if options.retry:
641             rebase_remote = "rebaseon"
642             retry_task = [("retry",
643                             '''set -e
644                             git remote add -t %s %s %s
645                             git fetch %s
646                             while :; do
647                               sleep 60
648                               git describe %s/%s > old_remote_branch.desc
649                               git fetch %s
650                               git describe %s/%s > remote_branch.desc
651                               diff old_remote_branch.desc remote_branch.desc
652                             done
653                            ''' % (
654                                 rebase_branch, rebase_remote, rebase_url,
655                                rebase_remote,
656                                rebase_remote, rebase_branch,
657                                rebase_remote,
658                                rebase_remote, rebase_branch
659                             ),
660                             "test/plain")]
661
662             self.retry = builder('retry', retry_task, cp=False)
663             self.need_retry = False
664
665     def kill_kids(self):
666         if self.tail_proc is not None:
667             self.tail_proc.terminate()
668             self.tail_proc.wait()
669             self.tail_proc = None
670         if self.retry is not None:
671             self.retry.proc.terminate()
672             self.retry.proc.wait()
673             self.retry = None
674         for b in self.tlist:
675             if b.proc is not None:
676                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
677                 b.proc.terminate()
678                 b.proc.wait()
679                 b.proc = None
680
681     def wait_one(self):
682         while True:
683             none_running = True
684             for b in self.tlist:
685                 if b.proc is None:
686                     continue
687                 none_running = False
688                 b.status = b.proc.poll()
689                 if b.status is None:
690                     continue
691                 b.proc = None
692                 return b
693             if options.retry:
694                 ret = self.retry.proc.poll()
695                 if ret is not None:
696                     self.need_retry = True
697                     self.retry = None
698                     return None
699             if none_running:
700                 return None
701             time.sleep(0.1)
702
703     def run(self):
704         while True:
705             b = self.wait_one()
706             if options.retry and self.need_retry:
707                 self.kill_kids()
708                 do_print("retry needed")
709                 return (0, None, None, None, "retry")
710             if b is None:
711                 break
712             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
713                 self.kill_kids()
714                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
715             b.start_next()
716         self.kill_kids()
717         return (0, None, None, None, "All OK")
718
719     def write_system_info(self):
720         filename = 'system-info.txt'
721         f = open(filename, 'w')
722         for cmd in ['uname -a',
723                     'lsb_release -a',
724                     'free',
725                     'mount',
726                     'cat /proc/cpuinfo',
727                     'cc --version',
728                     'df -m .',
729                     'df -m %s' % testbase]:
730             out = run_cmd(cmd, output=True, checkfail=False)
731             print('### %s' % cmd, file=f)
732             print(out.decode('utf8', 'backslashreplace'), file=f)
733             print(file=f)
734         f.close()
735         return filename
736
737     def tarlogs(self, fname):
738         tar = tarfile.open(fname, "w:gz")
739         for b in self.tlist:
740             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
741             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
742         if os.path.exists("autobuild.log"):
743             tar.add("autobuild.log")
744         sys_info = self.write_system_info()
745         tar.add(sys_info)
746         tar.close()
747
748     def remove_logs(self):
749         for b in self.tlist:
750             os.unlink(b.stdout_path)
751             os.unlink(b.stderr_path)
752
753     def start_tail(self):
754         cmd = ["tail", "-f"]
755         for b in self.tlist:
756             cmd.append(b.stdout_path)
757             cmd.append(b.stderr_path)
758         self.tail_proc = Popen(cmd, close_fds=True)
759
760
761 def cleanup():
762     if options.nocleanup:
763         return
764     run_cmd("stat %s || true" % test_tmpdir, show=True)
765     run_cmd("stat %s" % testbase, show=True)
766     do_print("Cleaning up %r" % cleanup_list)
767     for d in cleanup_list:
768         run_cmd("rm -rf %s" % d)
769
770
771 def find_git_root():
772     '''get to the top of the git repo'''
773     p = os.getcwd()
774     while p != '/':
775         if os.path.isdir(os.path.join(p, ".git")):
776             return p
777         p = os.path.abspath(os.path.join(p, '..'))
778     return None
779
780
781 def daemonize(logfile):
782     pid = os.fork()
783     if pid == 0:  # Parent
784         os.setsid()
785         pid = os.fork()
786         if pid != 0:  # Actual daemon
787             os._exit(0)
788     else:  # Grandparent
789         os._exit(0)
790
791     import resource      # Resource usage information.
792     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
793     if maxfd == resource.RLIM_INFINITY:
794         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
795     for fd in range(0, maxfd):
796         try:
797             os.close(fd)
798         except OSError:
799             pass
800     os.open(logfile, os.O_RDWR | os.O_CREAT)
801     os.dup2(0, 1)
802     os.dup2(0, 2)
803
804
805 def write_pidfile(fname):
806     '''write a pid file, cleanup on exit'''
807     f = open(fname, mode='w')
808     f.write("%u\n" % os.getpid())
809     f.close()
810
811
812 def rebase_tree(rebase_url, rebase_branch="master"):
813     rebase_remote = "rebaseon"
814     do_print("Rebasing on %s" % rebase_url)
815     run_cmd("git describe HEAD", show=True, dir=test_master)
816     run_cmd("git remote add -t %s %s %s" %
817             (rebase_branch, rebase_remote, rebase_url),
818             show=True, dir=test_master)
819     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
820     if options.fix_whitespace:
821         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
822                 (rebase_remote, rebase_branch),
823                 show=True, dir=test_master)
824     else:
825         run_cmd("git rebase --force-rebase %s/%s" %
826                 (rebase_remote, rebase_branch),
827                 show=True, dir=test_master)
828     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
829                    (rebase_remote, rebase_branch),
830                    dir=test_master, output=True)
831     if diff == '':
832         do_print("No differences between HEAD and %s/%s - exiting" %
833                  (rebase_remote, rebase_branch))
834         sys.exit(0)
835     run_cmd("git describe %s/%s" %
836             (rebase_remote, rebase_branch),
837             show=True, dir=test_master)
838     run_cmd("git describe HEAD", show=True, dir=test_master)
839     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
840             (rebase_remote, rebase_branch),
841             show=True, dir=test_master)
842
843
844 def push_to(push_url, push_branch="master"):
845     push_remote = "pushto"
846     do_print("Pushing to %s" % push_url)
847     if options.mark:
848         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
849         run_cmd("git commit --amend -c HEAD", dir=test_master)
850         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
851         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
852     run_cmd("git remote add -t %s %s %s" %
853             (push_branch, push_remote, push_url),
854             show=True, dir=test_master)
855     run_cmd("git push %s +HEAD:%s" %
856             (push_remote, push_branch),
857             show=True, dir=test_master)
858
859
860 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
861
862 gitroot = find_git_root()
863 if gitroot is None:
864     raise Exception("Failed to find git root")
865
866 parser = OptionParser()
867 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
868 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
869 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
870 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
871                   default=def_testbase)
872 parser.add_option("", "--passcmd", help="command to run on success", default=None)
873 parser.add_option("", "--verbose", help="show all commands as they are run",
874                   default=False, action="store_true")
875 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
876                   default=None, type='str')
877 parser.add_option("", "--pushto", help="push to a git url on success",
878                   default=None, type='str')
879 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
880                   default=False, action="store_true")
881 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
882                   default=False, action="store_true")
883 parser.add_option("", "--retry", help="automatically retry if master changes",
884                   default=False, action="store_true")
885 parser.add_option("", "--email", help="send email to the given address on failure",
886                   type='str', default=None)
887 parser.add_option("", "--email-from", help="send email from the given address",
888                   type='str', default="autobuild@samba.org")
889 parser.add_option("", "--email-server", help="send email via the given server",
890                   type='str', default='localhost')
891 parser.add_option("", "--always-email", help="always send email, even on success",
892                   action="store_true")
893 parser.add_option("", "--daemon", help="daemonize after initial setup",
894                   action="store_true")
895 parser.add_option("", "--branch", help="the branch to work on (default=master)",
896                   default="master", type='str')
897 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
898                   default=gitroot, type='str')
899 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
900                   default=False, action="store_true")
901 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
902                   default='')
903
904
905 def send_email(subject, text, log_tar):
906     if options.email is None:
907         do_print("not sending email because the recipient is not set")
908         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
909                  (subject, text))
910         return
911     outer = MIMEMultipart()
912     outer['Subject'] = subject
913     outer['To'] = options.email
914     outer['From'] = options.email_from
915     outer['Date'] = email.utils.formatdate(localtime=True)
916     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
917     outer.attach(MIMEText(text, 'plain'))
918     if options.attach_logs:
919         fp = open(log_tar, 'rb')
920         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
921         fp.close()
922         # Set the filename parameter
923         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
924         outer.attach(msg)
925     content = outer.as_string()
926     s = smtplib.SMTP(options.email_server)
927     email_user = os.getenv('SMTP_USERNAME')
928     email_password = os.getenv('SMTP_PASSWORD')
929     if email_user is not None:
930         s.starttls()
931         s.login(email_user, email_password)
932
933     s.sendmail(options.email_from, [options.email], content)
934     s.set_debuglevel(1)
935     s.quit()
936
937
938 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
939                   elapsed_time, log_base=None, add_log_tail=True):
940     '''send an email to options.email about the failure'''
941     elapsed_minutes = elapsed_time / 60.0
942     if log_base is None:
943         log_base = gitroot
944     text = '''
945 Dear Developer,
946
947 Your autobuild on %s failed after %.1f minutes
948 when trying to test %s with the following error:
949
950    %s
951
952 the autobuild has been abandoned. Please fix the error and resubmit.
953
954 A summary of the autobuild process is here:
955
956   %s/autobuild.log
957 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
958
959     if options.restrict_tests:
960         text += """
961 The build was restricted to tests matching %s\n""" % options.restrict_tests
962
963     if failed_task != 'rebase':
964         text += '''
965 You can see logs of the failed task here:
966
967   %s/%s.stdout
968   %s/%s.stderr
969
970 or you can get full logs of all tasks in this job here:
971
972   %s/logs.tar.gz
973
974 The top commit for the tree that was built was:
975
976 %s
977
978 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
979
980     if add_log_tail:
981         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
982         lines = f.readlines()
983         log_tail = "".join(lines[-50:])
984         num_lines = len(lines)
985         if num_lines < 50:
986             # Also include stderr (compile failures) if < 50 lines of stdout
987             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
988             log_tail += "".join(f.readlines()[-(50 - num_lines):])
989
990         text += '''
991 The last 50 lines of log messages:
992
993 %s
994     ''' % log_tail
995         f.close()
996
997     logs = os.path.join(gitroot, 'logs.tar.gz')
998     send_email('autobuild[%s] failure on %s for task %s during %s'
999                % (options.branch, platform.node(), failed_task, failed_stage),
1000                text, logs)
1001
1002
1003 def email_success(elapsed_time, log_base=None):
1004     '''send an email to options.email about a successful build'''
1005     if log_base is None:
1006         log_base = gitroot
1007     text = '''
1008 Dear Developer,
1009
1010 Your autobuild on %s has succeeded after %.1f minutes.
1011
1012 ''' % (platform.node(), elapsed_time / 60.)
1013
1014     if options.restrict_tests:
1015         text += """
1016 The build was restricted to tests matching %s\n""" % options.restrict_tests
1017
1018     if options.keeplogs:
1019         text += '''
1020
1021 you can get full logs of all tasks in this job here:
1022
1023   %s/logs.tar.gz
1024
1025 ''' % log_base
1026
1027     text += '''
1028 The top commit for the tree that was built was:
1029
1030 %s
1031 ''' % top_commit_msg
1032
1033     logs = os.path.join(gitroot, 'logs.tar.gz')
1034     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1035                text, logs)
1036
1037
1038 (options, args) = parser.parse_args()
1039
1040 if options.retry:
1041     if options.rebase is None:
1042         raise Exception('You can only use --retry if you also rebase')
1043
1044 testbase = "%s/b%u" % (options.testbase, os.getpid())
1045 test_master = "%s/master" % testbase
1046 test_prefix = "%s/prefix" % testbase
1047 test_tmpdir = "%s/tmp" % testbase
1048 os.environ['TMPDIR'] = test_tmpdir
1049
1050 # get the top commit message, for emails
1051 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1052 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
1053
1054 try:
1055     os.makedirs(testbase)
1056 except Exception as reason:
1057     raise Exception("Unable to create %s : %s" % (testbase, reason))
1058 cleanup_list.append(testbase)
1059
1060 if options.daemon:
1061     logfile = os.path.join(testbase, "log")
1062     do_print("Forking into the background, writing progress to %s" % logfile)
1063     daemonize(logfile)
1064
1065 write_pidfile(gitroot + "/autobuild.pid")
1066
1067 start_time = time.time()
1068
1069 while True:
1070     try:
1071         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1072         os.makedirs(test_tmpdir)
1073         # The waf uninstall code removes empty directories all the way
1074         # up the tree.  Creating a file in test_tmpdir stops it from
1075         # being removed.
1076         run_cmd("touch %s" % os.path.join(test_tmpdir,
1077                                           ".directory-is-not-empty"), show=True)
1078         run_cmd("stat %s" % test_tmpdir, show=True)
1079         run_cmd("stat %s" % testbase, show=True)
1080         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1081     except Exception:
1082         cleanup()
1083         raise
1084
1085     try:
1086         try:
1087             if options.rebase is not None:
1088                 rebase_tree(options.rebase, rebase_branch=options.branch)
1089         except Exception:
1090             cleanup_list.append(gitroot + "/autobuild.pid")
1091             cleanup()
1092             elapsed_time = time.time() - start_time
1093             email_failure(-1, 'rebase', 'rebase', 'rebase',
1094                           'rebase on %s failed' % options.branch,
1095                           elapsed_time, log_base=options.log_base)
1096             sys.exit(1)
1097         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1098         if options.tail:
1099             blist.start_tail()
1100         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1101         if status != 0 or errstr != "retry":
1102             break
1103         cleanup()
1104     except Exception:
1105         cleanup()
1106         raise
1107
1108 cleanup_list.append(gitroot + "/autobuild.pid")
1109
1110 do_print(errstr)
1111
1112 blist.kill_kids()
1113 if options.tail:
1114     do_print("waiting for tail to flush")
1115     time.sleep(1)
1116
1117 elapsed_time = time.time() - start_time
1118 if status == 0:
1119     if options.passcmd is not None:
1120         do_print("Running passcmd: %s" % options.passcmd)
1121         run_cmd(options.passcmd, dir=test_master)
1122     if options.pushto is not None:
1123         push_to(options.pushto, push_branch=options.branch)
1124     if options.keeplogs or options.attach_logs:
1125         blist.tarlogs("logs.tar.gz")
1126         do_print("Logs in logs.tar.gz")
1127     if options.always_email:
1128         email_success(elapsed_time, log_base=options.log_base)
1129     blist.remove_logs()
1130     cleanup()
1131     do_print(errstr)
1132     sys.exit(0)
1133
1134 # something failed, gather a tar of the logs
1135 blist.tarlogs("logs.tar.gz")
1136
1137 if options.email is not None:
1138     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1139                   elapsed_time, log_base=options.log_base)
1140 else:
1141     elapsed_minutes = elapsed_time / 60.0
1142     print('''
1143
1144 ####################################################################
1145
1146 AUTOBUILD FAILURE
1147
1148 Your autobuild[%s] on %s failed after %.1f minutes
1149 when trying to test %s with the following error:
1150
1151    %s
1152
1153 the autobuild has been abandoned. Please fix the error and resubmit.
1154
1155 ####################################################################
1156
1157 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1158
1159 cleanup()
1160 do_print(errstr)
1161 do_print("Logs in logs.tar.gz")
1162 sys.exit(status)