PEP8: fix E202: whitespace before ')'
[samba.git] / script / autobuild.py
1 #!/usr/bin/env python
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call,Popen, PIPE
8 import os, tarfile, sys, time
9 from optparse import OptionParser
10 import smtplib
11 import email
12 from email.mime.text import MIMEText
13 from email.mime.base import MIMEBase
14 from email.mime.application import MIMEApplication
15 from email.mime.multipart import MIMEMultipart
16 from distutils.sysconfig import get_python_lib
17 import platform
18
19 os.environ["PYTHONUNBUFFERED"] = "1"
20
21 # This speeds up testing remarkably.
22 os.environ['TDB_NO_FSYNC'] = '1'
23
24 cleanup_list = []
25
26 builddirs = {
27     "ctdb"    : "ctdb",
28     "samba"  : ".",
29     "samba-nt4"  : ".",
30     "samba-fileserver"  : ".",
31     "samba-xc" : ".",
32     "samba-o3" : ".",
33     "samba-ctdb" : ".",
34     "samba-libs"  : ".",
35     "samba-static"  : ".",
36     "samba-test-only"  : ".",
37     "samba-none-env"  : ".",
38     "samba-ad-dc"  : ".",
39     "samba-ad-dc-2"  : ".",
40     "samba-systemkrb5"  : ".",
41     "samba-nopython"  : ".",
42     "ldb"     : "lib/ldb",
43     "tdb"     : "lib/tdb",
44     "talloc"  : "lib/talloc",
45     "replace" : "lib/replace",
46     "tevent"  : "lib/tevent",
47     "pidl"    : "pidl",
48     "pass"    : ".",
49     "fail"    : ".",
50     "retry"   : "."
51 }
52
53 defaulttasks = ["ctdb",
54                  "samba",
55                  "samba-nt4",
56                  "samba-fileserver",
57                  "samba-xc",
58                  "samba-o3",
59                  "samba-ctdb",
60                  "samba-libs",
61                  "samba-static",
62                  "samba-none-env",
63                  "samba-ad-dc",
64                  "samba-ad-dc-2",
65                  "samba-systemkrb5",
66                  "samba-nopython",
67                  "ldb",
68                  "tdb",
69                  "talloc",
70                  "replace",
71                  "tevent",
72                  "pidl"]
73
74 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
75     defaulttasks.remove("samba-o3")
76
77 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
78 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
79
80 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
81 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
82 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
83 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
84 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
85 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
86 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
87
88 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
89     extra_python = ""
90 else:
91     extra_python = "--extra-python=/usr/bin/python3"
92
93 tasks = {
94     "ctdb" : [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
95                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
96                ("make", "make all", "text/plain"),
97                ("install", "make install", "text/plain"),
98                ("test", "make autotest", "text/plain"),
99                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
100                ("clean", "make clean", "text/plain")],
101
102     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
103     "samba" : [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
104                 ("make", "make -j", "text/plain"),
105                 ("test", "make test FAIL_IMMEDIATELY=1 "
106                  "TESTS='--exclude-env=none "
107                  "--exclude-env=nt4_dc "
108                  "--exclude-env=nt4_member "
109                  "--exclude-env=ad_dc "
110                  "--exclude-env=fl2003dc "
111                  "--exclude-env=fl2008r2dc "
112                  "--exclude-env=ad_member "
113                  "--exclude-env=ad_member_idmap_rid "
114                  "--exclude-env=ad_member_idmap_ad "
115                  "--exclude-env=chgdcpass "
116                  "--exclude-env=vampire_2000_dc "
117                  "--exclude-env=fl2000dc "
118                  "--exclude-env=fileserver'",
119                  "text/plain"),
120                 ("install", "make install", "text/plain"),
121                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
122                 ("clean", "make clean", "text/plain")],
123
124     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
125     "samba-nt4" : [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
126                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
127                     ("make", "make -j", "text/plain"),
128                     ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
129                     ("install", "make install", "text/plain"),
130                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
131                     ("clean", "make clean", "text/plain")],
132
133     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
134     "samba-fileserver" : [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
135                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json-audit --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
136                            ("make", "make -j", "text/plain"),
137                            ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
138                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
139
140     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
141     "samba-ad-dc" : [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
142                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
143                       ("make", "make -j", "text/plain"),
144                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='"
145                        "--include-env=ad_dc "
146                        "--include-env=fl2003dc "
147                        "--include-env=fl2008r2dc "
148                        "--include-env=ad_member "
149                        "--include-env=ad_member_idmap_rid "
150                        "--include-env=ad_member_idmap_ad'", "text/plain"),
151                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
152
153     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
154     "samba-ad-dc-2" : [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
155                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
156                         ("make", "make -j", "text/plain"),
157                         ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
158                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
159
160     "samba-test-only" : [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
161                           ("make", "make -j", "text/plain"),
162                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain")],
163
164     # Test cross-compile infrastructure
165     "samba-xc" : [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
166                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
167                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
168                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
169                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
170                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
171                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
172
173     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
174     "samba-o3" : [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
175                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
176                    ("make", "make -j", "text/plain"),
177                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
178                    ("install", "make install", "text/plain"),
179                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
180                    ("clean", "make clean", "text/plain")],
181
182     "samba-ctdb" : [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
183
184                      # make sure we have tdb around:
185                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
186                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
187                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
188
189
190                      # build samba with cluster support (also building ctdb):
191                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
192                      ("samba-make", "make", "text/plain"),
193                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
194                      ("samba-install", "make install", "text/plain"),
195                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
196
197                      # clean up:
198                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
199                      ("clean", "make clean", "text/plain"),
200                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
201
202     "samba-libs" : [
203                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
204                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
205                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
206                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
207
208                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
209                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
210                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
211
212                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
213                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
214                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
215
216                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
217                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
218                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
219
220                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
221                       ("nondevel-make", "make -j", "text/plain"),
222                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
223                       ("nondevel-install", "make install", "text/plain"),
224                       ("nondevel-dist", "make dist", "text/plain"),
225
226                       # retry with all modules shared
227                       ("allshared-distclean", "make distclean", "text/plain"),
228                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
229                       ("allshared-make", "make -j", "text/plain")],
230
231     "samba-none-env" : [
232                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
233                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
234                       ("make", "make -j", "text/plain"),
235                       ("test", "make test "
236                        "FAIL_IMMEDIATELY=1 "
237                        "TESTS='--include-env=none'",
238                        "text/plain")],
239
240     "samba-static" : [
241                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
242                       # build with all modules static
243                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
244                       ("allstatic-make", "make -j", "text/plain"),
245                       ("allstatic-test", "make test "
246                        "FAIL_IMMEDIATELY=1 "
247                        "TESTS='samba3.smb2.create.*nt4_dc'",
248                        "text/plain"),
249
250                       # retry without any required modules
251                       ("none-distclean", "make distclean", "text/plain"),
252                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
253                       ("none-make", "make -j", "text/plain"),
254
255                       # retry with nonshared smbd and smbtorture
256                       ("nonshared-distclean", "make distclean", "text/plain"),
257                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
258                       ("nonshared-make", "make -j", "text/plain")],
259
260     "samba-systemkrb5" : [
261                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
262                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
263                       ("make", "make -j", "text/plain"),
264                       # we currently cannot run a full make test, a limited list of tests could be run
265                       # via "make test TESTS=sometests"
266                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
267                       ("install", "make install", "text/plain"),
268                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
269                       ("clean", "make clean", "text/plain")
270                       ],
271
272     # Test Samba without python still builds.  When this test fails
273     # due to more use of Python, the expectations is that the newly
274     # failing part of the code should be disabled when
275     # --disable-python is set (rather than major work being done to
276     # support this environment).  The target here is for vendors
277     # shipping a minimal smbd.
278     "samba-nopython" : [
279                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
280                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
281                       ("make", "make -j", "text/plain"),
282                       ("install", "make install", "text/plain"),
283                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
284                       ("clean", "make clean", "text/plain"),
285
286                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
287                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
288                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
289
290                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
291                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
292                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
293
294                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
295                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
296                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
297
298                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
299                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
300                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
301
302                       # retry against installed library packages
303                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
304                       ("libs-make", "make -j", "text/plain"),
305                       ("libs-install", "make install", "text/plain"),
306                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
307                       ("libs-clean", "make clean", "text/plain")
308                       ],
309
310
311
312     "ldb" : [
313               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
314               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
315               ("make", "make", "text/plain"),
316               ("install", "make install", "text/plain"),
317               ("test", "make test", "text/plain"),
318               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
319               ("make-no-lmdb", "make", "text/plain"),
320               ("install-no-lmdb", "make install", "text/plain"),
321               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
322               ("distcheck", "make distcheck", "text/plain"),
323               ("clean", "make clean", "text/plain")],
324
325     "tdb" : [
326               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
327               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
328               ("make", "make", "text/plain"),
329               ("install", "make install", "text/plain"),
330               ("test", "make test", "text/plain"),
331               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
332               ("distcheck", "make distcheck", "text/plain"),
333               ("clean", "make clean", "text/plain")],
334
335     "talloc" : [
336                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
337                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
338                  ("make", "make", "text/plain"),
339                  ("install", "make install", "text/plain"),
340                  ("test", "make test", "text/plain"),
341                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
342                  ("distcheck", "make distcheck", "text/plain"),
343                  ("clean", "make clean", "text/plain")],
344
345     "replace" : [
346                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
347                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
348                   ("make", "make", "text/plain"),
349                   ("install", "make install", "text/plain"),
350                   ("test", "make test", "text/plain"),
351                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
352                   ("distcheck", "make distcheck", "text/plain"),
353                   ("clean", "make clean", "text/plain")],
354
355     "tevent" : [
356                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
357                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
358                  ("make", "make", "text/plain"),
359                  ("install", "make install", "text/plain"),
360                  ("test", "make test", "text/plain"),
361                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
362                  ("distcheck", "make distcheck", "text/plain"),
363                  ("clean", "make clean", "text/plain")],
364
365     "pidl" : [
366         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
367         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
368         ("touch", "touch *.yp", "text/plain"),
369         ("make", "make", "text/plain"),
370         ("test", "make test", "text/plain"),
371         ("install", "make install", "text/plain"),
372         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
373         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
374         ("clean", "make clean", "text/plain")],
375
376     # these are useful for debugging autobuild
377     'pass' : [("pass", 'echo passing && /bin/true', "text/plain")],
378     'fail' : [("fail", 'echo failing && /bin/false', "text/plain")]
379 }
380
381 def do_print(msg):
382     print("%s" % msg)
383     sys.stdout.flush()
384     sys.stderr.flush()
385
386 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
387     if show is None:
388         show = options.verbose
389     if show:
390         do_print("Running: '%s' in '%s'" % (cmd, dir))
391     if output:
392         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
393     elif checkfail:
394         return check_call(cmd, shell=True, cwd=dir)
395     else:
396         return call(cmd, shell=True, cwd=dir)
397
398
399 class builder(object):
400     '''handle build of one directory'''
401
402     def __init__(self, name, sequence, cp=True):
403         self.name = name
404         self.dir = builddirs[name]
405
406         self.tag = self.name.replace('/', '_')
407         self.sequence = sequence
408         self.next = 0
409         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
410         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
411         if options.verbose:
412             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
413             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
414         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
415         self.stdout = open(self.stdout_path, 'w')
416         self.stderr = open(self.stderr_path, 'w')
417         self.stdin  = open("/dev/null", 'r')
418         self.sdir = "%s/%s" % (testbase, self.tag)
419         self.prefix = "%s/%s" % (test_prefix, self.tag)
420         run_cmd("rm -rf %s" % self.sdir)
421         run_cmd("rm -rf %s" % self.prefix)
422         if cp:
423             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
424         else:
425             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
426         self.start_next()
427
428     def start_next(self):
429         if self.next == len(self.sequence):
430             if not options.nocleanup:
431                 run_cmd("rm -rf %s" % self.sdir)
432                 run_cmd("rm -rf %s" % self.prefix)
433             do_print('%s: Completed OK' % self.name)
434             self.done = True
435             return
436         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
437         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
438         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
439         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
440         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
441         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
442 #        if self.output_mime_type == "text/x-subunit":
443 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
444         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
445         cwd = os.getcwd()
446         os.chdir("%s/%s" % (self.sdir, self.dir))
447         self.proc = Popen(self.cmd, shell=True,
448                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
449         os.chdir(cwd)
450         self.next += 1
451
452
453 class buildlist(object):
454     '''handle build of multiple directories'''
455
456     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
457         global tasks
458         self.tlist = []
459         self.tail_proc = None
460         self.retry = None
461         if tasknames == []:
462             if options.restrict_tests:
463                 tasknames = ["samba-test-only"]
464             else:
465                 tasknames = defaulttasks
466         else:
467             # If we are only running one test,
468             # do not sleep randomly to wait for it to start
469             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
470
471         for n in tasknames:
472             b = builder(n, tasks[n], cp=n is not "pidl")
473             self.tlist.append(b)
474         if options.retry:
475             rebase_remote = "rebaseon"
476             retry_task = [("retry",
477                             '''set -e
478                             git remote add -t %s %s %s
479                             git fetch %s
480                             while :; do
481                               sleep 60
482                               git describe %s/%s > old_remote_branch.desc
483                               git fetch %s
484                               git describe %s/%s > remote_branch.desc
485                               diff old_remote_branch.desc remote_branch.desc
486                             done
487                            ''' % (
488                                 rebase_branch, rebase_remote, rebase_url,
489                                rebase_remote,
490                                rebase_remote, rebase_branch,
491                                rebase_remote,
492                                rebase_remote, rebase_branch
493                             ),
494                             "test/plain")]
495
496             self.retry = builder('retry', retry_task, cp=False)
497             self.need_retry = False
498
499     def kill_kids(self):
500         if self.tail_proc is not None:
501             self.tail_proc.terminate()
502             self.tail_proc.wait()
503             self.tail_proc = None
504         if self.retry is not None:
505             self.retry.proc.terminate()
506             self.retry.proc.wait()
507             self.retry = None
508         for b in self.tlist:
509             if b.proc is not None:
510                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
511                 b.proc.terminate()
512                 b.proc.wait()
513                 b.proc = None
514
515     def wait_one(self):
516         while True:
517             none_running = True
518             for b in self.tlist:
519                 if b.proc is None:
520                     continue
521                 none_running = False
522                 b.status = b.proc.poll()
523                 if b.status is None:
524                     continue
525                 b.proc = None
526                 return b
527             if options.retry:
528                 ret = self.retry.proc.poll()
529                 if ret is not None:
530                     self.need_retry = True
531                     self.retry = None
532                     return None
533             if none_running:
534                 return None
535             time.sleep(0.1)
536
537     def run(self):
538         while True:
539             b = self.wait_one()
540             if options.retry and self.need_retry:
541                 self.kill_kids()
542                 do_print("retry needed")
543                 return (0, None, None, None, "retry")
544             if b is None:
545                 break
546             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
547                 self.kill_kids()
548                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
549             b.start_next()
550         self.kill_kids()
551         return (0, None, None, None, "All OK")
552
553     def write_system_info(self):
554         filename = 'system-info.txt'
555         f = open(filename, 'w')
556         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
557                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
558             print('### %s' % cmd, file=f)
559             print(run_cmd(cmd, output=True, checkfail=False), file=f)
560             print(file=f)
561         f.close()
562         return filename
563
564     def tarlogs(self, fname):
565         tar = tarfile.open(fname, "w:gz")
566         for b in self.tlist:
567             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
568             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
569         if os.path.exists("autobuild.log"):
570             tar.add("autobuild.log")
571         sys_info = self.write_system_info()
572         tar.add(sys_info)
573         tar.close()
574
575     def remove_logs(self):
576         for b in self.tlist:
577             os.unlink(b.stdout_path)
578             os.unlink(b.stderr_path)
579
580     def start_tail(self):
581         cwd = os.getcwd()
582         cmd = "tail -f *.stdout *.stderr"
583         os.chdir(gitroot)
584         self.tail_proc = Popen(cmd, shell=True)
585         os.chdir(cwd)
586
587
588 def cleanup():
589     if options.nocleanup:
590         return
591     run_cmd("stat %s || true" % test_tmpdir, show=True)
592     run_cmd("stat %s" % testbase, show=True)
593     do_print("Cleaning up ....")
594     for d in cleanup_list:
595         run_cmd("rm -rf %s" % d)
596
597
598 def find_git_root():
599     '''get to the top of the git repo'''
600     p=os.getcwd()
601     while p != '/':
602         if os.path.isdir(os.path.join(p, ".git")):
603             return p
604         p = os.path.abspath(os.path.join(p, '..'))
605     return None
606
607
608 def daemonize(logfile):
609     pid = os.fork()
610     if pid == 0: # Parent
611         os.setsid()
612         pid = os.fork()
613         if pid != 0: # Actual daemon
614             os._exit(0)
615     else: # Grandparent
616         os._exit(0)
617
618     import resource      # Resource usage information.
619     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
620     if maxfd == resource.RLIM_INFINITY:
621         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
622     for fd in range(0, maxfd):
623         try:
624             os.close(fd)
625         except OSError:
626             pass
627     os.open(logfile, os.O_RDWR | os.O_CREAT)
628     os.dup2(0, 1)
629     os.dup2(0, 2)
630
631 def write_pidfile(fname):
632     '''write a pid file, cleanup on exit'''
633     f = open(fname, mode='w')
634     f.write("%u\n" % os.getpid())
635     f.close()
636
637
638 def rebase_tree(rebase_url, rebase_branch = "master"):
639     rebase_remote = "rebaseon"
640     do_print("Rebasing on %s" % rebase_url)
641     run_cmd("git describe HEAD", show=True, dir=test_master)
642     run_cmd("git remote add -t %s %s %s" %
643             (rebase_branch, rebase_remote, rebase_url),
644             show=True, dir=test_master)
645     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
646     if options.fix_whitespace:
647         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
648                 (rebase_remote, rebase_branch),
649                 show=True, dir=test_master)
650     else:
651         run_cmd("git rebase --force-rebase %s/%s" %
652                 (rebase_remote, rebase_branch),
653                 show=True, dir=test_master)
654     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
655                    (rebase_remote, rebase_branch),
656                    dir=test_master, output=True)
657     if diff == '':
658         do_print("No differences between HEAD and %s/%s - exiting" %
659                  (rebase_remote, rebase_branch))
660         sys.exit(0)
661     run_cmd("git describe %s/%s" %
662             (rebase_remote, rebase_branch),
663             show=True, dir=test_master)
664     run_cmd("git describe HEAD", show=True, dir=test_master)
665     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
666             (rebase_remote, rebase_branch),
667             show=True, dir=test_master)
668
669 def push_to(push_url, push_branch = "master"):
670     push_remote = "pushto"
671     do_print("Pushing to %s" % push_url)
672     if options.mark:
673         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
674         run_cmd("git commit --amend -c HEAD", dir=test_master)
675         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
676         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
677     run_cmd("git remote add -t %s %s %s" %
678             (push_branch, push_remote, push_url),
679             show=True, dir=test_master)
680     run_cmd("git push %s +HEAD:%s" %
681             (push_remote, push_branch),
682             show=True, dir=test_master)
683
684 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
685
686 gitroot = find_git_root()
687 if gitroot is None:
688     raise Exception("Failed to find git root")
689
690 parser = OptionParser()
691 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
692 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
693 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
694 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
695                   default=def_testbase)
696 parser.add_option("", "--passcmd", help="command to run on success", default=None)
697 parser.add_option("", "--verbose", help="show all commands as they are run",
698                   default=False, action="store_true")
699 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
700                   default=None, type='str')
701 parser.add_option("", "--pushto", help="push to a git url on success",
702                   default=None, type='str')
703 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
704                   default=False, action="store_true")
705 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
706                   default=False, action="store_true")
707 parser.add_option("", "--retry", help="automatically retry if master changes",
708                   default=False, action="store_true")
709 parser.add_option("", "--email", help="send email to the given address on failure",
710                   type='str', default=None)
711 parser.add_option("", "--email-from", help="send email from the given address",
712                   type='str', default="autobuild@samba.org")
713 parser.add_option("", "--email-server", help="send email via the given server",
714                   type='str', default='localhost')
715 parser.add_option("", "--always-email", help="always send email, even on success",
716                   action="store_true")
717 parser.add_option("", "--daemon", help="daemonize after initial setup",
718                   action="store_true")
719 parser.add_option("", "--branch", help="the branch to work on (default=master)",
720                   default="master", type='str')
721 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
722                   default=gitroot, type='str')
723 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
724                   default=False, action="store_true")
725 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
726                   default='')
727
728 def send_email(subject, text, log_tar):
729     if options.email is None:
730         do_print("not sending email because the recipient is not set")
731         do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
732                  (subject, text))
733         return
734     outer = MIMEMultipart()
735     outer['Subject'] = subject
736     outer['To'] = options.email
737     outer['From'] = options.email_from
738     outer['Date'] = email.utils.formatdate(localtime = True)
739     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
740     outer.attach(MIMEText(text, 'plain'))
741     if options.attach_logs:
742         fp = open(log_tar, 'rb')
743         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
744         fp.close()
745         # Set the filename parameter
746         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
747         outer.attach(msg)
748     content = outer.as_string()
749     s = smtplib.SMTP(options.email_server)
750     s.sendmail(options.email_from, [options.email], content)
751     s.set_debuglevel(1)
752     s.quit()
753
754 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
755                   elapsed_time, log_base=None, add_log_tail=True):
756     '''send an email to options.email about the failure'''
757     elapsed_minutes = elapsed_time / 60.0
758     user = os.getenv("USER")
759     if log_base is None:
760         log_base = gitroot
761     text = '''
762 Dear Developer,
763
764 Your autobuild on %s failed after %.1f minutes
765 when trying to test %s with the following error:
766
767    %s
768
769 the autobuild has been abandoned. Please fix the error and resubmit.
770
771 A summary of the autobuild process is here:
772
773   %s/autobuild.log
774 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
775
776     if options.restrict_tests:
777         text += """
778 The build was restricted to tests matching %s\n""" % options.restrict_tests
779
780     if failed_task != 'rebase':
781         text += '''
782 You can see logs of the failed task here:
783
784   %s/%s.stdout
785   %s/%s.stderr
786
787 or you can get full logs of all tasks in this job here:
788
789   %s/logs.tar.gz
790
791 The top commit for the tree that was built was:
792
793 %s
794
795 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
796
797     if add_log_tail:
798         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
799         lines = f.readlines()
800         log_tail = "".join(lines[-50:])
801         num_lines = len(lines)
802         if num_lines < 50:
803             # Also include stderr (compile failures) if < 50 lines of stdout
804             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
805             log_tail += "".join(f.readlines()[-(50-num_lines):])
806
807         text += '''
808 The last 50 lines of log messages:
809
810 %s
811     ''' % log_tail
812         f.close()
813
814     logs = os.path.join(gitroot, 'logs.tar.gz')
815     send_email('autobuild[%s] failure on %s for task %s during %s'
816                % (options.branch, platform.node(), failed_task, failed_stage),
817                text, logs)
818
819 def email_success(elapsed_time, log_base=None):
820     '''send an email to options.email about a successful build'''
821     user = os.getenv("USER")
822     if log_base is None:
823         log_base = gitroot
824     text = '''
825 Dear Developer,
826
827 Your autobuild on %s has succeeded after %.1f minutes.
828
829 ''' % (platform.node(), elapsed_time / 60.)
830
831     if options.restrict_tests:
832         text += """
833 The build was restricted to tests matching %s\n""" % options.restrict_tests
834
835     if options.keeplogs:
836         text += '''
837
838 you can get full logs of all tasks in this job here:
839
840   %s/logs.tar.gz
841
842 ''' % log_base
843
844     text += '''
845 The top commit for the tree that was built was:
846
847 %s
848 ''' % top_commit_msg
849
850     logs = os.path.join(gitroot, 'logs.tar.gz')
851     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
852                text, logs)
853
854
855 (options, args) = parser.parse_args()
856
857 if options.retry:
858     if options.rebase is None:
859         raise Exception('You can only use --retry if you also rebase')
860
861 testbase = "%s/b%u" % (options.testbase, os.getpid())
862 test_master = "%s/master" % testbase
863 test_prefix = "%s/prefix" % testbase
864 test_tmpdir = "%s/tmp" % testbase
865 os.environ['TMPDIR'] = test_tmpdir
866
867 # get the top commit message, for emails
868 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
869
870 try:
871     os.makedirs(testbase)
872 except Exception as reason:
873     raise Exception("Unable to create %s : %s" % (testbase, reason))
874 cleanup_list.append(testbase)
875
876 if options.daemon:
877     logfile = os.path.join(testbase, "log")
878     do_print("Forking into the background, writing progress to %s" % logfile)
879     daemonize(logfile)
880
881 write_pidfile(gitroot + "/autobuild.pid")
882
883 start_time = time.time()
884
885 while True:
886     try:
887         run_cmd("rm -rf %s" % test_tmpdir, show=True)
888         os.makedirs(test_tmpdir)
889         # The waf uninstall code removes empty directories all the way
890         # up the tree.  Creating a file in test_tmpdir stops it from
891         # being removed.
892         run_cmd("touch %s" % os.path.join(test_tmpdir,
893                                           ".directory-is-not-empty"), show=True)
894         run_cmd("stat %s" % test_tmpdir, show=True)
895         run_cmd("stat %s" % testbase, show=True)
896         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
897     except Exception:
898         cleanup()
899         raise
900
901     try:
902         try:
903             if options.rebase is not None:
904                 rebase_tree(options.rebase, rebase_branch=options.branch)
905         except Exception:
906             cleanup_list.append(gitroot + "/autobuild.pid")
907             cleanup()
908             elapsed_time = time.time() - start_time
909             email_failure(-1, 'rebase', 'rebase', 'rebase',
910                           'rebase on %s failed' % options.branch,
911                           elapsed_time, log_base=options.log_base)
912             sys.exit(1)
913         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
914         if options.tail:
915             blist.start_tail()
916         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
917         if status != 0 or errstr != "retry":
918             break
919         cleanup()
920     except Exception:
921         cleanup()
922         raise
923
924 cleanup_list.append(gitroot + "/autobuild.pid")
925
926 do_print(errstr)
927
928 blist.kill_kids()
929 if options.tail:
930     do_print("waiting for tail to flush")
931     time.sleep(1)
932
933 elapsed_time = time.time() - start_time
934 if status == 0:
935     if options.passcmd is not None:
936         do_print("Running passcmd: %s" % options.passcmd)
937         run_cmd(options.passcmd, dir=test_master)
938     if options.pushto is not None:
939         push_to(options.pushto, push_branch=options.branch)
940     if options.keeplogs or options.attach_logs:
941         blist.tarlogs("logs.tar.gz")
942         do_print("Logs in logs.tar.gz")
943     if options.always_email:
944         email_success(elapsed_time, log_base=options.log_base)
945     blist.remove_logs()
946     cleanup()
947     do_print(errstr)
948     sys.exit(0)
949
950 # something failed, gather a tar of the logs
951 blist.tarlogs("logs.tar.gz")
952
953 if options.email is not None:
954     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
955                   elapsed_time, log_base=options.log_base)
956 else:
957     elapsed_minutes = elapsed_time / 60.0
958     print('''
959
960 ####################################################################
961
962 AUTOBUILD FAILURE
963
964 Your autobuild[%s] on %s failed after %.1f minutes
965 when trying to test %s with the following error:
966
967    %s
968
969 the autobuild has been abandoned. Please fix the error and resubmit.
970
971 ####################################################################
972
973 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
974
975 cleanup()
976 do_print(errstr)
977 do_print("Logs in logs.tar.gz")
978 sys.exit(status)