e00eebda1c1ed35df60d7985589ae776fc191c9a
[samba.git] / script / autobuild.py
1 #!/usr/bin/env python
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from __future__ import print_function
7 from subprocess import call, check_call,Popen, PIPE
8 import os, tarfile, sys, time
9 from optparse import OptionParser
10 import smtplib
11 import email
12 from email.mime.text import MIMEText
13 from email.mime.base import MIMEBase
14 from email.mime.application import MIMEApplication
15 from email.mime.multipart import MIMEMultipart
16 from distutils.sysconfig import get_python_lib
17 import platform
18
19 os.environ["PYTHONUNBUFFERED"] = "1"
20
21 # This speeds up testing remarkably.
22 os.environ['TDB_NO_FSYNC'] = '1'
23
24 cleanup_list = []
25
26 builddirs = {
27     "ctdb"    : "ctdb",
28     "samba"  : ".",
29     "samba-nt4"  : ".",
30     "samba-fileserver"  : ".",
31     "samba-xc" : ".",
32     "samba-o3" : ".",
33     "samba-ctdb" : ".",
34     "samba-libs"  : ".",
35     "samba-static"  : ".",
36     "samba-test-only"  : ".",
37     "samba-none-env"  : ".",
38     "samba-ad-dc"  : ".",
39     "samba-ad-dc-2"  : ".",
40     "samba-systemkrb5"  : ".",
41     "samba-nopython"  : ".",
42     "ldb"     : "lib/ldb",
43     "tdb"     : "lib/tdb",
44     "talloc"  : "lib/talloc",
45     "replace" : "lib/replace",
46     "tevent"  : "lib/tevent",
47     "pidl"    : "pidl",
48     "pass"    : ".",
49     "fail"    : ".",
50     "retry"   : "."
51     }
52
53 defaulttasks = [ "ctdb",
54                  "samba",
55                  "samba-nt4",
56                  "samba-fileserver",
57                  "samba-xc",
58                  "samba-o3",
59                  "samba-ctdb",
60                  "samba-libs",
61                  "samba-static",
62                  "samba-none-env",
63                  "samba-ad-dc",
64                  "samba-ad-dc-2",
65                  "samba-systemkrb5",
66                  "samba-nopython",
67                  "ldb",
68                  "tdb",
69                  "talloc",
70                  "replace",
71                  "tevent",
72                  "pidl" ]
73
74 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
75     defaulttasks.remove("samba-o3")
76
77 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
78 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
79
80 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
81 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
82 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
83 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX} ${EXTRA_PYTHON}"
84 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE"
85 samba_libs_configure_samba = samba_libs_configure_base + " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
86
87 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
88     extra_python = ""
89 else:
90     extra_python = "--extra-python=/usr/bin/python3"
91
92 tasks = {
93     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
94                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
95                ("make", "make all", "text/plain"),
96                ("install", "make install", "text/plain"),
97                ("test", "make autotest", "text/plain"),
98                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
99                ("clean", "make clean", "text/plain") ],
100
101     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
102     "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
103                 ("make", "make -j", "text/plain"),
104                 ("test", "make test FAIL_IMMEDIATELY=1 "
105                  "TESTS='--exclude-env=none "
106                  "--exclude-env=nt4_dc "
107                  "--exclude-env=nt4_member "
108                  "--exclude-env=ad_dc "
109                  "--exclude-env=fl2003dc "
110                  "--exclude-env=fl2008r2dc "
111                  "--exclude-env=ad_member "
112                  "--exclude-env=ad_member_idmap_rid "
113                  "--exclude-env=ad_member_idmap_ad "
114                  "--exclude-env=chgdcpass "
115                  "--exclude-env=vampire_2000_dc "
116                  "--exclude-env=fl2000dc "
117                  "--exclude-env=fileserver'",
118                  "text/plain"),
119                 ("install", "make install", "text/plain"),
120                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
121                 ("clean", "make clean", "text/plain") ],
122
123     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
124     "samba-nt4" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
125                        ("make", "make -j", "text/plain"),
126                        ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
127                        ("install", "make install", "text/plain"),
128                        ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
129                        ("clean", "make clean", "text/plain") ],
130
131     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
132     "samba-fileserver" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
133                       ("configure", "./configure.developer --without-ad-dc --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
134                       ("make", "make -j", "text/plain"),
135                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
136                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
137
138     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
139     "samba-ad-dc" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
140                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
141                       ("make", "make -j", "text/plain"),
142                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='"
143                        "--include-env=ad_dc "
144                        "--include-env=fl2003dc "
145                        "--include-env=fl2008r2dc "
146                        "--include-env=ad_member "
147                        "--include-env=ad_member_idmap_rid "
148                        "--include-env=ad_member_idmap_ad'", "text/plain"),
149                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
150
151     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
152     "samba-ad-dc-2" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
153                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
154                       ("make", "make -j", "text/plain"),
155                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
156                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
157
158     "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
159                           ("make", "make -j", "text/plain"),
160                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
161
162     # Test cross-compile infrastructure
163     "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
164                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
165                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
166                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
167                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
168                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
169
170     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
171     "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
172                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
173                    ("make", "make -j", "text/plain"),
174                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
175                    ("install", "make install", "text/plain"),
176                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
177                    ("clean", "make clean", "text/plain") ],
178
179     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
180
181                      # make sure we have tdb around:
182                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
183                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
184                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
185
186
187                      # build samba with cluster support (also building ctdb):
188                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
189                      ("samba-make", "make", "text/plain"),
190                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
191                      ("samba-install", "make install", "text/plain"),
192                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
193
194                      # clean up:
195                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
196                      ("clean", "make clean", "text/plain"),
197                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
198
199     "samba-libs" : [
200                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
201                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
202                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
203                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
204
205                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
206                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
207                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
208
209                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
210                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
211                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
212
213                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
214                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
215                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
216
217                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
218                       ("nondevel-make", "make -j", "text/plain"),
219                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
220                       ("nondevel-install", "make install", "text/plain"),
221                       ("nondevel-dist", "make dist", "text/plain"),
222
223                       # retry with all modules shared
224                       ("allshared-distclean", "make distclean", "text/plain"),
225                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
226                       ("allshared-make", "make -j", "text/plain")],
227
228     "samba-none-env" : [
229                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
230                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
231                       ("make", "make -j", "text/plain"),
232                       ("test", "make test "
233                        "FAIL_IMMEDIATELY=1 "
234                        "TESTS='--include-env=none'",
235                        "text/plain")],
236
237     "samba-static" : [
238                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
239                       # build with all modules static
240                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
241                       ("allstatic-make", "make -j", "text/plain"),
242
243                       # retry without any required modules
244                       ("none-distclean", "make distclean", "text/plain"),
245                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
246                       ("none-make", "make -j", "text/plain"),
247
248                       # retry with nonshared smbd and smbtorture
249                       ("nonshared-distclean", "make distclean", "text/plain"),
250                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
251                       ("nonshared-make", "make -j", "text/plain")],
252
253     "samba-systemkrb5" : [
254                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
255                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
256                       ("make", "make -j", "text/plain"),
257                       # we currently cannot run a full make test, a limited list of tests could be run
258                       # via "make test TESTS=sometests"
259                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
260                       ("install", "make install", "text/plain"),
261                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
262                       ("clean", "make clean", "text/plain")
263                       ],
264
265     # Test Samba without python still builds.  When this test fails
266     # due to more use of Python, the expectations is that the newly
267     # failing part of the code should be disabled when
268     # --disable-python is set (rather than major work being done to
269     # support this environment).  The target here is for vendors
270     # shipping a minimal smbd.
271     "samba-nopython" : [
272                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
273                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
274                       ("make", "make -j", "text/plain"),
275                       ("install", "make install", "text/plain"),
276                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
277                       ("clean", "make clean", "text/plain")
278                       ],
279
280
281
282     "ldb" : [
283               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
284               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
285               ("make", "make", "text/plain"),
286               ("install", "make install", "text/plain"),
287               ("test", "make test", "text/plain"),
288               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
289               ("distcheck", "make distcheck", "text/plain"),
290               ("clean", "make clean", "text/plain") ],
291
292     "tdb" : [
293               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
294               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
295               ("make", "make", "text/plain"),
296               ("install", "make install", "text/plain"),
297               ("test", "make test", "text/plain"),
298               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
299               ("distcheck", "make distcheck", "text/plain"),
300               ("clean", "make clean", "text/plain") ],
301
302     "talloc" : [
303                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
304                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
305                  ("make", "make", "text/plain"),
306                  ("install", "make install", "text/plain"),
307                  ("test", "make test", "text/plain"),
308                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
309                  ("distcheck", "make distcheck", "text/plain"),
310                  ("clean", "make clean", "text/plain") ],
311
312     "replace" : [
313                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
314                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
315                   ("make", "make", "text/plain"),
316                   ("install", "make install", "text/plain"),
317                   ("test", "make test", "text/plain"),
318                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
319                   ("distcheck", "make distcheck", "text/plain"),
320                   ("clean", "make clean", "text/plain") ],
321
322     "tevent" : [
323                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
324                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
325                  ("make", "make", "text/plain"),
326                  ("install", "make install", "text/plain"),
327                  ("test", "make test", "text/plain"),
328                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
329                  ("distcheck", "make distcheck", "text/plain"),
330                  ("clean", "make clean", "text/plain") ],
331
332     "pidl" : [
333                ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
334                ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
335                ("touch", "touch *.yp", "text/plain"),
336                ("make", "make", "text/plain"),
337                ("test", "make test", "text/plain"),
338                ("install", "make install", "text/plain"),
339                ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
340                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
341                ("clean", "make clean", "text/plain") ],
342
343     # these are useful for debugging autobuild
344     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
345     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
346 }
347
348 def do_print(msg):
349     print("%s" % msg)
350     sys.stdout.flush()
351     sys.stderr.flush()
352
353 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
354     if show is None:
355         show = options.verbose
356     if show:
357         do_print("Running: '%s' in '%s'" % (cmd, dir))
358     if output:
359         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
360     elif checkfail:
361         return check_call(cmd, shell=True, cwd=dir)
362     else:
363         return call(cmd, shell=True, cwd=dir)
364
365
366 class builder(object):
367     '''handle build of one directory'''
368
369     def __init__(self, name, sequence, cp=True):
370         self.name = name
371         self.dir = builddirs[name]
372
373         self.tag = self.name.replace('/', '_')
374         self.sequence = sequence
375         self.next = 0
376         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
377         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
378         if options.verbose:
379             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
380             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
381         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
382         self.stdout = open(self.stdout_path, 'w')
383         self.stderr = open(self.stderr_path, 'w')
384         self.stdin  = open("/dev/null", 'r')
385         self.sdir = "%s/%s" % (testbase, self.tag)
386         self.prefix = "%s/%s" % (test_prefix, self.tag)
387         run_cmd("rm -rf %s" % self.sdir)
388         run_cmd("rm -rf %s" % self.prefix)
389         if cp:
390             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
391         else:
392             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
393         self.start_next()
394
395     def start_next(self):
396         if self.next == len(self.sequence):
397             if not options.nocleanup:
398                 run_cmd("rm -rf %s" % self.sdir)
399                 run_cmd("rm -rf %s" % self.prefix)
400             do_print('%s: Completed OK' % self.name)
401             self.done = True
402             return
403         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
404         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
405         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
406         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
407         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
408         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
409 #        if self.output_mime_type == "text/x-subunit":
410 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
411         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
412         cwd = os.getcwd()
413         os.chdir("%s/%s" % (self.sdir, self.dir))
414         self.proc = Popen(self.cmd, shell=True,
415                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
416         os.chdir(cwd)
417         self.next += 1
418
419
420 class buildlist(object):
421     '''handle build of multiple directories'''
422
423     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
424         global tasks
425         self.tlist = []
426         self.tail_proc = None
427         self.retry = None
428         if tasknames == []:
429             if options.restrict_tests:
430                 tasknames = ["samba-test-only"]
431             else:
432                 tasknames = defaulttasks
433         else:
434             # If we are only running one test,
435             # do not sleep randomly to wait for it to start
436             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
437
438         for n in tasknames:
439             b = builder(n, tasks[n], cp=n is not "pidl")
440             self.tlist.append(b)
441         if options.retry:
442             rebase_remote = "rebaseon"
443             retry_task = [ ("retry",
444                             '''set -e
445                             git remote add -t %s %s %s
446                             git fetch %s
447                             while :; do
448                               sleep 60
449                               git describe %s/%s > old_remote_branch.desc
450                               git fetch %s
451                               git describe %s/%s > remote_branch.desc
452                               diff old_remote_branch.desc remote_branch.desc
453                             done
454                            ''' % (
455                                rebase_branch, rebase_remote, rebase_url,
456                                rebase_remote,
457                                rebase_remote, rebase_branch,
458                                rebase_remote,
459                                rebase_remote, rebase_branch
460                            ),
461                            "test/plain" ) ]
462
463             self.retry = builder('retry', retry_task, cp=False)
464             self.need_retry = False
465
466     def kill_kids(self):
467         if self.tail_proc is not None:
468             self.tail_proc.terminate()
469             self.tail_proc.wait()
470             self.tail_proc = None
471         if self.retry is not None:
472             self.retry.proc.terminate()
473             self.retry.proc.wait()
474             self.retry = None
475         for b in self.tlist:
476             if b.proc is not None:
477                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
478                 b.proc.terminate()
479                 b.proc.wait()
480                 b.proc = None
481
482     def wait_one(self):
483         while True:
484             none_running = True
485             for b in self.tlist:
486                 if b.proc is None:
487                     continue
488                 none_running = False
489                 b.status = b.proc.poll()
490                 if b.status is None:
491                     continue
492                 b.proc = None
493                 return b
494             if options.retry:
495                 ret = self.retry.proc.poll()
496                 if ret is not None:
497                     self.need_retry = True
498                     self.retry = None
499                     return None
500             if none_running:
501                 return None
502             time.sleep(0.1)
503
504     def run(self):
505         while True:
506             b = self.wait_one()
507             if options.retry and self.need_retry:
508                 self.kill_kids()
509                 do_print("retry needed")
510                 return (0, None, None, None, "retry")
511             if b is None:
512                 break
513             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
514                 self.kill_kids()
515                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
516             b.start_next()
517         self.kill_kids()
518         return (0, None, None, None, "All OK")
519
520     def write_system_info(self):
521         filename = 'system-info.txt'
522         f = open(filename, 'w')
523         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo']:
524             print('### %s' % cmd, file=f)
525             print(run_cmd(cmd, output=True, checkfail=False), file=f)
526             print(file=f)
527         f.close()
528         return filename
529
530     def tarlogs(self, fname):
531         tar = tarfile.open(fname, "w:gz")
532         for b in self.tlist:
533             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
534             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
535         if os.path.exists("autobuild.log"):
536             tar.add("autobuild.log")
537         sys_info = self.write_system_info()
538         tar.add(sys_info)
539         tar.close()
540
541     def remove_logs(self):
542         for b in self.tlist:
543             os.unlink(b.stdout_path)
544             os.unlink(b.stderr_path)
545
546     def start_tail(self):
547         cwd = os.getcwd()
548         cmd = "tail -f *.stdout *.stderr"
549         os.chdir(gitroot)
550         self.tail_proc = Popen(cmd, shell=True)
551         os.chdir(cwd)
552
553
554 def cleanup():
555     if options.nocleanup:
556         return
557     run_cmd("stat %s || true" % test_tmpdir, show=True)
558     run_cmd("stat %s" % testbase, show=True)
559     do_print("Cleaning up ....")
560     for d in cleanup_list:
561         run_cmd("rm -rf %s" % d)
562
563
564 def find_git_root():
565     '''get to the top of the git repo'''
566     p=os.getcwd()
567     while p != '/':
568         if os.path.isdir(os.path.join(p, ".git")):
569             return p
570         p = os.path.abspath(os.path.join(p, '..'))
571     return None
572
573
574 def daemonize(logfile):
575     pid = os.fork()
576     if pid == 0: # Parent
577         os.setsid()
578         pid = os.fork()
579         if pid != 0: # Actual daemon
580             os._exit(0)
581     else: # Grandparent
582         os._exit(0)
583
584     import resource      # Resource usage information.
585     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
586     if maxfd == resource.RLIM_INFINITY:
587         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
588     for fd in range(0, maxfd):
589         try:
590             os.close(fd)
591         except OSError:
592             pass
593     os.open(logfile, os.O_RDWR | os.O_CREAT)
594     os.dup2(0, 1)
595     os.dup2(0, 2)
596
597 def write_pidfile(fname):
598     '''write a pid file, cleanup on exit'''
599     f = open(fname, mode='w')
600     f.write("%u\n" % os.getpid())
601     f.close()
602
603
604 def rebase_tree(rebase_url, rebase_branch = "master"):
605     rebase_remote = "rebaseon"
606     do_print("Rebasing on %s" % rebase_url)
607     run_cmd("git describe HEAD", show=True, dir=test_master)
608     run_cmd("git remote add -t %s %s %s" %
609             (rebase_branch, rebase_remote, rebase_url),
610             show=True, dir=test_master)
611     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
612     if options.fix_whitespace:
613         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
614                 (rebase_remote, rebase_branch),
615                 show=True, dir=test_master)
616     else:
617         run_cmd("git rebase --force-rebase %s/%s" %
618                 (rebase_remote, rebase_branch),
619                 show=True, dir=test_master)
620     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
621                    (rebase_remote, rebase_branch),
622                    dir=test_master, output=True)
623     if diff == '':
624         do_print("No differences between HEAD and %s/%s - exiting" %
625               (rebase_remote, rebase_branch))
626         sys.exit(0)
627     run_cmd("git describe %s/%s" %
628             (rebase_remote, rebase_branch),
629             show=True, dir=test_master)
630     run_cmd("git describe HEAD", show=True, dir=test_master)
631     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
632             (rebase_remote, rebase_branch),
633             show=True, dir=test_master)
634
635 def push_to(push_url, push_branch = "master"):
636     push_remote = "pushto"
637     do_print("Pushing to %s" % push_url)
638     if options.mark:
639         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
640         run_cmd("git commit --amend -c HEAD", dir=test_master)
641         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
642         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
643     run_cmd("git remote add -t %s %s %s" %
644             (push_branch, push_remote, push_url),
645             show=True, dir=test_master)
646     run_cmd("git push %s +HEAD:%s" %
647             (push_remote, push_branch),
648             show=True, dir=test_master)
649
650 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
651
652 gitroot = find_git_root()
653 if gitroot is None:
654     raise Exception("Failed to find git root")
655
656 parser = OptionParser()
657 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
658 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
659 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
660 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
661                   default=def_testbase)
662 parser.add_option("", "--passcmd", help="command to run on success", default=None)
663 parser.add_option("", "--verbose", help="show all commands as they are run",
664                   default=False, action="store_true")
665 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
666                   default=None, type='str')
667 parser.add_option("", "--pushto", help="push to a git url on success",
668                   default=None, type='str')
669 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
670                   default=False, action="store_true")
671 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
672                   default=False, action="store_true")
673 parser.add_option("", "--retry", help="automatically retry if master changes",
674                   default=False, action="store_true")
675 parser.add_option("", "--email", help="send email to the given address on failure",
676                   type='str', default=None)
677 parser.add_option("", "--email-from", help="send email from the given address",
678                   type='str', default="autobuild@samba.org")
679 parser.add_option("", "--email-server", help="send email via the given server",
680                   type='str', default='localhost')
681 parser.add_option("", "--always-email", help="always send email, even on success",
682                   action="store_true")
683 parser.add_option("", "--daemon", help="daemonize after initial setup",
684                   action="store_true")
685 parser.add_option("", "--branch", help="the branch to work on (default=master)",
686                   default="master", type='str')
687 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
688                   default=gitroot, type='str')
689 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
690                   default=False, action="store_true")
691 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
692                   default='')
693
694 def send_email(subject, text, log_tar):
695     outer = MIMEMultipart()
696     outer['Subject'] = subject
697     outer['To'] = options.email
698     outer['From'] = options.email_from
699     outer['Date'] = email.utils.formatdate(localtime = True)
700     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
701     outer.attach(MIMEText(text, 'plain'))
702     if options.attach_logs:
703         fp = open(log_tar, 'rb')
704         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
705         fp.close()
706         # Set the filename parameter
707         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
708         outer.attach(msg)
709     content = outer.as_string()
710     s = smtplib.SMTP(options.email_server)
711     s.sendmail(options.email_from, [options.email], content)
712     s.set_debuglevel(1)
713     s.quit()
714
715 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
716                   elapsed_time, log_base=None, add_log_tail=True):
717     '''send an email to options.email about the failure'''
718     elapsed_minutes = elapsed_time / 60.0
719     user = os.getenv("USER")
720     if log_base is None:
721         log_base = gitroot
722     text = '''
723 Dear Developer,
724
725 Your autobuild on %s failed after %.1f minutes
726 when trying to test %s with the following error:
727
728    %s
729
730 the autobuild has been abandoned. Please fix the error and resubmit.
731
732 A summary of the autobuild process is here:
733
734   %s/autobuild.log
735 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
736
737     if options.restrict_tests:
738         text += """
739 The build was restricted to tests matching %s\n""" % options.restrict_tests
740
741     if failed_task != 'rebase':
742         text += '''
743 You can see logs of the failed task here:
744
745   %s/%s.stdout
746   %s/%s.stderr
747
748 or you can get full logs of all tasks in this job here:
749
750   %s/logs.tar.gz
751
752 The top commit for the tree that was built was:
753
754 %s
755
756 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
757
758     if add_log_tail:
759         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
760         lines = f.readlines()
761         log_tail = "".join(lines[-50:])
762         num_lines = len(lines)
763         if num_lines < 50:
764             # Also include stderr (compile failures) if < 50 lines of stdout
765             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
766             log_tail += "".join(f.readlines()[-(50-num_lines):])
767
768         text += '''
769 The last 50 lines of log messages:
770
771 %s
772     ''' % log_tail
773         f.close()
774
775     logs = os.path.join(gitroot, 'logs.tar.gz')
776     send_email('autobuild[%s] failure on %s for task %s during %s'
777                % (options.branch, platform.node(), failed_task, failed_stage),
778                text, logs)
779
780 def email_success(elapsed_time, log_base=None):
781     '''send an email to options.email about a successful build'''
782     user = os.getenv("USER")
783     if log_base is None:
784         log_base = gitroot
785     text = '''
786 Dear Developer,
787
788 Your autobuild on %s has succeeded after %.1f minutes.
789
790 ''' % (platform.node(), elapsed_time / 60.)
791
792     if options.restrict_tests:
793         text += """
794 The build was restricted to tests matching %s\n""" % options.restrict_tests
795
796     if options.keeplogs:
797         text += '''
798
799 you can get full logs of all tasks in this job here:
800
801   %s/logs.tar.gz
802
803 ''' % log_base
804
805     text += '''
806 The top commit for the tree that was built was:
807
808 %s
809 ''' % top_commit_msg
810
811     logs = os.path.join(gitroot, 'logs.tar.gz')
812     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
813                text, logs)
814
815
816 (options, args) = parser.parse_args()
817
818 if options.retry:
819     if options.rebase is None:
820         raise Exception('You can only use --retry if you also rebase')
821
822 testbase = "%s/b%u" % (options.testbase, os.getpid())
823 test_master = "%s/master" % testbase
824 test_prefix = "%s/prefix" % testbase
825 test_tmpdir = "%s/tmp" % testbase
826 os.environ['TMPDIR'] = test_tmpdir
827
828 # get the top commit message, for emails
829 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
830
831 try:
832     os.makedirs(testbase)
833 except Exception as reason:
834     raise Exception("Unable to create %s : %s" % (testbase, reason))
835 cleanup_list.append(testbase)
836
837 if options.daemon:
838     logfile = os.path.join(testbase, "log")
839     do_print("Forking into the background, writing progress to %s" % logfile)
840     daemonize(logfile)
841
842 write_pidfile(gitroot + "/autobuild.pid")
843
844 start_time = time.time()
845
846 while True:
847     try:
848         run_cmd("rm -rf %s" % test_tmpdir, show=True)
849         os.makedirs(test_tmpdir)
850         # The waf uninstall code removes empty directories all the way
851         # up the tree.  Creating a file in test_tmpdir stops it from
852         # being removed.
853         run_cmd("touch %s" % os.path.join(test_tmpdir,
854                                           ".directory-is-not-empty"), show=True)
855         run_cmd("stat %s" % test_tmpdir, show=True)
856         run_cmd("stat %s" % testbase, show=True)
857         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
858     except Exception:
859         cleanup()
860         raise
861
862     try:
863         try:
864             if options.rebase is not None:
865                 rebase_tree(options.rebase, rebase_branch=options.branch)
866         except Exception:
867             cleanup_list.append(gitroot + "/autobuild.pid")
868             cleanup()
869             elapsed_time = time.time() - start_time
870             email_failure(-1, 'rebase', 'rebase', 'rebase',
871                           'rebase on %s failed' % options.branch,
872                           elapsed_time, log_base=options.log_base)
873             sys.exit(1)
874         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
875         if options.tail:
876             blist.start_tail()
877         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
878         if status != 0 or errstr != "retry":
879             break
880         cleanup()
881     except Exception:
882         cleanup()
883         raise
884
885 cleanup_list.append(gitroot + "/autobuild.pid")
886
887 do_print(errstr)
888
889 blist.kill_kids()
890 if options.tail:
891     do_print("waiting for tail to flush")
892     time.sleep(1)
893
894 elapsed_time = time.time() - start_time
895 if status == 0:
896     if options.passcmd is not None:
897         do_print("Running passcmd: %s" % options.passcmd)
898         run_cmd(options.passcmd, dir=test_master)
899     if options.pushto is not None:
900         push_to(options.pushto, push_branch=options.branch)
901     if options.keeplogs or options.attach_logs:
902         blist.tarlogs("logs.tar.gz")
903         do_print("Logs in logs.tar.gz")
904     if options.always_email:
905         email_success(elapsed_time, log_base=options.log_base)
906     blist.remove_logs()
907     cleanup()
908     do_print(errstr)
909     sys.exit(0)
910
911 # something failed, gather a tar of the logs
912 blist.tarlogs("logs.tar.gz")
913
914 if options.email is not None:
915     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
916                   elapsed_time, log_base=options.log_base)
917 else:
918     elapsed_minutes = elapsed_time / 60.0
919     print('''
920
921 ####################################################################
922
923 AUTOBUILD FAILURE
924
925 Your autobuild[%s] on %s failed after %.1f minutes
926 when trying to test %s with the following error:
927
928    %s
929
930 the autobuild has been abandoned. Please fix the error and resubmit.
931
932 ####################################################################
933
934 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
935
936 cleanup()
937 do_print(errstr)
938 do_print("Logs in logs.tar.gz")
939 sys.exit(status)