2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # Copyright Jelmer Vernooij 2010
5 # released under GNU GPL v3 or later
8 from subprocess import call, check_call, Popen, PIPE
9 import os, tarfile, sys, time
10 from optparse import OptionParser
12 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../selftest"))
13 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools"))
14 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python"))
18 from email.mime.text import MIMEText
20 samba_master = os.getenv('SAMBA_MASTER', 'git://git.samba.org/samba.git')
21 samba_master_ssh = os.getenv('SAMBA_MASTER_SSH', 'git+ssh://git.samba.org/data/git/samba.git')
25 os.putenv('CC', "ccache gcc")
28 "source3" : [ ("autogen", "./autogen.sh", "text/plain"),
29 ("configure", "./configure.developer ${PREFIX}", "text/plain"),
30 ("make basics", "make basics", "text/plain"),
31 ("make", "make -j 4 everything", "text/plain"), # don't use too many processes
32 ("install", "make install", "text/plain"),
33 ("test", "TDB_NO_FSYNC=1 make subunit-test", "text/x-subunit") ],
35 "source4" : [ ("configure", "./configure.developer ${PREFIX}", "text/plain"),
36 ("make", "make -j", "text/plain"),
37 ("install", "make install", "text/plain"),
38 ("test", "TDB_NO_FSYNC=1 make subunit-test", "text/x-subunit") ],
40 "source4/lib/ldb" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
41 ("make", "make -j", "text/plain"),
42 ("install", "make install", "text/plain"),
43 ("test", "make test", "text/plain") ],
45 "lib/tdb" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
46 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
47 ("make", "make -j", "text/plain"),
48 ("install", "make install", "text/plain"),
49 ("test", "make test", "text/plain") ],
51 "lib/talloc" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
52 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
53 ("make", "make -j", "text/plain"),
54 ("install", "make install", "text/plain"),
55 ("test", "make test", "text/x-subunit"), ],
57 "lib/replace" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
58 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
59 ("make", "make -j", "text/plain"),
60 ("install", "make install", "text/plain"),
61 ("test", "make test", "text/plain"), ],
63 "lib/tevent" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
64 ("make", "make -j", "text/plain"),
65 ("install", "make install", "text/plain"),
66 ("test", "make test", "text/plain"), ],
69 retry_task = [ ( "retry",
71 git remote add -t master master %s
75 git describe master/master > old_master.desc
77 git describe master/master > master.desc
78 diff old_master.desc master.desc
80 ''' % samba_master, "test/plain" ) ]
83 def run_cmd(cmd, dir=None, show=None, output=False, checkfail=True, shell=False):
85 show = options.verbose
87 print("Running: '%s' in '%s'" % (cmd, dir))
89 return Popen(cmd, stdout=PIPE, cwd=dir, shell=shell).communicate()[0]
91 return check_call(cmd, cwd=dir, shell=shell)
93 return call(cmd, cwd=dir, shell=shell)
96 class TreeStageBuilder(object):
97 """Handle building of a particular stage for a tree.
100 def __init__(self, tree, name, command, fail_quickly=False):
103 self.command = command
104 self.fail_quickly = fail_quickly
106 self.stdin = open(os.devnull, 'r')
109 raise NotImplementedError(self.start)
112 self.status = self.proc.poll()
116 if self.proc is not None:
118 run_cmd(["killbysubdir", self.tree.sdir], checkfail=False)
120 # killbysubdir doesn't exist ?
122 self.proc.terminate()
127 def failure_reason(self):
128 return "failed '%s' with status %d" % (self.cmd, self.status)
132 return (os.WIFSIGNALED(self.status) or os.WEXITSTATUS(self.status) != 0)
135 class PlainTreeStageBuilder(TreeStageBuilder):
138 print '%s: [%s] Running %s' % (self.name, self.name, self.command)
139 self.proc = Popen(self.command, shell=True, cwd=self.tree.dir,
140 stdout=self.tree.stdout, stderr=self.tree.stderr,
144 class AbortingTestResult(subunithelper.TestsuiteEnabledTestResult):
146 def __init__(self, stage):
147 super(AbortingTestResult, self).__init__()
150 def addError(self, test, details=None):
151 self.stage.proc.terminate()
153 def addFailure(self, test, details=None):
154 self.stage.proc.terminate()
157 class SubunitTreeStageBuilder(TreeStageBuilder):
159 def __init__(self, tree, name, command, fail_quickly=False):
160 super(SubunitTreeStageBuilder, self).__init__(tree, name, command,
162 self.failed_tests = []
163 self.subunit_path = os.path.join(gitroot,
164 "%s.%s.subunit" % (self.tree.tag, self.name))
165 self.tree.logfiles.append(
166 (self.subunit_path, os.path.basename(self.subunit_path)))
167 self.subunit = open(self.subunit_path, 'w')
169 formatter = subunithelper.PlainFormatter(False, True, {})
170 clients = [formatter, subunit.TestProtocolClient(self.subunit)]
172 clients.append(AbortingTestResult(self))
173 self.subunit_server = subunit.TestProtocolServer(
174 testtools.MultiTestResult(*clients),
179 print '%s: [%s] Running' % (self.tree.name, self.name)
180 self.proc = Popen(self.command, shell=True, cwd=self.tree.dir,
181 stdout=PIPE, stderr=self.tree.stderr, stdin=self.stdin)
182 fd = self.proc.stdout.fileno()
183 fl = fcntl.fcntl(fd, fcntl.F_GETFL)
184 fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
188 data = self.proc.stdout.read()
192 self.tree.stdout.write(data)
193 self.buffered += data
195 for l in self.buffered.splitlines(True):
197 self.subunit_server.lineReceived(l)
200 self.buffered = buffered
201 self.status = self.proc.poll()
202 if self.status is not None:
207 class TreeBuilder(object):
208 '''handle build of one directory'''
210 def __init__(self, name, sequence, fail_quickly=False):
212 self.fail_quickly = fail_quickly
214 self.tag = self.name.replace('/', '_')
215 self.sequence = sequence
217 self.stdout_path = os.path.join(gitroot, "%s.stdout" % (self.tag, ))
218 self.stderr_path = os.path.join(gitroot, "%s.stderr" % (self.tag, ))
219 self.logfiles = [(self.stdout_path, os.path.basename(self.stdout_path)),
220 (self.stderr_path, os.path.basename(self.stderr_path))]
222 print("stdout for %s in %s" % (self.name, self.stdout_path))
223 print("stderr for %s in %s" % (self.name, self.stderr_path))
224 if os.path.exists(self.stdout_path):
225 os.unlink(self.stdout_path)
226 if os.path.exists(self.stderr_path):
227 os.unlink(self.stderr_path)
228 self.stdout = open(self.stdout_path, 'w')
229 self.stderr = open(self.stderr_path, 'w')
230 self.sdir = os.path.join(testbase, self.tag)
231 if name in ['pass', 'fail', 'retry']:
234 self.dir = os.path.join(self.sdir, self.name)
235 self.prefix = os.path.join(testbase, "prefix", self.tag)
236 run_cmd(["rm", "-rf", self.sdir])
237 cleanup_list.append(self.sdir)
238 cleanup_list.append(self.prefix)
239 os.makedirs(self.sdir)
240 run_cmd(["rm", "-rf", self.sdir])
241 run_cmd(["git", "clone", "--shared", gitroot, self.sdir])
244 def start_next(self):
245 if self.next == len(self.sequence):
246 print '%s: Completed OK' % self.name
251 (stage_name, cmd, output_mime_type) = self.sequence[self.next]
252 cmd = cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
253 if output_mime_type == "text/plain":
254 self.stage = PlainTreeStageBuilder(self, stage_name, cmd,
256 elif output_mime_type == "text/x-subunit":
257 self.stage = SubunitTreeStageBuilder(self, stage_name, cmd,
260 raise Exception("Unknown output mime type %s" % output_mime_type)
264 def remove_logs(self):
265 for path, name in self.logfiles:
270 return self.stage.status
273 return self.stage.poll()
276 if self.stage is not None:
282 if self.stage is None:
284 return self.stage.failed
287 def failure_reason(self):
288 return "%s: [%s] %s" % (self.name, self.stage.name,
289 self.stage.failure_reason)
292 class BuildList(object):
293 '''handle build of multiple directories'''
295 def __init__(self, tasklist, tasknames):
298 self.tail_proc = None
300 if tasknames == ['pass']:
301 tasks = { 'pass' : [ ("pass", '/bin/true', "text/plain") ]}
302 if tasknames == ['fail']:
303 tasks = { 'fail' : [ ("fail", '/bin/false', "text/plain") ]}
307 b = TreeBuilder(n, tasks[n], not options.fail_slowly)
310 self.retry = TreeBuilder('retry', retry_task,
311 not options.fail_slowly)
312 self.need_retry = False
315 if self.tail_proc is not None:
316 self.tail_proc.terminate()
317 self.tail_proc.wait()
318 self.tail_proc = None
319 if self.retry is not None:
320 self.retry.proc.terminate()
321 self.retry.proc.wait()
338 ret = self.retry.proc.poll()
340 self.need_retry = True
350 if options.retry and self.need_retry:
352 print("retry needed")
353 return (0, None, None, None, "retry")
358 return (b.status, b.name, b.stage, b.tag, b.failure_reason)
361 return (0, None, None, None, "All OK")
363 def tarlogs(self, fname):
364 tar = tarfile.open(fname, "w:gz")
366 for (path, name) in b.logfiles:
367 tar.add(path, arcname=name)
368 if os.path.exists("autobuild.log"):
369 tar.add("autobuild.log")
372 def remove_logs(self):
376 def start_tail(self):
377 cmd = "tail -f *.stdout *.stderr"
378 self.tail_proc = Popen(cmd, shell=True, cwd=gitroot)
382 if options.nocleanup:
384 print("Cleaning up ....")
385 for d in cleanup_list:
386 run_cmd(["rm", "-rf", d])
389 def find_git_root(p):
390 '''get to the top of the git repo'''
392 if os.path.isdir(os.path.join(p, ".git")):
394 p = os.path.abspath(os.path.join(p, '..'))
398 def daemonize(logfile):
400 if pid == 0: # Parent
403 if pid != 0: # Actual daemon
408 import resource # Resource usage information.
409 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
410 if maxfd == resource.RLIM_INFINITY:
411 maxfd = 1024 # Rough guess at maximum number of open file descriptors.
412 for fd in range(0, maxfd):
417 os.open(logfile, os.O_RDWR | os.O_CREAT)
422 def rebase_tree(url):
423 print("Rebasing on %s" % url)
424 run_cmd(["git", "remote", "add", "-t", "master", "master", url], show=True, dir=test_master)
425 run_cmd(["git", "fetch", "master"], show=True, dir=test_master)
426 if options.fix_whitespace:
427 run_cmd(["git", "rebase", "--whitespace=fix", "master/master"], show=True, dir=test_master)
429 run_cmd(["git", "rebase", "master/master"], show=True, dir=test_master)
430 diff = run_cmd(["git", "--no-pager", "diff", "HEAD", "master/master"], dir=test_master, output=True)
432 print("No differences between HEAD and master/master - exiting")
436 print("Pushing to %s" % url)
438 run_cmd("EDITOR=script/commit_mark.sh git commit --amend -c HEAD", dir=test_master, shell=True)
439 # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
440 # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
441 run_cmd(["git", "remote", "add", "-t", "master", "pushto", url], show=True, dir=test_master)
442 run_cmd(["git", "push", "pushto", "+HEAD:master"], show=True, dir=test_master)
444 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
446 parser = OptionParser()
447 parser.add_option("", "--repository", help="repository to run tests for", default=None, type=str)
448 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
449 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
450 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
451 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
452 default=def_testbase)
453 parser.add_option("", "--passcmd", help="command to run on success", default=None)
454 parser.add_option("", "--verbose", help="show all commands as they are run",
455 default=False, action="store_true")
456 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
457 default=None, type='str')
458 parser.add_option("", "--rebase-master", help="rebase on %s before testing" % samba_master,
459 default=False, action='store_true')
460 parser.add_option("", "--pushto", help="push to a git url on success",
461 default=None, type='str')
462 parser.add_option("", "--push-master", help="push to %s on success" % samba_master_ssh,
463 default=False, action='store_true')
464 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
465 default=False, action="store_true")
466 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
467 default=False, action="store_true")
468 parser.add_option("", "--retry", help="automatically retry if master changes",
469 default=False, action="store_true")
470 parser.add_option("", "--email", help="send email to the given address on failure",
471 type='str', default=None)
472 parser.add_option("", "--always-email", help="always send email, even on success",
474 parser.add_option("", "--daemon", help="daemonize after initial setup",
476 parser.add_option("", "--fail-slowly", help="continue running tests even after one has already failed",
480 def email_failure(status, failed_task, failed_stage, failed_tag, errstr):
481 '''send an email to options.email about the failure'''
482 user = os.getenv("USER")
486 Your autobuild failed when trying to test %s with the following error:
489 the autobuild has been abandoned. Please fix the error and resubmit.
491 You can see logs of the failed task here:
493 http://git.samba.org/%s/samba-autobuild/%s.stdout
494 http://git.samba.org/%s/samba-autobuild/%s.stderr
496 A summary of the autobuild process is here:
498 http://git.samba.org/%s/samba-autobuild/autobuild.log
500 or you can get full logs of all tasks in this job here:
502 http://git.samba.org/%s/samba-autobuild/logs.tar.gz
504 The top commit for the tree that was built was:
508 ''' % (failed_task, errstr, user, failed_tag, user, failed_tag, user, user, top_commit_msg)
510 msg['Subject'] = 'autobuild failure for task %s during %s' % (failed_task, failed_stage)
511 msg['From'] = 'autobuild@samba.org'
512 msg['To'] = options.email
516 s.sendmail(msg['From'], [msg['To']], msg.as_string())
520 '''send an email to options.email about a successful build'''
521 user = os.getenv("USER")
525 Your autobuild has succeeded.
532 you can get full logs of all tasks in this job here:
534 http://git.samba.org/%s/samba-autobuild/logs.tar.gz
539 The top commit for the tree that was built was:
545 msg['Subject'] = 'autobuild success'
546 msg['From'] = 'autobuild@samba.org'
547 msg['To'] = options.email
551 s.sendmail(msg['From'], [msg['To']], msg.as_string())
555 (options, args) = parser.parse_args()
558 if not options.rebase_master and options.rebase is None:
559 raise Exception('You can only use --retry if you also rebase')
561 testbase = os.path.join(options.testbase, "b%u" % (os.getpid(),))
562 test_master = os.path.join(testbase, "master")
564 if options.repository is not None:
565 repository = options.repository
567 repository = os.getcwd()
569 gitroot = find_git_root(repository)
571 raise Exception("Failed to find git root under %s" % repository)
573 # get the top commit message, for emails
574 top_commit_msg = run_cmd(["git", "log", "-1"], dir=gitroot, output=True)
577 os.makedirs(testbase)
578 except Exception, reason:
579 raise Exception("Unable to create %s : %s" % (testbase, reason))
580 cleanup_list.append(testbase)
583 logfile = os.path.join(testbase, "log")
584 print "Forking into the background, writing progress to %s" % logfile
589 run_cmd(["rm", "-rf", test_master])
590 cleanup_list.append(test_master)
591 run_cmd(["git", "clone", "--shared", gitroot, test_master])
597 if options.rebase is not None:
598 rebase_tree(options.rebase)
599 elif options.rebase_master:
600 rebase_tree(samba_master)
601 blist = BuildList(tasks, args)
604 (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
605 if status != 0 or errstr != "retry":
614 print("waiting for tail to flush")
619 if options.passcmd is not None:
620 print("Running passcmd: %s" % options.passcmd)
621 run_cmd(options.passcmd, dir=test_master, shell=True)
622 if options.pushto is not None:
623 push_to(options.pushto)
624 elif options.push_master:
625 push_to(samba_master_ssh)
627 blist.tarlogs("logs.tar.gz")
628 print("Logs in logs.tar.gz")
629 if options.always_email:
635 # something failed, gather a tar of the logs
636 blist.tarlogs("logs.tar.gz")
638 if options.email is not None:
639 email_failure(status, failed_task, failed_stage, failed_tag, errstr)
643 print("Logs in logs.tar.gz")