land: Implement --fail-immediately directly in Python, provide subunit file.
[samba.git] / script / land.py
1 #!/usr/bin/env python
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # Copyright Jelmer Vernooij 2010
5 # released under GNU GPL v3 or later
6
7 import fcntl
8 from subprocess import call, check_call, Popen, PIPE
9 import os, tarfile, sys, time
10 from optparse import OptionParser
11 import smtplib
12 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../selftest"))
13 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/testtools"))
14 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/subunit/python"))
15 import subunit
16 import testtools
17 import subunithelper
18 from email.mime.text import MIMEText
19
20 samba_master = os.getenv('SAMBA_MASTER', 'git://git.samba.org/samba.git')
21 samba_master_ssh = os.getenv('SAMBA_MASTER_SSH', 'git+ssh://git.samba.org/data/git/samba.git')
22
23 cleanup_list = []
24
25 os.putenv('CC', "ccache gcc")
26
27 tasks = {
28     "source3" : [ ("autogen", "./autogen.sh", "text/plain"),
29                   ("configure", "./configure.developer ${PREFIX}", "text/plain"),
30                   ("make basics", "make basics", "text/plain"),
31                   ("make", "make -j 4 everything", "text/plain"), # don't use too many processes
32                   ("install", "make install", "text/plain"),
33                   ("test", "TDB_NO_FSYNC=1 make subunit-test", "text/x-subunit") ],
34
35     "source4" : [ ("configure", "./configure.developer ${PREFIX}", "text/plain"),
36                   ("make", "make -j", "text/plain"),
37                   ("install", "make install", "text/plain"),
38                   ("test", "TDB_NO_FSYNC=1 make subunit-test", "text/x-subunit") ],
39
40     "source4/lib/ldb" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
41                           ("make", "make -j", "text/plain"),
42                           ("install", "make install", "text/plain"),
43                           ("test", "make test", "text/plain") ],
44
45     "lib/tdb" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
46                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
47                   ("make", "make -j", "text/plain"),
48                   ("install", "make install", "text/plain"),
49                   ("test", "make test", "text/plain") ],
50
51     "lib/talloc" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
52                      ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
53                      ("make", "make -j", "text/plain"),
54                      ("install", "make install", "text/plain"),
55                      ("test", "make test", "text/x-subunit"), ],
56
57     "lib/replace" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
58                       ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
59                       ("make", "make -j", "text/plain"),
60                       ("install", "make install", "text/plain"),
61                       ("test", "make test", "text/plain"), ],
62
63     "lib/tevent" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
64                      ("make", "make -j", "text/plain"),
65                      ("install", "make install", "text/plain"),
66                      ("test", "make test", "text/plain"), ],
67 }
68
69 retry_task = [ ( "retry",
70                  '''set -e
71                 git remote add -t master master %s
72                 git fetch master
73                 while :; do
74                   sleep 60
75                   git describe master/master > old_master.desc
76                   git fetch master
77                   git describe master/master > master.desc
78                   diff old_master.desc master.desc
79                 done
80                ''' % samba_master, "test/plain" ) ]
81
82
83 def run_cmd(cmd, dir=None, show=None, output=False, checkfail=True, shell=False):
84     if show is None:
85         show = options.verbose
86     if show:
87         print("Running: '%s' in '%s'" % (cmd, dir))
88     if output:
89         return Popen(cmd, stdout=PIPE, cwd=dir, shell=shell).communicate()[0]
90     elif checkfail:
91         return check_call(cmd, cwd=dir, shell=shell)
92     else:
93         return call(cmd, cwd=dir, shell=shell)
94
95
96 class TreeStageBuilder(object):
97     """Handle building of a particular stage for a tree.
98     """
99
100     def __init__(self, tree, name, command, fail_quickly=False):
101         self.tree = tree
102         self.name = name
103         self.command = command
104         self.fail_quickly = fail_quickly
105         self.status = None
106         self.stdin = open(os.devnull, 'r')
107
108     def start(self):
109         raise NotImplementedError(self.start)
110
111     def poll(self):
112         self.status = self.proc.poll()
113         return self.status
114
115     def kill(self):
116         if self.proc is not None:
117             try:
118                 run_cmd(["killbysubdir", self.tree.sdir], checkfail=False)
119             except OSError:
120                 # killbysubdir doesn't exist ?
121                 pass
122             self.proc.terminate()
123             self.proc.wait()
124             self.proc = None
125
126     @property
127     def failure_reason(self):
128         return "failed '%s' with status %d" % (self.cmd, self.status)
129
130     @property
131     def failed(self):
132         return (os.WIFSIGNALED(self.status) or os.WEXITSTATUS(self.status) != 0)
133
134
135 class PlainTreeStageBuilder(TreeStageBuilder):
136
137     def start(self):
138         print '%s: [%s] Running %s' % (self.name, self.name, self.command)
139         self.proc = Popen(self.command, shell=True, cwd=self.tree.dir,
140                           stdout=self.tree.stdout, stderr=self.tree.stderr,
141                           stdin=self.stdin)
142
143
144 class AbortingTestResult(subunithelper.TestsuiteEnabledTestResult):
145
146     def __init__(self, stage):
147         super(AbortingTestResult, self).__init__()
148         self.stage = stage
149
150     def addError(self, test, details=None):
151         self.stage.proc.terminate()
152
153     def addFailure(self, test, details=None):
154         self.stage.proc.terminate()
155
156
157 class SubunitTreeStageBuilder(TreeStageBuilder):
158
159     def __init__(self, tree, name, command, fail_quickly=False):
160         super(SubunitTreeStageBuilder, self).__init__(tree, name, command,
161                 fail_quickly)
162         self.failed_tests = []
163         self.subunit_path = os.path.join(gitroot,
164             "%s.%s.subunit" % (self.tree.tag, self.name))
165         self.tree.logfiles.append(
166             (self.subunit_path, os.path.basename(self.subunit_path)))
167         self.subunit = open(self.subunit_path, 'w')
168
169         formatter = subunithelper.PlainFormatter(False, True, {})
170         clients = [formatter, subunit.TestProtocolClient(self.subunit)]
171         if fail_quickly:
172             clients.append(AbortingTestResult(self))
173         self.subunit_server = subunit.TestProtocolServer(
174             testtools.MultiTestResult(*clients),
175             self.subunit)
176         self.buffered = ""
177
178     def start(self):
179         print '%s: [%s] Running' % (self.tree.name, self.name)
180         self.proc = Popen(self.command, shell=True, cwd=self.tree.dir,
181             stdout=PIPE, stderr=self.tree.stderr, stdin=self.stdin)
182         fd = self.proc.stdout.fileno()
183         fl = fcntl.fcntl(fd, fcntl.F_GETFL)
184         fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
185
186     def poll(self):
187         try:
188             data = self.proc.stdout.read()
189         except IOError:
190             return None
191         else:
192             self.tree.stdout.write(data)
193             self.buffered += data
194             buffered = ""
195             for l in self.buffered.splitlines(True):
196                 if l[-1] == "\n":
197                     self.subunit_server.lineReceived(l)
198                 else:
199                     buffered += l
200             self.buffered = buffered
201             self.status = self.proc.poll()
202             if self.status is not None:
203                 self.subunit.close()
204                 import pdb; pdb.set_trace()
205             return self.status
206
207
208 class TreeBuilder(object):
209     '''handle build of one directory'''
210
211     def __init__(self, name, sequence, fail_quickly=False):
212         self.name = name
213         self.fail_quickly = fail_quickly
214
215         self.tag = self.name.replace('/', '_')
216         self.sequence = sequence
217         self.next = 0
218         self.stdout_path = os.path.join(gitroot, "%s.stdout" % (self.tag, ))
219         self.stderr_path = os.path.join(gitroot, "%s.stderr" % (self.tag, ))
220         self.logfiles = [(self.stdout_path, os.path.basename(self.stdout_path)),
221                          (self.stderr_path, os.path.basename(self.stderr_path))]
222         if options.verbose:
223             print("stdout for %s in %s" % (self.name, self.stdout_path))
224             print("stderr for %s in %s" % (self.name, self.stderr_path))
225         if os.path.exists(self.stdout_path):
226             os.unlink(self.stdout_path)
227         if os.path.exists(self.stderr_path):
228             os.unlink(self.stderr_path)
229         self.stdout = open(self.stdout_path, 'w')
230         self.stderr = open(self.stderr_path, 'w')
231         self.sdir = os.path.join(testbase, self.tag)
232         if name in ['pass', 'fail', 'retry']:
233             self.dir = self.sdir
234         else:
235             self.dir = os.path.join(self.sdir, self.name)
236         self.prefix = os.path.join(testbase, "prefix", self.tag)
237         run_cmd(["rm", "-rf", self.sdir])
238         cleanup_list.append(self.sdir)
239         cleanup_list.append(self.prefix)
240         os.makedirs(self.sdir)
241         run_cmd(["rm",  "-rf", self.sdir])
242         run_cmd(["git", "clone", "--shared", gitroot, self.sdir])
243         self.start_next()
244
245     def start_next(self):
246         if self.next == len(self.sequence):
247             print '%s: Completed OK' % self.name
248             self.done = True
249             self.stdout.close()
250             self.stderr.close()
251             return
252         (stage_name, cmd, output_mime_type) = self.sequence[self.next]
253         cmd = cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
254         if output_mime_type == "text/plain":
255             self.stage = PlainTreeStageBuilder(self, stage_name, cmd,
256                 self.fail_quickly)
257         elif output_mime_type == "text/x-subunit":
258             self.stage = SubunitTreeStageBuilder(self, stage_name, cmd,
259                 self.fail_quickly)
260         else:
261             raise Exception("Unknown output mime type %s" % output_mime_type)
262         self.stage.start()
263         self.next += 1
264
265     def remove_logs(self):
266         for path, name in self.logfiles:
267             os.unlink(path)
268
269     @property
270     def status(self):
271         return self.stage.status
272
273     def poll(self):
274         return self.stage.poll()
275
276     def kill(self):
277         if self.stage is not None:
278             self.stage.kill()
279             self.stage = None
280
281     @property
282     def failed(self):
283         if self.stage is None:
284             return False
285         return self.stage.failed
286
287     @property
288     def failure_reason(self):
289         return "%s: [%s] %s" % (self.name, self.stage.name,
290             self.stage.failure_reason)
291
292
293 class BuildList(object):
294     '''handle build of multiple directories'''
295
296     def __init__(self, tasklist, tasknames):
297         global tasks
298         self.tlist = []
299         self.tail_proc = None
300         self.retry = None
301         if tasknames == ['pass']:
302             tasks = { 'pass' : [ ("pass", '/bin/true', "text/plain") ]}
303         if tasknames == ['fail']:
304             tasks = { 'fail' : [ ("fail", '/bin/false', "text/plain") ]}
305         if tasknames == []:
306             tasknames = tasklist
307         for n in tasknames:
308             b = TreeBuilder(n, tasks[n], not options.fail_slowly)
309             self.tlist.append(b)
310         if options.retry:
311             self.retry = TreeBuilder('retry', retry_task,
312                 not options.fail_slowly)
313             self.need_retry = False
314
315     def kill_kids(self):
316         if self.tail_proc is not None:
317             self.tail_proc.terminate()
318             self.tail_proc.wait()
319             self.tail_proc = None
320         if self.retry is not None:
321             self.retry.proc.terminate()
322             self.retry.proc.wait()
323             self.retry = None
324         for b in self.tlist:
325             b.kill()
326
327     def wait_one(self):
328         while True:
329             none_running = True
330             for b in self.tlist:
331                 if b.stage is None:
332                     continue
333                 none_running = False
334                 if b.poll() is None:
335                     continue
336                 b.stage = None
337                 return b
338             if options.retry:
339                 ret = self.retry.proc.poll()
340                 if ret is not None:
341                     self.need_retry = True
342                     self.retry = None
343                     return None
344             if none_running:
345                 return None
346             time.sleep(0.1)
347
348     def run(self):
349         while True:
350             b = self.wait_one()
351             if options.retry and self.need_retry:
352                 self.kill_kids()
353                 print("retry needed")
354                 return (0, None, None, None, "retry")
355             if b is None:
356                 break
357             if b.failed:
358                 self.kill_kids()
359                 return (b.status, b.name, b.stage, b.tag, b.failure_reason)
360             b.start_next()
361         self.kill_kids()
362         return (0, None, None, None, "All OK")
363
364     def tarlogs(self, fname):
365         tar = tarfile.open(fname, "w:gz")
366         for b in self.tlist:
367             for (path, name) in b.logfiles:
368                 tar.add(path, arcname=name)
369         if os.path.exists("autobuild.log"):
370             tar.add("autobuild.log")
371         tar.close()
372
373     def remove_logs(self):
374         for b in self.tlist:
375             b.remove_logs()
376
377     def start_tail(self):
378         cmd = "tail -f *.stdout *.stderr"
379         self.tail_proc = Popen(cmd, shell=True, cwd=gitroot)
380
381
382 def cleanup():
383     if options.nocleanup:
384         return
385     print("Cleaning up ....")
386     for d in cleanup_list:
387         run_cmd(["rm", "-rf", d])
388
389
390 def find_git_root(p):
391     '''get to the top of the git repo'''
392     while p != '/':
393         if os.path.isdir(os.path.join(p, ".git")):
394             return p
395         p = os.path.abspath(os.path.join(p, '..'))
396     return None
397
398
399 def daemonize(logfile):
400     pid = os.fork()
401     if pid == 0: # Parent
402         os.setsid()
403         pid = os.fork()
404         if pid != 0: # Actual daemon
405             os._exit(0)
406     else: # Grandparent
407         os._exit(0)
408
409     import resource      # Resource usage information.
410     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
411     if maxfd == resource.RLIM_INFINITY:
412         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
413     for fd in range(0, maxfd):
414         try:
415             os.close(fd)
416         except OSError:
417             pass
418     os.open(logfile, os.O_RDWR | os.O_CREAT)
419     os.dup2(0, 1)
420     os.dup2(0, 2)
421
422
423 def rebase_tree(url):
424     print("Rebasing on %s" % url)
425     run_cmd(["git", "remote", "add", "-t", "master", "master", url], show=True, dir=test_master)
426     run_cmd(["git", "fetch", "master"], show=True, dir=test_master)
427     if options.fix_whitespace:
428         run_cmd(["git", "rebase", "--whitespace=fix", "master/master"], show=True, dir=test_master)
429     else:
430         run_cmd(["git", "rebase", "master/master"], show=True, dir=test_master)
431     diff = run_cmd(["git", "--no-pager", "diff", "HEAD", "master/master"], dir=test_master, output=True)
432     if diff == '':
433         print("No differences between HEAD and master/master - exiting")
434         sys.exit(0)
435
436 def push_to(url):
437     print("Pushing to %s" % url)
438     if options.mark:
439         run_cmd("EDITOR=script/commit_mark.sh git commit --amend -c HEAD", dir=test_master, shell=True)
440         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
441         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
442     run_cmd(["git", "remote", "add", "-t", "master", "pushto", url], show=True, dir=test_master)
443     run_cmd(["git", "push", "pushto", "+HEAD:master"], show=True, dir=test_master)
444
445 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
446
447 parser = OptionParser()
448 parser.add_option("", "--repository", help="repository to run tests for", default=None, type=str)
449 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
450 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
451 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
452 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
453                   default=def_testbase)
454 parser.add_option("", "--passcmd", help="command to run on success", default=None)
455 parser.add_option("", "--verbose", help="show all commands as they are run",
456                   default=False, action="store_true")
457 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
458                   default=None, type='str')
459 parser.add_option("", "--rebase-master", help="rebase on %s before testing" % samba_master,
460                   default=False, action='store_true')
461 parser.add_option("", "--pushto", help="push to a git url on success",
462                   default=None, type='str')
463 parser.add_option("", "--push-master", help="push to %s on success" % samba_master_ssh,
464                   default=False, action='store_true')
465 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
466                   default=False, action="store_true")
467 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
468                   default=False, action="store_true")
469 parser.add_option("", "--retry", help="automatically retry if master changes",
470                   default=False, action="store_true")
471 parser.add_option("", "--email", help="send email to the given address on failure",
472                   type='str', default=None)
473 parser.add_option("", "--always-email", help="always send email, even on success",
474                   action="store_true")
475 parser.add_option("", "--daemon", help="daemonize after initial setup",
476                   action="store_true")
477 parser.add_option("", "--fail-slowly", help="continue running tests even after one has already failed",
478                   action="store_true")
479
480
481 def email_failure(status, failed_task, failed_stage, failed_tag, errstr):
482     '''send an email to options.email about the failure'''
483     user = os.getenv("USER")
484     text = '''
485 Dear Developer,
486
487 Your autobuild failed when trying to test %s with the following error:
488    %s
489
490 the autobuild has been abandoned. Please fix the error and resubmit.
491
492 You can see logs of the failed task here:
493
494   http://git.samba.org/%s/samba-autobuild/%s.stdout
495   http://git.samba.org/%s/samba-autobuild/%s.stderr
496
497 A summary of the autobuild process is here:
498
499   http://git.samba.org/%s/samba-autobuild/autobuild.log
500
501 or you can get full logs of all tasks in this job here:
502
503   http://git.samba.org/%s/samba-autobuild/logs.tar.gz
504
505 The top commit for the tree that was built was:
506
507 %s
508
509 ''' % (failed_task, errstr, user, failed_tag, user, failed_tag, user, user, top_commit_msg)
510     msg = MIMEText(text)
511     msg['Subject'] = 'autobuild failure for task %s during %s' % (failed_task, failed_stage)
512     msg['From'] = 'autobuild@samba.org'
513     msg['To'] = options.email
514
515     s = smtplib.SMTP()
516     s.connect()
517     s.sendmail(msg['From'], [msg['To']], msg.as_string())
518     s.quit()
519
520 def email_success():
521     '''send an email to options.email about a successful build'''
522     user = os.getenv("USER")
523     text = '''
524 Dear Developer,
525
526 Your autobuild has succeeded.
527
528 '''
529
530     if options.keeplogs:
531         text += '''
532
533 you can get full logs of all tasks in this job here:
534
535   http://git.samba.org/%s/samba-autobuild/logs.tar.gz
536
537 ''' % user
538
539     text += '''
540 The top commit for the tree that was built was:
541
542 %s
543 ''' % top_commit_msg
544
545     msg = MIMEText(text)
546     msg['Subject'] = 'autobuild success'
547     msg['From'] = 'autobuild@samba.org'
548     msg['To'] = options.email
549
550     s = smtplib.SMTP()
551     s.connect()
552     s.sendmail(msg['From'], [msg['To']], msg.as_string())
553     s.quit()
554
555
556 (options, args) = parser.parse_args()
557
558 if options.retry:
559     if not options.rebase_master and options.rebase is None:
560         raise Exception('You can only use --retry if you also rebase')
561
562 testbase = os.path.join(options.testbase, "b%u" % (os.getpid(),))
563 test_master = os.path.join(testbase, "master")
564
565 if options.repository is not None:
566     repository = options.repository
567 else:
568     repository = os.getcwd()
569
570 gitroot = find_git_root(repository)
571 if gitroot is None:
572     raise Exception("Failed to find git root under %s" % repository)
573
574 # get the top commit message, for emails
575 top_commit_msg = run_cmd(["git", "log", "-1"], dir=gitroot, output=True)
576
577 try:
578     os.makedirs(testbase)
579 except Exception, reason:
580     raise Exception("Unable to create %s : %s" % (testbase, reason))
581 cleanup_list.append(testbase)
582
583 if options.daemon:
584     logfile = os.path.join(testbase, "log")
585     print "Forking into the background, writing progress to %s" % logfile
586     daemonize(logfile)
587
588 while True:
589     try:
590         run_cmd(["rm", "-rf", test_master])
591         cleanup_list.append(test_master)
592         run_cmd(["git", "clone", "--shared", gitroot, test_master])
593     except:
594         cleanup()
595         raise
596
597     try:
598         if options.rebase is not None:
599             rebase_tree(options.rebase)
600         elif options.rebase_master:
601             rebase_tree(samba_master)
602         blist = BuildList(tasks, args)
603         if options.tail:
604             blist.start_tail()
605         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
606         if status != 0 or errstr != "retry":
607             break
608         cleanup()
609     except:
610         cleanup()
611         raise
612
613 blist.kill_kids()
614 if options.tail:
615     print("waiting for tail to flush")
616     time.sleep(1)
617
618 if status == 0:
619     print errstr
620     if options.passcmd is not None:
621         print("Running passcmd: %s" % options.passcmd)
622         run_cmd(options.passcmd, dir=test_master, shell=True)
623     if options.pushto is not None:
624         push_to(options.pushto)
625     elif options.push_master:
626         push_to(samba_master_ssh)
627     if options.keeplogs:
628         blist.tarlogs("logs.tar.gz")
629         print("Logs in logs.tar.gz")
630     if options.always_email:
631         email_success()
632     blist.remove_logs()
633     cleanup()
634     print(errstr)
635 else:
636     # something failed, gather a tar of the logs
637     blist.tarlogs("logs.tar.gz")
638
639     if options.email is not None:
640         email_failure(status, failed_task, failed_stage, failed_tag, errstr)
641
642     cleanup()
643     print(errstr)
644     print("Logs in logs.tar.gz")
645 sys.exit(status)