script/autobuild: don't use make -j for small projects
[sfrench/samba-autobuild/.git] / script / autobuild.py
1 #!/usr/bin/env python
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from subprocess import call, check_call,Popen, PIPE
7 import os, tarfile, sys, time
8 from optparse import OptionParser
9 import smtplib
10 from email.mime.text import MIMEText
11 from distutils.sysconfig import get_python_lib
12
13 samba_master = os.getenv('SAMBA_MASTER', 'git://git.samba.org/samba.git')
14 samba_master_ssh = os.getenv('SAMBA_MASTER_SSH', 'git+ssh://git.samba.org/data/git/samba.git')
15
16 cleanup_list = []
17
18 builddirs = {
19     "samba3"  : "source3",
20     "samba4"  : ".",
21     "samba4-libs"  : ".",
22     "ldb"     : "lib/ldb",
23     "tdb"     : "lib/tdb",
24     "ntdb"    : "lib/ntdb",
25     "talloc"  : "lib/talloc",
26     "replace" : "lib/replace",
27     "tevent"  : "lib/tevent",
28     "pidl"    : "pidl",
29     "pass"    : ".",
30     "fail"    : ".",
31     "retry"   : "."
32     }
33
34 defaulttasks = [ "samba3", "samba4", "samba4-libs", "ldb", "tdb", "ntdb", "talloc", "replace", "tevent", "pidl" ]
35
36 tasks = {
37     "samba3" : [ ("autogen", "./autogen.sh", "text/plain"),
38                  ("configure", "./configure.developer ${PREFIX}", "text/plain"),
39                  ("make basics", "make basics", "text/plain"),
40                  # we split 'make -j 4', 'make bin/smbtorture4' and 'make -j 4 everything'
41                  # because it makes it much easier to find errors.
42                  ("make", "make -j 4", "text/plain"), # don't use too many processes
43                  ("make bin/smbtorture4", "make -j 4 bin/smbtorture4", "text/plain"),
44                  ("make everything", "make -j 4 everything", "text/plain"),
45                  ("install", "make install", "text/plain"),
46                  ("test", "TDB_NO_FSYNC=1 make test FAIL_IMMEDIATELY=1", "text/plain"),
47                  ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
48                  ("clean", "make clean", "text/plain") ],
49
50     # We have 'test' before 'install' because, 'test' should work without 'install'
51     "samba4" : [ ("configure", "./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab", "text/plain"),
52                  ("make", "make -j", "text/plain"),
53                  ("test", "TDB_NO_FSYNC=1 make test FAIL_IMMEDIATELY=1", "text/plain"),
54                  ("install", "make install", "text/plain"),
55                  ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
56                  ("clean", "make clean", "text/plain") ],
57
58     "samba4-libs" : [ ("talloc-configure", "cd lib/talloc && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
59                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
60                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
61
62                       ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
63                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
64                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
65
66                       ("tevent-configure", "cd lib/tevent && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
67                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
68                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
69
70                       ("ldb-configure", "cd lib/ldb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
71                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
72                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
73
74                       ("configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=!talloc,!tdb,!pytdb,!ldb,!pyldb,!tevent,!pytevent --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
75                       ("make", "make", "text/plain"),
76                       ("install", "make install", "text/plain")],
77
78     "ldb" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
79               ("make", "make", "text/plain"),
80               ("install", "make install", "text/plain"),
81               ("test", "TDB_NO_FSYNC=1 make test", "text/plain"),
82               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
83               ("distcheck", "make distcheck", "text/plain"),
84               ("clean", "make clean", "text/plain") ],
85
86     # We don't use TDB_NO_FSYNC=1 here, because we want to test the transaction code
87     "tdb" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
88               ("make", "make", "text/plain"),
89               ("install", "make install", "text/plain"),
90               ("test", "make test", "text/plain"),
91               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
92               ("distcheck", "make distcheck", "text/plain"),
93               ("clean", "make clean", "text/plain") ],
94
95     "ntdb" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
96                ("make", "make", "text/plain"),
97                ("install", "make install", "text/plain"),
98                ("test", "make test", "text/plain"),
99                ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
100                ("distcheck", "make distcheck", "text/plain"),
101                ("clean", "make clean", "text/plain") ],
102
103     "talloc" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
104                  ("make", "make", "text/plain"),
105                  ("install", "make install", "text/plain"),
106                  ("test", "make test", "text/plain"),
107                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
108                  ("distcheck", "make distcheck", "text/plain"),
109                  ("clean", "make clean", "text/plain") ],
110
111     "replace" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
112                   ("make", "make", "text/plain"),
113                   ("install", "make install", "text/plain"),
114                   ("test", "make test", "text/plain"),
115                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
116                   ("distcheck", "make distcheck", "text/plain"),
117                   ("clean", "make clean", "text/plain") ],
118
119     "tevent" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
120                  ("make", "make", "text/plain"),
121                  ("install", "make install", "text/plain"),
122                  ("test", "make test", "text/plain"),
123                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
124                  ("distcheck", "make distcheck", "text/plain"),
125                  ("clean", "make clean", "text/plain") ],
126
127     "pidl" : [ ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
128                ("touch", "touch *.yp", "text/plain"),
129                ("make", "make", "text/plain"),
130                ("test", "make test", "text/plain"),
131                ("install", "make install", "text/plain"),
132                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
133                ("clean", "make clean", "text/plain") ],
134
135     # these are useful for debugging autobuild
136     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
137     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
138 }
139
140 retry_task = [ ( "retry",
141                  '''set -e
142                 git remote add -t master master %s
143                 git fetch master
144                 while :; do
145                   sleep 60
146                   git describe master/master > old_master.desc
147                   git fetch master
148                   git describe master/master > master.desc
149                   diff old_master.desc master.desc
150                 done
151                ''' % samba_master, "test/plain" ) ]
152
153 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
154     if show is None:
155         show = options.verbose
156     if show:
157         print("Running: '%s' in '%s'" % (cmd, dir))
158     if output:
159         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
160     elif checkfail:
161         return check_call(cmd, shell=True, cwd=dir)
162     else:
163         return call(cmd, shell=True, cwd=dir)
164
165
166 class builder(object):
167     '''handle build of one directory'''
168
169     def __init__(self, name, sequence):
170         self.name = name
171         self.dir = builddirs[name]
172
173         self.tag = self.name.replace('/', '_')
174         self.sequence = sequence
175         self.next = 0
176         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
177         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
178         if options.verbose:
179             print("stdout for %s in %s" % (self.name, self.stdout_path))
180             print("stderr for %s in %s" % (self.name, self.stderr_path))
181         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
182         self.stdout = open(self.stdout_path, 'w')
183         self.stderr = open(self.stderr_path, 'w')
184         self.stdin  = open("/dev/null", 'r')
185         self.sdir = "%s/%s" % (testbase, self.tag)
186         self.prefix = "%s/prefix/%s" % (testbase, self.tag)
187         run_cmd("rm -rf %s" % self.sdir)
188         cleanup_list.append(self.sdir)
189         cleanup_list.append(self.prefix)
190         os.makedirs(self.sdir)
191         run_cmd("rm -rf %s" % self.sdir)
192         run_cmd("git clone --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
193         self.start_next()
194
195     def start_next(self):
196         if self.next == len(self.sequence):
197             print '%s: Completed OK' % self.name
198             self.done = True
199             return
200         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
201         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
202         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
203         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
204 #        if self.output_mime_type == "text/x-subunit":
205 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
206         print '%s: [%s] Running %s' % (self.name, self.stage, self.cmd)
207         cwd = os.getcwd()
208         os.chdir("%s/%s" % (self.sdir, self.dir))
209         self.proc = Popen(self.cmd, shell=True,
210                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
211         os.chdir(cwd)
212         self.next += 1
213
214
215 class buildlist(object):
216     '''handle build of multiple directories'''
217
218     def __init__(self, tasklist, tasknames):
219         global tasks
220         self.tlist = []
221         self.tail_proc = None
222         self.retry = None
223         if tasknames == []:
224             tasknames = defaulttasks
225         for n in tasknames:
226             b = builder(n, tasks[n])
227             self.tlist.append(b)
228         if options.retry:
229             self.retry = builder('retry', retry_task)
230             self.need_retry = False
231
232     def kill_kids(self):
233         if self.tail_proc is not None:
234             self.tail_proc.terminate()
235             self.tail_proc.wait()
236             self.tail_proc = None
237         if self.retry is not None:
238             self.retry.proc.terminate()
239             self.retry.proc.wait()
240             self.retry = None
241         for b in self.tlist:
242             if b.proc is not None:
243                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
244                 b.proc.terminate()
245                 b.proc.wait()
246                 b.proc = None
247
248     def wait_one(self):
249         while True:
250             none_running = True
251             for b in self.tlist:
252                 if b.proc is None:
253                     continue
254                 none_running = False
255                 b.status = b.proc.poll()
256                 if b.status is None:
257                     continue
258                 b.proc = None
259                 return b
260             if options.retry:
261                 ret = self.retry.proc.poll()
262                 if ret is not None:
263                     self.need_retry = True
264                     self.retry = None
265                     return None
266             if none_running:
267                 return None
268             time.sleep(0.1)
269
270     def run(self):
271         while True:
272             b = self.wait_one()
273             if options.retry and self.need_retry:
274                 self.kill_kids()
275                 print("retry needed")
276                 return (0, None, None, None, "retry")
277             if b is None:
278                 break
279             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
280                 self.kill_kids()
281                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
282             b.start_next()
283         self.kill_kids()
284         return (0, None, None, None, "All OK")
285
286     def tarlogs(self, fname):
287         tar = tarfile.open(fname, "w:gz")
288         for b in self.tlist:
289             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
290             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
291         if os.path.exists("autobuild.log"):
292             tar.add("autobuild.log")
293         tar.close()
294
295     def remove_logs(self):
296         for b in self.tlist:
297             os.unlink(b.stdout_path)
298             os.unlink(b.stderr_path)
299
300     def start_tail(self):
301         cwd = os.getcwd()
302         cmd = "tail -f *.stdout *.stderr"
303         os.chdir(gitroot)
304         self.tail_proc = Popen(cmd, shell=True)
305         os.chdir(cwd)
306
307
308 def cleanup():
309     if options.nocleanup:
310         return
311     print("Cleaning up ....")
312     for d in cleanup_list:
313         run_cmd("rm -rf %s" % d)
314
315
316 def find_git_root():
317     '''get to the top of the git repo'''
318     p=os.getcwd()
319     while p != '/':
320         if os.path.isdir(os.path.join(p, ".git")):
321             return p
322         p = os.path.abspath(os.path.join(p, '..'))
323     return None
324
325
326 def daemonize(logfile):
327     pid = os.fork()
328     if pid == 0: # Parent
329         os.setsid()
330         pid = os.fork()
331         if pid != 0: # Actual daemon
332             os._exit(0)
333     else: # Grandparent
334         os._exit(0)
335
336     import resource      # Resource usage information.
337     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
338     if maxfd == resource.RLIM_INFINITY:
339         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
340     for fd in range(0, maxfd):
341         try:
342             os.close(fd)
343         except OSError:
344             pass
345     os.open(logfile, os.O_RDWR | os.O_CREAT)
346     os.dup2(0, 1)
347     os.dup2(0, 2)
348
349 def write_pidfile(fname):
350     '''write a pid file, cleanup on exit'''
351     f = open(fname, mode='w')
352     f.write("%u\n" % os.getpid())
353     f.close()
354
355
356 def rebase_tree(url):
357     print("Rebasing on %s" % url)
358     run_cmd("git describe HEAD", show=True, dir=test_master)
359     run_cmd("git remote add -t master master %s" % url, show=True, dir=test_master)
360     run_cmd("git fetch master", show=True, dir=test_master)
361     if options.fix_whitespace:
362         run_cmd("git rebase --whitespace=fix master/master", show=True, dir=test_master)
363     else:
364         run_cmd("git rebase master/master", show=True, dir=test_master)
365     diff = run_cmd("git --no-pager diff HEAD master/master", dir=test_master, output=True)
366     if diff == '':
367         print("No differences between HEAD and master/master - exiting")
368         sys.exit(0)
369     run_cmd("git describe master/master", show=True, dir=test_master)
370     run_cmd("git describe HEAD", show=True, dir=test_master)
371     run_cmd("git --no-pager diff --stat HEAD master/master", show=True, dir=test_master)
372
373 def push_to(url):
374     print("Pushing to %s" % url)
375     if options.mark:
376         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
377         run_cmd("git commit --amend -c HEAD", dir=test_master)
378         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
379         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
380     run_cmd("git remote add -t master pushto %s" % url, show=True, dir=test_master)
381     run_cmd("git push pushto +HEAD:master", show=True, dir=test_master)
382
383 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
384
385 parser = OptionParser()
386 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
387 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
388 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
389 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
390                   default=def_testbase)
391 parser.add_option("", "--passcmd", help="command to run on success", default=None)
392 parser.add_option("", "--verbose", help="show all commands as they are run",
393                   default=False, action="store_true")
394 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
395                   default=None, type='str')
396 parser.add_option("", "--rebase-master", help="rebase on %s before testing" % samba_master,
397                   default=False, action='store_true')
398 parser.add_option("", "--pushto", help="push to a git url on success",
399                   default=None, type='str')
400 parser.add_option("", "--push-master", help="push to %s on success" % samba_master_ssh,
401                   default=False, action='store_true')
402 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
403                   default=False, action="store_true")
404 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
405                   default=False, action="store_true")
406 parser.add_option("", "--retry", help="automatically retry if master changes",
407                   default=False, action="store_true")
408 parser.add_option("", "--email", help="send email to the given address on failure",
409                   type='str', default=None)
410 parser.add_option("", "--always-email", help="always send email, even on success",
411                   action="store_true")
412 parser.add_option("", "--daemon", help="daemonize after initial setup",
413                   action="store_true")
414
415
416 def email_failure(status, failed_task, failed_stage, failed_tag, errstr):
417     '''send an email to options.email about the failure'''
418     user = os.getenv("USER")
419     text = '''
420 Dear Developer,
421
422 Your autobuild failed when trying to test %s with the following error:
423    %s
424
425 the autobuild has been abandoned. Please fix the error and resubmit.
426
427 A summary of the autobuild process is here:
428
429   http://git.samba.org/%s/samba-autobuild/autobuild.log
430 ''' % (failed_task, errstr, user)
431     
432     if failed_task != 'rebase':
433         text += '''
434 You can see logs of the failed task here:
435
436   http://git.samba.org/%s/samba-autobuild/%s.stdout
437   http://git.samba.org/%s/samba-autobuild/%s.stderr
438
439 or you can get full logs of all tasks in this job here:
440
441   http://git.samba.org/%s/samba-autobuild/logs.tar.gz
442
443 The top commit for the tree that was built was:
444
445 %s
446
447 ''' % (user, failed_tag, user, failed_tag, user, top_commit_msg)
448     msg = MIMEText(text)
449     msg['Subject'] = 'autobuild failure for task %s during %s' % (failed_task, failed_stage)
450     msg['From'] = 'autobuild@samba.org'
451     msg['To'] = options.email
452
453     s = smtplib.SMTP()
454     s.connect()
455     s.sendmail(msg['From'], [msg['To']], msg.as_string())
456     s.quit()
457
458 def email_success():
459     '''send an email to options.email about a successful build'''
460     user = os.getenv("USER")
461     text = '''
462 Dear Developer,
463
464 Your autobuild has succeeded.
465
466 '''
467
468     if options.keeplogs:
469         text += '''
470
471 you can get full logs of all tasks in this job here:
472
473   http://git.samba.org/%s/samba-autobuild/logs.tar.gz
474
475 ''' % user
476
477     text += '''
478 The top commit for the tree that was built was:
479
480 %s
481 ''' % top_commit_msg
482
483     msg = MIMEText(text)
484     msg['Subject'] = 'autobuild success'
485     msg['From'] = 'autobuild@samba.org'
486     msg['To'] = options.email
487
488     s = smtplib.SMTP()
489     s.connect()
490     s.sendmail(msg['From'], [msg['To']], msg.as_string())
491     s.quit()
492
493
494 (options, args) = parser.parse_args()
495
496 if options.retry:
497     if not options.rebase_master and options.rebase is None:
498         raise Exception('You can only use --retry if you also rebase')
499
500 testbase = "%s/b%u" % (options.testbase, os.getpid())
501 test_master = "%s/master" % testbase
502
503 gitroot = find_git_root()
504 if gitroot is None:
505     raise Exception("Failed to find git root")
506
507 # get the top commit message, for emails
508 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
509
510 try:
511     os.makedirs(testbase)
512 except Exception, reason:
513     raise Exception("Unable to create %s : %s" % (testbase, reason))
514 cleanup_list.append(testbase)
515
516 if options.daemon:
517     logfile = os.path.join(testbase, "log")
518     print "Forking into the background, writing progress to %s" % logfile
519     daemonize(logfile)
520
521 write_pidfile(gitroot + "/autobuild.pid")
522
523 while True:
524     try:
525         run_cmd("rm -rf %s" % test_master)
526         cleanup_list.append(test_master)
527         run_cmd("git clone --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
528     except Exception:
529         cleanup()
530         raise
531
532     try:
533         try:
534             if options.rebase is not None:
535                 rebase_tree(options.rebase)
536             elif options.rebase_master:
537                 rebase_tree(samba_master)
538         except Exception:
539             cleanup_list.append(gitroot + "/autobuild.pid")
540             cleanup()
541             email_failure(-1, 'rebase', 'rebase', 'rebase', 'rebase on master failed')
542             sys.exit(1)
543         blist = buildlist(tasks, args)
544         if options.tail:
545             blist.start_tail()
546         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
547         if status != 0 or errstr != "retry":
548             break
549         cleanup()
550     except Exception:
551         cleanup()
552         raise
553
554 cleanup_list.append(gitroot + "/autobuild.pid")
555
556 blist.kill_kids()
557 if options.tail:
558     print("waiting for tail to flush")
559     time.sleep(1)
560
561 if status == 0:
562     print errstr
563     if options.passcmd is not None:
564         print("Running passcmd: %s" % options.passcmd)
565         run_cmd(options.passcmd, dir=test_master)
566     if options.pushto is not None:
567         push_to(options.pushto)
568     elif options.push_master:
569         push_to(samba_master_ssh)
570     if options.keeplogs:
571         blist.tarlogs("logs.tar.gz")
572         print("Logs in logs.tar.gz")
573     if options.always_email:
574         email_success()
575     blist.remove_logs()
576     cleanup()
577     print(errstr)
578     sys.exit(0)
579
580 # something failed, gather a tar of the logs
581 blist.tarlogs("logs.tar.gz")
582
583 if options.email is not None:
584     email_failure(status, failed_task, failed_stage, failed_tag, errstr)
585
586 cleanup()
587 print(errstr)
588 print("Logs in logs.tar.gz")
589 sys.exit(status)