ldb: make ldb a top level library for Samba 4.0
[sfrench/samba-autobuild/.git] / script / autobuild.py
1 #!/usr/bin/env python
2 # run tests on all Samba subprojects and push to a git tree on success
3 # Copyright Andrew Tridgell 2010
4 # released under GNU GPL v3 or later
5
6 from subprocess import call, check_call,Popen, PIPE
7 import os, tarfile, sys, time
8 from optparse import OptionParser
9 import smtplib
10 from email.mime.text import MIMEText
11
12 samba_master = os.getenv('SAMBA_MASTER', 'git://git.samba.org/samba.git')
13 samba_master_ssh = os.getenv('SAMBA_MASTER_SSH', 'git+ssh://git.samba.org/data/git/samba.git')
14
15 cleanup_list = []
16
17 builddirs = {
18     "samba3"  : "source3",
19     "samba3-waf": "source3",
20     "samba4"  : ".",
21     "ldb"     : "lib/ldb",
22     "tdb"     : "lib/tdb",
23     "talloc"  : "lib/talloc",
24     "replace" : "lib/replace",
25     "tevent"  : "lib/tevent",
26     "pidl"    : "pidl",
27     "pass"    : ".",
28     "fail"    : ".",
29     "retry"   : "."
30     }
31
32 defaulttasks = [ "samba3", "samba3-waf", "samba4", "ldb", "tdb", "talloc", "replace", "tevent", "pidl" ]
33
34 tasks = {
35     "samba3" : [ ("autogen", "./autogen.sh", "text/plain"),
36                  ("configure", "./configure.developer ${PREFIX}", "text/plain"),
37                  ("make basics", "make basics", "text/plain"),
38                  ("make", "make -j 4 everything", "text/plain"), # don't use too many processes
39                  ("install", "make install", "text/plain"),
40                  ("test", "TDB_NO_FSYNC=1 make test FAIL_IMMEDIATELY=1", "text/plain"),
41                  ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
42                  ("clean", "make clean", "text/plain") ],
43
44     "samba3-waf" : [ ("autogen", "./autogen-waf.sh", "text/plain"),
45                  ("configure", "./configure.developer ${PREFIX}", "text/plain"),
46                  ("make", "make -j", "text/plain"),
47                  ("install", "make install", "text/plain"),
48                  ("clean", "make clean", "text/plain") ],
49
50     # We have 'test' before 'install' because, 'test' should work without 'install'
51     "samba4" : [ ("configure", "./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab", "text/plain"),
52                  ("make", "make -j", "text/plain"),
53                  ("test", "TDB_NO_FSYNC=1 make test FAIL_IMMEDIATELY=1", "text/plain"),
54                  ("install", "make install", "text/plain"),
55                  ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
56                  ("clean", "make clean", "text/plain") ],
57
58     "ldb" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
59               ("make", "make -j", "text/plain"),
60               ("install", "make install", "text/plain"),
61               ("test", "TDB_NO_FSYNC=1 make test", "text/plain"),
62               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
63               ("distcheck", "make distcheck", "text/plain"),
64               ("clean", "make clean", "text/plain") ],
65
66     # We don't use TDB_NO_FSYNC=1 here, because we want to test the transaction code
67     "tdb" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
68               ("make", "make -j", "text/plain"),
69               ("install", "make install", "text/plain"),
70               ("test", "make test", "text/plain"),
71               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
72               ("distcheck", "make distcheck", "text/plain"),
73               ("clean", "make clean", "text/plain") ],
74
75     "talloc" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
76                  ("make", "make -j", "text/plain"),
77                  ("install", "make install", "text/plain"),
78                  ("test", "make test", "text/plain"),
79                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
80                  ("distcheck", "make distcheck", "text/plain"),
81                  ("clean", "make clean", "text/plain") ],
82
83     "replace" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
84                   ("make", "make -j", "text/plain"),
85                   ("install", "make install", "text/plain"),
86                   ("test", "make test", "text/plain"),
87                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
88                   ("distcheck", "make distcheck", "text/plain"),
89                   ("clean", "make clean", "text/plain") ],
90
91     "tevent" : [ ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
92                  ("make", "make -j", "text/plain"),
93                  ("install", "make install", "text/plain"),
94                  ("test", "make test", "text/plain"),
95                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
96                  ("distcheck", "make distcheck", "text/plain"),
97                  ("clean", "make clean", "text/plain") ],
98
99     "pidl" : [ ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
100                ("touch", "touch *.yp", "text/plain"),
101                ("make", "make", "text/plain"),
102                ("test", "make test", "text/plain"),
103                ("install", "make install", "text/plain"),
104                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
105                ("clean", "make clean", "text/plain") ],
106
107     # these are useful for debugging autobuild
108     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
109     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
110 }
111
112 retry_task = [ ( "retry",
113                  '''set -e
114                 git remote add -t master master %s
115                 git fetch master
116                 while :; do
117                   sleep 60
118                   git describe master/master > old_master.desc
119                   git fetch master
120                   git describe master/master > master.desc
121                   diff old_master.desc master.desc
122                 done
123                ''' % samba_master, "test/plain" ) ]
124
125 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
126     if show is None:
127         show = options.verbose
128     if show:
129         print("Running: '%s' in '%s'" % (cmd, dir))
130     if output:
131         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
132     elif checkfail:
133         return check_call(cmd, shell=True, cwd=dir)
134     else:
135         return call(cmd, shell=True, cwd=dir)
136
137
138 class builder(object):
139     '''handle build of one directory'''
140
141     def __init__(self, name, sequence):
142         self.name = name
143         self.dir = builddirs[name]
144
145         self.tag = self.name.replace('/', '_')
146         self.sequence = sequence
147         self.next = 0
148         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
149         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
150         if options.verbose:
151             print("stdout for %s in %s" % (self.name, self.stdout_path))
152             print("stderr for %s in %s" % (self.name, self.stderr_path))
153         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
154         self.stdout = open(self.stdout_path, 'w')
155         self.stderr = open(self.stderr_path, 'w')
156         self.stdin  = open("/dev/null", 'r')
157         self.sdir = "%s/%s" % (testbase, self.tag)
158         self.prefix = "%s/prefix/%s" % (testbase, self.tag)
159         run_cmd("rm -rf %s" % self.sdir)
160         cleanup_list.append(self.sdir)
161         cleanup_list.append(self.prefix)
162         os.makedirs(self.sdir)
163         run_cmd("rm -rf %s" % self.sdir)
164         run_cmd("git clone --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
165         self.start_next()
166
167     def start_next(self):
168         if self.next == len(self.sequence):
169             print '%s: Completed OK' % self.name
170             self.done = True
171             return
172         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
173         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
174         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
175 #        if self.output_mime_type == "text/x-subunit":
176 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
177         print '%s: [%s] Running %s' % (self.name, self.stage, self.cmd)
178         cwd = os.getcwd()
179         os.chdir("%s/%s" % (self.sdir, self.dir))
180         self.proc = Popen(self.cmd, shell=True,
181                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
182         os.chdir(cwd)
183         self.next += 1
184
185
186 class buildlist(object):
187     '''handle build of multiple directories'''
188
189     def __init__(self, tasklist, tasknames):
190         global tasks
191         self.tlist = []
192         self.tail_proc = None
193         self.retry = None
194         if tasknames == []:
195             tasknames = defaulttasks
196         for n in tasknames:
197             b = builder(n, tasks[n])
198             self.tlist.append(b)
199         if options.retry:
200             self.retry = builder('retry', retry_task)
201             self.need_retry = False
202
203     def kill_kids(self):
204         if self.tail_proc is not None:
205             self.tail_proc.terminate()
206             self.tail_proc.wait()
207             self.tail_proc = None
208         if self.retry is not None:
209             self.retry.proc.terminate()
210             self.retry.proc.wait()
211             self.retry = None
212         for b in self.tlist:
213             if b.proc is not None:
214                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
215                 b.proc.terminate()
216                 b.proc.wait()
217                 b.proc = None
218
219     def wait_one(self):
220         while True:
221             none_running = True
222             for b in self.tlist:
223                 if b.proc is None:
224                     continue
225                 none_running = False
226                 b.status = b.proc.poll()
227                 if b.status is None:
228                     continue
229                 b.proc = None
230                 return b
231             if options.retry:
232                 ret = self.retry.proc.poll()
233                 if ret is not None:
234                     self.need_retry = True
235                     self.retry = None
236                     return None
237             if none_running:
238                 return None
239             time.sleep(0.1)
240
241     def run(self):
242         while True:
243             b = self.wait_one()
244             if options.retry and self.need_retry:
245                 self.kill_kids()
246                 print("retry needed")
247                 return (0, None, None, None, "retry")
248             if b is None:
249                 break
250             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
251                 self.kill_kids()
252                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
253             b.start_next()
254         self.kill_kids()
255         return (0, None, None, None, "All OK")
256
257     def tarlogs(self, fname):
258         tar = tarfile.open(fname, "w:gz")
259         for b in self.tlist:
260             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
261             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
262         if os.path.exists("autobuild.log"):
263             tar.add("autobuild.log")
264         tar.close()
265
266     def remove_logs(self):
267         for b in self.tlist:
268             os.unlink(b.stdout_path)
269             os.unlink(b.stderr_path)
270
271     def start_tail(self):
272         cwd = os.getcwd()
273         cmd = "tail -f *.stdout *.stderr"
274         os.chdir(gitroot)
275         self.tail_proc = Popen(cmd, shell=True)
276         os.chdir(cwd)
277
278
279 def cleanup():
280     if options.nocleanup:
281         return
282     print("Cleaning up ....")
283     for d in cleanup_list:
284         run_cmd("rm -rf %s" % d)
285
286
287 def find_git_root():
288     '''get to the top of the git repo'''
289     p=os.getcwd()
290     while p != '/':
291         if os.path.isdir(os.path.join(p, ".git")):
292             return p
293         p = os.path.abspath(os.path.join(p, '..'))
294     return None
295
296
297 def daemonize(logfile):
298     pid = os.fork()
299     if pid == 0: # Parent
300         os.setsid()
301         pid = os.fork()
302         if pid != 0: # Actual daemon
303             os._exit(0)
304     else: # Grandparent
305         os._exit(0)
306
307     import resource      # Resource usage information.
308     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
309     if maxfd == resource.RLIM_INFINITY:
310         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
311     for fd in range(0, maxfd):
312         try:
313             os.close(fd)
314         except OSError:
315             pass
316     os.open(logfile, os.O_RDWR | os.O_CREAT)
317     os.dup2(0, 1)
318     os.dup2(0, 2)
319
320 def write_pidfile(fname):
321     '''write a pid file, cleanup on exit'''
322     f = open(fname, mode='w')
323     f.write("%u\n" % os.getpid())
324     f.close()
325
326
327 def rebase_tree(url):
328     print("Rebasing on %s" % url)
329     run_cmd("git describe HEAD", show=True, dir=test_master)
330     run_cmd("git remote add -t master master %s" % url, show=True, dir=test_master)
331     run_cmd("git fetch master", show=True, dir=test_master)
332     if options.fix_whitespace:
333         run_cmd("git rebase --whitespace=fix master/master", show=True, dir=test_master)
334     else:
335         run_cmd("git rebase master/master", show=True, dir=test_master)
336     diff = run_cmd("git --no-pager diff HEAD master/master", dir=test_master, output=True)
337     if diff == '':
338         print("No differences between HEAD and master/master - exiting")
339         sys.exit(0)
340     run_cmd("git describe master/master", show=True, dir=test_master)
341     run_cmd("git describe HEAD", show=True, dir=test_master)
342     run_cmd("git --no-pager diff --stat HEAD master/master", show=True, dir=test_master)
343
344 def push_to(url):
345     print("Pushing to %s" % url)
346     if options.mark:
347         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
348         run_cmd("git commit --amend -c HEAD", dir=test_master)
349         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
350         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
351     run_cmd("git remote add -t master pushto %s" % url, show=True, dir=test_master)
352     run_cmd("git push pushto +HEAD:master", show=True, dir=test_master)
353
354 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
355
356 parser = OptionParser()
357 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
358 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
359 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
360 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
361                   default=def_testbase)
362 parser.add_option("", "--passcmd", help="command to run on success", default=None)
363 parser.add_option("", "--verbose", help="show all commands as they are run",
364                   default=False, action="store_true")
365 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
366                   default=None, type='str')
367 parser.add_option("", "--rebase-master", help="rebase on %s before testing" % samba_master,
368                   default=False, action='store_true')
369 parser.add_option("", "--pushto", help="push to a git url on success",
370                   default=None, type='str')
371 parser.add_option("", "--push-master", help="push to %s on success" % samba_master_ssh,
372                   default=False, action='store_true')
373 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
374                   default=False, action="store_true")
375 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
376                   default=False, action="store_true")
377 parser.add_option("", "--retry", help="automatically retry if master changes",
378                   default=False, action="store_true")
379 parser.add_option("", "--email", help="send email to the given address on failure",
380                   type='str', default=None)
381 parser.add_option("", "--always-email", help="always send email, even on success",
382                   action="store_true")
383 parser.add_option("", "--daemon", help="daemonize after initial setup",
384                   action="store_true")
385
386
387 def email_failure(status, failed_task, failed_stage, failed_tag, errstr):
388     '''send an email to options.email about the failure'''
389     user = os.getenv("USER")
390     text = '''
391 Dear Developer,
392
393 Your autobuild failed when trying to test %s with the following error:
394    %s
395
396 the autobuild has been abandoned. Please fix the error and resubmit.
397
398 A summary of the autobuild process is here:
399
400   http://git.samba.org/%s/samba-autobuild/autobuild.log
401 ''' % (failed_task, errstr, user)
402     
403     if failed_task != 'rebase':
404         text += '''
405 You can see logs of the failed task here:
406
407   http://git.samba.org/%s/samba-autobuild/%s.stdout
408   http://git.samba.org/%s/samba-autobuild/%s.stderr
409
410 or you can get full logs of all tasks in this job here:
411
412   http://git.samba.org/%s/samba-autobuild/logs.tar.gz
413
414 The top commit for the tree that was built was:
415
416 %s
417
418 ''' % (user, failed_tag, user, failed_tag, user, top_commit_msg)
419     msg = MIMEText(text)
420     msg['Subject'] = 'autobuild failure for task %s during %s' % (failed_task, failed_stage)
421     msg['From'] = 'autobuild@samba.org'
422     msg['To'] = options.email
423
424     s = smtplib.SMTP()
425     s.connect()
426     s.sendmail(msg['From'], [msg['To']], msg.as_string())
427     s.quit()
428
429 def email_success():
430     '''send an email to options.email about a successful build'''
431     user = os.getenv("USER")
432     text = '''
433 Dear Developer,
434
435 Your autobuild has succeeded.
436
437 '''
438
439     if options.keeplogs:
440         text += '''
441
442 you can get full logs of all tasks in this job here:
443
444   http://git.samba.org/%s/samba-autobuild/logs.tar.gz
445
446 ''' % user
447
448     text += '''
449 The top commit for the tree that was built was:
450
451 %s
452 ''' % top_commit_msg
453
454     msg = MIMEText(text)
455     msg['Subject'] = 'autobuild success'
456     msg['From'] = 'autobuild@samba.org'
457     msg['To'] = options.email
458
459     s = smtplib.SMTP()
460     s.connect()
461     s.sendmail(msg['From'], [msg['To']], msg.as_string())
462     s.quit()
463
464
465 (options, args) = parser.parse_args()
466
467 if options.retry:
468     if not options.rebase_master and options.rebase is None:
469         raise Exception('You can only use --retry if you also rebase')
470
471 testbase = "%s/b%u" % (options.testbase, os.getpid())
472 test_master = "%s/master" % testbase
473
474 gitroot = find_git_root()
475 if gitroot is None:
476     raise Exception("Failed to find git root")
477
478 # get the top commit message, for emails
479 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
480
481 try:
482     os.makedirs(testbase)
483 except Exception, reason:
484     raise Exception("Unable to create %s : %s" % (testbase, reason))
485 cleanup_list.append(testbase)
486
487 if options.daemon:
488     logfile = os.path.join(testbase, "log")
489     print "Forking into the background, writing progress to %s" % logfile
490     daemonize(logfile)
491
492 write_pidfile(gitroot + "/autobuild.pid")
493
494 while True:
495     try:
496         run_cmd("rm -rf %s" % test_master)
497         cleanup_list.append(test_master)
498         run_cmd("git clone --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
499     except:
500         cleanup()
501         raise
502
503     try:
504         try:
505             if options.rebase is not None:
506                 rebase_tree(options.rebase)
507             elif options.rebase_master:
508                 rebase_tree(samba_master)
509         except:
510             email_failure(-1, 'rebase', 'rebase', 'rebase', 'rebase on master failed')
511             sys.exit(1)
512         blist = buildlist(tasks, args)
513         if options.tail:
514             blist.start_tail()
515         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
516         if status != 0 or errstr != "retry":
517             break
518         cleanup()
519     except:
520         cleanup()
521         raise
522
523 cleanup_list.append(gitroot + "/autobuild.pid")
524
525 blist.kill_kids()
526 if options.tail:
527     print("waiting for tail to flush")
528     time.sleep(1)
529
530 if status == 0:
531     print errstr
532     if options.passcmd is not None:
533         print("Running passcmd: %s" % options.passcmd)
534         run_cmd(options.passcmd, dir=test_master)
535     if options.pushto is not None:
536         push_to(options.pushto)
537     elif options.push_master:
538         push_to(samba_master_ssh)
539     if options.keeplogs:
540         blist.tarlogs("logs.tar.gz")
541         print("Logs in logs.tar.gz")
542     if options.always_email:
543         email_success()
544     blist.remove_logs()
545     cleanup()
546     print(errstr)
547     sys.exit(0)
548
549 # something failed, gather a tar of the logs
550 blist.tarlogs("logs.tar.gz")
551
552 if options.email is not None:
553     email_failure(status, failed_task, failed_stage, failed_tag, errstr)
554
555 cleanup()
556 print(errstr)
557 print("Logs in logs.tar.gz")
558 sys.exit(status)