script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call,Popen, PIPE
   8 import os, tarfile, sys, time
   9 from optparse import OptionParser
  10 import smtplib
  11 import email
  12 from email.mime.text import MIMEText
  13 from email.mime.base import MIMEBase
  14 from email.mime.application import MIMEApplication
  15 from email.mime.multipart import MIMEMultipart
  16 from distutils.sysconfig import get_python_lib
  17 import platform
  18
  19 os.environ["PYTHONUNBUFFERED"] = "1"
  20
  21 # This speeds up testing remarkably.
  22 os.environ['TDB_NO_FSYNC'] = '1'
  23
  24 cleanup_list = []
  25
  26 builddirs = {
  27     "ctdb"    : "ctdb",
  28     "samba"  : ".",
  29     "samba-xc" : ".",
  30     "samba-o3" : ".",
  31     "samba-ctdb" : ".",
  32     "samba-libs"  : ".",
  33     "samba-static"  : ".",
  34     "samba-test-only"  : ".",
  35     "samba-systemkrb5"  : ".",
  36     "samba-nopython"  : ".",
  37     "ldb"     : "lib/ldb",
  38     "tdb"     : "lib/tdb",
  39     "talloc"  : "lib/talloc",
  40     "replace" : "lib/replace",
  41     "tevent"  : "lib/tevent",
  42     "pidl"    : "pidl",
  43     "pass"    : ".",
  44     "fail"    : ".",
  45     "retry"   : "."
  46     }
  47
  48 defaulttasks = [ "ctdb",
  49                  "samba",
  50                  "samba-xc",
  51                  "samba-o3",
  52                  "samba-ctdb",
  53                  "samba-libs",
  54                  "samba-static",
  55                  "samba-systemkrb5",
  56                  "samba-nopython",
  57                  "ldb",
  58                  "tdb",
  59                  "talloc",
  60                  "replace",
  61                  "tevent",
  62                  "pidl" ]
  63
  64 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  65     defaulttasks.remove("samba-o3")
  66
  67 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  68 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  69
  70 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  71 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  72 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  73 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX} ${EXTRA_PYTHON}"
  74 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE"
  75 samba_libs_configure_samba = samba_libs_configure_base + " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
  76
  77 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  78     extra_python = ""
  79 else:
  80     extra_python = "--extra-python=/usr/bin/python3"
  81
  82 tasks = {
  83     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  84                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  85                ("make", "make all", "text/plain"),
  86                ("install", "make install", "text/plain"),
  87                ("test", "make autotest", "text/plain"),
  88                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  89                ("clean", "make clean", "text/plain") ],
  90
  91     # We have 'test' before 'install' because, 'test' should work without 'install'
  92     "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  93                 ("make", "make -j", "text/plain"),
  94                 ("test", "make test FAIL_IMMEDIATELY=1", "text/plain"),
  95                 ("install", "make install", "text/plain"),
  96                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
  97                 ("clean", "make clean", "text/plain") ],
  98
  99     "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 100                           ("make", "make -j", "text/plain"),
 101                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
 102
 103     # Test cross-compile infrastructure
 104     "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 105                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 106                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 107                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 108                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 109                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 110
 111     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 112     "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 113                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 114                    ("make", "make -j", "text/plain"),
 115                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 116                    ("install", "make install", "text/plain"),
 117                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 118                    ("clean", "make clean", "text/plain") ],
 119
 120     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 121
 122                      # make sure we have tdb around:
 123                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 124                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 125                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 126
 127
 128                      # build samba with cluster support (also building ctdb):
 129                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 130                      ("samba-make", "make", "text/plain"),
 131                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 132                      ("samba-install", "make install", "text/plain"),
 133                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 134
 135                      # clean up:
 136                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 137                      ("clean", "make clean", "text/plain"),
 138                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
 139
 140     "samba-libs" : [
 141                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 142                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 143                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 144                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 145
 146                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 147                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 148                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 149
 150                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 151                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 152                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 153
 154                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 155                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 156                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 157
 158                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 159                       ("nondevel-make", "make -j", "text/plain"),
 160                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 161                       ("nondevel-install", "make install", "text/plain"),
 162                       ("nondevel-dist", "make dist", "text/plain"),
 163
 164                       # retry with all modules shared
 165                       ("allshared-distclean", "make distclean", "text/plain"),
 166                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 167                       ("allshared-make", "make -j", "text/plain")],
 168
 169     "samba-static" : [
 170                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 171                       # build with all modules static
 172                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 173                       ("allstatic-make", "make -j", "text/plain"),
 174
 175                       # retry without any required modules
 176                       ("none-distclean", "make distclean", "text/plain"),
 177                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 178                       ("none-make", "make -j", "text/plain"),
 179
 180                       # retry with nonshared smbd and smbtorture
 181                       ("nonshared-distclean", "make distclean", "text/plain"),
 182                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 183                       ("nonshared-make", "make -j", "text/plain")],
 184
 185     "samba-systemkrb5" : [
 186                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 187                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 188                       ("make", "make -j", "text/plain"),
 189                       # we currently cannot run a full make test, a limited list of tests could be run
 190                       # via "make test TESTS=sometests"
 191                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 192                       ("install", "make install", "text/plain"),
 193                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 194                       ("clean", "make clean", "text/plain")
 195                       ],
 196
 197     # Test Samba without python still builds.  When this test fails
 198     # due to more use of Python, the expectations is that the newly
 199     # failing part of the code should be disabled when
 200     # --disable-python is set (rather than major work being done to
 201     # support this environment).  The target here is for vendors
 202     # shipping a minimal smbd.
 203     "samba-nopython" : [
 204                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 205                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 206                       ("make", "make -j", "text/plain"),
 207                       ("install", "make install", "text/plain"),
 208                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 209                       ("clean", "make clean", "text/plain")
 210                       ],
 211
 212
 213
 214     "ldb" : [
 215               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 216               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 217               ("make", "make", "text/plain"),
 218               ("install", "make install", "text/plain"),
 219               ("test", "make test", "text/plain"),
 220               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 221               ("distcheck", "make distcheck", "text/plain"),
 222               ("clean", "make clean", "text/plain") ],
 223
 224     "tdb" : [
 225               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 226               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 227               ("make", "make", "text/plain"),
 228               ("install", "make install", "text/plain"),
 229               ("test", "make test", "text/plain"),
 230               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 231               ("distcheck", "make distcheck", "text/plain"),
 232               ("clean", "make clean", "text/plain") ],
 233
 234     "talloc" : [
 235                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 236                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 237                  ("make", "make", "text/plain"),
 238                  ("install", "make install", "text/plain"),
 239                  ("test", "make test", "text/plain"),
 240                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 241                  ("distcheck", "make distcheck", "text/plain"),
 242                  ("clean", "make clean", "text/plain") ],
 243
 244     "replace" : [
 245                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 246                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 247                   ("make", "make", "text/plain"),
 248                   ("install", "make install", "text/plain"),
 249                   ("test", "make test", "text/plain"),
 250                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 251                   ("distcheck", "make distcheck", "text/plain"),
 252                   ("clean", "make clean", "text/plain") ],
 253
 254     "tevent" : [
 255                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 256                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 257                  ("make", "make", "text/plain"),
 258                  ("install", "make install", "text/plain"),
 259                  ("test", "make test", "text/plain"),
 260                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 261                  ("distcheck", "make distcheck", "text/plain"),
 262                  ("clean", "make clean", "text/plain") ],
 263
 264     "pidl" : [
 265                ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 266                ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 267                ("touch", "touch *.yp", "text/plain"),
 268                ("make", "make", "text/plain"),
 269                ("test", "make test", "text/plain"),
 270                ("install", "make install", "text/plain"),
 271                ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 272                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 273                ("clean", "make clean", "text/plain") ],
 274
 275     # these are useful for debugging autobuild
 276     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
 277     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
 278 }
 279
 280 def do_print(msg):
 281     print("%s" % msg)
 282     sys.stdout.flush()
 283     sys.stderr.flush()
 284
 285 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 286     if show is None:
 287         show = options.verbose
 288     if show:
 289         do_print("Running: '%s' in '%s'" % (cmd, dir))
 290     if output:
 291         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 292     elif checkfail:
 293         return check_call(cmd, shell=True, cwd=dir)
 294     else:
 295         return call(cmd, shell=True, cwd=dir)
 296
 297
 298 class builder(object):
 299     '''handle build of one directory'''
 300
 301     def __init__(self, name, sequence, cp=True):
 302         self.name = name
 303         self.dir = builddirs[name]
 304
 305         self.tag = self.name.replace('/', '_')
 306         self.sequence = sequence
 307         self.next = 0
 308         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 309         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 310         if options.verbose:
 311             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 312             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 313         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 314         self.stdout = open(self.stdout_path, 'w')
 315         self.stderr = open(self.stderr_path, 'w')
 316         self.stdin  = open("/dev/null", 'r')
 317         self.sdir = "%s/%s" % (testbase, self.tag)
 318         self.prefix = "%s/%s" % (test_prefix, self.tag)
 319         run_cmd("rm -rf %s" % self.sdir)
 320         run_cmd("rm -rf %s" % self.prefix)
 321         if cp:
 322             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 323         else:
 324             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 325         self.start_next()
 326
 327     def start_next(self):
 328         if self.next == len(self.sequence):
 329             if not options.nocleanup:
 330                 run_cmd("rm -rf %s" % self.sdir)
 331                 run_cmd("rm -rf %s" % self.prefix)
 332             do_print('%s: Completed OK' % self.name)
 333             self.done = True
 334             return
 335         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 336         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 337         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 338         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 339         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 340         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 341 #        if self.output_mime_type == "text/x-subunit":
 342 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 343         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 344         cwd = os.getcwd()
 345         os.chdir("%s/%s" % (self.sdir, self.dir))
 346         self.proc = Popen(self.cmd, shell=True,
 347                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 348         os.chdir(cwd)
 349         self.next += 1
 350
 351
 352 class buildlist(object):
 353     '''handle build of multiple directories'''
 354
 355     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 356         global tasks
 357         self.tlist = []
 358         self.tail_proc = None
 359         self.retry = None
 360         if tasknames == []:
 361             if options.restrict_tests:
 362                 tasknames = ["samba-test-only"]
 363             else:
 364                 tasknames = defaulttasks
 365         else:
 366             # If we are only running one test,
 367             # do not sleep randomly to wait for it to start
 368             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 369
 370         for n in tasknames:
 371             b = builder(n, tasks[n], cp=n is not "pidl")
 372             self.tlist.append(b)
 373         if options.retry:
 374             rebase_remote = "rebaseon"
 375             retry_task = [ ("retry",
 376                             '''set -e
 377                             git remote add -t %s %s %s
 378                             git fetch %s
 379                             while :; do
 380                               sleep 60
 381                               git describe %s/%s > old_remote_branch.desc
 382                               git fetch %s
 383                               git describe %s/%s > remote_branch.desc
 384                               diff old_remote_branch.desc remote_branch.desc
 385                             done
 386                            ''' % (
 387                                rebase_branch, rebase_remote, rebase_url,
 388                                rebase_remote,
 389                                rebase_remote, rebase_branch,
 390                                rebase_remote,
 391                                rebase_remote, rebase_branch
 392                            ),
 393                            "test/plain" ) ]
 394
 395             self.retry = builder('retry', retry_task, cp=False)
 396             self.need_retry = False
 397
 398     def kill_kids(self):
 399         if self.tail_proc is not None:
 400             self.tail_proc.terminate()
 401             self.tail_proc.wait()
 402             self.tail_proc = None
 403         if self.retry is not None:
 404             self.retry.proc.terminate()
 405             self.retry.proc.wait()
 406             self.retry = None
 407         for b in self.tlist:
 408             if b.proc is not None:
 409                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 410                 b.proc.terminate()
 411                 b.proc.wait()
 412                 b.proc = None
 413
 414     def wait_one(self):
 415         while True:
 416             none_running = True
 417             for b in self.tlist:
 418                 if b.proc is None:
 419                     continue
 420                 none_running = False
 421                 b.status = b.proc.poll()
 422                 if b.status is None:
 423                     continue
 424                 b.proc = None
 425                 return b
 426             if options.retry:
 427                 ret = self.retry.proc.poll()
 428                 if ret is not None:
 429                     self.need_retry = True
 430                     self.retry = None
 431                     return None
 432             if none_running:
 433                 return None
 434             time.sleep(0.1)
 435
 436     def run(self):
 437         while True:
 438             b = self.wait_one()
 439             if options.retry and self.need_retry:
 440                 self.kill_kids()
 441                 do_print("retry needed")
 442                 return (0, None, None, None, "retry")
 443             if b is None:
 444                 break
 445             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 446                 self.kill_kids()
 447                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 448             b.start_next()
 449         self.kill_kids()
 450         return (0, None, None, None, "All OK")
 451
 452     def write_system_info(self):
 453         filename = 'system-info.txt'
 454         f = open(filename, 'w')
 455         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo']:
 456             print('### %s' % cmd, file=f)
 457             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 458             print(file=f)
 459         f.close()
 460         return filename
 461
 462     def tarlogs(self, fname):
 463         tar = tarfile.open(fname, "w:gz")
 464         for b in self.tlist:
 465             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 466             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 467         if os.path.exists("autobuild.log"):
 468             tar.add("autobuild.log")
 469         sys_info = self.write_system_info()
 470         tar.add(sys_info)
 471         tar.close()
 472
 473     def remove_logs(self):
 474         for b in self.tlist:
 475             os.unlink(b.stdout_path)
 476             os.unlink(b.stderr_path)
 477
 478     def start_tail(self):
 479         cwd = os.getcwd()
 480         cmd = "tail -f *.stdout *.stderr"
 481         os.chdir(gitroot)
 482         self.tail_proc = Popen(cmd, shell=True)
 483         os.chdir(cwd)
 484
 485
 486 def cleanup():
 487     if options.nocleanup:
 488         return
 489     run_cmd("stat %s || true" % test_tmpdir, show=True)
 490     run_cmd("stat %s" % testbase, show=True)
 491     do_print("Cleaning up ....")
 492     for d in cleanup_list:
 493         run_cmd("rm -rf %s" % d)
 494
 495
 496 def find_git_root():
 497     '''get to the top of the git repo'''
 498     p=os.getcwd()
 499     while p != '/':
 500         if os.path.isdir(os.path.join(p, ".git")):
 501             return p
 502         p = os.path.abspath(os.path.join(p, '..'))
 503     return None
 504
 505
 506 def daemonize(logfile):
 507     pid = os.fork()
 508     if pid == 0: # Parent
 509         os.setsid()
 510         pid = os.fork()
 511         if pid != 0: # Actual daemon
 512             os._exit(0)
 513     else: # Grandparent
 514         os._exit(0)
 515
 516     import resource      # Resource usage information.
 517     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 518     if maxfd == resource.RLIM_INFINITY:
 519         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
 520     for fd in range(0, maxfd):
 521         try:
 522             os.close(fd)
 523         except OSError:
 524             pass
 525     os.open(logfile, os.O_RDWR | os.O_CREAT)
 526     os.dup2(0, 1)
 527     os.dup2(0, 2)
 528
 529 def write_pidfile(fname):
 530     '''write a pid file, cleanup on exit'''
 531     f = open(fname, mode='w')
 532     f.write("%u\n" % os.getpid())
 533     f.close()
 534
 535
 536 def rebase_tree(rebase_url, rebase_branch = "master"):
 537     rebase_remote = "rebaseon"
 538     do_print("Rebasing on %s" % rebase_url)
 539     run_cmd("git describe HEAD", show=True, dir=test_master)
 540     run_cmd("git remote add -t %s %s %s" %
 541             (rebase_branch, rebase_remote, rebase_url),
 542             show=True, dir=test_master)
 543     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 544     if options.fix_whitespace:
 545         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 546                 (rebase_remote, rebase_branch),
 547                 show=True, dir=test_master)
 548     else:
 549         run_cmd("git rebase --force-rebase %s/%s" %
 550                 (rebase_remote, rebase_branch),
 551                 show=True, dir=test_master)
 552     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 553                    (rebase_remote, rebase_branch),
 554                    dir=test_master, output=True)
 555     if diff == '':
 556         do_print("No differences between HEAD and %s/%s - exiting" %
 557               (rebase_remote, rebase_branch))
 558         sys.exit(0)
 559     run_cmd("git describe %s/%s" %
 560             (rebase_remote, rebase_branch),
 561             show=True, dir=test_master)
 562     run_cmd("git describe HEAD", show=True, dir=test_master)
 563     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 564             (rebase_remote, rebase_branch),
 565             show=True, dir=test_master)
 566
 567 def push_to(push_url, push_branch = "master"):
 568     push_remote = "pushto"
 569     do_print("Pushing to %s" % push_url)
 570     if options.mark:
 571         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 572         run_cmd("git commit --amend -c HEAD", dir=test_master)
 573         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 574         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 575     run_cmd("git remote add -t %s %s %s" %
 576             (push_branch, push_remote, push_url),
 577             show=True, dir=test_master)
 578     run_cmd("git push %s +HEAD:%s" %
 579             (push_remote, push_branch),
 580             show=True, dir=test_master)
 581
 582 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 583
 584 gitroot = find_git_root()
 585 if gitroot is None:
 586     raise Exception("Failed to find git root")
 587
 588 parser = OptionParser()
 589 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 590 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 591 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 592 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 593                   default=def_testbase)
 594 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 595 parser.add_option("", "--verbose", help="show all commands as they are run",
 596                   default=False, action="store_true")
 597 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 598                   default=None, type='str')
 599 parser.add_option("", "--pushto", help="push to a git url on success",
 600                   default=None, type='str')
 601 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 602                   default=False, action="store_true")
 603 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 604                   default=False, action="store_true")
 605 parser.add_option("", "--retry", help="automatically retry if master changes",
 606                   default=False, action="store_true")
 607 parser.add_option("", "--email", help="send email to the given address on failure",
 608                   type='str', default=None)
 609 parser.add_option("", "--email-from", help="send email from the given address",
 610                   type='str', default="autobuild@samba.org")
 611 parser.add_option("", "--email-server", help="send email via the given server",
 612                   type='str', default='localhost')
 613 parser.add_option("", "--always-email", help="always send email, even on success",
 614                   action="store_true")
 615 parser.add_option("", "--daemon", help="daemonize after initial setup",
 616                   action="store_true")
 617 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 618                   default="master", type='str')
 619 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 620                   default=gitroot, type='str')
 621 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 622                   default=False, action="store_true")
 623 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 624                   default='')
 625
 626 def send_email(subject, text, log_tar):
 627     outer = MIMEMultipart()
 628     outer['Subject'] = subject
 629     outer['To'] = options.email
 630     outer['From'] = options.email_from
 631     outer['Date'] = email.utils.formatdate(localtime = True)
 632     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 633     outer.attach(MIMEText(text, 'plain'))
 634     if options.attach_logs:
 635         fp = open(log_tar, 'rb')
 636         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 637         fp.close()
 638         # Set the filename parameter
 639         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 640         outer.attach(msg)
 641     content = outer.as_string()
 642     s = smtplib.SMTP(options.email_server)
 643     s.sendmail(options.email_from, [options.email], content)
 644     s.set_debuglevel(1)
 645     s.quit()
 646
 647 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 648                   elapsed_time, log_base=None, add_log_tail=True):
 649     '''send an email to options.email about the failure'''
 650     elapsed_minutes = elapsed_time / 60.0
 651     user = os.getenv("USER")
 652     if log_base is None:
 653         log_base = gitroot
 654     text = '''
 655 Dear Developer,
 656
 657 Your autobuild on %s failed after %.1f minutes
 658 when trying to test %s with the following error:
 659
 660    %s
 661
 662 the autobuild has been abandoned. Please fix the error and resubmit.
 663
 664 A summary of the autobuild process is here:
 665
 666   %s/autobuild.log
 667 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 668
 669     if options.restrict_tests:
 670         text += """
 671 The build was restricted to tests matching %s\n""" % options.restrict_tests
 672
 673     if failed_task != 'rebase':
 674         text += '''
 675 You can see logs of the failed task here:
 676
 677   %s/%s.stdout
 678   %s/%s.stderr
 679
 680 or you can get full logs of all tasks in this job here:
 681
 682   %s/logs.tar.gz
 683
 684 The top commit for the tree that was built was:
 685
 686 %s
 687
 688 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 689
 690     if add_log_tail:
 691         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 692         lines = f.readlines()
 693         log_tail = "".join(lines[-50:])
 694         num_lines = len(lines)
 695         if num_lines < 50:
 696             # Also include stderr (compile failures) if < 50 lines of stdout
 697             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 698             log_tail += "".join(f.readlines()[-(50-num_lines):])
 699
 700         text += '''
 701 The last 50 lines of log messages:
 702
 703 %s
 704     ''' % log_tail
 705         f.close()
 706
 707     logs = os.path.join(gitroot, 'logs.tar.gz')
 708     send_email('autobuild[%s] failure on %s for task %s during %s'
 709                % (options.branch, platform.node(), failed_task, failed_stage),
 710                text, logs)
 711
 712 def email_success(elapsed_time, log_base=None):
 713     '''send an email to options.email about a successful build'''
 714     user = os.getenv("USER")
 715     if log_base is None:
 716         log_base = gitroot
 717     text = '''
 718 Dear Developer,
 719
 720 Your autobuild on %s has succeeded after %.1f minutes.
 721
 722 ''' % (platform.node(), elapsed_time / 60.)
 723
 724     if options.restrict_tests:
 725         text += """
 726 The build was restricted to tests matching %s\n""" % options.restrict_tests
 727
 728     if options.keeplogs:
 729         text += '''
 730
 731 you can get full logs of all tasks in this job here:
 732
 733   %s/logs.tar.gz
 734
 735 ''' % log_base
 736
 737     text += '''
 738 The top commit for the tree that was built was:
 739
 740 %s
 741 ''' % top_commit_msg
 742
 743     logs = os.path.join(gitroot, 'logs.tar.gz')
 744     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 745                text, logs)
 746
 747
 748 (options, args) = parser.parse_args()
 749
 750 if options.retry:
 751     if options.rebase is None:
 752         raise Exception('You can only use --retry if you also rebase')
 753
 754 testbase = "%s/b%u" % (options.testbase, os.getpid())
 755 test_master = "%s/master" % testbase
 756 test_prefix = "%s/prefix" % testbase
 757 test_tmpdir = "%s/tmp" % testbase
 758 os.environ['TMPDIR'] = test_tmpdir
 759
 760 # get the top commit message, for emails
 761 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 762
 763 try:
 764     os.makedirs(testbase)
 765 except Exception as reason:
 766     raise Exception("Unable to create %s : %s" % (testbase, reason))
 767 cleanup_list.append(testbase)
 768
 769 if options.daemon:
 770     logfile = os.path.join(testbase, "log")
 771     do_print("Forking into the background, writing progress to %s" % logfile)
 772     daemonize(logfile)
 773
 774 write_pidfile(gitroot + "/autobuild.pid")
 775
 776 start_time = time.time()
 777
 778 while True:
 779     try:
 780         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 781         os.makedirs(test_tmpdir)
 782         # The waf uninstall code removes empty directories all the way
 783         # up the tree.  Creating a file in test_tmpdir stops it from
 784         # being removed.
 785         run_cmd("touch %s" % os.path.join(test_tmpdir,
 786                                           ".directory-is-not-empty"), show=True)
 787         run_cmd("stat %s" % test_tmpdir, show=True)
 788         run_cmd("stat %s" % testbase, show=True)
 789         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 790     except Exception:
 791         cleanup()
 792         raise
 793
 794     try:
 795         try:
 796             if options.rebase is not None:
 797                 rebase_tree(options.rebase, rebase_branch=options.branch)
 798         except Exception:
 799             cleanup_list.append(gitroot + "/autobuild.pid")
 800             cleanup()
 801             elapsed_time = time.time() - start_time
 802             email_failure(-1, 'rebase', 'rebase', 'rebase',
 803                           'rebase on %s failed' % options.branch,
 804                           elapsed_time, log_base=options.log_base)
 805             sys.exit(1)
 806         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 807         if options.tail:
 808             blist.start_tail()
 809         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 810         if status != 0 or errstr != "retry":
 811             break
 812         cleanup()
 813     except Exception:
 814         cleanup()
 815         raise
 816
 817 cleanup_list.append(gitroot + "/autobuild.pid")
 818
 819 do_print(errstr)
 820
 821 blist.kill_kids()
 822 if options.tail:
 823     do_print("waiting for tail to flush")
 824     time.sleep(1)
 825
 826 elapsed_time = time.time() - start_time
 827 if status == 0:
 828     if options.passcmd is not None:
 829         do_print("Running passcmd: %s" % options.passcmd)
 830         run_cmd(options.passcmd, dir=test_master)
 831     if options.pushto is not None:
 832         push_to(options.pushto, push_branch=options.branch)
 833     if options.keeplogs or options.attach_logs:
 834         blist.tarlogs("logs.tar.gz")
 835         do_print("Logs in logs.tar.gz")
 836     if options.always_email:
 837         email_success(elapsed_time, log_base=options.log_base)
 838     blist.remove_logs()
 839     cleanup()
 840     do_print(errstr)
 841     sys.exit(0)
 842
 843 # something failed, gather a tar of the logs
 844 blist.tarlogs("logs.tar.gz")
 845
 846 if options.email is not None:
 847     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 848                   elapsed_time, log_base=options.log_base)
 849 else:
 850     elapsed_minutes = elapsed_time / 60.0
 851     print('''
 852
 853 ####################################################################
 854
 855 AUTOBUILD FAILURE
 856
 857 Your autobuild[%s] on %s failed after %.1f minutes
 858 when trying to test %s with the following error:
 859
 860    %s
 861
 862 the autobuild has been abandoned. Please fix the error and resubmit.
 863
 864 ####################################################################
 865
 866 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 867
 868 cleanup()
 869 do_print(errstr)
 870 do_print("Logs in logs.tar.gz")
 871 sys.exit(status)