script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call,Popen, PIPE
   8 import os, tarfile, sys, time
   9 from optparse import OptionParser
  10 import smtplib
  11 import email
  12 from email.mime.text import MIMEText
  13 from email.mime.base import MIMEBase
  14 from email.mime.application import MIMEApplication
  15 from email.mime.multipart import MIMEMultipart
  16 from distutils.sysconfig import get_python_lib
  17 import platform
  18
  19 os.environ["PYTHONUNBUFFERED"] = "1"
  20
  21 # This speeds up testing remarkably.
  22 os.environ['TDB_NO_FSYNC'] = '1'
  23
  24 cleanup_list = []
  25
  26 builddirs = {
  27     "ctdb"    : "ctdb",
  28     "samba"  : ".",
  29     "samba-nt4"  : ".",
  30     "samba-xc" : ".",
  31     "samba-o3" : ".",
  32     "samba-ctdb" : ".",
  33     "samba-libs"  : ".",
  34     "samba-static"  : ".",
  35     "samba-test-only"  : ".",
  36     "samba-none-env"  : ".",
  37     "samba-systemkrb5"  : ".",
  38     "samba-nopython"  : ".",
  39     "ldb"     : "lib/ldb",
  40     "tdb"     : "lib/tdb",
  41     "talloc"  : "lib/talloc",
  42     "replace" : "lib/replace",
  43     "tevent"  : "lib/tevent",
  44     "pidl"    : "pidl",
  45     "pass"    : ".",
  46     "fail"    : ".",
  47     "retry"   : "."
  48     }
  49
  50 defaulttasks = [ "ctdb",
  51                  "samba",
  52                  "samba-nt4",
  53                  "samba-xc",
  54                  "samba-o3",
  55                  "samba-ctdb",
  56                  "samba-libs",
  57                  "samba-static",
  58                  "samba-none-env",
  59                  "samba-systemkrb5",
  60                  "samba-nopython",
  61                  "ldb",
  62                  "tdb",
  63                  "talloc",
  64                  "replace",
  65                  "tevent",
  66                  "pidl" ]
  67
  68 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  69     defaulttasks.remove("samba-o3")
  70
  71 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  72 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  73
  74 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  75 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  76 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  77 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX} ${EXTRA_PYTHON}"
  78 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE"
  79 samba_libs_configure_samba = samba_libs_configure_base + " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
  80
  81 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  82     extra_python = ""
  83 else:
  84     extra_python = "--extra-python=/usr/bin/python3"
  85
  86 tasks = {
  87     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  88                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  89                ("make", "make all", "text/plain"),
  90                ("install", "make install", "text/plain"),
  91                ("test", "make autotest", "text/plain"),
  92                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  93                ("clean", "make clean", "text/plain") ],
  94
  95     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
  96     "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  97                 ("make", "make -j", "text/plain"),
  98                 ("test", "make test FAIL_IMMEDIATELY=1 "
  99                  "TESTS='--exclude-env=none "
 100                  "--exclude-env=nt4_dc "
 101                  "--exclude-env=nt4_member'", "text/plain"),
 102                 ("install", "make install", "text/plain"),
 103                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 104                 ("clean", "make clean", "text/plain") ],
 105
 106     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 107     "samba-nt4" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 108                        ("make", "make -j", "text/plain"),
 109                        ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 110                        ("install", "make install", "text/plain"),
 111                        ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 112                        ("clean", "make clean", "text/plain") ],
 113
 114     "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 115                           ("make", "make -j", "text/plain"),
 116                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
 117
 118     # Test cross-compile infrastructure
 119     "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 120                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 121                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 122                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 123                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 124                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 125
 126     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 127     "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 128                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 129                    ("make", "make -j", "text/plain"),
 130                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 131                    ("install", "make install", "text/plain"),
 132                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 133                    ("clean", "make clean", "text/plain") ],
 134
 135     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 136
 137                      # make sure we have tdb around:
 138                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 139                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 140                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 141
 142
 143                      # build samba with cluster support (also building ctdb):
 144                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 145                      ("samba-make", "make", "text/plain"),
 146                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 147                      ("samba-install", "make install", "text/plain"),
 148                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 149
 150                      # clean up:
 151                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 152                      ("clean", "make clean", "text/plain"),
 153                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
 154
 155     "samba-libs" : [
 156                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 157                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 158                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 159                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 160
 161                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 162                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 163                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 164
 165                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 166                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 167                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 168
 169                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 170                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 171                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 172
 173                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 174                       ("nondevel-make", "make -j", "text/plain"),
 175                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 176                       ("nondevel-install", "make install", "text/plain"),
 177                       ("nondevel-dist", "make dist", "text/plain"),
 178
 179                       # retry with all modules shared
 180                       ("allshared-distclean", "make distclean", "text/plain"),
 181                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 182                       ("allshared-make", "make -j", "text/plain")],
 183
 184     "samba-none-env" : [
 185                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 186                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 187                       ("make", "make -j", "text/plain"),
 188                       ("test", "make test "
 189                        "FAIL_IMMEDIATELY=1 "
 190                        "TESTS='--include-env=none'",
 191                        "text/plain")],
 192
 193     "samba-static" : [
 194                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 195                       # build with all modules static
 196                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 197                       ("allstatic-make", "make -j", "text/plain"),
 198
 199                       # retry without any required modules
 200                       ("none-distclean", "make distclean", "text/plain"),
 201                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 202                       ("none-make", "make -j", "text/plain"),
 203
 204                       # retry with nonshared smbd and smbtorture
 205                       ("nonshared-distclean", "make distclean", "text/plain"),
 206                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 207                       ("nonshared-make", "make -j", "text/plain")],
 208
 209     "samba-systemkrb5" : [
 210                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 211                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 212                       ("make", "make -j", "text/plain"),
 213                       # we currently cannot run a full make test, a limited list of tests could be run
 214                       # via "make test TESTS=sometests"
 215                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 216                       ("install", "make install", "text/plain"),
 217                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 218                       ("clean", "make clean", "text/plain")
 219                       ],
 220
 221     # Test Samba without python still builds.  When this test fails
 222     # due to more use of Python, the expectations is that the newly
 223     # failing part of the code should be disabled when
 224     # --disable-python is set (rather than major work being done to
 225     # support this environment).  The target here is for vendors
 226     # shipping a minimal smbd.
 227     "samba-nopython" : [
 228                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 229                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 230                       ("make", "make -j", "text/plain"),
 231                       ("install", "make install", "text/plain"),
 232                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 233                       ("clean", "make clean", "text/plain")
 234                       ],
 235
 236
 237
 238     "ldb" : [
 239               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 240               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 241               ("make", "make", "text/plain"),
 242               ("install", "make install", "text/plain"),
 243               ("test", "make test", "text/plain"),
 244               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 245               ("distcheck", "make distcheck", "text/plain"),
 246               ("clean", "make clean", "text/plain") ],
 247
 248     "tdb" : [
 249               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 250               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 251               ("make", "make", "text/plain"),
 252               ("install", "make install", "text/plain"),
 253               ("test", "make test", "text/plain"),
 254               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 255               ("distcheck", "make distcheck", "text/plain"),
 256               ("clean", "make clean", "text/plain") ],
 257
 258     "talloc" : [
 259                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 260                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 261                  ("make", "make", "text/plain"),
 262                  ("install", "make install", "text/plain"),
 263                  ("test", "make test", "text/plain"),
 264                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 265                  ("distcheck", "make distcheck", "text/plain"),
 266                  ("clean", "make clean", "text/plain") ],
 267
 268     "replace" : [
 269                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 270                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 271                   ("make", "make", "text/plain"),
 272                   ("install", "make install", "text/plain"),
 273                   ("test", "make test", "text/plain"),
 274                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 275                   ("distcheck", "make distcheck", "text/plain"),
 276                   ("clean", "make clean", "text/plain") ],
 277
 278     "tevent" : [
 279                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 280                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 281                  ("make", "make", "text/plain"),
 282                  ("install", "make install", "text/plain"),
 283                  ("test", "make test", "text/plain"),
 284                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 285                  ("distcheck", "make distcheck", "text/plain"),
 286                  ("clean", "make clean", "text/plain") ],
 287
 288     "pidl" : [
 289                ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 290                ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 291                ("touch", "touch *.yp", "text/plain"),
 292                ("make", "make", "text/plain"),
 293                ("test", "make test", "text/plain"),
 294                ("install", "make install", "text/plain"),
 295                ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 296                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 297                ("clean", "make clean", "text/plain") ],
 298
 299     # these are useful for debugging autobuild
 300     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
 301     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
 302 }
 303
 304 def do_print(msg):
 305     print("%s" % msg)
 306     sys.stdout.flush()
 307     sys.stderr.flush()
 308
 309 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 310     if show is None:
 311         show = options.verbose
 312     if show:
 313         do_print("Running: '%s' in '%s'" % (cmd, dir))
 314     if output:
 315         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 316     elif checkfail:
 317         return check_call(cmd, shell=True, cwd=dir)
 318     else:
 319         return call(cmd, shell=True, cwd=dir)
 320
 321
 322 class builder(object):
 323     '''handle build of one directory'''
 324
 325     def __init__(self, name, sequence, cp=True):
 326         self.name = name
 327         self.dir = builddirs[name]
 328
 329         self.tag = self.name.replace('/', '_')
 330         self.sequence = sequence
 331         self.next = 0
 332         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 333         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 334         if options.verbose:
 335             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 336             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 337         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 338         self.stdout = open(self.stdout_path, 'w')
 339         self.stderr = open(self.stderr_path, 'w')
 340         self.stdin  = open("/dev/null", 'r')
 341         self.sdir = "%s/%s" % (testbase, self.tag)
 342         self.prefix = "%s/%s" % (test_prefix, self.tag)
 343         run_cmd("rm -rf %s" % self.sdir)
 344         run_cmd("rm -rf %s" % self.prefix)
 345         if cp:
 346             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 347         else:
 348             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 349         self.start_next()
 350
 351     def start_next(self):
 352         if self.next == len(self.sequence):
 353             if not options.nocleanup:
 354                 run_cmd("rm -rf %s" % self.sdir)
 355                 run_cmd("rm -rf %s" % self.prefix)
 356             do_print('%s: Completed OK' % self.name)
 357             self.done = True
 358             return
 359         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 360         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 361         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 362         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 363         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 364         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 365 #        if self.output_mime_type == "text/x-subunit":
 366 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 367         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 368         cwd = os.getcwd()
 369         os.chdir("%s/%s" % (self.sdir, self.dir))
 370         self.proc = Popen(self.cmd, shell=True,
 371                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 372         os.chdir(cwd)
 373         self.next += 1
 374
 375
 376 class buildlist(object):
 377     '''handle build of multiple directories'''
 378
 379     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 380         global tasks
 381         self.tlist = []
 382         self.tail_proc = None
 383         self.retry = None
 384         if tasknames == []:
 385             if options.restrict_tests:
 386                 tasknames = ["samba-test-only"]
 387             else:
 388                 tasknames = defaulttasks
 389         else:
 390             # If we are only running one test,
 391             # do not sleep randomly to wait for it to start
 392             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 393
 394         for n in tasknames:
 395             b = builder(n, tasks[n], cp=n is not "pidl")
 396             self.tlist.append(b)
 397         if options.retry:
 398             rebase_remote = "rebaseon"
 399             retry_task = [ ("retry",
 400                             '''set -e
 401                             git remote add -t %s %s %s
 402                             git fetch %s
 403                             while :; do
 404                               sleep 60
 405                               git describe %s/%s > old_remote_branch.desc
 406                               git fetch %s
 407                               git describe %s/%s > remote_branch.desc
 408                               diff old_remote_branch.desc remote_branch.desc
 409                             done
 410                            ''' % (
 411                                rebase_branch, rebase_remote, rebase_url,
 412                                rebase_remote,
 413                                rebase_remote, rebase_branch,
 414                                rebase_remote,
 415                                rebase_remote, rebase_branch
 416                            ),
 417                            "test/plain" ) ]
 418
 419             self.retry = builder('retry', retry_task, cp=False)
 420             self.need_retry = False
 421
 422     def kill_kids(self):
 423         if self.tail_proc is not None:
 424             self.tail_proc.terminate()
 425             self.tail_proc.wait()
 426             self.tail_proc = None
 427         if self.retry is not None:
 428             self.retry.proc.terminate()
 429             self.retry.proc.wait()
 430             self.retry = None
 431         for b in self.tlist:
 432             if b.proc is not None:
 433                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 434                 b.proc.terminate()
 435                 b.proc.wait()
 436                 b.proc = None
 437
 438     def wait_one(self):
 439         while True:
 440             none_running = True
 441             for b in self.tlist:
 442                 if b.proc is None:
 443                     continue
 444                 none_running = False
 445                 b.status = b.proc.poll()
 446                 if b.status is None:
 447                     continue
 448                 b.proc = None
 449                 return b
 450             if options.retry:
 451                 ret = self.retry.proc.poll()
 452                 if ret is not None:
 453                     self.need_retry = True
 454                     self.retry = None
 455                     return None
 456             if none_running:
 457                 return None
 458             time.sleep(0.1)
 459
 460     def run(self):
 461         while True:
 462             b = self.wait_one()
 463             if options.retry and self.need_retry:
 464                 self.kill_kids()
 465                 do_print("retry needed")
 466                 return (0, None, None, None, "retry")
 467             if b is None:
 468                 break
 469             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 470                 self.kill_kids()
 471                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 472             b.start_next()
 473         self.kill_kids()
 474         return (0, None, None, None, "All OK")
 475
 476     def write_system_info(self):
 477         filename = 'system-info.txt'
 478         f = open(filename, 'w')
 479         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo']:
 480             print('### %s' % cmd, file=f)
 481             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 482             print(file=f)
 483         f.close()
 484         return filename
 485
 486     def tarlogs(self, fname):
 487         tar = tarfile.open(fname, "w:gz")
 488         for b in self.tlist:
 489             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 490             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 491         if os.path.exists("autobuild.log"):
 492             tar.add("autobuild.log")
 493         sys_info = self.write_system_info()
 494         tar.add(sys_info)
 495         tar.close()
 496
 497     def remove_logs(self):
 498         for b in self.tlist:
 499             os.unlink(b.stdout_path)
 500             os.unlink(b.stderr_path)
 501
 502     def start_tail(self):
 503         cwd = os.getcwd()
 504         cmd = "tail -f *.stdout *.stderr"
 505         os.chdir(gitroot)
 506         self.tail_proc = Popen(cmd, shell=True)
 507         os.chdir(cwd)
 508
 509
 510 def cleanup():
 511     if options.nocleanup:
 512         return
 513     run_cmd("stat %s || true" % test_tmpdir, show=True)
 514     run_cmd("stat %s" % testbase, show=True)
 515     do_print("Cleaning up ....")
 516     for d in cleanup_list:
 517         run_cmd("rm -rf %s" % d)
 518
 519
 520 def find_git_root():
 521     '''get to the top of the git repo'''
 522     p=os.getcwd()
 523     while p != '/':
 524         if os.path.isdir(os.path.join(p, ".git")):
 525             return p
 526         p = os.path.abspath(os.path.join(p, '..'))
 527     return None
 528
 529
 530 def daemonize(logfile):
 531     pid = os.fork()
 532     if pid == 0: # Parent
 533         os.setsid()
 534         pid = os.fork()
 535         if pid != 0: # Actual daemon
 536             os._exit(0)
 537     else: # Grandparent
 538         os._exit(0)
 539
 540     import resource      # Resource usage information.
 541     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 542     if maxfd == resource.RLIM_INFINITY:
 543         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
 544     for fd in range(0, maxfd):
 545         try:
 546             os.close(fd)
 547         except OSError:
 548             pass
 549     os.open(logfile, os.O_RDWR | os.O_CREAT)
 550     os.dup2(0, 1)
 551     os.dup2(0, 2)
 552
 553 def write_pidfile(fname):
 554     '''write a pid file, cleanup on exit'''
 555     f = open(fname, mode='w')
 556     f.write("%u\n" % os.getpid())
 557     f.close()
 558
 559
 560 def rebase_tree(rebase_url, rebase_branch = "master"):
 561     rebase_remote = "rebaseon"
 562     do_print("Rebasing on %s" % rebase_url)
 563     run_cmd("git describe HEAD", show=True, dir=test_master)
 564     run_cmd("git remote add -t %s %s %s" %
 565             (rebase_branch, rebase_remote, rebase_url),
 566             show=True, dir=test_master)
 567     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 568     if options.fix_whitespace:
 569         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 570                 (rebase_remote, rebase_branch),
 571                 show=True, dir=test_master)
 572     else:
 573         run_cmd("git rebase --force-rebase %s/%s" %
 574                 (rebase_remote, rebase_branch),
 575                 show=True, dir=test_master)
 576     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 577                    (rebase_remote, rebase_branch),
 578                    dir=test_master, output=True)
 579     if diff == '':
 580         do_print("No differences between HEAD and %s/%s - exiting" %
 581               (rebase_remote, rebase_branch))
 582         sys.exit(0)
 583     run_cmd("git describe %s/%s" %
 584             (rebase_remote, rebase_branch),
 585             show=True, dir=test_master)
 586     run_cmd("git describe HEAD", show=True, dir=test_master)
 587     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 588             (rebase_remote, rebase_branch),
 589             show=True, dir=test_master)
 590
 591 def push_to(push_url, push_branch = "master"):
 592     push_remote = "pushto"
 593     do_print("Pushing to %s" % push_url)
 594     if options.mark:
 595         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 596         run_cmd("git commit --amend -c HEAD", dir=test_master)
 597         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 598         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 599     run_cmd("git remote add -t %s %s %s" %
 600             (push_branch, push_remote, push_url),
 601             show=True, dir=test_master)
 602     run_cmd("git push %s +HEAD:%s" %
 603             (push_remote, push_branch),
 604             show=True, dir=test_master)
 605
 606 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 607
 608 gitroot = find_git_root()
 609 if gitroot is None:
 610     raise Exception("Failed to find git root")
 611
 612 parser = OptionParser()
 613 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 614 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 615 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 616 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 617                   default=def_testbase)
 618 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 619 parser.add_option("", "--verbose", help="show all commands as they are run",
 620                   default=False, action="store_true")
 621 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 622                   default=None, type='str')
 623 parser.add_option("", "--pushto", help="push to a git url on success",
 624                   default=None, type='str')
 625 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 626                   default=False, action="store_true")
 627 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 628                   default=False, action="store_true")
 629 parser.add_option("", "--retry", help="automatically retry if master changes",
 630                   default=False, action="store_true")
 631 parser.add_option("", "--email", help="send email to the given address on failure",
 632                   type='str', default=None)
 633 parser.add_option("", "--email-from", help="send email from the given address",
 634                   type='str', default="autobuild@samba.org")
 635 parser.add_option("", "--email-server", help="send email via the given server",
 636                   type='str', default='localhost')
 637 parser.add_option("", "--always-email", help="always send email, even on success",
 638                   action="store_true")
 639 parser.add_option("", "--daemon", help="daemonize after initial setup",
 640                   action="store_true")
 641 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 642                   default="master", type='str')
 643 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 644                   default=gitroot, type='str')
 645 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 646                   default=False, action="store_true")
 647 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 648                   default='')
 649
 650 def send_email(subject, text, log_tar):
 651     outer = MIMEMultipart()
 652     outer['Subject'] = subject
 653     outer['To'] = options.email
 654     outer['From'] = options.email_from
 655     outer['Date'] = email.utils.formatdate(localtime = True)
 656     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 657     outer.attach(MIMEText(text, 'plain'))
 658     if options.attach_logs:
 659         fp = open(log_tar, 'rb')
 660         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 661         fp.close()
 662         # Set the filename parameter
 663         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 664         outer.attach(msg)
 665     content = outer.as_string()
 666     s = smtplib.SMTP(options.email_server)
 667     s.sendmail(options.email_from, [options.email], content)
 668     s.set_debuglevel(1)
 669     s.quit()
 670
 671 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 672                   elapsed_time, log_base=None, add_log_tail=True):
 673     '''send an email to options.email about the failure'''
 674     elapsed_minutes = elapsed_time / 60.0
 675     user = os.getenv("USER")
 676     if log_base is None:
 677         log_base = gitroot
 678     text = '''
 679 Dear Developer,
 680
 681 Your autobuild on %s failed after %.1f minutes
 682 when trying to test %s with the following error:
 683
 684    %s
 685
 686 the autobuild has been abandoned. Please fix the error and resubmit.
 687
 688 A summary of the autobuild process is here:
 689
 690   %s/autobuild.log
 691 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 692
 693     if options.restrict_tests:
 694         text += """
 695 The build was restricted to tests matching %s\n""" % options.restrict_tests
 696
 697     if failed_task != 'rebase':
 698         text += '''
 699 You can see logs of the failed task here:
 700
 701   %s/%s.stdout
 702   %s/%s.stderr
 703
 704 or you can get full logs of all tasks in this job here:
 705
 706   %s/logs.tar.gz
 707
 708 The top commit for the tree that was built was:
 709
 710 %s
 711
 712 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 713
 714     if add_log_tail:
 715         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 716         lines = f.readlines()
 717         log_tail = "".join(lines[-50:])
 718         num_lines = len(lines)
 719         if num_lines < 50:
 720             # Also include stderr (compile failures) if < 50 lines of stdout
 721             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 722             log_tail += "".join(f.readlines()[-(50-num_lines):])
 723
 724         text += '''
 725 The last 50 lines of log messages:
 726
 727 %s
 728     ''' % log_tail
 729         f.close()
 730
 731     logs = os.path.join(gitroot, 'logs.tar.gz')
 732     send_email('autobuild[%s] failure on %s for task %s during %s'
 733                % (options.branch, platform.node(), failed_task, failed_stage),
 734                text, logs)
 735
 736 def email_success(elapsed_time, log_base=None):
 737     '''send an email to options.email about a successful build'''
 738     user = os.getenv("USER")
 739     if log_base is None:
 740         log_base = gitroot
 741     text = '''
 742 Dear Developer,
 743
 744 Your autobuild on %s has succeeded after %.1f minutes.
 745
 746 ''' % (platform.node(), elapsed_time / 60.)
 747
 748     if options.restrict_tests:
 749         text += """
 750 The build was restricted to tests matching %s\n""" % options.restrict_tests
 751
 752     if options.keeplogs:
 753         text += '''
 754
 755 you can get full logs of all tasks in this job here:
 756
 757   %s/logs.tar.gz
 758
 759 ''' % log_base
 760
 761     text += '''
 762 The top commit for the tree that was built was:
 763
 764 %s
 765 ''' % top_commit_msg
 766
 767     logs = os.path.join(gitroot, 'logs.tar.gz')
 768     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 769                text, logs)
 770
 771
 772 (options, args) = parser.parse_args()
 773
 774 if options.retry:
 775     if options.rebase is None:
 776         raise Exception('You can only use --retry if you also rebase')
 777
 778 testbase = "%s/b%u" % (options.testbase, os.getpid())
 779 test_master = "%s/master" % testbase
 780 test_prefix = "%s/prefix" % testbase
 781 test_tmpdir = "%s/tmp" % testbase
 782 os.environ['TMPDIR'] = test_tmpdir
 783
 784 # get the top commit message, for emails
 785 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 786
 787 try:
 788     os.makedirs(testbase)
 789 except Exception as reason:
 790     raise Exception("Unable to create %s : %s" % (testbase, reason))
 791 cleanup_list.append(testbase)
 792
 793 if options.daemon:
 794     logfile = os.path.join(testbase, "log")
 795     do_print("Forking into the background, writing progress to %s" % logfile)
 796     daemonize(logfile)
 797
 798 write_pidfile(gitroot + "/autobuild.pid")
 799
 800 start_time = time.time()
 801
 802 while True:
 803     try:
 804         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 805         os.makedirs(test_tmpdir)
 806         # The waf uninstall code removes empty directories all the way
 807         # up the tree.  Creating a file in test_tmpdir stops it from
 808         # being removed.
 809         run_cmd("touch %s" % os.path.join(test_tmpdir,
 810                                           ".directory-is-not-empty"), show=True)
 811         run_cmd("stat %s" % test_tmpdir, show=True)
 812         run_cmd("stat %s" % testbase, show=True)
 813         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 814     except Exception:
 815         cleanup()
 816         raise
 817
 818     try:
 819         try:
 820             if options.rebase is not None:
 821                 rebase_tree(options.rebase, rebase_branch=options.branch)
 822         except Exception:
 823             cleanup_list.append(gitroot + "/autobuild.pid")
 824             cleanup()
 825             elapsed_time = time.time() - start_time
 826             email_failure(-1, 'rebase', 'rebase', 'rebase',
 827                           'rebase on %s failed' % options.branch,
 828                           elapsed_time, log_base=options.log_base)
 829             sys.exit(1)
 830         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 831         if options.tail:
 832             blist.start_tail()
 833         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 834         if status != 0 or errstr != "retry":
 835             break
 836         cleanup()
 837     except Exception:
 838         cleanup()
 839         raise
 840
 841 cleanup_list.append(gitroot + "/autobuild.pid")
 842
 843 do_print(errstr)
 844
 845 blist.kill_kids()
 846 if options.tail:
 847     do_print("waiting for tail to flush")
 848     time.sleep(1)
 849
 850 elapsed_time = time.time() - start_time
 851 if status == 0:
 852     if options.passcmd is not None:
 853         do_print("Running passcmd: %s" % options.passcmd)
 854         run_cmd(options.passcmd, dir=test_master)
 855     if options.pushto is not None:
 856         push_to(options.pushto, push_branch=options.branch)
 857     if options.keeplogs or options.attach_logs:
 858         blist.tarlogs("logs.tar.gz")
 859         do_print("Logs in logs.tar.gz")
 860     if options.always_email:
 861         email_success(elapsed_time, log_base=options.log_base)
 862     blist.remove_logs()
 863     cleanup()
 864     do_print(errstr)
 865     sys.exit(0)
 866
 867 # something failed, gather a tar of the logs
 868 blist.tarlogs("logs.tar.gz")
 869
 870 if options.email is not None:
 871     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 872                   elapsed_time, log_base=options.log_base)
 873 else:
 874     elapsed_minutes = elapsed_time / 60.0
 875     print('''
 876
 877 ####################################################################
 878
 879 AUTOBUILD FAILURE
 880
 881 Your autobuild[%s] on %s failed after %.1f minutes
 882 when trying to test %s with the following error:
 883
 884    %s
 885
 886 the autobuild has been abandoned. Please fix the error and resubmit.
 887
 888 ####################################################################
 889
 890 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 891
 892 cleanup()
 893 do_print(errstr)
 894 do_print("Logs in logs.tar.gz")
 895 sys.exit(status)