script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.isdir(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--passcmd", help="command to run on success", default=None)
  60 parser.add_option("--verbose", help="show all commands as they are run",
  61                   default=False, action="store_true")
  62 parser.add_option("--rebase", help="rebase on the given tree before testing",
  63                   default=None, type='str')
  64 parser.add_option("--pushto", help="push to a git url on success",
  65                   default=None, type='str')
  66 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  67                   default=False, action="store_true")
  68 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  69                   default=False, action="store_true")
  70 parser.add_option("--retry", help="automatically retry if master changes",
  71                   default=False, action="store_true")
  72 parser.add_option("--email", help="send email to the given address on failure",
  73                   type='str', default=None)
  74 parser.add_option("--email-from", help="send email from the given address",
  75                   type='str', default="autobuild@samba.org")
  76 parser.add_option("--email-server", help="send email via the given server",
  77                   type='str', default='localhost')
  78 parser.add_option("--always-email", help="always send email, even on success",
  79                   action="store_true")
  80 parser.add_option("--daemon", help="daemonize after initial setup",
  81                   action="store_true")
  82 parser.add_option("--branch", help="the branch to work on (default=master)",
  83                   default="master", type='str')
  84 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  85                   default=gitroot, type='str')
  86 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  87                   default=False, action="store_true")
  88 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  89                   default='')
  90 parser.add_option("--enable-coverage", dest='enable_coverage',
  91                   action="store_const", const='--enable-coverage', default='',
  92                   help="Add --enable-coverage option while configure")
  93
  94 (options, args) = parser.parse_args()
  95
  96 if options.retry:
  97     if options.rebase is None:
  98         raise Exception('You can only use --retry if you also rebase')
  99
 100 testbase = "%s/b%u" % (options.testbase, os.getpid())
 101 test_master = "%s/master" % testbase
 102 test_prefix = "%s/prefix" % testbase
 103 test_tmpdir = "%s/tmp" % testbase
 104 os.environ['TMPDIR'] = test_tmpdir
 105
 106 if options.enable_coverage:
 107     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 108 else:
 109     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 110
 111 if args:
 112     # If we are only running specific test,
 113     # do not sleep randomly to wait for it to start
 114     def random_sleep(low, high):
 115         return 'sleep 1'
 116 else:
 117     def random_sleep(low, high):
 118         return 'sleep {}'.format(random.randint(low, high))
 119
 120 cleanup_list = []
 121
 122 builddirs = {
 123     "ctdb": "ctdb",
 124     "samba": ".",
 125     "samba-nt4": ".",
 126     "samba-fileserver": ".",
 127     "samba-ad-member": ".",
 128     "samba-xc": ".",
 129     "samba-o3": ".",
 130     "samba-ctdb": ".",
 131     "samba-libs": ".",
 132     "samba-static": ".",
 133     "samba-none-env": ".",
 134     "samba-ad-dc-1": ".",
 135     "samba-ad-dc-2": ".",
 136     "samba-ad-dc-3": ".",
 137     "samba-ad-dc-4": ".",
 138     "samba-ad-dc-5": ".",
 139     "samba-ad-dc-6": ".",
 140     "samba-ad-dc-ntvfs": ".",
 141     "samba-ad-dc-backup": ".",
 142     "samba-systemkrb5": ".",
 143     "samba-nopython": ".",
 144     "samba-nopython-py2": ".",
 145     "samba-schemaupgrade": ".",
 146     "ldb": "lib/ldb",
 147     "tdb": "lib/tdb",
 148     "talloc": "lib/talloc",
 149     "replace": "lib/replace",
 150     "tevent": "lib/tevent",
 151     "pidl": "pidl"
 152 }
 153
 154 defaulttasks = list(builddirs.keys())
 155
 156 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 157     defaulttasks.remove("samba-o3")
 158
 159 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
 160 samba_configure_params = " ${ENABLE_COVERAGE} --picky-developer ${PREFIX} --with-profiling-data"
 161
 162 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 163 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 164 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 165 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug --picky-developer -C ${PREFIX}"
 166 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 167 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 168 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 169
 170
 171 def format_option(name, value=None):
 172     """Format option as str list."""
 173     if value is None:  # boolean option
 174         return [name]
 175     if not isinstance(value, list):  # single value option
 176         value = [value]
 177     # repeatable option
 178     return ['{}={}'.format(name, item) for item in value]
 179
 180
 181 def make_test(
 182         cmd='make test',
 183         FAIL_IMMEDIATELY=1,
 184         TESTS='',
 185         include_envs=None,
 186         exclude_envs=None):
 187
 188     test_options = []
 189     if include_envs:
 190         test_options = format_option('--include-env', include_envs)
 191     if exclude_envs:
 192         test_options = format_option('--exclude-env', exclude_envs)
 193     if test_options:
 194         # join envs options to original test options
 195         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 196
 197     _options = []
 198     if FAIL_IMMEDIATELY:
 199         _options.append('FAIL_IMMEDIATELY=1')
 200     if TESTS:
 201         _options.append("TESTS='{}'".format(TESTS))
 202
 203     return ' '.join([cmd] + _options)
 204
 205
 206 tasks = {
 207     "ctdb": [
 208         ("random-sleep", random_sleep(300, 900)),
 209         ("configure", "./configure " + ctdb_configure_params),
 210         ("make", "make all"),
 211         ("install", "make install"),
 212         ("test", "make autotest"),
 213         ("check-clean-tree", "../script/clean-source-tree.sh"),
 214         ("clean", "make clean"),
 215         ],
 216
 217     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 218     "samba": [
 219         ("random-sleep", random_sleep(300, 900)),
 220         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 221         ("make", "make -j"),
 222         ("test", make_test(exclude_envs=[
 223             "none",
 224             "nt4_dc",
 225             "nt4_dc_schannel",
 226             "nt4_member",
 227             "ad_dc",
 228             "ad_dc_backup",
 229             "ad_dc_ntvfs",
 230             "ad_dc_default",
 231             "ad_dc_slowtests",
 232             "ad_dc_no_nss",
 233             "ad_dc_no_ntlm",
 234             "fl2003dc",
 235             "fl2008dc",
 236             "fl2008r2dc",
 237             "ad_member",
 238             "ad_member_idmap_rid",
 239             "ad_member_idmap_ad",
 240             "ad_member_rfc2307",
 241             "chgdcpass",
 242             "vampire_2000_dc",
 243             "fl2000dc",
 244             "fileserver",
 245             "maptoguest",
 246             "simpleserver",
 247             "backupfromdc",
 248             "restoredc",
 249             "renamedc",
 250             "offlinebackupdc",
 251             "labdc",
 252             "preforkrestartdc",
 253             "proclimitdc",
 254             "promoted_dc",
 255             "vampire_dc",
 256             "rodc",
 257             "ad_dc_default",
 258             "ad_dc_slowtests",
 259             "schema_pair_dc",
 260             "schema_dc",
 261             ])),
 262         ("lcov", LCOV_CMD),
 263         ("install", "make install"),
 264         ("check-clean-tree", "script/clean-source-tree.sh"),
 265         ("clean", "make clean"),
 266         ],
 267
 268     "samba-nt4": [
 269         ("random-sleep", random_sleep(300, 900)),
 270         ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params),
 271         ("make", "make -j"),
 272         ("test", make_test(include_envs=[
 273             "nt4_dc",
 274             "nt4_dc_schannel",
 275             "nt4_member",
 276             ])),
 277         ("lcov", LCOV_CMD),
 278         ("install", "make install"),
 279         ("check-clean-tree", "script/clean-source-tree.sh"),
 280         ("clean", "make clean"),
 281         ],
 282
 283     "samba-fileserver": [
 284         ("random-sleep", random_sleep(300, 900)),
 285         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
 286         ("make", "make -j"),
 287         ("test", make_test(include_envs=[
 288             "fileserver",
 289             "maptoguest",
 290             "simpleserver",
 291             ])),
 292         ("lcov", LCOV_CMD),
 293         ("check-clean-tree", "script/clean-source-tree.sh"),
 294         ],
 295
 296     "samba-ad-member": [
 297         ("random-sleep", random_sleep(300, 900)),
 298         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 299         ("make", "make -j"),
 300         ("test", make_test(include_envs=[
 301             "ad_member",
 302             "ad_member_idmap_rid",
 303             "ad_member_idmap_ad",
 304             "ad_member_rfc2307",
 305             ])),
 306         ("lcov", LCOV_CMD),
 307         ("check-clean-tree", "script/clean-source-tree.sh"),
 308         ],
 309
 310     "samba-ad-dc-1": [
 311         ("random-sleep", random_sleep(1, 1)),
 312         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 313         ("make", "make -j"),
 314         ("test", make_test(include_envs=[
 315             "ad_dc",
 316             "ad_dc_no_nss",
 317             "ad_dc_no_ntlm",
 318             ])),
 319         ("lcov", LCOV_CMD),
 320         ("check-clean-tree", "script/clean-source-tree.sh"),
 321         ],
 322
 323     "samba-ad-dc-2": [
 324         ("random-sleep", random_sleep(1, 1)),
 325         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 326         ("make", "make -j"),
 327         ("test", make_test(include_envs=[
 328             "vampire_dc",
 329             "vampire_2000_dc",
 330             "rodc",
 331             ])),
 332         ("lcov", LCOV_CMD),
 333         ("check-clean-tree", "script/clean-source-tree.sh"),
 334         ],
 335
 336     "samba-ad-dc-3": [
 337         ("random-sleep", random_sleep(1, 1)),
 338         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 339         ("make", "make -j"),
 340         ("test", make_test(include_envs=[
 341             "promoted_dc",
 342             "chgdcpass",
 343             "preforkrestartdc",
 344             "proclimitdc",
 345             ])),
 346         ("lcov", LCOV_CMD),
 347         ("check-clean-tree", "script/clean-source-tree.sh"),
 348         ],
 349
 350     "samba-ad-dc-4": [
 351         ("random-sleep", random_sleep(1, 1)),
 352         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 353         ("make", "make -j"),
 354         ("test", make_test(include_envs=[
 355             "fl2000dc",
 356             "fl2003dc",
 357             "fl2008dc",
 358             "fl2008r2dc",
 359             ])),
 360         ("lcov", LCOV_CMD),
 361         ("check-clean-tree", "script/clean-source-tree.sh"),
 362         ],
 363
 364     "samba-ad-dc-5": [
 365         ("random-sleep", random_sleep(1, 1)),
 366         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 367         ("make", "make -j"),
 368         ("test", make_test(include_envs=["ad_dc_default"])),
 369         ("lcov", LCOV_CMD),
 370         ("check-clean-tree", "script/clean-source-tree.sh"),
 371         ],
 372
 373     "samba-ad-dc-6": [
 374         ("random-sleep", random_sleep(1, 1)),
 375         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 376         ("make", "make -j"),
 377         ("test", make_test(include_envs=["ad_dc_slowtests"])),
 378         ("lcov", LCOV_CMD),
 379         ("check-clean-tree", "script/clean-source-tree.sh"),
 380         ],
 381
 382     "samba-schemaupgrade": [
 383         ("random-sleep", random_sleep(1, 1)),
 384         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 385         ("make", "make -j"),
 386         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 387         ("lcov", LCOV_CMD),
 388         ("check-clean-tree", "script/clean-source-tree.sh"),
 389         ],
 390
 391     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 392     # This is currently the longest task, so we don't randomly delay it.
 393     "samba-ad-dc-ntvfs": [
 394         ("random-sleep", random_sleep(1, 1)),
 395         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 396         ("make", "make -j"),
 397         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 398         ("lcov", LCOV_CMD),
 399         ("check-clean-tree", "script/clean-source-tree.sh"),
 400         ],
 401
 402     # run the backup/restore testenvs separately as they're fairly standalone
 403     # (and CI seems to max out at ~8 different DCs running at once)
 404     "samba-ad-dc-backup": [
 405         ("random-sleep", random_sleep(300, 900)),
 406         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 407         ("make", "make -j"),
 408         ("test", make_test(include_envs=[
 409             "backupfromdc",
 410             "restoredc",
 411             "renamedc",
 412             "offlinebackupdc",
 413             "labdc",
 414             "ad_dc_backup",
 415             ])),
 416         ("lcov", LCOV_CMD),
 417         ("check-clean-tree", "script/clean-source-tree.sh"),
 418         ],
 419
 420     "samba-test-only": [
 421         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
 422         ("make", "make -j"),
 423         ("test", make_test(TESTS="${TESTS}")),
 424         ("lcov", LCOV_CMD),
 425         ],
 426
 427     # Test cross-compile infrastructure
 428     "samba-xc": [
 429         ("random-sleep", random_sleep(900, 1500)),
 430         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 431         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 432             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 433         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 434             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 435         ("compare-results", "script/compare_cc_results.py "
 436             "./bin/c4che/default{} "
 437             "./bin-xe/c4che/default{} "
 438             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 439         ],
 440
 441     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 442     "samba-o3": [
 443         ("random-sleep", random_sleep(300, 900)),
 444         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
 445         ("make", "make -j"),
 446         ("test", make_test(cmd='make quicktest', include_envs=["ad_dc"])),
 447         ("lcov", LCOV_CMD),
 448         ("install", "make install"),
 449         ("check-clean-tree", "script/clean-source-tree.sh"),
 450         ("clean", "make clean"),
 451         ],
 452
 453     "samba-ctdb": [
 454         ("random-sleep", random_sleep(900, 1500)),
 455
 456         # make sure we have tdb around:
 457         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 458         ("tdb-make", "cd lib/tdb && make"),
 459         ("tdb-install", "cd lib/tdb && make install"),
 460
 461         # build samba with cluster support (also building ctdb):
 462         ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb"),
 463         ("samba-make", "make"),
 464         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 465         ("samba-install", "make install"),
 466         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 467
 468         # clean up:
 469         ("check-clean-tree", "script/clean-source-tree.sh"),
 470         ("clean", "make clean"),
 471         ("ctdb-clean", "cd ./ctdb && make clean"),
 472         ],
 473
 474     "samba-libs": [
 475         ("random-sleep", random_sleep(300, 900)),
 476         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 477         ("talloc-make", "cd lib/talloc && make"),
 478         ("talloc-install", "cd lib/talloc && make install"),
 479
 480         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 481         ("tdb-make", "cd lib/tdb && make"),
 482         ("tdb-install", "cd lib/tdb && make install"),
 483
 484         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 485         ("tevent-make", "cd lib/tevent && make"),
 486         ("tevent-install", "cd lib/tevent && make install"),
 487
 488         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 489         ("ldb-make", "cd lib/ldb && make"),
 490         ("ldb-install", "cd lib/ldb && make install"),
 491
 492         ("nondevel-configure", "./configure ${PREFIX}"),
 493         ("nondevel-make", "make -j"),
 494         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 495         ("nondevel-install", "make install"),
 496         ("nondevel-dist", "make dist"),
 497
 498         # retry with all modules shared
 499         ("allshared-distclean", "make distclean"),
 500         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 501         ("allshared-make", "make -j"),
 502         ],
 503
 504     "samba-none-env": [
 505         ("random-sleep", random_sleep(1, 1)),
 506         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 507         ("make", "make -j"),
 508         ("test", make_test(include_envs=["none"])),
 509         ("lcov", LCOV_CMD),
 510         ],
 511
 512     "samba-static": [
 513         ("random-sleep", random_sleep(1, 1)),
 514         # build with all modules static
 515         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 516         ("allstatic-make", "make -j"),
 517         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 518         ("lcov", LCOV_CMD),
 519
 520         # retry without any required modules
 521         ("none-distclean", "make distclean"),
 522         ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 523         ("none-make", "make -j"),
 524
 525         # retry with nonshared smbd and smbtorture
 526         ("nonshared-distclean", "make distclean"),
 527         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 528         ("nonshared-make", "make -j"),
 529         ],
 530
 531     "samba-systemkrb5": [
 532         ("random-sleep", random_sleep(900, 1500)),
 533         ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --with-experimental-mit-ad-dc"),
 534         ("make", "make -j"),
 535         # we currently cannot run a full make test, a limited list of tests could be run
 536         # via "make test TESTS=sometests"
 537         ("test", make_test(include_envs=["ktest"])),
 538         ("lcov", LCOV_CMD),
 539         ("install", "make install"),
 540         ("check-clean-tree", "script/clean-source-tree.sh"),
 541         ("clean", "make clean"),
 542         ],
 543
 544     # Test Samba without python still builds.  When this test fails
 545     # due to more use of Python, the expectations is that the newly
 546     # failing part of the code should be disabled when
 547     # --disable-python is set (rather than major work being done to
 548     # support this environment).  The target here is for vendors
 549     # shipping a minimal smbd.
 550     "samba-nopython": [
 551         ("random-sleep", random_sleep(300, 900)),
 552         ("configure", "./configure.developer ${ENABLE_COVERAGE} --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 553         ("make", "make -j"),
 554         ("install", "make install"),
 555         ("find-python", "script/find_python.sh ${PREFIX}"),
 556         ("test", "make test-nopython"),
 557         ("lcov", LCOV_CMD),
 558         ("check-clean-tree", "script/clean-source-tree.sh"),
 559         ("clean", "make clean"),
 560
 561         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 562         ("talloc-make", "cd lib/talloc && make"),
 563         ("talloc-install", "cd lib/talloc && make install"),
 564
 565         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 566         ("tdb-make", "cd lib/tdb && make"),
 567         ("tdb-install", "cd lib/tdb && make install"),
 568
 569         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 570         ("tevent-make", "cd lib/tevent && make"),
 571         ("tevent-install", "cd lib/tevent && make install"),
 572
 573         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 574         ("ldb-make", "cd lib/ldb && make"),
 575         ("ldb-install", "cd lib/ldb && make install"),
 576
 577         # retry against installed library packages
 578         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 579         ("libs-make", "make -j"),
 580         ("libs-install", "make install"),
 581         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 582         ("libs-clean", "make clean"),
 583         ],
 584
 585     # check we can do the same thing using python2
 586     "samba-nopython-py2": [
 587         ("random-sleep", random_sleep(300, 900)),
 588         ("configure", "PYTHON=python2 ./configure.developer ${ENABLE_COVERAGE} --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 589         ("make", "PYTHON=python2 make -j"),
 590         ("install", "PYTHON=python2 make install"),
 591         ("find-python", "script/find_python.sh ${PREFIX}"),
 592         ("test", "make test-nopython"),
 593         ("lcov", LCOV_CMD),
 594         ("check-clean-tree", "script/clean-source-tree.sh"),
 595         ("clean", "PYTHON=python2 make clean"),
 596
 597         ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 598         ("talloc-make", "cd lib/talloc && PYTHON=python2 make"),
 599         ("talloc-install", "cd lib/talloc && PYTHON=python2 make install"),
 600
 601         ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 602         ("tdb-make", "cd lib/tdb && PYTHON=python2 make"),
 603         ("tdb-install", "cd lib/tdb && PYTHON=python2 make install"),
 604
 605         ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 606         ("tevent-make", "cd lib/tevent && PYTHON=python2 make"),
 607         ("tevent-install", "cd lib/tevent && PYTHON=python2 make install"),
 608
 609         ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 610         ("ldb-make", "cd lib/ldb && PYTHON=python2 make"),
 611         ("ldb-install", "cd lib/ldb && PYTHON=python2 make install"),
 612
 613         # retry against installed library packages
 614         ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 615         ("libs-make", "PYTHON=python2 make -j"),
 616         ("libs-install", "PYTHON=python2 make install"),
 617         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 618         ("libs-clean", "PYTHON=python2 make clean"),
 619         ],
 620
 621     "ldb": [
 622         ("random-sleep", random_sleep(60, 600)),
 623         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 624         ("make", "make"),
 625         ("install", "make install"),
 626         ("test", "make test"),
 627         ("lcov", LCOV_CMD),
 628         ("clean", "make clean"),
 629         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 630         ("make-no-lmdb", "make"),
 631         ("test-no-lmdb", "make test"),
 632         ("lcov-no-lmdb", LCOV_CMD),
 633         ("install-no-lmdb", "make install"),
 634         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 635         ("distcheck", "make distcheck"),
 636         ("clean", "make clean"),
 637         ],
 638
 639     "tdb": [
 640         ("random-sleep", random_sleep(60, 600)),
 641         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 642         ("make", "make"),
 643         ("install", "make install"),
 644         ("test", "make test"),
 645         ("lcov", LCOV_CMD),
 646         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 647         ("distcheck", "make distcheck"),
 648         ("clean", "make clean"),
 649         ],
 650
 651     "talloc": [
 652         ("random-sleep", random_sleep(60, 600)),
 653         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 654         ("make", "make"),
 655         ("install", "make install"),
 656         ("test", "make test"),
 657         ("lcov", LCOV_CMD),
 658         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 659         ("distcheck", "make distcheck"),
 660         ("clean", "make clean"),
 661         ],
 662
 663     "replace": [
 664         ("random-sleep", random_sleep(60, 600)),
 665         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 666         ("make", "make"),
 667         ("install", "make install"),
 668         ("test", "make test"),
 669         ("lcov", LCOV_CMD),
 670         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 671         ("distcheck", "make distcheck"),
 672         ("clean", "make clean"),
 673         ],
 674
 675     "tevent": [
 676         ("random-sleep", random_sleep(60, 600)),
 677         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 678         ("make", "make"),
 679         ("install", "make install"),
 680         ("test", "make test"),
 681         ("lcov", LCOV_CMD),
 682         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 683         ("distcheck", "make distcheck"),
 684         ("clean", "make clean"),
 685         ],
 686
 687     "pidl": [
 688         ("random-sleep", random_sleep(60, 600)),
 689         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 690         ("touch", "touch *.yp"),
 691         ("make", "make"),
 692         ("test", "make test"),
 693         ("install", "make install"),
 694         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 695         ("check-clean-tree", "../script/clean-source-tree.sh"),
 696         ("clean", "make clean"),
 697         ],
 698
 699     # these are useful for debugging autobuild
 700     'pass': [("pass", 'echo passing && /bin/true')],
 701     'fail': [("fail", 'echo failing && /bin/false')],
 702 }
 703
 704
 705 def do_print(msg):
 706     print("%s" % msg)
 707     sys.stdout.flush()
 708     sys.stderr.flush()
 709
 710
 711 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 712     if show is None:
 713         show = options.verbose
 714     if show:
 715         do_print("Running: '%s' in '%s'" % (cmd, dir))
 716     if output:
 717         out = check_output([cmd], shell=True, cwd=dir)
 718         return out.decode(encoding='utf-8', errors='backslashreplace')
 719     elif checkfail:
 720         return check_call(cmd, shell=True, cwd=dir)
 721     else:
 722         return call(cmd, shell=True, cwd=dir)
 723
 724
 725 class builder(object):
 726     '''handle build of one directory'''
 727
 728     def __init__(self, name, sequence, cp=True):
 729         self.name = name
 730         if name in builddirs:
 731             self.dir = builddirs[name]
 732         else:
 733             self.dir = "."
 734
 735         self.tag = self.name.replace('/', '_')
 736         self.sequence = sequence
 737         self.next = 0
 738         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 739         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 740         if options.verbose:
 741             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 742             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 743         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 744         self.stdout = open(self.stdout_path, 'w')
 745         self.stderr = open(self.stderr_path, 'w')
 746         self.stdin  = open("/dev/null", 'r')
 747         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 748         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
 749         self.prefix = "%s/%s" % (test_prefix, self.tag)
 750         run_cmd("rm -rf %s" % self.test_source_dir)
 751         run_cmd("rm -rf %s" % self.prefix)
 752         if cp:
 753             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 754         else:
 755             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 756         self.start_next()
 757
 758     def start_next(self):
 759         if self.next == len(self.sequence):
 760             if not options.nocleanup:
 761                 run_cmd("rm -rf %s" % self.test_source_dir)
 762                 run_cmd("rm -rf %s" % self.prefix)
 763             do_print('%s: Completed OK' % self.name)
 764             self.done = True
 765             return
 766         (self.stage, self.cmd) = self.sequence[self.next]
 767         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 768         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 769         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 770         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 771         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 772         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 773         self.cmd = self.cmd.replace("${NAME}", self.name)
 774         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 775         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
 776         self.proc = Popen(self.cmd, shell=True,
 777                           close_fds=True, cwd=self.cwd,
 778                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 779         self.next += 1
 780
 781
 782 class buildlist(object):
 783     '''handle build of multiple directories'''
 784
 785     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 786         global tasks
 787         self.tlist = []
 788         self.tail_proc = None
 789         self.retry = None
 790         if tasknames == []:
 791             if options.restrict_tests:
 792                 tasknames = ["samba-test-only"]
 793             else:
 794                 tasknames = defaulttasks
 795
 796         for n in tasknames:
 797             b = builder(n, tasks[n], cp=n is not "pidl")
 798             self.tlist.append(b)
 799         if options.retry:
 800             rebase_remote = "rebaseon"
 801             retry_task = [("retry",
 802                             '''set -e
 803                             git remote add -t %s %s %s
 804                             git fetch %s
 805                             while :; do
 806                               sleep 60
 807                               git describe %s/%s > old_remote_branch.desc
 808                               git fetch %s
 809                               git describe %s/%s > remote_branch.desc
 810                               diff old_remote_branch.desc remote_branch.desc
 811                             done
 812                            ''' % (
 813                                rebase_branch, rebase_remote, rebase_url,
 814                                rebase_remote,
 815                                rebase_remote, rebase_branch,
 816                                rebase_remote,
 817                                rebase_remote, rebase_branch
 818                             ))]
 819
 820             self.retry = builder('retry', retry_task, cp=False)
 821             self.need_retry = False
 822
 823     def kill_kids(self):
 824         if self.tail_proc is not None:
 825             self.tail_proc.terminate()
 826             self.tail_proc.wait()
 827             self.tail_proc = None
 828         if self.retry is not None:
 829             self.retry.proc.terminate()
 830             self.retry.proc.wait()
 831             self.retry = None
 832         for b in self.tlist:
 833             if b.proc is not None:
 834                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 835                 b.proc.terminate()
 836                 b.proc.wait()
 837                 b.proc = None
 838
 839     def wait_one(self):
 840         while True:
 841             none_running = True
 842             for b in self.tlist:
 843                 if b.proc is None:
 844                     continue
 845                 none_running = False
 846                 b.status = b.proc.poll()
 847                 if b.status is None:
 848                     continue
 849                 b.proc = None
 850                 return b
 851             if options.retry:
 852                 ret = self.retry.proc.poll()
 853                 if ret is not None:
 854                     self.need_retry = True
 855                     self.retry = None
 856                     return None
 857             if none_running:
 858                 return None
 859             time.sleep(0.1)
 860
 861     def run(self):
 862         while True:
 863             b = self.wait_one()
 864             if options.retry and self.need_retry:
 865                 self.kill_kids()
 866                 do_print("retry needed")
 867                 return (0, None, None, None, "retry")
 868             if b is None:
 869                 break
 870             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 871                 self.kill_kids()
 872                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 873             b.start_next()
 874         self.kill_kids()
 875         return (0, None, None, None, "All OK")
 876
 877     def write_system_info(self):
 878         filename = 'system-info.txt'
 879         f = open(filename, 'w')
 880         for cmd in ['uname -a',
 881                     'lsb_release -a',
 882                     'free',
 883                     'mount',
 884                     'cat /proc/cpuinfo',
 885                     'cc --version',
 886                     'df -m .',
 887                     'df -m %s' % testbase]:
 888             out = run_cmd(cmd, output=True, checkfail=False)
 889             print('### %s' % cmd, file=f)
 890             print(out, file=f)
 891             print(file=f)
 892         f.close()
 893         return filename
 894
 895     def tarlogs(self, fname):
 896         tar = tarfile.open(fname, "w:gz")
 897         for b in self.tlist:
 898             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 899             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 900         if os.path.exists("autobuild.log"):
 901             tar.add("autobuild.log")
 902         sys_info = self.write_system_info()
 903         tar.add(sys_info)
 904         tar.close()
 905
 906     def remove_logs(self):
 907         for b in self.tlist:
 908             os.unlink(b.stdout_path)
 909             os.unlink(b.stderr_path)
 910
 911     def start_tail(self):
 912         cmd = ["tail", "-f"]
 913         for b in self.tlist:
 914             cmd.append(b.stdout_path)
 915             cmd.append(b.stderr_path)
 916         self.tail_proc = Popen(cmd, close_fds=True)
 917
 918
 919 def cleanup():
 920     if options.nocleanup:
 921         return
 922     run_cmd("stat %s || true" % test_tmpdir, show=True)
 923     run_cmd("stat %s" % testbase, show=True)
 924     do_print("Cleaning up %r" % cleanup_list)
 925     for d in cleanup_list:
 926         run_cmd("rm -rf %s" % d)
 927
 928
 929 def daemonize(logfile):
 930     pid = os.fork()
 931     if pid == 0:  # Parent
 932         os.setsid()
 933         pid = os.fork()
 934         if pid != 0:  # Actual daemon
 935             os._exit(0)
 936     else:  # Grandparent
 937         os._exit(0)
 938
 939     import resource      # Resource usage information.
 940     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 941     if maxfd == resource.RLIM_INFINITY:
 942         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 943     for fd in range(0, maxfd):
 944         try:
 945             os.close(fd)
 946         except OSError:
 947             pass
 948     os.open(logfile, os.O_RDWR | os.O_CREAT)
 949     os.dup2(0, 1)
 950     os.dup2(0, 2)
 951
 952
 953 def write_pidfile(fname):
 954     '''write a pid file, cleanup on exit'''
 955     f = open(fname, mode='w')
 956     f.write("%u\n" % os.getpid())
 957     f.close()
 958
 959
 960 def rebase_tree(rebase_url, rebase_branch="master"):
 961     rebase_remote = "rebaseon"
 962     do_print("Rebasing on %s" % rebase_url)
 963     run_cmd("git describe HEAD", show=True, dir=test_master)
 964     run_cmd("git remote add -t %s %s %s" %
 965             (rebase_branch, rebase_remote, rebase_url),
 966             show=True, dir=test_master)
 967     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 968     if options.fix_whitespace:
 969         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 970                 (rebase_remote, rebase_branch),
 971                 show=True, dir=test_master)
 972     else:
 973         run_cmd("git rebase --force-rebase %s/%s" %
 974                 (rebase_remote, rebase_branch),
 975                 show=True, dir=test_master)
 976     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 977                    (rebase_remote, rebase_branch),
 978                    dir=test_master, output=True)
 979     if diff == '':
 980         do_print("No differences between HEAD and %s/%s - exiting" %
 981                  (rebase_remote, rebase_branch))
 982         sys.exit(0)
 983     run_cmd("git describe %s/%s" %
 984             (rebase_remote, rebase_branch),
 985             show=True, dir=test_master)
 986     run_cmd("git describe HEAD", show=True, dir=test_master)
 987     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 988             (rebase_remote, rebase_branch),
 989             show=True, dir=test_master)
 990
 991
 992 def push_to(push_url, push_branch="master"):
 993     push_remote = "pushto"
 994     do_print("Pushing to %s" % push_url)
 995     if options.mark:
 996         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 997         run_cmd("git commit --amend -c HEAD", dir=test_master)
 998         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 999         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1000     run_cmd("git remote add -t %s %s %s" %
1001             (push_branch, push_remote, push_url),
1002             show=True, dir=test_master)
1003     run_cmd("git push %s +HEAD:%s" %
1004             (push_remote, push_branch),
1005             show=True, dir=test_master)
1006
1007
1008 def send_email(subject, text, log_tar):
1009     if options.email is None:
1010         do_print("not sending email because the recipient is not set")
1011         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1012                  (subject, text))
1013         return
1014     outer = MIMEMultipart()
1015     outer['Subject'] = subject
1016     outer['To'] = options.email
1017     outer['From'] = options.email_from
1018     outer['Date'] = email.utils.formatdate(localtime=True)
1019     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1020     outer.attach(MIMEText(text, 'plain'))
1021     if options.attach_logs:
1022         fp = open(log_tar, 'rb')
1023         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1024         fp.close()
1025         # Set the filename parameter
1026         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1027         outer.attach(msg)
1028     content = outer.as_string()
1029     s = smtplib.SMTP(options.email_server)
1030     email_user = os.getenv('SMTP_USERNAME')
1031     email_password = os.getenv('SMTP_PASSWORD')
1032     if email_user is not None:
1033         s.starttls()
1034         s.login(email_user, email_password)
1035
1036     s.sendmail(options.email_from, [options.email], content)
1037     s.set_debuglevel(1)
1038     s.quit()
1039
1040
1041 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1042                   elapsed_time, log_base=None, add_log_tail=True):
1043     '''send an email to options.email about the failure'''
1044     elapsed_minutes = elapsed_time / 60.0
1045     if log_base is None:
1046         log_base = gitroot
1047     text = '''
1048 Dear Developer,
1049
1050 Your autobuild on %s failed after %.1f minutes
1051 when trying to test %s with the following error:
1052
1053    %s
1054
1055 the autobuild has been abandoned. Please fix the error and resubmit.
1056
1057 A summary of the autobuild process is here:
1058
1059   %s/autobuild.log
1060 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1061
1062     if options.restrict_tests:
1063         text += """
1064 The build was restricted to tests matching %s\n""" % options.restrict_tests
1065
1066     if failed_task != 'rebase':
1067         text += '''
1068 You can see logs of the failed task here:
1069
1070   %s/%s.stdout
1071   %s/%s.stderr
1072
1073 or you can get full logs of all tasks in this job here:
1074
1075   %s/logs.tar.gz
1076
1077 The top commit for the tree that was built was:
1078
1079 %s
1080
1081 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1082
1083     if add_log_tail:
1084         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1085         lines = f.readlines()
1086         log_tail = "".join(lines[-50:])
1087         num_lines = len(lines)
1088         if num_lines < 50:
1089             # Also include stderr (compile failures) if < 50 lines of stdout
1090             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1091             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1092
1093         text += '''
1094 The last 50 lines of log messages:
1095
1096 %s
1097     ''' % log_tail
1098         f.close()
1099
1100     logs = os.path.join(gitroot, 'logs.tar.gz')
1101     send_email('autobuild[%s] failure on %s for task %s during %s'
1102                % (options.branch, platform.node(), failed_task, failed_stage),
1103                text, logs)
1104
1105
1106 def email_success(elapsed_time, log_base=None):
1107     '''send an email to options.email about a successful build'''
1108     if log_base is None:
1109         log_base = gitroot
1110     text = '''
1111 Dear Developer,
1112
1113 Your autobuild on %s has succeeded after %.1f minutes.
1114
1115 ''' % (platform.node(), elapsed_time / 60.)
1116
1117     if options.restrict_tests:
1118         text += """
1119 The build was restricted to tests matching %s\n""" % options.restrict_tests
1120
1121     if options.keeplogs:
1122         text += '''
1123
1124 you can get full logs of all tasks in this job here:
1125
1126   %s/logs.tar.gz
1127
1128 ''' % log_base
1129
1130     text += '''
1131 The top commit for the tree that was built was:
1132
1133 %s
1134 ''' % top_commit_msg
1135
1136     logs = os.path.join(gitroot, 'logs.tar.gz')
1137     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1138                text, logs)
1139
1140
1141 # get the top commit message, for emails
1142 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1143
1144 try:
1145     os.makedirs(testbase)
1146 except Exception as reason:
1147     raise Exception("Unable to create %s : %s" % (testbase, reason))
1148 cleanup_list.append(testbase)
1149
1150 if options.daemon:
1151     logfile = os.path.join(testbase, "log")
1152     do_print("Forking into the background, writing progress to %s" % logfile)
1153     daemonize(logfile)
1154
1155 write_pidfile(gitroot + "/autobuild.pid")
1156
1157 start_time = time.time()
1158
1159 while True:
1160     try:
1161         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1162         os.makedirs(test_tmpdir)
1163         # The waf uninstall code removes empty directories all the way
1164         # up the tree.  Creating a file in test_tmpdir stops it from
1165         # being removed.
1166         run_cmd("touch %s" % os.path.join(test_tmpdir,
1167                                           ".directory-is-not-empty"), show=True)
1168         run_cmd("stat %s" % test_tmpdir, show=True)
1169         run_cmd("stat %s" % testbase, show=True)
1170         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1171     except Exception:
1172         cleanup()
1173         raise
1174
1175     try:
1176         try:
1177             if options.rebase is not None:
1178                 rebase_tree(options.rebase, rebase_branch=options.branch)
1179         except Exception:
1180             cleanup_list.append(gitroot + "/autobuild.pid")
1181             cleanup()
1182             elapsed_time = time.time() - start_time
1183             email_failure(-1, 'rebase', 'rebase', 'rebase',
1184                           'rebase on %s failed' % options.branch,
1185                           elapsed_time, log_base=options.log_base)
1186             sys.exit(1)
1187         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1188         if options.tail:
1189             blist.start_tail()
1190         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1191         if status != 0 or errstr != "retry":
1192             break
1193         cleanup()
1194     except Exception:
1195         cleanup()
1196         raise
1197
1198 cleanup_list.append(gitroot + "/autobuild.pid")
1199
1200 do_print(errstr)
1201
1202 blist.kill_kids()
1203 if options.tail:
1204     do_print("waiting for tail to flush")
1205     time.sleep(1)
1206
1207 elapsed_time = time.time() - start_time
1208 if status == 0:
1209     if options.passcmd is not None:
1210         do_print("Running passcmd: %s" % options.passcmd)
1211         run_cmd(options.passcmd, dir=test_master)
1212     if options.pushto is not None:
1213         push_to(options.pushto, push_branch=options.branch)
1214     if options.keeplogs or options.attach_logs:
1215         blist.tarlogs("logs.tar.gz")
1216         do_print("Logs in logs.tar.gz")
1217     if options.always_email:
1218         email_success(elapsed_time, log_base=options.log_base)
1219     blist.remove_logs()
1220     cleanup()
1221     do_print(errstr)
1222     sys.exit(0)
1223
1224 # something failed, gather a tar of the logs
1225 blist.tarlogs("logs.tar.gz")
1226
1227 if options.email is not None:
1228     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1229                   elapsed_time, log_base=options.log_base)
1230 else:
1231     elapsed_minutes = elapsed_time / 60.0
1232     print('''
1233
1234 ####################################################################
1235
1236 AUTOBUILD FAILURE
1237
1238 Your autobuild[%s] on %s failed after %.1f minutes
1239 when trying to test %s with the following error:
1240
1241    %s
1242
1243 the autobuild has been abandoned. Please fix the error and resubmit.
1244
1245 ####################################################################
1246
1247 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1248
1249 cleanup()
1250 do_print(errstr)
1251 do_print("Logs in logs.tar.gz")
1252 sys.exit(status)