script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.isdir(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--passcmd", help="command to run on success", default=None)
  60 parser.add_option("--verbose", help="show all commands as they are run",
  61                   default=False, action="store_true")
  62 parser.add_option("--rebase", help="rebase on the given tree before testing",
  63                   default=None, type='str')
  64 parser.add_option("--pushto", help="push to a git url on success",
  65                   default=None, type='str')
  66 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  67                   default=False, action="store_true")
  68 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  69                   default=False, action="store_true")
  70 parser.add_option("--retry", help="automatically retry if master changes",
  71                   default=False, action="store_true")
  72 parser.add_option("--email", help="send email to the given address on failure",
  73                   type='str', default=None)
  74 parser.add_option("--email-from", help="send email from the given address",
  75                   type='str', default="autobuild@samba.org")
  76 parser.add_option("--email-server", help="send email via the given server",
  77                   type='str', default='localhost')
  78 parser.add_option("--always-email", help="always send email, even on success",
  79                   action="store_true")
  80 parser.add_option("--daemon", help="daemonize after initial setup",
  81                   action="store_true")
  82 parser.add_option("--branch", help="the branch to work on (default=master)",
  83                   default="master", type='str')
  84 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  85                   default=gitroot, type='str')
  86 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  87                   default=False, action="store_true")
  88 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  89                   default='')
  90 parser.add_option("--enable-coverage", dest='enable_coverage',
  91                   action="store_const", const='--enable-coverage', default='',
  92                   help="Add --enable-coverage option while configure")
  93
  94 (options, args) = parser.parse_args()
  95
  96 if options.retry:
  97     if options.rebase is None:
  98         raise Exception('You can only use --retry if you also rebase')
  99
 100 testbase = "%s/b%u" % (options.testbase, os.getpid())
 101 test_master = "%s/master" % testbase
 102 test_prefix = "%s/prefix" % testbase
 103 test_tmpdir = "%s/tmp" % testbase
 104 os.environ['TMPDIR'] = test_tmpdir
 105
 106 if options.enable_coverage:
 107     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 108 else:
 109     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 110
 111 if args:
 112     # If we are only running specific test,
 113     # do not sleep randomly to wait for it to start
 114     def random_sleep(low, high):
 115         return 'sleep 1'
 116 else:
 117     def random_sleep(low, high):
 118         return 'sleep {}'.format(random.randint(low, high))
 119
 120 cleanup_list = []
 121
 122 builddirs = {
 123     "ctdb": "ctdb",
 124     "samba": ".",
 125     "samba-nt4": ".",
 126     "samba-fileserver": ".",
 127     "samba-ad-member": ".",
 128     "samba-xc": ".",
 129     "samba-o3": ".",
 130     "samba-ctdb": ".",
 131     "samba-libs": ".",
 132     "samba-static": ".",
 133     "samba-none-env": ".",
 134     "samba-ad-dc-1": ".",
 135     "samba-ad-dc-2": ".",
 136     "samba-ad-dc-3": ".",
 137     "samba-ad-dc-4": ".",
 138     "samba-ad-dc-5": ".",
 139     "samba-ad-dc-6": ".",
 140     "samba-ad-dc-ntvfs": ".",
 141     "samba-ad-dc-backup": ".",
 142     "samba-systemkrb5": ".",
 143     "samba-nopython": ".",
 144     "samba-nopython-py2": ".",
 145     "samba-schemaupgrade": ".",
 146     "ldb": "lib/ldb",
 147     "tdb": "lib/tdb",
 148     "talloc": "lib/talloc",
 149     "replace": "lib/replace",
 150     "tevent": "lib/tevent",
 151     "pidl": "pidl"
 152 }
 153
 154 defaulttasks = list(builddirs.keys())
 155
 156 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 157     defaulttasks.remove("samba-o3")
 158
 159 ctdb_configure_params = " --enable-developer ${PREFIX}"
 160 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 161
 162 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 163 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 164 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 165 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 166 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 167 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 168 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 169
 170
 171 def format_option(name, value=None):
 172     """Format option as str list."""
 173     if value is None:  # boolean option
 174         return [name]
 175     if not isinstance(value, list):  # single value option
 176         value = [value]
 177     # repeatable option
 178     return ['{}={}'.format(name, item) for item in value]
 179
 180
 181 def make_test(
 182         cmd='make test',
 183         FAIL_IMMEDIATELY=1,
 184         TESTS='',
 185         include_envs=None,
 186         exclude_envs=None):
 187
 188     test_options = []
 189     if include_envs:
 190         test_options = format_option('--include-env', include_envs)
 191     if exclude_envs:
 192         test_options = format_option('--exclude-env', exclude_envs)
 193     if test_options:
 194         # join envs options to original test options
 195         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 196
 197     _options = []
 198     if FAIL_IMMEDIATELY:
 199         _options.append('FAIL_IMMEDIATELY=1')
 200     if TESTS:
 201         _options.append("TESTS='{}'".format(TESTS))
 202
 203     return ' '.join([cmd] + _options)
 204
 205
 206 tasks = {
 207     "ctdb": [
 208         ("random-sleep", random_sleep(300, 900)),
 209         ("configure", "./configure " + ctdb_configure_params),
 210         ("make", "make all"),
 211         ("install", "make install"),
 212         ("test", "make autotest"),
 213         ("check-clean-tree", "../script/clean-source-tree.sh"),
 214         ("clean", "make clean"),
 215         ],
 216
 217     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 218     "samba": [
 219         ("random-sleep", random_sleep(300, 900)),
 220         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 221         ("make", "make -j"),
 222         ("test", make_test(exclude_envs=[
 223             "none",
 224             "nt4_dc",
 225             "nt4_dc_schannel",
 226             "nt4_member",
 227             "ad_dc",
 228             "ad_dc_backup",
 229             "ad_dc_ntvfs",
 230             "ad_dc_default",
 231             "ad_dc_slowtests",
 232             "ad_dc_no_nss",
 233             "ad_dc_no_ntlm",
 234             "fl2003dc",
 235             "fl2008dc",
 236             "fl2008r2dc",
 237             "ad_member",
 238             "ad_member_idmap_rid",
 239             "ad_member_idmap_ad",
 240             "ad_member_rfc2307",
 241             "chgdcpass",
 242             "vampire_2000_dc",
 243             "fl2000dc",
 244             "fileserver",
 245             "maptoguest",
 246             "simpleserver",
 247             "backupfromdc",
 248             "restoredc",
 249             "renamedc",
 250             "offlinebackupdc",
 251             "labdc",
 252             "preforkrestartdc",
 253             "proclimitdc",
 254             "promoted_dc",
 255             "vampire_dc",
 256             "rodc",
 257             "ad_dc_default",
 258             "ad_dc_slowtests",
 259             "schema_pair_dc",
 260             "schema_dc",
 261             ])),
 262         ("lcov", LCOV_CMD),
 263         ("install", "make install"),
 264         ("check-clean-tree", "script/clean-source-tree.sh"),
 265         ("clean", "make clean"),
 266         ],
 267
 268     "samba-nt4": [
 269         ("random-sleep", random_sleep(300, 900)),
 270         ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params),
 271         ("make", "make -j"),
 272         ("test", make_test(include_envs=[
 273             "nt4_dc",
 274             "nt4_dc_schannel",
 275             "nt4_member",
 276             ])),
 277         ("lcov", LCOV_CMD),
 278         ("install", "make install"),
 279         ("check-clean-tree", "script/clean-source-tree.sh"),
 280         ("clean", "make clean"),
 281         ],
 282
 283     "samba-fileserver": [
 284         ("random-sleep", random_sleep(300, 900)),
 285         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
 286         ("make", "make -j"),
 287         ("test", make_test(include_envs=[
 288             "fileserver",
 289             "maptoguest",
 290             "simpleserver",
 291             ])),
 292         ("lcov", LCOV_CMD),
 293         ("check-clean-tree", "script/clean-source-tree.sh"),
 294         ],
 295
 296     "samba-ad-member": [
 297         ("random-sleep", random_sleep(300, 900)),
 298         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 299         ("make", "make -j"),
 300         ("test", make_test(include_envs=[
 301             "ad_member",
 302             "ad_member_idmap_rid",
 303             "ad_member_idmap_ad",
 304             "ad_member_rfc2307",
 305             ])),
 306         ("lcov", LCOV_CMD),
 307         ("check-clean-tree", "script/clean-source-tree.sh"),
 308         ],
 309
 310     "samba-ad-dc-1": [
 311         ("random-sleep", random_sleep(1, 1)),
 312         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 313         ("make", "make -j"),
 314         ("test", make_test(include_envs=[
 315             "ad_dc",
 316             "ad_dc_no_nss",
 317             "ad_dc_no_ntlm",
 318             ])),
 319         ("lcov", LCOV_CMD),
 320         ("check-clean-tree", "script/clean-source-tree.sh"),
 321         ],
 322
 323     "samba-ad-dc-2": [
 324         ("random-sleep", random_sleep(1, 1)),
 325         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 326         ("make", "make -j"),
 327         ("test", make_test(include_envs=[
 328             "vampire_dc",
 329             "vampire_2000_dc",
 330             "rodc",
 331             ])),
 332         ("lcov", LCOV_CMD),
 333         ("check-clean-tree", "script/clean-source-tree.sh"),
 334         ],
 335
 336     "samba-ad-dc-3": [
 337         ("random-sleep", random_sleep(1, 1)),
 338         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 339         ("make", "make -j"),
 340         ("test", make_test(include_envs=[
 341             "promoted_dc",
 342             "chgdcpass",
 343             "preforkrestartdc",
 344             "proclimitdc",
 345             ])),
 346         ("lcov", LCOV_CMD),
 347         ("check-clean-tree", "script/clean-source-tree.sh"),
 348         ],
 349
 350     "samba-ad-dc-4": [
 351         ("random-sleep", random_sleep(1, 1)),
 352         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 353         ("make", "make -j"),
 354         ("test", make_test(include_envs=[
 355             "fl2000dc",
 356             "fl2003dc",
 357             "fl2008dc",
 358             "fl2008r2dc",
 359             ])),
 360         ("lcov", LCOV_CMD),
 361         ("check-clean-tree", "script/clean-source-tree.sh"),
 362         ],
 363
 364     "samba-ad-dc-5": [
 365         ("random-sleep", random_sleep(1, 1)),
 366         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 367         ("make", "make -j"),
 368         ("test", make_test(include_envs=["ad_dc_default"])),
 369         ("lcov", LCOV_CMD),
 370         ("check-clean-tree", "script/clean-source-tree.sh"),
 371         ],
 372
 373     "samba-ad-dc-6": [
 374         ("random-sleep", random_sleep(1, 1)),
 375         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 376         ("make", "make -j"),
 377         ("test", make_test(include_envs=["ad_dc_slowtests"])),
 378         ("lcov", LCOV_CMD),
 379         ("check-clean-tree", "script/clean-source-tree.sh"),
 380         ],
 381
 382     "samba-schemaupgrade": [
 383         ("random-sleep", random_sleep(1, 1)),
 384         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 385         ("make", "make -j"),
 386         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 387         ("lcov", LCOV_CMD),
 388         ("check-clean-tree", "script/clean-source-tree.sh"),
 389         ],
 390
 391     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 392     # This is currently the longest task, so we don't randomly delay it.
 393     "samba-ad-dc-ntvfs": [
 394         ("random-sleep", random_sleep(1, 1)),
 395         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 396         ("make", "make -j"),
 397         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 398         ("lcov", LCOV_CMD),
 399         ("check-clean-tree", "script/clean-source-tree.sh"),
 400         ],
 401
 402     # run the backup/restore testenvs separately as they're fairly standalone
 403     # (and CI seems to max out at ~8 different DCs running at once)
 404     "samba-ad-dc-backup": [
 405         ("random-sleep", random_sleep(300, 900)),
 406         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 407         ("make", "make -j"),
 408         ("test", make_test(include_envs=[
 409             "backupfromdc",
 410             "restoredc",
 411             "renamedc",
 412             "offlinebackupdc",
 413             "labdc",
 414             "ad_dc_backup",
 415             ])),
 416         ("lcov", LCOV_CMD),
 417         ("check-clean-tree", "script/clean-source-tree.sh"),
 418         ],
 419
 420     "samba-test-only": [
 421         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
 422         ("make", "make -j"),
 423         ("test", make_test(TESTS="${TESTS}")),
 424         ("lcov", LCOV_CMD),
 425         ],
 426
 427     # Test cross-compile infrastructure
 428     "samba-xc": [
 429         ("random-sleep", random_sleep(900, 1500)),
 430         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 431         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 432             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 433         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 434             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 435         ("compare-results", "script/compare_cc_results.py "
 436             "./bin/c4che/default{} "
 437             "./bin-xe/c4che/default{} "
 438             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 439         ],
 440
 441     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 442     "samba-o3": [
 443         ("random-sleep", random_sleep(300, 900)),
 444         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
 445         ("make", "make -j"),
 446         ("test", make_test(cmd='make quicktest', include_envs=["ad_dc"])),
 447         ("lcov", LCOV_CMD),
 448         ("install", "make install"),
 449         ("check-clean-tree", "script/clean-source-tree.sh"),
 450         ("clean", "make clean"),
 451         ],
 452
 453     "samba-ctdb": [
 454         ("random-sleep", random_sleep(900, 1500)),
 455
 456         # make sure we have tdb around:
 457         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 458         ("tdb-make", "cd lib/tdb && make"),
 459         ("tdb-install", "cd lib/tdb && make install"),
 460
 461         # build samba with cluster support (also building ctdb):
 462         ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb"),
 463         ("samba-make", "make"),
 464         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 465         ("samba-install", "make install"),
 466         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 467
 468         # clean up:
 469         ("check-clean-tree", "script/clean-source-tree.sh"),
 470         ("clean", "make clean"),
 471         ("ctdb-clean", "cd ./ctdb && make clean"),
 472         ],
 473
 474     "samba-libs": [
 475         ("random-sleep", random_sleep(300, 900)),
 476         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 477         ("talloc-make", "cd lib/talloc && make"),
 478         ("talloc-install", "cd lib/talloc && make install"),
 479
 480         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 481         ("tdb-make", "cd lib/tdb && make"),
 482         ("tdb-install", "cd lib/tdb && make install"),
 483
 484         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 485         ("tevent-make", "cd lib/tevent && make"),
 486         ("tevent-install", "cd lib/tevent && make install"),
 487
 488         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 489         ("ldb-make", "cd lib/ldb && make"),
 490         ("ldb-install", "cd lib/ldb && make install"),
 491
 492         ("nondevel-configure", "./configure ${PREFIX}"),
 493         ("nondevel-make", "make -j"),
 494         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 495         ("nondevel-install", "make install"),
 496         ("nondevel-dist", "make dist"),
 497
 498         # retry with all modules shared
 499         ("allshared-distclean", "make distclean"),
 500         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 501         ("allshared-make", "make -j"),
 502         ],
 503
 504     "samba-none-env": [
 505         ("random-sleep", random_sleep(1, 1)),
 506         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 507         ("make", "make -j"),
 508         ("test", make_test(include_envs=["none"])),
 509         ("lcov", LCOV_CMD),
 510         ],
 511
 512     "samba-static": [
 513         ("random-sleep", random_sleep(1, 1)),
 514         # build with all modules static
 515         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 516         ("allstatic-make", "make -j"),
 517         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 518         ("lcov", LCOV_CMD),
 519
 520         # retry without any required modules
 521         ("none-distclean", "make distclean"),
 522         ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 523         ("none-make", "make -j"),
 524
 525         # retry with nonshared smbd and smbtorture
 526         ("nonshared-distclean", "make distclean"),
 527         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 528         ("nonshared-make", "make -j"),
 529         ],
 530
 531     "samba-systemkrb5": [
 532         ("random-sleep", random_sleep(900, 1500)),
 533         ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --with-experimental-mit-ad-dc"),
 534         ("make", "make -j"),
 535         # we currently cannot run a full make test, a limited list of tests could be run
 536         # via "make test TESTS=sometests"
 537         ("test", make_test(include_envs=["ktest"])),
 538         ("lcov", LCOV_CMD),
 539         ("install", "make install"),
 540         ("check-clean-tree", "script/clean-source-tree.sh"),
 541         ("clean", "make clean"),
 542         ],
 543
 544     # Test Samba without python still builds.  When this test fails
 545     # due to more use of Python, the expectations is that the newly
 546     # failing part of the code should be disabled when
 547     # --disable-python is set (rather than major work being done to
 548     # support this environment).  The target here is for vendors
 549     # shipping a minimal smbd.
 550     "samba-nopython": [
 551         ("random-sleep", random_sleep(300, 900)),
 552         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 553         ("make", "make -j"),
 554         ("install", "make install"),
 555         ("find-python", "script/find_python.sh ${PREFIX}"),
 556         ("test", "make test-nopython"),
 557         ("lcov", LCOV_CMD),
 558         ("check-clean-tree", "script/clean-source-tree.sh"),
 559         ("clean", "make clean"),
 560
 561         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 562         ("talloc-make", "cd lib/talloc && make"),
 563         ("talloc-install", "cd lib/talloc && make install"),
 564
 565         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 566         ("tdb-make", "cd lib/tdb && make"),
 567         ("tdb-install", "cd lib/tdb && make install"),
 568
 569         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 570         ("tevent-make", "cd lib/tevent && make"),
 571         ("tevent-install", "cd lib/tevent && make install"),
 572
 573         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 574         ("ldb-make", "cd lib/ldb && make"),
 575         ("ldb-install", "cd lib/ldb && make install"),
 576
 577         # retry against installed library packages
 578         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 579         ("libs-make", "make -j"),
 580         ("libs-install", "make install"),
 581         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 582         ("libs-clean", "make clean"),
 583         ],
 584
 585     # check we can do the same thing using python2
 586     "samba-nopython-py2": [
 587         ("random-sleep", random_sleep(300, 900)),
 588         ("configure", "PYTHON=python2 ./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 589         ("make", "PYTHON=python2 make -j"),
 590         ("install", "PYTHON=python2 make install"),
 591         ("find-python", "script/find_python.sh ${PREFIX}"),
 592         ("test", "make test-nopython"),
 593         ("lcov", LCOV_CMD),
 594         ("check-clean-tree", "script/clean-source-tree.sh"),
 595         ("clean", "PYTHON=python2 make clean"),
 596
 597         ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 598         ("talloc-make", "cd lib/talloc && PYTHON=python2 make"),
 599         ("talloc-install", "cd lib/talloc && PYTHON=python2 make install"),
 600
 601         ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 602         ("tdb-make", "cd lib/tdb && PYTHON=python2 make"),
 603         ("tdb-install", "cd lib/tdb && PYTHON=python2 make install"),
 604
 605         ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 606         ("tevent-make", "cd lib/tevent && PYTHON=python2 make"),
 607         ("tevent-install", "cd lib/tevent && PYTHON=python2 make install"),
 608
 609         ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 610         ("ldb-make", "cd lib/ldb && PYTHON=python2 make"),
 611         ("ldb-install", "cd lib/ldb && PYTHON=python2 make install"),
 612
 613         # retry against installed library packages
 614         ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 615         ("libs-make", "PYTHON=python2 make -j"),
 616         ("libs-install", "PYTHON=python2 make install"),
 617         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 618         ("libs-clean", "PYTHON=python2 make clean"),
 619         ],
 620
 621     "ldb": [
 622         ("random-sleep", random_sleep(60, 600)),
 623         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 624         ("make", "make"),
 625         ("install", "make install"),
 626         ("test", "make test"),
 627         ("lcov", LCOV_CMD),
 628         ("clean", "make clean"),
 629         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 630         ("make-no-lmdb", "make"),
 631         ("test-no-lmdb", "make test"),
 632         ("lcov-no-lmdb", LCOV_CMD),
 633         ("install-no-lmdb", "make install"),
 634         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 635         ("distcheck", "make distcheck"),
 636         ("clean", "make clean"),
 637         ],
 638
 639     "tdb": [
 640         ("random-sleep", random_sleep(60, 600)),
 641         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 642         ("make", "make"),
 643         ("install", "make install"),
 644         ("test", "make test"),
 645         ("lcov", LCOV_CMD),
 646         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 647         ("distcheck", "make distcheck"),
 648         ("clean", "make clean"),
 649         ],
 650
 651     "talloc": [
 652         ("random-sleep", random_sleep(60, 600)),
 653         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 654         ("make", "make"),
 655         ("install", "make install"),
 656         ("test", "make test"),
 657         ("lcov", LCOV_CMD),
 658         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 659         ("distcheck", "make distcheck"),
 660         ("clean", "make clean"),
 661         ],
 662
 663     "replace": [
 664         ("random-sleep", random_sleep(60, 600)),
 665         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 666         ("make", "make"),
 667         ("install", "make install"),
 668         ("test", "make test"),
 669         ("lcov", LCOV_CMD),
 670         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 671         ("distcheck", "make distcheck"),
 672         ("clean", "make clean"),
 673         ],
 674
 675     "tevent": [
 676         ("random-sleep", random_sleep(60, 600)),
 677         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 678         ("make", "make"),
 679         ("install", "make install"),
 680         ("test", "make test"),
 681         ("lcov", LCOV_CMD),
 682         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 683         ("distcheck", "make distcheck"),
 684         ("clean", "make clean"),
 685         ],
 686
 687     "pidl": [
 688         ("random-sleep", random_sleep(60, 600)),
 689         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 690         ("touch", "touch *.yp"),
 691         ("make", "make"),
 692         ("test", "make test"),
 693         ("install", "make install"),
 694         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 695         ("check-clean-tree", "../script/clean-source-tree.sh"),
 696         ("clean", "make clean"),
 697         ],
 698
 699     # these are useful for debugging autobuild
 700     'pass': [("pass", 'echo passing && /bin/true')],
 701     'fail': [("fail", 'echo failing && /bin/false')],
 702 }
 703
 704
 705 def do_print(msg):
 706     print("%s" % msg)
 707     sys.stdout.flush()
 708     sys.stderr.flush()
 709
 710
 711 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 712     if show is None:
 713         show = options.verbose
 714     if show:
 715         do_print("Running: '%s' in '%s'" % (cmd, dir))
 716     if output:
 717         out = check_output([cmd], shell=True, cwd=dir)
 718         return out.decode(encoding='utf-8', errors='backslashreplace')
 719     elif checkfail:
 720         return check_call(cmd, shell=True, cwd=dir)
 721     else:
 722         return call(cmd, shell=True, cwd=dir)
 723
 724
 725 class builder(object):
 726     '''handle build of one directory'''
 727
 728     def __init__(self, name, sequence, cp=True):
 729         self.name = name
 730         self.dir = builddirs.get(name, '.')
 731         self.tag = self.name.replace('/', '_')
 732         self.sequence = sequence
 733         self.next = 0
 734         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 735         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 736         if options.verbose:
 737             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 738             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 739         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 740         self.stdout = open(self.stdout_path, 'w')
 741         self.stderr = open(self.stderr_path, 'w')
 742         self.stdin  = open("/dev/null", 'r')
 743         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 744         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
 745         self.prefix = "%s/%s" % (test_prefix, self.tag)
 746         run_cmd("rm -rf %s" % self.test_source_dir)
 747         run_cmd("rm -rf %s" % self.prefix)
 748         if cp:
 749             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 750         else:
 751             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 752         self.start_next()
 753
 754     def start_next(self):
 755         if self.next == len(self.sequence):
 756             if not options.nocleanup:
 757                 run_cmd("rm -rf %s" % self.test_source_dir)
 758                 run_cmd("rm -rf %s" % self.prefix)
 759             do_print('%s: Completed OK' % self.name)
 760             self.done = True
 761             return
 762         (self.stage, self.cmd) = self.sequence[self.next]
 763         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 764         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 765         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 766         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 767         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 768         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 769         self.cmd = self.cmd.replace("${NAME}", self.name)
 770         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 771         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
 772         self.proc = Popen(self.cmd, shell=True,
 773                           close_fds=True, cwd=self.cwd,
 774                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 775         self.next += 1
 776
 777
 778 class buildlist(object):
 779     '''handle build of multiple directories'''
 780
 781     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 782         self.tail_proc = None
 783         self.retry = None
 784         if not tasknames:
 785             if options.restrict_tests:
 786                 tasknames = ["samba-test-only"]
 787             else:
 788                 tasknames = defaulttasks
 789
 790         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
 791
 792         if options.retry:
 793             rebase_remote = "rebaseon"
 794             retry_task = [("retry",
 795                             '''set -e
 796                             git remote add -t %s %s %s
 797                             git fetch %s
 798                             while :; do
 799                               sleep 60
 800                               git describe %s/%s > old_remote_branch.desc
 801                               git fetch %s
 802                               git describe %s/%s > remote_branch.desc
 803                               diff old_remote_branch.desc remote_branch.desc
 804                             done
 805                            ''' % (
 806                                rebase_branch, rebase_remote, rebase_url,
 807                                rebase_remote,
 808                                rebase_remote, rebase_branch,
 809                                rebase_remote,
 810                                rebase_remote, rebase_branch
 811                             ))]
 812
 813             self.retry = builder('retry', retry_task, cp=False)
 814             self.need_retry = False
 815
 816     def kill_kids(self):
 817         if self.tail_proc is not None:
 818             self.tail_proc.terminate()
 819             self.tail_proc.wait()
 820             self.tail_proc = None
 821         if self.retry is not None:
 822             self.retry.proc.terminate()
 823             self.retry.proc.wait()
 824             self.retry = None
 825         for b in self.tlist:
 826             if b.proc is not None:
 827                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 828                 b.proc.terminate()
 829                 b.proc.wait()
 830                 b.proc = None
 831
 832     def wait_one(self):
 833         while True:
 834             none_running = True
 835             for b in self.tlist:
 836                 if b.proc is None:
 837                     continue
 838                 none_running = False
 839                 b.status = b.proc.poll()
 840                 if b.status is None:
 841                     continue
 842                 b.proc = None
 843                 return b
 844             if options.retry:
 845                 ret = self.retry.proc.poll()
 846                 if ret is not None:
 847                     self.need_retry = True
 848                     self.retry = None
 849                     return None
 850             if none_running:
 851                 return None
 852             time.sleep(0.1)
 853
 854     def run(self):
 855         while True:
 856             b = self.wait_one()
 857             if options.retry and self.need_retry:
 858                 self.kill_kids()
 859                 do_print("retry needed")
 860                 return (0, None, None, None, "retry")
 861             if b is None:
 862                 break
 863             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 864                 self.kill_kids()
 865                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 866             b.start_next()
 867         self.kill_kids()
 868         return (0, None, None, None, "All OK")
 869
 870     def write_system_info(self, filename):
 871         with open(filename, 'w') as f:
 872             for cmd in ['uname -a',
 873                         'lsb_release -a',
 874                         'free',
 875                         'mount',
 876                         'cat /proc/cpuinfo',
 877                         'cc --version',
 878                         'df -m .',
 879                         'df -m %s' % testbase]:
 880                 out = run_cmd(cmd, output=True, checkfail=False)
 881                 print('### %s' % cmd, file=f)
 882                 print(out, file=f)
 883                 print(file=f)
 884
 885     def tarlogs(self, fname):
 886         with tarfile.open(fname, "w:gz") as tar:
 887             for b in self.tlist:
 888                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 889                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 890             if os.path.exists("autobuild.log"):
 891                 tar.add("autobuild.log")
 892             filename = 'system-info.txt'
 893             self.write_system_info(filename)
 894             tar.add(filename)
 895
 896     def remove_logs(self):
 897         for b in self.tlist:
 898             os.unlink(b.stdout_path)
 899             os.unlink(b.stderr_path)
 900
 901     def start_tail(self):
 902         cmd = ["tail", "-f"]
 903         for b in self.tlist:
 904             cmd.append(b.stdout_path)
 905             cmd.append(b.stderr_path)
 906         self.tail_proc = Popen(cmd, close_fds=True)
 907
 908
 909 def cleanup():
 910     if options.nocleanup:
 911         return
 912     run_cmd("stat %s || true" % test_tmpdir, show=True)
 913     run_cmd("stat %s" % testbase, show=True)
 914     do_print("Cleaning up %r" % cleanup_list)
 915     for d in cleanup_list:
 916         run_cmd("rm -rf %s" % d)
 917
 918
 919 def daemonize(logfile):
 920     pid = os.fork()
 921     if pid == 0:  # Parent
 922         os.setsid()
 923         pid = os.fork()
 924         if pid != 0:  # Actual daemon
 925             os._exit(0)
 926     else:  # Grandparent
 927         os._exit(0)
 928
 929     import resource      # Resource usage information.
 930     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 931     if maxfd == resource.RLIM_INFINITY:
 932         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 933     for fd in range(0, maxfd):
 934         try:
 935             os.close(fd)
 936         except OSError:
 937             pass
 938     os.open(logfile, os.O_RDWR | os.O_CREAT)
 939     os.dup2(0, 1)
 940     os.dup2(0, 2)
 941
 942
 943 def write_pidfile(fname):
 944     '''write a pid file, cleanup on exit'''
 945     with open(fname, mode='w') as f:
 946         f.write("%u\n" % os.getpid())
 947
 948
 949 def rebase_tree(rebase_url, rebase_branch="master"):
 950     rebase_remote = "rebaseon"
 951     do_print("Rebasing on %s" % rebase_url)
 952     run_cmd("git describe HEAD", show=True, dir=test_master)
 953     run_cmd("git remote add -t %s %s %s" %
 954             (rebase_branch, rebase_remote, rebase_url),
 955             show=True, dir=test_master)
 956     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 957     if options.fix_whitespace:
 958         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 959                 (rebase_remote, rebase_branch),
 960                 show=True, dir=test_master)
 961     else:
 962         run_cmd("git rebase --force-rebase %s/%s" %
 963                 (rebase_remote, rebase_branch),
 964                 show=True, dir=test_master)
 965     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 966                    (rebase_remote, rebase_branch),
 967                    dir=test_master, output=True)
 968     if diff == '':
 969         do_print("No differences between HEAD and %s/%s - exiting" %
 970                  (rebase_remote, rebase_branch))
 971         sys.exit(0)
 972     run_cmd("git describe %s/%s" %
 973             (rebase_remote, rebase_branch),
 974             show=True, dir=test_master)
 975     run_cmd("git describe HEAD", show=True, dir=test_master)
 976     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 977             (rebase_remote, rebase_branch),
 978             show=True, dir=test_master)
 979
 980
 981 def push_to(push_url, push_branch="master"):
 982     push_remote = "pushto"
 983     do_print("Pushing to %s" % push_url)
 984     if options.mark:
 985         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 986         run_cmd("git commit --amend -c HEAD", dir=test_master)
 987         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 988         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 989     run_cmd("git remote add -t %s %s %s" %
 990             (push_branch, push_remote, push_url),
 991             show=True, dir=test_master)
 992     run_cmd("git push %s +HEAD:%s" %
 993             (push_remote, push_branch),
 994             show=True, dir=test_master)
 995
 996
 997 def send_email(subject, text, log_tar):
 998     if options.email is None:
 999         do_print("not sending email because the recipient is not set")
1000         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1001                  (subject, text))
1002         return
1003     outer = MIMEMultipart()
1004     outer['Subject'] = subject
1005     outer['To'] = options.email
1006     outer['From'] = options.email_from
1007     outer['Date'] = email.utils.formatdate(localtime=True)
1008     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1009     outer.attach(MIMEText(text, 'plain'))
1010     if options.attach_logs:
1011         with open(log_tar, 'rb') as fp:
1012             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1013         # Set the filename parameter
1014         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1015         outer.attach(msg)
1016     content = outer.as_string()
1017     s = smtplib.SMTP(options.email_server)
1018     email_user = os.getenv('SMTP_USERNAME')
1019     email_password = os.getenv('SMTP_PASSWORD')
1020     if email_user is not None:
1021         s.starttls()
1022         s.login(email_user, email_password)
1023
1024     s.sendmail(options.email_from, [options.email], content)
1025     s.set_debuglevel(1)
1026     s.quit()
1027
1028
1029 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1030                   elapsed_time, log_base=None, add_log_tail=True):
1031     '''send an email to options.email about the failure'''
1032     elapsed_minutes = elapsed_time / 60.0
1033     if log_base is None:
1034         log_base = gitroot
1035     text = '''
1036 Dear Developer,
1037
1038 Your autobuild on %s failed after %.1f minutes
1039 when trying to test %s with the following error:
1040
1041    %s
1042
1043 the autobuild has been abandoned. Please fix the error and resubmit.
1044
1045 A summary of the autobuild process is here:
1046
1047   %s/autobuild.log
1048 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1049
1050     if options.restrict_tests:
1051         text += """
1052 The build was restricted to tests matching %s\n""" % options.restrict_tests
1053
1054     if failed_task != 'rebase':
1055         text += '''
1056 You can see logs of the failed task here:
1057
1058   %s/%s.stdout
1059   %s/%s.stderr
1060
1061 or you can get full logs of all tasks in this job here:
1062
1063   %s/logs.tar.gz
1064
1065 The top commit for the tree that was built was:
1066
1067 %s
1068
1069 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1070
1071     if add_log_tail:
1072         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1073         lines = f.readlines()
1074         log_tail = "".join(lines[-50:])
1075         num_lines = len(lines)
1076         if num_lines < 50:
1077             # Also include stderr (compile failures) if < 50 lines of stdout
1078             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1079             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1080
1081         text += '''
1082 The last 50 lines of log messages:
1083
1084 %s
1085     ''' % log_tail
1086         f.close()
1087
1088     logs = os.path.join(gitroot, 'logs.tar.gz')
1089     send_email('autobuild[%s] failure on %s for task %s during %s'
1090                % (options.branch, platform.node(), failed_task, failed_stage),
1091                text, logs)
1092
1093
1094 def email_success(elapsed_time, log_base=None):
1095     '''send an email to options.email about a successful build'''
1096     if log_base is None:
1097         log_base = gitroot
1098     text = '''
1099 Dear Developer,
1100
1101 Your autobuild on %s has succeeded after %.1f minutes.
1102
1103 ''' % (platform.node(), elapsed_time / 60.)
1104
1105     if options.restrict_tests:
1106         text += """
1107 The build was restricted to tests matching %s\n""" % options.restrict_tests
1108
1109     if options.keeplogs:
1110         text += '''
1111
1112 you can get full logs of all tasks in this job here:
1113
1114   %s/logs.tar.gz
1115
1116 ''' % log_base
1117
1118     text += '''
1119 The top commit for the tree that was built was:
1120
1121 %s
1122 ''' % top_commit_msg
1123
1124     logs = os.path.join(gitroot, 'logs.tar.gz')
1125     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1126                text, logs)
1127
1128
1129 # get the top commit message, for emails
1130 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1131
1132 try:
1133     os.makedirs(testbase)
1134 except Exception as reason:
1135     raise Exception("Unable to create %s : %s" % (testbase, reason))
1136 cleanup_list.append(testbase)
1137
1138 if options.daemon:
1139     logfile = os.path.join(testbase, "log")
1140     do_print("Forking into the background, writing progress to %s" % logfile)
1141     daemonize(logfile)
1142
1143 write_pidfile(gitroot + "/autobuild.pid")
1144
1145 start_time = time.time()
1146
1147 while True:
1148     try:
1149         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1150         os.makedirs(test_tmpdir)
1151         # The waf uninstall code removes empty directories all the way
1152         # up the tree.  Creating a file in test_tmpdir stops it from
1153         # being removed.
1154         run_cmd("touch %s" % os.path.join(test_tmpdir,
1155                                           ".directory-is-not-empty"), show=True)
1156         run_cmd("stat %s" % test_tmpdir, show=True)
1157         run_cmd("stat %s" % testbase, show=True)
1158         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1159     except Exception:
1160         cleanup()
1161         raise
1162
1163     try:
1164         if options.rebase is not None:
1165             rebase_tree(options.rebase, rebase_branch=options.branch)
1166     except Exception:
1167         cleanup_list.append(gitroot + "/autobuild.pid")
1168         cleanup()
1169         elapsed_time = time.time() - start_time
1170         email_failure(-1, 'rebase', 'rebase', 'rebase',
1171                       'rebase on %s failed' % options.branch,
1172                       elapsed_time, log_base=options.log_base)
1173         sys.exit(1)
1174
1175     try:
1176         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1177         if options.tail:
1178             blist.start_tail()
1179         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1180         if status != 0 or errstr != "retry":
1181             break
1182         cleanup()
1183     except Exception:
1184         cleanup()
1185         raise
1186
1187 cleanup_list.append(gitroot + "/autobuild.pid")
1188
1189 do_print(errstr)
1190
1191 blist.kill_kids()
1192 if options.tail:
1193     do_print("waiting for tail to flush")
1194     time.sleep(1)
1195
1196 elapsed_time = time.time() - start_time
1197 if status == 0:
1198     if options.passcmd is not None:
1199         do_print("Running passcmd: %s" % options.passcmd)
1200         run_cmd(options.passcmd, dir=test_master)
1201     if options.pushto is not None:
1202         push_to(options.pushto, push_branch=options.branch)
1203     if options.keeplogs or options.attach_logs:
1204         blist.tarlogs("logs.tar.gz")
1205         do_print("Logs in logs.tar.gz")
1206     if options.always_email:
1207         email_success(elapsed_time, log_base=options.log_base)
1208     blist.remove_logs()
1209     cleanup()
1210     do_print(errstr)
1211     sys.exit(0)
1212
1213 # something failed, gather a tar of the logs
1214 blist.tarlogs("logs.tar.gz")
1215
1216 if options.email is not None:
1217     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1218                   elapsed_time, log_base=options.log_base)
1219 else:
1220     elapsed_minutes = elapsed_time / 60.0
1221     print('''
1222
1223 ####################################################################
1224
1225 AUTOBUILD FAILURE
1226
1227 Your autobuild[%s] on %s failed after %.1f minutes
1228 when trying to test %s with the following error:
1229
1230    %s
1231
1232 the autobuild has been abandoned. Please fix the error and resubmit.
1233
1234 ####################################################################
1235
1236 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1237
1238 cleanup()
1239 do_print(errstr)
1240 do_print("Logs in logs.tar.gz")
1241 sys.exit(status)