script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.isdir(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--passcmd", help="command to run on success", default=None)
  60 parser.add_option("--verbose", help="show all commands as they are run",
  61                   default=False, action="store_true")
  62 parser.add_option("--rebase", help="rebase on the given tree before testing",
  63                   default=None, type='str')
  64 parser.add_option("--pushto", help="push to a git url on success",
  65                   default=None, type='str')
  66 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  67                   default=False, action="store_true")
  68 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  69                   default=False, action="store_true")
  70 parser.add_option("--retry", help="automatically retry if master changes",
  71                   default=False, action="store_true")
  72 parser.add_option("--email", help="send email to the given address on failure",
  73                   type='str', default=None)
  74 parser.add_option("--email-from", help="send email from the given address",
  75                   type='str', default="autobuild@samba.org")
  76 parser.add_option("--email-server", help="send email via the given server",
  77                   type='str', default='localhost')
  78 parser.add_option("--always-email", help="always send email, even on success",
  79                   action="store_true")
  80 parser.add_option("--daemon", help="daemonize after initial setup",
  81                   action="store_true")
  82 parser.add_option("--branch", help="the branch to work on (default=master)",
  83                   default="master", type='str')
  84 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  85                   default=gitroot, type='str')
  86 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  87                   default=False, action="store_true")
  88 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  89                   default='')
  90 parser.add_option("--enable-coverage", dest='enable_coverage',
  91                   action="store_const", const='--enable-coverage', default='',
  92                   help="Add --enable-coverage option while configure")
  93
  94 (options, args) = parser.parse_args()
  95
  96 if options.retry:
  97     if options.rebase is None:
  98         raise Exception('You can only use --retry if you also rebase')
  99
 100 testbase = "%s/b%u" % (options.testbase, os.getpid())
 101 test_master = "%s/master" % testbase
 102 test_prefix = "%s/prefix" % testbase
 103 test_tmpdir = "%s/tmp" % testbase
 104 os.environ['TMPDIR'] = test_tmpdir
 105
 106 if options.enable_coverage:
 107     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 108 else:
 109     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 110
 111 if args:
 112     # If we are only running specific test,
 113     # do not sleep randomly to wait for it to start
 114     def random_sleep(low, high):
 115         return 'sleep 1'
 116 else:
 117     def random_sleep(low, high):
 118         return 'sleep {}'.format(random.randint(low, high))
 119
 120 cleanup_list = []
 121
 122 builddirs = {
 123     "ctdb": "ctdb",
 124     "samba": ".",
 125     "samba-nt4": ".",
 126     "samba-fileserver": ".",
 127     "samba-ad-member": ".",
 128     "samba-ad-member-mitkrb5": ".",
 129     "samba-xc": ".",
 130     "samba-o3": ".",
 131     "samba-ctdb": ".",
 132     "samba-libs": ".",
 133     "samba-static": ".",
 134     "samba-none-env": ".",
 135     "samba-ad-dc-1": ".",
 136     "samba-ad-dc-1-mitkrb5": ".",
 137     "samba-ad-dc-2": ".",
 138     "samba-ad-dc-3": ".",
 139     "samba-ad-dc-4": ".",
 140     "samba-ad-dc-4-mitkrb5": ".",
 141     "samba-ad-dc-5": ".",
 142     "samba-ad-dc-6": ".",
 143     "samba-ad-dc-ntvfs": ".",
 144     "samba-ad-dc-backup": ".",
 145     "samba-nopython": ".",
 146     "samba-nopython-py2": ".",
 147     "samba-schemaupgrade": ".",
 148     "ldb": "lib/ldb",
 149     "tdb": "lib/tdb",
 150     "talloc": "lib/talloc",
 151     "replace": "lib/replace",
 152     "tevent": "lib/tevent",
 153     "pidl": "pidl"
 154 }
 155
 156 defaulttasks = list(builddirs.keys())
 157
 158 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 159     defaulttasks.remove("samba-o3")
 160
 161 ctdb_configure_params = " --enable-developer ${PREFIX}"
 162 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 163
 164 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 165 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 166 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 167 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 168 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 169 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 170 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 171
 172
 173 def format_option(name, value=None):
 174     """Format option as str list."""
 175     if value is None:  # boolean option
 176         return [name]
 177     if not isinstance(value, list):  # single value option
 178         value = [value]
 179     # repeatable option
 180     return ['{}={}'.format(name, item) for item in value]
 181
 182
 183 def make_test(
 184         cmd='make test',
 185         FAIL_IMMEDIATELY=1,
 186         TESTS='',
 187         include_envs=None,
 188         exclude_envs=None):
 189
 190     test_options = []
 191     if include_envs:
 192         test_options = format_option('--include-env', include_envs)
 193     if exclude_envs:
 194         test_options = format_option('--exclude-env', exclude_envs)
 195     if test_options:
 196         # join envs options to original test options
 197         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 198
 199     _options = []
 200     if FAIL_IMMEDIATELY:
 201         _options.append('FAIL_IMMEDIATELY=1')
 202     if TESTS:
 203         _options.append("TESTS='{}'".format(TESTS))
 204
 205     return ' '.join([cmd] + _options)
 206
 207
 208 tasks = {
 209     "ctdb": [
 210         ("random-sleep", random_sleep(300, 900)),
 211         ("configure", "./configure " + ctdb_configure_params),
 212         ("make", "make all"),
 213         ("install", "make install"),
 214         ("test", "make autotest"),
 215         ("check-clean-tree", "../script/clean-source-tree.sh"),
 216         ("clean", "make clean"),
 217         ],
 218
 219     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 220     "samba": [
 221         ("random-sleep", random_sleep(300, 900)),
 222         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 223         ("make", "make -j"),
 224         ("test", make_test(exclude_envs=[
 225             "none",
 226             "nt4_dc",
 227             "nt4_dc_schannel",
 228             "nt4_member",
 229             "ad_dc",
 230             "ad_dc_backup",
 231             "ad_dc_ntvfs",
 232             "ad_dc_default",
 233             "ad_dc_slowtests",
 234             "ad_dc_no_nss",
 235             "ad_dc_no_ntlm",
 236             "fl2003dc",
 237             "fl2008dc",
 238             "fl2008r2dc",
 239             "ad_member",
 240             "ad_member_idmap_rid",
 241             "ad_member_idmap_ad",
 242             "ad_member_rfc2307",
 243             "chgdcpass",
 244             "vampire_2000_dc",
 245             "fl2000dc",
 246             "fileserver",
 247             "maptoguest",
 248             "simpleserver",
 249             "backupfromdc",
 250             "restoredc",
 251             "renamedc",
 252             "offlinebackupdc",
 253             "labdc",
 254             "preforkrestartdc",
 255             "proclimitdc",
 256             "promoted_dc",
 257             "vampire_dc",
 258             "rodc",
 259             "ad_dc_default",
 260             "ad_dc_slowtests",
 261             "schema_pair_dc",
 262             "schema_dc",
 263             ])),
 264         ("lcov", LCOV_CMD),
 265         ("install", "make install"),
 266         ("check-clean-tree", "script/clean-source-tree.sh"),
 267         ("clean", "make clean"),
 268         ],
 269
 270     "samba-nt4": [
 271         ("random-sleep", random_sleep(300, 900)),
 272         ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params),
 273         ("make", "make -j"),
 274         ("test", make_test(include_envs=[
 275             "nt4_dc",
 276             "nt4_dc_schannel",
 277             "nt4_member",
 278             ])),
 279         ("lcov", LCOV_CMD),
 280         ("install", "make install"),
 281         ("check-clean-tree", "script/clean-source-tree.sh"),
 282         ("clean", "make clean"),
 283         ],
 284
 285     "samba-simpleserver": [
 286         ("random-sleep", random_sleep(300, 900)),
 287         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
 288         ("make", "make -j"),
 289         ("test", make_test(include_envs=[
 290             "simpleserver",
 291             ])),
 292         ("lcov", LCOV_CMD),
 293         ("check-clean-tree", "script/clean-source-tree.sh"),
 294         ],
 295
 296     "samba-fileserver": [
 297         ("random-sleep", random_sleep(300, 900)),
 298         ("configure", "./configure.developer --without-ad-dc --with-selftest-prefix=./bin/ab" + samba_configure_params),
 299         ("make", "make -j"),
 300         ("test", make_test(include_envs=[
 301             "fileserver",
 302             "maptoguest",
 303             ])),
 304         ("lcov", LCOV_CMD),
 305         ("check-clean-tree", "script/clean-source-tree.sh"),
 306         ],
 307
 308     "samba-ad-member": [
 309         ("random-sleep", random_sleep(300, 900)),
 310         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 311         ("make", "make -j"),
 312         ("test", make_test(include_envs=[
 313             "ad_member",
 314             "ad_member_idmap_rid",
 315             "ad_member_idmap_ad",
 316             "ad_member_rfc2307",
 317             ])),
 318         ("lcov", LCOV_CMD),
 319         ("check-clean-tree", "script/clean-source-tree.sh"),
 320         ],
 321
 322     "samba-ad-dc-1": [
 323         ("random-sleep", random_sleep(1, 1)),
 324         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 325         ("make", "make -j"),
 326         ("test", make_test(include_envs=[
 327             "ad_dc",
 328             "ad_dc_no_nss",
 329             "ad_dc_no_ntlm",
 330             ])),
 331         ("lcov", LCOV_CMD),
 332         ("check-clean-tree", "script/clean-source-tree.sh"),
 333         ],
 334
 335     "samba-ad-dc-2": [
 336         ("random-sleep", random_sleep(1, 1)),
 337         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 338         ("make", "make -j"),
 339         ("test", make_test(include_envs=[
 340             "vampire_dc",
 341             "vampire_2000_dc",
 342             "rodc",
 343             ])),
 344         ("lcov", LCOV_CMD),
 345         ("check-clean-tree", "script/clean-source-tree.sh"),
 346         ],
 347
 348     "samba-ad-dc-3": [
 349         ("random-sleep", random_sleep(1, 1)),
 350         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 351         ("make", "make -j"),
 352         ("test", make_test(include_envs=[
 353             "promoted_dc",
 354             "chgdcpass",
 355             "preforkrestartdc",
 356             "proclimitdc",
 357             ])),
 358         ("lcov", LCOV_CMD),
 359         ("check-clean-tree", "script/clean-source-tree.sh"),
 360         ],
 361
 362     "samba-ad-dc-4": [
 363         ("random-sleep", random_sleep(1, 1)),
 364         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 365         ("make", "make -j"),
 366         ("test", make_test(include_envs=[
 367             "fl2000dc",
 368             "fl2003dc",
 369             "fl2008dc",
 370             "fl2008r2dc",
 371             ])),
 372         ("lcov", LCOV_CMD),
 373         ("check-clean-tree", "script/clean-source-tree.sh"),
 374         ],
 375
 376     "samba-ad-dc-5": [
 377         ("random-sleep", random_sleep(1, 1)),
 378         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 379         ("make", "make -j"),
 380         ("test", make_test(include_envs=["ad_dc_default"])),
 381         ("lcov", LCOV_CMD),
 382         ("check-clean-tree", "script/clean-source-tree.sh"),
 383         ],
 384
 385     "samba-ad-dc-6": [
 386         ("random-sleep", random_sleep(1, 1)),
 387         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 388         ("make", "make -j"),
 389         ("test", make_test(include_envs=["ad_dc_slowtests"])),
 390         ("lcov", LCOV_CMD),
 391         ("check-clean-tree", "script/clean-source-tree.sh"),
 392         ],
 393
 394     "samba-schemaupgrade": [
 395         ("random-sleep", random_sleep(1, 1)),
 396         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 397         ("make", "make -j"),
 398         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 399         ("lcov", LCOV_CMD),
 400         ("check-clean-tree", "script/clean-source-tree.sh"),
 401         ],
 402
 403     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 404     # This is currently the longest task, so we don't randomly delay it.
 405     "samba-ad-dc-ntvfs": [
 406         ("random-sleep", random_sleep(1, 1)),
 407         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 408         ("make", "make -j"),
 409         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 410         ("lcov", LCOV_CMD),
 411         ("check-clean-tree", "script/clean-source-tree.sh"),
 412         ],
 413
 414     # run the backup/restore testenvs separately as they're fairly standalone
 415     # (and CI seems to max out at ~8 different DCs running at once)
 416     "samba-ad-dc-backup": [
 417         ("random-sleep", random_sleep(300, 900)),
 418         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 419         ("make", "make -j"),
 420         ("test", make_test(include_envs=[
 421             "backupfromdc",
 422             "restoredc",
 423             "renamedc",
 424             "offlinebackupdc",
 425             "labdc",
 426             "ad_dc_backup",
 427             ])),
 428         ("lcov", LCOV_CMD),
 429         ("check-clean-tree", "script/clean-source-tree.sh"),
 430         ],
 431
 432     "samba-ad-member-mitkrb5": [
 433         ("random-sleep", random_sleep(300, 900)),
 434         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 435         ("make", "make -j"),
 436         ("test", make_test(include_envs=[
 437             "ad_member",
 438             "ad_member_idmap_rid",
 439             "ad_member_idmap_ad",
 440             "ad_member_rfc2307",
 441             ])),
 442         ("lcov", LCOV_CMD),
 443         ("check-clean-tree", "script/clean-source-tree.sh"),
 444         ],
 445
 446     "samba-ad-dc-1-mitkrb5": [
 447         ("random-sleep", random_sleep(1, 1)),
 448         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 449         ("make", "make -j"),
 450         ("test", make_test(include_envs=[
 451             "ad_dc",
 452             "ad_dc_no_nss",
 453             "ad_dc_no_ntlm",
 454             ])),
 455         ("lcov", LCOV_CMD),
 456         ("check-clean-tree", "script/clean-source-tree.sh"),
 457         ],
 458
 459     "samba-ad-dc-4-mitkrb5": [
 460         ("random-sleep", random_sleep(1, 1)),
 461         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 462         ("make", "make -j"),
 463         ("test", make_test(include_envs=[
 464             "fl2000dc",
 465             "fl2003dc",
 466             "fl2008dc",
 467             "fl2008r2dc",
 468             ])),
 469         ("lcov", LCOV_CMD),
 470         ("check-clean-tree", "script/clean-source-tree.sh"),
 471         ],
 472
 473     "samba-test-only": [
 474         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
 475         ("make", "make -j"),
 476         ("test", make_test(TESTS="${TESTS}")),
 477         ("lcov", LCOV_CMD),
 478         ],
 479
 480     # Test cross-compile infrastructure
 481     "samba-xc": [
 482         ("random-sleep", random_sleep(900, 1500)),
 483         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 484         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 485             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 486         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 487             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 488         ("compare-results", "script/compare_cc_results.py "
 489             "./bin/c4che/default{} "
 490             "./bin-xe/c4che/default{} "
 491             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 492         ],
 493
 494     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 495     "samba-o3": [
 496         ("random-sleep", random_sleep(300, 900)),
 497         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
 498         ("make", "make -j"),
 499         ("test", make_test(cmd='make quicktest', include_envs=["ad_dc"])),
 500         ("lcov", LCOV_CMD),
 501         ("install", "make install"),
 502         ("check-clean-tree", "script/clean-source-tree.sh"),
 503         ("clean", "make clean"),
 504         ],
 505
 506     "samba-ctdb": [
 507         ("random-sleep", random_sleep(900, 1500)),
 508
 509         # make sure we have tdb around:
 510         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 511         ("tdb-make", "cd lib/tdb && make"),
 512         ("tdb-install", "cd lib/tdb && make install"),
 513
 514         # build samba with cluster support (also building ctdb):
 515         ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb"),
 516         ("samba-make", "make"),
 517         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 518         ("samba-install", "make install"),
 519         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 520
 521         # clean up:
 522         ("check-clean-tree", "script/clean-source-tree.sh"),
 523         ("clean", "make clean"),
 524         ("ctdb-clean", "cd ./ctdb && make clean"),
 525         ],
 526
 527     "samba-libs": [
 528         ("random-sleep", random_sleep(300, 900)),
 529         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 530         ("talloc-make", "cd lib/talloc && make"),
 531         ("talloc-install", "cd lib/talloc && make install"),
 532
 533         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 534         ("tdb-make", "cd lib/tdb && make"),
 535         ("tdb-install", "cd lib/tdb && make install"),
 536
 537         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 538         ("tevent-make", "cd lib/tevent && make"),
 539         ("tevent-install", "cd lib/tevent && make install"),
 540
 541         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 542         ("ldb-make", "cd lib/ldb && make"),
 543         ("ldb-install", "cd lib/ldb && make install"),
 544
 545         ("nondevel-configure", "./configure ${PREFIX}"),
 546         ("nondevel-make", "make -j"),
 547         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 548         ("nondevel-install", "make install"),
 549         ("nondevel-dist", "make dist"),
 550
 551         # retry with all modules shared
 552         ("allshared-distclean", "make distclean"),
 553         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 554         ("allshared-make", "make -j"),
 555         ],
 556
 557     "samba-none-env": [
 558         ("random-sleep", random_sleep(1, 1)),
 559         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 560         ("make", "make -j"),
 561         ("test", make_test(include_envs=["none"])),
 562         ("lcov", LCOV_CMD),
 563         ],
 564
 565     "samba-static": [
 566         ("random-sleep", random_sleep(1, 1)),
 567         # build with all modules static
 568         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 569         ("allstatic-make", "make -j"),
 570         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 571         ("lcov", LCOV_CMD),
 572
 573         # retry without any required modules
 574         ("none-distclean", "make distclean"),
 575         ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 576         ("none-make", "make -j"),
 577
 578         # retry with nonshared smbd and smbtorture
 579         ("nonshared-distclean", "make distclean"),
 580         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 581         ("nonshared-make", "make -j"),
 582         ],
 583
 584     # Test Samba without python still builds.  When this test fails
 585     # due to more use of Python, the expectations is that the newly
 586     # failing part of the code should be disabled when
 587     # --disable-python is set (rather than major work being done to
 588     # support this environment).  The target here is for vendors
 589     # shipping a minimal smbd.
 590     "samba-nopython": [
 591         ("random-sleep", random_sleep(300, 900)),
 592         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 593         ("make", "make -j"),
 594         ("install", "make install"),
 595         ("find-python", "script/find_python.sh ${PREFIX}"),
 596         ("test", "make test-nopython"),
 597         ("lcov", LCOV_CMD),
 598         ("check-clean-tree", "script/clean-source-tree.sh"),
 599         ("clean", "make clean"),
 600
 601         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 602         ("talloc-make", "cd lib/talloc && make"),
 603         ("talloc-install", "cd lib/talloc && make install"),
 604
 605         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 606         ("tdb-make", "cd lib/tdb && make"),
 607         ("tdb-install", "cd lib/tdb && make install"),
 608
 609         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 610         ("tevent-make", "cd lib/tevent && make"),
 611         ("tevent-install", "cd lib/tevent && make install"),
 612
 613         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 614         ("ldb-make", "cd lib/ldb && make"),
 615         ("ldb-install", "cd lib/ldb && make install"),
 616
 617         # retry against installed library packages
 618         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 619         ("libs-make", "make -j"),
 620         ("libs-install", "make install"),
 621         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 622         ("libs-clean", "make clean"),
 623         ],
 624
 625     # check we can do the same thing using python2
 626     "samba-nopython-py2": [
 627         ("random-sleep", random_sleep(300, 900)),
 628         ("configure", "PYTHON=python2 ./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 629         ("make", "PYTHON=python2 make -j"),
 630         ("install", "PYTHON=python2 make install"),
 631         ("find-python", "script/find_python.sh ${PREFIX}"),
 632         ("test", "make test-nopython"),
 633         ("lcov", LCOV_CMD),
 634         ("check-clean-tree", "script/clean-source-tree.sh"),
 635         ("clean", "PYTHON=python2 make clean"),
 636
 637         ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 638         ("talloc-make", "cd lib/talloc && PYTHON=python2 make"),
 639         ("talloc-install", "cd lib/talloc && PYTHON=python2 make install"),
 640
 641         ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 642         ("tdb-make", "cd lib/tdb && PYTHON=python2 make"),
 643         ("tdb-install", "cd lib/tdb && PYTHON=python2 make install"),
 644
 645         ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 646         ("tevent-make", "cd lib/tevent && PYTHON=python2 make"),
 647         ("tevent-install", "cd lib/tevent && PYTHON=python2 make install"),
 648
 649         ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 650         ("ldb-make", "cd lib/ldb && PYTHON=python2 make"),
 651         ("ldb-install", "cd lib/ldb && PYTHON=python2 make install"),
 652
 653         # retry against installed library packages
 654         ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 655         ("libs-make", "PYTHON=python2 make -j"),
 656         ("libs-install", "PYTHON=python2 make install"),
 657         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 658         ("libs-clean", "PYTHON=python2 make clean"),
 659         ],
 660
 661     "ldb": [
 662         ("random-sleep", random_sleep(60, 600)),
 663         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 664         ("make", "make"),
 665         ("install", "make install"),
 666         ("test", "make test"),
 667         ("lcov", LCOV_CMD),
 668         ("clean", "make clean"),
 669         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 670         ("make-no-lmdb", "make"),
 671         ("test-no-lmdb", "make test"),
 672         ("lcov-no-lmdb", LCOV_CMD),
 673         ("install-no-lmdb", "make install"),
 674         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 675         ("distcheck", "make distcheck"),
 676         ("clean", "make clean"),
 677         ],
 678
 679     "tdb": [
 680         ("random-sleep", random_sleep(60, 600)),
 681         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 682         ("make", "make"),
 683         ("install", "make install"),
 684         ("test", "make test"),
 685         ("lcov", LCOV_CMD),
 686         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 687         ("distcheck", "make distcheck"),
 688         ("clean", "make clean"),
 689         ],
 690
 691     "talloc": [
 692         ("random-sleep", random_sleep(60, 600)),
 693         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 694         ("make", "make"),
 695         ("install", "make install"),
 696         ("test", "make test"),
 697         ("lcov", LCOV_CMD),
 698         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 699         ("distcheck", "make distcheck"),
 700         ("clean", "make clean"),
 701         ],
 702
 703     "replace": [
 704         ("random-sleep", random_sleep(60, 600)),
 705         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 706         ("make", "make"),
 707         ("install", "make install"),
 708         ("test", "make test"),
 709         ("lcov", LCOV_CMD),
 710         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 711         ("distcheck", "make distcheck"),
 712         ("clean", "make clean"),
 713         ],
 714
 715     "tevent": [
 716         ("random-sleep", random_sleep(60, 600)),
 717         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 718         ("make", "make"),
 719         ("install", "make install"),
 720         ("test", "make test"),
 721         ("lcov", LCOV_CMD),
 722         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 723         ("distcheck", "make distcheck"),
 724         ("clean", "make clean"),
 725         ],
 726
 727     "pidl": [
 728         ("random-sleep", random_sleep(60, 600)),
 729         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 730         ("touch", "touch *.yp"),
 731         ("make", "make"),
 732         ("test", "make test"),
 733         ("install", "make install"),
 734         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 735         ("check-clean-tree", "../script/clean-source-tree.sh"),
 736         ("clean", "make clean"),
 737         ],
 738
 739     # these are useful for debugging autobuild
 740     'pass': [("pass", 'echo passing && /bin/true')],
 741     'fail': [("fail", 'echo failing && /bin/false')],
 742 }
 743
 744
 745 def do_print(msg):
 746     print("%s" % msg)
 747     sys.stdout.flush()
 748     sys.stderr.flush()
 749
 750
 751 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 752     if show is None:
 753         show = options.verbose
 754     if show:
 755         do_print("Running: '%s' in '%s'" % (cmd, dir))
 756     if output:
 757         out = check_output([cmd], shell=True, cwd=dir)
 758         return out.decode(encoding='utf-8', errors='backslashreplace')
 759     elif checkfail:
 760         return check_call(cmd, shell=True, cwd=dir)
 761     else:
 762         return call(cmd, shell=True, cwd=dir)
 763
 764
 765 class builder(object):
 766     '''handle build of one directory'''
 767
 768     def __init__(self, name, sequence, cp=True):
 769         self.name = name
 770         self.dir = builddirs.get(name, '.')
 771         self.tag = self.name.replace('/', '_')
 772         self.sequence = sequence
 773         self.next = 0
 774         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 775         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 776         if options.verbose:
 777             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 778             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 779         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 780         self.stdout = open(self.stdout_path, 'w')
 781         self.stderr = open(self.stderr_path, 'w')
 782         self.stdin  = open("/dev/null", 'r')
 783         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 784         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
 785         self.prefix = "%s/%s" % (test_prefix, self.tag)
 786         run_cmd("rm -rf %s" % self.test_source_dir)
 787         run_cmd("rm -rf %s" % self.prefix)
 788         if cp:
 789             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 790         else:
 791             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 792         self.start_next()
 793
 794     def start_next(self):
 795         if self.next == len(self.sequence):
 796             if not options.nocleanup:
 797                 run_cmd("rm -rf %s" % self.test_source_dir)
 798                 run_cmd("rm -rf %s" % self.prefix)
 799             do_print('%s: Completed OK' % self.name)
 800             self.done = True
 801             return
 802         (self.stage, self.cmd) = self.sequence[self.next]
 803         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 804         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 805         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 806         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 807         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 808         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 809         self.cmd = self.cmd.replace("${NAME}", self.name)
 810         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 811         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
 812         self.proc = Popen(self.cmd, shell=True,
 813                           close_fds=True, cwd=self.cwd,
 814                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 815         self.next += 1
 816
 817
 818 class buildlist(object):
 819     '''handle build of multiple directories'''
 820
 821     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 822         self.tail_proc = None
 823         self.retry = None
 824         if not tasknames:
 825             if options.restrict_tests:
 826                 tasknames = ["samba-test-only"]
 827             else:
 828                 tasknames = defaulttasks
 829
 830         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
 831
 832         if options.retry:
 833             rebase_remote = "rebaseon"
 834             retry_task = [("retry",
 835                             '''set -e
 836                             git remote add -t %s %s %s
 837                             git fetch %s
 838                             while :; do
 839                               sleep 60
 840                               git describe %s/%s > old_remote_branch.desc
 841                               git fetch %s
 842                               git describe %s/%s > remote_branch.desc
 843                               diff old_remote_branch.desc remote_branch.desc
 844                             done
 845                            ''' % (
 846                                rebase_branch, rebase_remote, rebase_url,
 847                                rebase_remote,
 848                                rebase_remote, rebase_branch,
 849                                rebase_remote,
 850                                rebase_remote, rebase_branch
 851                             ))]
 852
 853             self.retry = builder('retry', retry_task, cp=False)
 854             self.need_retry = False
 855
 856     def kill_kids(self):
 857         if self.tail_proc is not None:
 858             self.tail_proc.terminate()
 859             self.tail_proc.wait()
 860             self.tail_proc = None
 861         if self.retry is not None:
 862             self.retry.proc.terminate()
 863             self.retry.proc.wait()
 864             self.retry = None
 865         for b in self.tlist:
 866             if b.proc is not None:
 867                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 868                 b.proc.terminate()
 869                 b.proc.wait()
 870                 b.proc = None
 871
 872     def wait_one(self):
 873         while True:
 874             none_running = True
 875             for b in self.tlist:
 876                 if b.proc is None:
 877                     continue
 878                 none_running = False
 879                 b.status = b.proc.poll()
 880                 if b.status is None:
 881                     continue
 882                 b.proc = None
 883                 return b
 884             if options.retry:
 885                 ret = self.retry.proc.poll()
 886                 if ret is not None:
 887                     self.need_retry = True
 888                     self.retry = None
 889                     return None
 890             if none_running:
 891                 return None
 892             time.sleep(0.1)
 893
 894     def run(self):
 895         while True:
 896             b = self.wait_one()
 897             if options.retry and self.need_retry:
 898                 self.kill_kids()
 899                 do_print("retry needed")
 900                 return (0, None, None, None, "retry")
 901             if b is None:
 902                 break
 903             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 904                 self.kill_kids()
 905                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 906             b.start_next()
 907         self.kill_kids()
 908         return (0, None, None, None, "All OK")
 909
 910     def write_system_info(self, filename):
 911         with open(filename, 'w') as f:
 912             for cmd in ['uname -a',
 913                         'lsb_release -a',
 914                         'free',
 915                         'mount',
 916                         'cat /proc/cpuinfo',
 917                         'cc --version',
 918                         'df -m .',
 919                         'df -m %s' % testbase]:
 920                 out = run_cmd(cmd, output=True, checkfail=False)
 921                 print('### %s' % cmd, file=f)
 922                 print(out, file=f)
 923                 print(file=f)
 924
 925     def tarlogs(self, fname):
 926         with tarfile.open(fname, "w:gz") as tar:
 927             for b in self.tlist:
 928                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 929                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 930             if os.path.exists("autobuild.log"):
 931                 tar.add("autobuild.log")
 932             filename = 'system-info.txt'
 933             self.write_system_info(filename)
 934             tar.add(filename)
 935
 936     def remove_logs(self):
 937         for b in self.tlist:
 938             os.unlink(b.stdout_path)
 939             os.unlink(b.stderr_path)
 940
 941     def start_tail(self):
 942         cmd = ["tail", "-f"]
 943         for b in self.tlist:
 944             cmd.append(b.stdout_path)
 945             cmd.append(b.stderr_path)
 946         self.tail_proc = Popen(cmd, close_fds=True)
 947
 948
 949 def cleanup():
 950     if options.nocleanup:
 951         return
 952     run_cmd("stat %s || true" % test_tmpdir, show=True)
 953     run_cmd("stat %s" % testbase, show=True)
 954     do_print("Cleaning up %r" % cleanup_list)
 955     for d in cleanup_list:
 956         run_cmd("rm -rf %s" % d)
 957
 958
 959 def daemonize(logfile):
 960     pid = os.fork()
 961     if pid == 0:  # Parent
 962         os.setsid()
 963         pid = os.fork()
 964         if pid != 0:  # Actual daemon
 965             os._exit(0)
 966     else:  # Grandparent
 967         os._exit(0)
 968
 969     import resource      # Resource usage information.
 970     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 971     if maxfd == resource.RLIM_INFINITY:
 972         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 973     for fd in range(0, maxfd):
 974         try:
 975             os.close(fd)
 976         except OSError:
 977             pass
 978     os.open(logfile, os.O_RDWR | os.O_CREAT)
 979     os.dup2(0, 1)
 980     os.dup2(0, 2)
 981
 982
 983 def write_pidfile(fname):
 984     '''write a pid file, cleanup on exit'''
 985     with open(fname, mode='w') as f:
 986         f.write("%u\n" % os.getpid())
 987
 988
 989 def rebase_tree(rebase_url, rebase_branch="master"):
 990     rebase_remote = "rebaseon"
 991     do_print("Rebasing on %s" % rebase_url)
 992     run_cmd("git describe HEAD", show=True, dir=test_master)
 993     run_cmd("git remote add -t %s %s %s" %
 994             (rebase_branch, rebase_remote, rebase_url),
 995             show=True, dir=test_master)
 996     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 997     if options.fix_whitespace:
 998         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 999                 (rebase_remote, rebase_branch),
1000                 show=True, dir=test_master)
1001     else:
1002         run_cmd("git rebase --force-rebase %s/%s" %
1003                 (rebase_remote, rebase_branch),
1004                 show=True, dir=test_master)
1005     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1006                    (rebase_remote, rebase_branch),
1007                    dir=test_master, output=True)
1008     if diff == '':
1009         do_print("No differences between HEAD and %s/%s - exiting" %
1010                  (rebase_remote, rebase_branch))
1011         sys.exit(0)
1012     run_cmd("git describe %s/%s" %
1013             (rebase_remote, rebase_branch),
1014             show=True, dir=test_master)
1015     run_cmd("git describe HEAD", show=True, dir=test_master)
1016     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1017             (rebase_remote, rebase_branch),
1018             show=True, dir=test_master)
1019
1020
1021 def push_to(push_url, push_branch="master"):
1022     push_remote = "pushto"
1023     do_print("Pushing to %s" % push_url)
1024     if options.mark:
1025         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1026         run_cmd("git commit --amend -c HEAD", dir=test_master)
1027         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1028         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1029     run_cmd("git remote add -t %s %s %s" %
1030             (push_branch, push_remote, push_url),
1031             show=True, dir=test_master)
1032     run_cmd("git push %s +HEAD:%s" %
1033             (push_remote, push_branch),
1034             show=True, dir=test_master)
1035
1036
1037 def send_email(subject, text, log_tar):
1038     if options.email is None:
1039         do_print("not sending email because the recipient is not set")
1040         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1041                  (subject, text))
1042         return
1043     outer = MIMEMultipart()
1044     outer['Subject'] = subject
1045     outer['To'] = options.email
1046     outer['From'] = options.email_from
1047     outer['Date'] = email.utils.formatdate(localtime=True)
1048     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1049     outer.attach(MIMEText(text, 'plain'))
1050     if options.attach_logs:
1051         with open(log_tar, 'rb') as fp:
1052             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1053         # Set the filename parameter
1054         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1055         outer.attach(msg)
1056     content = outer.as_string()
1057     s = smtplib.SMTP(options.email_server)
1058     email_user = os.getenv('SMTP_USERNAME')
1059     email_password = os.getenv('SMTP_PASSWORD')
1060     if email_user is not None:
1061         s.starttls()
1062         s.login(email_user, email_password)
1063
1064     s.sendmail(options.email_from, [options.email], content)
1065     s.set_debuglevel(1)
1066     s.quit()
1067
1068
1069 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1070                   elapsed_time, log_base=None, add_log_tail=True):
1071     '''send an email to options.email about the failure'''
1072     elapsed_minutes = elapsed_time / 60.0
1073     if log_base is None:
1074         log_base = gitroot
1075     text = '''
1076 Dear Developer,
1077
1078 Your autobuild on %s failed after %.1f minutes
1079 when trying to test %s with the following error:
1080
1081    %s
1082
1083 the autobuild has been abandoned. Please fix the error and resubmit.
1084
1085 A summary of the autobuild process is here:
1086
1087   %s/autobuild.log
1088 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1089
1090     if options.restrict_tests:
1091         text += """
1092 The build was restricted to tests matching %s\n""" % options.restrict_tests
1093
1094     if failed_task != 'rebase':
1095         text += '''
1096 You can see logs of the failed task here:
1097
1098   %s/%s.stdout
1099   %s/%s.stderr
1100
1101 or you can get full logs of all tasks in this job here:
1102
1103   %s/logs.tar.gz
1104
1105 The top commit for the tree that was built was:
1106
1107 %s
1108
1109 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1110
1111     if add_log_tail:
1112         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1113         lines = f.readlines()
1114         log_tail = "".join(lines[-50:])
1115         num_lines = len(lines)
1116         if num_lines < 50:
1117             # Also include stderr (compile failures) if < 50 lines of stdout
1118             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1119             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1120
1121         text += '''
1122 The last 50 lines of log messages:
1123
1124 %s
1125     ''' % log_tail
1126         f.close()
1127
1128     logs = os.path.join(gitroot, 'logs.tar.gz')
1129     send_email('autobuild[%s] failure on %s for task %s during %s'
1130                % (options.branch, platform.node(), failed_task, failed_stage),
1131                text, logs)
1132
1133
1134 def email_success(elapsed_time, log_base=None):
1135     '''send an email to options.email about a successful build'''
1136     if log_base is None:
1137         log_base = gitroot
1138     text = '''
1139 Dear Developer,
1140
1141 Your autobuild on %s has succeeded after %.1f minutes.
1142
1143 ''' % (platform.node(), elapsed_time / 60.)
1144
1145     if options.restrict_tests:
1146         text += """
1147 The build was restricted to tests matching %s\n""" % options.restrict_tests
1148
1149     if options.keeplogs:
1150         text += '''
1151
1152 you can get full logs of all tasks in this job here:
1153
1154   %s/logs.tar.gz
1155
1156 ''' % log_base
1157
1158     text += '''
1159 The top commit for the tree that was built was:
1160
1161 %s
1162 ''' % top_commit_msg
1163
1164     logs = os.path.join(gitroot, 'logs.tar.gz')
1165     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1166                text, logs)
1167
1168
1169 # get the top commit message, for emails
1170 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1171
1172 try:
1173     os.makedirs(testbase)
1174 except Exception as reason:
1175     raise Exception("Unable to create %s : %s" % (testbase, reason))
1176 cleanup_list.append(testbase)
1177
1178 if options.daemon:
1179     logfile = os.path.join(testbase, "log")
1180     do_print("Forking into the background, writing progress to %s" % logfile)
1181     daemonize(logfile)
1182
1183 write_pidfile(gitroot + "/autobuild.pid")
1184
1185 start_time = time.time()
1186
1187 while True:
1188     try:
1189         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1190         os.makedirs(test_tmpdir)
1191         # The waf uninstall code removes empty directories all the way
1192         # up the tree.  Creating a file in test_tmpdir stops it from
1193         # being removed.
1194         run_cmd("touch %s" % os.path.join(test_tmpdir,
1195                                           ".directory-is-not-empty"), show=True)
1196         run_cmd("stat %s" % test_tmpdir, show=True)
1197         run_cmd("stat %s" % testbase, show=True)
1198         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1199     except Exception:
1200         cleanup()
1201         raise
1202
1203     try:
1204         if options.rebase is not None:
1205             rebase_tree(options.rebase, rebase_branch=options.branch)
1206     except Exception:
1207         cleanup_list.append(gitroot + "/autobuild.pid")
1208         cleanup()
1209         elapsed_time = time.time() - start_time
1210         email_failure(-1, 'rebase', 'rebase', 'rebase',
1211                       'rebase on %s failed' % options.branch,
1212                       elapsed_time, log_base=options.log_base)
1213         sys.exit(1)
1214
1215     try:
1216         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1217         if options.tail:
1218             blist.start_tail()
1219         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1220         if status != 0 or errstr != "retry":
1221             break
1222         cleanup()
1223     except Exception:
1224         cleanup()
1225         raise
1226
1227 cleanup_list.append(gitroot + "/autobuild.pid")
1228
1229 do_print(errstr)
1230
1231 blist.kill_kids()
1232 if options.tail:
1233     do_print("waiting for tail to flush")
1234     time.sleep(1)
1235
1236 elapsed_time = time.time() - start_time
1237 if status == 0:
1238     if options.passcmd is not None:
1239         do_print("Running passcmd: %s" % options.passcmd)
1240         run_cmd(options.passcmd, dir=test_master)
1241     if options.pushto is not None:
1242         push_to(options.pushto, push_branch=options.branch)
1243     if options.keeplogs or options.attach_logs:
1244         blist.tarlogs("logs.tar.gz")
1245         do_print("Logs in logs.tar.gz")
1246     if options.always_email:
1247         email_success(elapsed_time, log_base=options.log_base)
1248     blist.remove_logs()
1249     cleanup()
1250     do_print(errstr)
1251     sys.exit(0)
1252
1253 # something failed, gather a tar of the logs
1254 blist.tarlogs("logs.tar.gz")
1255
1256 if options.email is not None:
1257     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1258                   elapsed_time, log_base=options.log_base)
1259 else:
1260     elapsed_minutes = elapsed_time / 60.0
1261     print('''
1262
1263 ####################################################################
1264
1265 AUTOBUILD FAILURE
1266
1267 Your autobuild[%s] on %s failed after %.1f minutes
1268 when trying to test %s with the following error:
1269
1270    %s
1271
1272 the autobuild has been abandoned. Please fix the error and resubmit.
1273
1274 ####################################################################
1275
1276 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1277
1278 cleanup()
1279 do_print(errstr)
1280 do_print("Logs in logs.tar.gz")
1281 sys.exit(status)