script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.isdir(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--passcmd", help="command to run on success", default=None)
  60 parser.add_option("--verbose", help="show all commands as they are run",
  61                   default=False, action="store_true")
  62 parser.add_option("--rebase", help="rebase on the given tree before testing",
  63                   default=None, type='str')
  64 parser.add_option("--pushto", help="push to a git url on success",
  65                   default=None, type='str')
  66 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  67                   default=False, action="store_true")
  68 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  69                   default=False, action="store_true")
  70 parser.add_option("--retry", help="automatically retry if master changes",
  71                   default=False, action="store_true")
  72 parser.add_option("--email", help="send email to the given address on failure",
  73                   type='str', default=None)
  74 parser.add_option("--email-from", help="send email from the given address",
  75                   type='str', default="autobuild@samba.org")
  76 parser.add_option("--email-server", help="send email via the given server",
  77                   type='str', default='localhost')
  78 parser.add_option("--always-email", help="always send email, even on success",
  79                   action="store_true")
  80 parser.add_option("--daemon", help="daemonize after initial setup",
  81                   action="store_true")
  82 parser.add_option("--branch", help="the branch to work on (default=master)",
  83                   default="master", type='str')
  84 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  85                   default=gitroot, type='str')
  86 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  87                   default=False, action="store_true")
  88 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  89                   default='')
  90 parser.add_option("--enable-coverage", dest='enable_coverage',
  91                   action="store_const", const='--enable-coverage', default='',
  92                   help="Add --enable-coverage option while configure")
  93
  94 (options, args) = parser.parse_args()
  95
  96 if options.retry:
  97     if options.rebase is None:
  98         raise Exception('You can only use --retry if you also rebase')
  99
 100 testbase = "%s/b%u" % (options.testbase, os.getpid())
 101 test_master = "%s/master" % testbase
 102 test_prefix = "%s/prefix" % testbase
 103 test_tmpdir = "%s/tmp" % testbase
 104 os.environ['TMPDIR'] = test_tmpdir
 105
 106 if options.enable_coverage:
 107     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 108 else:
 109     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 110
 111 if args:
 112     # If we are only running specific test,
 113     # do not sleep randomly to wait for it to start
 114     def random_sleep(low, high):
 115         return 'sleep 1'
 116 else:
 117     def random_sleep(low, high):
 118         return 'sleep {}'.format(random.randint(low, high))
 119
 120 cleanup_list = []
 121
 122 builddirs = {
 123     "ctdb": "ctdb",
 124     "samba": ".",
 125     "samba-mitkrb5": ".",
 126     "samba-nt4": ".",
 127     "samba-fileserver": ".",
 128     "samba-ktest-heimdal": ".",
 129     "samba-admem": ".",
 130     "samba-admem-mit": ".",
 131     "samba-xc": ".",
 132     "samba-o3": ".",
 133     "samba-ctdb": ".",
 134     "samba-libs": ".",
 135     "samba-static": ".",
 136     "samba-none-env": ".",
 137     "samba-ad-dc-1": ".",
 138     "samba-ad-dc-1-mitkrb5": ".",
 139     "samba-ad-dc-2": ".",
 140     "samba-ad-dc-3": ".",
 141     "samba-ad-dc-4": ".",
 142     "samba-ad-dc-4-mitkrb5": ".",
 143     "samba-ad-dc-5": ".",
 144     "samba-ad-dc-6": ".",
 145     "samba-ad-dc-ntvfs": ".",
 146     "samba-ad-dc-backup": ".",
 147     "samba-nopython": ".",
 148     "samba-nopython-py2": ".",
 149     "samba-schemaupgrade": ".",
 150     "ldb": "lib/ldb",
 151     "tdb": "lib/tdb",
 152     "talloc": "lib/talloc",
 153     "replace": "lib/replace",
 154     "tevent": "lib/tevent",
 155     "pidl": "pidl"
 156 }
 157
 158 defaulttasks = list(builddirs.keys())
 159
 160 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 161     defaulttasks.remove("samba-o3")
 162
 163 ctdb_configure_params = " --enable-developer ${PREFIX}"
 164 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 165
 166 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 167 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 168 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 169 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 170 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 171 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 172 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 173
 174
 175 def format_option(name, value=None):
 176     """Format option as str list."""
 177     if value is None:  # boolean option
 178         return [name]
 179     if not isinstance(value, list):  # single value option
 180         value = [value]
 181     # repeatable option
 182     return ['{}={}'.format(name, item) for item in value]
 183
 184
 185 def make_test(
 186         cmd='make test',
 187         FAIL_IMMEDIATELY=1,
 188         TESTS='',
 189         include_envs=None,
 190         exclude_envs=None):
 191
 192     test_options = []
 193     if include_envs:
 194         test_options = format_option('--include-env', include_envs)
 195     if exclude_envs:
 196         test_options = format_option('--exclude-env', exclude_envs)
 197     if test_options:
 198         # join envs options to original test options
 199         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 200
 201     _options = []
 202     if FAIL_IMMEDIATELY:
 203         _options.append('FAIL_IMMEDIATELY=1')
 204     if TESTS:
 205         _options.append("TESTS='{}'".format(TESTS))
 206
 207     return ' '.join([cmd] + _options)
 208
 209
 210 tasks = {
 211     "ctdb": [
 212         ("random-sleep", random_sleep(300, 900)),
 213         ("configure", "./configure " + ctdb_configure_params),
 214         ("make", "make all"),
 215         ("install", "make install"),
 216         ("test", "make autotest"),
 217         ("check-clean-tree", "../script/clean-source-tree.sh"),
 218         ("clean", "make clean"),
 219         ],
 220
 221     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 222     "samba": [
 223         ("random-sleep", random_sleep(300, 900)),
 224         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 225         ("make", "make -j"),
 226         ("test", make_test(exclude_envs=[
 227             "none",
 228             "nt4_dc",
 229             "nt4_dc_schannel",
 230             "nt4_member",
 231             "ad_dc",
 232             "ad_dc_backup",
 233             "ad_dc_ntvfs",
 234             "ad_dc_default",
 235             "ad_dc_slowtests",
 236             "ad_dc_no_nss",
 237             "ad_dc_no_ntlm",
 238             "fl2003dc",
 239             "fl2008dc",
 240             "fl2008r2dc",
 241             "ad_member",
 242             "ad_member_idmap_rid",
 243             "ad_member_idmap_ad",
 244             "ad_member_rfc2307",
 245             "chgdcpass",
 246             "vampire_2000_dc",
 247             "fl2000dc",
 248             "fileserver",
 249             "maptoguest",
 250             "simpleserver",
 251             "backupfromdc",
 252             "restoredc",
 253             "renamedc",
 254             "offlinebackupdc",
 255             "labdc",
 256             "preforkrestartdc",
 257             "proclimitdc",
 258             "promoted_dc",
 259             "vampire_dc",
 260             "rodc",
 261             "ad_dc_default",
 262             "ad_dc_slowtests",
 263             "schema_pair_dc",
 264             "schema_dc",
 265             ])),
 266         ("lcov", LCOV_CMD),
 267         ("install", "make install"),
 268         ("check-clean-tree", "script/clean-source-tree.sh"),
 269         ("clean", "make clean"),
 270         ],
 271
 272     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 273     "samba-mitkrb5": [
 274         ("random-sleep", random_sleep(300, 900)),
 275         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 276         ("make", "make -j"),
 277         ("test", make_test(exclude_envs=[
 278             "none",
 279             "nt4_dc",
 280             "nt4_dc_schannel",
 281             "nt4_member",
 282             "ad_dc",
 283             "ad_dc_backup",
 284             "ad_dc_ntvfs",
 285             "ad_dc_default",
 286             "ad_dc_slowtests",
 287             "ad_dc_no_nss",
 288             "ad_dc_no_ntlm",
 289             "fl2003dc",
 290             "fl2008dc",
 291             "fl2008r2dc",
 292             "ad_member",
 293             "ad_member_idmap_rid",
 294             "ad_member_idmap_ad",
 295             "ad_member_rfc2307",
 296             "chgdcpass",
 297             "vampire_2000_dc",
 298             "fl2000dc",
 299             "fileserver",
 300             "maptoguest",
 301             "simpleserver",
 302             "backupfromdc",
 303             "restoredc",
 304             "renamedc",
 305             "offlinebackupdc",
 306             "labdc",
 307             "preforkrestartdc",
 308             "proclimitdc",
 309             "promoted_dc",
 310             "vampire_dc",
 311             "rodc",
 312             "ad_dc_default",
 313             "ad_dc_slowtests",
 314             "schema_pair_dc",
 315             "schema_dc",
 316             ])),
 317         ("lcov", LCOV_CMD),
 318         ("install", "make install"),
 319         ("check-clean-tree", "script/clean-source-tree.sh"),
 320         ("clean", "make clean"),
 321         ],
 322
 323     "samba-nt4": [
 324         ("random-sleep", random_sleep(300, 900)),
 325         ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params),
 326         ("make", "make -j"),
 327         ("test", make_test(include_envs=[
 328             "nt4_dc",
 329             "nt4_dc_schannel",
 330             "nt4_member",
 331             ])),
 332         ("lcov", LCOV_CMD),
 333         ("install", "make install"),
 334         ("check-clean-tree", "script/clean-source-tree.sh"),
 335         ("clean", "make clean"),
 336         ],
 337
 338     "samba-simpleserver": [
 339         ("random-sleep", random_sleep(300, 900)),
 340         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
 341         ("make", "make -j"),
 342         ("test", make_test(include_envs=[
 343             "simpleserver",
 344             ])),
 345         ("lcov", LCOV_CMD),
 346         ("check-clean-tree", "script/clean-source-tree.sh"),
 347         ],
 348
 349     "samba-fileserver": [
 350         ("random-sleep", random_sleep(300, 900)),
 351         ("configure", "./configure.developer --without-ad-dc --with-selftest-prefix=./bin/ab" + samba_configure_params),
 352         ("make", "make -j"),
 353         ("test", make_test(include_envs=[
 354             "fileserver",
 355             "maptoguest",
 356             ])),
 357         ("lcov", LCOV_CMD),
 358         ("check-clean-tree", "script/clean-source-tree.sh"),
 359         ],
 360
 361     "samba-ktest-heimdal": [
 362         ("random-sleep", random_sleep(300, 900)),
 363         ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab" + samba_configure_params),
 364         ("make", "make -j"),
 365         ("test", make_test(include_envs=[
 366             "ktest",
 367             ])),
 368         ("lcov", LCOV_CMD),
 369         ("check-clean-tree", "script/clean-source-tree.sh"),
 370         ],
 371
 372     "samba-admem": [
 373         ("random-sleep", random_sleep(300, 900)),
 374         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 375         ("make", "make -j"),
 376         ("test", make_test(include_envs=[
 377             "ad_member",
 378             "ad_member_idmap_rid",
 379             "ad_member_idmap_ad",
 380             "ad_member_rfc2307",
 381             ])),
 382         ("lcov", LCOV_CMD),
 383         ("check-clean-tree", "script/clean-source-tree.sh"),
 384         ],
 385
 386     "samba-ad-dc-1": [
 387         ("random-sleep", random_sleep(1, 1)),
 388         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 389         ("make", "make -j"),
 390         ("test", make_test(include_envs=[
 391             "ad_dc",
 392             "ad_dc_no_nss",
 393             "ad_dc_no_ntlm",
 394             ])),
 395         ("lcov", LCOV_CMD),
 396         ("check-clean-tree", "script/clean-source-tree.sh"),
 397         ],
 398
 399     "samba-ad-dc-2": [
 400         ("random-sleep", random_sleep(1, 1)),
 401         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 402         ("make", "make -j"),
 403         ("test", make_test(include_envs=[
 404             "vampire_dc",
 405             "vampire_2000_dc",
 406             "rodc",
 407             ])),
 408         ("lcov", LCOV_CMD),
 409         ("check-clean-tree", "script/clean-source-tree.sh"),
 410         ],
 411
 412     "samba-ad-dc-3": [
 413         ("random-sleep", random_sleep(1, 1)),
 414         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 415         ("make", "make -j"),
 416         ("test", make_test(include_envs=[
 417             "promoted_dc",
 418             "chgdcpass",
 419             "preforkrestartdc",
 420             "proclimitdc",
 421             ])),
 422         ("lcov", LCOV_CMD),
 423         ("check-clean-tree", "script/clean-source-tree.sh"),
 424         ],
 425
 426     "samba-ad-dc-4": [
 427         ("random-sleep", random_sleep(1, 1)),
 428         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 429         ("make", "make -j"),
 430         ("test", make_test(include_envs=[
 431             "fl2000dc",
 432             "fl2003dc",
 433             "fl2008dc",
 434             "fl2008r2dc",
 435             ])),
 436         ("lcov", LCOV_CMD),
 437         ("check-clean-tree", "script/clean-source-tree.sh"),
 438         ],
 439
 440     "samba-ad-dc-5": [
 441         ("random-sleep", random_sleep(1, 1)),
 442         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 443         ("make", "make -j"),
 444         ("test", make_test(include_envs=["ad_dc_default"])),
 445         ("lcov", LCOV_CMD),
 446         ("check-clean-tree", "script/clean-source-tree.sh"),
 447         ],
 448
 449     "samba-ad-dc-6": [
 450         ("random-sleep", random_sleep(1, 1)),
 451         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 452         ("make", "make -j"),
 453         ("test", make_test(include_envs=["ad_dc_slowtests"])),
 454         ("lcov", LCOV_CMD),
 455         ("check-clean-tree", "script/clean-source-tree.sh"),
 456         ],
 457
 458     "samba-schemaupgrade": [
 459         ("random-sleep", random_sleep(1, 1)),
 460         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 461         ("make", "make -j"),
 462         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 463         ("lcov", LCOV_CMD),
 464         ("check-clean-tree", "script/clean-source-tree.sh"),
 465         ],
 466
 467     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 468     # This is currently the longest task, so we don't randomly delay it.
 469     "samba-ad-dc-ntvfs": [
 470         ("random-sleep", random_sleep(1, 1)),
 471         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 472         ("make", "make -j"),
 473         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 474         ("lcov", LCOV_CMD),
 475         ("check-clean-tree", "script/clean-source-tree.sh"),
 476         ],
 477
 478     # run the backup/restore testenvs separately as they're fairly standalone
 479     # (and CI seems to max out at ~8 different DCs running at once)
 480     "samba-ad-dc-backup": [
 481         ("random-sleep", random_sleep(300, 900)),
 482         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 483         ("make", "make -j"),
 484         ("test", make_test(include_envs=[
 485             "backupfromdc",
 486             "restoredc",
 487             "renamedc",
 488             "offlinebackupdc",
 489             "labdc",
 490             "ad_dc_backup",
 491             ])),
 492         ("lcov", LCOV_CMD),
 493         ("check-clean-tree", "script/clean-source-tree.sh"),
 494         ],
 495
 496     "samba-admem-mit": [
 497         ("random-sleep", random_sleep(300, 900)),
 498         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 499         ("make", "make -j"),
 500         ("test", make_test(include_envs=[
 501             "ad_member",
 502             "ad_member_idmap_rid",
 503             "ad_member_idmap_ad",
 504             "ad_member_rfc2307",
 505             ])),
 506         ("lcov", LCOV_CMD),
 507         ("check-clean-tree", "script/clean-source-tree.sh"),
 508         ],
 509
 510     "samba-ad-dc-1-mitkrb5": [
 511         ("random-sleep", random_sleep(1, 1)),
 512         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 513         ("make", "make -j"),
 514         ("test", make_test(include_envs=[
 515             "ad_dc",
 516             "ad_dc_no_nss",
 517             "ad_dc_no_ntlm",
 518             ])),
 519         ("lcov", LCOV_CMD),
 520         ("check-clean-tree", "script/clean-source-tree.sh"),
 521         ],
 522
 523     "samba-ad-dc-4-mitkrb5": [
 524         ("random-sleep", random_sleep(1, 1)),
 525         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 526         ("make", "make -j"),
 527         ("test", make_test(include_envs=[
 528             "fl2000dc",
 529             "fl2003dc",
 530             "fl2008dc",
 531             "fl2008r2dc",
 532             ])),
 533         ("lcov", LCOV_CMD),
 534         ("check-clean-tree", "script/clean-source-tree.sh"),
 535         ],
 536
 537     "samba-test-only": [
 538         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
 539         ("make", "make -j"),
 540         ("test", make_test(TESTS="${TESTS}")),
 541         ("lcov", LCOV_CMD),
 542         ],
 543
 544     # Test cross-compile infrastructure
 545     "samba-xc": [
 546         ("random-sleep", random_sleep(900, 1500)),
 547         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 548         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 549             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 550         ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 551         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 552             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 553         ("compare-results", "script/compare_cc_results.py "
 554             "./bin/c4che/default{} "
 555             "./bin-xe/c4che/default{} "
 556             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 557         ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 558         ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 559             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 560         ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 561             " = \"'1234'\"".format(CACHE_SUFFIX)),
 562         ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 563         ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 564             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 565             " ; test $? -ne 0"),
 566         ],
 567
 568     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 569     "samba-o3": [
 570         ("random-sleep", random_sleep(300, 900)),
 571         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
 572         ("make", "make -j"),
 573         ("test", make_test(cmd='make quicktest', include_envs=["ad_dc"])),
 574         ("lcov", LCOV_CMD),
 575         ("install", "make install"),
 576         ("check-clean-tree", "script/clean-source-tree.sh"),
 577         ("clean", "make clean"),
 578         ],
 579
 580     "samba-ctdb": [
 581         ("random-sleep", random_sleep(900, 1500)),
 582
 583         # make sure we have tdb around:
 584         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 585         ("tdb-make", "cd lib/tdb && make"),
 586         ("tdb-install", "cd lib/tdb && make install"),
 587
 588         # build samba with cluster support (also building ctdb):
 589         ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb"),
 590         ("samba-make", "make"),
 591         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 592         ("samba-install", "make install"),
 593         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 594
 595         # clean up:
 596         ("check-clean-tree", "script/clean-source-tree.sh"),
 597         ("clean", "make clean"),
 598         ("ctdb-clean", "cd ./ctdb && make clean"),
 599         ],
 600
 601     "samba-libs": [
 602         ("random-sleep", random_sleep(300, 900)),
 603         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 604         ("talloc-make", "cd lib/talloc && make"),
 605         ("talloc-install", "cd lib/talloc && make install"),
 606
 607         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 608         ("tdb-make", "cd lib/tdb && make"),
 609         ("tdb-install", "cd lib/tdb && make install"),
 610
 611         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 612         ("tevent-make", "cd lib/tevent && make"),
 613         ("tevent-install", "cd lib/tevent && make install"),
 614
 615         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 616         ("ldb-make", "cd lib/ldb && make"),
 617         ("ldb-install", "cd lib/ldb && make install"),
 618
 619         ("nondevel-configure", "./configure ${PREFIX}"),
 620         ("nondevel-make", "make -j"),
 621         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 622         ("nondevel-install", "make install"),
 623         ("nondevel-dist", "make dist"),
 624
 625         # retry with all modules shared
 626         ("allshared-distclean", "make distclean"),
 627         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 628         ("allshared-make", "make -j"),
 629         ],
 630
 631     "samba-none-env": [
 632         ("random-sleep", random_sleep(1, 1)),
 633         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 634         ("make", "make -j"),
 635         ("test", make_test(include_envs=["none"])),
 636         ("lcov", LCOV_CMD),
 637         ],
 638
 639     "samba-static": [
 640         ("random-sleep", random_sleep(1, 1)),
 641         # build with all modules static
 642         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 643         ("allstatic-make", "make -j"),
 644         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 645         ("lcov", LCOV_CMD),
 646
 647         # retry without any required modules
 648         ("none-distclean", "make distclean"),
 649         ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 650         ("none-make", "make -j"),
 651
 652         # retry with nonshared smbd and smbtorture
 653         ("nonshared-distclean", "make distclean"),
 654         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 655         ("nonshared-make", "make -j"),
 656         ],
 657
 658     # Test Samba without python still builds.  When this test fails
 659     # due to more use of Python, the expectations is that the newly
 660     # failing part of the code should be disabled when
 661     # --disable-python is set (rather than major work being done to
 662     # support this environment).  The target here is for vendors
 663     # shipping a minimal smbd.
 664     "samba-nopython": [
 665         ("random-sleep", random_sleep(300, 900)),
 666         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 667         ("make", "make -j"),
 668         ("install", "make install"),
 669         ("find-python", "script/find_python.sh ${PREFIX}"),
 670         ("test", "make test-nopython"),
 671         ("lcov", LCOV_CMD),
 672         ("check-clean-tree", "script/clean-source-tree.sh"),
 673         ("clean", "make clean"),
 674
 675         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 676         ("talloc-make", "cd lib/talloc && make"),
 677         ("talloc-install", "cd lib/talloc && make install"),
 678
 679         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 680         ("tdb-make", "cd lib/tdb && make"),
 681         ("tdb-install", "cd lib/tdb && make install"),
 682
 683         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 684         ("tevent-make", "cd lib/tevent && make"),
 685         ("tevent-install", "cd lib/tevent && make install"),
 686
 687         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 688         ("ldb-make", "cd lib/ldb && make"),
 689         ("ldb-install", "cd lib/ldb && make install"),
 690
 691         # retry against installed library packages
 692         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 693         ("libs-make", "make -j"),
 694         ("libs-install", "make install"),
 695         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 696         ("libs-clean", "make clean"),
 697         ],
 698
 699     # check we can do the same thing using python2
 700     "samba-nopython-py2": [
 701         ("random-sleep", random_sleep(300, 900)),
 702         ("configure", "PYTHON=python2 ./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 703         ("make", "PYTHON=python2 make -j"),
 704         ("install", "PYTHON=python2 make install"),
 705         ("find-python", "script/find_python.sh ${PREFIX}"),
 706         ("test", "make test-nopython"),
 707         ("lcov", LCOV_CMD),
 708         ("check-clean-tree", "script/clean-source-tree.sh"),
 709         ("clean", "PYTHON=python2 make clean"),
 710
 711         ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 712         ("talloc-make", "cd lib/talloc && PYTHON=python2 make"),
 713         ("talloc-install", "cd lib/talloc && PYTHON=python2 make install"),
 714
 715         ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 716         ("tdb-make", "cd lib/tdb && PYTHON=python2 make"),
 717         ("tdb-install", "cd lib/tdb && PYTHON=python2 make install"),
 718
 719         ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 720         ("tevent-make", "cd lib/tevent && PYTHON=python2 make"),
 721         ("tevent-install", "cd lib/tevent && PYTHON=python2 make install"),
 722
 723         ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 724         ("ldb-make", "cd lib/ldb && PYTHON=python2 make"),
 725         ("ldb-install", "cd lib/ldb && PYTHON=python2 make install"),
 726
 727         # retry against installed library packages
 728         ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 729         ("libs-make", "PYTHON=python2 make -j"),
 730         ("libs-install", "PYTHON=python2 make install"),
 731         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 732         ("libs-clean", "PYTHON=python2 make clean"),
 733         ],
 734
 735     "ldb": [
 736         ("random-sleep", random_sleep(60, 600)),
 737         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 738         ("make", "make"),
 739         ("install", "make install"),
 740         ("test", "make test"),
 741         ("lcov", LCOV_CMD),
 742         ("clean", "make clean"),
 743         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 744         ("make-no-lmdb", "make"),
 745         ("test-no-lmdb", "make test"),
 746         ("lcov-no-lmdb", LCOV_CMD),
 747         ("install-no-lmdb", "make install"),
 748         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 749         ("distcheck", "make distcheck"),
 750         ("clean", "make clean"),
 751         ],
 752
 753     "tdb": [
 754         ("random-sleep", random_sleep(60, 600)),
 755         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 756         ("make", "make"),
 757         ("install", "make install"),
 758         ("test", "make test"),
 759         ("lcov", LCOV_CMD),
 760         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 761         ("distcheck", "make distcheck"),
 762         ("clean", "make clean"),
 763         ],
 764
 765     "talloc": [
 766         ("random-sleep", random_sleep(60, 600)),
 767         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 768         ("make", "make"),
 769         ("install", "make install"),
 770         ("test", "make test"),
 771         ("lcov", LCOV_CMD),
 772         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 773         ("distcheck", "make distcheck"),
 774         ("clean", "make clean"),
 775         ],
 776
 777     "replace": [
 778         ("random-sleep", random_sleep(60, 600)),
 779         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 780         ("make", "make"),
 781         ("install", "make install"),
 782         ("test", "make test"),
 783         ("lcov", LCOV_CMD),
 784         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 785         ("distcheck", "make distcheck"),
 786         ("clean", "make clean"),
 787         ],
 788
 789     "tevent": [
 790         ("random-sleep", random_sleep(60, 600)),
 791         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 792         ("make", "make"),
 793         ("install", "make install"),
 794         ("test", "make test"),
 795         ("lcov", LCOV_CMD),
 796         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 797         ("distcheck", "make distcheck"),
 798         ("clean", "make clean"),
 799         ],
 800
 801     "pidl": [
 802         ("random-sleep", random_sleep(60, 600)),
 803         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 804         ("touch", "touch *.yp"),
 805         ("make", "make"),
 806         ("test", "make test"),
 807         ("install", "make install"),
 808         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 809         ("check-clean-tree", "../script/clean-source-tree.sh"),
 810         ("clean", "make clean"),
 811         ],
 812
 813     # these are useful for debugging autobuild
 814     'pass': [("pass", 'echo passing && /bin/true')],
 815     'fail': [("fail", 'echo failing && /bin/false')],
 816 }
 817
 818
 819 def do_print(msg):
 820     print("%s" % msg)
 821     sys.stdout.flush()
 822     sys.stderr.flush()
 823
 824
 825 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 826     if show is None:
 827         show = options.verbose
 828     if show:
 829         do_print("Running: '%s' in '%s'" % (cmd, dir))
 830     if output:
 831         out = check_output([cmd], shell=True, cwd=dir)
 832         return out.decode(encoding='utf-8', errors='backslashreplace')
 833     elif checkfail:
 834         return check_call(cmd, shell=True, cwd=dir)
 835     else:
 836         return call(cmd, shell=True, cwd=dir)
 837
 838
 839 class builder(object):
 840     '''handle build of one directory'''
 841
 842     def __init__(self, name, sequence, cp=True):
 843         self.name = name
 844         self.dir = builddirs.get(name, '.')
 845         self.tag = self.name.replace('/', '_')
 846         self.sequence = sequence
 847         self.next = 0
 848         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 849         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 850         if options.verbose:
 851             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 852             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 853         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 854         self.stdout = open(self.stdout_path, 'w')
 855         self.stderr = open(self.stderr_path, 'w')
 856         self.stdin  = open("/dev/null", 'r')
 857         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 858         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
 859         self.prefix = "%s/%s" % (test_prefix, self.tag)
 860         run_cmd("rm -rf %s" % self.test_source_dir)
 861         run_cmd("rm -rf %s" % self.prefix)
 862         if cp:
 863             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 864         else:
 865             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 866         self.start_next()
 867
 868     def start_next(self):
 869         if self.next == len(self.sequence):
 870             if not options.nocleanup:
 871                 run_cmd("rm -rf %s" % self.test_source_dir)
 872                 run_cmd("rm -rf %s" % self.prefix)
 873             do_print('%s: Completed OK' % self.name)
 874             self.done = True
 875             return
 876         (self.stage, self.cmd) = self.sequence[self.next]
 877         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 878         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 879         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 880         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 881         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 882         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 883         self.cmd = self.cmd.replace("${NAME}", self.name)
 884         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 885         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
 886         self.proc = Popen(self.cmd, shell=True,
 887                           close_fds=True, cwd=self.cwd,
 888                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 889         self.next += 1
 890
 891
 892 class buildlist(object):
 893     '''handle build of multiple directories'''
 894
 895     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 896         self.tail_proc = None
 897         self.retry = None
 898         if not tasknames:
 899             if options.restrict_tests:
 900                 tasknames = ["samba-test-only"]
 901             else:
 902                 tasknames = defaulttasks
 903
 904         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
 905
 906         if options.retry:
 907             rebase_remote = "rebaseon"
 908             retry_task = [("retry",
 909                             '''set -e
 910                             git remote add -t %s %s %s
 911                             git fetch %s
 912                             while :; do
 913                               sleep 60
 914                               git describe %s/%s > old_remote_branch.desc
 915                               git fetch %s
 916                               git describe %s/%s > remote_branch.desc
 917                               diff old_remote_branch.desc remote_branch.desc
 918                             done
 919                            ''' % (
 920                                rebase_branch, rebase_remote, rebase_url,
 921                                rebase_remote,
 922                                rebase_remote, rebase_branch,
 923                                rebase_remote,
 924                                rebase_remote, rebase_branch
 925                             ))]
 926
 927             self.retry = builder('retry', retry_task, cp=False)
 928             self.need_retry = False
 929
 930     def kill_kids(self):
 931         if self.tail_proc is not None:
 932             self.tail_proc.terminate()
 933             self.tail_proc.wait()
 934             self.tail_proc = None
 935         if self.retry is not None:
 936             self.retry.proc.terminate()
 937             self.retry.proc.wait()
 938             self.retry = None
 939         for b in self.tlist:
 940             if b.proc is not None:
 941                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 942                 b.proc.terminate()
 943                 b.proc.wait()
 944                 b.proc = None
 945
 946     def wait_one(self):
 947         while True:
 948             none_running = True
 949             for b in self.tlist:
 950                 if b.proc is None:
 951                     continue
 952                 none_running = False
 953                 b.status = b.proc.poll()
 954                 if b.status is None:
 955                     continue
 956                 b.proc = None
 957                 return b
 958             if options.retry:
 959                 ret = self.retry.proc.poll()
 960                 if ret is not None:
 961                     self.need_retry = True
 962                     self.retry = None
 963                     return None
 964             if none_running:
 965                 return None
 966             time.sleep(0.1)
 967
 968     def run(self):
 969         while True:
 970             b = self.wait_one()
 971             if options.retry and self.need_retry:
 972                 self.kill_kids()
 973                 do_print("retry needed")
 974                 return (0, None, None, None, "retry")
 975             if b is None:
 976                 break
 977             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 978                 self.kill_kids()
 979                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 980             b.start_next()
 981         self.kill_kids()
 982         return (0, None, None, None, "All OK")
 983
 984     def write_system_info(self, filename):
 985         with open(filename, 'w') as f:
 986             for cmd in ['uname -a',
 987                         'lsb_release -a',
 988                         'free',
 989                         'mount',
 990                         'cat /proc/cpuinfo',
 991                         'cc --version',
 992                         'df -m .',
 993                         'df -m %s' % testbase]:
 994                 out = run_cmd(cmd, output=True, checkfail=False)
 995                 print('### %s' % cmd, file=f)
 996                 print(out, file=f)
 997                 print(file=f)
 998
 999     def tarlogs(self, fname):
1000         with tarfile.open(fname, "w:gz") as tar:
1001             for b in self.tlist:
1002                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1003                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1004             if os.path.exists("autobuild.log"):
1005                 tar.add("autobuild.log")
1006             filename = 'system-info.txt'
1007             self.write_system_info(filename)
1008             tar.add(filename)
1009
1010     def remove_logs(self):
1011         for b in self.tlist:
1012             os.unlink(b.stdout_path)
1013             os.unlink(b.stderr_path)
1014
1015     def start_tail(self):
1016         cmd = ["tail", "-f"]
1017         for b in self.tlist:
1018             cmd.append(b.stdout_path)
1019             cmd.append(b.stderr_path)
1020         self.tail_proc = Popen(cmd, close_fds=True)
1021
1022
1023 def cleanup():
1024     if options.nocleanup:
1025         return
1026     run_cmd("stat %s || true" % test_tmpdir, show=True)
1027     run_cmd("stat %s" % testbase, show=True)
1028     do_print("Cleaning up %r" % cleanup_list)
1029     for d in cleanup_list:
1030         run_cmd("rm -rf %s" % d)
1031
1032
1033 def daemonize(logfile):
1034     pid = os.fork()
1035     if pid == 0:  # Parent
1036         os.setsid()
1037         pid = os.fork()
1038         if pid != 0:  # Actual daemon
1039             os._exit(0)
1040     else:  # Grandparent
1041         os._exit(0)
1042
1043     import resource      # Resource usage information.
1044     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1045     if maxfd == resource.RLIM_INFINITY:
1046         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1047     for fd in range(0, maxfd):
1048         try:
1049             os.close(fd)
1050         except OSError:
1051             pass
1052     os.open(logfile, os.O_RDWR | os.O_CREAT)
1053     os.dup2(0, 1)
1054     os.dup2(0, 2)
1055
1056
1057 def write_pidfile(fname):
1058     '''write a pid file, cleanup on exit'''
1059     with open(fname, mode='w') as f:
1060         f.write("%u\n" % os.getpid())
1061
1062
1063 def rebase_tree(rebase_url, rebase_branch="master"):
1064     rebase_remote = "rebaseon"
1065     do_print("Rebasing on %s" % rebase_url)
1066     run_cmd("git describe HEAD", show=True, dir=test_master)
1067     run_cmd("git remote add -t %s %s %s" %
1068             (rebase_branch, rebase_remote, rebase_url),
1069             show=True, dir=test_master)
1070     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1071     if options.fix_whitespace:
1072         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1073                 (rebase_remote, rebase_branch),
1074                 show=True, dir=test_master)
1075     else:
1076         run_cmd("git rebase --force-rebase %s/%s" %
1077                 (rebase_remote, rebase_branch),
1078                 show=True, dir=test_master)
1079     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1080                    (rebase_remote, rebase_branch),
1081                    dir=test_master, output=True)
1082     if diff == '':
1083         do_print("No differences between HEAD and %s/%s - exiting" %
1084                  (rebase_remote, rebase_branch))
1085         sys.exit(0)
1086     run_cmd("git describe %s/%s" %
1087             (rebase_remote, rebase_branch),
1088             show=True, dir=test_master)
1089     run_cmd("git describe HEAD", show=True, dir=test_master)
1090     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1091             (rebase_remote, rebase_branch),
1092             show=True, dir=test_master)
1093
1094
1095 def push_to(push_url, push_branch="master"):
1096     push_remote = "pushto"
1097     do_print("Pushing to %s" % push_url)
1098     if options.mark:
1099         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1100         run_cmd("git commit --amend -c HEAD", dir=test_master)
1101         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1102         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1103     run_cmd("git remote add -t %s %s %s" %
1104             (push_branch, push_remote, push_url),
1105             show=True, dir=test_master)
1106     run_cmd("git push %s +HEAD:%s" %
1107             (push_remote, push_branch),
1108             show=True, dir=test_master)
1109
1110
1111 def send_email(subject, text, log_tar):
1112     if options.email is None:
1113         do_print("not sending email because the recipient is not set")
1114         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1115                  (subject, text))
1116         return
1117     outer = MIMEMultipart()
1118     outer['Subject'] = subject
1119     outer['To'] = options.email
1120     outer['From'] = options.email_from
1121     outer['Date'] = email.utils.formatdate(localtime=True)
1122     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1123     outer.attach(MIMEText(text, 'plain'))
1124     if options.attach_logs:
1125         with open(log_tar, 'rb') as fp:
1126             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1127         # Set the filename parameter
1128         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1129         outer.attach(msg)
1130     content = outer.as_string()
1131     s = smtplib.SMTP(options.email_server)
1132     email_user = os.getenv('SMTP_USERNAME')
1133     email_password = os.getenv('SMTP_PASSWORD')
1134     if email_user is not None:
1135         s.starttls()
1136         s.login(email_user, email_password)
1137
1138     s.sendmail(options.email_from, [options.email], content)
1139     s.set_debuglevel(1)
1140     s.quit()
1141
1142
1143 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1144                   elapsed_time, log_base=None, add_log_tail=True):
1145     '''send an email to options.email about the failure'''
1146     elapsed_minutes = elapsed_time / 60.0
1147     if log_base is None:
1148         log_base = gitroot
1149     text = '''
1150 Dear Developer,
1151
1152 Your autobuild on %s failed after %.1f minutes
1153 when trying to test %s with the following error:
1154
1155    %s
1156
1157 the autobuild has been abandoned. Please fix the error and resubmit.
1158
1159 A summary of the autobuild process is here:
1160
1161   %s/autobuild.log
1162 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1163
1164     if options.restrict_tests:
1165         text += """
1166 The build was restricted to tests matching %s\n""" % options.restrict_tests
1167
1168     if failed_task != 'rebase':
1169         text += '''
1170 You can see logs of the failed task here:
1171
1172   %s/%s.stdout
1173   %s/%s.stderr
1174
1175 or you can get full logs of all tasks in this job here:
1176
1177   %s/logs.tar.gz
1178
1179 The top commit for the tree that was built was:
1180
1181 %s
1182
1183 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1184
1185     if add_log_tail:
1186         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1187         lines = f.readlines()
1188         log_tail = "".join(lines[-50:])
1189         num_lines = len(lines)
1190         if num_lines < 50:
1191             # Also include stderr (compile failures) if < 50 lines of stdout
1192             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1193             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1194
1195         text += '''
1196 The last 50 lines of log messages:
1197
1198 %s
1199     ''' % log_tail
1200         f.close()
1201
1202     logs = os.path.join(gitroot, 'logs.tar.gz')
1203     send_email('autobuild[%s] failure on %s for task %s during %s'
1204                % (options.branch, platform.node(), failed_task, failed_stage),
1205                text, logs)
1206
1207
1208 def email_success(elapsed_time, log_base=None):
1209     '''send an email to options.email about a successful build'''
1210     if log_base is None:
1211         log_base = gitroot
1212     text = '''
1213 Dear Developer,
1214
1215 Your autobuild on %s has succeeded after %.1f minutes.
1216
1217 ''' % (platform.node(), elapsed_time / 60.)
1218
1219     if options.restrict_tests:
1220         text += """
1221 The build was restricted to tests matching %s\n""" % options.restrict_tests
1222
1223     if options.keeplogs:
1224         text += '''
1225
1226 you can get full logs of all tasks in this job here:
1227
1228   %s/logs.tar.gz
1229
1230 ''' % log_base
1231
1232     text += '''
1233 The top commit for the tree that was built was:
1234
1235 %s
1236 ''' % top_commit_msg
1237
1238     logs = os.path.join(gitroot, 'logs.tar.gz')
1239     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1240                text, logs)
1241
1242
1243 # get the top commit message, for emails
1244 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1245
1246 try:
1247     os.makedirs(testbase)
1248 except Exception as reason:
1249     raise Exception("Unable to create %s : %s" % (testbase, reason))
1250 cleanup_list.append(testbase)
1251
1252 if options.daemon:
1253     logfile = os.path.join(testbase, "log")
1254     do_print("Forking into the background, writing progress to %s" % logfile)
1255     daemonize(logfile)
1256
1257 write_pidfile(gitroot + "/autobuild.pid")
1258
1259 start_time = time.time()
1260
1261 while True:
1262     try:
1263         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1264         os.makedirs(test_tmpdir)
1265         # The waf uninstall code removes empty directories all the way
1266         # up the tree.  Creating a file in test_tmpdir stops it from
1267         # being removed.
1268         run_cmd("touch %s" % os.path.join(test_tmpdir,
1269                                           ".directory-is-not-empty"), show=True)
1270         run_cmd("stat %s" % test_tmpdir, show=True)
1271         run_cmd("stat %s" % testbase, show=True)
1272         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1273     except Exception:
1274         cleanup()
1275         raise
1276
1277     try:
1278         if options.rebase is not None:
1279             rebase_tree(options.rebase, rebase_branch=options.branch)
1280     except Exception:
1281         cleanup_list.append(gitroot + "/autobuild.pid")
1282         cleanup()
1283         elapsed_time = time.time() - start_time
1284         email_failure(-1, 'rebase', 'rebase', 'rebase',
1285                       'rebase on %s failed' % options.branch,
1286                       elapsed_time, log_base=options.log_base)
1287         sys.exit(1)
1288
1289     try:
1290         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1291         if options.tail:
1292             blist.start_tail()
1293         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1294         if status != 0 or errstr != "retry":
1295             break
1296         cleanup()
1297     except Exception:
1298         cleanup()
1299         raise
1300
1301 cleanup_list.append(gitroot + "/autobuild.pid")
1302
1303 do_print(errstr)
1304
1305 blist.kill_kids()
1306 if options.tail:
1307     do_print("waiting for tail to flush")
1308     time.sleep(1)
1309
1310 elapsed_time = time.time() - start_time
1311 if status == 0:
1312     if options.passcmd is not None:
1313         do_print("Running passcmd: %s" % options.passcmd)
1314         run_cmd(options.passcmd, dir=test_master)
1315     if options.pushto is not None:
1316         push_to(options.pushto, push_branch=options.branch)
1317     if options.keeplogs or options.attach_logs:
1318         blist.tarlogs("logs.tar.gz")
1319         do_print("Logs in logs.tar.gz")
1320     if options.always_email:
1321         email_success(elapsed_time, log_base=options.log_base)
1322     blist.remove_logs()
1323     cleanup()
1324     do_print(errstr)
1325     sys.exit(0)
1326
1327 # something failed, gather a tar of the logs
1328 blist.tarlogs("logs.tar.gz")
1329
1330 if options.email is not None:
1331     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1332                   elapsed_time, log_base=options.log_base)
1333 else:
1334     elapsed_minutes = elapsed_time / 60.0
1335     print('''
1336
1337 ####################################################################
1338
1339 AUTOBUILD FAILURE
1340
1341 Your autobuild[%s] on %s failed after %.1f minutes
1342 when trying to test %s with the following error:
1343
1344    %s
1345
1346 the autobuild has been abandoned. Please fix the error and resubmit.
1347
1348 ####################################################################
1349
1350 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1351
1352 cleanup()
1353 do_print(errstr)
1354 do_print("Logs in logs.tar.gz")
1355 sys.exit(status)