script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.exists(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  60                   default=None)
  61 parser.add_option("--passcmd", help="command to run on success", default=None)
  62 parser.add_option("--verbose", help="show all commands as they are run",
  63                   default=False, action="store_true")
  64 parser.add_option("--rebase", help="rebase on the given tree before testing",
  65                   default=None, type='str')
  66 parser.add_option("--pushto", help="push to a git url on success",
  67                   default=None, type='str')
  68 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  69                   default=False, action="store_true")
  70 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  71                   default=False, action="store_true")
  72 parser.add_option("--retry", help="automatically retry if master changes",
  73                   default=False, action="store_true")
  74 parser.add_option("--email", help="send email to the given address on failure",
  75                   type='str', default=None)
  76 parser.add_option("--email-from", help="send email from the given address",
  77                   type='str', default="autobuild@samba.org")
  78 parser.add_option("--email-server", help="send email via the given server",
  79                   type='str', default='localhost')
  80 parser.add_option("--always-email", help="always send email, even on success",
  81                   action="store_true")
  82 parser.add_option("--daemon", help="daemonize after initial setup",
  83                   action="store_true")
  84 parser.add_option("--branch", help="the branch to work on (default=master)",
  85                   default="master", type='str')
  86 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  87                   default=gitroot, type='str')
  88 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  89                   default=False, action="store_true")
  90 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  91                   default='')
  92 parser.add_option("--enable-coverage", dest='enable_coverage',
  93                   action="store_const", const='--enable-coverage', default='',
  94                   help="Add --enable-coverage option while configure")
  95
  96 (options, args) = parser.parse_args()
  97
  98 if options.retry:
  99     if options.rebase is None:
 100         raise Exception('You can only use --retry if you also rebase')
 101
 102 if options.full_testbase is not None:
 103     testbase = options.full_testbase
 104 else:
 105     testbase = "%s/b%u" % (options.testbase, os.getpid())
 106 test_master = "%s/master" % testbase
 107 test_prefix = "%s/prefix" % testbase
 108 test_tmpdir = "%s/tmp" % testbase
 109 os.environ['TMPDIR'] = test_tmpdir
 110
 111 if options.enable_coverage:
 112     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 113 else:
 114     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 115
 116 if args:
 117     # If we are only running specific test,
 118     # do not sleep randomly to wait for it to start
 119     def random_sleep(low, high):
 120         return 'sleep 1'
 121 else:
 122     def random_sleep(low, high):
 123         return 'sleep {}'.format(random.randint(low, high))
 124
 125 cleanup_list = []
 126
 127 builddirs = {
 128     "ctdb": "ctdb",
 129     "ldb": "lib/ldb",
 130     "tdb": "lib/tdb",
 131     "talloc": "lib/talloc",
 132     "replace": "lib/replace",
 133     "tevent": "lib/tevent",
 134     "pidl": "pidl",
 135     "docs-xml": "docs-xml"
 136 }
 137
 138 ctdb_configure_params = " --enable-developer ${PREFIX}"
 139 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 140
 141 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 142 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 143 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 144 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 145 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 146 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 147 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 148
 149
 150 def format_option(name, value=None):
 151     """Format option as str list."""
 152     if value is None:  # boolean option
 153         return [name]
 154     if not isinstance(value, list):  # single value option
 155         value = [value]
 156     # repeatable option
 157     return ['{}={}'.format(name, item) for item in value]
 158
 159
 160 def make_test(
 161         cmd='make test',
 162         FAIL_IMMEDIATELY=1,
 163         TESTS='',
 164         include_envs=None,
 165         exclude_envs=None):
 166
 167     test_options = []
 168     if include_envs:
 169         test_options = format_option('--include-env', include_envs)
 170     if exclude_envs:
 171         test_options = format_option('--exclude-env', exclude_envs)
 172     if test_options:
 173         # join envs options to original test options
 174         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 175
 176     _options = []
 177     if FAIL_IMMEDIATELY:
 178         _options.append('FAIL_IMMEDIATELY=1')
 179     if TESTS:
 180         _options.append("TESTS='{}'".format(TESTS))
 181
 182     return ' '.join([cmd] + _options)
 183
 184
 185 # When updating this list, also update .gitlab-ci.yml to add the job
 186 # and to make it a dependency of 'page' for the coverage report.
 187
 188 tasks = {
 189     "ctdb": [
 190         ("random-sleep", random_sleep(300, 900)),
 191         ("configure", "./configure " + ctdb_configure_params),
 192         ("make", "make all"),
 193         ("install", "make install"),
 194         ("test", "make autotest"),
 195         ("check-clean-tree", "../script/clean-source-tree.sh"),
 196         ("clean", "make clean"),
 197         ],
 198
 199     "docs-xml": [
 200         ("random-sleep", random_sleep(300, 900)),
 201         ("autoconf", "autoconf"),
 202         ("configure", "./configure"),
 203         ("make", "make html htmlman"),
 204         ("clean", "make clean"),
 205         ],
 206
 207     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 208     "samba": [
 209         ("random-sleep", random_sleep(300, 900)),
 210         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 211         ("make", "make -j"),
 212         ("test", make_test(exclude_envs=[
 213             "none",
 214             "nt4_dc",
 215             "nt4_dc_smb1",
 216             "nt4_dc_smb1_done",
 217             "nt4_dc_schannel",
 218             "nt4_member",
 219             "ad_dc",
 220             "ad_dc_smb1",
 221             "ad_dc_smb1_done",
 222             "ad_dc_backup",
 223             "ad_dc_ntvfs",
 224             "ad_dc_default",
 225             "ad_dc_default_smb1",
 226             "ad_dc_slowtests",
 227             "ad_dc_no_nss",
 228             "ad_dc_no_ntlm",
 229             "fl2003dc",
 230             "fl2008dc",
 231             "fl2008r2dc",
 232             "ad_member",
 233             "ad_member_idmap_rid",
 234             "ad_member_idmap_ad",
 235             "ad_member_rfc2307",
 236             "chgdcpass",
 237             "vampire_2000_dc",
 238             "fl2000dc",
 239             "fileserver",
 240             "fileserver_smb1",
 241             "fileserver_smb1_done",
 242             "maptoguest",
 243             "simpleserver",
 244             "backupfromdc",
 245             "restoredc",
 246             "renamedc",
 247             "offlinebackupdc",
 248             "labdc",
 249             "preforkrestartdc",
 250             "proclimitdc",
 251             "promoted_dc",
 252             "vampire_dc",
 253             "rodc",
 254             "ad_dc_default",
 255             "ad_dc_default_smb1",
 256             "ad_dc_default_smb1_done",
 257             "ad_dc_slowtests",
 258             "schema_pair_dc",
 259             "schema_dc",
 260             "clusteredmember_smb1",
 261             ])),
 262         ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 263         ("lcov", LCOV_CMD),
 264         ("install", "make install"),
 265         ("check-clean-tree", "script/clean-source-tree.sh"),
 266         ("clean", "make clean"),
 267         ],
 268
 269     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 270     "samba-mitkrb5": [
 271         ("random-sleep", random_sleep(300, 900)),
 272         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 273         ("make", "make -j"),
 274         ("test", make_test(exclude_envs=[
 275             "none",
 276             "nt4_dc",
 277             "nt4_dc_smb1",
 278             "nt4_dc_smb1_done",
 279             "nt4_dc_schannel",
 280             "nt4_member",
 281             "ad_dc",
 282             "ad_dc_smb1",
 283             "ad_dc_smb1_done",
 284             "ad_dc_backup",
 285             "ad_dc_ntvfs",
 286             "ad_dc_default",
 287             "ad_dc_default_smb1",
 288             "ad_dc_default_smb1_done",
 289             "ad_dc_slowtests",
 290             "ad_dc_no_nss",
 291             "ad_dc_no_ntlm",
 292             "fl2003dc",
 293             "fl2008dc",
 294             "fl2008r2dc",
 295             "ad_member",
 296             "ad_member_idmap_rid",
 297             "ad_member_idmap_ad",
 298             "ad_member_rfc2307",
 299             "chgdcpass",
 300             "vampire_2000_dc",
 301             "fl2000dc",
 302             "fileserver",
 303             "fileserver_smb1",
 304             "fileserver_smb1_done",
 305             "maptoguest",
 306             "simpleserver",
 307             "backupfromdc",
 308             "restoredc",
 309             "renamedc",
 310             "offlinebackupdc",
 311             "labdc",
 312             "preforkrestartdc",
 313             "proclimitdc",
 314             "promoted_dc",
 315             "vampire_dc",
 316             "rodc",
 317             "ad_dc_default",
 318             "ad_dc_default_smb1",
 319             "ad_dc_default_smb1_done",
 320             "ad_dc_slowtests",
 321             "schema_pair_dc",
 322             "schema_dc",
 323             "clusteredmember_smb1",
 324             ])),
 325         ("lcov", LCOV_CMD),
 326         ("install", "make install"),
 327         ("check-clean-tree", "script/clean-source-tree.sh"),
 328         ("clean", "make clean"),
 329         ],
 330
 331     "samba-nt4": [
 332         ("random-sleep", random_sleep(300, 900)),
 333         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
 334         ("make", "make -j"),
 335         ("test", make_test(include_envs=[
 336             "nt4_dc",
 337             "nt4_dc_smb1",
 338             "nt4_dc_smb1_done",
 339             "nt4_dc_schannel",
 340             "nt4_member",
 341             "simpleserver",
 342             ])),
 343         ("lcov", LCOV_CMD),
 344         ("check-clean-tree", "script/clean-source-tree.sh"),
 345         ("clean", "make clean"),
 346         ],
 347
 348     "samba-fileserver": [
 349         ("random-sleep", random_sleep(300, 900)),
 350         ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab" + samba_configure_params),
 351         ("make", "make -j"),
 352         ("test", make_test(include_envs=[
 353             "fileserver",
 354             "fileserver_smb1",
 355             "fileserver_smb1_done",
 356             "maptoguest",
 357             "ktest", # ktest is also tested in samba and samba-mitkrb5
 358                      # but is tested here against a system Heimdal
 359             ])),
 360         ("lcov", LCOV_CMD),
 361         ("check-clean-tree", "script/clean-source-tree.sh"),
 362         ],
 363
 364     "samba-admem": [
 365         ("random-sleep", random_sleep(300, 900)),
 366         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 367         ("make", "make -j"),
 368         ("test", make_test(include_envs=[
 369             "ad_member",
 370             "ad_member_idmap_rid",
 371             "ad_member_idmap_ad",
 372             "ad_member_rfc2307",
 373             ])),
 374         ("lcov", LCOV_CMD),
 375         ("check-clean-tree", "script/clean-source-tree.sh"),
 376         ],
 377
 378     "samba-ad-dc-1": [
 379         ("random-sleep", random_sleep(1, 1)),
 380         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 381         ("make", "make -j"),
 382         ("test", make_test(include_envs=[
 383             "ad_dc",
 384             "ad_dc_smb1",
 385             "ad_dc_smb1_done",
 386             "ad_dc_no_nss",
 387             "ad_dc_no_ntlm",
 388             ])),
 389         ("lcov", LCOV_CMD),
 390         ("check-clean-tree", "script/clean-source-tree.sh"),
 391         ],
 392
 393     "samba-ad-dc-2": [
 394         ("random-sleep", random_sleep(1, 1)),
 395         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 396         ("make", "make -j"),
 397         ("test", make_test(include_envs=[
 398             "vampire_dc",
 399             "vampire_2000_dc",
 400             "rodc",
 401             ])),
 402         ("lcov", LCOV_CMD),
 403         ("check-clean-tree", "script/clean-source-tree.sh"),
 404         ],
 405
 406     "samba-ad-dc-3": [
 407         ("random-sleep", random_sleep(1, 1)),
 408         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 409         ("make", "make -j"),
 410         ("test", make_test(include_envs=[
 411             "promoted_dc",
 412             "chgdcpass",
 413             "preforkrestartdc",
 414             "proclimitdc",
 415             ])),
 416         ("lcov", LCOV_CMD),
 417         ("check-clean-tree", "script/clean-source-tree.sh"),
 418         ],
 419
 420     "samba-ad-dc-4": [
 421         ("random-sleep", random_sleep(1, 1)),
 422         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 423         ("make", "make -j"),
 424         ("test", make_test(include_envs=[
 425             "fl2000dc",
 426             "fl2003dc",
 427             "fl2008dc",
 428             "fl2008r2dc",
 429             ])),
 430         ("lcov", LCOV_CMD),
 431         ("check-clean-tree", "script/clean-source-tree.sh"),
 432         ],
 433
 434     "samba-ad-dc-5": [
 435         ("random-sleep", random_sleep(1, 1)),
 436         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 437         ("make", "make -j"),
 438         ("test", make_test(include_envs=[
 439             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 440         ("lcov", LCOV_CMD),
 441         ("check-clean-tree", "script/clean-source-tree.sh"),
 442         ],
 443
 444     "samba-ad-dc-6": [
 445         ("random-sleep", random_sleep(1, 1)),
 446         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 447         ("make", "make -j"),
 448         ("test", make_test(include_envs=["ad_dc_slowtests"])),
 449         ("lcov", LCOV_CMD),
 450         ("check-clean-tree", "script/clean-source-tree.sh"),
 451         ],
 452
 453     "samba-schemaupgrade": [
 454         ("random-sleep", random_sleep(1, 1)),
 455         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 456         ("make", "make -j"),
 457         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 458         ("lcov", LCOV_CMD),
 459         ("check-clean-tree", "script/clean-source-tree.sh"),
 460         ],
 461
 462     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 463     # This is currently the longest task, so we don't randomly delay it.
 464     "samba-ad-dc-ntvfs": [
 465         ("random-sleep", random_sleep(1, 1)),
 466         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 467         ("make", "make -j"),
 468         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 469         ("lcov", LCOV_CMD),
 470         ("check-clean-tree", "script/clean-source-tree.sh"),
 471         ],
 472
 473     # Test fips compliance
 474     "samba-fips": [
 475         ("random-sleep", random_sleep(100, 500)),
 476         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 477         ("make", "make -j"),
 478         ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 479         ("lcov", LCOV_CMD),
 480         ("check-clean-tree", "script/clean-source-tree.sh"),
 481         ],
 482
 483     # run the backup/restore testenvs separately as they're fairly standalone
 484     # (and CI seems to max out at ~8 different DCs running at once)
 485     "samba-ad-dc-backup": [
 486         ("random-sleep", random_sleep(300, 900)),
 487         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 488         ("make", "make -j"),
 489         ("test", make_test(include_envs=[
 490             "backupfromdc",
 491             "restoredc",
 492             "renamedc",
 493             "offlinebackupdc",
 494             "labdc",
 495             "ad_dc_backup",
 496             ])),
 497         ("lcov", LCOV_CMD),
 498         ("check-clean-tree", "script/clean-source-tree.sh"),
 499         ],
 500
 501     "samba-admem-mit": [
 502         ("random-sleep", random_sleep(1, 1)),
 503         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 504         ("make", "make -j"),
 505         ("test", make_test(include_envs=[
 506             "ad_member",
 507             "ad_member_idmap_rid",
 508             "ad_member_idmap_ad",
 509             "ad_member_rfc2307",
 510             ])),
 511         ("lcov", LCOV_CMD),
 512         ("check-clean-tree", "script/clean-source-tree.sh"),
 513         ],
 514
 515     "samba-ad-dc-1-mitkrb5": [
 516         ("random-sleep", random_sleep(1, 1)),
 517         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 518         ("make", "make -j"),
 519         ("test", make_test(include_envs=[
 520             "ad_dc",
 521             "ad_dc_smb1",
 522             "ad_dc_smb1_done",
 523             "ad_dc_no_nss",
 524             "ad_dc_no_ntlm",
 525             ])),
 526         ("lcov", LCOV_CMD),
 527         ("check-clean-tree", "script/clean-source-tree.sh"),
 528         ],
 529
 530     "samba-ad-dc-4-mitkrb5": [
 531         ("random-sleep", random_sleep(1, 1)),
 532         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 533         ("make", "make -j"),
 534         ("test", make_test(include_envs=[
 535             "fl2000dc",
 536             "fl2003dc",
 537             "fl2008dc",
 538             "fl2008r2dc",
 539             ])),
 540         ("lcov", LCOV_CMD),
 541         ("check-clean-tree", "script/clean-source-tree.sh"),
 542         ],
 543
 544     "samba-test-only": [
 545         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
 546         ("make", "make -j"),
 547         ("test", make_test(TESTS="${TESTS}")),
 548         ("lcov", LCOV_CMD),
 549         ],
 550
 551     # Test cross-compile infrastructure
 552     "samba-xc": [
 553         ("random-sleep", random_sleep(900, 1500)),
 554         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 555         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 556             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 557         ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 558         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 559             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 560         ("compare-results", "script/compare_cc_results.py "
 561             "./bin/c4che/default{} "
 562             "./bin-xe/c4che/default{} "
 563             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 564         ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 565         ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 566             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 567         ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 568             " = \"'1234'\"".format(CACHE_SUFFIX)),
 569         ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 570         ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 571             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 572             " ; test $? -ne 0"),
 573         ],
 574
 575     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 576     "samba-o3": [
 577         ("random-sleep", random_sleep(300, 900)),
 578         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
 579         ("make", "make -j"),
 580         ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 581         ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 582         ("lcov", LCOV_CMD),
 583         ("install", "make install"),
 584         ("check-clean-tree", "script/clean-source-tree.sh"),
 585         ("clean", "make clean"),
 586         ],
 587
 588     "samba-ctdb": [
 589         ("random-sleep", random_sleep(900, 1500)),
 590
 591         # make sure we have tdb around:
 592         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 593         ("tdb-make", "cd lib/tdb && make"),
 594         ("tdb-install", "cd lib/tdb && make install"),
 595
 596         # build samba with cluster support (also building ctdb):
 597         ("samba-configure",
 598          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 599          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 600          "./configure.developer ${PREFIX} "
 601          "--with-selftest-prefix=./bin/ab "
 602          "--with-cluster-support "
 603          "--without-ad-dc "
 604          "--bundled-libraries=!tdb"),
 605         ("samba-make", "make"),
 606         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 607         ("samba-install", "make install"),
 608         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 609
 610         ("test",
 611          make_test(cmd='make test',
 612                    include_envs=["clusteredmember_smb1"])
 613         ),
 614
 615         # clean up:
 616         ("check-clean-tree", "script/clean-source-tree.sh"),
 617         ("clean", "make clean"),
 618         ("ctdb-clean", "cd ./ctdb && make clean"),
 619         ],
 620
 621     "samba-libs": [
 622         ("random-sleep", random_sleep(300, 900)),
 623         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 624         ("talloc-make", "cd lib/talloc && make"),
 625         ("talloc-install", "cd lib/talloc && make install"),
 626
 627         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 628         ("tdb-make", "cd lib/tdb && make"),
 629         ("tdb-install", "cd lib/tdb && make install"),
 630
 631         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 632         ("tevent-make", "cd lib/tevent && make"),
 633         ("tevent-install", "cd lib/tevent && make install"),
 634
 635         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 636         ("ldb-make", "cd lib/ldb && make"),
 637         ("ldb-install", "cd lib/ldb && make install"),
 638
 639         ("nondevel-configure", "./configure ${PREFIX}"),
 640         ("nondevel-make", "make -j"),
 641         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 642         ("nondevel-install", "make install"),
 643         ("nondevel-dist", "make dist"),
 644
 645         # retry with all modules shared
 646         ("allshared-distclean", "make distclean"),
 647         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 648         ("allshared-make", "make -j"),
 649         ],
 650
 651     "samba-fuzz": [
 652         # build the fuzzers (static) via the oss-fuzz script
 653         ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 654         ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl"),
 655         ("fuzzers-check", "./lib/fuzzing/oss-fuzz/check_build.sh ${PREFIX_DIR}")
 656         ],
 657
 658     # * Test smbd and smbtorture can build semi-static
 659     #
 660     # * Test Samba without python still builds.
 661     #
 662     # When this test fails due to more use of Python, the expectations
 663     # is that the newly failing part of the code should be disabled
 664     # when --disable-python is set (rather than major work being done
 665     # to support this environment).
 666     #
 667     # The target here is for vendors shipping a minimal smbd.
 668     "samba-minimal-smbd": [
 669         ("random-sleep", random_sleep(300, 900)),
 670
 671         # build with all modules static
 672         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 673         ("allstatic-make", "make -j"),
 674         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 675         ("lcov", LCOV_CMD),
 676
 677         # retry without any required modules
 678         ("none-distclean", "make distclean"),
 679         ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 680         ("none-make", "make -j"),
 681
 682         # retry with nonshared smbd and smbtorture
 683         ("nonshared-distclean", "make distclean"),
 684         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 685         ("nonshared-make", "make -j"),
 686
 687         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 688         ("make", "make -j"),
 689         ("find-python", "script/find_python.sh ${PREFIX}"),
 690         ("test", "make test-nopython"),
 691         ("lcov", LCOV_CMD),
 692         ("check-clean-tree", "script/clean-source-tree.sh"),
 693         ("clean", "make clean"),
 694
 695         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 696         ("talloc-make", "cd lib/talloc && make"),
 697         ("talloc-install", "cd lib/talloc && make install"),
 698
 699         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 700         ("tdb-make", "cd lib/tdb && make"),
 701         ("tdb-install", "cd lib/tdb && make install"),
 702
 703         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 704         ("tevent-make", "cd lib/tevent && make"),
 705         ("tevent-install", "cd lib/tevent && make install"),
 706
 707         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 708         ("ldb-make", "cd lib/ldb && make"),
 709         ("ldb-install", "cd lib/ldb && make install"),
 710
 711         # retry against installed library packages
 712         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 713         ("libs-make", "make -j"),
 714         ("libs-install", "make install"),
 715         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 716         ("libs-clean", "make clean"),
 717
 718         ],
 719
 720     "ldb": [
 721         ("random-sleep", random_sleep(60, 600)),
 722         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 723         ("make", "make"),
 724         ("install", "make install"),
 725         ("test", "make test"),
 726         ("lcov", LCOV_CMD),
 727         ("clean", "make clean"),
 728         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 729         ("make-no-lmdb", "make"),
 730         ("test-no-lmdb", "make test"),
 731         ("lcov-no-lmdb", LCOV_CMD),
 732         ("install-no-lmdb", "make install"),
 733         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 734         ("distcheck", "make distcheck"),
 735         ("clean", "make clean"),
 736         ],
 737
 738     "tdb": [
 739         ("random-sleep", random_sleep(60, 600)),
 740         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 741         ("make", "make"),
 742         ("install", "make install"),
 743         ("test", "make test"),
 744         ("lcov", LCOV_CMD),
 745         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 746         ("distcheck", "make distcheck"),
 747         ("clean", "make clean"),
 748         ],
 749
 750     "talloc": [
 751         ("random-sleep", random_sleep(60, 600)),
 752         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 753         ("make", "make"),
 754         ("install", "make install"),
 755         ("test", "make test"),
 756         ("lcov", LCOV_CMD),
 757         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 758         ("distcheck", "make distcheck"),
 759         ("clean", "make clean"),
 760         ],
 761
 762     "replace": [
 763         ("random-sleep", random_sleep(60, 600)),
 764         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 765         ("make", "make"),
 766         ("install", "make install"),
 767         ("test", "make test"),
 768         ("lcov", LCOV_CMD),
 769         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 770         ("distcheck", "make distcheck"),
 771         ("clean", "make clean"),
 772         ],
 773
 774     "tevent": [
 775         ("random-sleep", random_sleep(60, 600)),
 776         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 777         ("make", "make"),
 778         ("install", "make install"),
 779         ("test", "make test"),
 780         ("lcov", LCOV_CMD),
 781         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 782         ("distcheck", "make distcheck"),
 783         ("clean", "make clean"),
 784         ],
 785
 786     "pidl": [
 787         ("random-sleep", random_sleep(60, 600)),
 788         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 789         ("touch", "touch *.yp"),
 790         ("make", "make"),
 791         ("test", "make test"),
 792         ("install", "make install"),
 793         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 794         ("check-clean-tree", "../script/clean-source-tree.sh"),
 795         ("clean", "make clean"),
 796         ],
 797
 798     # these are useful for debugging autobuild
 799     'pass': [("pass", 'echo passing && /bin/true')],
 800     'fail': [("fail", 'echo failing && /bin/false')],
 801 }
 802
 803 defaulttasks = list(tasks.keys())
 804
 805 defaulttasks.remove("pass")
 806 defaulttasks.remove("fail")
 807 defaulttasks.remove("samba-test-only")
 808 defaulttasks.remove("samba-fuzz")
 809 defaulttasks.remove("samba-fips")
 810 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 811     defaulttasks.remove("samba-o3")
 812
 813
 814 def do_print(msg):
 815     print("%s" % msg)
 816     sys.stdout.flush()
 817     sys.stderr.flush()
 818
 819
 820 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 821     if show is None:
 822         show = options.verbose
 823     if show:
 824         do_print("Running: '%s' in '%s'" % (cmd, dir))
 825     if output:
 826         out = check_output([cmd], shell=True, cwd=dir)
 827         return out.decode(encoding='utf-8', errors='backslashreplace')
 828     elif checkfail:
 829         return check_call(cmd, shell=True, cwd=dir)
 830     else:
 831         return call(cmd, shell=True, cwd=dir)
 832
 833
 834 class builder(object):
 835     '''handle build of one directory'''
 836
 837     def __init__(self, name, sequence, cp=True):
 838         self.name = name
 839         self.dir = builddirs.get(name, '.')
 840         self.tag = self.name.replace('/', '_')
 841         self.sequence = sequence
 842         self.next = 0
 843         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 844         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 845         if options.verbose:
 846             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 847             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 848         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 849         self.stdout = open(self.stdout_path, 'w')
 850         self.stderr = open(self.stderr_path, 'w')
 851         self.stdin  = open("/dev/null", 'r')
 852         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 853         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
 854         self.prefix = "%s/%s" % (test_prefix, self.tag)
 855         run_cmd("rm -rf %s" % self.test_source_dir)
 856         run_cmd("rm -rf %s" % self.prefix)
 857         if cp:
 858             run_cmd("cp -R -a -l %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 859         else:
 860             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 861         self.start_next()
 862
 863     def start_next(self):
 864         if self.next == len(self.sequence):
 865             if not options.nocleanup:
 866                 run_cmd("rm -rf %s" % self.test_source_dir)
 867                 run_cmd("rm -rf %s" % self.prefix)
 868             do_print('%s: Completed OK' % self.name)
 869             self.done = True
 870             return
 871         (self.stage, self.cmd) = self.sequence[self.next]
 872         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 873         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 874         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 875         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 876         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 877         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 878         self.cmd = self.cmd.replace("${NAME}", self.name)
 879         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 880         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
 881         self.proc = Popen(self.cmd, shell=True,
 882                           close_fds=True, cwd=self.cwd,
 883                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 884         self.next += 1
 885
 886
 887 class buildlist(object):
 888     '''handle build of multiple directories'''
 889
 890     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 891         self.tail_proc = None
 892         self.retry = None
 893         if not tasknames:
 894             if options.restrict_tests:
 895                 tasknames = ["samba-test-only"]
 896             else:
 897                 tasknames = defaulttasks
 898
 899         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
 900
 901         if options.retry:
 902             rebase_remote = "rebaseon"
 903             retry_task = [("retry",
 904                             '''set -e
 905                             git remote add -t %s %s %s
 906                             git fetch %s
 907                             while :; do
 908                               sleep 60
 909                               git describe %s/%s > old_remote_branch.desc
 910                               git fetch %s
 911                               git describe %s/%s > remote_branch.desc
 912                               diff old_remote_branch.desc remote_branch.desc
 913                             done
 914                            ''' % (
 915                                rebase_branch, rebase_remote, rebase_url,
 916                                rebase_remote,
 917                                rebase_remote, rebase_branch,
 918                                rebase_remote,
 919                                rebase_remote, rebase_branch
 920                             ))]
 921
 922             self.retry = builder('retry', retry_task, cp=False)
 923             self.need_retry = False
 924
 925     def kill_kids(self):
 926         if self.tail_proc is not None:
 927             self.tail_proc.terminate()
 928             self.tail_proc.wait()
 929             self.tail_proc = None
 930         if self.retry is not None:
 931             self.retry.proc.terminate()
 932             self.retry.proc.wait()
 933             self.retry = None
 934         for b in self.tlist:
 935             if b.proc is not None:
 936                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 937                 b.proc.terminate()
 938                 b.proc.wait()
 939                 b.proc = None
 940
 941     def wait_one(self):
 942         while True:
 943             none_running = True
 944             for b in self.tlist:
 945                 if b.proc is None:
 946                     continue
 947                 none_running = False
 948                 b.status = b.proc.poll()
 949                 if b.status is None:
 950                     continue
 951                 b.proc = None
 952                 return b
 953             if options.retry:
 954                 ret = self.retry.proc.poll()
 955                 if ret is not None:
 956                     self.need_retry = True
 957                     self.retry = None
 958                     return None
 959             if none_running:
 960                 return None
 961             time.sleep(0.1)
 962
 963     def run(self):
 964         while True:
 965             b = self.wait_one()
 966             if options.retry and self.need_retry:
 967                 self.kill_kids()
 968                 do_print("retry needed")
 969                 return (0, None, None, None, "retry")
 970             if b is None:
 971                 break
 972             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 973                 self.kill_kids()
 974                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 975             b.start_next()
 976         self.kill_kids()
 977         return (0, None, None, None, "All OK")
 978
 979     def write_system_info(self, filename):
 980         with open(filename, 'w') as f:
 981             for cmd in ['uname -a',
 982                         'lsb_release -a',
 983                         'free',
 984                         'mount',
 985                         'cat /proc/cpuinfo',
 986                         'cc --version',
 987                         'df -m .',
 988                         'df -m %s' % testbase]:
 989                 try:
 990                     out = run_cmd(cmd, output=True, checkfail=False)
 991                 except subprocess.CalledProcessError as e:
 992                     out = "<failed: %s>" % str(e)
 993                 print('### %s' % cmd, file=f)
 994                 print(out, file=f)
 995                 print(file=f)
 996
 997     def tarlogs(self, fname):
 998         with tarfile.open(fname, "w:gz") as tar:
 999             for b in self.tlist:
1000                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1001                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1002             if os.path.exists("autobuild.log"):
1003                 tar.add("autobuild.log")
1004             filename = 'system-info.txt'
1005             self.write_system_info(filename)
1006             tar.add(filename)
1007
1008     def remove_logs(self):
1009         for b in self.tlist:
1010             os.unlink(b.stdout_path)
1011             os.unlink(b.stderr_path)
1012
1013     def start_tail(self):
1014         cmd = ["tail", "-f"]
1015         for b in self.tlist:
1016             cmd.append(b.stdout_path)
1017             cmd.append(b.stderr_path)
1018         self.tail_proc = Popen(cmd, close_fds=True)
1019
1020
1021 def cleanup():
1022     if options.nocleanup:
1023         return
1024     run_cmd("stat %s || true" % test_tmpdir, show=True)
1025     run_cmd("stat %s" % testbase, show=True)
1026     do_print("Cleaning up %r" % cleanup_list)
1027     for d in cleanup_list:
1028         run_cmd("rm -rf %s" % d)
1029
1030
1031 def daemonize(logfile):
1032     pid = os.fork()
1033     if pid == 0:  # Parent
1034         os.setsid()
1035         pid = os.fork()
1036         if pid != 0:  # Actual daemon
1037             os._exit(0)
1038     else:  # Grandparent
1039         os._exit(0)
1040
1041     import resource      # Resource usage information.
1042     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1043     if maxfd == resource.RLIM_INFINITY:
1044         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1045     for fd in range(0, maxfd):
1046         try:
1047             os.close(fd)
1048         except OSError:
1049             pass
1050     os.open(logfile, os.O_RDWR | os.O_CREAT)
1051     os.dup2(0, 1)
1052     os.dup2(0, 2)
1053
1054
1055 def write_pidfile(fname):
1056     '''write a pid file, cleanup on exit'''
1057     with open(fname, mode='w') as f:
1058         f.write("%u\n" % os.getpid())
1059
1060
1061 def rebase_tree(rebase_url, rebase_branch="master"):
1062     rebase_remote = "rebaseon"
1063     do_print("Rebasing on %s" % rebase_url)
1064     run_cmd("git describe HEAD", show=True, dir=test_master)
1065     run_cmd("git remote add -t %s %s %s" %
1066             (rebase_branch, rebase_remote, rebase_url),
1067             show=True, dir=test_master)
1068     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1069     if options.fix_whitespace:
1070         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1071                 (rebase_remote, rebase_branch),
1072                 show=True, dir=test_master)
1073     else:
1074         run_cmd("git rebase --force-rebase %s/%s" %
1075                 (rebase_remote, rebase_branch),
1076                 show=True, dir=test_master)
1077     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1078                    (rebase_remote, rebase_branch),
1079                    dir=test_master, output=True)
1080     if diff == '':
1081         do_print("No differences between HEAD and %s/%s - exiting" %
1082                  (rebase_remote, rebase_branch))
1083         sys.exit(0)
1084     run_cmd("git describe %s/%s" %
1085             (rebase_remote, rebase_branch),
1086             show=True, dir=test_master)
1087     run_cmd("git describe HEAD", show=True, dir=test_master)
1088     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1089             (rebase_remote, rebase_branch),
1090             show=True, dir=test_master)
1091
1092
1093 def push_to(push_url, push_branch="master"):
1094     push_remote = "pushto"
1095     do_print("Pushing to %s" % push_url)
1096     if options.mark:
1097         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1098         run_cmd("git commit --amend -c HEAD", dir=test_master)
1099         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1100         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1101     run_cmd("git remote add -t %s %s %s" %
1102             (push_branch, push_remote, push_url),
1103             show=True, dir=test_master)
1104     run_cmd("git push %s +HEAD:%s" %
1105             (push_remote, push_branch),
1106             show=True, dir=test_master)
1107
1108
1109 def send_email(subject, text, log_tar):
1110     if options.email is None:
1111         do_print("not sending email because the recipient is not set")
1112         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1113                  (subject, text))
1114         return
1115     outer = MIMEMultipart()
1116     outer['Subject'] = subject
1117     outer['To'] = options.email
1118     outer['From'] = options.email_from
1119     outer['Date'] = email.utils.formatdate(localtime=True)
1120     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1121     outer.attach(MIMEText(text, 'plain'))
1122     if options.attach_logs:
1123         with open(log_tar, 'rb') as fp:
1124             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1125         # Set the filename parameter
1126         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1127         outer.attach(msg)
1128     content = outer.as_string()
1129     s = smtplib.SMTP(options.email_server)
1130     email_user = os.getenv('SMTP_USERNAME')
1131     email_password = os.getenv('SMTP_PASSWORD')
1132     if email_user is not None:
1133         s.starttls()
1134         s.login(email_user, email_password)
1135
1136     s.sendmail(options.email_from, [options.email], content)
1137     s.set_debuglevel(1)
1138     s.quit()
1139
1140
1141 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1142                   elapsed_time, log_base=None, add_log_tail=True):
1143     '''send an email to options.email about the failure'''
1144     elapsed_minutes = elapsed_time / 60.0
1145     if log_base is None:
1146         log_base = gitroot
1147     text = '''
1148 Dear Developer,
1149
1150 Your autobuild on %s failed after %.1f minutes
1151 when trying to test %s with the following error:
1152
1153    %s
1154
1155 the autobuild has been abandoned. Please fix the error and resubmit.
1156
1157 A summary of the autobuild process is here:
1158
1159   %s/autobuild.log
1160 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1161
1162     if options.restrict_tests:
1163         text += """
1164 The build was restricted to tests matching %s\n""" % options.restrict_tests
1165
1166     if failed_task != 'rebase':
1167         text += '''
1168 You can see logs of the failed task here:
1169
1170   %s/%s.stdout
1171   %s/%s.stderr
1172
1173 or you can get full logs of all tasks in this job here:
1174
1175   %s/logs.tar.gz
1176
1177 The top commit for the tree that was built was:
1178
1179 %s
1180
1181 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1182
1183     if add_log_tail:
1184         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1185         lines = f.readlines()
1186         log_tail = "".join(lines[-50:])
1187         num_lines = len(lines)
1188         if num_lines < 50:
1189             # Also include stderr (compile failures) if < 50 lines of stdout
1190             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1191             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1192
1193         text += '''
1194 The last 50 lines of log messages:
1195
1196 %s
1197     ''' % log_tail
1198         f.close()
1199
1200     logs = os.path.join(gitroot, 'logs.tar.gz')
1201     send_email('autobuild[%s] failure on %s for task %s during %s'
1202                % (options.branch, platform.node(), failed_task, failed_stage),
1203                text, logs)
1204
1205
1206 def email_success(elapsed_time, log_base=None):
1207     '''send an email to options.email about a successful build'''
1208     if log_base is None:
1209         log_base = gitroot
1210     text = '''
1211 Dear Developer,
1212
1213 Your autobuild on %s has succeeded after %.1f minutes.
1214
1215 ''' % (platform.node(), elapsed_time / 60.)
1216
1217     if options.restrict_tests:
1218         text += """
1219 The build was restricted to tests matching %s\n""" % options.restrict_tests
1220
1221     if options.keeplogs:
1222         text += '''
1223
1224 you can get full logs of all tasks in this job here:
1225
1226   %s/logs.tar.gz
1227
1228 ''' % log_base
1229
1230     text += '''
1231 The top commit for the tree that was built was:
1232
1233 %s
1234 ''' % top_commit_msg
1235
1236     logs = os.path.join(gitroot, 'logs.tar.gz')
1237     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1238                text, logs)
1239
1240
1241 # get the top commit message, for emails
1242 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1243
1244 try:
1245     os.makedirs(testbase)
1246 except Exception as reason:
1247     raise Exception("Unable to create %s : %s" % (testbase, reason))
1248 cleanup_list.append(testbase)
1249
1250 if options.daemon:
1251     logfile = os.path.join(testbase, "log")
1252     do_print("Forking into the background, writing progress to %s" % logfile)
1253     daemonize(logfile)
1254
1255 write_pidfile(gitroot + "/autobuild.pid")
1256
1257 start_time = time.time()
1258
1259 while True:
1260     try:
1261         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1262         os.makedirs(test_tmpdir)
1263         # The waf uninstall code removes empty directories all the way
1264         # up the tree.  Creating a file in test_tmpdir stops it from
1265         # being removed.
1266         run_cmd("touch %s" % os.path.join(test_tmpdir,
1267                                           ".directory-is-not-empty"), show=True)
1268         run_cmd("stat %s" % test_tmpdir, show=True)
1269         run_cmd("stat %s" % testbase, show=True)
1270         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1271     except Exception:
1272         cleanup()
1273         raise
1274
1275     try:
1276         if options.rebase is not None:
1277             rebase_tree(options.rebase, rebase_branch=options.branch)
1278     except Exception:
1279         cleanup_list.append(gitroot + "/autobuild.pid")
1280         cleanup()
1281         elapsed_time = time.time() - start_time
1282         email_failure(-1, 'rebase', 'rebase', 'rebase',
1283                       'rebase on %s failed' % options.branch,
1284                       elapsed_time, log_base=options.log_base)
1285         sys.exit(1)
1286
1287     try:
1288         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1289         if options.tail:
1290             blist.start_tail()
1291         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1292         if status != 0 or errstr != "retry":
1293             break
1294         cleanup()
1295     except Exception:
1296         cleanup()
1297         raise
1298
1299 cleanup_list.append(gitroot + "/autobuild.pid")
1300
1301 do_print(errstr)
1302
1303 blist.kill_kids()
1304 if options.tail:
1305     do_print("waiting for tail to flush")
1306     time.sleep(1)
1307
1308 elapsed_time = time.time() - start_time
1309 if status == 0:
1310     if options.passcmd is not None:
1311         do_print("Running passcmd: %s" % options.passcmd)
1312         run_cmd(options.passcmd, dir=test_master)
1313     if options.pushto is not None:
1314         push_to(options.pushto, push_branch=options.branch)
1315     if options.keeplogs or options.attach_logs:
1316         blist.tarlogs("logs.tar.gz")
1317         do_print("Logs in logs.tar.gz")
1318     if options.always_email:
1319         email_success(elapsed_time, log_base=options.log_base)
1320     blist.remove_logs()
1321     cleanup()
1322     do_print(errstr)
1323     sys.exit(0)
1324
1325 # something failed, gather a tar of the logs
1326 blist.tarlogs("logs.tar.gz")
1327
1328 if options.email is not None:
1329     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1330                   elapsed_time, log_base=options.log_base)
1331 else:
1332     elapsed_minutes = elapsed_time / 60.0
1333     print('''
1334
1335 ####################################################################
1336
1337 AUTOBUILD FAILURE
1338
1339 Your autobuild[%s] on %s failed after %.1f minutes
1340 when trying to test %s with the following error:
1341
1342    %s
1343
1344 the autobuild has been abandoned. Please fix the error and resubmit.
1345
1346 ####################################################################
1347
1348 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1349
1350 cleanup()
1351 do_print(errstr)
1352 do_print("Logs in logs.tar.gz")
1353 sys.exit(status)