script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 # allow autobuild to run within git rebase -i
  35 if "GIT_DIR" in os.environ:
  36     del os.environ["GIT_DIR"]
  37 if "GIT_WORK_TREE" in os.environ:
  38     del os.environ["GIT_WORK_TREE"]
  39
  40 def find_git_root():
  41     '''get to the top of the git repo'''
  42     p = os.getcwd()
  43     while p != '/':
  44         if os.path.exists(os.path.join(p, ".git")):
  45             return p
  46         p = os.path.abspath(os.path.join(p, '..'))
  47     return None
  48
  49
  50 gitroot = find_git_root()
  51 if gitroot is None:
  52     raise Exception("Failed to find git root")
  53
  54
  55 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  56
  57 parser = OptionParser()
  58 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  59 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  60 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  61 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  62 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  63                   default=def_testbase)
  64 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  65                   default=None)
  66 parser.add_option("--passcmd", help="command to run on success", default=None)
  67 parser.add_option("--verbose", help="show all commands as they are run",
  68                   default=False, action="store_true")
  69 parser.add_option("--rebase", help="rebase on the given tree before testing",
  70                   default=None, type='str')
  71 parser.add_option("--pushto", help="push to a git url on success",
  72                   default=None, type='str')
  73 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  74                   default=False, action="store_true")
  75 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  76                   default=False, action="store_true")
  77 parser.add_option("--retry", help="automatically retry if master changes",
  78                   default=False, action="store_true")
  79 parser.add_option("--email", help="send email to the given address on failure",
  80                   type='str', default=None)
  81 parser.add_option("--email-from", help="send email from the given address",
  82                   type='str', default="autobuild@samba.org")
  83 parser.add_option("--email-server", help="send email via the given server",
  84                   type='str', default='localhost')
  85 parser.add_option("--always-email", help="always send email, even on success",
  86                   action="store_true")
  87 parser.add_option("--daemon", help="daemonize after initial setup",
  88                   action="store_true")
  89 parser.add_option("--branch", help="the branch to work on (default=master)",
  90                   default="master", type='str')
  91 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  92                   default=gitroot, type='str')
  93 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  94                   default=False, action="store_true")
  95 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  96                   default='')
  97 parser.add_option("--enable-coverage", dest='enable_coverage',
  98                   action="store_const", const='--enable-coverage', default='',
  99                   help="Add --enable-coverage option while configure")
 100
 101 (options, args) = parser.parse_args()
 102
 103 if options.retry:
 104     if options.rebase is None:
 105         raise Exception('You can only use --retry if you also rebase')
 106
 107 if options.full_testbase is not None:
 108     testbase = options.full_testbase
 109 else:
 110     testbase = "%s/b%u" % (options.testbase, os.getpid())
 111 test_master = "%s/master" % testbase
 112 test_prefix = "%s/prefix" % testbase
 113 test_tmpdir = "%s/tmp" % testbase
 114 os.environ['TMPDIR'] = test_tmpdir
 115
 116 if options.enable_coverage:
 117     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 118 else:
 119     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 120
 121 if options.enable_coverage:
 122     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 123 else:
 124     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 125
 126 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 127
 128
 129 def check_symbols(sofile, expected_symbols=""):
 130     return "objdump --dynamic-syms " + sofile + " | " + \
 131            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 132
 133 if args:
 134     # If we are only running specific test,
 135     # do not sleep randomly to wait for it to start
 136     def random_sleep(low, high):
 137         return 'sleep 1'
 138 else:
 139     def random_sleep(low, high):
 140         return 'sleep {}'.format(random.randint(low, high))
 141
 142 cleanup_list = []
 143
 144 builddirs = {
 145     "ctdb": "ctdb",
 146     "ldb": "lib/ldb",
 147     "tdb": "lib/tdb",
 148     "talloc": "lib/talloc",
 149     "replace": "lib/replace",
 150     "tevent": "lib/tevent",
 151     "pidl": "pidl",
 152     "docs-xml": "docs-xml"
 153 }
 154
 155 ctdb_configure_params = " --enable-developer ${PREFIX}"
 156 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 157
 158 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 159 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 160 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 161 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 162 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 163 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 164 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 165
 166
 167 def format_option(name, value=None):
 168     """Format option as str list."""
 169     if value is None:  # boolean option
 170         return [name]
 171     if not isinstance(value, list):  # single value option
 172         value = [value]
 173     # repeatable option
 174     return ['{}={}'.format(name, item) for item in value]
 175
 176
 177 def make_test(
 178         cmd='make testonly',
 179         INJECT_SELFTEST_PREFIX=1,
 180         TESTS='',
 181         include_envs=None,
 182         exclude_envs=None):
 183
 184     test_options = []
 185     if include_envs:
 186         test_options = format_option('--include-env', include_envs)
 187     if exclude_envs:
 188         test_options = format_option('--exclude-env', exclude_envs)
 189     if test_options:
 190         # join envs options to original test options
 191         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 192
 193     _options = []
 194
 195     # Allow getting a full CI with
 196     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 197
 198     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 199
 200     if int(FAIL_IMMEDIATELY):
 201         _options.append('FAIL_IMMEDIATELY=1')
 202     if TESTS:
 203         _options.append("TESTS='{}'".format(TESTS))
 204
 205     if INJECT_SELFTEST_PREFIX:
 206         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 207         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 208
 209     return ' '.join([cmd] + _options)
 210
 211
 212 # When updating this list, also update .gitlab-ci.yml to add the job
 213 # and to make it a dependency of 'page' for the coverage report.
 214
 215 tasks = {
 216     "ctdb": {
 217         "sequence": [
 218             ("random-sleep", random_sleep(300, 900)),
 219             ("configure", "./configure " + ctdb_configure_params),
 220             ("make", "make all"),
 221             ("install", "make install"),
 222             ("test", "make autotest"),
 223             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 224             ("clean", "make clean"),
 225         ],
 226     },
 227     "docs-xml": {
 228         "sequence": [
 229             ("random-sleep", random_sleep(300, 900)),
 230             ("autoconf", "autoconf"),
 231             ("configure", "./configure"),
 232             ("make", "make html htmlman"),
 233             ("publish-docs", PUBLISH_DOCS),
 234             ("clean", "make clean"),
 235         ],
 236     },
 237
 238     "samba-def-build": {
 239         "git-clone-required": True,
 240         "sequence": [
 241             ("configure", "./configure.developer" + samba_configure_params),
 242             ("make", "make -j"),
 243             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 244             ("chmod-R-a-w", "chmod -R a-w ."),
 245         ],
 246     },
 247
 248     "samba-mit-build": {
 249         "git-clone-required": True,
 250         "sequence": [
 251             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 252             ("make", "make -j"),
 253             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 254             ("chmod-R-a-w", "chmod -R a-w ."),
 255         ],
 256     },
 257
 258     "samba-nt4-build": {
 259         "git-clone-required": True,
 260         "sequence": [
 261             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 262             ("make", "make -j"),
 263             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 264             ("chmod-R-a-w", "chmod -R a-w ."),
 265         ],
 266     },
 267
 268     "samba-h5l-build": {
 269         "git-clone-required": True,
 270         "sequence": [
 271             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 272             ("make", "make -j"),
 273             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 274             ("chmod-R-a-w", "chmod -R a-w ."),
 275         ],
 276     },
 277
 278     "samba-without-smb1-build": {
 279         "git-clone-required": True,
 280         "sequence": [
 281             ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
 282             ("make", "make -j"),
 283             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 284             ("chmod-R-a-w", "chmod -R a-w ."),
 285         ],
 286     },
 287
 288     "samba-no-opath-build": {
 289         "git-clone-required": True,
 290         "sequence": [
 291             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 292             ("make", "make -j"),
 293             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 294             ("chmod-R-a-w", "chmod -R a-w ."),
 295         ],
 296     },
 297
 298     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 299     "samba": {
 300         "sequence": [
 301             ("random-sleep", random_sleep(300, 900)),
 302             ("configure", "./configure.developer" + samba_configure_params),
 303             ("make", "make -j"),
 304             ("test", make_test(exclude_envs=[
 305             "none",
 306             "nt4_dc",
 307             "nt4_dc_smb1",
 308             "nt4_dc_smb1_done",
 309             "nt4_dc_schannel",
 310             "nt4_member",
 311             "ad_dc",
 312             "ad_dc_smb1",
 313             "ad_dc_smb1_done",
 314             "ad_dc_backup",
 315             "ad_dc_ntvfs",
 316             "ad_dc_default",
 317             "ad_dc_default_smb1",
 318             "ad_dc_slowtests",
 319             "ad_dc_no_nss",
 320             "ad_dc_no_ntlm",
 321             "fl2003dc",
 322             "fl2008dc",
 323             "fl2008r2dc",
 324             "ad_member",
 325             "ad_member_idmap_rid",
 326             "admem_idmap_autorid",
 327             "ad_member_idmap_ad",
 328             "ad_member_rfc2307",
 329             "ad_member_oneway",
 330             "chgdcpass",
 331             "vampire_2000_dc",
 332             "fl2000dc",
 333             "fileserver",
 334             "fileserver_smb1",
 335             "fileserver_smb1_done",
 336             "maptoguest",
 337             "simpleserver",
 338             "backupfromdc",
 339             "restoredc",
 340             "renamedc",
 341             "offlinebackupdc",
 342             "labdc",
 343             "preforkrestartdc",
 344             "proclimitdc",
 345             "promoted_dc",
 346             "vampire_dc",
 347             "rodc",
 348             "ad_dc_default",
 349             "ad_dc_default_smb1",
 350             "ad_dc_default_smb1_done",
 351             "ad_dc_slowtests",
 352             "schema_pair_dc",
 353             "schema_dc",
 354             "clusteredmember",
 355             "ad_dc_fips",
 356             "ad_member_fips",
 357             ])),
 358             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 359             ("lcov", LCOV_CMD),
 360             ("install", "make install"),
 361             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 362             ("clean", "make clean"),
 363         ],
 364     },
 365
 366     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 367     "samba-mitkrb5": {
 368         "sequence": [
 369             ("random-sleep", random_sleep(300, 900)),
 370             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 371             ("make", "make -j"),
 372             ("test", make_test(exclude_envs=[
 373             "none",
 374             "nt4_dc",
 375             "nt4_dc_smb1",
 376             "nt4_dc_smb1_done",
 377             "nt4_dc_schannel",
 378             "nt4_member",
 379             "ad_dc",
 380             "ad_dc_smb1",
 381             "ad_dc_smb1_done",
 382             "ad_dc_backup",
 383             "ad_dc_ntvfs",
 384             "ad_dc_default",
 385             "ad_dc_default_smb1",
 386             "ad_dc_default_smb1_done",
 387             "ad_dc_slowtests",
 388             "ad_dc_no_nss",
 389             "ad_dc_no_ntlm",
 390             "fl2003dc",
 391             "fl2008dc",
 392             "fl2008r2dc",
 393             "ad_member",
 394             "ad_member_idmap_rid",
 395             "admem_idmap_autorid",
 396             "ad_member_idmap_ad",
 397             "ad_member_rfc2307",
 398             "ad_member_oneway",
 399             "chgdcpass",
 400             "vampire_2000_dc",
 401             "fl2000dc",
 402             "fileserver",
 403             "fileserver_smb1",
 404             "fileserver_smb1_done",
 405             "maptoguest",
 406             "simpleserver",
 407             "backupfromdc",
 408             "restoredc",
 409             "renamedc",
 410             "offlinebackupdc",
 411             "labdc",
 412             "preforkrestartdc",
 413             "proclimitdc",
 414             "promoted_dc",
 415             "vampire_dc",
 416             "rodc",
 417             "ad_dc_default",
 418             "ad_dc_default_smb1",
 419             "ad_dc_default_smb1_done",
 420             "ad_dc_slowtests",
 421             "schema_pair_dc",
 422             "schema_dc",
 423             "clusteredmember",
 424             "ad_dc_fips",
 425             "ad_member_fips",
 426             ])),
 427             ("lcov", LCOV_CMD),
 428             ("install", "make install"),
 429             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 430             ("clean", "make clean"),
 431         ],
 432     },
 433
 434     "samba-nt4": {
 435         "dependency": "samba-nt4-build",
 436         "sequence": [
 437             ("random-sleep", random_sleep(300, 900)),
 438             ("test", make_test(include_envs=[
 439             "nt4_dc",
 440             "nt4_dc_smb1",
 441             "nt4_dc_smb1_done",
 442             "nt4_dc_schannel",
 443             "nt4_member",
 444             "simpleserver",
 445             ])),
 446             ("lcov", LCOV_CMD),
 447             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 448         ],
 449     },
 450
 451     "samba-fileserver": {
 452         "dependency": "samba-h5l-build",
 453         "sequence": [
 454             ("random-sleep", random_sleep(300, 900)),
 455             ("test", make_test(include_envs=[
 456             "fileserver",
 457             "fileserver_smb1",
 458             "fileserver_smb1_done",
 459             "maptoguest",
 460             "ktest", # ktest is also tested in samba-ktest-mit samba
 461                      # and samba-mitkrb5 but is tested here against
 462                      # a system Heimdal
 463             ])),
 464             ("lcov", LCOV_CMD),
 465             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 466         ],
 467     },
 468
 469     "samba-fileserver-without-smb1": {
 470         "dependency": "samba-without-smb1-build",
 471         "sequence": [
 472             ("random-sleep", random_sleep(300, 900)),
 473             ("test", make_test(include_envs=["fileserver"])),
 474             ("lcov", LCOV_CMD),
 475             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 476         ],
 477     },
 478
 479     # This is a full build without the AD DC so we test the build with
 480     # MIT Kerberos from the current system.  Runtime behaviour is
 481     # confirmed via the ktest (static ccache and keytab) environment
 482
 483     "samba-ktest-mit": {
 484         "sequence": [
 485             ("random-sleep", random_sleep(300, 900)),
 486             ("configure", "./configure.developer --without-ad-dc --with-system-mitkrb5 " + samba_configure_params),
 487             ("make", "make -j"),
 488             ("test", make_test(include_envs=[
 489             "ktest", # ktest is also tested in fileserver, samba and
 490                      # samba-mitkrb5 but is tested here against a
 491                      # system MIT krb5
 492             ])),
 493             ("lcov", LCOV_CMD),
 494             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 495         ],
 496     },
 497
 498     "samba-admem": {
 499         "dependency": "samba-def-build",
 500         "sequence": [
 501             ("random-sleep", random_sleep(300, 900)),
 502             ("test", make_test(include_envs=[
 503             "ad_member",
 504             "ad_member_idmap_rid",
 505             "admem_idmap_autorid",
 506             "ad_member_idmap_ad",
 507             "ad_member_rfc2307",
 508             "ad_member_offlogon",
 509             ])),
 510             ("lcov", LCOV_CMD),
 511             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 512         ],
 513     },
 514
 515     "samba-no-opath1": {
 516         "dependency": "samba-no-opath-build",
 517         "sequence": [
 518             ("random-sleep", random_sleep(300, 900)),
 519             ("test", make_test(
 520                 cmd="make testonly DISABLE_OPATH=1",
 521                 include_envs=[
 522                 "nt4_dc",
 523                 "nt4_dc_smb1",
 524                 "nt4_dc_smb1_done",
 525                 "nt4_dc_schannel",
 526                 "nt4_member",
 527                 "simpleserver",
 528                 ])),
 529             ("lcov", LCOV_CMD),
 530             ("check-clean-tree", "script/clean-source-tree.sh"),
 531         ],
 532     },
 533
 534     "samba-no-opath2": {
 535         "dependency": "samba-no-opath-build",
 536         "sequence": [
 537             ("random-sleep", random_sleep(300, 900)),
 538             ("test", make_test(
 539                 cmd="make testonly DISABLE_OPATH=1",
 540                 include_envs=[
 541                 "fileserver",
 542                 "fileserver_smb1",
 543                 "fileserver_smb1_done",
 544                 ])),
 545             ("lcov", LCOV_CMD),
 546             ("check-clean-tree", "script/clean-source-tree.sh"),
 547         ],
 548     },
 549
 550     "samba-ad-dc-1": {
 551         "dependency": "samba-def-build",
 552         "sequence": [
 553             ("random-sleep", random_sleep(1, 1)),
 554             ("test", make_test(include_envs=[
 555             "ad_dc",
 556             "ad_dc_smb1",
 557             "ad_dc_smb1_done",
 558             "ad_dc_no_nss",
 559             "ad_dc_no_ntlm",
 560             ])),
 561             ("lcov", LCOV_CMD),
 562             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 563         ],
 564     },
 565
 566     "samba-ad-dc-2": {
 567         "dependency": "samba-def-build",
 568         "sequence": [
 569             ("random-sleep", random_sleep(1, 1)),
 570             ("test", make_test(include_envs=[
 571             "vampire_dc",
 572             "vampire_2000_dc",
 573             "rodc",
 574             ])),
 575             ("lcov", LCOV_CMD),
 576             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 577         ],
 578     },
 579
 580     "samba-ad-dc-3": {
 581         "dependency": "samba-def-build",
 582         "sequence": [
 583             ("random-sleep", random_sleep(1, 1)),
 584             ("test", make_test(include_envs=[
 585             "promoted_dc",
 586             "chgdcpass",
 587             "preforkrestartdc",
 588             "proclimitdc",
 589             ])),
 590             ("lcov", LCOV_CMD),
 591             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 592         ],
 593     },
 594
 595     "samba-ad-dc-4a": {
 596         "dependency": "samba-def-build",
 597         "sequence": [
 598             ("random-sleep", random_sleep(1, 1)),
 599             ("test", make_test(include_envs=[
 600             "fl2000dc",
 601             "ad_member_oneway",
 602             "fl2003dc",
 603             ])),
 604             ("lcov", LCOV_CMD),
 605             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 606         ],
 607     },
 608     "samba-ad-dc-4b": {
 609         "dependency": "samba-def-build",
 610         "sequence": [
 611             ("random-sleep", random_sleep(1, 1)),
 612             ("test", make_test(include_envs=[
 613             "fl2008dc",
 614             "fl2008r2dc",
 615             ])),
 616             ("lcov", LCOV_CMD),
 617             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 618         ],
 619     },
 620
 621     "samba-ad-dc-5": {
 622         "dependency": "samba-def-build",
 623         "sequence": [
 624             ("random-sleep", random_sleep(1, 1)),
 625             ("test", make_test(include_envs=[
 626             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 627             ("lcov", LCOV_CMD),
 628             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 629         ],
 630     },
 631
 632     "samba-ad-dc-6": {
 633         "dependency": "samba-def-build",
 634         "sequence": [
 635             ("random-sleep", random_sleep(1, 1)),
 636             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 637             ("lcov", LCOV_CMD),
 638             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 639         ],
 640     },
 641
 642     "samba-schemaupgrade": {
 643         "dependency": "samba-def-build",
 644         "sequence": [
 645             ("random-sleep", random_sleep(1, 1)),
 646             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 647             ("lcov", LCOV_CMD),
 648             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 649         ],
 650     },
 651
 652     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 653     # This is currently the longest task, so we don't randomly delay it.
 654     "samba-ad-dc-ntvfs": {
 655         "dependency": "samba-def-build",
 656         "sequence": [
 657             ("random-sleep", random_sleep(1, 1)),
 658             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 659             ("lcov", LCOV_CMD),
 660             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 661         ],
 662     },
 663
 664     # Test fips compliance
 665     "samba-fips": {
 666         "dependency": "samba-mit-build",
 667         "sequence": [
 668             ("random-sleep", random_sleep(1, 1)),
 669             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 670             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 671             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 672         ],
 673     },
 674
 675     # run the backup/restore testenvs separately as they're fairly standalone
 676     # (and CI seems to max out at ~3 different DCs running at once)
 677     "samba-ad-back1": {
 678         "dependency": "samba-def-build",
 679         "sequence": [
 680             ("random-sleep", random_sleep(300, 900)),
 681             ("test", make_test(include_envs=[
 682             "backupfromdc",
 683             "restoredc",
 684             "renamedc",
 685             ])),
 686             ("lcov", LCOV_CMD),
 687             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 688         ],
 689     },
 690     "samba-ad-back2": {
 691         "dependency": "samba-def-build",
 692         "sequence": [
 693             ("random-sleep", random_sleep(300, 900)),
 694             ("test", make_test(include_envs=[
 695             "backupfromdc",
 696             "offlinebackupdc",
 697             "labdc",
 698             ])),
 699             ("lcov", LCOV_CMD),
 700             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 701         ],
 702     },
 703
 704     "samba-admem-mit": {
 705         "dependency": "samba-mit-build",
 706         "sequence": [
 707             ("random-sleep", random_sleep(1, 1)),
 708             ("test", make_test(include_envs=[
 709             "ad_member",
 710             "ad_member_idmap_rid",
 711             "admem_idmap_autorid",
 712             "ad_member_idmap_ad",
 713             "ad_member_rfc2307",
 714             "ad_member_offlogon",
 715             ])),
 716             ("lcov", LCOV_CMD),
 717             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 718         ],
 719     },
 720
 721     "samba-addc-mit-1": {
 722         "dependency": "samba-mit-build",
 723         "sequence": [
 724             ("random-sleep", random_sleep(1, 1)),
 725             ("test", make_test(include_envs=[
 726             "ad_dc",
 727             "ad_dc_smb1",
 728             "ad_dc_smb1_done",
 729             "ad_dc_no_nss",
 730             "ad_dc_no_ntlm",
 731             ])),
 732             ("lcov", LCOV_CMD),
 733             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 734         ],
 735     },
 736
 737     "samba-addc-mit-4a": {
 738         "dependency": "samba-mit-build",
 739         "sequence": [
 740             ("random-sleep", random_sleep(1, 1)),
 741             ("test", make_test(include_envs=[
 742             "fl2000dc",
 743             "ad_member_oneway",
 744             "fl2003dc",
 745             ])),
 746             ("lcov", LCOV_CMD),
 747             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 748         ],
 749     },
 750     "samba-addc-mit-4b": {
 751         "dependency": "samba-mit-build",
 752         "sequence": [
 753             ("random-sleep", random_sleep(1, 1)),
 754             ("test", make_test(include_envs=[
 755             "fl2008dc",
 756             "fl2008r2dc",
 757             ])),
 758             ("lcov", LCOV_CMD),
 759             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 760         ],
 761     },
 762
 763     "samba-test-only": {
 764         "sequence": [
 765             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 766             ("make", "make -j"),
 767             ("test", make_test(TESTS="${TESTS}")),
 768             ("lcov", LCOV_CMD),
 769         ],
 770     },
 771
 772     # Test cross-compile infrastructure
 773     "samba-xc": {
 774         "sequence": [
 775             ("random-sleep", random_sleep(900, 1500)),
 776             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 777             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 778             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 779             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 780             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 781             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 782             ("compare-results", "script/compare_cc_results.py "
 783             "./bin/c4che/default{} "
 784             "./bin-xe/c4che/default{} "
 785             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 786             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 787             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 788             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 789             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 790             " = \"'1234'\"".format(CACHE_SUFFIX)),
 791             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 792             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 793             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 794             " ; test $? -ne 0"),
 795         ],
 796     },
 797
 798     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 799     "samba-o3": {
 800         "sequence": [
 801             ("random-sleep", random_sleep(300, 900)),
 802             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 803             ("make", "make -j"),
 804             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 805             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 806             ("lcov", LCOV_CMD),
 807             ("install", "make install"),
 808             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 809             ("clean", "make clean"),
 810         ],
 811     },
 812
 813     "samba-ctdb": {
 814         "sequence": [
 815             ("random-sleep", random_sleep(900, 1500)),
 816
 817         # make sure we have tdb around:
 818             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 819             ("tdb-make", "cd lib/tdb && make"),
 820             ("tdb-install", "cd lib/tdb && make install"),
 821
 822         # build samba with cluster support (also building ctdb):
 823             ("samba-configure",
 824          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 825          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 826          "./configure.developer ${PREFIX} "
 827          "--with-selftest-prefix=./bin/ab "
 828          "--with-cluster-support "
 829          "--without-ad-dc "
 830          "--bundled-libraries=!tdb"),
 831             ("samba-make", "make"),
 832             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 833             ("samba-install", "make install"),
 834             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 835
 836             ("test", make_test(
 837                 cmd='make test',
 838                 INJECT_SELFTEST_PREFIX=0,
 839                 include_envs=["clusteredmember"])
 840             ),
 841
 842         # clean up:
 843             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 844             ("clean", "make clean"),
 845             ("ctdb-clean", "cd ./ctdb && make clean"),
 846         ],
 847     },
 848
 849     "samba-libs": {
 850         "sequence": [
 851             ("random-sleep", random_sleep(300, 900)),
 852             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 853             ("talloc-make", "cd lib/talloc && make"),
 854             ("talloc-install", "cd lib/talloc && make install"),
 855
 856             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 857             ("tdb-make", "cd lib/tdb && make"),
 858             ("tdb-install", "cd lib/tdb && make install"),
 859
 860             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 861             ("tevent-make", "cd lib/tevent && make"),
 862             ("tevent-install", "cd lib/tevent && make install"),
 863
 864             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 865             ("ldb-make", "cd lib/ldb && make"),
 866             ("ldb-install", "cd lib/ldb && make install"),
 867
 868             ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
 869             ("nondevel-make", "make -j"),
 870             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 871             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 872             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 873             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 874             ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 875             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 876             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 877             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 878             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 879             ("nondevel-no-public-nss_winbind",
 880                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 881             ("nondevel-no-public-nss_wins",
 882                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 883             ("nondevel-no-public-libwbclient",
 884                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 885             ("nondevel-no-public-pam_winbind",
 886                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 887             ("nondevel-no-public-winbind_krb5_locator",
 888                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 889             ("nondevel-no-public-async_dns_krb5_locator",
 890                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 891             ("nondevel-install", "make -j install"),
 892             ("nondevel-dist", "make dist"),
 893
 894             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 895             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 896             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 897             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
 898             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 899             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 900             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 901             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 902             ("prefix-no-public-nss_winbind",
 903                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 904             ("prefix-no-public-nss_wins",
 905                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
 906             ("prefix-no-public-libwbclient",
 907                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
 908             ("prefix-no-public-pam_winbind",
 909                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
 910             ("prefix-no-public-winbind_krb5_locator",
 911                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
 912                               "service_locator")),
 913             ("prefix-no-public-async_dns_krb5_locator",
 914                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
 915                               "service_locator")),
 916
 917             # retry with all modules shared
 918             ("allshared-distclean", "make distclean"),
 919             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 920             ("allshared-make", "make -j"),
 921             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 922             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 923             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 924             ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 925             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 926             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 927             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 928             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 929             ("allshared-no-public-nss_winbind",
 930                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 931             ("allshared-no-public-nss_wins",
 932                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 933             ("allshared-no-public-libwbclient",
 934                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 935             ("allshared-no-public-pam_winbind",
 936                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 937             ("allshared-no-public-winbind_krb5_locator",
 938                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 939             ("allshared-no-public-async_dns_krb5_locator",
 940                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 941         ],
 942     },
 943
 944     "samba-fuzz": {
 945         "sequence": [
 946         # build the fuzzers (static) via the oss-fuzz script
 947             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 948             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 949         ],
 950     },
 951
 952     # * Test smbd and smbtorture can build semi-static
 953     #
 954     # * Test Samba without python still builds.
 955     #
 956     # When this test fails due to more use of Python, the expectations
 957     # is that the newly failing part of the code should be disabled
 958     # when --disable-python is set (rather than major work being done
 959     # to support this environment).
 960     #
 961     # The target here is for vendors shipping a minimal smbd.
 962     "samba-minimal-smbd": {
 963         "sequence": [
 964             ("random-sleep", random_sleep(300, 900)),
 965
 966         # build with all modules static
 967             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 968             ("allstatic-make", "make -j"),
 969             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 970             ("allstatic-lcov", LCOV_CMD),
 971
 972         # retry with nonshared smbd and smbtorture
 973             ("nonshared-distclean", "make distclean"),
 974             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 975             ("nonshared-make", "make -j"),
 976             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 977             # TODO ("nonshared-lcov", LCOV_CMD),
 978
 979             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 980             ("clean", "make clean"),
 981         ],
 982     },
 983
 984     "samba-nopython": {
 985         "sequence": [
 986             ("random-sleep", random_sleep(300, 900)),
 987
 988             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 989             ("make", "make -j"),
 990             ("find-python", "script/find_python.sh ${PREFIX}"),
 991             ("test", "make test-nopython"),
 992             ("lcov", LCOV_CMD),
 993             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 994             ("clean", "make clean"),
 995
 996             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 997             ("talloc-make", "cd lib/talloc && make"),
 998             ("talloc-install", "cd lib/talloc && make install"),
 999
1000             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1001             ("tdb-make", "cd lib/tdb && make"),
1002             ("tdb-install", "cd lib/tdb && make install"),
1003
1004             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1005             ("tevent-make", "cd lib/tevent && make"),
1006             ("tevent-install", "cd lib/tevent && make install"),
1007
1008             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1009             ("ldb-make", "cd lib/ldb && make"),
1010             ("ldb-install", "cd lib/ldb && make install"),
1011
1012         # retry against installed library packages, but no required modules
1013             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
1014             ("libs-make", "make -j"),
1015             ("libs-install", "make install"),
1016             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1017             ("libs-clean", "make clean"),
1018
1019         ],
1020     },
1021
1022     "ldb": {
1023         "sequence": [
1024             ("random-sleep", random_sleep(60, 600)),
1025             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1026             ("make", "make"),
1027             ("install", "make install"),
1028             ("test", "make test"),
1029             ("lcov", LCOV_CMD),
1030             ("clean", "make clean"),
1031             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
1032             ("make-no-lmdb", "make"),
1033             ("test-no-lmdb", "make test"),
1034             ("lcov-no-lmdb", LCOV_CMD),
1035             ("install-no-lmdb", "make install"),
1036             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1037             ("distcheck", "make distcheck"),
1038             ("clean", "make clean"),
1039         ],
1040     },
1041
1042     "tdb": {
1043         "sequence": [
1044             ("random-sleep", random_sleep(60, 600)),
1045             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1046             ("make", "make"),
1047             ("install", "make install"),
1048             ("test", "make test"),
1049             ("lcov", LCOV_CMD),
1050             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1051             ("distcheck", "make distcheck"),
1052             ("clean", "make clean"),
1053         ],
1054     },
1055
1056     "talloc": {
1057         "sequence": [
1058             ("random-sleep", random_sleep(60, 600)),
1059             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1060             ("make", "make"),
1061             ("install", "make install"),
1062             ("test", "make test"),
1063             ("lcov", LCOV_CMD),
1064             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1065             ("distcheck", "make distcheck"),
1066             ("clean", "make clean"),
1067         ],
1068     },
1069
1070     "replace": {
1071         "sequence": [
1072             ("random-sleep", random_sleep(60, 600)),
1073             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1074             ("make", "make"),
1075             ("install", "make install"),
1076             ("test", "make test"),
1077             ("lcov", LCOV_CMD),
1078             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1079             ("distcheck", "make distcheck"),
1080             ("clean", "make clean"),
1081         ],
1082     },
1083
1084     "tevent": {
1085         "sequence": [
1086             ("random-sleep", random_sleep(60, 600)),
1087             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1088             ("make", "make"),
1089             ("install", "make install"),
1090             ("test", "make test"),
1091             ("lcov", LCOV_CMD),
1092             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1093             ("distcheck", "make distcheck"),
1094             ("clean", "make clean"),
1095         ],
1096     },
1097
1098     "pidl": {
1099         "git-clone-required": True,
1100         "sequence": [
1101             ("random-sleep", random_sleep(60, 600)),
1102             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1103             ("touch", "touch *.yp"),
1104             ("make", "make"),
1105             ("test", "make test"),
1106             ("install", "make install"),
1107             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1108             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1109             ("clean", "make clean"),
1110         ],
1111     },
1112
1113     # these are useful for debugging autobuild
1114     "pass": {
1115         "sequence": [
1116             ("pass", 'echo passing && /bin/true'),
1117         ],
1118     },
1119     "fail": {
1120         "sequence": [
1121             ("fail", 'echo failing && /bin/false'),
1122         ],
1123     },
1124 }
1125
1126 defaulttasks = list(tasks.keys())
1127
1128 defaulttasks.remove("pass")
1129 defaulttasks.remove("fail")
1130
1131 # The build tasks will be brought in by the test tasks as needed
1132 defaulttasks.remove("samba-def-build")
1133 defaulttasks.remove("samba-nt4-build")
1134 defaulttasks.remove("samba-mit-build")
1135 defaulttasks.remove("samba-h5l-build")
1136 defaulttasks.remove("samba-no-opath-build")
1137
1138 # This is not a normal test, but a task to support manually running
1139 # one test under autobuild
1140 defaulttasks.remove("samba-test-only")
1141
1142 # Only built on GitLab CI and not in the default autobuild because it
1143 # uses too much space (4GB of semi-static binaries)
1144 defaulttasks.remove("samba-fuzz")
1145
1146 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1147 # container where a simulated FIPS mode is possible.
1148 defaulttasks.remove("samba-fips")
1149
1150 # The MIT build runs on a current Fedora where an up to date MIT KDC
1151 # is already packaged.  This avoids needing to backport a current MIT
1152 # to the default Ubuntu 18.04, particularly during development, and
1153 # the need to install on the shared sn-devel-184.
1154
1155 defaulttasks.remove("samba-mitkrb5")
1156 defaulttasks.remove("samba-admem-mit")
1157 defaulttasks.remove("samba-addc-mit-1")
1158 defaulttasks.remove("samba-addc-mit-4a")
1159 defaulttasks.remove("samba-addc-mit-4b")
1160
1161 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1162     defaulttasks.remove("samba-o3")
1163
1164
1165 def do_print(msg):
1166     print("%s" % msg)
1167     sys.stdout.flush()
1168     sys.stderr.flush()
1169
1170
1171 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1172     if show is None:
1173         show = options.verbose
1174     if show:
1175         do_print("Running: '%s' in '%s'" % (cmd, dir))
1176     if output:
1177         out = check_output([cmd], shell=True, cwd=dir)
1178         return out.decode(encoding='utf-8', errors='backslashreplace')
1179     elif checkfail:
1180         return check_call(cmd, shell=True, cwd=dir)
1181     else:
1182         return call(cmd, shell=True, cwd=dir)
1183
1184 def rmdir_force(dirname, re_raise=True):
1185     try:
1186         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1187                 dirname, dirname, dirname), output=True, show=True)
1188     except CalledProcessError as e:
1189         do_print("Failed: '%s'" % (str(e)))
1190         run_cmd("tree %s" % dirname, output=True, show=True)
1191         if re_raise:
1192             raise
1193         return False
1194     return True
1195
1196 class builder(object):
1197     '''handle build of one directory'''
1198
1199     def __init__(self, name, definition):
1200         self.name = name
1201         self.dir = builddirs.get(name, '.')
1202         self.tag = self.name.replace('/', '_')
1203         self.definition = definition
1204         self.sequence = definition["sequence"]
1205         self.git_clone_required = False
1206         if "git-clone-required" in definition:
1207             self.git_clone_required = bool(definition["git-clone-required"])
1208         self.proc = None
1209         self.done = False
1210         self.next = 0
1211         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1212         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1213         if options.verbose:
1214             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1215             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1216         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1217         self.stdout = open(self.stdout_path, 'w')
1218         self.stderr = open(self.stderr_path, 'w')
1219         self.stdin  = open("/dev/null", 'r')
1220         self.builder_dir = "%s/%s" % (testbase, self.tag)
1221         self.test_source_dir = self.builder_dir
1222         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1223         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1224         self.prefix = "%s/%s" % (test_prefix, self.tag)
1225         self.consumers = []
1226         self.producer = None
1227
1228         if self.git_clone_required:
1229             assert "dependency" not in definition
1230
1231     def mark_existing(self):
1232         do_print('%s: Mark as existing dependency' % self.name)
1233         self.next = len(self.sequence)
1234         self.done = True
1235
1236     def add_consumer(self, consumer):
1237         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1238         consumer.producer = self
1239         consumer.test_source_dir = self.test_source_dir
1240         self.consumers.append(consumer)
1241
1242     def start_next(self):
1243         if self.producer is not None:
1244             if not self.producer.done:
1245                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1246                 return
1247
1248         if self.next == 0:
1249             rmdir_force(self.builder_dir)
1250             rmdir_force(self.prefix)
1251             if self.producer is not None:
1252                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1253             elif not self.git_clone_required:
1254                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1255             else:
1256                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1257
1258         if self.next == len(self.sequence):
1259             if not self.done:
1260                 do_print('%s: Completed OK' % self.name)
1261                 self.done = True
1262             if not options.nocleanup and len(self.consumers) == 0:
1263                 do_print('%s: Cleaning up' % self.name)
1264                 rmdir_force(self.builder_dir)
1265                 rmdir_force(self.prefix)
1266             for consumer in self.consumers:
1267                 if consumer.next != 0:
1268                     continue
1269                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1270                 consumer.start_next()
1271             if self.producer is not None:
1272                 self.producer.consumers.remove(self)
1273                 assert self.producer.done
1274                 self.producer.start_next()
1275             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1276             return
1277         (self.stage, self.cmd) = self.sequence[self.next]
1278         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
1279         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1280         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1281         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1282         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1283         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1284         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1285         self.cmd = self.cmd.replace("${NAME}", self.name)
1286         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1287         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1288         self.proc = Popen(self.cmd, shell=True,
1289                           close_fds=True, cwd=self.cwd,
1290                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1291         self.next += 1
1292
1293 def expand_dependencies(n):
1294     deps = list()
1295     if "dependency" in tasks[n]:
1296         depname = tasks[n]["dependency"]
1297         assert depname in tasks
1298         sdeps = expand_dependencies(depname)
1299         assert n not in sdeps
1300         for sdep in sdeps:
1301             deps.append(sdep)
1302         deps.append(depname)
1303     return deps
1304
1305
1306 class buildlist(object):
1307     '''handle build of multiple directories'''
1308
1309     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1310         self.tail_proc = None
1311         self.retry = None
1312         if not tasknames:
1313             if options.restrict_tests:
1314                 tasknames = ["samba-test-only"]
1315             else:
1316                 tasknames = defaulttasks
1317
1318         given_tasknames = tasknames.copy()
1319         implicit_tasknames = []
1320         for n in given_tasknames:
1321             deps = expand_dependencies(n)
1322             for dep in deps:
1323                 if dep in given_tasknames:
1324                     continue
1325                 if dep in implicit_tasknames:
1326                     continue
1327                 implicit_tasknames.append(dep)
1328
1329         tasknames = implicit_tasknames.copy()
1330         tasknames.extend(given_tasknames)
1331         do_print("given_tasknames: %s" % given_tasknames)
1332         do_print("implicit_tasknames: %s" % implicit_tasknames)
1333         do_print("tasknames: %s" % tasknames)
1334         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1335
1336         if options.retry:
1337             rebase_remote = "rebaseon"
1338             retry_task = {
1339                     "git-clone-required": True,
1340                     "sequence": [
1341                             ("retry",
1342                             '''set -e
1343                             git remote add -t %s %s %s
1344                             git fetch %s
1345                             while :; do
1346                               sleep 60
1347                               git describe %s/%s > old_remote_branch.desc
1348                               git fetch %s
1349                               git describe %s/%s > remote_branch.desc
1350                               diff old_remote_branch.desc remote_branch.desc
1351                             done
1352                            ''' % (
1353                                rebase_branch, rebase_remote, rebase_url,
1354                                rebase_remote,
1355                                rebase_remote, rebase_branch,
1356                                rebase_remote,
1357                                rebase_remote, rebase_branch
1358                             ))]}
1359
1360             self.retry = builder('retry', retry_task)
1361             self.need_retry = False
1362
1363         if options.skip_dependencies:
1364             for b in self.tlist:
1365                 if b.name in implicit_tasknames:
1366                     b.mark_existing()
1367
1368         for b in self.tlist:
1369             do_print("b.name=%s" % b.name)
1370             if "dependency" not in b.definition:
1371                 continue
1372             depname = b.definition["dependency"]
1373             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1374             for p in self.tlist:
1375                 if p.name == depname:
1376                     p.add_consumer(b)
1377
1378     def kill_kids(self):
1379         if self.tail_proc is not None:
1380             self.tail_proc.terminate()
1381             self.tail_proc.wait()
1382             self.tail_proc = None
1383         if self.retry is not None:
1384             self.retry.proc.terminate()
1385             self.retry.proc.wait()
1386             self.retry = None
1387         for b in self.tlist:
1388             if b.proc is not None:
1389                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1390                 b.proc.terminate()
1391                 b.proc.wait()
1392                 b.proc = None
1393
1394     def wait_one(self):
1395         while True:
1396             none_running = True
1397             for b in self.tlist:
1398                 if b.proc is None:
1399                     continue
1400                 none_running = False
1401                 b.status = b.proc.poll()
1402                 if b.status is None:
1403                     continue
1404                 b.proc = None
1405                 return b
1406             if options.retry:
1407                 ret = self.retry.proc.poll()
1408                 if ret is not None:
1409                     self.need_retry = True
1410                     self.retry = None
1411                     return None
1412             if none_running:
1413                 return None
1414             time.sleep(0.1)
1415
1416     def run(self):
1417         for b in self.tlist:
1418             b.start_next()
1419         if options.retry:
1420             self.retry.start_next()
1421         while True:
1422             b = self.wait_one()
1423             if options.retry and self.need_retry:
1424                 self.kill_kids()
1425                 do_print("retry needed")
1426                 return (0, None, None, None, "retry")
1427             if b is None:
1428                 break
1429             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1430                 self.kill_kids()
1431                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1432             b.start_next()
1433         self.kill_kids()
1434         return (0, None, None, None, "All OK")
1435
1436     def write_system_info(self, filename):
1437         with open(filename, 'w') as f:
1438             for cmd in ['uname -a',
1439                         'lsb_release -a',
1440                         'free',
1441                         'mount',
1442                         'cat /proc/cpuinfo',
1443                         'cc --version',
1444                         'df -m .',
1445                         'df -m %s' % testbase]:
1446                 try:
1447                     out = run_cmd(cmd, output=True, checkfail=False)
1448                 except CalledProcessError as e:
1449                     out = "<failed: %s>" % str(e)
1450                 print('### %s' % cmd, file=f)
1451                 print(out, file=f)
1452                 print(file=f)
1453
1454     def tarlogs(self, fname):
1455         with tarfile.open(fname, "w:gz") as tar:
1456             for b in self.tlist:
1457                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1458                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1459             if os.path.exists("autobuild.log"):
1460                 tar.add("autobuild.log")
1461             filename = 'system-info.txt'
1462             self.write_system_info(filename)
1463             tar.add(filename)
1464
1465     def remove_logs(self):
1466         for b in self.tlist:
1467             os.unlink(b.stdout_path)
1468             os.unlink(b.stderr_path)
1469
1470     def start_tail(self):
1471         cmd = ["tail", "-f"]
1472         for b in self.tlist:
1473             cmd.append(b.stdout_path)
1474             cmd.append(b.stderr_path)
1475         self.tail_proc = Popen(cmd, close_fds=True)
1476
1477
1478 def cleanup(do_raise=False):
1479     if options.nocleanup:
1480         return
1481     run_cmd("stat %s || true" % test_tmpdir, show=True)
1482     run_cmd("stat %s" % testbase, show=True)
1483     do_print("Cleaning up %r" % cleanup_list)
1484     for d in cleanup_list:
1485         ok = rmdir_force(d, re_raise=False)
1486         if ok:
1487             continue
1488         if os.path.isdir(d):
1489             do_print("Killing, waiting and retry")
1490             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1491         else:
1492             do_print("Waiting and retry")
1493         time.sleep(1)
1494         rmdir_force(d, re_raise=do_raise)
1495
1496
1497 def daemonize(logfile):
1498     pid = os.fork()
1499     if pid == 0:  # Parent
1500         os.setsid()
1501         pid = os.fork()
1502         if pid != 0:  # Actual daemon
1503             os._exit(0)
1504     else:  # Grandparent
1505         os._exit(0)
1506
1507     import resource      # Resource usage information.
1508     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1509     if maxfd == resource.RLIM_INFINITY:
1510         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1511     for fd in range(0, maxfd):
1512         try:
1513             os.close(fd)
1514         except OSError:
1515             pass
1516     os.open(logfile, os.O_RDWR | os.O_CREAT)
1517     os.dup2(0, 1)
1518     os.dup2(0, 2)
1519
1520
1521 def write_pidfile(fname):
1522     '''write a pid file, cleanup on exit'''
1523     with open(fname, mode='w') as f:
1524         f.write("%u\n" % os.getpid())
1525
1526
1527 def rebase_tree(rebase_url, rebase_branch="master"):
1528     rebase_remote = "rebaseon"
1529     do_print("Rebasing on %s" % rebase_url)
1530     run_cmd("git describe HEAD", show=True, dir=test_master)
1531     run_cmd("git remote add -t %s %s %s" %
1532             (rebase_branch, rebase_remote, rebase_url),
1533             show=True, dir=test_master)
1534     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1535     if options.fix_whitespace:
1536         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1537                 (rebase_remote, rebase_branch),
1538                 show=True, dir=test_master)
1539     else:
1540         run_cmd("git rebase --force-rebase %s/%s" %
1541                 (rebase_remote, rebase_branch),
1542                 show=True, dir=test_master)
1543     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1544                    (rebase_remote, rebase_branch),
1545                    dir=test_master, output=True)
1546     if diff == '':
1547         do_print("No differences between HEAD and %s/%s - exiting" %
1548                  (rebase_remote, rebase_branch))
1549         sys.exit(0)
1550     run_cmd("git describe %s/%s" %
1551             (rebase_remote, rebase_branch),
1552             show=True, dir=test_master)
1553     run_cmd("git describe HEAD", show=True, dir=test_master)
1554     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1555             (rebase_remote, rebase_branch),
1556             show=True, dir=test_master)
1557
1558
1559 def push_to(push_url, push_branch="master"):
1560     push_remote = "pushto"
1561     do_print("Pushing to %s" % push_url)
1562     if options.mark:
1563         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1564         run_cmd("git commit --amend -c HEAD", dir=test_master)
1565         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1566         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1567     run_cmd("git remote add -t %s %s %s" %
1568             (push_branch, push_remote, push_url),
1569             show=True, dir=test_master)
1570     run_cmd("git push %s +HEAD:%s" %
1571             (push_remote, push_branch),
1572             show=True, dir=test_master)
1573
1574
1575 def send_email(subject, text, log_tar):
1576     if options.email is None:
1577         do_print("not sending email because the recipient is not set")
1578         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1579                  (subject, text))
1580         return
1581     outer = MIMEMultipart()
1582     outer['Subject'] = subject
1583     outer['To'] = options.email
1584     outer['From'] = options.email_from
1585     outer['Date'] = email.utils.formatdate(localtime=True)
1586     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1587     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1588     if options.attach_logs:
1589         with open(log_tar, 'rb') as fp:
1590             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1591         # Set the filename parameter
1592         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1593         outer.attach(msg)
1594     content = outer.as_string()
1595     s = smtplib.SMTP(options.email_server)
1596     email_user = os.getenv('SMTP_USERNAME')
1597     email_password = os.getenv('SMTP_PASSWORD')
1598     if email_user is not None:
1599         s.starttls()
1600         s.login(email_user, email_password)
1601
1602     s.sendmail(options.email_from, [options.email], content)
1603     s.set_debuglevel(1)
1604     s.quit()
1605
1606
1607 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1608                   elapsed_time, log_base=None, add_log_tail=True):
1609     '''send an email to options.email about the failure'''
1610     elapsed_minutes = elapsed_time / 60.0
1611     if log_base is None:
1612         log_base = gitroot
1613     text = '''
1614 Dear Developer,
1615
1616 Your autobuild on %s failed after %.1f minutes
1617 when trying to test %s with the following error:
1618
1619    %s
1620
1621 the autobuild has been abandoned. Please fix the error and resubmit.
1622
1623 A summary of the autobuild process is here:
1624
1625   %s/autobuild.log
1626 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1627
1628     if options.restrict_tests:
1629         text += """
1630 The build was restricted to tests matching %s\n""" % options.restrict_tests
1631
1632     if failed_task != 'rebase':
1633         text += '''
1634 You can see logs of the failed task here:
1635
1636   %s/%s.stdout
1637   %s/%s.stderr
1638
1639 or you can get full logs of all tasks in this job here:
1640
1641   %s/logs.tar.gz
1642
1643 The top commit for the tree that was built was:
1644
1645 %s
1646
1647 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1648
1649     if add_log_tail:
1650         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1651         lines = f.readlines()
1652         log_tail = "".join(lines[-50:])
1653         num_lines = len(lines)
1654         if num_lines < 50:
1655             # Also include stderr (compile failures) if < 50 lines of stdout
1656             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1657             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1658
1659         text += '''
1660 The last 50 lines of log messages:
1661
1662 %s
1663     ''' % log_tail
1664         f.close()
1665
1666     logs = os.path.join(gitroot, 'logs.tar.gz')
1667     send_email('autobuild[%s] failure on %s for task %s during %s'
1668                % (options.branch, platform.node(), failed_task, failed_stage),
1669                text, logs)
1670
1671
1672 def email_success(elapsed_time, log_base=None):
1673     '''send an email to options.email about a successful build'''
1674     if log_base is None:
1675         log_base = gitroot
1676     text = '''
1677 Dear Developer,
1678
1679 Your autobuild on %s has succeeded after %.1f minutes.
1680
1681 ''' % (platform.node(), elapsed_time / 60.)
1682
1683     if options.restrict_tests:
1684         text += """
1685 The build was restricted to tests matching %s\n""" % options.restrict_tests
1686
1687     if options.keeplogs:
1688         text += '''
1689
1690 you can get full logs of all tasks in this job here:
1691
1692   %s/logs.tar.gz
1693
1694 ''' % log_base
1695
1696     text += '''
1697 The top commit for the tree that was built was:
1698
1699 %s
1700 ''' % top_commit_msg
1701
1702     logs = os.path.join(gitroot, 'logs.tar.gz')
1703     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1704                text, logs)
1705
1706
1707 # get the top commit message, for emails
1708 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1709
1710 try:
1711     if options.skip_dependencies:
1712         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1713     else:
1714         os.makedirs(testbase)
1715 except Exception as reason:
1716     raise Exception("Unable to create %s : %s" % (testbase, reason))
1717 cleanup_list.append(testbase)
1718
1719 if options.daemon:
1720     logfile = os.path.join(testbase, "log")
1721     do_print("Forking into the background, writing progress to %s" % logfile)
1722     daemonize(logfile)
1723
1724 write_pidfile(gitroot + "/autobuild.pid")
1725
1726 start_time = time.time()
1727
1728 while True:
1729     try:
1730         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1731         os.makedirs(test_tmpdir)
1732         # The waf uninstall code removes empty directories all the way
1733         # up the tree.  Creating a file in test_tmpdir stops it from
1734         # being removed.
1735         run_cmd("touch %s" % os.path.join(test_tmpdir,
1736                                           ".directory-is-not-empty"), show=True)
1737         run_cmd("stat %s" % test_tmpdir, show=True)
1738         run_cmd("stat %s" % testbase, show=True)
1739         if options.skip_dependencies:
1740             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1741         else:
1742             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1743     except Exception:
1744         cleanup()
1745         raise
1746
1747     try:
1748         if options.rebase is not None:
1749             rebase_tree(options.rebase, rebase_branch=options.branch)
1750     except Exception:
1751         cleanup_list.append(gitroot + "/autobuild.pid")
1752         cleanup()
1753         elapsed_time = time.time() - start_time
1754         email_failure(-1, 'rebase', 'rebase', 'rebase',
1755                       'rebase on %s failed' % options.branch,
1756                       elapsed_time, log_base=options.log_base)
1757         sys.exit(1)
1758
1759     try:
1760         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1761         if options.tail:
1762             blist.start_tail()
1763         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1764         if status != 0 or errstr != "retry":
1765             break
1766         cleanup(do_raise=True)
1767     except Exception:
1768         cleanup()
1769         raise
1770
1771 cleanup_list.append(gitroot + "/autobuild.pid")
1772
1773 do_print(errstr)
1774
1775 blist.kill_kids()
1776 if options.tail:
1777     do_print("waiting for tail to flush")
1778     time.sleep(1)
1779
1780 elapsed_time = time.time() - start_time
1781 if status == 0:
1782     if options.passcmd is not None:
1783         do_print("Running passcmd: %s" % options.passcmd)
1784         run_cmd(options.passcmd, dir=test_master)
1785     if options.pushto is not None:
1786         push_to(options.pushto, push_branch=options.branch)
1787     if options.keeplogs or options.attach_logs:
1788         blist.tarlogs("logs.tar.gz")
1789         do_print("Logs in logs.tar.gz")
1790     if options.always_email:
1791         email_success(elapsed_time, log_base=options.log_base)
1792     blist.remove_logs()
1793     cleanup()
1794     do_print(errstr)
1795     sys.exit(0)
1796
1797 # something failed, gather a tar of the logs
1798 blist.tarlogs("logs.tar.gz")
1799
1800 if options.email is not None:
1801     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1802                   elapsed_time, log_base=options.log_base)
1803 else:
1804     elapsed_minutes = elapsed_time / 60.0
1805     print('''
1806
1807 ####################################################################
1808
1809 AUTOBUILD FAILURE
1810
1811 Your autobuild[%s] on %s failed after %.1f minutes
1812 when trying to test %s with the following error:
1813
1814    %s
1815
1816 the autobuild has been abandoned. Please fix the error and resubmit.
1817
1818 ####################################################################
1819
1820 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1821
1822 cleanup()
1823 do_print(errstr)
1824 do_print("Logs in logs.tar.gz")
1825 sys.exit(status)