script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34
  35 def find_git_root():
  36     '''get to the top of the git repo'''
  37     p = os.getcwd()
  38     while p != '/':
  39         if os.path.exists(os.path.join(p, ".git")):
  40             return p
  41         p = os.path.abspath(os.path.join(p, '..'))
  42     return None
  43
  44
  45 gitroot = find_git_root()
  46 if gitroot is None:
  47     raise Exception("Failed to find git root")
  48
  49
  50 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  51
  52 parser = OptionParser()
  53 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  54 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  55 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  56 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  60                   default=None)
  61 parser.add_option("--passcmd", help="command to run on success", default=None)
  62 parser.add_option("--verbose", help="show all commands as they are run",
  63                   default=False, action="store_true")
  64 parser.add_option("--rebase", help="rebase on the given tree before testing",
  65                   default=None, type='str')
  66 parser.add_option("--pushto", help="push to a git url on success",
  67                   default=None, type='str')
  68 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  69                   default=False, action="store_true")
  70 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  71                   default=False, action="store_true")
  72 parser.add_option("--retry", help="automatically retry if master changes",
  73                   default=False, action="store_true")
  74 parser.add_option("--email", help="send email to the given address on failure",
  75                   type='str', default=None)
  76 parser.add_option("--email-from", help="send email from the given address",
  77                   type='str', default="autobuild@samba.org")
  78 parser.add_option("--email-server", help="send email via the given server",
  79                   type='str', default='localhost')
  80 parser.add_option("--always-email", help="always send email, even on success",
  81                   action="store_true")
  82 parser.add_option("--daemon", help="daemonize after initial setup",
  83                   action="store_true")
  84 parser.add_option("--branch", help="the branch to work on (default=master)",
  85                   default="master", type='str')
  86 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  87                   default=gitroot, type='str')
  88 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  89                   default=False, action="store_true")
  90 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  91                   default='')
  92 parser.add_option("--enable-coverage", dest='enable_coverage',
  93                   action="store_const", const='--enable-coverage', default='',
  94                   help="Add --enable-coverage option while configure")
  95
  96 (options, args) = parser.parse_args()
  97
  98 if options.retry:
  99     if options.rebase is None:
 100         raise Exception('You can only use --retry if you also rebase')
 101
 102 if options.full_testbase is not None:
 103     testbase = options.full_testbase
 104 else:
 105     testbase = "%s/b%u" % (options.testbase, os.getpid())
 106 test_master = "%s/master" % testbase
 107 test_prefix = "%s/prefix" % testbase
 108 test_tmpdir = "%s/tmp" % testbase
 109 os.environ['TMPDIR'] = test_tmpdir
 110
 111 if options.enable_coverage:
 112     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 113 else:
 114     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 115
 116 if options.enable_coverage:
 117     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 118 else:
 119     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 120
 121 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 122
 123 if args:
 124     # If we are only running specific test,
 125     # do not sleep randomly to wait for it to start
 126     def random_sleep(low, high):
 127         return 'sleep 1'
 128 else:
 129     def random_sleep(low, high):
 130         return 'sleep {}'.format(random.randint(low, high))
 131
 132 cleanup_list = []
 133
 134 builddirs = {
 135     "ctdb": "ctdb",
 136     "ldb": "lib/ldb",
 137     "tdb": "lib/tdb",
 138     "talloc": "lib/talloc",
 139     "replace": "lib/replace",
 140     "tevent": "lib/tevent",
 141     "pidl": "pidl",
 142     "docs-xml": "docs-xml"
 143 }
 144
 145 ctdb_configure_params = " --enable-developer ${PREFIX}"
 146 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 147
 148 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 149 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 150 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 151 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 152 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 153 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 154 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 155
 156
 157 def format_option(name, value=None):
 158     """Format option as str list."""
 159     if value is None:  # boolean option
 160         return [name]
 161     if not isinstance(value, list):  # single value option
 162         value = [value]
 163     # repeatable option
 164     return ['{}={}'.format(name, item) for item in value]
 165
 166
 167 def make_test(
 168         cmd='make testonly',
 169         FAIL_IMMEDIATELY=1,
 170         INJECT_SELFTEST_PREFIX=1,
 171         TESTS='',
 172         include_envs=None,
 173         exclude_envs=None):
 174
 175     test_options = []
 176     if include_envs:
 177         test_options = format_option('--include-env', include_envs)
 178     if exclude_envs:
 179         test_options = format_option('--exclude-env', exclude_envs)
 180     if test_options:
 181         # join envs options to original test options
 182         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 183
 184     _options = []
 185     if FAIL_IMMEDIATELY:
 186         _options.append('FAIL_IMMEDIATELY=1')
 187     if TESTS:
 188         _options.append("TESTS='{}'".format(TESTS))
 189
 190     if INJECT_SELFTEST_PREFIX:
 191         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 192         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 193
 194     return ' '.join([cmd] + _options)
 195
 196
 197 # When updating this list, also update .gitlab-ci.yml to add the job
 198 # and to make it a dependency of 'page' for the coverage report.
 199
 200 tasks = {
 201     "ctdb": {
 202         "sequence": [
 203             ("random-sleep", random_sleep(300, 900)),
 204             ("configure", "./configure " + ctdb_configure_params),
 205             ("make", "make all"),
 206             ("install", "make install"),
 207             ("test", "make autotest"),
 208             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 209             ("clean", "make clean"),
 210         ],
 211     },
 212     "docs-xml": {
 213         "sequence": [
 214             ("random-sleep", random_sleep(300, 900)),
 215             ("autoconf", "autoconf"),
 216             ("configure", "./configure"),
 217             ("make", "make html htmlman"),
 218             ("publish-docs", PUBLISH_DOCS),
 219             ("clean", "make clean"),
 220         ],
 221     },
 222
 223     "samba-def-build": {
 224         "git-clone-required": True,
 225         "sequence": [
 226             ("configure", "./configure.developer" + samba_configure_params),
 227             ("make", "make -j"),
 228             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 229             ("chmod-R-a-w", "chmod -R a-w ."),
 230         ],
 231     },
 232
 233     "samba-mit-build": {
 234         "git-clone-required": True,
 235         "sequence": [
 236             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 237             ("make", "make -j"),
 238             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 239             ("chmod-R-a-w", "chmod -R a-w ."),
 240         ],
 241     },
 242
 243     "samba-nt4-build": {
 244         "git-clone-required": True,
 245         "sequence": [
 246             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 247             ("make", "make -j"),
 248             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 249             ("chmod-R-a-w", "chmod -R a-w ."),
 250         ],
 251     },
 252
 253     "samba-h5l-build": {
 254         "git-clone-required": True,
 255         "sequence": [
 256             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 257             ("make", "make -j"),
 258             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 259             ("chmod-R-a-w", "chmod -R a-w ."),
 260         ],
 261     },
 262
 263     "samba-no-opath-build": {
 264         "git-clone-required": True,
 265         "sequence": [
 266             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 267             ("make", "make -j"),
 268             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 269             ("chmod-R-a-w", "chmod -R a-w ."),
 270         ],
 271     },
 272
 273     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 274     "samba": {
 275         "sequence": [
 276             ("random-sleep", random_sleep(300, 900)),
 277             ("configure", "./configure.developer" + samba_configure_params),
 278             ("make", "make -j"),
 279             ("test", make_test(exclude_envs=[
 280             "none",
 281             "nt4_dc",
 282             "nt4_dc_smb1",
 283             "nt4_dc_smb1_done",
 284             "nt4_dc_schannel",
 285             "nt4_member",
 286             "ad_dc",
 287             "ad_dc_smb1",
 288             "ad_dc_smb1_done",
 289             "ad_dc_backup",
 290             "ad_dc_ntvfs",
 291             "ad_dc_default",
 292             "ad_dc_default_smb1",
 293             "ad_dc_slowtests",
 294             "ad_dc_no_nss",
 295             "ad_dc_no_ntlm",
 296             "fl2003dc",
 297             "fl2008dc",
 298             "fl2008r2dc",
 299             "ad_member",
 300             "ad_member_idmap_rid",
 301             "ad_member_idmap_ad",
 302             "ad_member_rfc2307",
 303             "ad_member_oneway",
 304             "chgdcpass",
 305             "vampire_2000_dc",
 306             "fl2000dc",
 307             "fileserver",
 308             "fileserver_smb1",
 309             "fileserver_smb1_done",
 310             "maptoguest",
 311             "simpleserver",
 312             "backupfromdc",
 313             "restoredc",
 314             "renamedc",
 315             "offlinebackupdc",
 316             "labdc",
 317             "preforkrestartdc",
 318             "proclimitdc",
 319             "promoted_dc",
 320             "vampire_dc",
 321             "rodc",
 322             "ad_dc_default",
 323             "ad_dc_default_smb1",
 324             "ad_dc_default_smb1_done",
 325             "ad_dc_slowtests",
 326             "schema_pair_dc",
 327             "schema_dc",
 328             "clusteredmember",
 329             "ad_dc_fips",
 330             "ad_member_fips",
 331             ])),
 332             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 333             ("lcov", LCOV_CMD),
 334             ("install", "make install"),
 335             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 336             ("clean", "make clean"),
 337         ],
 338     },
 339
 340     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 341     "samba-mitkrb5": {
 342         "sequence": [
 343             ("random-sleep", random_sleep(300, 900)),
 344             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 345             ("make", "make -j"),
 346             ("test", make_test(exclude_envs=[
 347             "none",
 348             "nt4_dc",
 349             "nt4_dc_smb1",
 350             "nt4_dc_smb1_done",
 351             "nt4_dc_schannel",
 352             "nt4_member",
 353             "ad_dc",
 354             "ad_dc_smb1",
 355             "ad_dc_smb1_done",
 356             "ad_dc_backup",
 357             "ad_dc_ntvfs",
 358             "ad_dc_default",
 359             "ad_dc_default_smb1",
 360             "ad_dc_default_smb1_done",
 361             "ad_dc_slowtests",
 362             "ad_dc_no_nss",
 363             "ad_dc_no_ntlm",
 364             "fl2003dc",
 365             "fl2008dc",
 366             "fl2008r2dc",
 367             "ad_member",
 368             "ad_member_idmap_rid",
 369             "ad_member_idmap_ad",
 370             "ad_member_rfc2307",
 371             "ad_member_oneway",
 372             "chgdcpass",
 373             "vampire_2000_dc",
 374             "fl2000dc",
 375             "fileserver",
 376             "fileserver_smb1",
 377             "fileserver_smb1_done",
 378             "maptoguest",
 379             "simpleserver",
 380             "backupfromdc",
 381             "restoredc",
 382             "renamedc",
 383             "offlinebackupdc",
 384             "labdc",
 385             "preforkrestartdc",
 386             "proclimitdc",
 387             "promoted_dc",
 388             "vampire_dc",
 389             "rodc",
 390             "ad_dc_default",
 391             "ad_dc_default_smb1",
 392             "ad_dc_default_smb1_done",
 393             "ad_dc_slowtests",
 394             "schema_pair_dc",
 395             "schema_dc",
 396             "clusteredmember",
 397             "ad_dc_fips",
 398             "ad_member_fips",
 399             ])),
 400             ("lcov", LCOV_CMD),
 401             ("install", "make install"),
 402             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 403             ("clean", "make clean"),
 404         ],
 405     },
 406
 407     "samba-nt4": {
 408         "dependency": "samba-nt4-build",
 409         "sequence": [
 410             ("random-sleep", random_sleep(300, 900)),
 411             ("test", make_test(include_envs=[
 412             "nt4_dc",
 413             "nt4_dc_smb1",
 414             "nt4_dc_smb1_done",
 415             "nt4_dc_schannel",
 416             "nt4_member",
 417             "simpleserver",
 418             ])),
 419             ("lcov", LCOV_CMD),
 420             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 421         ],
 422     },
 423
 424     "samba-fileserver": {
 425         "dependency": "samba-h5l-build",
 426         "sequence": [
 427             ("random-sleep", random_sleep(300, 900)),
 428             ("test", make_test(include_envs=[
 429             "fileserver",
 430             "fileserver_smb1",
 431             "fileserver_smb1_done",
 432             "maptoguest",
 433             "ktest", # ktest is also tested in samba and samba-mitkrb5
 434                      # but is tested here against a system Heimdal
 435             ])),
 436             ("lcov", LCOV_CMD),
 437             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 438         ],
 439     },
 440
 441     "samba-admem": {
 442         "dependency": "samba-def-build",
 443         "sequence": [
 444             ("random-sleep", random_sleep(300, 900)),
 445             ("test", make_test(include_envs=[
 446             "ad_member",
 447             "ad_member_idmap_rid",
 448             "ad_member_idmap_ad",
 449             "ad_member_rfc2307",
 450             "ad_member_offlogon",
 451             ])),
 452             ("lcov", LCOV_CMD),
 453             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 454         ],
 455     },
 456
 457     "samba-no-opath1": {
 458         "dependency": "samba-no-opath-build",
 459         "sequence": [
 460             ("random-sleep", random_sleep(300, 900)),
 461             ("test", make_test(
 462                 cmd="make testonly DISABLE_OPATH=1",
 463                 include_envs=[
 464                 "nt4_dc",
 465                 "nt4_dc_smb1",
 466                 "nt4_dc_smb1_done",
 467                 "nt4_dc_schannel",
 468                 "nt4_member",
 469                 "simpleserver",
 470                 ])),
 471             ("lcov", LCOV_CMD),
 472             ("check-clean-tree", "script/clean-source-tree.sh"),
 473         ],
 474     },
 475
 476     "samba-no-opath2": {
 477         "dependency": "samba-no-opath-build",
 478         "sequence": [
 479             ("random-sleep", random_sleep(300, 900)),
 480             ("test", make_test(
 481                 cmd="make testonly DISABLE_OPATH=1",
 482                 include_envs=[
 483                 "fileserver",
 484                 "fileserver_smb1",
 485                 "fileserver_smb1_done",
 486                 ])),
 487             ("lcov", LCOV_CMD),
 488             ("check-clean-tree", "script/clean-source-tree.sh"),
 489         ],
 490     },
 491
 492     "samba-ad-dc-1": {
 493         "dependency": "samba-def-build",
 494         "sequence": [
 495             ("random-sleep", random_sleep(1, 1)),
 496             ("test", make_test(include_envs=[
 497             "ad_dc",
 498             "ad_dc_smb1",
 499             "ad_dc_smb1_done",
 500             "ad_dc_no_nss",
 501             "ad_dc_no_ntlm",
 502             ])),
 503             ("lcov", LCOV_CMD),
 504             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 505         ],
 506     },
 507
 508     "samba-ad-dc-2": {
 509         "dependency": "samba-def-build",
 510         "sequence": [
 511             ("random-sleep", random_sleep(1, 1)),
 512             ("test", make_test(include_envs=[
 513             "vampire_dc",
 514             "vampire_2000_dc",
 515             "rodc",
 516             ])),
 517             ("lcov", LCOV_CMD),
 518             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 519         ],
 520     },
 521
 522     "samba-ad-dc-3": {
 523         "dependency": "samba-def-build",
 524         "sequence": [
 525             ("random-sleep", random_sleep(1, 1)),
 526             ("test", make_test(include_envs=[
 527             "promoted_dc",
 528             "chgdcpass",
 529             "preforkrestartdc",
 530             "proclimitdc",
 531             ])),
 532             ("lcov", LCOV_CMD),
 533             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 534         ],
 535     },
 536
 537     "samba-ad-dc-4a": {
 538         "dependency": "samba-def-build",
 539         "sequence": [
 540             ("random-sleep", random_sleep(1, 1)),
 541             ("test", make_test(include_envs=[
 542             "fl2000dc",
 543             "ad_member_oneway",
 544             "fl2003dc",
 545             ])),
 546             ("lcov", LCOV_CMD),
 547             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 548         ],
 549     },
 550     "samba-ad-dc-4b": {
 551         "dependency": "samba-def-build",
 552         "sequence": [
 553             ("random-sleep", random_sleep(1, 1)),
 554             ("test", make_test(include_envs=[
 555             "fl2008dc",
 556             "fl2008r2dc",
 557             ])),
 558             ("lcov", LCOV_CMD),
 559             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 560         ],
 561     },
 562
 563     "samba-ad-dc-5": {
 564         "dependency": "samba-def-build",
 565         "sequence": [
 566             ("random-sleep", random_sleep(1, 1)),
 567             ("test", make_test(include_envs=[
 568             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 569             ("lcov", LCOV_CMD),
 570             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 571         ],
 572     },
 573
 574     "samba-ad-dc-6": {
 575         "dependency": "samba-def-build",
 576         "sequence": [
 577             ("random-sleep", random_sleep(1, 1)),
 578             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 579             ("lcov", LCOV_CMD),
 580             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 581         ],
 582     },
 583
 584     "samba-schemaupgrade": {
 585         "dependency": "samba-def-build",
 586         "sequence": [
 587             ("random-sleep", random_sleep(1, 1)),
 588             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 589             ("lcov", LCOV_CMD),
 590             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 591         ],
 592     },
 593
 594     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 595     # This is currently the longest task, so we don't randomly delay it.
 596     "samba-ad-dc-ntvfs": {
 597         "dependency": "samba-def-build",
 598         "sequence": [
 599             ("random-sleep", random_sleep(1, 1)),
 600             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 601             ("lcov", LCOV_CMD),
 602             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 603         ],
 604     },
 605
 606     # Test fips compliance
 607     "samba-fips": {
 608         "dependency": "samba-mit-build",
 609         "sequence": [
 610             ("random-sleep", random_sleep(1, 1)),
 611             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 612             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 613             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 614         ],
 615     },
 616
 617     # run the backup/restore testenvs separately as they're fairly standalone
 618     # (and CI seems to max out at ~3 different DCs running at once)
 619     "samba-ad-back1": {
 620         "dependency": "samba-def-build",
 621         "sequence": [
 622             ("random-sleep", random_sleep(300, 900)),
 623             ("test", make_test(include_envs=[
 624             "backupfromdc",
 625             "restoredc",
 626             "renamedc",
 627             ])),
 628             ("lcov", LCOV_CMD),
 629             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 630         ],
 631     },
 632     "samba-ad-back2": {
 633         "dependency": "samba-def-build",
 634         "sequence": [
 635             ("random-sleep", random_sleep(300, 900)),
 636             ("test", make_test(include_envs=[
 637             "backupfromdc",
 638             "offlinebackupdc",
 639             "labdc",
 640             ])),
 641             ("lcov", LCOV_CMD),
 642             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 643         ],
 644     },
 645
 646     "samba-admem-mit": {
 647         "dependency": "samba-mit-build",
 648         "sequence": [
 649             ("random-sleep", random_sleep(1, 1)),
 650             ("test", make_test(include_envs=[
 651             "ad_member",
 652             "ad_member_idmap_rid",
 653             "ad_member_idmap_ad",
 654             "ad_member_rfc2307",
 655             "ad_member_offlogon",
 656             ])),
 657             ("lcov", LCOV_CMD),
 658             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 659         ],
 660     },
 661
 662     "samba-addc-mit-1": {
 663         "dependency": "samba-mit-build",
 664         "sequence": [
 665             ("random-sleep", random_sleep(1, 1)),
 666             ("test", make_test(include_envs=[
 667             "ad_dc",
 668             "ad_dc_smb1",
 669             "ad_dc_smb1_done",
 670             "ad_dc_no_nss",
 671             "ad_dc_no_ntlm",
 672             ])),
 673             ("lcov", LCOV_CMD),
 674             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 675         ],
 676     },
 677
 678     "samba-addc-mit-4a": {
 679         "dependency": "samba-mit-build",
 680         "sequence": [
 681             ("random-sleep", random_sleep(1, 1)),
 682             ("test", make_test(include_envs=[
 683             "fl2000dc",
 684             "ad_member_oneway",
 685             "fl2003dc",
 686             ])),
 687             ("lcov", LCOV_CMD),
 688             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 689         ],
 690     },
 691     "samba-addc-mit-4b": {
 692         "dependency": "samba-mit-build",
 693         "sequence": [
 694             ("random-sleep", random_sleep(1, 1)),
 695             ("test", make_test(include_envs=[
 696             "fl2000dc",
 697             "fl2003dc",
 698             ])),
 699             ("lcov", LCOV_CMD),
 700             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 701         ],
 702     },
 703
 704     "samba-test-only": {
 705         "sequence": [
 706             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 707             ("make", "make -j"),
 708             ("test", make_test(TESTS="${TESTS}")),
 709             ("lcov", LCOV_CMD),
 710         ],
 711     },
 712
 713     # Test cross-compile infrastructure
 714     "samba-xc": {
 715         "sequence": [
 716             ("random-sleep", random_sleep(900, 1500)),
 717             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 718             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 719             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 720             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 721             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 722             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 723             ("compare-results", "script/compare_cc_results.py "
 724             "./bin/c4che/default{} "
 725             "./bin-xe/c4che/default{} "
 726             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 727             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 728             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 729             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 730             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 731             " = \"'1234'\"".format(CACHE_SUFFIX)),
 732             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 733             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 734             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 735             " ; test $? -ne 0"),
 736         ],
 737     },
 738
 739     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 740     "samba-o3": {
 741         "sequence": [
 742             ("random-sleep", random_sleep(300, 900)),
 743             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 744             ("make", "make -j"),
 745             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 746             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 747             ("lcov", LCOV_CMD),
 748             ("install", "make install"),
 749             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 750             ("clean", "make clean"),
 751         ],
 752     },
 753
 754     "samba-ctdb": {
 755         "sequence": [
 756             ("random-sleep", random_sleep(900, 1500)),
 757
 758         # make sure we have tdb around:
 759             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 760             ("tdb-make", "cd lib/tdb && make"),
 761             ("tdb-install", "cd lib/tdb && make install"),
 762
 763         # build samba with cluster support (also building ctdb):
 764             ("samba-configure",
 765          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 766          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 767          "./configure.developer ${PREFIX} "
 768          "--with-selftest-prefix=./bin/ab "
 769          "--enable-clangdb "
 770          "--with-cluster-support "
 771          "--without-ad-dc "
 772          "--bundled-libraries=!tdb"),
 773             ("samba-make", "make"),
 774             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 775             ("samba-install", "make install"),
 776             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 777
 778             ("test", make_test(
 779                 cmd='make test',
 780                 INJECT_SELFTEST_PREFIX=0,
 781                 include_envs=["clusteredmember"])
 782             ),
 783
 784         # clean up:
 785             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 786             ("clean", "make clean"),
 787             ("ctdb-clean", "cd ./ctdb && make clean"),
 788         ],
 789     },
 790
 791     "samba-libs": {
 792         "sequence": [
 793             ("random-sleep", random_sleep(300, 900)),
 794             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 795             ("talloc-make", "cd lib/talloc && make"),
 796             ("talloc-install", "cd lib/talloc && make install"),
 797
 798             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 799             ("tdb-make", "cd lib/tdb && make"),
 800             ("tdb-install", "cd lib/tdb && make install"),
 801
 802             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 803             ("tevent-make", "cd lib/tevent && make"),
 804             ("tevent-install", "cd lib/tevent && make install"),
 805
 806             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 807             ("ldb-make", "cd lib/ldb && make"),
 808             ("ldb-install", "cd lib/ldb && make install"),
 809
 810             ("nondevel-configure", "./configure ${PREFIX}"),
 811             ("nondevel-make", "make -j"),
 812             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 813             ("nondevel-install", "make install"),
 814             ("nondevel-dist", "make dist"),
 815
 816         # retry with all modules shared
 817             ("allshared-distclean", "make distclean"),
 818             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 819             ("allshared-make", "make -j"),
 820         ],
 821     },
 822
 823     "samba-fuzz": {
 824         "sequence": [
 825         # build the fuzzers (static) via the oss-fuzz script
 826             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 827             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 828         ],
 829     },
 830
 831     # * Test smbd and smbtorture can build semi-static
 832     #
 833     # * Test Samba without python still builds.
 834     #
 835     # When this test fails due to more use of Python, the expectations
 836     # is that the newly failing part of the code should be disabled
 837     # when --disable-python is set (rather than major work being done
 838     # to support this environment).
 839     #
 840     # The target here is for vendors shipping a minimal smbd.
 841     "samba-minimal-smbd": {
 842         "sequence": [
 843             ("random-sleep", random_sleep(300, 900)),
 844
 845         # build with all modules static
 846             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 847             ("allstatic-make", "make -j"),
 848             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 849             ("allstatic-lcov", LCOV_CMD),
 850
 851         # retry with nonshared smbd and smbtorture
 852             ("nonshared-distclean", "make distclean"),
 853             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 854             ("nonshared-make", "make -j"),
 855             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 856             # TODO ("nonshared-lcov", LCOV_CMD),
 857
 858             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 859             ("clean", "make clean"),
 860         ],
 861     },
 862
 863     "samba-nopython": {
 864         "sequence": [
 865             ("random-sleep", random_sleep(300, 900)),
 866
 867             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 868             ("make", "make -j"),
 869             ("find-python", "script/find_python.sh ${PREFIX}"),
 870             ("test", "make test-nopython"),
 871             ("lcov", LCOV_CMD),
 872             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 873             ("clean", "make clean"),
 874
 875             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 876             ("talloc-make", "cd lib/talloc && make"),
 877             ("talloc-install", "cd lib/talloc && make install"),
 878
 879             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 880             ("tdb-make", "cd lib/tdb && make"),
 881             ("tdb-install", "cd lib/tdb && make install"),
 882
 883             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 884             ("tevent-make", "cd lib/tevent && make"),
 885             ("tevent-install", "cd lib/tevent && make install"),
 886
 887             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 888             ("ldb-make", "cd lib/ldb && make"),
 889             ("ldb-install", "cd lib/ldb && make install"),
 890
 891         # retry against installed library packages, but no required modules
 892             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 893             ("libs-make", "make -j"),
 894             ("libs-install", "make install"),
 895             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 896             ("libs-clean", "make clean"),
 897
 898         ],
 899     },
 900
 901     "ldb": {
 902         "sequence": [
 903             ("random-sleep", random_sleep(60, 600)),
 904             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 905             ("make", "make"),
 906             ("install", "make install"),
 907             ("test", "make test"),
 908             ("lcov", LCOV_CMD),
 909             ("clean", "make clean"),
 910             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 911             ("make-no-lmdb", "make"),
 912             ("test-no-lmdb", "make test"),
 913             ("lcov-no-lmdb", LCOV_CMD),
 914             ("install-no-lmdb", "make install"),
 915             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 916             ("distcheck", "make distcheck"),
 917             ("clean", "make clean"),
 918         ],
 919     },
 920
 921     "tdb": {
 922         "sequence": [
 923             ("random-sleep", random_sleep(60, 600)),
 924             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 925             ("make", "make"),
 926             ("install", "make install"),
 927             ("test", "make test"),
 928             ("lcov", LCOV_CMD),
 929             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 930             ("distcheck", "make distcheck"),
 931             ("clean", "make clean"),
 932         ],
 933     },
 934
 935     "talloc": {
 936         "sequence": [
 937             ("random-sleep", random_sleep(60, 600)),
 938             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 939             ("make", "make"),
 940             ("install", "make install"),
 941             ("test", "make test"),
 942             ("lcov", LCOV_CMD),
 943             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 944             ("distcheck", "make distcheck"),
 945             ("clean", "make clean"),
 946         ],
 947     },
 948
 949     "replace": {
 950         "sequence": [
 951             ("random-sleep", random_sleep(60, 600)),
 952             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 953             ("make", "make"),
 954             ("install", "make install"),
 955             ("test", "make test"),
 956             ("lcov", LCOV_CMD),
 957             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 958             ("distcheck", "make distcheck"),
 959             ("clean", "make clean"),
 960         ],
 961     },
 962
 963     "tevent": {
 964         "sequence": [
 965             ("random-sleep", random_sleep(60, 600)),
 966             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 967             ("make", "make"),
 968             ("install", "make install"),
 969             ("test", "make test"),
 970             ("lcov", LCOV_CMD),
 971             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 972             ("distcheck", "make distcheck"),
 973             ("clean", "make clean"),
 974         ],
 975     },
 976
 977     "pidl": {
 978         "git-clone-required": True,
 979         "sequence": [
 980             ("random-sleep", random_sleep(60, 600)),
 981             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 982             ("touch", "touch *.yp"),
 983             ("make", "make"),
 984             ("test", "make test"),
 985             ("install", "make install"),
 986             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 987             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 988             ("clean", "make clean"),
 989         ],
 990     },
 991
 992     # these are useful for debugging autobuild
 993     "pass": {
 994         "sequence": [
 995             ("pass", 'echo passing && /bin/true'),
 996         ],
 997     },
 998     "fail": {
 999         "sequence": [
1000             ("fail", 'echo failing && /bin/false'),
1001         ],
1002     },
1003 }
1004
1005 defaulttasks = list(tasks.keys())
1006
1007 defaulttasks.remove("pass")
1008 defaulttasks.remove("fail")
1009
1010 # The build tasks will be brought in by the test tasks as needed
1011 defaulttasks.remove("samba-def-build")
1012 defaulttasks.remove("samba-nt4-build")
1013 defaulttasks.remove("samba-mit-build")
1014 defaulttasks.remove("samba-h5l-build")
1015 defaulttasks.remove("samba-no-opath-build")
1016
1017 # This is not a normal test, but a task to support manually running
1018 # one test under autobuild
1019 defaulttasks.remove("samba-test-only")
1020
1021 # Only built on GitLab CI and not in the default autobuild because it
1022 # uses too much space (4GB of semi-static binaries)
1023 defaulttasks.remove("samba-fuzz")
1024
1025 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1026 # container where a simulated FIPS mode is possible.
1027 defaulttasks.remove("samba-fips")
1028
1029 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1030     defaulttasks.remove("samba-o3")
1031
1032
1033 def do_print(msg):
1034     print("%s" % msg)
1035     sys.stdout.flush()
1036     sys.stderr.flush()
1037
1038
1039 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1040     if show is None:
1041         show = options.verbose
1042     if show:
1043         do_print("Running: '%s' in '%s'" % (cmd, dir))
1044     if output:
1045         out = check_output([cmd], shell=True, cwd=dir)
1046         return out.decode(encoding='utf-8', errors='backslashreplace')
1047     elif checkfail:
1048         return check_call(cmd, shell=True, cwd=dir)
1049     else:
1050         return call(cmd, shell=True, cwd=dir)
1051
1052 def rmdir_force(dirname, re_raise=True):
1053     try:
1054         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1055                 dirname, dirname, dirname), output=True, show=True)
1056     except CalledProcessError as e:
1057         do_print("Failed: '%s'" % (str(e)))
1058         run_cmd("tree %s" % dirname, output=True, show=True)
1059         if re_raise:
1060             raise
1061         return False
1062     return True
1063
1064 class builder(object):
1065     '''handle build of one directory'''
1066
1067     def __init__(self, name, definition):
1068         self.name = name
1069         self.dir = builddirs.get(name, '.')
1070         self.tag = self.name.replace('/', '_')
1071         self.definition = definition
1072         self.sequence = definition["sequence"]
1073         self.git_clone_required = False
1074         if "git-clone-required" in definition:
1075             self.git_clone_required = bool(definition["git-clone-required"])
1076         self.proc = None
1077         self.done = False
1078         self.next = 0
1079         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1080         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1081         if options.verbose:
1082             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1083             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1084         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1085         self.stdout = open(self.stdout_path, 'w')
1086         self.stderr = open(self.stderr_path, 'w')
1087         self.stdin  = open("/dev/null", 'r')
1088         self.builder_dir = "%s/%s" % (testbase, self.tag)
1089         self.test_source_dir = self.builder_dir
1090         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1091         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1092         self.prefix = "%s/%s" % (test_prefix, self.tag)
1093         self.consumers = []
1094         self.producer = None
1095
1096         if self.git_clone_required:
1097             assert "dependency" not in definition
1098
1099     def mark_existing(self):
1100         do_print('%s: Mark as existing dependency' % self.name)
1101         self.next = len(self.sequence)
1102         self.done = True
1103
1104     def add_consumer(self, consumer):
1105         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1106         consumer.producer = self
1107         consumer.test_source_dir = self.test_source_dir
1108         self.consumers.append(consumer)
1109
1110     def start_next(self):
1111         if self.producer is not None:
1112             if not self.producer.done:
1113                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1114                 return
1115
1116         if self.next == 0:
1117             rmdir_force(self.builder_dir)
1118             rmdir_force(self.prefix)
1119             if self.producer is not None:
1120                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1121             elif not self.git_clone_required:
1122                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1123             else:
1124                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1125
1126         if self.next == len(self.sequence):
1127             if not self.done:
1128                 do_print('%s: Completed OK' % self.name)
1129                 self.done = True
1130             if not options.nocleanup and len(self.consumers) == 0:
1131                 do_print('%s: Cleaning up' % self.name)
1132                 rmdir_force(self.builder_dir)
1133                 rmdir_force(self.prefix)
1134             for consumer in self.consumers:
1135                 if consumer.next != 0:
1136                     continue
1137                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1138                 consumer.start_next()
1139             if self.producer is not None:
1140                 self.producer.consumers.remove(self)
1141                 assert self.producer.done
1142                 self.producer.start_next()
1143             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1144             return
1145         (self.stage, self.cmd) = self.sequence[self.next]
1146         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
1147         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1148         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1149         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1150         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1151         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1152         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1153         self.cmd = self.cmd.replace("${NAME}", self.name)
1154         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1155         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1156         self.proc = Popen(self.cmd, shell=True,
1157                           close_fds=True, cwd=self.cwd,
1158                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1159         self.next += 1
1160
1161 def expand_dependencies(n):
1162     deps = list()
1163     if "dependency" in tasks[n]:
1164         depname = tasks[n]["dependency"]
1165         assert depname in tasks
1166         sdeps = expand_dependencies(depname)
1167         assert n not in sdeps
1168         for sdep in sdeps:
1169             deps.append(sdep)
1170         deps.append(depname)
1171     return deps
1172
1173
1174 class buildlist(object):
1175     '''handle build of multiple directories'''
1176
1177     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1178         self.tail_proc = None
1179         self.retry = None
1180         if not tasknames:
1181             if options.restrict_tests:
1182                 tasknames = ["samba-test-only"]
1183             else:
1184                 tasknames = defaulttasks
1185
1186         given_tasknames = tasknames.copy()
1187         implicit_tasknames = []
1188         for n in given_tasknames:
1189             deps = expand_dependencies(n)
1190             for dep in deps:
1191                 if dep in given_tasknames:
1192                     continue
1193                 if dep in implicit_tasknames:
1194                     continue
1195                 implicit_tasknames.append(dep)
1196
1197         tasknames = implicit_tasknames.copy()
1198         tasknames.extend(given_tasknames)
1199         do_print("given_tasknames: %s" % given_tasknames)
1200         do_print("implicit_tasknames: %s" % implicit_tasknames)
1201         do_print("tasknames: %s" % tasknames)
1202         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1203
1204         if options.retry:
1205             rebase_remote = "rebaseon"
1206             retry_task = {
1207                     "git-clone-required": True,
1208                     "sequence": [
1209                             ("retry",
1210                             '''set -e
1211                             git remote add -t %s %s %s
1212                             git fetch %s
1213                             while :; do
1214                               sleep 60
1215                               git describe %s/%s > old_remote_branch.desc
1216                               git fetch %s
1217                               git describe %s/%s > remote_branch.desc
1218                               diff old_remote_branch.desc remote_branch.desc
1219                             done
1220                            ''' % (
1221                                rebase_branch, rebase_remote, rebase_url,
1222                                rebase_remote,
1223                                rebase_remote, rebase_branch,
1224                                rebase_remote,
1225                                rebase_remote, rebase_branch
1226                             ))]}
1227
1228             self.retry = builder('retry', retry_task)
1229             self.need_retry = False
1230
1231         if options.skip_dependencies:
1232             for b in self.tlist:
1233                 if b.name in implicit_tasknames:
1234                     b.mark_existing()
1235
1236         for b in self.tlist:
1237             do_print("b.name=%s" % b.name)
1238             if "dependency" not in b.definition:
1239                 continue
1240             depname = b.definition["dependency"]
1241             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1242             for p in self.tlist:
1243                 if p.name == depname:
1244                     p.add_consumer(b)
1245
1246     def kill_kids(self):
1247         if self.tail_proc is not None:
1248             self.tail_proc.terminate()
1249             self.tail_proc.wait()
1250             self.tail_proc = None
1251         if self.retry is not None:
1252             self.retry.proc.terminate()
1253             self.retry.proc.wait()
1254             self.retry = None
1255         for b in self.tlist:
1256             if b.proc is not None:
1257                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1258                 b.proc.terminate()
1259                 b.proc.wait()
1260                 b.proc = None
1261
1262     def wait_one(self):
1263         while True:
1264             none_running = True
1265             for b in self.tlist:
1266                 if b.proc is None:
1267                     continue
1268                 none_running = False
1269                 b.status = b.proc.poll()
1270                 if b.status is None:
1271                     continue
1272                 b.proc = None
1273                 return b
1274             if options.retry:
1275                 ret = self.retry.proc.poll()
1276                 if ret is not None:
1277                     self.need_retry = True
1278                     self.retry = None
1279                     return None
1280             if none_running:
1281                 return None
1282             time.sleep(0.1)
1283
1284     def run(self):
1285         for b in self.tlist:
1286             b.start_next()
1287         if options.retry:
1288             self.retry.start_next()
1289         while True:
1290             b = self.wait_one()
1291             if options.retry and self.need_retry:
1292                 self.kill_kids()
1293                 do_print("retry needed")
1294                 return (0, None, None, None, "retry")
1295             if b is None:
1296                 break
1297             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1298                 self.kill_kids()
1299                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1300             b.start_next()
1301         self.kill_kids()
1302         return (0, None, None, None, "All OK")
1303
1304     def write_system_info(self, filename):
1305         with open(filename, 'w') as f:
1306             for cmd in ['uname -a',
1307                         'lsb_release -a',
1308                         'free',
1309                         'mount',
1310                         'cat /proc/cpuinfo',
1311                         'cc --version',
1312                         'df -m .',
1313                         'df -m %s' % testbase]:
1314                 try:
1315                     out = run_cmd(cmd, output=True, checkfail=False)
1316                 except CalledProcessError as e:
1317                     out = "<failed: %s>" % str(e)
1318                 print('### %s' % cmd, file=f)
1319                 print(out, file=f)
1320                 print(file=f)
1321
1322     def tarlogs(self, fname):
1323         with tarfile.open(fname, "w:gz") as tar:
1324             for b in self.tlist:
1325                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1326                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1327             if os.path.exists("autobuild.log"):
1328                 tar.add("autobuild.log")
1329             filename = 'system-info.txt'
1330             self.write_system_info(filename)
1331             tar.add(filename)
1332
1333     def remove_logs(self):
1334         for b in self.tlist:
1335             os.unlink(b.stdout_path)
1336             os.unlink(b.stderr_path)
1337
1338     def start_tail(self):
1339         cmd = ["tail", "-f"]
1340         for b in self.tlist:
1341             cmd.append(b.stdout_path)
1342             cmd.append(b.stderr_path)
1343         self.tail_proc = Popen(cmd, close_fds=True)
1344
1345
1346 def cleanup(do_raise=False):
1347     if options.nocleanup:
1348         return
1349     run_cmd("stat %s || true" % test_tmpdir, show=True)
1350     run_cmd("stat %s" % testbase, show=True)
1351     do_print("Cleaning up %r" % cleanup_list)
1352     for d in cleanup_list:
1353         ok = rmdir_force(d, re_raise=False)
1354         if ok:
1355             continue
1356         if os.path.isdir(d):
1357             do_print("Killing, waiting and retry")
1358             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1359         else:
1360             do_print("Waiting and retry")
1361         time.sleep(1)
1362         rmdir_force(d, re_raise=do_raise)
1363
1364
1365 def daemonize(logfile):
1366     pid = os.fork()
1367     if pid == 0:  # Parent
1368         os.setsid()
1369         pid = os.fork()
1370         if pid != 0:  # Actual daemon
1371             os._exit(0)
1372     else:  # Grandparent
1373         os._exit(0)
1374
1375     import resource      # Resource usage information.
1376     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1377     if maxfd == resource.RLIM_INFINITY:
1378         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1379     for fd in range(0, maxfd):
1380         try:
1381             os.close(fd)
1382         except OSError:
1383             pass
1384     os.open(logfile, os.O_RDWR | os.O_CREAT)
1385     os.dup2(0, 1)
1386     os.dup2(0, 2)
1387
1388
1389 def write_pidfile(fname):
1390     '''write a pid file, cleanup on exit'''
1391     with open(fname, mode='w') as f:
1392         f.write("%u\n" % os.getpid())
1393
1394
1395 def rebase_tree(rebase_url, rebase_branch="master"):
1396     rebase_remote = "rebaseon"
1397     do_print("Rebasing on %s" % rebase_url)
1398     run_cmd("git describe HEAD", show=True, dir=test_master)
1399     run_cmd("git remote add -t %s %s %s" %
1400             (rebase_branch, rebase_remote, rebase_url),
1401             show=True, dir=test_master)
1402     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1403     if options.fix_whitespace:
1404         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1405                 (rebase_remote, rebase_branch),
1406                 show=True, dir=test_master)
1407     else:
1408         run_cmd("git rebase --force-rebase %s/%s" %
1409                 (rebase_remote, rebase_branch),
1410                 show=True, dir=test_master)
1411     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1412                    (rebase_remote, rebase_branch),
1413                    dir=test_master, output=True)
1414     if diff == '':
1415         do_print("No differences between HEAD and %s/%s - exiting" %
1416                  (rebase_remote, rebase_branch))
1417         sys.exit(0)
1418     run_cmd("git describe %s/%s" %
1419             (rebase_remote, rebase_branch),
1420             show=True, dir=test_master)
1421     run_cmd("git describe HEAD", show=True, dir=test_master)
1422     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1423             (rebase_remote, rebase_branch),
1424             show=True, dir=test_master)
1425
1426
1427 def push_to(push_url, push_branch="master"):
1428     push_remote = "pushto"
1429     do_print("Pushing to %s" % push_url)
1430     if options.mark:
1431         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1432         run_cmd("git commit --amend -c HEAD", dir=test_master)
1433         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1434         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1435     run_cmd("git remote add -t %s %s %s" %
1436             (push_branch, push_remote, push_url),
1437             show=True, dir=test_master)
1438     run_cmd("git push %s +HEAD:%s" %
1439             (push_remote, push_branch),
1440             show=True, dir=test_master)
1441
1442
1443 def send_email(subject, text, log_tar):
1444     if options.email is None:
1445         do_print("not sending email because the recipient is not set")
1446         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1447                  (subject, text))
1448         return
1449     outer = MIMEMultipart()
1450     outer['Subject'] = subject
1451     outer['To'] = options.email
1452     outer['From'] = options.email_from
1453     outer['Date'] = email.utils.formatdate(localtime=True)
1454     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1455     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1456     if options.attach_logs:
1457         with open(log_tar, 'rb') as fp:
1458             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1459         # Set the filename parameter
1460         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1461         outer.attach(msg)
1462     content = outer.as_string()
1463     s = smtplib.SMTP(options.email_server)
1464     email_user = os.getenv('SMTP_USERNAME')
1465     email_password = os.getenv('SMTP_PASSWORD')
1466     if email_user is not None:
1467         s.starttls()
1468         s.login(email_user, email_password)
1469
1470     s.sendmail(options.email_from, [options.email], content)
1471     s.set_debuglevel(1)
1472     s.quit()
1473
1474
1475 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1476                   elapsed_time, log_base=None, add_log_tail=True):
1477     '''send an email to options.email about the failure'''
1478     elapsed_minutes = elapsed_time / 60.0
1479     if log_base is None:
1480         log_base = gitroot
1481     text = '''
1482 Dear Developer,
1483
1484 Your autobuild on %s failed after %.1f minutes
1485 when trying to test %s with the following error:
1486
1487    %s
1488
1489 the autobuild has been abandoned. Please fix the error and resubmit.
1490
1491 A summary of the autobuild process is here:
1492
1493   %s/autobuild.log
1494 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1495
1496     if options.restrict_tests:
1497         text += """
1498 The build was restricted to tests matching %s\n""" % options.restrict_tests
1499
1500     if failed_task != 'rebase':
1501         text += '''
1502 You can see logs of the failed task here:
1503
1504   %s/%s.stdout
1505   %s/%s.stderr
1506
1507 or you can get full logs of all tasks in this job here:
1508
1509   %s/logs.tar.gz
1510
1511 The top commit for the tree that was built was:
1512
1513 %s
1514
1515 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1516
1517     if add_log_tail:
1518         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1519         lines = f.readlines()
1520         log_tail = "".join(lines[-50:])
1521         num_lines = len(lines)
1522         if num_lines < 50:
1523             # Also include stderr (compile failures) if < 50 lines of stdout
1524             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1525             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1526
1527         text += '''
1528 The last 50 lines of log messages:
1529
1530 %s
1531     ''' % log_tail
1532         f.close()
1533
1534     logs = os.path.join(gitroot, 'logs.tar.gz')
1535     send_email('autobuild[%s] failure on %s for task %s during %s'
1536                % (options.branch, platform.node(), failed_task, failed_stage),
1537                text, logs)
1538
1539
1540 def email_success(elapsed_time, log_base=None):
1541     '''send an email to options.email about a successful build'''
1542     if log_base is None:
1543         log_base = gitroot
1544     text = '''
1545 Dear Developer,
1546
1547 Your autobuild on %s has succeeded after %.1f minutes.
1548
1549 ''' % (platform.node(), elapsed_time / 60.)
1550
1551     if options.restrict_tests:
1552         text += """
1553 The build was restricted to tests matching %s\n""" % options.restrict_tests
1554
1555     if options.keeplogs:
1556         text += '''
1557
1558 you can get full logs of all tasks in this job here:
1559
1560   %s/logs.tar.gz
1561
1562 ''' % log_base
1563
1564     text += '''
1565 The top commit for the tree that was built was:
1566
1567 %s
1568 ''' % top_commit_msg
1569
1570     logs = os.path.join(gitroot, 'logs.tar.gz')
1571     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1572                text, logs)
1573
1574
1575 # get the top commit message, for emails
1576 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1577
1578 try:
1579     if options.skip_dependencies:
1580         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1581     else:
1582         os.makedirs(testbase)
1583 except Exception as reason:
1584     raise Exception("Unable to create %s : %s" % (testbase, reason))
1585 cleanup_list.append(testbase)
1586
1587 if options.daemon:
1588     logfile = os.path.join(testbase, "log")
1589     do_print("Forking into the background, writing progress to %s" % logfile)
1590     daemonize(logfile)
1591
1592 write_pidfile(gitroot + "/autobuild.pid")
1593
1594 start_time = time.time()
1595
1596 while True:
1597     try:
1598         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1599         os.makedirs(test_tmpdir)
1600         # The waf uninstall code removes empty directories all the way
1601         # up the tree.  Creating a file in test_tmpdir stops it from
1602         # being removed.
1603         run_cmd("touch %s" % os.path.join(test_tmpdir,
1604                                           ".directory-is-not-empty"), show=True)
1605         run_cmd("stat %s" % test_tmpdir, show=True)
1606         run_cmd("stat %s" % testbase, show=True)
1607         if options.skip_dependencies:
1608             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1609         else:
1610             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1611     except Exception:
1612         cleanup()
1613         raise
1614
1615     try:
1616         if options.rebase is not None:
1617             rebase_tree(options.rebase, rebase_branch=options.branch)
1618     except Exception:
1619         cleanup_list.append(gitroot + "/autobuild.pid")
1620         cleanup()
1621         elapsed_time = time.time() - start_time
1622         email_failure(-1, 'rebase', 'rebase', 'rebase',
1623                       'rebase on %s failed' % options.branch,
1624                       elapsed_time, log_base=options.log_base)
1625         sys.exit(1)
1626
1627     try:
1628         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1629         if options.tail:
1630             blist.start_tail()
1631         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1632         if status != 0 or errstr != "retry":
1633             break
1634         cleanup(do_raise=True)
1635     except Exception:
1636         cleanup()
1637         raise
1638
1639 cleanup_list.append(gitroot + "/autobuild.pid")
1640
1641 do_print(errstr)
1642
1643 blist.kill_kids()
1644 if options.tail:
1645     do_print("waiting for tail to flush")
1646     time.sleep(1)
1647
1648 elapsed_time = time.time() - start_time
1649 if status == 0:
1650     if options.passcmd is not None:
1651         do_print("Running passcmd: %s" % options.passcmd)
1652         run_cmd(options.passcmd, dir=test_master)
1653     if options.pushto is not None:
1654         push_to(options.pushto, push_branch=options.branch)
1655     if options.keeplogs or options.attach_logs:
1656         blist.tarlogs("logs.tar.gz")
1657         do_print("Logs in logs.tar.gz")
1658     if options.always_email:
1659         email_success(elapsed_time, log_base=options.log_base)
1660     blist.remove_logs()
1661     cleanup()
1662     do_print(errstr)
1663     sys.exit(0)
1664
1665 # something failed, gather a tar of the logs
1666 blist.tarlogs("logs.tar.gz")
1667
1668 if options.email is not None:
1669     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1670                   elapsed_time, log_base=options.log_base)
1671 else:
1672     elapsed_minutes = elapsed_time / 60.0
1673     print('''
1674
1675 ####################################################################
1676
1677 AUTOBUILD FAILURE
1678
1679 Your autobuild[%s] on %s failed after %.1f minutes
1680 when trying to test %s with the following error:
1681
1682    %s
1683
1684 the autobuild has been abandoned. Please fix the error and resubmit.
1685
1686 ####################################################################
1687
1688 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1689
1690 cleanup()
1691 do_print(errstr)
1692 do_print("Logs in logs.tar.gz")
1693 sys.exit(status)