script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.exists(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  58 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  59                   default=def_testbase)
  60 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  61                   default=None)
  62 parser.add_option("--passcmd", help="command to run on success", default=None)
  63 parser.add_option("--verbose", help="show all commands as they are run",
  64                   default=False, action="store_true")
  65 parser.add_option("--rebase", help="rebase on the given tree before testing",
  66                   default=None, type='str')
  67 parser.add_option("--pushto", help="push to a git url on success",
  68                   default=None, type='str')
  69 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  70                   default=False, action="store_true")
  71 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  72                   default=False, action="store_true")
  73 parser.add_option("--retry", help="automatically retry if master changes",
  74                   default=False, action="store_true")
  75 parser.add_option("--email", help="send email to the given address on failure",
  76                   type='str', default=None)
  77 parser.add_option("--email-from", help="send email from the given address",
  78                   type='str', default="autobuild@samba.org")
  79 parser.add_option("--email-server", help="send email via the given server",
  80                   type='str', default='localhost')
  81 parser.add_option("--always-email", help="always send email, even on success",
  82                   action="store_true")
  83 parser.add_option("--daemon", help="daemonize after initial setup",
  84                   action="store_true")
  85 parser.add_option("--branch", help="the branch to work on (default=master)",
  86                   default="master", type='str')
  87 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  88                   default=gitroot, type='str')
  89 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  90                   default=False, action="store_true")
  91 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  92                   default='')
  93 parser.add_option("--enable-coverage", dest='enable_coverage',
  94                   action="store_const", const='--enable-coverage', default='',
  95                   help="Add --enable-coverage option while configure")
  96
  97 (options, args) = parser.parse_args()
  98
  99 if options.retry:
 100     if options.rebase is None:
 101         raise Exception('You can only use --retry if you also rebase')
 102
 103 if options.full_testbase is not None:
 104     testbase = options.full_testbase
 105 else:
 106     testbase = "%s/b%u" % (options.testbase, os.getpid())
 107 test_master = "%s/master" % testbase
 108 test_prefix = "%s/prefix" % testbase
 109 test_tmpdir = "%s/tmp" % testbase
 110 os.environ['TMPDIR'] = test_tmpdir
 111
 112 if options.enable_coverage:
 113     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 114 else:
 115     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 116
 117 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 118
 119 if args:
 120     # If we are only running specific test,
 121     # do not sleep randomly to wait for it to start
 122     def random_sleep(low, high):
 123         return 'sleep 1'
 124 else:
 125     def random_sleep(low, high):
 126         return 'sleep {}'.format(random.randint(low, high))
 127
 128 cleanup_list = []
 129
 130 builddirs = {
 131     "ctdb": "ctdb",
 132     "ldb": "lib/ldb",
 133     "tdb": "lib/tdb",
 134     "talloc": "lib/talloc",
 135     "replace": "lib/replace",
 136     "tevent": "lib/tevent",
 137     "pidl": "pidl",
 138     "docs-xml": "docs-xml"
 139 }
 140
 141 ctdb_configure_params = " --enable-developer ${PREFIX}"
 142 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 143
 144 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 145 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 146 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 147 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 148 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 149 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 150 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 151
 152
 153 def format_option(name, value=None):
 154     """Format option as str list."""
 155     if value is None:  # boolean option
 156         return [name]
 157     if not isinstance(value, list):  # single value option
 158         value = [value]
 159     # repeatable option
 160     return ['{}={}'.format(name, item) for item in value]
 161
 162
 163 def make_test(
 164         cmd='make testonly',
 165         FAIL_IMMEDIATELY=1,
 166         INJECT_SELFTEST_PREFIX=1,
 167         TESTS='',
 168         include_envs=None,
 169         exclude_envs=None):
 170
 171     test_options = []
 172     if include_envs:
 173         test_options = format_option('--include-env', include_envs)
 174     if exclude_envs:
 175         test_options = format_option('--exclude-env', exclude_envs)
 176     if test_options:
 177         # join envs options to original test options
 178         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 179
 180     _options = []
 181     if FAIL_IMMEDIATELY:
 182         _options.append('FAIL_IMMEDIATELY=1')
 183     if TESTS:
 184         _options.append("TESTS='{}'".format(TESTS))
 185
 186     if INJECT_SELFTEST_PREFIX:
 187         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 188         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 189
 190     return ' '.join([cmd] + _options)
 191
 192
 193 # When updating this list, also update .gitlab-ci.yml to add the job
 194 # and to make it a dependency of 'page' for the coverage report.
 195
 196 tasks = {
 197     "ctdb": {
 198         "sequence": [
 199             ("random-sleep", random_sleep(300, 900)),
 200             ("configure", "./configure " + ctdb_configure_params),
 201             ("make", "make all"),
 202             ("install", "make install"),
 203             ("test", "make autotest"),
 204             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 205             ("clean", "make clean"),
 206         ],
 207     },
 208     "docs-xml": {
 209         "sequence": [
 210             ("random-sleep", random_sleep(300, 900)),
 211             ("autoconf", "autoconf"),
 212             ("configure", "./configure"),
 213             ("make", "make html htmlman"),
 214             ("clean", "make clean"),
 215         ],
 216     },
 217
 218     "samba-def-build": {
 219         "git-clone-required": True,
 220         "sequence": [
 221             ("configure", "./configure.developer" + samba_configure_params),
 222             ("make", "make -j"),
 223             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 224             ("chmod-R-a-w", "chmod -R a-w ."),
 225         ],
 226     },
 227
 228     "samba-mit-build": {
 229         "git-clone-required": True,
 230         "sequence": [
 231             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 232             ("make", "make -j"),
 233             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 234             ("chmod-R-a-w", "chmod -R a-w ."),
 235         ],
 236     },
 237
 238     "samba-nt4-build": {
 239         "git-clone-required": True,
 240         "sequence": [
 241             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 242             ("make", "make -j"),
 243             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 244             ("chmod-R-a-w", "chmod -R a-w ."),
 245         ],
 246     },
 247
 248     "samba-h5l-build": {
 249         "git-clone-required": True,
 250         "sequence": [
 251             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 252             ("make", "make -j"),
 253             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 254             ("chmod-R-a-w", "chmod -R a-w ."),
 255         ],
 256     },
 257
 258     "samba-no-opath-build": {
 259         "git-clone-required": True,
 260         "sequence": [
 261             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 262             ("make", "make -j"),
 263             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 264             ("chmod-R-a-w", "chmod -R a-w ."),
 265         ],
 266     },
 267
 268     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 269     "samba": {
 270         "sequence": [
 271             ("random-sleep", random_sleep(300, 900)),
 272             ("configure", "./configure.developer" + samba_configure_params),
 273             ("make", "make -j"),
 274             ("test", make_test(exclude_envs=[
 275             "none",
 276             "nt4_dc",
 277             "nt4_dc_smb1",
 278             "nt4_dc_smb1_done",
 279             "nt4_dc_schannel",
 280             "nt4_member",
 281             "ad_dc",
 282             "ad_dc_smb1",
 283             "ad_dc_smb1_done",
 284             "ad_dc_backup",
 285             "ad_dc_ntvfs",
 286             "ad_dc_default",
 287             "ad_dc_default_smb1",
 288             "ad_dc_slowtests",
 289             "ad_dc_no_nss",
 290             "ad_dc_no_ntlm",
 291             "fl2003dc",
 292             "fl2008dc",
 293             "fl2008r2dc",
 294             "ad_member",
 295             "ad_member_idmap_rid",
 296             "ad_member_idmap_ad",
 297             "ad_member_rfc2307",
 298             "chgdcpass",
 299             "vampire_2000_dc",
 300             "fl2000dc",
 301             "fileserver",
 302             "fileserver_smb1",
 303             "fileserver_smb1_done",
 304             "maptoguest",
 305             "simpleserver",
 306             "backupfromdc",
 307             "restoredc",
 308             "renamedc",
 309             "offlinebackupdc",
 310             "labdc",
 311             "preforkrestartdc",
 312             "proclimitdc",
 313             "promoted_dc",
 314             "vampire_dc",
 315             "rodc",
 316             "ad_dc_default",
 317             "ad_dc_default_smb1",
 318             "ad_dc_default_smb1_done",
 319             "ad_dc_slowtests",
 320             "schema_pair_dc",
 321             "schema_dc",
 322             "clusteredmember",
 323             ])),
 324             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 325             ("lcov", LCOV_CMD),
 326             ("install", "make install"),
 327             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 328             ("clean", "make clean"),
 329         ],
 330     },
 331
 332     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 333     "samba-mitkrb5": {
 334         "sequence": [
 335             ("random-sleep", random_sleep(300, 900)),
 336             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 337             ("make", "make -j"),
 338             ("test", make_test(exclude_envs=[
 339             "none",
 340             "nt4_dc",
 341             "nt4_dc_smb1",
 342             "nt4_dc_smb1_done",
 343             "nt4_dc_schannel",
 344             "nt4_member",
 345             "ad_dc",
 346             "ad_dc_smb1",
 347             "ad_dc_smb1_done",
 348             "ad_dc_backup",
 349             "ad_dc_ntvfs",
 350             "ad_dc_default",
 351             "ad_dc_default_smb1",
 352             "ad_dc_default_smb1_done",
 353             "ad_dc_slowtests",
 354             "ad_dc_no_nss",
 355             "ad_dc_no_ntlm",
 356             "fl2003dc",
 357             "fl2008dc",
 358             "fl2008r2dc",
 359             "ad_member",
 360             "ad_member_idmap_rid",
 361             "ad_member_idmap_ad",
 362             "ad_member_rfc2307",
 363             "chgdcpass",
 364             "vampire_2000_dc",
 365             "fl2000dc",
 366             "fileserver",
 367             "fileserver_smb1",
 368             "fileserver_smb1_done",
 369             "maptoguest",
 370             "simpleserver",
 371             "backupfromdc",
 372             "restoredc",
 373             "renamedc",
 374             "offlinebackupdc",
 375             "labdc",
 376             "preforkrestartdc",
 377             "proclimitdc",
 378             "promoted_dc",
 379             "vampire_dc",
 380             "rodc",
 381             "ad_dc_default",
 382             "ad_dc_default_smb1",
 383             "ad_dc_default_smb1_done",
 384             "ad_dc_slowtests",
 385             "schema_pair_dc",
 386             "schema_dc",
 387             "clusteredmember",
 388             ])),
 389             ("lcov", LCOV_CMD),
 390             ("install", "make install"),
 391             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 392             ("clean", "make clean"),
 393         ],
 394     },
 395
 396     "samba-nt4": {
 397         "dependency": "samba-nt4-build",
 398         "sequence": [
 399             ("random-sleep", random_sleep(300, 900)),
 400             ("test", make_test(include_envs=[
 401             "nt4_dc",
 402             "nt4_dc_smb1",
 403             "nt4_dc_smb1_done",
 404             "nt4_dc_schannel",
 405             "nt4_member",
 406             "simpleserver",
 407             ])),
 408             ("lcov", LCOV_CMD),
 409             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 410         ],
 411     },
 412
 413     "samba-fileserver": {
 414         "dependency": "samba-h5l-build",
 415         "sequence": [
 416             ("random-sleep", random_sleep(300, 900)),
 417             ("test", make_test(include_envs=[
 418             "fileserver",
 419             "fileserver_smb1",
 420             "fileserver_smb1_done",
 421             "maptoguest",
 422             "ktest", # ktest is also tested in samba and samba-mitkrb5
 423                      # but is tested here against a system Heimdal
 424             ])),
 425             ("lcov", LCOV_CMD),
 426             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 427         ],
 428     },
 429
 430     "samba-admem": {
 431         "dependency": "samba-def-build",
 432         "sequence": [
 433             ("random-sleep", random_sleep(300, 900)),
 434             ("test", make_test(include_envs=[
 435             "ad_member",
 436             "ad_member_idmap_rid",
 437             "ad_member_idmap_ad",
 438             "ad_member_rfc2307",
 439             ])),
 440             ("lcov", LCOV_CMD),
 441             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 442         ],
 443     },
 444
 445     "samba-no-opath1": {
 446         "dependency": "samba-no-opath-build",
 447         "sequence": [
 448             ("random-sleep", random_sleep(300, 900)),
 449             ("test", make_test(
 450                 cmd="make testonly DISABLE_OPATH=1",
 451                 include_envs=[
 452                 "nt4_dc",
 453                 "nt4_dc_smb1",
 454                 "nt4_dc_smb1_done",
 455                 "nt4_dc_schannel",
 456                 "nt4_member",
 457                 "simpleserver",
 458                 ])),
 459             ("lcov", LCOV_CMD),
 460             ("check-clean-tree", "script/clean-source-tree.sh"),
 461         ],
 462     },
 463
 464     "samba-no-opath2": {
 465         "dependency": "samba-no-opath-build",
 466         "sequence": [
 467             ("random-sleep", random_sleep(300, 900)),
 468             ("test", make_test(
 469                 cmd="make testonly DISABLE_OPATH=1",
 470                 include_envs=[
 471                 "fileserver",
 472                 "fileserver_smb1",
 473                 "fileserver_smb1_done",
 474                 ])),
 475             ("lcov", LCOV_CMD),
 476             ("check-clean-tree", "script/clean-source-tree.sh"),
 477         ],
 478     },
 479
 480     "samba-ad-dc-1": {
 481         "dependency": "samba-def-build",
 482         "sequence": [
 483             ("random-sleep", random_sleep(1, 1)),
 484             ("test", make_test(include_envs=[
 485             "ad_dc",
 486             "ad_dc_smb1",
 487             "ad_dc_smb1_done",
 488             "ad_dc_no_nss",
 489             "ad_dc_no_ntlm",
 490             ])),
 491             ("lcov", LCOV_CMD),
 492             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 493         ],
 494     },
 495
 496     "samba-ad-dc-2": {
 497         "dependency": "samba-def-build",
 498         "sequence": [
 499             ("random-sleep", random_sleep(1, 1)),
 500             ("test", make_test(include_envs=[
 501             "vampire_dc",
 502             "vampire_2000_dc",
 503             "rodc",
 504             ])),
 505             ("lcov", LCOV_CMD),
 506             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 507         ],
 508     },
 509
 510     "samba-ad-dc-3": {
 511         "dependency": "samba-def-build",
 512         "sequence": [
 513             ("random-sleep", random_sleep(1, 1)),
 514             ("test", make_test(include_envs=[
 515             "promoted_dc",
 516             "chgdcpass",
 517             "preforkrestartdc",
 518             "proclimitdc",
 519             ])),
 520             ("lcov", LCOV_CMD),
 521             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 522         ],
 523     },
 524
 525     "samba-ad-dc-4": {
 526         "dependency": "samba-def-build",
 527         "sequence": [
 528             ("random-sleep", random_sleep(1, 1)),
 529             ("test", make_test(include_envs=[
 530             "fl2000dc",
 531             "fl2003dc",
 532             "fl2008dc",
 533             "fl2008r2dc",
 534             ])),
 535             ("lcov", LCOV_CMD),
 536             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 537         ],
 538     },
 539
 540     "samba-ad-dc-5": {
 541         "dependency": "samba-def-build",
 542         "sequence": [
 543             ("random-sleep", random_sleep(1, 1)),
 544             ("test", make_test(include_envs=[
 545             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 546             ("lcov", LCOV_CMD),
 547             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 548         ],
 549     },
 550
 551     "samba-ad-dc-6": {
 552         "dependency": "samba-def-build",
 553         "sequence": [
 554             ("random-sleep", random_sleep(1, 1)),
 555             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 556             ("lcov", LCOV_CMD),
 557             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 558         ],
 559     },
 560
 561     "samba-schemaupgrade": {
 562         "dependency": "samba-def-build",
 563         "sequence": [
 564             ("random-sleep", random_sleep(1, 1)),
 565             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 566             ("lcov", LCOV_CMD),
 567             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 568         ],
 569     },
 570
 571     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 572     # This is currently the longest task, so we don't randomly delay it.
 573     "samba-ad-dc-ntvfs": {
 574         "dependency": "samba-def-build",
 575         "sequence": [
 576             ("random-sleep", random_sleep(1, 1)),
 577             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 578             ("lcov", LCOV_CMD),
 579             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 580         ],
 581     },
 582
 583     # Test fips compliance
 584     "samba-fips": {
 585         "dependency": "samba-mit-build",
 586         "sequence": [
 587             ("random-sleep", random_sleep(1, 1)),
 588             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 589             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 590             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 591         ],
 592     },
 593
 594     # run the backup/restore testenvs separately as they're fairly standalone
 595     # (and CI seems to max out at ~3 different DCs running at once)
 596     "samba-ad-back1": {
 597         "dependency": "samba-def-build",
 598         "sequence": [
 599             ("random-sleep", random_sleep(300, 900)),
 600             ("test", make_test(include_envs=[
 601             "backupfromdc",
 602             "restoredc",
 603             "renamedc",
 604             ])),
 605             ("lcov", LCOV_CMD),
 606             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 607         ],
 608     },
 609     "samba-ad-back2": {
 610         "dependency": "samba-def-build",
 611         "sequence": [
 612             ("random-sleep", random_sleep(300, 900)),
 613             ("test", make_test(include_envs=[
 614             "backupfromdc",
 615             "offlinebackupdc",
 616             "labdc",
 617             ])),
 618             ("lcov", LCOV_CMD),
 619             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 620         ],
 621     },
 622
 623     "samba-admem-mit": {
 624         "dependency": "samba-mit-build",
 625         "sequence": [
 626             ("random-sleep", random_sleep(1, 1)),
 627             ("test", make_test(include_envs=[
 628             "ad_member",
 629             "ad_member_idmap_rid",
 630             "ad_member_idmap_ad",
 631             "ad_member_rfc2307",
 632             ])),
 633             ("lcov", LCOV_CMD),
 634             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 635         ],
 636     },
 637
 638     "samba-ad-dc-1-mitkrb5": {
 639         "dependency": "samba-mit-build",
 640         "sequence": [
 641             ("random-sleep", random_sleep(1, 1)),
 642             ("test", make_test(include_envs=[
 643             "ad_dc",
 644             "ad_dc_smb1",
 645             "ad_dc_smb1_done",
 646             "ad_dc_no_nss",
 647             "ad_dc_no_ntlm",
 648             ])),
 649             ("lcov", LCOV_CMD),
 650             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 651         ],
 652     },
 653
 654     "samba-ad-dc-4-mitkrb5": {
 655         "dependency": "samba-mit-build",
 656         "sequence": [
 657             ("random-sleep", random_sleep(1, 1)),
 658             ("test", make_test(include_envs=[
 659             "fl2000dc",
 660             "fl2003dc",
 661             "fl2008dc",
 662             "fl2008r2dc",
 663             ])),
 664             ("lcov", LCOV_CMD),
 665             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 666         ],
 667     },
 668
 669     "samba-test-only": {
 670         "sequence": [
 671             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 672             ("make", "make -j"),
 673             ("test", make_test(TESTS="${TESTS}")),
 674             ("lcov", LCOV_CMD),
 675         ],
 676     },
 677
 678     # Test cross-compile infrastructure
 679     "samba-xc": {
 680         "sequence": [
 681             ("random-sleep", random_sleep(900, 1500)),
 682             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 683             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 684             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 685             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 686             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 687             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 688             ("compare-results", "script/compare_cc_results.py "
 689             "./bin/c4che/default{} "
 690             "./bin-xe/c4che/default{} "
 691             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 692             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 693             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 694             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 695             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 696             " = \"'1234'\"".format(CACHE_SUFFIX)),
 697             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 698             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 699             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 700             " ; test $? -ne 0"),
 701         ],
 702     },
 703
 704     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 705     "samba-o3": {
 706         "sequence": [
 707             ("random-sleep", random_sleep(300, 900)),
 708             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 709             ("make", "make -j"),
 710             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 711             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 712             ("lcov", LCOV_CMD),
 713             ("install", "make install"),
 714             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 715             ("clean", "make clean"),
 716         ],
 717     },
 718
 719     "samba-ctdb": {
 720         "sequence": [
 721             ("random-sleep", random_sleep(900, 1500)),
 722
 723         # make sure we have tdb around:
 724             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 725             ("tdb-make", "cd lib/tdb && make"),
 726             ("tdb-install", "cd lib/tdb && make install"),
 727
 728         # build samba with cluster support (also building ctdb):
 729             ("samba-configure",
 730          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 731          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 732          "./configure.developer ${PREFIX} "
 733          "--with-selftest-prefix=./bin/ab "
 734          "--enable-clangdb "
 735          "--with-cluster-support "
 736          "--without-ad-dc "
 737          "--bundled-libraries=!tdb"),
 738             ("samba-make", "make"),
 739             ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 740             ("samba-install", "make install"),
 741             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 742
 743             ("test", make_test(
 744                 cmd='make test',
 745                 INJECT_SELFTEST_PREFIX=0,
 746                 include_envs=["clusteredmember"])
 747             ),
 748
 749         # clean up:
 750             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 751             ("clean", "make clean"),
 752             ("ctdb-clean", "cd ./ctdb && make clean"),
 753         ],
 754     },
 755
 756     "samba-libs": {
 757         "sequence": [
 758             ("random-sleep", random_sleep(300, 900)),
 759             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 760             ("talloc-make", "cd lib/talloc && make"),
 761             ("talloc-install", "cd lib/talloc && make install"),
 762
 763             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 764             ("tdb-make", "cd lib/tdb && make"),
 765             ("tdb-install", "cd lib/tdb && make install"),
 766
 767             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 768             ("tevent-make", "cd lib/tevent && make"),
 769             ("tevent-install", "cd lib/tevent && make install"),
 770
 771             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 772             ("ldb-make", "cd lib/ldb && make"),
 773             ("ldb-install", "cd lib/ldb && make install"),
 774
 775             ("nondevel-configure", "./configure ${PREFIX}"),
 776             ("nondevel-make", "make -j"),
 777             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 778             ("nondevel-install", "make install"),
 779             ("nondevel-dist", "make dist"),
 780
 781         # retry with all modules shared
 782             ("allshared-distclean", "make distclean"),
 783             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 784             ("allshared-make", "make -j"),
 785         ],
 786     },
 787
 788     "samba-fuzz": {
 789         "sequence": [
 790         # build the fuzzers (static) via the oss-fuzz script
 791             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 792             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 793         ],
 794     },
 795
 796     # * Test smbd and smbtorture can build semi-static
 797     #
 798     # * Test Samba without python still builds.
 799     #
 800     # When this test fails due to more use of Python, the expectations
 801     # is that the newly failing part of the code should be disabled
 802     # when --disable-python is set (rather than major work being done
 803     # to support this environment).
 804     #
 805     # The target here is for vendors shipping a minimal smbd.
 806     "samba-minimal-smbd": {
 807         "sequence": [
 808             ("random-sleep", random_sleep(300, 900)),
 809
 810         # build with all modules static
 811             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 812             ("allstatic-make", "make -j"),
 813             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 814             ("lcov", LCOV_CMD),
 815
 816         # retry with nonshared smbd and smbtorture
 817             ("nonshared-distclean", "make distclean"),
 818             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 819             ("nonshared-make", "make -j"),
 820
 821             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 822             ("make", "make -j"),
 823             ("find-python", "script/find_python.sh ${PREFIX}"),
 824             ("test", "make test-nopython"),
 825             ("lcov", LCOV_CMD),
 826             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 827             ("clean", "make clean"),
 828
 829             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 830             ("talloc-make", "cd lib/talloc && make"),
 831             ("talloc-install", "cd lib/talloc && make install"),
 832
 833             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 834             ("tdb-make", "cd lib/tdb && make"),
 835             ("tdb-install", "cd lib/tdb && make install"),
 836
 837             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 838             ("tevent-make", "cd lib/tevent && make"),
 839             ("tevent-install", "cd lib/tevent && make install"),
 840
 841             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 842             ("ldb-make", "cd lib/ldb && make"),
 843             ("ldb-install", "cd lib/ldb && make install"),
 844
 845         # retry against installed library packages, but no required modules
 846             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 847             ("libs-make", "make -j"),
 848             ("libs-install", "make install"),
 849             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 850             ("libs-clean", "make clean"),
 851
 852         ],
 853     },
 854
 855     "ldb": {
 856         "sequence": [
 857             ("random-sleep", random_sleep(60, 600)),
 858             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 859             ("make", "make"),
 860             ("install", "make install"),
 861             ("test", "make test"),
 862             ("lcov", LCOV_CMD),
 863             ("clean", "make clean"),
 864             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 865             ("make-no-lmdb", "make"),
 866             ("test-no-lmdb", "make test"),
 867             ("lcov-no-lmdb", LCOV_CMD),
 868             ("install-no-lmdb", "make install"),
 869             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 870             ("distcheck", "make distcheck"),
 871             ("clean", "make clean"),
 872         ],
 873     },
 874
 875     "tdb": {
 876         "sequence": [
 877             ("random-sleep", random_sleep(60, 600)),
 878             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 879             ("make", "make"),
 880             ("install", "make install"),
 881             ("test", "make test"),
 882             ("lcov", LCOV_CMD),
 883             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 884             ("distcheck", "make distcheck"),
 885             ("clean", "make clean"),
 886         ],
 887     },
 888
 889     "talloc": {
 890         "sequence": [
 891             ("random-sleep", random_sleep(60, 600)),
 892             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 893             ("make", "make"),
 894             ("install", "make install"),
 895             ("test", "make test"),
 896             ("lcov", LCOV_CMD),
 897             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 898             ("distcheck", "make distcheck"),
 899             ("clean", "make clean"),
 900         ],
 901     },
 902
 903     "replace": {
 904         "sequence": [
 905             ("random-sleep", random_sleep(60, 600)),
 906             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 907             ("make", "make"),
 908             ("install", "make install"),
 909             ("test", "make test"),
 910             ("lcov", LCOV_CMD),
 911             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 912             ("distcheck", "make distcheck"),
 913             ("clean", "make clean"),
 914         ],
 915     },
 916
 917     "tevent": {
 918         "sequence": [
 919             ("random-sleep", random_sleep(60, 600)),
 920             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 921             ("make", "make"),
 922             ("install", "make install"),
 923             ("test", "make test"),
 924             ("lcov", LCOV_CMD),
 925             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 926             ("distcheck", "make distcheck"),
 927             ("clean", "make clean"),
 928         ],
 929     },
 930
 931     "pidl": {
 932         "git-clone-required": True,
 933         "sequence": [
 934             ("random-sleep", random_sleep(60, 600)),
 935             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 936             ("touch", "touch *.yp"),
 937             ("make", "make"),
 938             ("test", "make test"),
 939             ("install", "make install"),
 940             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 941             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 942             ("clean", "make clean"),
 943         ],
 944     },
 945
 946     # these are useful for debugging autobuild
 947     "pass": {
 948         "sequence": [
 949             ("pass", 'echo passing && /bin/true'),
 950         ],
 951     },
 952     "fail": {
 953         "sequence": [
 954             ("fail", 'echo failing && /bin/false'),
 955         ],
 956     },
 957 }
 958
 959 defaulttasks = list(tasks.keys())
 960
 961 defaulttasks.remove("pass")
 962 defaulttasks.remove("fail")
 963 defaulttasks.remove("samba-def-build")
 964 defaulttasks.remove("samba-nt4-build")
 965 defaulttasks.remove("samba-mit-build")
 966 defaulttasks.remove("samba-h5l-build")
 967 defaulttasks.remove("samba-no-opath-build")
 968 defaulttasks.remove("samba-test-only")
 969 defaulttasks.remove("samba-fuzz")
 970 defaulttasks.remove("samba-fips")
 971 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 972     defaulttasks.remove("samba-o3")
 973
 974
 975 def do_print(msg):
 976     print("%s" % msg)
 977     sys.stdout.flush()
 978     sys.stderr.flush()
 979
 980
 981 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 982     if show is None:
 983         show = options.verbose
 984     if show:
 985         do_print("Running: '%s' in '%s'" % (cmd, dir))
 986     if output:
 987         out = check_output([cmd], shell=True, cwd=dir)
 988         return out.decode(encoding='utf-8', errors='backslashreplace')
 989     elif checkfail:
 990         return check_call(cmd, shell=True, cwd=dir)
 991     else:
 992         return call(cmd, shell=True, cwd=dir)
 993
 994 def rmdir_force(dirname, re_raise=True):
 995     try:
 996         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
 997                 dirname, dirname, dirname), output=True, show=True)
 998     except CalledProcessError as e:
 999         do_print("Failed: '%s'" % (str(e)))
1000         run_cmd("tree %s" % dirname, output=True, show=True)
1001         if re_raise:
1002             raise
1003         return False
1004     return True
1005
1006 class builder(object):
1007     '''handle build of one directory'''
1008
1009     def __init__(self, name, definition):
1010         self.name = name
1011         self.dir = builddirs.get(name, '.')
1012         self.tag = self.name.replace('/', '_')
1013         self.definition = definition
1014         self.sequence = definition["sequence"]
1015         self.git_clone_required = False
1016         if "git-clone-required" in definition:
1017             self.git_clone_required = bool(definition["git-clone-required"])
1018         self.proc = None
1019         self.done = False
1020         self.next = 0
1021         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1022         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1023         if options.verbose:
1024             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1025             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1026         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1027         self.stdout = open(self.stdout_path, 'w')
1028         self.stderr = open(self.stderr_path, 'w')
1029         self.stdin  = open("/dev/null", 'r')
1030         self.builder_dir = "%s/%s" % (testbase, self.tag)
1031         self.test_source_dir = self.builder_dir
1032         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1033         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1034         self.prefix = "%s/%s" % (test_prefix, self.tag)
1035         self.consumers = []
1036         self.producer = None
1037
1038         if self.git_clone_required:
1039             assert "dependency" not in definition
1040
1041     def mark_existing(self):
1042         do_print('%s: Mark as existing dependency' % self.name)
1043         self.next = len(self.sequence)
1044         self.done = True
1045
1046     def add_consumer(self, consumer):
1047         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1048         consumer.producer = self
1049         consumer.test_source_dir = self.test_source_dir
1050         self.consumers.append(consumer)
1051
1052     def start_next(self):
1053         if self.producer is not None:
1054             if not self.producer.done:
1055                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1056                 return
1057
1058         if self.next == 0:
1059             rmdir_force(self.builder_dir)
1060             rmdir_force(self.prefix)
1061             if self.producer is not None:
1062                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1063             elif not self.git_clone_required:
1064                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1065             else:
1066                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1067
1068         if self.next == len(self.sequence):
1069             if not self.done:
1070                 do_print('%s: Completed OK' % self.name)
1071                 self.done = True
1072             if not options.nocleanup and len(self.consumers) == 0:
1073                 do_print('%s: Cleaning up' % self.name)
1074                 rmdir_force(self.builder_dir)
1075                 rmdir_force(self.prefix)
1076             for consumer in self.consumers:
1077                 if consumer.next != 0:
1078                     continue
1079                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1080                 consumer.start_next()
1081             if self.producer is not None:
1082                 self.producer.consumers.remove(self)
1083                 assert self.producer.done
1084                 self.producer.start_next()
1085             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1086             return
1087         (self.stage, self.cmd) = self.sequence[self.next]
1088         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
1089         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1090         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1091         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1092         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1093         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1094         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1095         self.cmd = self.cmd.replace("${NAME}", self.name)
1096         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1097         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1098         self.proc = Popen(self.cmd, shell=True,
1099                           close_fds=True, cwd=self.cwd,
1100                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1101         self.next += 1
1102
1103 def expand_dependencies(n):
1104     deps = list()
1105     if "dependency" in tasks[n]:
1106         depname = tasks[n]["dependency"]
1107         assert depname in tasks
1108         sdeps = expand_dependencies(depname)
1109         assert n not in sdeps
1110         for sdep in sdeps:
1111             deps.append(sdep)
1112         deps.append(depname)
1113     return deps
1114
1115
1116 class buildlist(object):
1117     '''handle build of multiple directories'''
1118
1119     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1120         self.tail_proc = None
1121         self.retry = None
1122         if not tasknames:
1123             if options.restrict_tests:
1124                 tasknames = ["samba-test-only"]
1125             else:
1126                 tasknames = defaulttasks
1127
1128         given_tasknames = tasknames.copy()
1129         implicit_tasknames = []
1130         for n in given_tasknames:
1131             deps = expand_dependencies(n)
1132             for dep in deps:
1133                 if dep in given_tasknames:
1134                     continue
1135                 if dep in implicit_tasknames:
1136                     continue
1137                 implicit_tasknames.append(dep)
1138
1139         tasknames = implicit_tasknames.copy()
1140         tasknames.extend(given_tasknames)
1141         do_print("given_tasknames: %s" % given_tasknames)
1142         do_print("implicit_tasknames: %s" % implicit_tasknames)
1143         do_print("tasknames: %s" % tasknames)
1144         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1145
1146         if options.retry:
1147             rebase_remote = "rebaseon"
1148             retry_task = {
1149                     "git-clone-required": True,
1150                     "sequence": [
1151                             ("retry",
1152                             '''set -e
1153                             git remote add -t %s %s %s
1154                             git fetch %s
1155                             while :; do
1156                               sleep 60
1157                               git describe %s/%s > old_remote_branch.desc
1158                               git fetch %s
1159                               git describe %s/%s > remote_branch.desc
1160                               diff old_remote_branch.desc remote_branch.desc
1161                             done
1162                            ''' % (
1163                                rebase_branch, rebase_remote, rebase_url,
1164                                rebase_remote,
1165                                rebase_remote, rebase_branch,
1166                                rebase_remote,
1167                                rebase_remote, rebase_branch
1168                             ))]}
1169
1170             self.retry = builder('retry', retry_task)
1171             self.need_retry = False
1172
1173         if options.skip_dependencies:
1174             for b in self.tlist:
1175                 if b.name in implicit_tasknames:
1176                     b.mark_existing()
1177
1178         for b in self.tlist:
1179             do_print("b.name=%s" % b.name)
1180             if "dependency" not in b.definition:
1181                 continue
1182             depname = b.definition["dependency"]
1183             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1184             for p in self.tlist:
1185                 if p.name == depname:
1186                     p.add_consumer(b)
1187
1188     def kill_kids(self):
1189         if self.tail_proc is not None:
1190             self.tail_proc.terminate()
1191             self.tail_proc.wait()
1192             self.tail_proc = None
1193         if self.retry is not None:
1194             self.retry.proc.terminate()
1195             self.retry.proc.wait()
1196             self.retry = None
1197         for b in self.tlist:
1198             if b.proc is not None:
1199                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1200                 b.proc.terminate()
1201                 b.proc.wait()
1202                 b.proc = None
1203
1204     def wait_one(self):
1205         while True:
1206             none_running = True
1207             for b in self.tlist:
1208                 if b.proc is None:
1209                     continue
1210                 none_running = False
1211                 b.status = b.proc.poll()
1212                 if b.status is None:
1213                     continue
1214                 b.proc = None
1215                 return b
1216             if options.retry:
1217                 ret = self.retry.proc.poll()
1218                 if ret is not None:
1219                     self.need_retry = True
1220                     self.retry = None
1221                     return None
1222             if none_running:
1223                 return None
1224             time.sleep(0.1)
1225
1226     def run(self):
1227         for b in self.tlist:
1228             b.start_next()
1229         if options.retry:
1230             self.retry.start_next()
1231         while True:
1232             b = self.wait_one()
1233             if options.retry and self.need_retry:
1234                 self.kill_kids()
1235                 do_print("retry needed")
1236                 return (0, None, None, None, "retry")
1237             if b is None:
1238                 break
1239             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1240                 self.kill_kids()
1241                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1242             b.start_next()
1243         self.kill_kids()
1244         return (0, None, None, None, "All OK")
1245
1246     def write_system_info(self, filename):
1247         with open(filename, 'w') as f:
1248             for cmd in ['uname -a',
1249                         'lsb_release -a',
1250                         'free',
1251                         'mount',
1252                         'cat /proc/cpuinfo',
1253                         'cc --version',
1254                         'df -m .',
1255                         'df -m %s' % testbase]:
1256                 try:
1257                     out = run_cmd(cmd, output=True, checkfail=False)
1258                 except CalledProcessError as e:
1259                     out = "<failed: %s>" % str(e)
1260                 print('### %s' % cmd, file=f)
1261                 print(out, file=f)
1262                 print(file=f)
1263
1264     def tarlogs(self, fname):
1265         with tarfile.open(fname, "w:gz") as tar:
1266             for b in self.tlist:
1267                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1268                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1269             if os.path.exists("autobuild.log"):
1270                 tar.add("autobuild.log")
1271             filename = 'system-info.txt'
1272             self.write_system_info(filename)
1273             tar.add(filename)
1274
1275     def remove_logs(self):
1276         for b in self.tlist:
1277             os.unlink(b.stdout_path)
1278             os.unlink(b.stderr_path)
1279
1280     def start_tail(self):
1281         cmd = ["tail", "-f"]
1282         for b in self.tlist:
1283             cmd.append(b.stdout_path)
1284             cmd.append(b.stderr_path)
1285         self.tail_proc = Popen(cmd, close_fds=True)
1286
1287
1288 def cleanup(do_raise=False):
1289     if options.nocleanup:
1290         return
1291     run_cmd("stat %s || true" % test_tmpdir, show=True)
1292     run_cmd("stat %s" % testbase, show=True)
1293     do_print("Cleaning up %r" % cleanup_list)
1294     for d in cleanup_list:
1295         ok = rmdir_force(d, re_raise=False)
1296         if ok:
1297             continue
1298         if os.path.isdir(d):
1299             do_print("Killing, waiting and retry")
1300             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1301         else:
1302             do_print("Waiting and retry")
1303         time.sleep(1)
1304         rmdir_force(d, re_raise=do_raise)
1305
1306
1307 def daemonize(logfile):
1308     pid = os.fork()
1309     if pid == 0:  # Parent
1310         os.setsid()
1311         pid = os.fork()
1312         if pid != 0:  # Actual daemon
1313             os._exit(0)
1314     else:  # Grandparent
1315         os._exit(0)
1316
1317     import resource      # Resource usage information.
1318     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1319     if maxfd == resource.RLIM_INFINITY:
1320         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1321     for fd in range(0, maxfd):
1322         try:
1323             os.close(fd)
1324         except OSError:
1325             pass
1326     os.open(logfile, os.O_RDWR | os.O_CREAT)
1327     os.dup2(0, 1)
1328     os.dup2(0, 2)
1329
1330
1331 def write_pidfile(fname):
1332     '''write a pid file, cleanup on exit'''
1333     with open(fname, mode='w') as f:
1334         f.write("%u\n" % os.getpid())
1335
1336
1337 def rebase_tree(rebase_url, rebase_branch="master"):
1338     rebase_remote = "rebaseon"
1339     do_print("Rebasing on %s" % rebase_url)
1340     run_cmd("git describe HEAD", show=True, dir=test_master)
1341     run_cmd("git remote add -t %s %s %s" %
1342             (rebase_branch, rebase_remote, rebase_url),
1343             show=True, dir=test_master)
1344     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1345     if options.fix_whitespace:
1346         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1347                 (rebase_remote, rebase_branch),
1348                 show=True, dir=test_master)
1349     else:
1350         run_cmd("git rebase --force-rebase %s/%s" %
1351                 (rebase_remote, rebase_branch),
1352                 show=True, dir=test_master)
1353     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1354                    (rebase_remote, rebase_branch),
1355                    dir=test_master, output=True)
1356     if diff == '':
1357         do_print("No differences between HEAD and %s/%s - exiting" %
1358                  (rebase_remote, rebase_branch))
1359         sys.exit(0)
1360     run_cmd("git describe %s/%s" %
1361             (rebase_remote, rebase_branch),
1362             show=True, dir=test_master)
1363     run_cmd("git describe HEAD", show=True, dir=test_master)
1364     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1365             (rebase_remote, rebase_branch),
1366             show=True, dir=test_master)
1367
1368
1369 def push_to(push_url, push_branch="master"):
1370     push_remote = "pushto"
1371     do_print("Pushing to %s" % push_url)
1372     if options.mark:
1373         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1374         run_cmd("git commit --amend -c HEAD", dir=test_master)
1375         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1376         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1377     run_cmd("git remote add -t %s %s %s" %
1378             (push_branch, push_remote, push_url),
1379             show=True, dir=test_master)
1380     run_cmd("git push %s +HEAD:%s" %
1381             (push_remote, push_branch),
1382             show=True, dir=test_master)
1383
1384
1385 def send_email(subject, text, log_tar):
1386     if options.email is None:
1387         do_print("not sending email because the recipient is not set")
1388         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1389                  (subject, text))
1390         return
1391     outer = MIMEMultipart()
1392     outer['Subject'] = subject
1393     outer['To'] = options.email
1394     outer['From'] = options.email_from
1395     outer['Date'] = email.utils.formatdate(localtime=True)
1396     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1397     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1398     if options.attach_logs:
1399         with open(log_tar, 'rb') as fp:
1400             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1401         # Set the filename parameter
1402         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1403         outer.attach(msg)
1404     content = outer.as_string()
1405     s = smtplib.SMTP(options.email_server)
1406     email_user = os.getenv('SMTP_USERNAME')
1407     email_password = os.getenv('SMTP_PASSWORD')
1408     if email_user is not None:
1409         s.starttls()
1410         s.login(email_user, email_password)
1411
1412     s.sendmail(options.email_from, [options.email], content)
1413     s.set_debuglevel(1)
1414     s.quit()
1415
1416
1417 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1418                   elapsed_time, log_base=None, add_log_tail=True):
1419     '''send an email to options.email about the failure'''
1420     elapsed_minutes = elapsed_time / 60.0
1421     if log_base is None:
1422         log_base = gitroot
1423     text = '''
1424 Dear Developer,
1425
1426 Your autobuild on %s failed after %.1f minutes
1427 when trying to test %s with the following error:
1428
1429    %s
1430
1431 the autobuild has been abandoned. Please fix the error and resubmit.
1432
1433 A summary of the autobuild process is here:
1434
1435   %s/autobuild.log
1436 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1437
1438     if options.restrict_tests:
1439         text += """
1440 The build was restricted to tests matching %s\n""" % options.restrict_tests
1441
1442     if failed_task != 'rebase':
1443         text += '''
1444 You can see logs of the failed task here:
1445
1446   %s/%s.stdout
1447   %s/%s.stderr
1448
1449 or you can get full logs of all tasks in this job here:
1450
1451   %s/logs.tar.gz
1452
1453 The top commit for the tree that was built was:
1454
1455 %s
1456
1457 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1458
1459     if add_log_tail:
1460         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1461         lines = f.readlines()
1462         log_tail = "".join(lines[-50:])
1463         num_lines = len(lines)
1464         if num_lines < 50:
1465             # Also include stderr (compile failures) if < 50 lines of stdout
1466             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1467             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1468
1469         text += '''
1470 The last 50 lines of log messages:
1471
1472 %s
1473     ''' % log_tail
1474         f.close()
1475
1476     logs = os.path.join(gitroot, 'logs.tar.gz')
1477     send_email('autobuild[%s] failure on %s for task %s during %s'
1478                % (options.branch, platform.node(), failed_task, failed_stage),
1479                text, logs)
1480
1481
1482 def email_success(elapsed_time, log_base=None):
1483     '''send an email to options.email about a successful build'''
1484     if log_base is None:
1485         log_base = gitroot
1486     text = '''
1487 Dear Developer,
1488
1489 Your autobuild on %s has succeeded after %.1f minutes.
1490
1491 ''' % (platform.node(), elapsed_time / 60.)
1492
1493     if options.restrict_tests:
1494         text += """
1495 The build was restricted to tests matching %s\n""" % options.restrict_tests
1496
1497     if options.keeplogs:
1498         text += '''
1499
1500 you can get full logs of all tasks in this job here:
1501
1502   %s/logs.tar.gz
1503
1504 ''' % log_base
1505
1506     text += '''
1507 The top commit for the tree that was built was:
1508
1509 %s
1510 ''' % top_commit_msg
1511
1512     logs = os.path.join(gitroot, 'logs.tar.gz')
1513     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1514                text, logs)
1515
1516
1517 # get the top commit message, for emails
1518 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1519
1520 try:
1521     if options.skip_dependencies:
1522         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1523     else:
1524         os.makedirs(testbase)
1525 except Exception as reason:
1526     raise Exception("Unable to create %s : %s" % (testbase, reason))
1527 cleanup_list.append(testbase)
1528
1529 if options.daemon:
1530     logfile = os.path.join(testbase, "log")
1531     do_print("Forking into the background, writing progress to %s" % logfile)
1532     daemonize(logfile)
1533
1534 write_pidfile(gitroot + "/autobuild.pid")
1535
1536 start_time = time.time()
1537
1538 while True:
1539     try:
1540         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1541         os.makedirs(test_tmpdir)
1542         # The waf uninstall code removes empty directories all the way
1543         # up the tree.  Creating a file in test_tmpdir stops it from
1544         # being removed.
1545         run_cmd("touch %s" % os.path.join(test_tmpdir,
1546                                           ".directory-is-not-empty"), show=True)
1547         run_cmd("stat %s" % test_tmpdir, show=True)
1548         run_cmd("stat %s" % testbase, show=True)
1549         if options.skip_dependencies:
1550             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1551         else:
1552             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1553     except Exception:
1554         cleanup()
1555         raise
1556
1557     try:
1558         if options.rebase is not None:
1559             rebase_tree(options.rebase, rebase_branch=options.branch)
1560     except Exception:
1561         cleanup_list.append(gitroot + "/autobuild.pid")
1562         cleanup()
1563         elapsed_time = time.time() - start_time
1564         email_failure(-1, 'rebase', 'rebase', 'rebase',
1565                       'rebase on %s failed' % options.branch,
1566                       elapsed_time, log_base=options.log_base)
1567         sys.exit(1)
1568
1569     try:
1570         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1571         if options.tail:
1572             blist.start_tail()
1573         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1574         if status != 0 or errstr != "retry":
1575             break
1576         cleanup(do_raise=True)
1577     except Exception:
1578         cleanup()
1579         raise
1580
1581 cleanup_list.append(gitroot + "/autobuild.pid")
1582
1583 do_print(errstr)
1584
1585 blist.kill_kids()
1586 if options.tail:
1587     do_print("waiting for tail to flush")
1588     time.sleep(1)
1589
1590 elapsed_time = time.time() - start_time
1591 if status == 0:
1592     if options.passcmd is not None:
1593         do_print("Running passcmd: %s" % options.passcmd)
1594         run_cmd(options.passcmd, dir=test_master)
1595     if options.pushto is not None:
1596         push_to(options.pushto, push_branch=options.branch)
1597     if options.keeplogs or options.attach_logs:
1598         blist.tarlogs("logs.tar.gz")
1599         do_print("Logs in logs.tar.gz")
1600     if options.always_email:
1601         email_success(elapsed_time, log_base=options.log_base)
1602     blist.remove_logs()
1603     cleanup()
1604     do_print(errstr)
1605     sys.exit(0)
1606
1607 # something failed, gather a tar of the logs
1608 blist.tarlogs("logs.tar.gz")
1609
1610 if options.email is not None:
1611     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1612                   elapsed_time, log_base=options.log_base)
1613 else:
1614     elapsed_minutes = elapsed_time / 60.0
1615     print('''
1616
1617 ####################################################################
1618
1619 AUTOBUILD FAILURE
1620
1621 Your autobuild[%s] on %s failed after %.1f minutes
1622 when trying to test %s with the following error:
1623
1624    %s
1625
1626 the autobuild has been abandoned. Please fix the error and resubmit.
1627
1628 ####################################################################
1629
1630 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1631
1632 cleanup()
1633 do_print(errstr)
1634 do_print("Logs in logs.tar.gz")
1635 sys.exit(status)