script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.exists(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  58 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  59                   default=def_testbase)
  60 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  61                   default=None)
  62 parser.add_option("--passcmd", help="command to run on success", default=None)
  63 parser.add_option("--verbose", help="show all commands as they are run",
  64                   default=False, action="store_true")
  65 parser.add_option("--rebase", help="rebase on the given tree before testing",
  66                   default=None, type='str')
  67 parser.add_option("--pushto", help="push to a git url on success",
  68                   default=None, type='str')
  69 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  70                   default=False, action="store_true")
  71 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  72                   default=False, action="store_true")
  73 parser.add_option("--retry", help="automatically retry if master changes",
  74                   default=False, action="store_true")
  75 parser.add_option("--email", help="send email to the given address on failure",
  76                   type='str', default=None)
  77 parser.add_option("--email-from", help="send email from the given address",
  78                   type='str', default="autobuild@samba.org")
  79 parser.add_option("--email-server", help="send email via the given server",
  80                   type='str', default='localhost')
  81 parser.add_option("--always-email", help="always send email, even on success",
  82                   action="store_true")
  83 parser.add_option("--daemon", help="daemonize after initial setup",
  84                   action="store_true")
  85 parser.add_option("--branch", help="the branch to work on (default=master)",
  86                   default="master", type='str')
  87 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  88                   default=gitroot, type='str')
  89 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  90                   default=False, action="store_true")
  91 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  92                   default='')
  93 parser.add_option("--enable-coverage", dest='enable_coverage',
  94                   action="store_const", const='--enable-coverage', default='',
  95                   help="Add --enable-coverage option while configure")
  96
  97 (options, args) = parser.parse_args()
  98
  99 if options.retry:
 100     if options.rebase is None:
 101         raise Exception('You can only use --retry if you also rebase')
 102
 103 if options.full_testbase is not None:
 104     testbase = options.full_testbase
 105 else:
 106     testbase = "%s/b%u" % (options.testbase, os.getpid())
 107 test_master = "%s/master" % testbase
 108 test_prefix = "%s/prefix" % testbase
 109 test_tmpdir = "%s/tmp" % testbase
 110 os.environ['TMPDIR'] = test_tmpdir
 111
 112 if options.enable_coverage:
 113     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 114 else:
 115     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 116
 117 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 118
 119 if args:
 120     # If we are only running specific test,
 121     # do not sleep randomly to wait for it to start
 122     def random_sleep(low, high):
 123         return 'sleep 1'
 124 else:
 125     def random_sleep(low, high):
 126         return 'sleep {}'.format(random.randint(low, high))
 127
 128 cleanup_list = []
 129
 130 builddirs = {
 131     "ctdb": "ctdb",
 132     "ldb": "lib/ldb",
 133     "tdb": "lib/tdb",
 134     "talloc": "lib/talloc",
 135     "replace": "lib/replace",
 136     "tevent": "lib/tevent",
 137     "pidl": "pidl",
 138     "docs-xml": "docs-xml"
 139 }
 140
 141 ctdb_configure_params = " --enable-developer ${PREFIX}"
 142 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 143
 144 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 145 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 146 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 147 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 148 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 149 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 150 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 151
 152
 153 def format_option(name, value=None):
 154     """Format option as str list."""
 155     if value is None:  # boolean option
 156         return [name]
 157     if not isinstance(value, list):  # single value option
 158         value = [value]
 159     # repeatable option
 160     return ['{}={}'.format(name, item) for item in value]
 161
 162
 163 def make_test(
 164         cmd='make testonly',
 165         FAIL_IMMEDIATELY=1,
 166         INJECT_SELFTEST_PREFIX=1,
 167         TESTS='',
 168         include_envs=None,
 169         exclude_envs=None):
 170
 171     test_options = []
 172     if include_envs:
 173         test_options = format_option('--include-env', include_envs)
 174     if exclude_envs:
 175         test_options = format_option('--exclude-env', exclude_envs)
 176     if test_options:
 177         # join envs options to original test options
 178         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 179
 180     _options = []
 181     if FAIL_IMMEDIATELY:
 182         _options.append('FAIL_IMMEDIATELY=1')
 183     if TESTS:
 184         _options.append("TESTS='{}'".format(TESTS))
 185
 186     if INJECT_SELFTEST_PREFIX:
 187         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 188         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 189
 190     return ' '.join([cmd] + _options)
 191
 192
 193 # When updating this list, also update .gitlab-ci.yml to add the job
 194 # and to make it a dependency of 'page' for the coverage report.
 195
 196 tasks = {
 197     "ctdb": {
 198         "sequence": [
 199             ("random-sleep", random_sleep(300, 900)),
 200             ("configure", "./configure " + ctdb_configure_params),
 201             ("make", "make all"),
 202             ("install", "make install"),
 203             ("test", "make autotest"),
 204             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 205             ("clean", "make clean"),
 206         ],
 207     },
 208     "docs-xml": {
 209         "sequence": [
 210             ("random-sleep", random_sleep(300, 900)),
 211             ("autoconf", "autoconf"),
 212             ("configure", "./configure"),
 213             ("make", "make html htmlman"),
 214             ("clean", "make clean"),
 215         ],
 216     },
 217
 218     "samba-def-build": {
 219         "git-clone-required": True,
 220         "sequence": [
 221             ("configure", "./configure.developer" + samba_configure_params),
 222             ("make", "make -j"),
 223             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 224             ("chmod-R-a-w", "chmod -R a-w ."),
 225         ],
 226     },
 227
 228     "samba-mit-build": {
 229         "git-clone-required": True,
 230         "sequence": [
 231             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 232             ("make", "make -j"),
 233             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 234             ("chmod-R-a-w", "chmod -R a-w ."),
 235         ],
 236     },
 237
 238     "samba-nt4-build": {
 239         "git-clone-required": True,
 240         "sequence": [
 241             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 242             ("make", "make -j"),
 243             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 244             ("chmod-R-a-w", "chmod -R a-w ."),
 245         ],
 246     },
 247
 248     "samba-h5l-build": {
 249         "git-clone-required": True,
 250         "sequence": [
 251             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 252             ("make", "make -j"),
 253             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 254             ("chmod-R-a-w", "chmod -R a-w ."),
 255         ],
 256     },
 257
 258     "samba-no-opath-build": {
 259         "git-clone-required": True,
 260         "sequence": [
 261             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 262             ("make", "make -j"),
 263             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 264             ("chmod-R-a-w", "chmod -R a-w ."),
 265         ],
 266     },
 267
 268     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 269     "samba": {
 270         "sequence": [
 271             ("random-sleep", random_sleep(300, 900)),
 272             ("configure", "./configure.developer" + samba_configure_params),
 273             ("make", "make -j"),
 274             ("test", make_test(exclude_envs=[
 275             "none",
 276             "nt4_dc",
 277             "nt4_dc_smb1",
 278             "nt4_dc_smb1_done",
 279             "nt4_dc_schannel",
 280             "nt4_member",
 281             "ad_dc",
 282             "ad_dc_smb1",
 283             "ad_dc_smb1_done",
 284             "ad_dc_backup",
 285             "ad_dc_ntvfs",
 286             "ad_dc_default",
 287             "ad_dc_default_smb1",
 288             "ad_dc_slowtests",
 289             "ad_dc_no_nss",
 290             "ad_dc_no_ntlm",
 291             "fl2003dc",
 292             "fl2008dc",
 293             "fl2008r2dc",
 294             "ad_member",
 295             "ad_member_idmap_rid",
 296             "ad_member_idmap_ad",
 297             "ad_member_rfc2307",
 298             "chgdcpass",
 299             "vampire_2000_dc",
 300             "fl2000dc",
 301             "fileserver",
 302             "fileserver_smb1",
 303             "fileserver_smb1_done",
 304             "maptoguest",
 305             "simpleserver",
 306             "backupfromdc",
 307             "restoredc",
 308             "renamedc",
 309             "offlinebackupdc",
 310             "labdc",
 311             "preforkrestartdc",
 312             "proclimitdc",
 313             "promoted_dc",
 314             "vampire_dc",
 315             "rodc",
 316             "ad_dc_default",
 317             "ad_dc_default_smb1",
 318             "ad_dc_default_smb1_done",
 319             "ad_dc_slowtests",
 320             "schema_pair_dc",
 321             "schema_dc",
 322             "clusteredmember",
 323             ])),
 324             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 325             ("lcov", LCOV_CMD),
 326             ("install", "make install"),
 327             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 328             ("clean", "make clean"),
 329         ],
 330     },
 331
 332     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 333     "samba-mitkrb5": {
 334         "sequence": [
 335             ("random-sleep", random_sleep(300, 900)),
 336             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 337             ("make", "make -j"),
 338             ("test", make_test(exclude_envs=[
 339             "none",
 340             "nt4_dc",
 341             "nt4_dc_smb1",
 342             "nt4_dc_smb1_done",
 343             "nt4_dc_schannel",
 344             "nt4_member",
 345             "ad_dc",
 346             "ad_dc_smb1",
 347             "ad_dc_smb1_done",
 348             "ad_dc_backup",
 349             "ad_dc_ntvfs",
 350             "ad_dc_default",
 351             "ad_dc_default_smb1",
 352             "ad_dc_default_smb1_done",
 353             "ad_dc_slowtests",
 354             "ad_dc_no_nss",
 355             "ad_dc_no_ntlm",
 356             "fl2003dc",
 357             "fl2008dc",
 358             "fl2008r2dc",
 359             "ad_member",
 360             "ad_member_idmap_rid",
 361             "ad_member_idmap_ad",
 362             "ad_member_rfc2307",
 363             "chgdcpass",
 364             "vampire_2000_dc",
 365             "fl2000dc",
 366             "fileserver",
 367             "fileserver_smb1",
 368             "fileserver_smb1_done",
 369             "maptoguest",
 370             "simpleserver",
 371             "backupfromdc",
 372             "restoredc",
 373             "renamedc",
 374             "offlinebackupdc",
 375             "labdc",
 376             "preforkrestartdc",
 377             "proclimitdc",
 378             "promoted_dc",
 379             "vampire_dc",
 380             "rodc",
 381             "ad_dc_default",
 382             "ad_dc_default_smb1",
 383             "ad_dc_default_smb1_done",
 384             "ad_dc_slowtests",
 385             "schema_pair_dc",
 386             "schema_dc",
 387             "clusteredmember",
 388             ])),
 389             ("lcov", LCOV_CMD),
 390             ("install", "make install"),
 391             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 392             ("clean", "make clean"),
 393         ],
 394     },
 395
 396     "samba-nt4": {
 397         "dependency": "samba-nt4-build",
 398         "sequence": [
 399             ("random-sleep", random_sleep(300, 900)),
 400             ("test", make_test(include_envs=[
 401             "nt4_dc",
 402             "nt4_dc_smb1",
 403             "nt4_dc_smb1_done",
 404             "nt4_dc_schannel",
 405             "nt4_member",
 406             "simpleserver",
 407             ])),
 408             ("lcov", LCOV_CMD),
 409             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 410         ],
 411     },
 412
 413     "samba-fileserver": {
 414         "dependency": "samba-h5l-build",
 415         "sequence": [
 416             ("random-sleep", random_sleep(300, 900)),
 417             ("test", make_test(include_envs=[
 418             "fileserver",
 419             "fileserver_smb1",
 420             "fileserver_smb1_done",
 421             "maptoguest",
 422             "ktest", # ktest is also tested in samba and samba-mitkrb5
 423                      # but is tested here against a system Heimdal
 424             ])),
 425             ("lcov", LCOV_CMD),
 426             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 427         ],
 428     },
 429
 430     "samba-admem": {
 431         "dependency": "samba-def-build",
 432         "sequence": [
 433             ("random-sleep", random_sleep(300, 900)),
 434             ("test", make_test(include_envs=[
 435             "ad_member",
 436             "ad_member_idmap_rid",
 437             "ad_member_idmap_ad",
 438             "ad_member_rfc2307",
 439             ])),
 440             ("lcov", LCOV_CMD),
 441             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 442         ],
 443     },
 444
 445     "samba-no-opath1": {
 446         "dependency": "samba-no-opath-build",
 447         "sequence": [
 448             ("random-sleep", random_sleep(300, 900)),
 449             ("test", make_test(
 450                 cmd="make testonly DISABLE_OPATH=1",
 451                 include_envs=[
 452                 "nt4_dc",
 453                 "nt4_dc_smb1",
 454                 "nt4_dc_smb1_done",
 455                 "nt4_dc_schannel",
 456                 "nt4_member",
 457                 "simpleserver",
 458                 ])),
 459             ("lcov", LCOV_CMD),
 460             ("check-clean-tree", "script/clean-source-tree.sh"),
 461         ],
 462     },
 463
 464     "samba-no-opath2": {
 465         "dependency": "samba-no-opath-build",
 466         "sequence": [
 467             ("random-sleep", random_sleep(300, 900)),
 468             ("test", make_test(
 469                 cmd="make testonly DISABLE_OPATH=1",
 470                 include_envs=[
 471                 "fileserver",
 472                 "fileserver_smb1",
 473                 "fileserver_smb1_done",
 474                 ])),
 475             ("lcov", LCOV_CMD),
 476             ("check-clean-tree", "script/clean-source-tree.sh"),
 477         ],
 478     },
 479
 480     "samba-ad-dc-1": {
 481         "dependency": "samba-def-build",
 482         "sequence": [
 483             ("random-sleep", random_sleep(1, 1)),
 484             ("test", make_test(include_envs=[
 485             "ad_dc",
 486             "ad_dc_smb1",
 487             "ad_dc_smb1_done",
 488             "ad_dc_no_nss",
 489             "ad_dc_no_ntlm",
 490             ])),
 491             ("lcov", LCOV_CMD),
 492             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 493         ],
 494     },
 495
 496     "samba-ad-dc-2": {
 497         "dependency": "samba-def-build",
 498         "sequence": [
 499             ("random-sleep", random_sleep(1, 1)),
 500             ("test", make_test(include_envs=[
 501             "vampire_dc",
 502             "vampire_2000_dc",
 503             "rodc",
 504             ])),
 505             ("lcov", LCOV_CMD),
 506             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 507         ],
 508     },
 509
 510     "samba-ad-dc-3": {
 511         "dependency": "samba-def-build",
 512         "sequence": [
 513             ("random-sleep", random_sleep(1, 1)),
 514             ("test", make_test(include_envs=[
 515             "promoted_dc",
 516             "chgdcpass",
 517             "preforkrestartdc",
 518             "proclimitdc",
 519             ])),
 520             ("lcov", LCOV_CMD),
 521             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 522         ],
 523     },
 524
 525     "samba-ad-dc-4a": {
 526         "dependency": "samba-def-build",
 527         "sequence": [
 528             ("random-sleep", random_sleep(1, 1)),
 529             ("test", make_test(include_envs=[
 530             "fl2000dc",
 531             "fl2003dc",
 532             ])),
 533             ("lcov", LCOV_CMD),
 534             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 535         ],
 536     },
 537     "samba-ad-dc-4b": {
 538         "dependency": "samba-def-build",
 539         "sequence": [
 540             ("random-sleep", random_sleep(1, 1)),
 541             ("test", make_test(include_envs=[
 542             "fl2008dc",
 543             "fl2008r2dc",
 544             ])),
 545             ("lcov", LCOV_CMD),
 546             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 547         ],
 548     },
 549
 550     "samba-ad-dc-5": {
 551         "dependency": "samba-def-build",
 552         "sequence": [
 553             ("random-sleep", random_sleep(1, 1)),
 554             ("test", make_test(include_envs=[
 555             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 556             ("lcov", LCOV_CMD),
 557             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 558         ],
 559     },
 560
 561     "samba-ad-dc-6": {
 562         "dependency": "samba-def-build",
 563         "sequence": [
 564             ("random-sleep", random_sleep(1, 1)),
 565             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 566             ("lcov", LCOV_CMD),
 567             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 568         ],
 569     },
 570
 571     "samba-schemaupgrade": {
 572         "dependency": "samba-def-build",
 573         "sequence": [
 574             ("random-sleep", random_sleep(1, 1)),
 575             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 576             ("lcov", LCOV_CMD),
 577             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 578         ],
 579     },
 580
 581     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 582     # This is currently the longest task, so we don't randomly delay it.
 583     "samba-ad-dc-ntvfs": {
 584         "dependency": "samba-def-build",
 585         "sequence": [
 586             ("random-sleep", random_sleep(1, 1)),
 587             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 588             ("lcov", LCOV_CMD),
 589             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 590         ],
 591     },
 592
 593     # Test fips compliance
 594     "samba-fips": {
 595         "dependency": "samba-mit-build",
 596         "sequence": [
 597             ("random-sleep", random_sleep(1, 1)),
 598             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 599             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 600             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 601         ],
 602     },
 603
 604     # run the backup/restore testenvs separately as they're fairly standalone
 605     # (and CI seems to max out at ~3 different DCs running at once)
 606     "samba-ad-back1": {
 607         "dependency": "samba-def-build",
 608         "sequence": [
 609             ("random-sleep", random_sleep(300, 900)),
 610             ("test", make_test(include_envs=[
 611             "backupfromdc",
 612             "restoredc",
 613             "renamedc",
 614             ])),
 615             ("lcov", LCOV_CMD),
 616             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 617         ],
 618     },
 619     "samba-ad-back2": {
 620         "dependency": "samba-def-build",
 621         "sequence": [
 622             ("random-sleep", random_sleep(300, 900)),
 623             ("test", make_test(include_envs=[
 624             "backupfromdc",
 625             "offlinebackupdc",
 626             "labdc",
 627             ])),
 628             ("lcov", LCOV_CMD),
 629             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 630         ],
 631     },
 632
 633     "samba-admem-mit": {
 634         "dependency": "samba-mit-build",
 635         "sequence": [
 636             ("random-sleep", random_sleep(1, 1)),
 637             ("test", make_test(include_envs=[
 638             "ad_member",
 639             "ad_member_idmap_rid",
 640             "ad_member_idmap_ad",
 641             "ad_member_rfc2307",
 642             ])),
 643             ("lcov", LCOV_CMD),
 644             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 645         ],
 646     },
 647
 648     "samba-ad-dc-1-mitkrb5": {
 649         "dependency": "samba-mit-build",
 650         "sequence": [
 651             ("random-sleep", random_sleep(1, 1)),
 652             ("test", make_test(include_envs=[
 653             "ad_dc",
 654             "ad_dc_smb1",
 655             "ad_dc_smb1_done",
 656             "ad_dc_no_nss",
 657             "ad_dc_no_ntlm",
 658             ])),
 659             ("lcov", LCOV_CMD),
 660             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 661         ],
 662     },
 663
 664     "samba-ad-dc-4a-mitkrb5": {
 665         "dependency": "samba-mit-build",
 666         "sequence": [
 667             ("random-sleep", random_sleep(1, 1)),
 668             ("test", make_test(include_envs=[
 669             "fl2000dc",
 670             "fl2003dc",
 671             ])),
 672             ("lcov", LCOV_CMD),
 673             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 674         ],
 675     },
 676     "samba-ad-dc-4b-mitkrb5": {
 677         "dependency": "samba-mit-build",
 678         "sequence": [
 679             ("random-sleep", random_sleep(1, 1)),
 680             ("test", make_test(include_envs=[
 681             "fl2000dc",
 682             "fl2003dc",
 683             ])),
 684             ("lcov", LCOV_CMD),
 685             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 686         ],
 687     },
 688
 689     "samba-test-only": {
 690         "sequence": [
 691             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 692             ("make", "make -j"),
 693             ("test", make_test(TESTS="${TESTS}")),
 694             ("lcov", LCOV_CMD),
 695         ],
 696     },
 697
 698     # Test cross-compile infrastructure
 699     "samba-xc": {
 700         "sequence": [
 701             ("random-sleep", random_sleep(900, 1500)),
 702             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 703             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 704             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 705             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 706             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 707             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 708             ("compare-results", "script/compare_cc_results.py "
 709             "./bin/c4che/default{} "
 710             "./bin-xe/c4che/default{} "
 711             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 712             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 713             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 714             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 715             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 716             " = \"'1234'\"".format(CACHE_SUFFIX)),
 717             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 718             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 719             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 720             " ; test $? -ne 0"),
 721         ],
 722     },
 723
 724     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 725     "samba-o3": {
 726         "sequence": [
 727             ("random-sleep", random_sleep(300, 900)),
 728             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 729             ("make", "make -j"),
 730             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 731             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 732             ("lcov", LCOV_CMD),
 733             ("install", "make install"),
 734             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 735             ("clean", "make clean"),
 736         ],
 737     },
 738
 739     "samba-ctdb": {
 740         "sequence": [
 741             ("random-sleep", random_sleep(900, 1500)),
 742
 743         # make sure we have tdb around:
 744             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 745             ("tdb-make", "cd lib/tdb && make"),
 746             ("tdb-install", "cd lib/tdb && make install"),
 747
 748         # build samba with cluster support (also building ctdb):
 749             ("samba-configure",
 750          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 751          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 752          "./configure.developer ${PREFIX} "
 753          "--with-selftest-prefix=./bin/ab "
 754          "--enable-clangdb "
 755          "--with-cluster-support "
 756          "--without-ad-dc "
 757          "--bundled-libraries=!tdb"),
 758             ("samba-make", "make"),
 759             ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 760             ("samba-install", "make install"),
 761             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 762
 763             ("test", make_test(
 764                 cmd='make test',
 765                 INJECT_SELFTEST_PREFIX=0,
 766                 include_envs=["clusteredmember"])
 767             ),
 768
 769         # clean up:
 770             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 771             ("clean", "make clean"),
 772             ("ctdb-clean", "cd ./ctdb && make clean"),
 773         ],
 774     },
 775
 776     "samba-libs": {
 777         "sequence": [
 778             ("random-sleep", random_sleep(300, 900)),
 779             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 780             ("talloc-make", "cd lib/talloc && make"),
 781             ("talloc-install", "cd lib/talloc && make install"),
 782
 783             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 784             ("tdb-make", "cd lib/tdb && make"),
 785             ("tdb-install", "cd lib/tdb && make install"),
 786
 787             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 788             ("tevent-make", "cd lib/tevent && make"),
 789             ("tevent-install", "cd lib/tevent && make install"),
 790
 791             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 792             ("ldb-make", "cd lib/ldb && make"),
 793             ("ldb-install", "cd lib/ldb && make install"),
 794
 795             ("nondevel-configure", "./configure ${PREFIX}"),
 796             ("nondevel-make", "make -j"),
 797             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 798             ("nondevel-install", "make install"),
 799             ("nondevel-dist", "make dist"),
 800
 801         # retry with all modules shared
 802             ("allshared-distclean", "make distclean"),
 803             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 804             ("allshared-make", "make -j"),
 805         ],
 806     },
 807
 808     "samba-fuzz": {
 809         "sequence": [
 810         # build the fuzzers (static) via the oss-fuzz script
 811             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 812             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 813         ],
 814     },
 815
 816     # * Test smbd and smbtorture can build semi-static
 817     #
 818     # * Test Samba without python still builds.
 819     #
 820     # When this test fails due to more use of Python, the expectations
 821     # is that the newly failing part of the code should be disabled
 822     # when --disable-python is set (rather than major work being done
 823     # to support this environment).
 824     #
 825     # The target here is for vendors shipping a minimal smbd.
 826     "samba-minimal-smbd": {
 827         "sequence": [
 828             ("random-sleep", random_sleep(300, 900)),
 829
 830         # build with all modules static
 831             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 832             ("allstatic-make", "make -j"),
 833             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 834             ("allstatic-lcov", LCOV_CMD),
 835
 836         # retry with nonshared smbd and smbtorture
 837             ("nonshared-distclean", "make distclean"),
 838             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 839             ("nonshared-make", "make -j"),
 840             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 841             # TODO ("nonshared-lcov", LCOV_CMD),
 842
 843             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 844             ("clean", "make clean"),
 845         ],
 846     },
 847
 848     "samba-nopython": {
 849         "sequence": [
 850             ("random-sleep", random_sleep(300, 900)),
 851
 852             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 853             ("make", "make -j"),
 854             ("find-python", "script/find_python.sh ${PREFIX}"),
 855             ("test", "make test-nopython"),
 856             ("lcov", LCOV_CMD),
 857             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 858             ("clean", "make clean"),
 859
 860             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 861             ("talloc-make", "cd lib/talloc && make"),
 862             ("talloc-install", "cd lib/talloc && make install"),
 863
 864             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 865             ("tdb-make", "cd lib/tdb && make"),
 866             ("tdb-install", "cd lib/tdb && make install"),
 867
 868             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 869             ("tevent-make", "cd lib/tevent && make"),
 870             ("tevent-install", "cd lib/tevent && make install"),
 871
 872             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 873             ("ldb-make", "cd lib/ldb && make"),
 874             ("ldb-install", "cd lib/ldb && make install"),
 875
 876         # retry against installed library packages, but no required modules
 877             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 878             ("libs-make", "make -j"),
 879             ("libs-install", "make install"),
 880             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 881             ("libs-clean", "make clean"),
 882
 883         ],
 884     },
 885
 886     "ldb": {
 887         "sequence": [
 888             ("random-sleep", random_sleep(60, 600)),
 889             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 890             ("make", "make"),
 891             ("install", "make install"),
 892             ("test", "make test"),
 893             ("lcov", LCOV_CMD),
 894             ("clean", "make clean"),
 895             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 896             ("make-no-lmdb", "make"),
 897             ("test-no-lmdb", "make test"),
 898             ("lcov-no-lmdb", LCOV_CMD),
 899             ("install-no-lmdb", "make install"),
 900             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 901             ("distcheck", "make distcheck"),
 902             ("clean", "make clean"),
 903         ],
 904     },
 905
 906     "tdb": {
 907         "sequence": [
 908             ("random-sleep", random_sleep(60, 600)),
 909             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 910             ("make", "make"),
 911             ("install", "make install"),
 912             ("test", "make test"),
 913             ("lcov", LCOV_CMD),
 914             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 915             ("distcheck", "make distcheck"),
 916             ("clean", "make clean"),
 917         ],
 918     },
 919
 920     "talloc": {
 921         "sequence": [
 922             ("random-sleep", random_sleep(60, 600)),
 923             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 924             ("make", "make"),
 925             ("install", "make install"),
 926             ("test", "make test"),
 927             ("lcov", LCOV_CMD),
 928             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 929             ("distcheck", "make distcheck"),
 930             ("clean", "make clean"),
 931         ],
 932     },
 933
 934     "replace": {
 935         "sequence": [
 936             ("random-sleep", random_sleep(60, 600)),
 937             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 938             ("make", "make"),
 939             ("install", "make install"),
 940             ("test", "make test"),
 941             ("lcov", LCOV_CMD),
 942             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 943             ("distcheck", "make distcheck"),
 944             ("clean", "make clean"),
 945         ],
 946     },
 947
 948     "tevent": {
 949         "sequence": [
 950             ("random-sleep", random_sleep(60, 600)),
 951             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 952             ("make", "make"),
 953             ("install", "make install"),
 954             ("test", "make test"),
 955             ("lcov", LCOV_CMD),
 956             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 957             ("distcheck", "make distcheck"),
 958             ("clean", "make clean"),
 959         ],
 960     },
 961
 962     "pidl": {
 963         "git-clone-required": True,
 964         "sequence": [
 965             ("random-sleep", random_sleep(60, 600)),
 966             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 967             ("touch", "touch *.yp"),
 968             ("make", "make"),
 969             ("test", "make test"),
 970             ("install", "make install"),
 971             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 972             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 973             ("clean", "make clean"),
 974         ],
 975     },
 976
 977     # these are useful for debugging autobuild
 978     "pass": {
 979         "sequence": [
 980             ("pass", 'echo passing && /bin/true'),
 981         ],
 982     },
 983     "fail": {
 984         "sequence": [
 985             ("fail", 'echo failing && /bin/false'),
 986         ],
 987     },
 988 }
 989
 990 defaulttasks = list(tasks.keys())
 991
 992 defaulttasks.remove("pass")
 993 defaulttasks.remove("fail")
 994 defaulttasks.remove("samba-def-build")
 995 defaulttasks.remove("samba-nt4-build")
 996 defaulttasks.remove("samba-mit-build")
 997 defaulttasks.remove("samba-h5l-build")
 998 defaulttasks.remove("samba-no-opath-build")
 999 defaulttasks.remove("samba-test-only")
1000 defaulttasks.remove("samba-fuzz")
1001 defaulttasks.remove("samba-fips")
1002 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1003     defaulttasks.remove("samba-o3")
1004
1005
1006 def do_print(msg):
1007     print("%s" % msg)
1008     sys.stdout.flush()
1009     sys.stderr.flush()
1010
1011
1012 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1013     if show is None:
1014         show = options.verbose
1015     if show:
1016         do_print("Running: '%s' in '%s'" % (cmd, dir))
1017     if output:
1018         out = check_output([cmd], shell=True, cwd=dir)
1019         return out.decode(encoding='utf-8', errors='backslashreplace')
1020     elif checkfail:
1021         return check_call(cmd, shell=True, cwd=dir)
1022     else:
1023         return call(cmd, shell=True, cwd=dir)
1024
1025 def rmdir_force(dirname, re_raise=True):
1026     try:
1027         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1028                 dirname, dirname, dirname), output=True, show=True)
1029     except CalledProcessError as e:
1030         do_print("Failed: '%s'" % (str(e)))
1031         run_cmd("tree %s" % dirname, output=True, show=True)
1032         if re_raise:
1033             raise
1034         return False
1035     return True
1036
1037 class builder(object):
1038     '''handle build of one directory'''
1039
1040     def __init__(self, name, definition):
1041         self.name = name
1042         self.dir = builddirs.get(name, '.')
1043         self.tag = self.name.replace('/', '_')
1044         self.definition = definition
1045         self.sequence = definition["sequence"]
1046         self.git_clone_required = False
1047         if "git-clone-required" in definition:
1048             self.git_clone_required = bool(definition["git-clone-required"])
1049         self.proc = None
1050         self.done = False
1051         self.next = 0
1052         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1053         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1054         if options.verbose:
1055             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1056             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1057         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1058         self.stdout = open(self.stdout_path, 'w')
1059         self.stderr = open(self.stderr_path, 'w')
1060         self.stdin  = open("/dev/null", 'r')
1061         self.builder_dir = "%s/%s" % (testbase, self.tag)
1062         self.test_source_dir = self.builder_dir
1063         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1064         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1065         self.prefix = "%s/%s" % (test_prefix, self.tag)
1066         self.consumers = []
1067         self.producer = None
1068
1069         if self.git_clone_required:
1070             assert "dependency" not in definition
1071
1072     def mark_existing(self):
1073         do_print('%s: Mark as existing dependency' % self.name)
1074         self.next = len(self.sequence)
1075         self.done = True
1076
1077     def add_consumer(self, consumer):
1078         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1079         consumer.producer = self
1080         consumer.test_source_dir = self.test_source_dir
1081         self.consumers.append(consumer)
1082
1083     def start_next(self):
1084         if self.producer is not None:
1085             if not self.producer.done:
1086                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1087                 return
1088
1089         if self.next == 0:
1090             rmdir_force(self.builder_dir)
1091             rmdir_force(self.prefix)
1092             if self.producer is not None:
1093                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1094             elif not self.git_clone_required:
1095                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1096             else:
1097                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1098
1099         if self.next == len(self.sequence):
1100             if not self.done:
1101                 do_print('%s: Completed OK' % self.name)
1102                 self.done = True
1103             if not options.nocleanup and len(self.consumers) == 0:
1104                 do_print('%s: Cleaning up' % self.name)
1105                 rmdir_force(self.builder_dir)
1106                 rmdir_force(self.prefix)
1107             for consumer in self.consumers:
1108                 if consumer.next != 0:
1109                     continue
1110                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1111                 consumer.start_next()
1112             if self.producer is not None:
1113                 self.producer.consumers.remove(self)
1114                 assert self.producer.done
1115                 self.producer.start_next()
1116             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1117             return
1118         (self.stage, self.cmd) = self.sequence[self.next]
1119         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
1120         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1121         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1122         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1123         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1124         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1125         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1126         self.cmd = self.cmd.replace("${NAME}", self.name)
1127         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1128         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1129         self.proc = Popen(self.cmd, shell=True,
1130                           close_fds=True, cwd=self.cwd,
1131                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1132         self.next += 1
1133
1134 def expand_dependencies(n):
1135     deps = list()
1136     if "dependency" in tasks[n]:
1137         depname = tasks[n]["dependency"]
1138         assert depname in tasks
1139         sdeps = expand_dependencies(depname)
1140         assert n not in sdeps
1141         for sdep in sdeps:
1142             deps.append(sdep)
1143         deps.append(depname)
1144     return deps
1145
1146
1147 class buildlist(object):
1148     '''handle build of multiple directories'''
1149
1150     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1151         self.tail_proc = None
1152         self.retry = None
1153         if not tasknames:
1154             if options.restrict_tests:
1155                 tasknames = ["samba-test-only"]
1156             else:
1157                 tasknames = defaulttasks
1158
1159         given_tasknames = tasknames.copy()
1160         implicit_tasknames = []
1161         for n in given_tasknames:
1162             deps = expand_dependencies(n)
1163             for dep in deps:
1164                 if dep in given_tasknames:
1165                     continue
1166                 if dep in implicit_tasknames:
1167                     continue
1168                 implicit_tasknames.append(dep)
1169
1170         tasknames = implicit_tasknames.copy()
1171         tasknames.extend(given_tasknames)
1172         do_print("given_tasknames: %s" % given_tasknames)
1173         do_print("implicit_tasknames: %s" % implicit_tasknames)
1174         do_print("tasknames: %s" % tasknames)
1175         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1176
1177         if options.retry:
1178             rebase_remote = "rebaseon"
1179             retry_task = {
1180                     "git-clone-required": True,
1181                     "sequence": [
1182                             ("retry",
1183                             '''set -e
1184                             git remote add -t %s %s %s
1185                             git fetch %s
1186                             while :; do
1187                               sleep 60
1188                               git describe %s/%s > old_remote_branch.desc
1189                               git fetch %s
1190                               git describe %s/%s > remote_branch.desc
1191                               diff old_remote_branch.desc remote_branch.desc
1192                             done
1193                            ''' % (
1194                                rebase_branch, rebase_remote, rebase_url,
1195                                rebase_remote,
1196                                rebase_remote, rebase_branch,
1197                                rebase_remote,
1198                                rebase_remote, rebase_branch
1199                             ))]}
1200
1201             self.retry = builder('retry', retry_task)
1202             self.need_retry = False
1203
1204         if options.skip_dependencies:
1205             for b in self.tlist:
1206                 if b.name in implicit_tasknames:
1207                     b.mark_existing()
1208
1209         for b in self.tlist:
1210             do_print("b.name=%s" % b.name)
1211             if "dependency" not in b.definition:
1212                 continue
1213             depname = b.definition["dependency"]
1214             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1215             for p in self.tlist:
1216                 if p.name == depname:
1217                     p.add_consumer(b)
1218
1219     def kill_kids(self):
1220         if self.tail_proc is not None:
1221             self.tail_proc.terminate()
1222             self.tail_proc.wait()
1223             self.tail_proc = None
1224         if self.retry is not None:
1225             self.retry.proc.terminate()
1226             self.retry.proc.wait()
1227             self.retry = None
1228         for b in self.tlist:
1229             if b.proc is not None:
1230                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1231                 b.proc.terminate()
1232                 b.proc.wait()
1233                 b.proc = None
1234
1235     def wait_one(self):
1236         while True:
1237             none_running = True
1238             for b in self.tlist:
1239                 if b.proc is None:
1240                     continue
1241                 none_running = False
1242                 b.status = b.proc.poll()
1243                 if b.status is None:
1244                     continue
1245                 b.proc = None
1246                 return b
1247             if options.retry:
1248                 ret = self.retry.proc.poll()
1249                 if ret is not None:
1250                     self.need_retry = True
1251                     self.retry = None
1252                     return None
1253             if none_running:
1254                 return None
1255             time.sleep(0.1)
1256
1257     def run(self):
1258         for b in self.tlist:
1259             b.start_next()
1260         if options.retry:
1261             self.retry.start_next()
1262         while True:
1263             b = self.wait_one()
1264             if options.retry and self.need_retry:
1265                 self.kill_kids()
1266                 do_print("retry needed")
1267                 return (0, None, None, None, "retry")
1268             if b is None:
1269                 break
1270             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1271                 self.kill_kids()
1272                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1273             b.start_next()
1274         self.kill_kids()
1275         return (0, None, None, None, "All OK")
1276
1277     def write_system_info(self, filename):
1278         with open(filename, 'w') as f:
1279             for cmd in ['uname -a',
1280                         'lsb_release -a',
1281                         'free',
1282                         'mount',
1283                         'cat /proc/cpuinfo',
1284                         'cc --version',
1285                         'df -m .',
1286                         'df -m %s' % testbase]:
1287                 try:
1288                     out = run_cmd(cmd, output=True, checkfail=False)
1289                 except CalledProcessError as e:
1290                     out = "<failed: %s>" % str(e)
1291                 print('### %s' % cmd, file=f)
1292                 print(out, file=f)
1293                 print(file=f)
1294
1295     def tarlogs(self, fname):
1296         with tarfile.open(fname, "w:gz") as tar:
1297             for b in self.tlist:
1298                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1299                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1300             if os.path.exists("autobuild.log"):
1301                 tar.add("autobuild.log")
1302             filename = 'system-info.txt'
1303             self.write_system_info(filename)
1304             tar.add(filename)
1305
1306     def remove_logs(self):
1307         for b in self.tlist:
1308             os.unlink(b.stdout_path)
1309             os.unlink(b.stderr_path)
1310
1311     def start_tail(self):
1312         cmd = ["tail", "-f"]
1313         for b in self.tlist:
1314             cmd.append(b.stdout_path)
1315             cmd.append(b.stderr_path)
1316         self.tail_proc = Popen(cmd, close_fds=True)
1317
1318
1319 def cleanup(do_raise=False):
1320     if options.nocleanup:
1321         return
1322     run_cmd("stat %s || true" % test_tmpdir, show=True)
1323     run_cmd("stat %s" % testbase, show=True)
1324     do_print("Cleaning up %r" % cleanup_list)
1325     for d in cleanup_list:
1326         ok = rmdir_force(d, re_raise=False)
1327         if ok:
1328             continue
1329         if os.path.isdir(d):
1330             do_print("Killing, waiting and retry")
1331             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1332         else:
1333             do_print("Waiting and retry")
1334         time.sleep(1)
1335         rmdir_force(d, re_raise=do_raise)
1336
1337
1338 def daemonize(logfile):
1339     pid = os.fork()
1340     if pid == 0:  # Parent
1341         os.setsid()
1342         pid = os.fork()
1343         if pid != 0:  # Actual daemon
1344             os._exit(0)
1345     else:  # Grandparent
1346         os._exit(0)
1347
1348     import resource      # Resource usage information.
1349     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1350     if maxfd == resource.RLIM_INFINITY:
1351         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1352     for fd in range(0, maxfd):
1353         try:
1354             os.close(fd)
1355         except OSError:
1356             pass
1357     os.open(logfile, os.O_RDWR | os.O_CREAT)
1358     os.dup2(0, 1)
1359     os.dup2(0, 2)
1360
1361
1362 def write_pidfile(fname):
1363     '''write a pid file, cleanup on exit'''
1364     with open(fname, mode='w') as f:
1365         f.write("%u\n" % os.getpid())
1366
1367
1368 def rebase_tree(rebase_url, rebase_branch="master"):
1369     rebase_remote = "rebaseon"
1370     do_print("Rebasing on %s" % rebase_url)
1371     run_cmd("git describe HEAD", show=True, dir=test_master)
1372     run_cmd("git remote add -t %s %s %s" %
1373             (rebase_branch, rebase_remote, rebase_url),
1374             show=True, dir=test_master)
1375     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1376     if options.fix_whitespace:
1377         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1378                 (rebase_remote, rebase_branch),
1379                 show=True, dir=test_master)
1380     else:
1381         run_cmd("git rebase --force-rebase %s/%s" %
1382                 (rebase_remote, rebase_branch),
1383                 show=True, dir=test_master)
1384     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1385                    (rebase_remote, rebase_branch),
1386                    dir=test_master, output=True)
1387     if diff == '':
1388         do_print("No differences between HEAD and %s/%s - exiting" %
1389                  (rebase_remote, rebase_branch))
1390         sys.exit(0)
1391     run_cmd("git describe %s/%s" %
1392             (rebase_remote, rebase_branch),
1393             show=True, dir=test_master)
1394     run_cmd("git describe HEAD", show=True, dir=test_master)
1395     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1396             (rebase_remote, rebase_branch),
1397             show=True, dir=test_master)
1398
1399
1400 def push_to(push_url, push_branch="master"):
1401     push_remote = "pushto"
1402     do_print("Pushing to %s" % push_url)
1403     if options.mark:
1404         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1405         run_cmd("git commit --amend -c HEAD", dir=test_master)
1406         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1407         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1408     run_cmd("git remote add -t %s %s %s" %
1409             (push_branch, push_remote, push_url),
1410             show=True, dir=test_master)
1411     run_cmd("git push %s +HEAD:%s" %
1412             (push_remote, push_branch),
1413             show=True, dir=test_master)
1414
1415
1416 def send_email(subject, text, log_tar):
1417     if options.email is None:
1418         do_print("not sending email because the recipient is not set")
1419         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1420                  (subject, text))
1421         return
1422     outer = MIMEMultipart()
1423     outer['Subject'] = subject
1424     outer['To'] = options.email
1425     outer['From'] = options.email_from
1426     outer['Date'] = email.utils.formatdate(localtime=True)
1427     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1428     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1429     if options.attach_logs:
1430         with open(log_tar, 'rb') as fp:
1431             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1432         # Set the filename parameter
1433         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1434         outer.attach(msg)
1435     content = outer.as_string()
1436     s = smtplib.SMTP(options.email_server)
1437     email_user = os.getenv('SMTP_USERNAME')
1438     email_password = os.getenv('SMTP_PASSWORD')
1439     if email_user is not None:
1440         s.starttls()
1441         s.login(email_user, email_password)
1442
1443     s.sendmail(options.email_from, [options.email], content)
1444     s.set_debuglevel(1)
1445     s.quit()
1446
1447
1448 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1449                   elapsed_time, log_base=None, add_log_tail=True):
1450     '''send an email to options.email about the failure'''
1451     elapsed_minutes = elapsed_time / 60.0
1452     if log_base is None:
1453         log_base = gitroot
1454     text = '''
1455 Dear Developer,
1456
1457 Your autobuild on %s failed after %.1f minutes
1458 when trying to test %s with the following error:
1459
1460    %s
1461
1462 the autobuild has been abandoned. Please fix the error and resubmit.
1463
1464 A summary of the autobuild process is here:
1465
1466   %s/autobuild.log
1467 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1468
1469     if options.restrict_tests:
1470         text += """
1471 The build was restricted to tests matching %s\n""" % options.restrict_tests
1472
1473     if failed_task != 'rebase':
1474         text += '''
1475 You can see logs of the failed task here:
1476
1477   %s/%s.stdout
1478   %s/%s.stderr
1479
1480 or you can get full logs of all tasks in this job here:
1481
1482   %s/logs.tar.gz
1483
1484 The top commit for the tree that was built was:
1485
1486 %s
1487
1488 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1489
1490     if add_log_tail:
1491         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1492         lines = f.readlines()
1493         log_tail = "".join(lines[-50:])
1494         num_lines = len(lines)
1495         if num_lines < 50:
1496             # Also include stderr (compile failures) if < 50 lines of stdout
1497             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1498             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1499
1500         text += '''
1501 The last 50 lines of log messages:
1502
1503 %s
1504     ''' % log_tail
1505         f.close()
1506
1507     logs = os.path.join(gitroot, 'logs.tar.gz')
1508     send_email('autobuild[%s] failure on %s for task %s during %s'
1509                % (options.branch, platform.node(), failed_task, failed_stage),
1510                text, logs)
1511
1512
1513 def email_success(elapsed_time, log_base=None):
1514     '''send an email to options.email about a successful build'''
1515     if log_base is None:
1516         log_base = gitroot
1517     text = '''
1518 Dear Developer,
1519
1520 Your autobuild on %s has succeeded after %.1f minutes.
1521
1522 ''' % (platform.node(), elapsed_time / 60.)
1523
1524     if options.restrict_tests:
1525         text += """
1526 The build was restricted to tests matching %s\n""" % options.restrict_tests
1527
1528     if options.keeplogs:
1529         text += '''
1530
1531 you can get full logs of all tasks in this job here:
1532
1533   %s/logs.tar.gz
1534
1535 ''' % log_base
1536
1537     text += '''
1538 The top commit for the tree that was built was:
1539
1540 %s
1541 ''' % top_commit_msg
1542
1543     logs = os.path.join(gitroot, 'logs.tar.gz')
1544     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1545                text, logs)
1546
1547
1548 # get the top commit message, for emails
1549 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1550
1551 try:
1552     if options.skip_dependencies:
1553         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1554     else:
1555         os.makedirs(testbase)
1556 except Exception as reason:
1557     raise Exception("Unable to create %s : %s" % (testbase, reason))
1558 cleanup_list.append(testbase)
1559
1560 if options.daemon:
1561     logfile = os.path.join(testbase, "log")
1562     do_print("Forking into the background, writing progress to %s" % logfile)
1563     daemonize(logfile)
1564
1565 write_pidfile(gitroot + "/autobuild.pid")
1566
1567 start_time = time.time()
1568
1569 while True:
1570     try:
1571         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1572         os.makedirs(test_tmpdir)
1573         # The waf uninstall code removes empty directories all the way
1574         # up the tree.  Creating a file in test_tmpdir stops it from
1575         # being removed.
1576         run_cmd("touch %s" % os.path.join(test_tmpdir,
1577                                           ".directory-is-not-empty"), show=True)
1578         run_cmd("stat %s" % test_tmpdir, show=True)
1579         run_cmd("stat %s" % testbase, show=True)
1580         if options.skip_dependencies:
1581             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1582         else:
1583             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1584     except Exception:
1585         cleanup()
1586         raise
1587
1588     try:
1589         if options.rebase is not None:
1590             rebase_tree(options.rebase, rebase_branch=options.branch)
1591     except Exception:
1592         cleanup_list.append(gitroot + "/autobuild.pid")
1593         cleanup()
1594         elapsed_time = time.time() - start_time
1595         email_failure(-1, 'rebase', 'rebase', 'rebase',
1596                       'rebase on %s failed' % options.branch,
1597                       elapsed_time, log_base=options.log_base)
1598         sys.exit(1)
1599
1600     try:
1601         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1602         if options.tail:
1603             blist.start_tail()
1604         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1605         if status != 0 or errstr != "retry":
1606             break
1607         cleanup(do_raise=True)
1608     except Exception:
1609         cleanup()
1610         raise
1611
1612 cleanup_list.append(gitroot + "/autobuild.pid")
1613
1614 do_print(errstr)
1615
1616 blist.kill_kids()
1617 if options.tail:
1618     do_print("waiting for tail to flush")
1619     time.sleep(1)
1620
1621 elapsed_time = time.time() - start_time
1622 if status == 0:
1623     if options.passcmd is not None:
1624         do_print("Running passcmd: %s" % options.passcmd)
1625         run_cmd(options.passcmd, dir=test_master)
1626     if options.pushto is not None:
1627         push_to(options.pushto, push_branch=options.branch)
1628     if options.keeplogs or options.attach_logs:
1629         blist.tarlogs("logs.tar.gz")
1630         do_print("Logs in logs.tar.gz")
1631     if options.always_email:
1632         email_success(elapsed_time, log_base=options.log_base)
1633     blist.remove_logs()
1634     cleanup()
1635     do_print(errstr)
1636     sys.exit(0)
1637
1638 # something failed, gather a tar of the logs
1639 blist.tarlogs("logs.tar.gz")
1640
1641 if options.email is not None:
1642     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1643                   elapsed_time, log_base=options.log_base)
1644 else:
1645     elapsed_minutes = elapsed_time / 60.0
1646     print('''
1647
1648 ####################################################################
1649
1650 AUTOBUILD FAILURE
1651
1652 Your autobuild[%s] on %s failed after %.1f minutes
1653 when trying to test %s with the following error:
1654
1655    %s
1656
1657 the autobuild has been abandoned. Please fix the error and resubmit.
1658
1659 ####################################################################
1660
1661 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1662
1663 cleanup()
1664 do_print(errstr)
1665 do_print("Logs in logs.tar.gz")
1666 sys.exit(status)