script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, check_output, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 import random
  13 from optparse import OptionParser
  14 import smtplib
  15 import email
  16 from email.mime.text import MIMEText
  17 from email.mime.base import MIMEBase
  18 from email.mime.application import MIMEApplication
  19 from email.mime.multipart import MIMEMultipart
  20 from distutils.sysconfig import get_python_lib
  21 import platform
  22
  23 try:
  24     from waflib.Build import CACHE_SUFFIX
  25 except ImportError:
  26     sys.path.insert(0, "./third_party/waf")
  27     from waflib.Build import CACHE_SUFFIX
  28
  29
  30 os.environ["PYTHONUNBUFFERED"] = "1"
  31
  32 # This speeds up testing remarkably.
  33 os.environ['TDB_NO_FSYNC'] = '1'
  34
  35
  36 def find_git_root():
  37     '''get to the top of the git repo'''
  38     p = os.getcwd()
  39     while p != '/':
  40         if os.path.isdir(os.path.join(p, ".git")):
  41             return p
  42         p = os.path.abspath(os.path.join(p, '..'))
  43     return None
  44
  45
  46 gitroot = find_git_root()
  47 if gitroot is None:
  48     raise Exception("Failed to find git root")
  49
  50
  51 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  52
  53 parser = OptionParser()
  54 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  55 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  56 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--passcmd", help="command to run on success", default=None)
  60 parser.add_option("--verbose", help="show all commands as they are run",
  61                   default=False, action="store_true")
  62 parser.add_option("--rebase", help="rebase on the given tree before testing",
  63                   default=None, type='str')
  64 parser.add_option("--pushto", help="push to a git url on success",
  65                   default=None, type='str')
  66 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  67                   default=False, action="store_true")
  68 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  69                   default=False, action="store_true")
  70 parser.add_option("--retry", help="automatically retry if master changes",
  71                   default=False, action="store_true")
  72 parser.add_option("--email", help="send email to the given address on failure",
  73                   type='str', default=None)
  74 parser.add_option("--email-from", help="send email from the given address",
  75                   type='str', default="autobuild@samba.org")
  76 parser.add_option("--email-server", help="send email via the given server",
  77                   type='str', default='localhost')
  78 parser.add_option("--always-email", help="always send email, even on success",
  79                   action="store_true")
  80 parser.add_option("--daemon", help="daemonize after initial setup",
  81                   action="store_true")
  82 parser.add_option("--branch", help="the branch to work on (default=master)",
  83                   default="master", type='str')
  84 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  85                   default=gitroot, type='str')
  86 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  87                   default=False, action="store_true")
  88 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  89                   default='')
  90 parser.add_option("--enable-coverage", dest='enable_coverage',
  91                   action="store_const", const='--enable-coverage', default='',
  92                   help="Add --enable-coverage option while configure")
  93
  94 (options, args) = parser.parse_args()
  95
  96 if options.retry:
  97     if options.rebase is None:
  98         raise Exception('You can only use --retry if you also rebase')
  99
 100 testbase = "%s/b%u" % (options.testbase, os.getpid())
 101 test_master = "%s/master" % testbase
 102 test_prefix = "%s/prefix" % testbase
 103 test_tmpdir = "%s/tmp" % testbase
 104 os.environ['TMPDIR'] = test_tmpdir
 105
 106 if options.enable_coverage:
 107     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 108 else:
 109     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 110
 111 if args:
 112     # If we are only running specific test,
 113     # do not sleep randomly to wait for it to start
 114     def random_sleep(low, high):
 115         return 'sleep 1'
 116 else:
 117     def random_sleep(low, high):
 118         return 'sleep {}'.format(random.randint(low, high))
 119
 120 cleanup_list = []
 121
 122 builddirs = {
 123     "ctdb": "ctdb",
 124     "ldb": "lib/ldb",
 125     "tdb": "lib/tdb",
 126     "talloc": "lib/talloc",
 127     "replace": "lib/replace",
 128     "tevent": "lib/tevent",
 129     "pidl": "pidl"
 130 }
 131
 132 ctdb_configure_params = " --enable-developer ${PREFIX}"
 133 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 134
 135 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 136 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 137 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 138 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 139 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 140 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 141 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 142
 143
 144 def format_option(name, value=None):
 145     """Format option as str list."""
 146     if value is None:  # boolean option
 147         return [name]
 148     if not isinstance(value, list):  # single value option
 149         value = [value]
 150     # repeatable option
 151     return ['{}={}'.format(name, item) for item in value]
 152
 153
 154 def make_test(
 155         cmd='make test',
 156         FAIL_IMMEDIATELY=1,
 157         TESTS='',
 158         include_envs=None,
 159         exclude_envs=None):
 160
 161     test_options = []
 162     if include_envs:
 163         test_options = format_option('--include-env', include_envs)
 164     if exclude_envs:
 165         test_options = format_option('--exclude-env', exclude_envs)
 166     if test_options:
 167         # join envs options to original test options
 168         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 169
 170     _options = []
 171     if FAIL_IMMEDIATELY:
 172         _options.append('FAIL_IMMEDIATELY=1')
 173     if TESTS:
 174         _options.append("TESTS='{}'".format(TESTS))
 175
 176     return ' '.join([cmd] + _options)
 177
 178
 179 # When updating this list, also update .gitlab-ci.yml to add the job
 180 # and to make it a dependency of 'page' for the coverage report.
 181
 182 tasks = {
 183     "ctdb": [
 184         ("random-sleep", random_sleep(300, 900)),
 185         ("configure", "./configure " + ctdb_configure_params),
 186         ("make", "make all"),
 187         ("install", "make install"),
 188         ("test", "make autotest"),
 189         ("check-clean-tree", "../script/clean-source-tree.sh"),
 190         ("clean", "make clean"),
 191         ],
 192
 193     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 194     "samba": [
 195         ("random-sleep", random_sleep(300, 900)),
 196         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 197         ("make", "make -j"),
 198         ("test", make_test(exclude_envs=[
 199             "none",
 200             "nt4_dc",
 201             "nt4_dc_schannel",
 202             "nt4_member",
 203             "ad_dc",
 204             "ad_dc_backup",
 205             "ad_dc_ntvfs",
 206             "ad_dc_default",
 207             "ad_dc_slowtests",
 208             "ad_dc_no_nss",
 209             "ad_dc_no_ntlm",
 210             "fl2003dc",
 211             "fl2008dc",
 212             "fl2008r2dc",
 213             "ad_member",
 214             "ad_member_idmap_rid",
 215             "ad_member_idmap_ad",
 216             "ad_member_rfc2307",
 217             "chgdcpass",
 218             "vampire_2000_dc",
 219             "fl2000dc",
 220             "fileserver",
 221             "maptoguest",
 222             "simpleserver",
 223             "backupfromdc",
 224             "restoredc",
 225             "renamedc",
 226             "offlinebackupdc",
 227             "labdc",
 228             "preforkrestartdc",
 229             "proclimitdc",
 230             "promoted_dc",
 231             "vampire_dc",
 232             "rodc",
 233             "ad_dc_default",
 234             "ad_dc_slowtests",
 235             "schema_pair_dc",
 236             "schema_dc",
 237             "clusteredmember",
 238             ])),
 239         ("lcov", LCOV_CMD),
 240         ("install", "make install"),
 241         ("check-clean-tree", "script/clean-source-tree.sh"),
 242         ("clean", "make clean"),
 243         ],
 244
 245     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 246     "samba-mitkrb5": [
 247         ("random-sleep", random_sleep(300, 900)),
 248         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 249         ("make", "make -j"),
 250         ("test", make_test(exclude_envs=[
 251             "none",
 252             "nt4_dc",
 253             "nt4_dc_schannel",
 254             "nt4_member",
 255             "ad_dc",
 256             "ad_dc_backup",
 257             "ad_dc_ntvfs",
 258             "ad_dc_default",
 259             "ad_dc_slowtests",
 260             "ad_dc_no_nss",
 261             "ad_dc_no_ntlm",
 262             "fl2003dc",
 263             "fl2008dc",
 264             "fl2008r2dc",
 265             "ad_member",
 266             "ad_member_idmap_rid",
 267             "ad_member_idmap_ad",
 268             "ad_member_rfc2307",
 269             "chgdcpass",
 270             "vampire_2000_dc",
 271             "fl2000dc",
 272             "fileserver",
 273             "maptoguest",
 274             "simpleserver",
 275             "backupfromdc",
 276             "restoredc",
 277             "renamedc",
 278             "offlinebackupdc",
 279             "labdc",
 280             "preforkrestartdc",
 281             "proclimitdc",
 282             "promoted_dc",
 283             "vampire_dc",
 284             "rodc",
 285             "ad_dc_default",
 286             "ad_dc_slowtests",
 287             "schema_pair_dc",
 288             "schema_dc",
 289             "clusteredmember",
 290             ])),
 291         ("lcov", LCOV_CMD),
 292         ("install", "make install"),
 293         ("check-clean-tree", "script/clean-source-tree.sh"),
 294         ("clean", "make clean"),
 295         ],
 296
 297     "samba-nt4": [
 298         ("random-sleep", random_sleep(300, 900)),
 299         ("configure", "./configure.developer --without-ads --without-ad-dc --with-selftest-prefix=./bin/ab" + samba_configure_params),
 300         ("make", "make -j"),
 301         ("test", make_test(include_envs=[
 302             "nt4_dc",
 303             "nt4_dc_schannel",
 304             "nt4_member",
 305             ])),
 306         ("lcov", LCOV_CMD),
 307         ("install", "make install"),
 308         ("check-clean-tree", "script/clean-source-tree.sh"),
 309         ("clean", "make clean"),
 310         ],
 311
 312     "samba-simpleserver": [
 313         ("random-sleep", random_sleep(300, 900)),
 314         ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params),
 315         ("make", "make -j"),
 316         ("test", make_test(include_envs=[
 317             "simpleserver",
 318             ])),
 319         ("lcov", LCOV_CMD),
 320         ("check-clean-tree", "script/clean-source-tree.sh"),
 321         ],
 322
 323     "samba-fileserver": [
 324         ("random-sleep", random_sleep(300, 900)),
 325         ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab" + samba_configure_params),
 326         ("make", "make -j"),
 327         ("test", make_test(include_envs=[
 328             "fileserver",
 329             "maptoguest",
 330             "ktest", # ktest is also tested in samba and samba-mitkrb5
 331                      # but is tested here against a system Heimdal
 332             ])),
 333         ("lcov", LCOV_CMD),
 334         ("check-clean-tree", "script/clean-source-tree.sh"),
 335         ],
 336
 337     "samba-admem": [
 338         ("random-sleep", random_sleep(300, 900)),
 339         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 340         ("make", "make -j"),
 341         ("test", make_test(include_envs=[
 342             "ad_member",
 343             "ad_member_idmap_rid",
 344             "ad_member_idmap_ad",
 345             "ad_member_rfc2307",
 346             ])),
 347         ("lcov", LCOV_CMD),
 348         ("check-clean-tree", "script/clean-source-tree.sh"),
 349         ],
 350
 351     "samba-ad-dc-1": [
 352         ("random-sleep", random_sleep(1, 1)),
 353         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 354         ("make", "make -j"),
 355         ("test", make_test(include_envs=[
 356             "ad_dc",
 357             "ad_dc_no_nss",
 358             "ad_dc_no_ntlm",
 359             ])),
 360         ("lcov", LCOV_CMD),
 361         ("check-clean-tree", "script/clean-source-tree.sh"),
 362         ],
 363
 364     "samba-ad-dc-2": [
 365         ("random-sleep", random_sleep(1, 1)),
 366         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 367         ("make", "make -j"),
 368         ("test", make_test(include_envs=[
 369             "vampire_dc",
 370             "vampire_2000_dc",
 371             "rodc",
 372             ])),
 373         ("lcov", LCOV_CMD),
 374         ("check-clean-tree", "script/clean-source-tree.sh"),
 375         ],
 376
 377     "samba-ad-dc-3": [
 378         ("random-sleep", random_sleep(1, 1)),
 379         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 380         ("make", "make -j"),
 381         ("test", make_test(include_envs=[
 382             "promoted_dc",
 383             "chgdcpass",
 384             "preforkrestartdc",
 385             "proclimitdc",
 386             ])),
 387         ("lcov", LCOV_CMD),
 388         ("check-clean-tree", "script/clean-source-tree.sh"),
 389         ],
 390
 391     "samba-ad-dc-4": [
 392         ("random-sleep", random_sleep(1, 1)),
 393         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 394         ("make", "make -j"),
 395         ("test", make_test(include_envs=[
 396             "fl2000dc",
 397             "fl2003dc",
 398             "fl2008dc",
 399             "fl2008r2dc",
 400             ])),
 401         ("lcov", LCOV_CMD),
 402         ("check-clean-tree", "script/clean-source-tree.sh"),
 403         ],
 404
 405     "samba-ad-dc-5": [
 406         ("random-sleep", random_sleep(1, 1)),
 407         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 408         ("make", "make -j"),
 409         ("test", make_test(include_envs=["ad_dc_default"])),
 410         ("lcov", LCOV_CMD),
 411         ("check-clean-tree", "script/clean-source-tree.sh"),
 412         ],
 413
 414     "samba-ad-dc-6": [
 415         ("random-sleep", random_sleep(1, 1)),
 416         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 417         ("make", "make -j"),
 418         ("test", make_test(include_envs=["ad_dc_slowtests"])),
 419         ("lcov", LCOV_CMD),
 420         ("check-clean-tree", "script/clean-source-tree.sh"),
 421         ],
 422
 423     "samba-schemaupgrade": [
 424         ("random-sleep", random_sleep(1, 1)),
 425         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 426         ("make", "make -j"),
 427         ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 428         ("lcov", LCOV_CMD),
 429         ("check-clean-tree", "script/clean-source-tree.sh"),
 430         ],
 431
 432     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 433     # This is currently the longest task, so we don't randomly delay it.
 434     "samba-ad-dc-ntvfs": [
 435         ("random-sleep", random_sleep(1, 1)),
 436         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 437         ("make", "make -j"),
 438         ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 439         ("lcov", LCOV_CMD),
 440         ("check-clean-tree", "script/clean-source-tree.sh"),
 441         ],
 442
 443     # run the backup/restore testenvs separately as they're fairly standalone
 444     # (and CI seems to max out at ~8 different DCs running at once)
 445     "samba-ad-dc-backup": [
 446         ("random-sleep", random_sleep(300, 900)),
 447         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 448         ("make", "make -j"),
 449         ("test", make_test(include_envs=[
 450             "backupfromdc",
 451             "restoredc",
 452             "renamedc",
 453             "offlinebackupdc",
 454             "labdc",
 455             "ad_dc_backup",
 456             ])),
 457         ("lcov", LCOV_CMD),
 458         ("check-clean-tree", "script/clean-source-tree.sh"),
 459         ],
 460
 461     "samba-admem-mit": [
 462         ("random-sleep", random_sleep(300, 900)),
 463         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 464         ("make", "make -j"),
 465         ("test", make_test(include_envs=[
 466             "ad_member",
 467             "ad_member_idmap_rid",
 468             "ad_member_idmap_ad",
 469             "ad_member_rfc2307",
 470             ])),
 471         ("lcov", LCOV_CMD),
 472         ("check-clean-tree", "script/clean-source-tree.sh"),
 473         ],
 474
 475     "samba-ad-dc-1-mitkrb5": [
 476         ("random-sleep", random_sleep(1, 1)),
 477         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 478         ("make", "make -j"),
 479         ("test", make_test(include_envs=[
 480             "ad_dc",
 481             "ad_dc_no_nss",
 482             "ad_dc_no_ntlm",
 483             ])),
 484         ("lcov", LCOV_CMD),
 485         ("check-clean-tree", "script/clean-source-tree.sh"),
 486         ],
 487
 488     "samba-ad-dc-4-mitkrb5": [
 489         ("random-sleep", random_sleep(1, 1)),
 490         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 491         ("make", "make -j"),
 492         ("test", make_test(include_envs=[
 493             "fl2000dc",
 494             "fl2003dc",
 495             "fl2008dc",
 496             "fl2008r2dc",
 497             ])),
 498         ("lcov", LCOV_CMD),
 499         ("check-clean-tree", "script/clean-source-tree.sh"),
 500         ],
 501
 502     "samba-test-only": [
 503         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params),
 504         ("make", "make -j"),
 505         ("test", make_test(TESTS="${TESTS}")),
 506         ("lcov", LCOV_CMD),
 507         ],
 508
 509     # Test cross-compile infrastructure
 510     "samba-xc": [
 511         ("random-sleep", random_sleep(900, 1500)),
 512         ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 513         ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 514             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 515         ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 516         ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 517             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 518         ("compare-results", "script/compare_cc_results.py "
 519             "./bin/c4che/default{} "
 520             "./bin-xe/c4che/default{} "
 521             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 522         ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 523         ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 524             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 525         ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 526             " = \"'1234'\"".format(CACHE_SUFFIX)),
 527         ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 528         ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 529             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 530             " ; test $? -ne 0"),
 531         ],
 532
 533     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 534     "samba-o3": [
 535         ("random-sleep", random_sleep(300, 900)),
 536         ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params),
 537         ("make", "make -j"),
 538         ("test", make_test(cmd='make test', include_envs=["none"])),
 539         ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc"])),
 540         ("lcov", LCOV_CMD),
 541         ("install", "make install"),
 542         ("check-clean-tree", "script/clean-source-tree.sh"),
 543         ("clean", "make clean"),
 544         ],
 545
 546     "samba-ctdb": [
 547         ("random-sleep", random_sleep(900, 1500)),
 548
 549         # make sure we have tdb around:
 550         ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 551         ("tdb-make", "cd lib/tdb && make"),
 552         ("tdb-install", "cd lib/tdb && make install"),
 553
 554         # build samba with cluster support (also building ctdb):
 555         ("samba-configure",
 556          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 557          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 558          "./configure.developer ${PREFIX} "
 559          "--with-selftest-prefix=./bin/ab "
 560          "--with-cluster-support "
 561          "--without-ad-dc "
 562          "--bundled-libraries=!tdb"),
 563         ("samba-make", "make"),
 564         ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"),
 565         ("samba-install", "make install"),
 566         ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 567
 568         ("test",
 569          make_test(cmd='make test',
 570                    include_envs=["clusteredmember"])
 571         ),
 572
 573         # clean up:
 574         ("check-clean-tree", "script/clean-source-tree.sh"),
 575         ("clean", "make clean"),
 576         ("ctdb-clean", "cd ./ctdb && make clean"),
 577         ],
 578
 579     "samba-libs": [
 580         ("random-sleep", random_sleep(300, 900)),
 581         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 582         ("talloc-make", "cd lib/talloc && make"),
 583         ("talloc-install", "cd lib/talloc && make install"),
 584
 585         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 586         ("tdb-make", "cd lib/tdb && make"),
 587         ("tdb-install", "cd lib/tdb && make install"),
 588
 589         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 590         ("tevent-make", "cd lib/tevent && make"),
 591         ("tevent-install", "cd lib/tevent && make install"),
 592
 593         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 594         ("ldb-make", "cd lib/ldb && make"),
 595         ("ldb-install", "cd lib/ldb && make install"),
 596
 597         ("nondevel-configure", "./configure ${PREFIX}"),
 598         ("nondevel-make", "make -j"),
 599         ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 600         ("nondevel-install", "make install"),
 601         ("nondevel-dist", "make dist"),
 602
 603         # retry with all modules shared
 604         ("allshared-distclean", "make distclean"),
 605         ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 606         ("allshared-make", "make -j"),
 607         ],
 608
 609     "samba-static": [
 610         ("random-sleep", random_sleep(1, 1)),
 611         # build with all modules static
 612         ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 613         ("allstatic-make", "make -j"),
 614         ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 615         ("lcov", LCOV_CMD),
 616
 617         # retry without any required modules
 618         ("none-distclean", "make distclean"),
 619         ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 620         ("none-make", "make -j"),
 621
 622         # retry with nonshared smbd and smbtorture
 623         ("nonshared-distclean", "make distclean"),
 624         ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 625         ("nonshared-make", "make -j")
 626         ],
 627
 628     "samba-fuzz": [
 629         # build the fuzzers (static) via the oss-fuzz script
 630         ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 631         ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl"),
 632         ("fuzzers-check", "./lib/fuzzing/oss-fuzz/check_build.sh ${PREFIX_DIR}")
 633         ],
 634
 635     # Test Samba without python still builds.  When this test fails
 636     # due to more use of Python, the expectations is that the newly
 637     # failing part of the code should be disabled when
 638     # --disable-python is set (rather than major work being done to
 639     # support this environment).  The target here is for vendors
 640     # shipping a minimal smbd.
 641     "samba-nopython": [
 642         ("random-sleep", random_sleep(300, 900)),
 643         ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 644         ("make", "make -j"),
 645         ("install", "make install"),
 646         ("find-python", "script/find_python.sh ${PREFIX}"),
 647         ("test", "make test-nopython"),
 648         ("lcov", LCOV_CMD),
 649         ("check-clean-tree", "script/clean-source-tree.sh"),
 650         ("clean", "make clean"),
 651
 652         ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 653         ("talloc-make", "cd lib/talloc && make"),
 654         ("talloc-install", "cd lib/talloc && make install"),
 655
 656         ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 657         ("tdb-make", "cd lib/tdb && make"),
 658         ("tdb-install", "cd lib/tdb && make install"),
 659
 660         ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 661         ("tevent-make", "cd lib/tevent && make"),
 662         ("tevent-install", "cd lib/tevent && make install"),
 663
 664         ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 665         ("ldb-make", "cd lib/ldb && make"),
 666         ("ldb-install", "cd lib/ldb && make install"),
 667
 668         # retry against installed library packages
 669         ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 670         ("libs-make", "make -j"),
 671         ("libs-install", "make install"),
 672         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 673         ("libs-clean", "make clean"),
 674         ],
 675
 676     # check we can do the same thing using python2
 677     "samba-nopython-py2": [
 678         ("random-sleep", random_sleep(300, 900)),
 679         ("configure", "PYTHON=python2 ./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 680         ("make", "PYTHON=python2 make -j"),
 681         ("install", "PYTHON=python2 make install"),
 682         ("find-python", "script/find_python.sh ${PREFIX}"),
 683         ("test", "make test-nopython"),
 684         ("lcov", LCOV_CMD),
 685         ("check-clean-tree", "script/clean-source-tree.sh"),
 686         ("clean", "PYTHON=python2 make clean"),
 687
 688         ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 689         ("talloc-make", "cd lib/talloc && PYTHON=python2 make"),
 690         ("talloc-install", "cd lib/talloc && PYTHON=python2 make install"),
 691
 692         ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 693         ("tdb-make", "cd lib/tdb && PYTHON=python2 make"),
 694         ("tdb-install", "cd lib/tdb && PYTHON=python2 make install"),
 695
 696         ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 697         ("tevent-make", "cd lib/tevent && PYTHON=python2 make"),
 698         ("tevent-install", "cd lib/tevent && PYTHON=python2 make install"),
 699
 700         ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 701         ("ldb-make", "cd lib/ldb && PYTHON=python2 make"),
 702         ("ldb-install", "cd lib/ldb && PYTHON=python2 make install"),
 703
 704         # retry against installed library packages
 705         ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"),
 706         ("libs-make", "PYTHON=python2 make -j"),
 707         ("libs-install", "PYTHON=python2 make install"),
 708         ("libs-check-clean-tree", "script/clean-source-tree.sh"),
 709         ("libs-clean", "PYTHON=python2 make clean"),
 710         ],
 711
 712     "ldb": [
 713         ("random-sleep", random_sleep(60, 600)),
 714         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 715         ("make", "make"),
 716         ("install", "make install"),
 717         ("test", "make test"),
 718         ("lcov", LCOV_CMD),
 719         ("clean", "make clean"),
 720         ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
 721         ("make-no-lmdb", "make"),
 722         ("test-no-lmdb", "make test"),
 723         ("lcov-no-lmdb", LCOV_CMD),
 724         ("install-no-lmdb", "make install"),
 725         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 726         ("distcheck", "make distcheck"),
 727         ("clean", "make clean"),
 728         ],
 729
 730     "tdb": [
 731         ("random-sleep", random_sleep(60, 600)),
 732         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 733         ("make", "make"),
 734         ("install", "make install"),
 735         ("test", "make test"),
 736         ("lcov", LCOV_CMD),
 737         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 738         ("distcheck", "make distcheck"),
 739         ("clean", "make clean"),
 740         ],
 741
 742     "talloc": [
 743         ("random-sleep", random_sleep(60, 600)),
 744         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 745         ("make", "make"),
 746         ("install", "make install"),
 747         ("test", "make test"),
 748         ("lcov", LCOV_CMD),
 749         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 750         ("distcheck", "make distcheck"),
 751         ("clean", "make clean"),
 752         ],
 753
 754     "replace": [
 755         ("random-sleep", random_sleep(60, 600)),
 756         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 757         ("make", "make"),
 758         ("install", "make install"),
 759         ("test", "make test"),
 760         ("lcov", LCOV_CMD),
 761         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 762         ("distcheck", "make distcheck"),
 763         ("clean", "make clean"),
 764         ],
 765
 766     "tevent": [
 767         ("random-sleep", random_sleep(60, 600)),
 768         ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 769         ("make", "make"),
 770         ("install", "make install"),
 771         ("test", "make test"),
 772         ("lcov", LCOV_CMD),
 773         ("check-clean-tree", "../../script/clean-source-tree.sh"),
 774         ("distcheck", "make distcheck"),
 775         ("clean", "make clean"),
 776         ],
 777
 778     "pidl": [
 779         ("random-sleep", random_sleep(60, 600)),
 780         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
 781         ("touch", "touch *.yp"),
 782         ("make", "make"),
 783         ("test", "make test"),
 784         ("install", "make install"),
 785         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
 786         ("check-clean-tree", "../script/clean-source-tree.sh"),
 787         ("clean", "make clean"),
 788         ],
 789
 790     # these are useful for debugging autobuild
 791     'pass': [("pass", 'echo passing && /bin/true')],
 792     'fail': [("fail", 'echo failing && /bin/false')],
 793 }
 794
 795 defaulttasks = list(tasks.keys())
 796
 797 defaulttasks.remove("pass")
 798 defaulttasks.remove("fail")
 799 defaulttasks.remove("samba-test-only")
 800 defaulttasks.remove("samba-fuzz")
 801 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 802     defaulttasks.remove("samba-o3")
 803
 804
 805 def do_print(msg):
 806     print("%s" % msg)
 807     sys.stdout.flush()
 808     sys.stderr.flush()
 809
 810
 811 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 812     if show is None:
 813         show = options.verbose
 814     if show:
 815         do_print("Running: '%s' in '%s'" % (cmd, dir))
 816     if output:
 817         out = check_output([cmd], shell=True, cwd=dir)
 818         return out.decode(encoding='utf-8', errors='backslashreplace')
 819     elif checkfail:
 820         return check_call(cmd, shell=True, cwd=dir)
 821     else:
 822         return call(cmd, shell=True, cwd=dir)
 823
 824
 825 class builder(object):
 826     '''handle build of one directory'''
 827
 828     def __init__(self, name, sequence, cp=True):
 829         self.name = name
 830         self.dir = builddirs.get(name, '.')
 831         self.tag = self.name.replace('/', '_')
 832         self.sequence = sequence
 833         self.next = 0
 834         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 835         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 836         if options.verbose:
 837             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 838             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 839         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 840         self.stdout = open(self.stdout_path, 'w')
 841         self.stderr = open(self.stderr_path, 'w')
 842         self.stdin  = open("/dev/null", 'r')
 843         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 844         self.cwd = "%s/%s" % (self.test_source_dir, self.dir)
 845         self.prefix = "%s/%s" % (test_prefix, self.tag)
 846         run_cmd("rm -rf %s" % self.test_source_dir)
 847         run_cmd("rm -rf %s" % self.prefix)
 848         if cp:
 849             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 850         else:
 851             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 852         self.start_next()
 853
 854     def start_next(self):
 855         if self.next == len(self.sequence):
 856             if not options.nocleanup:
 857                 run_cmd("rm -rf %s" % self.test_source_dir)
 858                 run_cmd("rm -rf %s" % self.prefix)
 859             do_print('%s: Completed OK' % self.name)
 860             self.done = True
 861             return
 862         (self.stage, self.cmd) = self.sequence[self.next]
 863         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 864         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 865         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 866         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 867         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 868         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 869         self.cmd = self.cmd.replace("${NAME}", self.name)
 870         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 871         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
 872         self.proc = Popen(self.cmd, shell=True,
 873                           close_fds=True, cwd=self.cwd,
 874                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 875         self.next += 1
 876
 877
 878 class buildlist(object):
 879     '''handle build of multiple directories'''
 880
 881     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 882         self.tail_proc = None
 883         self.retry = None
 884         if not tasknames:
 885             if options.restrict_tests:
 886                 tasknames = ["samba-test-only"]
 887             else:
 888                 tasknames = defaulttasks
 889
 890         self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames]
 891
 892         if options.retry:
 893             rebase_remote = "rebaseon"
 894             retry_task = [("retry",
 895                             '''set -e
 896                             git remote add -t %s %s %s
 897                             git fetch %s
 898                             while :; do
 899                               sleep 60
 900                               git describe %s/%s > old_remote_branch.desc
 901                               git fetch %s
 902                               git describe %s/%s > remote_branch.desc
 903                               diff old_remote_branch.desc remote_branch.desc
 904                             done
 905                            ''' % (
 906                                rebase_branch, rebase_remote, rebase_url,
 907                                rebase_remote,
 908                                rebase_remote, rebase_branch,
 909                                rebase_remote,
 910                                rebase_remote, rebase_branch
 911                             ))]
 912
 913             self.retry = builder('retry', retry_task, cp=False)
 914             self.need_retry = False
 915
 916     def kill_kids(self):
 917         if self.tail_proc is not None:
 918             self.tail_proc.terminate()
 919             self.tail_proc.wait()
 920             self.tail_proc = None
 921         if self.retry is not None:
 922             self.retry.proc.terminate()
 923             self.retry.proc.wait()
 924             self.retry = None
 925         for b in self.tlist:
 926             if b.proc is not None:
 927                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 928                 b.proc.terminate()
 929                 b.proc.wait()
 930                 b.proc = None
 931
 932     def wait_one(self):
 933         while True:
 934             none_running = True
 935             for b in self.tlist:
 936                 if b.proc is None:
 937                     continue
 938                 none_running = False
 939                 b.status = b.proc.poll()
 940                 if b.status is None:
 941                     continue
 942                 b.proc = None
 943                 return b
 944             if options.retry:
 945                 ret = self.retry.proc.poll()
 946                 if ret is not None:
 947                     self.need_retry = True
 948                     self.retry = None
 949                     return None
 950             if none_running:
 951                 return None
 952             time.sleep(0.1)
 953
 954     def run(self):
 955         while True:
 956             b = self.wait_one()
 957             if options.retry and self.need_retry:
 958                 self.kill_kids()
 959                 do_print("retry needed")
 960                 return (0, None, None, None, "retry")
 961             if b is None:
 962                 break
 963             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 964                 self.kill_kids()
 965                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 966             b.start_next()
 967         self.kill_kids()
 968         return (0, None, None, None, "All OK")
 969
 970     def write_system_info(self, filename):
 971         with open(filename, 'w') as f:
 972             for cmd in ['uname -a',
 973                         'lsb_release -a',
 974                         'free',
 975                         'mount',
 976                         'cat /proc/cpuinfo',
 977                         'cc --version',
 978                         'df -m .',
 979                         'df -m %s' % testbase]:
 980                 out = run_cmd(cmd, output=True, checkfail=False)
 981                 print('### %s' % cmd, file=f)
 982                 print(out, file=f)
 983                 print(file=f)
 984
 985     def tarlogs(self, fname):
 986         with tarfile.open(fname, "w:gz") as tar:
 987             for b in self.tlist:
 988                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 989                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 990             if os.path.exists("autobuild.log"):
 991                 tar.add("autobuild.log")
 992             filename = 'system-info.txt'
 993             self.write_system_info(filename)
 994             tar.add(filename)
 995
 996     def remove_logs(self):
 997         for b in self.tlist:
 998             os.unlink(b.stdout_path)
 999             os.unlink(b.stderr_path)
1000
1001     def start_tail(self):
1002         cmd = ["tail", "-f"]
1003         for b in self.tlist:
1004             cmd.append(b.stdout_path)
1005             cmd.append(b.stderr_path)
1006         self.tail_proc = Popen(cmd, close_fds=True)
1007
1008
1009 def cleanup():
1010     if options.nocleanup:
1011         return
1012     run_cmd("stat %s || true" % test_tmpdir, show=True)
1013     run_cmd("stat %s" % testbase, show=True)
1014     do_print("Cleaning up %r" % cleanup_list)
1015     for d in cleanup_list:
1016         run_cmd("rm -rf %s" % d)
1017
1018
1019 def daemonize(logfile):
1020     pid = os.fork()
1021     if pid == 0:  # Parent
1022         os.setsid()
1023         pid = os.fork()
1024         if pid != 0:  # Actual daemon
1025             os._exit(0)
1026     else:  # Grandparent
1027         os._exit(0)
1028
1029     import resource      # Resource usage information.
1030     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1031     if maxfd == resource.RLIM_INFINITY:
1032         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1033     for fd in range(0, maxfd):
1034         try:
1035             os.close(fd)
1036         except OSError:
1037             pass
1038     os.open(logfile, os.O_RDWR | os.O_CREAT)
1039     os.dup2(0, 1)
1040     os.dup2(0, 2)
1041
1042
1043 def write_pidfile(fname):
1044     '''write a pid file, cleanup on exit'''
1045     with open(fname, mode='w') as f:
1046         f.write("%u\n" % os.getpid())
1047
1048
1049 def rebase_tree(rebase_url, rebase_branch="master"):
1050     rebase_remote = "rebaseon"
1051     do_print("Rebasing on %s" % rebase_url)
1052     run_cmd("git describe HEAD", show=True, dir=test_master)
1053     run_cmd("git remote add -t %s %s %s" %
1054             (rebase_branch, rebase_remote, rebase_url),
1055             show=True, dir=test_master)
1056     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1057     if options.fix_whitespace:
1058         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1059                 (rebase_remote, rebase_branch),
1060                 show=True, dir=test_master)
1061     else:
1062         run_cmd("git rebase --force-rebase %s/%s" %
1063                 (rebase_remote, rebase_branch),
1064                 show=True, dir=test_master)
1065     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1066                    (rebase_remote, rebase_branch),
1067                    dir=test_master, output=True)
1068     if diff == '':
1069         do_print("No differences between HEAD and %s/%s - exiting" %
1070                  (rebase_remote, rebase_branch))
1071         sys.exit(0)
1072     run_cmd("git describe %s/%s" %
1073             (rebase_remote, rebase_branch),
1074             show=True, dir=test_master)
1075     run_cmd("git describe HEAD", show=True, dir=test_master)
1076     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1077             (rebase_remote, rebase_branch),
1078             show=True, dir=test_master)
1079
1080
1081 def push_to(push_url, push_branch="master"):
1082     push_remote = "pushto"
1083     do_print("Pushing to %s" % push_url)
1084     if options.mark:
1085         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1086         run_cmd("git commit --amend -c HEAD", dir=test_master)
1087         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1088         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1089     run_cmd("git remote add -t %s %s %s" %
1090             (push_branch, push_remote, push_url),
1091             show=True, dir=test_master)
1092     run_cmd("git push %s +HEAD:%s" %
1093             (push_remote, push_branch),
1094             show=True, dir=test_master)
1095
1096
1097 def send_email(subject, text, log_tar):
1098     if options.email is None:
1099         do_print("not sending email because the recipient is not set")
1100         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1101                  (subject, text))
1102         return
1103     outer = MIMEMultipart()
1104     outer['Subject'] = subject
1105     outer['To'] = options.email
1106     outer['From'] = options.email_from
1107     outer['Date'] = email.utils.formatdate(localtime=True)
1108     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1109     outer.attach(MIMEText(text, 'plain'))
1110     if options.attach_logs:
1111         with open(log_tar, 'rb') as fp:
1112             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1113         # Set the filename parameter
1114         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1115         outer.attach(msg)
1116     content = outer.as_string()
1117     s = smtplib.SMTP(options.email_server)
1118     email_user = os.getenv('SMTP_USERNAME')
1119     email_password = os.getenv('SMTP_PASSWORD')
1120     if email_user is not None:
1121         s.starttls()
1122         s.login(email_user, email_password)
1123
1124     s.sendmail(options.email_from, [options.email], content)
1125     s.set_debuglevel(1)
1126     s.quit()
1127
1128
1129 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1130                   elapsed_time, log_base=None, add_log_tail=True):
1131     '''send an email to options.email about the failure'''
1132     elapsed_minutes = elapsed_time / 60.0
1133     if log_base is None:
1134         log_base = gitroot
1135     text = '''
1136 Dear Developer,
1137
1138 Your autobuild on %s failed after %.1f minutes
1139 when trying to test %s with the following error:
1140
1141    %s
1142
1143 the autobuild has been abandoned. Please fix the error and resubmit.
1144
1145 A summary of the autobuild process is here:
1146
1147   %s/autobuild.log
1148 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1149
1150     if options.restrict_tests:
1151         text += """
1152 The build was restricted to tests matching %s\n""" % options.restrict_tests
1153
1154     if failed_task != 'rebase':
1155         text += '''
1156 You can see logs of the failed task here:
1157
1158   %s/%s.stdout
1159   %s/%s.stderr
1160
1161 or you can get full logs of all tasks in this job here:
1162
1163   %s/logs.tar.gz
1164
1165 The top commit for the tree that was built was:
1166
1167 %s
1168
1169 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1170
1171     if add_log_tail:
1172         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1173         lines = f.readlines()
1174         log_tail = "".join(lines[-50:])
1175         num_lines = len(lines)
1176         if num_lines < 50:
1177             # Also include stderr (compile failures) if < 50 lines of stdout
1178             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1179             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1180
1181         text += '''
1182 The last 50 lines of log messages:
1183
1184 %s
1185     ''' % log_tail
1186         f.close()
1187
1188     logs = os.path.join(gitroot, 'logs.tar.gz')
1189     send_email('autobuild[%s] failure on %s for task %s during %s'
1190                % (options.branch, platform.node(), failed_task, failed_stage),
1191                text, logs)
1192
1193
1194 def email_success(elapsed_time, log_base=None):
1195     '''send an email to options.email about a successful build'''
1196     if log_base is None:
1197         log_base = gitroot
1198     text = '''
1199 Dear Developer,
1200
1201 Your autobuild on %s has succeeded after %.1f minutes.
1202
1203 ''' % (platform.node(), elapsed_time / 60.)
1204
1205     if options.restrict_tests:
1206         text += """
1207 The build was restricted to tests matching %s\n""" % options.restrict_tests
1208
1209     if options.keeplogs:
1210         text += '''
1211
1212 you can get full logs of all tasks in this job here:
1213
1214   %s/logs.tar.gz
1215
1216 ''' % log_base
1217
1218     text += '''
1219 The top commit for the tree that was built was:
1220
1221 %s
1222 ''' % top_commit_msg
1223
1224     logs = os.path.join(gitroot, 'logs.tar.gz')
1225     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1226                text, logs)
1227
1228
1229 # get the top commit message, for emails
1230 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1231
1232 try:
1233     os.makedirs(testbase)
1234 except Exception as reason:
1235     raise Exception("Unable to create %s : %s" % (testbase, reason))
1236 cleanup_list.append(testbase)
1237
1238 if options.daemon:
1239     logfile = os.path.join(testbase, "log")
1240     do_print("Forking into the background, writing progress to %s" % logfile)
1241     daemonize(logfile)
1242
1243 write_pidfile(gitroot + "/autobuild.pid")
1244
1245 start_time = time.time()
1246
1247 while True:
1248     try:
1249         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1250         os.makedirs(test_tmpdir)
1251         # The waf uninstall code removes empty directories all the way
1252         # up the tree.  Creating a file in test_tmpdir stops it from
1253         # being removed.
1254         run_cmd("touch %s" % os.path.join(test_tmpdir,
1255                                           ".directory-is-not-empty"), show=True)
1256         run_cmd("stat %s" % test_tmpdir, show=True)
1257         run_cmd("stat %s" % testbase, show=True)
1258         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1259     except Exception:
1260         cleanup()
1261         raise
1262
1263     try:
1264         if options.rebase is not None:
1265             rebase_tree(options.rebase, rebase_branch=options.branch)
1266     except Exception:
1267         cleanup_list.append(gitroot + "/autobuild.pid")
1268         cleanup()
1269         elapsed_time = time.time() - start_time
1270         email_failure(-1, 'rebase', 'rebase', 'rebase',
1271                       'rebase on %s failed' % options.branch,
1272                       elapsed_time, log_base=options.log_base)
1273         sys.exit(1)
1274
1275     try:
1276         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1277         if options.tail:
1278             blist.start_tail()
1279         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1280         if status != 0 or errstr != "retry":
1281             break
1282         cleanup()
1283     except Exception:
1284         cleanup()
1285         raise
1286
1287 cleanup_list.append(gitroot + "/autobuild.pid")
1288
1289 do_print(errstr)
1290
1291 blist.kill_kids()
1292 if options.tail:
1293     do_print("waiting for tail to flush")
1294     time.sleep(1)
1295
1296 elapsed_time = time.time() - start_time
1297 if status == 0:
1298     if options.passcmd is not None:
1299         do_print("Running passcmd: %s" % options.passcmd)
1300         run_cmd(options.passcmd, dir=test_master)
1301     if options.pushto is not None:
1302         push_to(options.pushto, push_branch=options.branch)
1303     if options.keeplogs or options.attach_logs:
1304         blist.tarlogs("logs.tar.gz")
1305         do_print("Logs in logs.tar.gz")
1306     if options.always_email:
1307         email_success(elapsed_time, log_base=options.log_base)
1308     blist.remove_logs()
1309     cleanup()
1310     do_print(errstr)
1311     sys.exit(0)
1312
1313 # something failed, gather a tar of the logs
1314 blist.tarlogs("logs.tar.gz")
1315
1316 if options.email is not None:
1317     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1318                   elapsed_time, log_base=options.log_base)
1319 else:
1320     elapsed_minutes = elapsed_time / 60.0
1321     print('''
1322
1323 ####################################################################
1324
1325 AUTOBUILD FAILURE
1326
1327 Your autobuild[%s] on %s failed after %.1f minutes
1328 when trying to test %s with the following error:
1329
1330    %s
1331
1332 the autobuild has been abandoned. Please fix the error and resubmit.
1333
1334 ####################################################################
1335
1336 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1337
1338 cleanup()
1339 do_print(errstr)
1340 do_print("Logs in logs.tar.gz")
1341 sys.exit(status)