script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34
  35 def find_git_root():
  36     '''get to the top of the git repo'''
  37     p = os.getcwd()
  38     while p != '/':
  39         if os.path.isdir(os.path.join(p, ".git")):
  40             return p
  41         p = os.path.abspath(os.path.join(p, '..'))
  42     return None
  43
  44
  45 gitroot = find_git_root()
  46 if gitroot is None:
  47     raise Exception("Failed to find git root")
  48
  49
  50 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  51
  52 parser = OptionParser()
  53 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
  54 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
  55 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
  56 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  57                   default=def_testbase)
  58 parser.add_option("", "--passcmd", help="command to run on success", default=None)
  59 parser.add_option("", "--verbose", help="show all commands as they are run",
  60                   default=False, action="store_true")
  61 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
  62                   default=None, type='str')
  63 parser.add_option("", "--pushto", help="push to a git url on success",
  64                   default=None, type='str')
  65 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
  66                   default=False, action="store_true")
  67 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
  68                   default=False, action="store_true")
  69 parser.add_option("", "--retry", help="automatically retry if master changes",
  70                   default=False, action="store_true")
  71 parser.add_option("", "--email", help="send email to the given address on failure",
  72                   type='str', default=None)
  73 parser.add_option("", "--email-from", help="send email from the given address",
  74                   type='str', default="autobuild@samba.org")
  75 parser.add_option("", "--email-server", help="send email via the given server",
  76                   type='str', default='localhost')
  77 parser.add_option("", "--always-email", help="always send email, even on success",
  78                   action="store_true")
  79 parser.add_option("", "--daemon", help="daemonize after initial setup",
  80                   action="store_true")
  81 parser.add_option("", "--branch", help="the branch to work on (default=master)",
  82                   default="master", type='str')
  83 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
  84                   default=gitroot, type='str')
  85 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
  86                   default=False, action="store_true")
  87 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
  88                   default='')
  89 parser.add_option("--enable-coverage", dest='enable_coverage',
  90                   action="store_const", const='--enable-coverage', default='',
  91                   help="Add --enable-coverage option while configure")
  92
  93 (options, args) = parser.parse_args()
  94
  95 if options.retry:
  96     if options.rebase is None:
  97         raise Exception('You can only use --retry if you also rebase')
  98
  99 testbase = "%s/b%u" % (options.testbase, os.getpid())
 100 test_master = "%s/master" % testbase
 101 test_prefix = "%s/prefix" % testbase
 102 test_tmpdir = "%s/tmp" % testbase
 103 os.environ['TMPDIR'] = test_tmpdir
 104
 105
 106 cleanup_list = []
 107
 108 builddirs = {
 109     "ctdb": "ctdb",
 110     "samba": ".",
 111     "samba-nt4": ".",
 112     "samba-fileserver": ".",
 113     "samba-ad-member": ".",
 114     "samba-xc": ".",
 115     "samba-o3": ".",
 116     "samba-ctdb": ".",
 117     "samba-libs": ".",
 118     "samba-static": ".",
 119     "samba-none-env": ".",
 120     "samba-ad-dc-1": ".",
 121     "samba-ad-dc-2": ".",
 122     "samba-ad-dc-3": ".",
 123     "samba-ad-dc-4": ".",
 124     "samba-ad-dc-5": ".",
 125     "samba-ad-dc-6": ".",
 126     "samba-ad-dc-ntvfs": ".",
 127     "samba-ad-dc-backup": ".",
 128     "samba-systemkrb5": ".",
 129     "samba-nopython": ".",
 130     "samba-nopython-py2": ".",
 131     "samba-schemaupgrade": ".",
 132     "ldb": "lib/ldb",
 133     "tdb": "lib/tdb",
 134     "talloc": "lib/talloc",
 135     "replace": "lib/replace",
 136     "tevent": "lib/tevent",
 137     "pidl": "pidl"
 138 }
 139
 140 defaulttasks = builddirs.keys()
 141
 142 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
 143     defaulttasks.remove("samba-o3")
 144
 145 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
 146 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
 147
 148 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 149 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 150 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 151 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
 152 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 153 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 154 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 155
 156 tasks = {
 157     "ctdb": [("random-sleep", "../script/random-sleep.sh 300 900", "text/plain"),
 158                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
 159                ("make", "make all", "text/plain"),
 160                ("install", "make install", "text/plain"),
 161                ("test", "make autotest", "text/plain"),
 162                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 163                ("clean", "make clean", "text/plain")],
 164
 165     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 166     "samba": [
 167                 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 168                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 169                 ("make", "make -j", "text/plain"),
 170                 ("test", "make test FAIL_IMMEDIATELY=1 "
 171                  "TESTS='--exclude-env=none "
 172                  "--exclude-env=nt4_dc "
 173                  "--exclude-env=nt4_dc_schannel "
 174                  "--exclude-env=nt4_member "
 175                  "--exclude-env=ad_dc "
 176                  "--exclude-env=ad_dc_backup "
 177                  "--exclude-env=ad_dc_ntvfs "
 178                  "--exclude-env=ad_dc_default "
 179                  "--exclude-env=ad_dc_slowtests "
 180                  "--exclude-env=ad_dc_no_nss "
 181                  "--exclude-env=ad_dc_no_ntlm "
 182                  "--exclude-env=fl2003dc "
 183                  "--exclude-env=fl2008dc "
 184                  "--exclude-env=fl2008r2dc "
 185                  "--exclude-env=ad_member "
 186                  "--exclude-env=ad_member_idmap_rid "
 187                  "--exclude-env=ad_member_idmap_ad "
 188                  "--exclude-env=ad_member_rfc2307 "
 189                  "--exclude-env=chgdcpass "
 190                  "--exclude-env=vampire_2000_dc "
 191                  "--exclude-env=fl2000dc "
 192                  "--exclude-env=fileserver "
 193                  "--exclude-env=maptoguest "
 194                  "--exclude-env=simpleserver "
 195                  "--exclude-env=backupfromdc "
 196                  "--exclude-env=restoredc "
 197                  "--exclude-env=renamedc "
 198                  "--exclude-env=offlinebackupdc "
 199                  "--exclude-env=labdc "
 200                  "--exclude-env=preforkrestartdc "
 201                  "--exclude-env=proclimitdc "
 202                  "--exclude-env=promoted_dc "
 203                  "--exclude-env=vampire_dc "
 204                  "--exclude-env=rodc "
 205                  "--exclude-env=ad_dc_default "
 206                  "--exclude-env=ad_dc_slowtests "
 207                  "--exclude-env=schema_pair_dc "
 208                  "--exclude-env=schema_dc "
 209                  "'",
 210                  "text/plain"),
 211                 ("install", "make install", "text/plain"),
 212                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 213                 ("clean", "make clean", "text/plain")],
 214
 215     "samba-nt4": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 216                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 217                     ("make", "make -j", "text/plain"),
 218                     ("test", "make test FAIL_IMMEDIATELY=1 "
 219                      "TESTS='"
 220                      "--include-env=nt4_dc "
 221                      "--include-env=nt4_dc_schannel "
 222                      "--include-env=nt4_member "
 223                      "'", "text/plain"),
 224                     ("install", "make install", "text/plain"),
 225                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 226                     ("clean", "make clean", "text/plain")],
 227
 228     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 229                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 230                            ("make", "make -j", "text/plain"),
 231                            ("test", "make test FAIL_IMMEDIATELY=1 "
 232                             "TESTS='"
 233                             "--include-env=fileserver "
 234                             "--include-env=maptoguest "
 235                             "--include-env=simpleserver "
 236                             "'", "text/plain"),
 237                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 238
 239     "samba-ad-member": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 240                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 241                         ("make", "make -j", "text/plain"),
 242                         ("test", "make test FAIL_IMMEDIATELY=1 "
 243                          "TESTS='"
 244                          "--include-env=ad_member "
 245                          "--include-env=ad_member_idmap_rid "
 246                          "--include-env=ad_member_idmap_ad "
 247                          "--include-env=ad_member_rfc2307 "
 248                          "'", "text/plain"),
 249                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 250
 251     "samba-ad-dc-1": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 252                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 253                       ("make", "make -j", "text/plain"),
 254                       ("test", "make test FAIL_IMMEDIATELY=1 "
 255                        "TESTS='--include-env=ad_dc "
 256                        "--include-env=ad_dc_no_nss "
 257                        "--include-env=ad_dc_no_ntlm "
 258                        "'", "text/plain"),
 259                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 260
 261     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 262                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 263                         ("make", "make -j", "text/plain"),
 264                         ("test", "make test FAIL_IMMEDIATELY=1 "
 265                          "TESTS='"
 266                          "--include-env=vampire_dc "
 267                          "--include-env=vampire_2000_dc "
 268                          "--include-env=rodc "
 269                          "'", "text/plain"),
 270                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 271
 272     "samba-ad-dc-3": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 273                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 274                         ("make", "make -j", "text/plain"),
 275                         ("test", "make test FAIL_IMMEDIATELY=1 "
 276                          "TESTS='"
 277                          "--include-env=promoted_dc "
 278                          "--include-env=chgdcpass "
 279                          "--include-env=preforkrestartdc "
 280                          "--include-env=proclimitdc "
 281                          "'", "text/plain"),
 282                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 283
 284     "samba-ad-dc-4": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 285                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 286                         ("make", "make -j", "text/plain"),
 287                         ("test", "make test FAIL_IMMEDIATELY=1 "
 288                          "TESTS='"
 289                          "--include-env=fl2000dc "
 290                          "--include-env=fl2003dc "
 291                          "--include-env=fl2008dc "
 292                          "--include-env=fl2008r2dc "
 293                          "'", "text/plain"),
 294                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 295
 296     "samba-ad-dc-5": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 297                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 298                         ("make", "make -j", "text/plain"),
 299                         ("test", "make test FAIL_IMMEDIATELY=1 "
 300                          "TESTS='"
 301                          "--include-env=ad_dc_default "
 302                          "'", "text/plain"),
 303                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 304
 305     "samba-ad-dc-6": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 306                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 307                         ("make", "make -j", "text/plain"),
 308                         ("test", "make test FAIL_IMMEDIATELY=1 "
 309                          "TESTS='"
 310                          "--include-env=ad_dc_slowtests "
 311                          "'", "text/plain"),
 312                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 313
 314
 315     "samba-schemaupgrade": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 316                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 317                         ("make", "make -j", "text/plain"),
 318                         ("test", "make test FAIL_IMMEDIATELY=1 "
 319                          "TESTS='"
 320                          "--include-env=schema_dc "
 321                          "--include-env=schema_pair_dc "
 322                          "'", "text/plain"),
 323                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 324
 325     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 326     # This is currently the longest task, so we don't randomly delay it.
 327     "samba-ad-dc-ntvfs": [
 328                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 329                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 330                       ("make", "make -j", "text/plain"),
 331                       ("test", "make test FAIL_IMMEDIATELY=1 "
 332                        "TESTS='"
 333                        "--include-env=ad_dc_ntvfs "
 334                        "'", "text/plain"),
 335                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 336
 337     # run the backup/restore testenvs separately as they're fairly standalone
 338     # (and CI seems to max out at ~8 different DCs running at once)
 339     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 340                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 341                         ("make", "make -j", "text/plain"),
 342                         ("test", "make test FAIL_IMMEDIATELY=1 "
 343                          "TESTS='--include-env=backupfromdc "
 344                          "--include-env=restoredc "
 345                          "--include-env=renamedc "
 346                          "--include-env=offlinebackupdc "
 347                          "--include-env=labdc "
 348                          "--include-env=ad_dc_backup "
 349                          "'", "text/plain"),
 350                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 351
 352     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 353                           ("make", "make -j", "text/plain"),
 354                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 355
 356     # Test cross-compile infrastructure
 357     "samba-xc": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 358                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 359                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 360                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 361                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 362                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 363                    ("compare-results", "script/compare_cc_results.py "
 364                     "./bin/c4che/default{} "
 365                     "./bin-xe/c4che/default{} "
 366                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 367
 368     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 369     "samba-o3": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 370                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 371                    ("make", "make -j", "text/plain"),
 372                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 373                     "TESTS='--include-env=ad_dc'", "text/plain"),
 374                    ("install", "make install", "text/plain"),
 375                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 376                    ("clean", "make clean", "text/plain")],
 377
 378     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 379
 380                      # make sure we have tdb around:
 381                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 382                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 383                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 384
 385
 386                      # build samba with cluster support (also building ctdb):
 387                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 388                      ("samba-make", "make", "text/plain"),
 389                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 390                      ("samba-install", "make install", "text/plain"),
 391                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 392
 393                      # clean up:
 394                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 395                      ("clean", "make clean", "text/plain"),
 396                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 397
 398     "samba-libs": [
 399                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 400                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 401                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 402                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 403
 404                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 405                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 406                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 407
 408                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 409                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 410                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 411
 412                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 413                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 414                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 415
 416                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 417                       ("nondevel-make", "make -j", "text/plain"),
 418                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 419                       ("nondevel-install", "make install", "text/plain"),
 420                       ("nondevel-dist", "make dist", "text/plain"),
 421
 422                       # retry with all modules shared
 423                       ("allshared-distclean", "make distclean", "text/plain"),
 424                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 425                       ("allshared-make", "make -j", "text/plain")],
 426
 427     "samba-none-env": [
 428                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 429                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 430                       ("make", "make -j", "text/plain"),
 431                       ("test", "make test "
 432                        "FAIL_IMMEDIATELY=1 "
 433                        "TESTS='--include-env=none'",
 434                        "text/plain")],
 435
 436     "samba-static": [
 437                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 438                       # build with all modules static
 439                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 440                       ("allstatic-make", "make -j", "text/plain"),
 441                       ("allstatic-test", "make test "
 442                        "FAIL_IMMEDIATELY=1 "
 443                        "TESTS='samba3.smb2.create.*nt4_dc'",
 444                        "text/plain"),
 445
 446                       # retry without any required modules
 447                       ("none-distclean", "make distclean", "text/plain"),
 448                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 449                       ("none-make", "make -j", "text/plain"),
 450
 451                       # retry with nonshared smbd and smbtorture
 452                       ("nonshared-distclean", "make distclean", "text/plain"),
 453                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 454                       ("nonshared-make", "make -j", "text/plain")],
 455
 456     "samba-systemkrb5": [
 457                       ("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 458                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --with-experimental-mit-ad-dc", "text/plain"),
 459                       ("make", "make -j", "text/plain"),
 460                       # we currently cannot run a full make test, a limited list of tests could be run
 461                       # via "make test TESTS=sometests"
 462                       ("test", "make test FAIL_IMMEDIATELY=1 "
 463                        "TESTS='--include-env=ktest'", "text/plain"),
 464                       ("install", "make install", "text/plain"),
 465                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 466                       ("clean", "make clean", "text/plain")
 467                       ],
 468
 469     # Test Samba without python still builds.  When this test fails
 470     # due to more use of Python, the expectations is that the newly
 471     # failing part of the code should be disabled when
 472     # --disable-python is set (rather than major work being done to
 473     # support this environment).  The target here is for vendors
 474     # shipping a minimal smbd.
 475     "samba-nopython": [
 476                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 477                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 478                       ("make", "make -j", "text/plain"),
 479                       ("install", "make install", "text/plain"),
 480                       ("find-python", "script/find_python.sh ${PREFIX}", "text/plain"),
 481                       ("test", "make test-nopython", "text/plain"),
 482                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 483                       ("clean", "make clean", "text/plain"),
 484
 485                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 486                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 487                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 488
 489                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 490                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 491                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 492
 493                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 494                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 495                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 496
 497                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 498                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 499                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 500
 501                       # retry against installed library packages
 502                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 503                       ("libs-make", "make -j", "text/plain"),
 504                       ("libs-install", "make install", "text/plain"),
 505                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 506                       ("libs-clean", "make clean", "text/plain")
 507                       ],
 508
 509     # check we can do the same thing using python2
 510     "samba-nopython-py2": [
 511                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 512                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 513                       ("make", "PYTHON=python2 make -j", "text/plain"),
 514                       ("install", "PYTHON=python2 make install", "text/plain"),
 515                       ("find-python", "script/find_python.sh ${PREFIX}", "text/plain"),
 516                       ("test", "make test-nopython", "text/plain"),
 517                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 518                       ("clean", "PYTHON=python2 make clean", "text/plain"),
 519
 520                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 521                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
 522                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
 523
 524                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 525                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
 526                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
 527
 528                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 529                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
 530                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
 531
 532                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 533                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
 534                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
 535
 536                       # retry against installed library packages
 537                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 538                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
 539                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
 540                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 541                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
 542                       ],
 543
 544     "ldb": [
 545               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 546               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 547               ("make", "make", "text/plain"),
 548               ("install", "make install", "text/plain"),
 549               ("test", "make test", "text/plain"),
 550               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
 551               ("make-no-lmdb", "make", "text/plain"),
 552               ("install-no-lmdb", "make install", "text/plain"),
 553               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 554               ("distcheck", "make distcheck", "text/plain"),
 555               ("clean", "make clean", "text/plain")],
 556
 557     "tdb": [
 558               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 559               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 560               ("make", "make", "text/plain"),
 561               ("install", "make install", "text/plain"),
 562               ("test", "make test", "text/plain"),
 563               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 564               ("distcheck", "make distcheck", "text/plain"),
 565               ("clean", "make clean", "text/plain")],
 566
 567     "talloc": [
 568                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 569                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 570                  ("make", "make", "text/plain"),
 571                  ("install", "make install", "text/plain"),
 572                  ("test", "make test", "text/plain"),
 573                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 574                  ("distcheck", "make distcheck", "text/plain"),
 575                  ("clean", "make clean", "text/plain")],
 576
 577     "replace": [
 578                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 579                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 580                   ("make", "make", "text/plain"),
 581                   ("install", "make install", "text/plain"),
 582                   ("test", "make test", "text/plain"),
 583                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 584                   ("distcheck", "make distcheck", "text/plain"),
 585                   ("clean", "make clean", "text/plain")],
 586
 587     "tevent": [
 588                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 589                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 590                  ("make", "make", "text/plain"),
 591                  ("install", "make install", "text/plain"),
 592                  ("test", "make test", "text/plain"),
 593                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 594                  ("distcheck", "make distcheck", "text/plain"),
 595                  ("clean", "make clean", "text/plain")],
 596
 597     "pidl": [
 598         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 599         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 600         ("touch", "touch *.yp", "text/plain"),
 601         ("make", "make", "text/plain"),
 602         ("test", "make test", "text/plain"),
 603         ("install", "make install", "text/plain"),
 604         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 605         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 606         ("clean", "make clean", "text/plain")],
 607
 608
 609     # these are useful for debugging autobuild
 610     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 611     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 612
 613
 614 }
 615
 616
 617 def do_print(msg):
 618     print("%s" % msg)
 619     sys.stdout.flush()
 620     sys.stderr.flush()
 621
 622
 623 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 624     if show is None:
 625         show = options.verbose
 626     if show:
 627         do_print("Running: '%s' in '%s'" % (cmd, dir))
 628     if output:
 629         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 630     elif checkfail:
 631         return check_call(cmd, shell=True, cwd=dir)
 632     else:
 633         return call(cmd, shell=True, cwd=dir)
 634
 635
 636 class builder(object):
 637     '''handle build of one directory'''
 638
 639     def __init__(self, name, sequence, cp=True):
 640         self.name = name
 641         if name in builddirs:
 642             self.dir = builddirs[name]
 643         else:
 644             self.dir = "."
 645
 646         self.tag = self.name.replace('/', '_')
 647         self.sequence = sequence
 648         self.next = 0
 649         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 650         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 651         if options.verbose:
 652             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 653             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 654         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 655         self.stdout = open(self.stdout_path, 'w')
 656         self.stderr = open(self.stderr_path, 'w')
 657         self.stdin  = open("/dev/null", 'r')
 658         self.test_source_dir = "%s/%s" % (testbase, self.tag)
 659         self.prefix = "%s/%s" % (test_prefix, self.tag)
 660         run_cmd("rm -rf %s" % self.test_source_dir)
 661         run_cmd("rm -rf %s" % self.prefix)
 662         if cp:
 663             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 664         else:
 665             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True)
 666         self.start_next()
 667
 668     def start_next(self):
 669         if self.next == len(self.sequence):
 670             if not options.nocleanup:
 671                 run_cmd("rm -rf %s" % self.test_source_dir)
 672                 run_cmd("rm -rf %s" % self.prefix)
 673             do_print('%s: Completed OK' % self.name)
 674             self.done = True
 675             return
 676         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 677         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 678         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 679         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 680         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 681         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
 682         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
 683         self.cmd = self.cmd.replace("${NAME}", self.name)
 684         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
 685 #        if self.output_mime_type == "text/x-subunit":
 686 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 687         cwd = os.getcwd()
 688         os.chdir("%s/%s" % (self.test_source_dir, self.dir))
 689         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 690         self.proc = Popen(self.cmd, shell=True,
 691                           close_fds=True,
 692                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 693         os.chdir(cwd)
 694         self.next += 1
 695
 696
 697 class buildlist(object):
 698     '''handle build of multiple directories'''
 699
 700     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 701         global tasks
 702         self.tlist = []
 703         self.tail_proc = None
 704         self.retry = None
 705         if tasknames == []:
 706             if options.restrict_tests:
 707                 tasknames = ["samba-test-only"]
 708             else:
 709                 tasknames = defaulttasks
 710         else:
 711             # If we are only running one test,
 712             # do not sleep randomly to wait for it to start
 713             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 714
 715         for n in tasknames:
 716             b = builder(n, tasks[n], cp=n is not "pidl")
 717             self.tlist.append(b)
 718         if options.retry:
 719             rebase_remote = "rebaseon"
 720             retry_task = [("retry",
 721                             '''set -e
 722                             git remote add -t %s %s %s
 723                             git fetch %s
 724                             while :; do
 725                               sleep 60
 726                               git describe %s/%s > old_remote_branch.desc
 727                               git fetch %s
 728                               git describe %s/%s > remote_branch.desc
 729                               diff old_remote_branch.desc remote_branch.desc
 730                             done
 731                            ''' % (
 732                                 rebase_branch, rebase_remote, rebase_url,
 733                                rebase_remote,
 734                                rebase_remote, rebase_branch,
 735                                rebase_remote,
 736                                rebase_remote, rebase_branch
 737                             ),
 738                             "test/plain")]
 739
 740             self.retry = builder('retry', retry_task, cp=False)
 741             self.need_retry = False
 742
 743     def kill_kids(self):
 744         if self.tail_proc is not None:
 745             self.tail_proc.terminate()
 746             self.tail_proc.wait()
 747             self.tail_proc = None
 748         if self.retry is not None:
 749             self.retry.proc.terminate()
 750             self.retry.proc.wait()
 751             self.retry = None
 752         for b in self.tlist:
 753             if b.proc is not None:
 754                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
 755                 b.proc.terminate()
 756                 b.proc.wait()
 757                 b.proc = None
 758
 759     def wait_one(self):
 760         while True:
 761             none_running = True
 762             for b in self.tlist:
 763                 if b.proc is None:
 764                     continue
 765                 none_running = False
 766                 b.status = b.proc.poll()
 767                 if b.status is None:
 768                     continue
 769                 b.proc = None
 770                 return b
 771             if options.retry:
 772                 ret = self.retry.proc.poll()
 773                 if ret is not None:
 774                     self.need_retry = True
 775                     self.retry = None
 776                     return None
 777             if none_running:
 778                 return None
 779             time.sleep(0.1)
 780
 781     def run(self):
 782         while True:
 783             b = self.wait_one()
 784             if options.retry and self.need_retry:
 785                 self.kill_kids()
 786                 do_print("retry needed")
 787                 return (0, None, None, None, "retry")
 788             if b is None:
 789                 break
 790             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 791                 self.kill_kids()
 792                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 793             b.start_next()
 794         self.kill_kids()
 795         return (0, None, None, None, "All OK")
 796
 797     def write_system_info(self):
 798         filename = 'system-info.txt'
 799         f = open(filename, 'w')
 800         for cmd in ['uname -a',
 801                     'lsb_release -a',
 802                     'free',
 803                     'mount',
 804                     'cat /proc/cpuinfo',
 805                     'cc --version',
 806                     'df -m .',
 807                     'df -m %s' % testbase]:
 808             out = run_cmd(cmd, output=True, checkfail=False)
 809             print('### %s' % cmd, file=f)
 810             print(out.decode('utf8', 'backslashreplace'), file=f)
 811             print(file=f)
 812         f.close()
 813         return filename
 814
 815     def tarlogs(self, fname):
 816         tar = tarfile.open(fname, "w:gz")
 817         for b in self.tlist:
 818             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 819             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 820         if os.path.exists("autobuild.log"):
 821             tar.add("autobuild.log")
 822         sys_info = self.write_system_info()
 823         tar.add(sys_info)
 824         tar.close()
 825
 826     def remove_logs(self):
 827         for b in self.tlist:
 828             os.unlink(b.stdout_path)
 829             os.unlink(b.stderr_path)
 830
 831     def start_tail(self):
 832         cmd = ["tail", "-f"]
 833         for b in self.tlist:
 834             cmd.append(b.stdout_path)
 835             cmd.append(b.stderr_path)
 836         self.tail_proc = Popen(cmd, close_fds=True)
 837
 838
 839 def cleanup():
 840     if options.nocleanup:
 841         return
 842     run_cmd("stat %s || true" % test_tmpdir, show=True)
 843     run_cmd("stat %s" % testbase, show=True)
 844     do_print("Cleaning up %r" % cleanup_list)
 845     for d in cleanup_list:
 846         run_cmd("rm -rf %s" % d)
 847
 848
 849 def daemonize(logfile):
 850     pid = os.fork()
 851     if pid == 0:  # Parent
 852         os.setsid()
 853         pid = os.fork()
 854         if pid != 0:  # Actual daemon
 855             os._exit(0)
 856     else:  # Grandparent
 857         os._exit(0)
 858
 859     import resource      # Resource usage information.
 860     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 861     if maxfd == resource.RLIM_INFINITY:
 862         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 863     for fd in range(0, maxfd):
 864         try:
 865             os.close(fd)
 866         except OSError:
 867             pass
 868     os.open(logfile, os.O_RDWR | os.O_CREAT)
 869     os.dup2(0, 1)
 870     os.dup2(0, 2)
 871
 872
 873 def write_pidfile(fname):
 874     '''write a pid file, cleanup on exit'''
 875     f = open(fname, mode='w')
 876     f.write("%u\n" % os.getpid())
 877     f.close()
 878
 879
 880 def rebase_tree(rebase_url, rebase_branch="master"):
 881     rebase_remote = "rebaseon"
 882     do_print("Rebasing on %s" % rebase_url)
 883     run_cmd("git describe HEAD", show=True, dir=test_master)
 884     run_cmd("git remote add -t %s %s %s" %
 885             (rebase_branch, rebase_remote, rebase_url),
 886             show=True, dir=test_master)
 887     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 888     if options.fix_whitespace:
 889         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 890                 (rebase_remote, rebase_branch),
 891                 show=True, dir=test_master)
 892     else:
 893         run_cmd("git rebase --force-rebase %s/%s" %
 894                 (rebase_remote, rebase_branch),
 895                 show=True, dir=test_master)
 896     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 897                    (rebase_remote, rebase_branch),
 898                    dir=test_master, output=True)
 899     if diff == '':
 900         do_print("No differences between HEAD and %s/%s - exiting" %
 901                  (rebase_remote, rebase_branch))
 902         sys.exit(0)
 903     run_cmd("git describe %s/%s" %
 904             (rebase_remote, rebase_branch),
 905             show=True, dir=test_master)
 906     run_cmd("git describe HEAD", show=True, dir=test_master)
 907     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 908             (rebase_remote, rebase_branch),
 909             show=True, dir=test_master)
 910
 911
 912 def push_to(push_url, push_branch="master"):
 913     push_remote = "pushto"
 914     do_print("Pushing to %s" % push_url)
 915     if options.mark:
 916         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 917         run_cmd("git commit --amend -c HEAD", dir=test_master)
 918         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 919         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 920     run_cmd("git remote add -t %s %s %s" %
 921             (push_branch, push_remote, push_url),
 922             show=True, dir=test_master)
 923     run_cmd("git push %s +HEAD:%s" %
 924             (push_remote, push_branch),
 925             show=True, dir=test_master)
 926
 927
 928 def send_email(subject, text, log_tar):
 929     if options.email is None:
 930         do_print("not sending email because the recipient is not set")
 931         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 932                  (subject, text))
 933         return
 934     outer = MIMEMultipart()
 935     outer['Subject'] = subject
 936     outer['To'] = options.email
 937     outer['From'] = options.email_from
 938     outer['Date'] = email.utils.formatdate(localtime=True)
 939     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 940     outer.attach(MIMEText(text, 'plain'))
 941     if options.attach_logs:
 942         fp = open(log_tar, 'rb')
 943         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 944         fp.close()
 945         # Set the filename parameter
 946         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 947         outer.attach(msg)
 948     content = outer.as_string()
 949     s = smtplib.SMTP(options.email_server)
 950     email_user = os.getenv('SMTP_USERNAME')
 951     email_password = os.getenv('SMTP_PASSWORD')
 952     if email_user is not None:
 953         s.starttls()
 954         s.login(email_user, email_password)
 955
 956     s.sendmail(options.email_from, [options.email], content)
 957     s.set_debuglevel(1)
 958     s.quit()
 959
 960
 961 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 962                   elapsed_time, log_base=None, add_log_tail=True):
 963     '''send an email to options.email about the failure'''
 964     elapsed_minutes = elapsed_time / 60.0
 965     if log_base is None:
 966         log_base = gitroot
 967     text = '''
 968 Dear Developer,
 969
 970 Your autobuild on %s failed after %.1f minutes
 971 when trying to test %s with the following error:
 972
 973    %s
 974
 975 the autobuild has been abandoned. Please fix the error and resubmit.
 976
 977 A summary of the autobuild process is here:
 978
 979   %s/autobuild.log
 980 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 981
 982     if options.restrict_tests:
 983         text += """
 984 The build was restricted to tests matching %s\n""" % options.restrict_tests
 985
 986     if failed_task != 'rebase':
 987         text += '''
 988 You can see logs of the failed task here:
 989
 990   %s/%s.stdout
 991   %s/%s.stderr
 992
 993 or you can get full logs of all tasks in this job here:
 994
 995   %s/logs.tar.gz
 996
 997 The top commit for the tree that was built was:
 998
 999 %s
1000
1001 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1002
1003     if add_log_tail:
1004         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1005         lines = f.readlines()
1006         log_tail = "".join(lines[-50:])
1007         num_lines = len(lines)
1008         if num_lines < 50:
1009             # Also include stderr (compile failures) if < 50 lines of stdout
1010             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1011             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1012
1013         text += '''
1014 The last 50 lines of log messages:
1015
1016 %s
1017     ''' % log_tail
1018         f.close()
1019
1020     logs = os.path.join(gitroot, 'logs.tar.gz')
1021     send_email('autobuild[%s] failure on %s for task %s during %s'
1022                % (options.branch, platform.node(), failed_task, failed_stage),
1023                text, logs)
1024
1025
1026 def email_success(elapsed_time, log_base=None):
1027     '''send an email to options.email about a successful build'''
1028     if log_base is None:
1029         log_base = gitroot
1030     text = '''
1031 Dear Developer,
1032
1033 Your autobuild on %s has succeeded after %.1f minutes.
1034
1035 ''' % (platform.node(), elapsed_time / 60.)
1036
1037     if options.restrict_tests:
1038         text += """
1039 The build was restricted to tests matching %s\n""" % options.restrict_tests
1040
1041     if options.keeplogs:
1042         text += '''
1043
1044 you can get full logs of all tasks in this job here:
1045
1046   %s/logs.tar.gz
1047
1048 ''' % log_base
1049
1050     text += '''
1051 The top commit for the tree that was built was:
1052
1053 %s
1054 ''' % top_commit_msg
1055
1056     logs = os.path.join(gitroot, 'logs.tar.gz')
1057     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1058                text, logs)
1059
1060
1061 # get the top commit message, for emails
1062 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1063 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
1064
1065 try:
1066     os.makedirs(testbase)
1067 except Exception as reason:
1068     raise Exception("Unable to create %s : %s" % (testbase, reason))
1069 cleanup_list.append(testbase)
1070
1071 if options.daemon:
1072     logfile = os.path.join(testbase, "log")
1073     do_print("Forking into the background, writing progress to %s" % logfile)
1074     daemonize(logfile)
1075
1076 write_pidfile(gitroot + "/autobuild.pid")
1077
1078 start_time = time.time()
1079
1080 while True:
1081     try:
1082         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1083         os.makedirs(test_tmpdir)
1084         # The waf uninstall code removes empty directories all the way
1085         # up the tree.  Creating a file in test_tmpdir stops it from
1086         # being removed.
1087         run_cmd("touch %s" % os.path.join(test_tmpdir,
1088                                           ".directory-is-not-empty"), show=True)
1089         run_cmd("stat %s" % test_tmpdir, show=True)
1090         run_cmd("stat %s" % testbase, show=True)
1091         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1092     except Exception:
1093         cleanup()
1094         raise
1095
1096     try:
1097         try:
1098             if options.rebase is not None:
1099                 rebase_tree(options.rebase, rebase_branch=options.branch)
1100         except Exception:
1101             cleanup_list.append(gitroot + "/autobuild.pid")
1102             cleanup()
1103             elapsed_time = time.time() - start_time
1104             email_failure(-1, 'rebase', 'rebase', 'rebase',
1105                           'rebase on %s failed' % options.branch,
1106                           elapsed_time, log_base=options.log_base)
1107             sys.exit(1)
1108         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1109         if options.tail:
1110             blist.start_tail()
1111         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1112         if status != 0 or errstr != "retry":
1113             break
1114         cleanup()
1115     except Exception:
1116         cleanup()
1117         raise
1118
1119 cleanup_list.append(gitroot + "/autobuild.pid")
1120
1121 do_print(errstr)
1122
1123 blist.kill_kids()
1124 if options.tail:
1125     do_print("waiting for tail to flush")
1126     time.sleep(1)
1127
1128 elapsed_time = time.time() - start_time
1129 if status == 0:
1130     if options.passcmd is not None:
1131         do_print("Running passcmd: %s" % options.passcmd)
1132         run_cmd(options.passcmd, dir=test_master)
1133     if options.pushto is not None:
1134         push_to(options.pushto, push_branch=options.branch)
1135     if options.keeplogs or options.attach_logs:
1136         blist.tarlogs("logs.tar.gz")
1137         do_print("Logs in logs.tar.gz")
1138     if options.always_email:
1139         email_success(elapsed_time, log_base=options.log_base)
1140     blist.remove_logs()
1141     cleanup()
1142     do_print(errstr)
1143     sys.exit(0)
1144
1145 # something failed, gather a tar of the logs
1146 blist.tarlogs("logs.tar.gz")
1147
1148 if options.email is not None:
1149     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1150                   elapsed_time, log_base=options.log_base)
1151 else:
1152     elapsed_minutes = elapsed_time / 60.0
1153     print('''
1154
1155 ####################################################################
1156
1157 AUTOBUILD FAILURE
1158
1159 Your autobuild[%s] on %s failed after %.1f minutes
1160 when trying to test %s with the following error:
1161
1162    %s
1163
1164 the autobuild has been abandoned. Please fix the error and resubmit.
1165
1166 ####################################################################
1167
1168 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1169
1170 cleanup()
1171 do_print(errstr)
1172 do_print("Logs in logs.tar.gz")
1173 sys.exit(status)