script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-py2": ".",
  40     "samba-nt4": ".",
  41     "samba-nt4-py2": ".",
  42     "samba-fileserver": ".",
  43     "samba-xc": ".",
  44     "samba-o3": ".",
  45     "samba-ctdb": ".",
  46     "samba-libs": ".",
  47     "samba-libs-py2": ".",
  48     "samba-static": ".",
  49     "samba-none-env": ".",
  50     "samba-ad-dc": ".",
  51     "samba-ad-dc-py2": ".",
  52     "samba-ad-dc-ntvfs": ".",
  53     "samba-ad-dc-ntvfs-py2": ".",
  54     "samba-ad-dc-2": ".",
  55     "samba-ad-dc-2-py2": ".",
  56     "samba-ad-dc-backup": ".",
  57     "samba-systemkrb5": ".",
  58     "samba-nopython": ".",
  59     "samba-buildpy2-only": ".",
  60     "ldb": "lib/ldb",
  61     "tdb": "lib/tdb",
  62     "talloc": "lib/talloc",
  63     "replace": "lib/replace",
  64     "tevent": "lib/tevent",
  65     "pidl": "pidl"
  66 }
  67
  68 defaulttasks = builddirs.keys()
  69
  70 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  71     defaulttasks.remove("samba-o3")
  72
  73 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  74 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  75
  76 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  77 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  78 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  79 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  80 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  81 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  82 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  83
  84 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  85     extra_python = ""
  86 else:
  87     extra_python = "--extra-python=/usr/bin/python2"
  88
  89 tasks = {
  90     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  91                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  92                ("make", "make all", "text/plain"),
  93                ("install", "make install", "text/plain"),
  94                ("test", "make autotest", "text/plain"),
  95                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  96                ("clean", "make clean", "text/plain")],
  97
  98     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
  99     "samba": [
 100                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 101                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 102                 ("make", "make -j", "text/plain"),
 103                 ("test", "make test FAIL_IMMEDIATELY=1 "
 104                  "TESTS='${PY3_ONLY}"
 105                  "--exclude-env=none "
 106                  "--exclude-env=nt4_dc "
 107                  "--exclude-env=nt4_member "
 108                  "--exclude-env=ad_dc "
 109                  "--exclude-env=ad_dc_ntvfs "
 110                  "--exclude-env=ad_dc_no_nss "
 111                  "--exclude-env=fl2003dc "
 112                  "--exclude-env=fl2008r2dc "
 113                  "--exclude-env=ad_member "
 114                  "--exclude-env=ad_member_idmap_rid "
 115                  "--exclude-env=ad_member_idmap_ad "
 116                  "--exclude-env=chgdcpass "
 117                  "--exclude-env=vampire_2000_dc "
 118                  "--exclude-env=fl2000dc "
 119                  "--exclude-env=fileserver "
 120                  "--exclude-env=backupfromdc "
 121                  "--exclude-env=restoredc "
 122                  "--exclude-env=renamedc "
 123                  "--exclude-env=offlinebackupdc "
 124                  "--exclude-env=labdc "
 125                  "'",
 126                  "text/plain"),
 127                 ("install", "make install", "text/plain"),
 128                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 129                 ("clean", "make clean", "text/plain")],
 130
 131     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 132                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 133                     ("make", "make -j", "text/plain"),
 134                     ("test", "make test FAIL_IMMEDIATELY=1 "
 135                      "TESTS='${PY3_ONLY}"
 136                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 137                     ("install", "make install", "text/plain"),
 138                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 139                     ("clean", "make clean", "text/plain")],
 140
 141     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 142                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 143                            ("make", "make -j", "text/plain"),
 144                            ("test", "make test FAIL_IMMEDIATELY=1 "
 145                             "TESTS='${PY3_ONLY}"
 146                             "--include-env=fileserver'", "text/plain"),
 147                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 148
 149     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 150                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 151                       ("make", "make -j", "text/plain"),
 152                       ("test", "make test FAIL_IMMEDIATELY=1 "
 153                        "TESTS='${PY3_ONLY}"
 154                        "--include-env=ad_dc "
 155                        "--include-env=fl2003dc "
 156                        "--include-env=fl2008r2dc "
 157                        "--include-env=ad_member "
 158                        "--include-env=ad_member_idmap_rid "
 159                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 160                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 161
 162     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 163                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 164                         ("make", "make -j", "text/plain"),
 165                         ("test", "make test FAIL_IMMEDIATELY=1 "
 166                          "TESTS='${PY3_ONLY}"
 167                          "--include-env=chgdcpass "
 168                          "--include-env=vampire_2000_dc "
 169                          "--include-env=fl2000dc "
 170                          "--include-env=ad_dc_no_nss "
 171                          "'",
 172                          "text/plain"),
 173                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 174
 175     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 176     # This is currently the longest task, so we don't randomly delay it.
 177     "samba-ad-dc-ntvfs": [
 178                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 179                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 180                       ("make", "make -j", "text/plain"),
 181                       ("test", "make test FAIL_IMMEDIATELY=1 "
 182                        "TESTS='${PY3_ONLY}"
 183                        "--include-env=ad_dc_ntvfs "
 184                        "'",
 185                        "text/plain"),
 186                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 187
 188     # run the backup/restore testenvs separately as they're fairly standalone
 189     # (and CI seems to max out at ~8 different DCs running at once)
 190     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 191                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 192                         ("make", "make -j", "text/plain"),
 193                         ("test", "make test FAIL_IMMEDIATELY=1 "
 194                          "TESTS='${PY3_ONLY}"
 195                          "--include-env=backupfromdc "
 196                          "--include-env=restoredc "
 197                          "--include-env=renamedc "
 198                          "--include-env=offlinebackupdc "
 199                          "--include-env=labdc "
 200                          "'",
 201                          "text/plain"),
 202                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 203
 204     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 205                           ("make", "make -j", "text/plain"),
 206                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 207
 208     # Test cross-compile infrastructure
 209     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 210                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 211                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 212                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 213                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 214                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 215                    ("compare-results", "script/compare_cc_results.py "
 216                     "./bin/c4che/default{} "
 217                     "./bin-xe/c4che/default{} "
 218                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 219
 220     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 221     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 222                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 223                    ("make", "make -j", "text/plain"),
 224                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 225                     "TESTS='${PY3_ONLY}"
 226                     "--include-env=ad_dc'", "text/plain"),
 227                    ("install", "make install", "text/plain"),
 228                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 229                    ("clean", "make clean", "text/plain")],
 230
 231     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 232
 233                      # make sure we have tdb around:
 234                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 235                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 236                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 237
 238
 239                      # build samba with cluster support (also building ctdb):
 240                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 241                      ("samba-make", "make", "text/plain"),
 242                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 243                      ("samba-install", "make install", "text/plain"),
 244                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 245
 246                      # clean up:
 247                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 248                      ("clean", "make clean", "text/plain"),
 249                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 250
 251     "samba-libs": [
 252                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 253                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 254                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 255                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 256
 257                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 258                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 259                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 260
 261                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 262                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 263                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 264
 265                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 266                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 267                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 268
 269                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 270                       ("nondevel-make", "make -j", "text/plain"),
 271                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 272                       ("nondevel-install", "make install", "text/plain"),
 273                       ("nondevel-dist", "make dist", "text/plain"),
 274
 275                       # retry with all modules shared
 276                       ("allshared-distclean", "make distclean", "text/plain"),
 277                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 278                       ("allshared-make", "make -j", "text/plain")],
 279
 280     "samba-none-env": [
 281                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 282                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 283                       ("make", "make -j", "text/plain"),
 284                       ("test", "make test "
 285                        "FAIL_IMMEDIATELY=1 "
 286                        "TESTS='${PY3_ONLY}"
 287                        "--include-env=none'",
 288                        "text/plain")],
 289
 290     "samba-static": [
 291                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 292                       # build with all modules static
 293                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 294                       ("allstatic-make", "make -j", "text/plain"),
 295                       ("allstatic-test", "make test "
 296                        "FAIL_IMMEDIATELY=1 "
 297                        "TESTS='samba3.smb2.create.*nt4_dc'",
 298                        "text/plain"),
 299
 300                       # retry without any required modules
 301                       ("none-distclean", "make distclean", "text/plain"),
 302                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 303                       ("none-make", "make -j", "text/plain"),
 304
 305                       # retry with nonshared smbd and smbtorture
 306                       ("nonshared-distclean", "make distclean", "text/plain"),
 307                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 308                       ("nonshared-make", "make -j", "text/plain")],
 309
 310     "samba-systemkrb5": [
 311                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 312                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 313                       ("make", "make -j", "text/plain"),
 314                       # we currently cannot run a full make test, a limited list of tests could be run
 315                       # via "make test TESTS=sometests"
 316                       ("test", "make test FAIL_IMMEDIATELY=1 "
 317                        "TESTS='${PY3_ONLY}"
 318                        "--include-env=ktest'", "text/plain"),
 319                       ("install", "make install", "text/plain"),
 320                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 321                       ("clean", "make clean", "text/plain")
 322                       ],
 323
 324     # Test Samba without python still builds.  When this test fails
 325     # due to more use of Python, the expectations is that the newly
 326     # failing part of the code should be disabled when
 327     # --disable-python is set (rather than major work being done to
 328     # support this environment).  The target here is for vendors
 329     # shipping a minimal smbd.
 330     "samba-nopython": [
 331                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 332                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 333                       ("make", "make -j", "text/plain"),
 334                       ("install", "make install", "text/plain"),
 335                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 336                       ("clean", "make clean", "text/plain"),
 337
 338                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 339                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 340                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 341
 342                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 343                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 344                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 345
 346                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 347                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 348                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 349
 350                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 351                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 352                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 353
 354                       # retry against installed library packages
 355                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 356                       ("libs-make", "make -j", "text/plain"),
 357                       ("libs-install", "make install", "text/plain"),
 358                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 359                       ("libs-clean", "make clean", "text/plain")
 360                       ],
 361
 362
 363
 364     "ldb": [
 365               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 366               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 367               ("make", "make", "text/plain"),
 368               ("install", "make install", "text/plain"),
 369               ("test", "make test", "text/plain"),
 370               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 371               ("make-no-lmdb", "make", "text/plain"),
 372               ("install-no-lmdb", "make install", "text/plain"),
 373               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 374               ("distcheck", "make distcheck", "text/plain"),
 375               ("clean", "make clean", "text/plain")],
 376
 377     "tdb": [
 378               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 379               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 380               ("make", "make", "text/plain"),
 381               ("install", "make install", "text/plain"),
 382               ("test", "make test", "text/plain"),
 383               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 384               ("distcheck", "make distcheck", "text/plain"),
 385               ("clean", "make clean", "text/plain")],
 386
 387     "talloc": [
 388                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 389                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 390                  ("make", "make", "text/plain"),
 391                  ("install", "make install", "text/plain"),
 392                  ("test", "make test", "text/plain"),
 393                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 394                  ("distcheck", "make distcheck", "text/plain"),
 395                  ("clean", "make clean", "text/plain")],
 396
 397     "replace": [
 398                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 399                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 400                   ("make", "make", "text/plain"),
 401                   ("install", "make install", "text/plain"),
 402                   ("test", "make test", "text/plain"),
 403                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 404                   ("distcheck", "make distcheck", "text/plain"),
 405                   ("clean", "make clean", "text/plain")],
 406
 407     "tevent": [
 408                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 409                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 410                  ("make", "make", "text/plain"),
 411                  ("install", "make install", "text/plain"),
 412                  ("test", "make test", "text/plain"),
 413                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 414                  ("distcheck", "make distcheck", "text/plain"),
 415                  ("clean", "make clean", "text/plain")],
 416
 417     "pidl": [
 418         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 419         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 420         ("touch", "touch *.yp", "text/plain"),
 421         ("make", "make", "text/plain"),
 422         ("test", "make test", "text/plain"),
 423         ("install", "make install", "text/plain"),
 424         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 425         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 426         ("clean", "make clean", "text/plain")],
 427
 428     "samba-buildpy2-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 429                    ("configure", "PYTHON='python' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
 430                    ("make", "PYTHON='python' make -j", "text/plain"),
 431                    ("install", "PYTHON='python' make install", "text/plain"),
 432                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 433                    ("clean", "PYTHON='python' make clean", "text/plain")],
 434
 435
 436     # these are useful for debugging autobuild
 437     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 438     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 439
 440
 441 }
 442
 443
 444 def do_print(msg):
 445     print("%s" % msg)
 446     sys.stdout.flush()
 447     sys.stderr.flush()
 448
 449
 450 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 451     if show is None:
 452         show = options.verbose
 453     if show:
 454         do_print("Running: '%s' in '%s'" % (cmd, dir))
 455     if output:
 456         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 457     elif checkfail:
 458         return check_call(cmd, shell=True, cwd=dir)
 459     else:
 460         return call(cmd, shell=True, cwd=dir)
 461
 462
 463 class builder(object):
 464     '''handle build of one directory'''
 465
 466     def __init__(self, name, sequence, cp=True, py3=False):
 467         self.name = name
 468         self.py3 = py3
 469         if name in builddirs:
 470             self.dir = builddirs[name]
 471         else:
 472             self.dir = "."
 473
 474         self.tag = self.name.replace('/', '_')
 475         self.sequence = sequence
 476         self.next = 0
 477         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 478         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 479         if options.verbose:
 480             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 481             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 482         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 483         self.stdout = open(self.stdout_path, 'w')
 484         self.stderr = open(self.stderr_path, 'w')
 485         self.stdin  = open("/dev/null", 'r')
 486         self.sdir = "%s/%s" % (testbase, self.tag)
 487         self.prefix = "%s/%s" % (test_prefix, self.tag)
 488         run_cmd("rm -rf %s" % self.sdir)
 489         run_cmd("rm -rf %s" % self.prefix)
 490         if cp:
 491             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 492         else:
 493             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 494         self.start_next()
 495
 496     def start_next(self):
 497         if self.next == len(self.sequence):
 498             if not options.nocleanup:
 499                 run_cmd("rm -rf %s" % self.sdir)
 500                 run_cmd("rm -rf %s" % self.prefix)
 501             do_print('%s: Completed OK' % self.name)
 502             self.done = True
 503             return
 504         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 505         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 506         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 507         if self.py3:
 508             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 509             # The trailing space is important
 510             self.cmd = self.cmd.replace("${PY3_ONLY}", "python2 ")
 511         else:
 512             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 513             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 514         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 515         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 516 #        if self.output_mime_type == "text/x-subunit":
 517 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 518         cwd = os.getcwd()
 519         os.chdir("%s/%s" % (self.sdir, self.dir))
 520         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 521         self.proc = Popen(self.cmd, shell=True,
 522                           close_fds=True,
 523                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 524         os.chdir(cwd)
 525         self.next += 1
 526
 527
 528 class buildlist(object):
 529     '''handle build of multiple directories'''
 530
 531     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 532         global tasks
 533         self.tlist = []
 534         self.tail_proc = None
 535         self.retry = None
 536         if tasknames == []:
 537             if options.restrict_tests:
 538                 tasknames = ["samba-test-only"]
 539             else:
 540                 tasknames = defaulttasks
 541         else:
 542             # If we are only running one test,
 543             # do not sleep randomly to wait for it to start
 544             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 545
 546         for n in tasknames:
 547             if n not in tasks and n.endswith("-py2"):
 548                 b = builder(n,
 549                             tasks[n[:-4]],
 550                             cp=n is not "pidl",
 551                             py3=True)
 552             else:
 553                 b = builder(n, tasks[n], cp=n is not "pidl")
 554             self.tlist.append(b)
 555         if options.retry:
 556             rebase_remote = "rebaseon"
 557             retry_task = [("retry",
 558                             '''set -e
 559                             git remote add -t %s %s %s
 560                             git fetch %s
 561                             while :; do
 562                               sleep 60
 563                               git describe %s/%s > old_remote_branch.desc
 564                               git fetch %s
 565                               git describe %s/%s > remote_branch.desc
 566                               diff old_remote_branch.desc remote_branch.desc
 567                             done
 568                            ''' % (
 569                                 rebase_branch, rebase_remote, rebase_url,
 570                                rebase_remote,
 571                                rebase_remote, rebase_branch,
 572                                rebase_remote,
 573                                rebase_remote, rebase_branch
 574                             ),
 575                             "test/plain")]
 576
 577             self.retry = builder('retry', retry_task, cp=False)
 578             self.need_retry = False
 579
 580     def kill_kids(self):
 581         if self.tail_proc is not None:
 582             self.tail_proc.terminate()
 583             self.tail_proc.wait()
 584             self.tail_proc = None
 585         if self.retry is not None:
 586             self.retry.proc.terminate()
 587             self.retry.proc.wait()
 588             self.retry = None
 589         for b in self.tlist:
 590             if b.proc is not None:
 591                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 592                 b.proc.terminate()
 593                 b.proc.wait()
 594                 b.proc = None
 595
 596     def wait_one(self):
 597         while True:
 598             none_running = True
 599             for b in self.tlist:
 600                 if b.proc is None:
 601                     continue
 602                 none_running = False
 603                 b.status = b.proc.poll()
 604                 if b.status is None:
 605                     continue
 606                 b.proc = None
 607                 return b
 608             if options.retry:
 609                 ret = self.retry.proc.poll()
 610                 if ret is not None:
 611                     self.need_retry = True
 612                     self.retry = None
 613                     return None
 614             if none_running:
 615                 return None
 616             time.sleep(0.1)
 617
 618     def run(self):
 619         while True:
 620             b = self.wait_one()
 621             if options.retry and self.need_retry:
 622                 self.kill_kids()
 623                 do_print("retry needed")
 624                 return (0, None, None, None, "retry")
 625             if b is None:
 626                 break
 627             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 628                 self.kill_kids()
 629                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 630             b.start_next()
 631         self.kill_kids()
 632         return (0, None, None, None, "All OK")
 633
 634     def write_system_info(self):
 635         filename = 'system-info.txt'
 636         f = open(filename, 'w')
 637         for cmd in ['uname -a',
 638                     'lsb_release -a',
 639                     'free',
 640                     'mount',
 641                     'cat /proc/cpuinfo',
 642                     'cc --version',
 643                     'df -m .',
 644                     'df -m %s' % testbase]:
 645             out = run_cmd(cmd, output=True, checkfail=False)
 646             print('### %s' % cmd, file=f)
 647             print(out.decode('utf8', 'backslashreplace'), file=f)
 648             print(file=f)
 649         f.close()
 650         return filename
 651
 652     def tarlogs(self, fname):
 653         tar = tarfile.open(fname, "w:gz")
 654         for b in self.tlist:
 655             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 656             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 657         if os.path.exists("autobuild.log"):
 658             tar.add("autobuild.log")
 659         sys_info = self.write_system_info()
 660         tar.add(sys_info)
 661         tar.close()
 662
 663     def remove_logs(self):
 664         for b in self.tlist:
 665             os.unlink(b.stdout_path)
 666             os.unlink(b.stderr_path)
 667
 668     def start_tail(self):
 669         cmd = ["tail", "-f"]
 670         for b in self.tlist:
 671             cmd.append(b.stdout_path)
 672             cmd.append(b.stderr_path)
 673         self.tail_proc = Popen(cmd, close_fds=True)
 674
 675
 676 def cleanup():
 677     if options.nocleanup:
 678         return
 679     run_cmd("stat %s || true" % test_tmpdir, show=True)
 680     run_cmd("stat %s" % testbase, show=True)
 681     do_print("Cleaning up %r" % cleanup_list)
 682     for d in cleanup_list:
 683         run_cmd("rm -rf %s" % d)
 684
 685
 686 def find_git_root():
 687     '''get to the top of the git repo'''
 688     p = os.getcwd()
 689     while p != '/':
 690         if os.path.isdir(os.path.join(p, ".git")):
 691             return p
 692         p = os.path.abspath(os.path.join(p, '..'))
 693     return None
 694
 695
 696 def daemonize(logfile):
 697     pid = os.fork()
 698     if pid == 0:  # Parent
 699         os.setsid()
 700         pid = os.fork()
 701         if pid != 0:  # Actual daemon
 702             os._exit(0)
 703     else:  # Grandparent
 704         os._exit(0)
 705
 706     import resource      # Resource usage information.
 707     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 708     if maxfd == resource.RLIM_INFINITY:
 709         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 710     for fd in range(0, maxfd):
 711         try:
 712             os.close(fd)
 713         except OSError:
 714             pass
 715     os.open(logfile, os.O_RDWR | os.O_CREAT)
 716     os.dup2(0, 1)
 717     os.dup2(0, 2)
 718
 719
 720 def write_pidfile(fname):
 721     '''write a pid file, cleanup on exit'''
 722     f = open(fname, mode='w')
 723     f.write("%u\n" % os.getpid())
 724     f.close()
 725
 726
 727 def rebase_tree(rebase_url, rebase_branch="master"):
 728     rebase_remote = "rebaseon"
 729     do_print("Rebasing on %s" % rebase_url)
 730     run_cmd("git describe HEAD", show=True, dir=test_master)
 731     run_cmd("git remote add -t %s %s %s" %
 732             (rebase_branch, rebase_remote, rebase_url),
 733             show=True, dir=test_master)
 734     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 735     if options.fix_whitespace:
 736         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 737                 (rebase_remote, rebase_branch),
 738                 show=True, dir=test_master)
 739     else:
 740         run_cmd("git rebase --force-rebase %s/%s" %
 741                 (rebase_remote, rebase_branch),
 742                 show=True, dir=test_master)
 743     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 744                    (rebase_remote, rebase_branch),
 745                    dir=test_master, output=True)
 746     if diff == '':
 747         do_print("No differences between HEAD and %s/%s - exiting" %
 748                  (rebase_remote, rebase_branch))
 749         sys.exit(0)
 750     run_cmd("git describe %s/%s" %
 751             (rebase_remote, rebase_branch),
 752             show=True, dir=test_master)
 753     run_cmd("git describe HEAD", show=True, dir=test_master)
 754     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 755             (rebase_remote, rebase_branch),
 756             show=True, dir=test_master)
 757
 758
 759 def push_to(push_url, push_branch="master"):
 760     push_remote = "pushto"
 761     do_print("Pushing to %s" % push_url)
 762     if options.mark:
 763         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 764         run_cmd("git commit --amend -c HEAD", dir=test_master)
 765         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 766         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 767     run_cmd("git remote add -t %s %s %s" %
 768             (push_branch, push_remote, push_url),
 769             show=True, dir=test_master)
 770     run_cmd("git push %s +HEAD:%s" %
 771             (push_remote, push_branch),
 772             show=True, dir=test_master)
 773
 774
 775 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 776
 777 gitroot = find_git_root()
 778 if gitroot is None:
 779     raise Exception("Failed to find git root")
 780
 781 parser = OptionParser()
 782 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 783 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 784 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 785 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 786                   default=def_testbase)
 787 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 788 parser.add_option("", "--verbose", help="show all commands as they are run",
 789                   default=False, action="store_true")
 790 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 791                   default=None, type='str')
 792 parser.add_option("", "--pushto", help="push to a git url on success",
 793                   default=None, type='str')
 794 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 795                   default=False, action="store_true")
 796 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 797                   default=False, action="store_true")
 798 parser.add_option("", "--retry", help="automatically retry if master changes",
 799                   default=False, action="store_true")
 800 parser.add_option("", "--email", help="send email to the given address on failure",
 801                   type='str', default=None)
 802 parser.add_option("", "--email-from", help="send email from the given address",
 803                   type='str', default="autobuild@samba.org")
 804 parser.add_option("", "--email-server", help="send email via the given server",
 805                   type='str', default='localhost')
 806 parser.add_option("", "--always-email", help="always send email, even on success",
 807                   action="store_true")
 808 parser.add_option("", "--daemon", help="daemonize after initial setup",
 809                   action="store_true")
 810 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 811                   default="master", type='str')
 812 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 813                   default=gitroot, type='str')
 814 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 815                   default=False, action="store_true")
 816 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 817                   default='')
 818
 819
 820 def send_email(subject, text, log_tar):
 821     if options.email is None:
 822         do_print("not sending email because the recipient is not set")
 823         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 824                  (subject, text))
 825         return
 826     outer = MIMEMultipart()
 827     outer['Subject'] = subject
 828     outer['To'] = options.email
 829     outer['From'] = options.email_from
 830     outer['Date'] = email.utils.formatdate(localtime=True)
 831     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 832     outer.attach(MIMEText(text, 'plain'))
 833     if options.attach_logs:
 834         fp = open(log_tar, 'rb')
 835         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 836         fp.close()
 837         # Set the filename parameter
 838         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 839         outer.attach(msg)
 840     content = outer.as_string()
 841     s = smtplib.SMTP(options.email_server)
 842     s.sendmail(options.email_from, [options.email], content)
 843     s.set_debuglevel(1)
 844     s.quit()
 845
 846
 847 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 848                   elapsed_time, log_base=None, add_log_tail=True):
 849     '''send an email to options.email about the failure'''
 850     elapsed_minutes = elapsed_time / 60.0
 851     if log_base is None:
 852         log_base = gitroot
 853     text = '''
 854 Dear Developer,
 855
 856 Your autobuild on %s failed after %.1f minutes
 857 when trying to test %s with the following error:
 858
 859    %s
 860
 861 the autobuild has been abandoned. Please fix the error and resubmit.
 862
 863 A summary of the autobuild process is here:
 864
 865   %s/autobuild.log
 866 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 867
 868     if options.restrict_tests:
 869         text += """
 870 The build was restricted to tests matching %s\n""" % options.restrict_tests
 871
 872     if failed_task != 'rebase':
 873         text += '''
 874 You can see logs of the failed task here:
 875
 876   %s/%s.stdout
 877   %s/%s.stderr
 878
 879 or you can get full logs of all tasks in this job here:
 880
 881   %s/logs.tar.gz
 882
 883 The top commit for the tree that was built was:
 884
 885 %s
 886
 887 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 888
 889     if add_log_tail:
 890         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 891         lines = f.readlines()
 892         log_tail = "".join(lines[-50:])
 893         num_lines = len(lines)
 894         if num_lines < 50:
 895             # Also include stderr (compile failures) if < 50 lines of stdout
 896             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 897             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 898
 899         text += '''
 900 The last 50 lines of log messages:
 901
 902 %s
 903     ''' % log_tail
 904         f.close()
 905
 906     logs = os.path.join(gitroot, 'logs.tar.gz')
 907     send_email('autobuild[%s] failure on %s for task %s during %s'
 908                % (options.branch, platform.node(), failed_task, failed_stage),
 909                text, logs)
 910
 911
 912 def email_success(elapsed_time, log_base=None):
 913     '''send an email to options.email about a successful build'''
 914     if log_base is None:
 915         log_base = gitroot
 916     text = '''
 917 Dear Developer,
 918
 919 Your autobuild on %s has succeeded after %.1f minutes.
 920
 921 ''' % (platform.node(), elapsed_time / 60.)
 922
 923     if options.restrict_tests:
 924         text += """
 925 The build was restricted to tests matching %s\n""" % options.restrict_tests
 926
 927     if options.keeplogs:
 928         text += '''
 929
 930 you can get full logs of all tasks in this job here:
 931
 932   %s/logs.tar.gz
 933
 934 ''' % log_base
 935
 936     text += '''
 937 The top commit for the tree that was built was:
 938
 939 %s
 940 ''' % top_commit_msg
 941
 942     logs = os.path.join(gitroot, 'logs.tar.gz')
 943     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 944                text, logs)
 945
 946
 947 (options, args) = parser.parse_args()
 948
 949 if options.retry:
 950     if options.rebase is None:
 951         raise Exception('You can only use --retry if you also rebase')
 952
 953 testbase = "%s/b%u" % (options.testbase, os.getpid())
 954 test_master = "%s/master" % testbase
 955 test_prefix = "%s/prefix" % testbase
 956 test_tmpdir = "%s/tmp" % testbase
 957 os.environ['TMPDIR'] = test_tmpdir
 958
 959 # get the top commit message, for emails
 960 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 961 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
 962
 963 try:
 964     os.makedirs(testbase)
 965 except Exception as reason:
 966     raise Exception("Unable to create %s : %s" % (testbase, reason))
 967 cleanup_list.append(testbase)
 968
 969 if options.daemon:
 970     logfile = os.path.join(testbase, "log")
 971     do_print("Forking into the background, writing progress to %s" % logfile)
 972     daemonize(logfile)
 973
 974 write_pidfile(gitroot + "/autobuild.pid")
 975
 976 start_time = time.time()
 977
 978 while True:
 979     try:
 980         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 981         os.makedirs(test_tmpdir)
 982         # The waf uninstall code removes empty directories all the way
 983         # up the tree.  Creating a file in test_tmpdir stops it from
 984         # being removed.
 985         run_cmd("touch %s" % os.path.join(test_tmpdir,
 986                                           ".directory-is-not-empty"), show=True)
 987         run_cmd("stat %s" % test_tmpdir, show=True)
 988         run_cmd("stat %s" % testbase, show=True)
 989         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 990     except Exception:
 991         cleanup()
 992         raise
 993
 994     try:
 995         try:
 996             if options.rebase is not None:
 997                 rebase_tree(options.rebase, rebase_branch=options.branch)
 998         except Exception:
 999             cleanup_list.append(gitroot + "/autobuild.pid")
1000             cleanup()
1001             elapsed_time = time.time() - start_time
1002             email_failure(-1, 'rebase', 'rebase', 'rebase',
1003                           'rebase on %s failed' % options.branch,
1004                           elapsed_time, log_base=options.log_base)
1005             sys.exit(1)
1006         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1007         if options.tail:
1008             blist.start_tail()
1009         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1010         if status != 0 or errstr != "retry":
1011             break
1012         cleanup()
1013     except Exception:
1014         cleanup()
1015         raise
1016
1017 cleanup_list.append(gitroot + "/autobuild.pid")
1018
1019 do_print(errstr)
1020
1021 blist.kill_kids()
1022 if options.tail:
1023     do_print("waiting for tail to flush")
1024     time.sleep(1)
1025
1026 elapsed_time = time.time() - start_time
1027 if status == 0:
1028     if options.passcmd is not None:
1029         do_print("Running passcmd: %s" % options.passcmd)
1030         run_cmd(options.passcmd, dir=test_master)
1031     if options.pushto is not None:
1032         push_to(options.pushto, push_branch=options.branch)
1033     if options.keeplogs or options.attach_logs:
1034         blist.tarlogs("logs.tar.gz")
1035         do_print("Logs in logs.tar.gz")
1036     if options.always_email:
1037         email_success(elapsed_time, log_base=options.log_base)
1038     blist.remove_logs()
1039     cleanup()
1040     do_print(errstr)
1041     sys.exit(0)
1042
1043 # something failed, gather a tar of the logs
1044 blist.tarlogs("logs.tar.gz")
1045
1046 if options.email is not None:
1047     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1048                   elapsed_time, log_base=options.log_base)
1049 else:
1050     elapsed_minutes = elapsed_time / 60.0
1051     print('''
1052
1053 ####################################################################
1054
1055 AUTOBUILD FAILURE
1056
1057 Your autobuild[%s] on %s failed after %.1f minutes
1058 when trying to test %s with the following error:
1059
1060    %s
1061
1062 the autobuild has been abandoned. Please fix the error and resubmit.
1063
1064 ####################################################################
1065
1066 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1067
1068 cleanup()
1069 do_print(errstr)
1070 do_print("Logs in logs.tar.gz")
1071 sys.exit(status)