script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-nt4": ".",
  40     "samba-fileserver": ".",
  41     "samba-xc": ".",
  42     "samba-o3": ".",
  43     "samba-ctdb": ".",
  44     "samba-libs": ".",
  45     "samba-static": ".",
  46     "samba-none-env": ".",
  47     "samba-ad-dc": ".",
  48     "samba-ad-dc-ntvfs": ".",
  49     "samba-ad-dc-2": ".",
  50     "samba-ad-dc-backup": ".",
  51     "samba-systemkrb5": ".",
  52     "samba-nopython": ".",
  53     "samba-nopython-py2": ".",
  54     "ldb": "lib/ldb",
  55     "tdb": "lib/tdb",
  56     "talloc": "lib/talloc",
  57     "replace": "lib/replace",
  58     "tevent": "lib/tevent",
  59     "pidl": "pidl"
  60 }
  61
  62 defaulttasks = builddirs.keys()
  63
  64 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  65     defaulttasks.remove("samba-o3")
  66
  67 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  68 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
  69
  70 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  71 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  72 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  73 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  74 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
  75 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  76 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
  77
  78 tasks = {
  79     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  80                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  81                ("make", "make all", "text/plain"),
  82                ("install", "make install", "text/plain"),
  83                ("test", "make autotest", "text/plain"),
  84                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  85                ("clean", "make clean", "text/plain")],
  86
  87     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
  88     "samba": [
  89                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
  90                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  91                 ("make", "make -j", "text/plain"),
  92                 ("test", "make test FAIL_IMMEDIATELY=1 "
  93                  "TESTS='--exclude-env=none "
  94                  "--exclude-env=nt4_dc "
  95                  "--exclude-env=nt4_member "
  96                  "--exclude-env=ad_dc "
  97                  "--exclude-env=ad_dc_ntvfs "
  98                  "--exclude-env=ad_dc_no_nss "
  99                  "--exclude-env=fl2003dc "
 100                  "--exclude-env=fl2008r2dc "
 101                  "--exclude-env=ad_member "
 102                  "--exclude-env=ad_member_idmap_rid "
 103                  "--exclude-env=ad_member_idmap_ad "
 104                  "--exclude-env=chgdcpass "
 105                  "--exclude-env=vampire_2000_dc "
 106                  "--exclude-env=fl2000dc "
 107                  "--exclude-env=fileserver "
 108                  "--exclude-env=backupfromdc "
 109                  "--exclude-env=restoredc "
 110                  "--exclude-env=renamedc "
 111                  "--exclude-env=offlinebackupdc "
 112                  "--exclude-env=labdc "
 113                  "'",
 114                  "text/plain"),
 115                 ("install", "make install", "text/plain"),
 116                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 117                 ("clean", "make clean", "text/plain")],
 118
 119     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 120                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 121                     ("make", "make -j", "text/plain"),
 122                     ("test", "make test FAIL_IMMEDIATELY=1 "
 123                      "TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 124                     ("install", "make install", "text/plain"),
 125                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 126                     ("clean", "make clean", "text/plain")],
 127
 128     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 129                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 130                            ("make", "make -j", "text/plain"),
 131                            ("test", "make test FAIL_IMMEDIATELY=1 "
 132                             "TESTS='--include-env=fileserver'", "text/plain"),
 133                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 134
 135     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 136                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 137                       ("make", "make -j", "text/plain"),
 138                       ("test", "make test FAIL_IMMEDIATELY=1 "
 139                        "TESTS='--include-env=ad_dc "
 140                        "--include-env=fl2003dc "
 141                        "--include-env=fl2008r2dc "
 142                        "--include-env=ad_member "
 143                        "--include-env=ad_member_idmap_rid "
 144                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 145                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 146
 147     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 148                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 149                         ("make", "make -j", "text/plain"),
 150                         ("test", "make test FAIL_IMMEDIATELY=1 "
 151                          "TESTS='--include-env=chgdcpass "
 152                          "--include-env=vampire_2000_dc "
 153                          "--include-env=fl2000dc "
 154                          "--include-env=ad_dc_no_nss "
 155                          "'",
 156                          "text/plain"),
 157                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 158
 159     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 160     # This is currently the longest task, so we don't randomly delay it.
 161     "samba-ad-dc-ntvfs": [
 162                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 163                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 164                       ("make", "make -j", "text/plain"),
 165                       ("test", "make test FAIL_IMMEDIATELY=1 "
 166                        "TESTS='--include-env=ad_dc_ntvfs'",
 167                        "text/plain"),
 168                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 169
 170     # run the backup/restore testenvs separately as they're fairly standalone
 171     # (and CI seems to max out at ~8 different DCs running at once)
 172     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 173                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 174                         ("make", "make -j", "text/plain"),
 175                         ("test", "make test FAIL_IMMEDIATELY=1 "
 176                          "TESTS='--include-env=backupfromdc "
 177                          "--include-env=restoredc "
 178                          "--include-env=renamedc "
 179                          "--include-env=offlinebackupdc "
 180                          "--include-env=labdc "
 181                          "'",
 182                          "text/plain"),
 183                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 184
 185     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 186                           ("make", "make -j", "text/plain"),
 187                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 188
 189     # Test cross-compile infrastructure
 190     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 191                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 192                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 193                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 194                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 195                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 196                    ("compare-results", "script/compare_cc_results.py "
 197                     "./bin/c4che/default{} "
 198                     "./bin-xe/c4che/default{} "
 199                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 200
 201     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 202     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 203                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 204                    ("make", "make -j", "text/plain"),
 205                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 206                     "TESTS='--include-env=ad_dc'", "text/plain"),
 207                    ("install", "make install", "text/plain"),
 208                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 209                    ("clean", "make clean", "text/plain")],
 210
 211     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 212
 213                      # make sure we have tdb around:
 214                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 215                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 216                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 217
 218
 219                      # build samba with cluster support (also building ctdb):
 220                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 221                      ("samba-make", "make", "text/plain"),
 222                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 223                      ("samba-install", "make install", "text/plain"),
 224                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 225
 226                      # clean up:
 227                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 228                      ("clean", "make clean", "text/plain"),
 229                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 230
 231     "samba-libs": [
 232                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 233                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 234                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 235                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 236
 237                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 238                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 239                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 240
 241                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 242                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 243                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 244
 245                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 246                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 247                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 248
 249                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 250                       ("nondevel-make", "make -j", "text/plain"),
 251                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 252                       ("nondevel-install", "make install", "text/plain"),
 253                       ("nondevel-dist", "make dist", "text/plain"),
 254
 255                       # retry with all modules shared
 256                       ("allshared-distclean", "make distclean", "text/plain"),
 257                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 258                       ("allshared-make", "make -j", "text/plain")],
 259
 260     "samba-none-env": [
 261                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 262                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 263                       ("make", "make -j", "text/plain"),
 264                       ("test", "make test "
 265                        "FAIL_IMMEDIATELY=1 "
 266                        "TESTS='--include-env=none'",
 267                        "text/plain")],
 268
 269     "samba-static": [
 270                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 271                       # build with all modules static
 272                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 273                       ("allstatic-make", "make -j", "text/plain"),
 274                       ("allstatic-test", "make test "
 275                        "FAIL_IMMEDIATELY=1 "
 276                        "TESTS='samba3.smb2.create.*nt4_dc'",
 277                        "text/plain"),
 278
 279                       # retry without any required modules
 280                       ("none-distclean", "make distclean", "text/plain"),
 281                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 282                       ("none-make", "make -j", "text/plain"),
 283
 284                       # retry with nonshared smbd and smbtorture
 285                       ("nonshared-distclean", "make distclean", "text/plain"),
 286                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 287                       ("nonshared-make", "make -j", "text/plain")],
 288
 289     "samba-systemkrb5": [
 290                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 291                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 292                       ("make", "make -j", "text/plain"),
 293                       # we currently cannot run a full make test, a limited list of tests could be run
 294                       # via "make test TESTS=sometests"
 295                       ("test", "make test FAIL_IMMEDIATELY=1 "
 296                        "TESTS='--include-env=ktest'", "text/plain"),
 297                       ("install", "make install", "text/plain"),
 298                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 299                       ("clean", "make clean", "text/plain")
 300                       ],
 301
 302     # Test Samba without python still builds.  When this test fails
 303     # due to more use of Python, the expectations is that the newly
 304     # failing part of the code should be disabled when
 305     # --disable-python is set (rather than major work being done to
 306     # support this environment).  The target here is for vendors
 307     # shipping a minimal smbd.
 308     "samba-nopython": [
 309                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 310                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 311                       ("make", "make -j", "text/plain"),
 312                       ("install", "make install", "text/plain"),
 313                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 314                       ("clean", "make clean", "text/plain"),
 315
 316                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 317                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 318                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 319
 320                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 321                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 322                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 323
 324                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 325                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 326                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 327
 328                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 329                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 330                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 331
 332                       # retry against installed library packages
 333                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 334                       ("libs-make", "make -j", "text/plain"),
 335                       ("libs-install", "make install", "text/plain"),
 336                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 337                       ("libs-clean", "make clean", "text/plain")
 338                       ],
 339
 340     # check we can do the same thing using python2
 341     "samba-nopython-py2": [
 342                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 343                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 344                       ("make", "PYTHON=python2 make -j", "text/plain"),
 345                       ("install", "PYTHON=python2 make install", "text/plain"),
 346                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 347                       ("clean", "PYTHON=python2 make clean", "text/plain"),
 348
 349                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 350                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
 351                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
 352
 353                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 354                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
 355                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
 356
 357                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 358                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
 359                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
 360
 361                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 362                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
 363                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
 364
 365                       # retry against installed library packages
 366                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 367                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
 368                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
 369                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 370                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
 371                       ],
 372
 373     "ldb": [
 374               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 375               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 376               ("make", "make", "text/plain"),
 377               ("install", "make install", "text/plain"),
 378               ("test", "make test", "text/plain"),
 379               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
 380               ("make-no-lmdb", "make", "text/plain"),
 381               ("install-no-lmdb", "make install", "text/plain"),
 382               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 383               ("distcheck", "make distcheck", "text/plain"),
 384               ("clean", "make clean", "text/plain")],
 385
 386     "tdb": [
 387               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 388               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 389               ("make", "make", "text/plain"),
 390               ("install", "make install", "text/plain"),
 391               ("test", "make test", "text/plain"),
 392               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 393               ("distcheck", "make distcheck", "text/plain"),
 394               ("clean", "make clean", "text/plain")],
 395
 396     "talloc": [
 397                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 398                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 399                  ("make", "make", "text/plain"),
 400                  ("install", "make install", "text/plain"),
 401                  ("test", "make test", "text/plain"),
 402                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 403                  ("distcheck", "make distcheck", "text/plain"),
 404                  ("clean", "make clean", "text/plain")],
 405
 406     "replace": [
 407                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 408                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 409                   ("make", "make", "text/plain"),
 410                   ("install", "make install", "text/plain"),
 411                   ("test", "make test", "text/plain"),
 412                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 413                   ("distcheck", "make distcheck", "text/plain"),
 414                   ("clean", "make clean", "text/plain")],
 415
 416     "tevent": [
 417                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 418                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 419                  ("make", "make", "text/plain"),
 420                  ("install", "make install", "text/plain"),
 421                  ("test", "make test", "text/plain"),
 422                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 423                  ("distcheck", "make distcheck", "text/plain"),
 424                  ("clean", "make clean", "text/plain")],
 425
 426     "pidl": [
 427         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 428         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 429         ("touch", "touch *.yp", "text/plain"),
 430         ("make", "make", "text/plain"),
 431         ("test", "make test", "text/plain"),
 432         ("install", "make install", "text/plain"),
 433         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 434         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 435         ("clean", "make clean", "text/plain")],
 436
 437
 438     # these are useful for debugging autobuild
 439     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 440     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 441
 442
 443 }
 444
 445
 446 def do_print(msg):
 447     print("%s" % msg)
 448     sys.stdout.flush()
 449     sys.stderr.flush()
 450
 451
 452 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 453     if show is None:
 454         show = options.verbose
 455     if show:
 456         do_print("Running: '%s' in '%s'" % (cmd, dir))
 457     if output:
 458         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 459     elif checkfail:
 460         return check_call(cmd, shell=True, cwd=dir)
 461     else:
 462         return call(cmd, shell=True, cwd=dir)
 463
 464
 465 class builder(object):
 466     '''handle build of one directory'''
 467
 468     def __init__(self, name, sequence, cp=True):
 469         self.name = name
 470         if name in builddirs:
 471             self.dir = builddirs[name]
 472         else:
 473             self.dir = "."
 474
 475         self.tag = self.name.replace('/', '_')
 476         self.sequence = sequence
 477         self.next = 0
 478         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 479         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 480         if options.verbose:
 481             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 482             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 483         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 484         self.stdout = open(self.stdout_path, 'w')
 485         self.stderr = open(self.stderr_path, 'w')
 486         self.stdin  = open("/dev/null", 'r')
 487         self.sdir = "%s/%s" % (testbase, self.tag)
 488         self.prefix = "%s/%s" % (test_prefix, self.tag)
 489         run_cmd("rm -rf %s" % self.sdir)
 490         run_cmd("rm -rf %s" % self.prefix)
 491         if cp:
 492             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 493         else:
 494             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 495         self.start_next()
 496
 497     def start_next(self):
 498         if self.next == len(self.sequence):
 499             if not options.nocleanup:
 500                 run_cmd("rm -rf %s" % self.sdir)
 501                 run_cmd("rm -rf %s" % self.prefix)
 502             do_print('%s: Completed OK' % self.name)
 503             self.done = True
 504             return
 505         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 506         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 507         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 508         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 509         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 510 #        if self.output_mime_type == "text/x-subunit":
 511 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 512         cwd = os.getcwd()
 513         os.chdir("%s/%s" % (self.sdir, self.dir))
 514         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 515         self.proc = Popen(self.cmd, shell=True,
 516                           close_fds=True,
 517                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 518         os.chdir(cwd)
 519         self.next += 1
 520
 521
 522 class buildlist(object):
 523     '''handle build of multiple directories'''
 524
 525     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 526         global tasks
 527         self.tlist = []
 528         self.tail_proc = None
 529         self.retry = None
 530         if tasknames == []:
 531             if options.restrict_tests:
 532                 tasknames = ["samba-test-only"]
 533             else:
 534                 tasknames = defaulttasks
 535         else:
 536             # If we are only running one test,
 537             # do not sleep randomly to wait for it to start
 538             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 539
 540         for n in tasknames:
 541             b = builder(n, tasks[n], cp=n is not "pidl")
 542             self.tlist.append(b)
 543         if options.retry:
 544             rebase_remote = "rebaseon"
 545             retry_task = [("retry",
 546                             '''set -e
 547                             git remote add -t %s %s %s
 548                             git fetch %s
 549                             while :; do
 550                               sleep 60
 551                               git describe %s/%s > old_remote_branch.desc
 552                               git fetch %s
 553                               git describe %s/%s > remote_branch.desc
 554                               diff old_remote_branch.desc remote_branch.desc
 555                             done
 556                            ''' % (
 557                                 rebase_branch, rebase_remote, rebase_url,
 558                                rebase_remote,
 559                                rebase_remote, rebase_branch,
 560                                rebase_remote,
 561                                rebase_remote, rebase_branch
 562                             ),
 563                             "test/plain")]
 564
 565             self.retry = builder('retry', retry_task, cp=False)
 566             self.need_retry = False
 567
 568     def kill_kids(self):
 569         if self.tail_proc is not None:
 570             self.tail_proc.terminate()
 571             self.tail_proc.wait()
 572             self.tail_proc = None
 573         if self.retry is not None:
 574             self.retry.proc.terminate()
 575             self.retry.proc.wait()
 576             self.retry = None
 577         for b in self.tlist:
 578             if b.proc is not None:
 579                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 580                 b.proc.terminate()
 581                 b.proc.wait()
 582                 b.proc = None
 583
 584     def wait_one(self):
 585         while True:
 586             none_running = True
 587             for b in self.tlist:
 588                 if b.proc is None:
 589                     continue
 590                 none_running = False
 591                 b.status = b.proc.poll()
 592                 if b.status is None:
 593                     continue
 594                 b.proc = None
 595                 return b
 596             if options.retry:
 597                 ret = self.retry.proc.poll()
 598                 if ret is not None:
 599                     self.need_retry = True
 600                     self.retry = None
 601                     return None
 602             if none_running:
 603                 return None
 604             time.sleep(0.1)
 605
 606     def run(self):
 607         while True:
 608             b = self.wait_one()
 609             if options.retry and self.need_retry:
 610                 self.kill_kids()
 611                 do_print("retry needed")
 612                 return (0, None, None, None, "retry")
 613             if b is None:
 614                 break
 615             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 616                 self.kill_kids()
 617                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 618             b.start_next()
 619         self.kill_kids()
 620         return (0, None, None, None, "All OK")
 621
 622     def write_system_info(self):
 623         filename = 'system-info.txt'
 624         f = open(filename, 'w')
 625         for cmd in ['uname -a',
 626                     'lsb_release -a',
 627                     'free',
 628                     'mount',
 629                     'cat /proc/cpuinfo',
 630                     'cc --version',
 631                     'df -m .',
 632                     'df -m %s' % testbase]:
 633             out = run_cmd(cmd, output=True, checkfail=False)
 634             print('### %s' % cmd, file=f)
 635             print(out.decode('utf8', 'backslashreplace'), file=f)
 636             print(file=f)
 637         f.close()
 638         return filename
 639
 640     def tarlogs(self, fname):
 641         tar = tarfile.open(fname, "w:gz")
 642         for b in self.tlist:
 643             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 644             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 645         if os.path.exists("autobuild.log"):
 646             tar.add("autobuild.log")
 647         sys_info = self.write_system_info()
 648         tar.add(sys_info)
 649         tar.close()
 650
 651     def remove_logs(self):
 652         for b in self.tlist:
 653             os.unlink(b.stdout_path)
 654             os.unlink(b.stderr_path)
 655
 656     def start_tail(self):
 657         cmd = ["tail", "-f"]
 658         for b in self.tlist:
 659             cmd.append(b.stdout_path)
 660             cmd.append(b.stderr_path)
 661         self.tail_proc = Popen(cmd, close_fds=True)
 662
 663
 664 def cleanup():
 665     if options.nocleanup:
 666         return
 667     run_cmd("stat %s || true" % test_tmpdir, show=True)
 668     run_cmd("stat %s" % testbase, show=True)
 669     do_print("Cleaning up %r" % cleanup_list)
 670     for d in cleanup_list:
 671         run_cmd("rm -rf %s" % d)
 672
 673
 674 def find_git_root():
 675     '''get to the top of the git repo'''
 676     p = os.getcwd()
 677     while p != '/':
 678         if os.path.isdir(os.path.join(p, ".git")):
 679             return p
 680         p = os.path.abspath(os.path.join(p, '..'))
 681     return None
 682
 683
 684 def daemonize(logfile):
 685     pid = os.fork()
 686     if pid == 0:  # Parent
 687         os.setsid()
 688         pid = os.fork()
 689         if pid != 0:  # Actual daemon
 690             os._exit(0)
 691     else:  # Grandparent
 692         os._exit(0)
 693
 694     import resource      # Resource usage information.
 695     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 696     if maxfd == resource.RLIM_INFINITY:
 697         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 698     for fd in range(0, maxfd):
 699         try:
 700             os.close(fd)
 701         except OSError:
 702             pass
 703     os.open(logfile, os.O_RDWR | os.O_CREAT)
 704     os.dup2(0, 1)
 705     os.dup2(0, 2)
 706
 707
 708 def write_pidfile(fname):
 709     '''write a pid file, cleanup on exit'''
 710     f = open(fname, mode='w')
 711     f.write("%u\n" % os.getpid())
 712     f.close()
 713
 714
 715 def rebase_tree(rebase_url, rebase_branch="master"):
 716     rebase_remote = "rebaseon"
 717     do_print("Rebasing on %s" % rebase_url)
 718     run_cmd("git describe HEAD", show=True, dir=test_master)
 719     run_cmd("git remote add -t %s %s %s" %
 720             (rebase_branch, rebase_remote, rebase_url),
 721             show=True, dir=test_master)
 722     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 723     if options.fix_whitespace:
 724         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 725                 (rebase_remote, rebase_branch),
 726                 show=True, dir=test_master)
 727     else:
 728         run_cmd("git rebase --force-rebase %s/%s" %
 729                 (rebase_remote, rebase_branch),
 730                 show=True, dir=test_master)
 731     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 732                    (rebase_remote, rebase_branch),
 733                    dir=test_master, output=True)
 734     if diff == '':
 735         do_print("No differences between HEAD and %s/%s - exiting" %
 736                  (rebase_remote, rebase_branch))
 737         sys.exit(0)
 738     run_cmd("git describe %s/%s" %
 739             (rebase_remote, rebase_branch),
 740             show=True, dir=test_master)
 741     run_cmd("git describe HEAD", show=True, dir=test_master)
 742     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 743             (rebase_remote, rebase_branch),
 744             show=True, dir=test_master)
 745
 746
 747 def push_to(push_url, push_branch="master"):
 748     push_remote = "pushto"
 749     do_print("Pushing to %s" % push_url)
 750     if options.mark:
 751         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 752         run_cmd("git commit --amend -c HEAD", dir=test_master)
 753         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 754         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 755     run_cmd("git remote add -t %s %s %s" %
 756             (push_branch, push_remote, push_url),
 757             show=True, dir=test_master)
 758     run_cmd("git push %s +HEAD:%s" %
 759             (push_remote, push_branch),
 760             show=True, dir=test_master)
 761
 762
 763 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 764
 765 gitroot = find_git_root()
 766 if gitroot is None:
 767     raise Exception("Failed to find git root")
 768
 769 parser = OptionParser()
 770 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 771 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 772 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 773 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 774                   default=def_testbase)
 775 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 776 parser.add_option("", "--verbose", help="show all commands as they are run",
 777                   default=False, action="store_true")
 778 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 779                   default=None, type='str')
 780 parser.add_option("", "--pushto", help="push to a git url on success",
 781                   default=None, type='str')
 782 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 783                   default=False, action="store_true")
 784 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 785                   default=False, action="store_true")
 786 parser.add_option("", "--retry", help="automatically retry if master changes",
 787                   default=False, action="store_true")
 788 parser.add_option("", "--email", help="send email to the given address on failure",
 789                   type='str', default=None)
 790 parser.add_option("", "--email-from", help="send email from the given address",
 791                   type='str', default="autobuild@samba.org")
 792 parser.add_option("", "--email-server", help="send email via the given server",
 793                   type='str', default='localhost')
 794 parser.add_option("", "--always-email", help="always send email, even on success",
 795                   action="store_true")
 796 parser.add_option("", "--daemon", help="daemonize after initial setup",
 797                   action="store_true")
 798 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 799                   default="master", type='str')
 800 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 801                   default=gitroot, type='str')
 802 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 803                   default=False, action="store_true")
 804 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 805                   default='')
 806
 807
 808 def send_email(subject, text, log_tar):
 809     if options.email is None:
 810         do_print("not sending email because the recipient is not set")
 811         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 812                  (subject, text))
 813         return
 814     outer = MIMEMultipart()
 815     outer['Subject'] = subject
 816     outer['To'] = options.email
 817     outer['From'] = options.email_from
 818     outer['Date'] = email.utils.formatdate(localtime=True)
 819     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 820     outer.attach(MIMEText(text, 'plain'))
 821     if options.attach_logs:
 822         fp = open(log_tar, 'rb')
 823         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 824         fp.close()
 825         # Set the filename parameter
 826         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 827         outer.attach(msg)
 828     content = outer.as_string()
 829     s = smtplib.SMTP(options.email_server)
 830     s.sendmail(options.email_from, [options.email], content)
 831     s.set_debuglevel(1)
 832     s.quit()
 833
 834
 835 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 836                   elapsed_time, log_base=None, add_log_tail=True):
 837     '''send an email to options.email about the failure'''
 838     elapsed_minutes = elapsed_time / 60.0
 839     if log_base is None:
 840         log_base = gitroot
 841     text = '''
 842 Dear Developer,
 843
 844 Your autobuild on %s failed after %.1f minutes
 845 when trying to test %s with the following error:
 846
 847    %s
 848
 849 the autobuild has been abandoned. Please fix the error and resubmit.
 850
 851 A summary of the autobuild process is here:
 852
 853   %s/autobuild.log
 854 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 855
 856     if options.restrict_tests:
 857         text += """
 858 The build was restricted to tests matching %s\n""" % options.restrict_tests
 859
 860     if failed_task != 'rebase':
 861         text += '''
 862 You can see logs of the failed task here:
 863
 864   %s/%s.stdout
 865   %s/%s.stderr
 866
 867 or you can get full logs of all tasks in this job here:
 868
 869   %s/logs.tar.gz
 870
 871 The top commit for the tree that was built was:
 872
 873 %s
 874
 875 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 876
 877     if add_log_tail:
 878         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 879         lines = f.readlines()
 880         log_tail = "".join(lines[-50:])
 881         num_lines = len(lines)
 882         if num_lines < 50:
 883             # Also include stderr (compile failures) if < 50 lines of stdout
 884             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 885             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 886
 887         text += '''
 888 The last 50 lines of log messages:
 889
 890 %s
 891     ''' % log_tail
 892         f.close()
 893
 894     logs = os.path.join(gitroot, 'logs.tar.gz')
 895     send_email('autobuild[%s] failure on %s for task %s during %s'
 896                % (options.branch, platform.node(), failed_task, failed_stage),
 897                text, logs)
 898
 899
 900 def email_success(elapsed_time, log_base=None):
 901     '''send an email to options.email about a successful build'''
 902     if log_base is None:
 903         log_base = gitroot
 904     text = '''
 905 Dear Developer,
 906
 907 Your autobuild on %s has succeeded after %.1f minutes.
 908
 909 ''' % (platform.node(), elapsed_time / 60.)
 910
 911     if options.restrict_tests:
 912         text += """
 913 The build was restricted to tests matching %s\n""" % options.restrict_tests
 914
 915     if options.keeplogs:
 916         text += '''
 917
 918 you can get full logs of all tasks in this job here:
 919
 920   %s/logs.tar.gz
 921
 922 ''' % log_base
 923
 924     text += '''
 925 The top commit for the tree that was built was:
 926
 927 %s
 928 ''' % top_commit_msg
 929
 930     logs = os.path.join(gitroot, 'logs.tar.gz')
 931     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 932                text, logs)
 933
 934
 935 (options, args) = parser.parse_args()
 936
 937 if options.retry:
 938     if options.rebase is None:
 939         raise Exception('You can only use --retry if you also rebase')
 940
 941 testbase = "%s/b%u" % (options.testbase, os.getpid())
 942 test_master = "%s/master" % testbase
 943 test_prefix = "%s/prefix" % testbase
 944 test_tmpdir = "%s/tmp" % testbase
 945 os.environ['TMPDIR'] = test_tmpdir
 946
 947 # get the top commit message, for emails
 948 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 949 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
 950
 951 try:
 952     os.makedirs(testbase)
 953 except Exception as reason:
 954     raise Exception("Unable to create %s : %s" % (testbase, reason))
 955 cleanup_list.append(testbase)
 956
 957 if options.daemon:
 958     logfile = os.path.join(testbase, "log")
 959     do_print("Forking into the background, writing progress to %s" % logfile)
 960     daemonize(logfile)
 961
 962 write_pidfile(gitroot + "/autobuild.pid")
 963
 964 start_time = time.time()
 965
 966 while True:
 967     try:
 968         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 969         os.makedirs(test_tmpdir)
 970         # The waf uninstall code removes empty directories all the way
 971         # up the tree.  Creating a file in test_tmpdir stops it from
 972         # being removed.
 973         run_cmd("touch %s" % os.path.join(test_tmpdir,
 974                                           ".directory-is-not-empty"), show=True)
 975         run_cmd("stat %s" % test_tmpdir, show=True)
 976         run_cmd("stat %s" % testbase, show=True)
 977         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 978     except Exception:
 979         cleanup()
 980         raise
 981
 982     try:
 983         try:
 984             if options.rebase is not None:
 985                 rebase_tree(options.rebase, rebase_branch=options.branch)
 986         except Exception:
 987             cleanup_list.append(gitroot + "/autobuild.pid")
 988             cleanup()
 989             elapsed_time = time.time() - start_time
 990             email_failure(-1, 'rebase', 'rebase', 'rebase',
 991                           'rebase on %s failed' % options.branch,
 992                           elapsed_time, log_base=options.log_base)
 993             sys.exit(1)
 994         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 995         if options.tail:
 996             blist.start_tail()
 997         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 998         if status != 0 or errstr != "retry":
 999             break
1000         cleanup()
1001     except Exception:
1002         cleanup()
1003         raise
1004
1005 cleanup_list.append(gitroot + "/autobuild.pid")
1006
1007 do_print(errstr)
1008
1009 blist.kill_kids()
1010 if options.tail:
1011     do_print("waiting for tail to flush")
1012     time.sleep(1)
1013
1014 elapsed_time = time.time() - start_time
1015 if status == 0:
1016     if options.passcmd is not None:
1017         do_print("Running passcmd: %s" % options.passcmd)
1018         run_cmd(options.passcmd, dir=test_master)
1019     if options.pushto is not None:
1020         push_to(options.pushto, push_branch=options.branch)
1021     if options.keeplogs or options.attach_logs:
1022         blist.tarlogs("logs.tar.gz")
1023         do_print("Logs in logs.tar.gz")
1024     if options.always_email:
1025         email_success(elapsed_time, log_base=options.log_base)
1026     blist.remove_logs()
1027     cleanup()
1028     do_print(errstr)
1029     sys.exit(0)
1030
1031 # something failed, gather a tar of the logs
1032 blist.tarlogs("logs.tar.gz")
1033
1034 if options.email is not None:
1035     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1036                   elapsed_time, log_base=options.log_base)
1037 else:
1038     elapsed_minutes = elapsed_time / 60.0
1039     print('''
1040
1041 ####################################################################
1042
1043 AUTOBUILD FAILURE
1044
1045 Your autobuild[%s] on %s failed after %.1f minutes
1046 when trying to test %s with the following error:
1047
1048    %s
1049
1050 the autobuild has been abandoned. Please fix the error and resubmit.
1051
1052 ####################################################################
1053
1054 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1055
1056 cleanup()
1057 do_print(errstr)
1058 do_print("Logs in logs.tar.gz")
1059 sys.exit(status)