script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-nt4": ".",
  40     "samba-fileserver": ".",
  41     "samba-xc": ".",
  42     "samba-o3": ".",
  43     "samba-ctdb": ".",
  44     "samba-libs": ".",
  45     "samba-static": ".",
  46     "samba-none-env": ".",
  47     "samba-ad-dc": ".",
  48     "samba-ad-dc-ntvfs": ".",
  49     "samba-ad-dc-2": ".",
  50     "samba-ad-dc-backup": ".",
  51     "samba-systemkrb5": ".",
  52     "samba-nopython": ".",
  53     "samba-nopython-py2": ".",
  54     "ldb": "lib/ldb",
  55     "tdb": "lib/tdb",
  56     "talloc": "lib/talloc",
  57     "replace": "lib/replace",
  58     "tevent": "lib/tevent",
  59     "pidl": "pidl"
  60 }
  61
  62 defaulttasks = builddirs.keys()
  63
  64 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  65     defaulttasks.remove("samba-o3")
  66
  67 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  68 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
  69
  70 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  71 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  72 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  73 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  74 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
  75 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  76 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
  77
  78 tasks = {
  79     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  80                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  81                ("make", "make all", "text/plain"),
  82                ("install", "make install", "text/plain"),
  83                ("test", "make autotest", "text/plain"),
  84                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  85                ("clean", "make clean", "text/plain")],
  86
  87     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
  88     "samba": [
  89                 ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
  90                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  91                 ("make", "make -j", "text/plain"),
  92                 ("test", "make test FAIL_IMMEDIATELY=1 "
  93                  "TESTS='--exclude-env=none "
  94                  "--exclude-env=nt4_dc "
  95                  "--exclude-env=nt4_member "
  96                  "--exclude-env=ad_dc "
  97                  "--exclude-env=ad_dc_ntvfs "
  98                  "--exclude-env=ad_dc_no_nss "
  99                  "--exclude-env=fl2003dc "
 100                  "--exclude-env=fl2008r2dc "
 101                  "--exclude-env=ad_member "
 102                  "--exclude-env=ad_member_idmap_rid "
 103                  "--exclude-env=ad_member_idmap_ad "
 104                  "--exclude-env=chgdcpass "
 105                  "--exclude-env=vampire_2000_dc "
 106                  "--exclude-env=fl2000dc "
 107                  "--exclude-env=fileserver "
 108                  "--exclude-env=backupfromdc "
 109                  "--exclude-env=restoredc "
 110                  "--exclude-env=renamedc "
 111                  "--exclude-env=offlinebackupdc "
 112                  "--exclude-env=labdc "
 113                  "'",
 114                  "text/plain"),
 115                 ("install", "make install", "text/plain"),
 116                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 117                 ("clean", "make clean", "text/plain")],
 118
 119     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 120                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 121                     ("make", "make -j", "text/plain"),
 122                     ("test", "make test FAIL_IMMEDIATELY=1 "
 123                      "TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 124                     ("install", "make install", "text/plain"),
 125                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 126                     ("clean", "make clean", "text/plain")],
 127
 128     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 129                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 130                            ("make", "make -j", "text/plain"),
 131                            ("test", "make test FAIL_IMMEDIATELY=1 "
 132                             "TESTS='--include-env=fileserver'", "text/plain"),
 133                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 134
 135     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 136                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 137                       ("make", "make -j", "text/plain"),
 138                       ("test", "make test FAIL_IMMEDIATELY=1 "
 139                        "TESTS='--include-env=ad_dc "
 140                        "--include-env=fl2003dc "
 141                        "--include-env=fl2008r2dc "
 142                        "--include-env=ad_member "
 143                        "--include-env=ad_member_idmap_rid "
 144                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 145                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 146
 147     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 148                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 149                         ("make", "make -j", "text/plain"),
 150                         ("test", "make test FAIL_IMMEDIATELY=1 "
 151                          "TESTS='--include-env=chgdcpass "
 152                          "--include-env=vampire_2000_dc "
 153                          "--include-env=fl2000dc "
 154                          "--include-env=ad_dc_no_nss "
 155                          "'",
 156                          "text/plain"),
 157                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 158
 159     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 160     # This is currently the longest task, so we don't randomly delay it.
 161     "samba-ad-dc-ntvfs": [
 162                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 163                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 164                       ("make", "make -j", "text/plain"),
 165                       ("test", "make test FAIL_IMMEDIATELY=1 "
 166                        "TESTS='--include-env=ad_dc_ntvfs'",
 167                        "text/plain"),
 168                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 169
 170     # run the backup/restore testenvs separately as they're fairly standalone
 171     # (and CI seems to max out at ~8 different DCs running at once)
 172     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 173                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 174                         ("make", "make -j", "text/plain"),
 175                         ("test", "make test FAIL_IMMEDIATELY=1 "
 176                          "TESTS='--include-env=backupfromdc "
 177                          "--include-env=restoredc "
 178                          "--include-env=renamedc "
 179                          "--include-env=offlinebackupdc "
 180                          "--include-env=labdc "
 181                          "'",
 182                          "text/plain"),
 183                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 184
 185     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 186                           ("make", "make -j", "text/plain"),
 187                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 188
 189     # Test cross-compile infrastructure
 190     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 191                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 192                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 193                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 194                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 195                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 196                    ("compare-results", "script/compare_cc_results.py "
 197                     "./bin/c4che/default{} "
 198                     "./bin-xe/c4che/default{} "
 199                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 200
 201     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 202     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 203                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 204                    ("make", "make -j", "text/plain"),
 205                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 206                     "TESTS='--include-env=ad_dc'", "text/plain"),
 207                    ("install", "make install", "text/plain"),
 208                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 209                    ("clean", "make clean", "text/plain")],
 210
 211     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 212
 213                      # make sure we have tdb around:
 214                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 215                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 216                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 217
 218
 219                      # build samba with cluster support (also building ctdb):
 220                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 221                      ("samba-make", "make", "text/plain"),
 222                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 223                      ("samba-install", "make install", "text/plain"),
 224                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 225
 226                      # clean up:
 227                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 228                      ("clean", "make clean", "text/plain"),
 229                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 230
 231     "samba-libs": [
 232                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 233                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 234                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 235                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 236
 237                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 238                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 239                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 240
 241                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 242                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 243                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 244
 245                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 246                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 247                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 248
 249                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 250                       ("nondevel-make", "make -j", "text/plain"),
 251                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 252                       ("nondevel-install", "make install", "text/plain"),
 253                       ("nondevel-dist", "make dist", "text/plain"),
 254
 255                       # retry with all modules shared
 256                       ("allshared-distclean", "make distclean", "text/plain"),
 257                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 258                       ("allshared-make", "make -j", "text/plain")],
 259
 260     "samba-none-env": [
 261                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 262                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 263                       ("make", "make -j", "text/plain"),
 264                       ("test", "make test "
 265                        "FAIL_IMMEDIATELY=1 "
 266                        "TESTS='--include-env=none'",
 267                        "text/plain")],
 268
 269     "samba-static": [
 270                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 271                       # build with all modules static
 272                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 273                       ("allstatic-make", "make -j", "text/plain"),
 274                       ("allstatic-test", "make test "
 275                        "FAIL_IMMEDIATELY=1 "
 276                        "TESTS='samba3.smb2.create.*nt4_dc'",
 277                        "text/plain"),
 278
 279                       # retry without any required modules
 280                       ("none-distclean", "make distclean", "text/plain"),
 281                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 282                       ("none-make", "make -j", "text/plain"),
 283
 284                       # retry with nonshared smbd and smbtorture
 285                       ("nonshared-distclean", "make distclean", "text/plain"),
 286                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 287                       ("nonshared-make", "make -j", "text/plain")],
 288
 289     "samba-systemkrb5": [
 290                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 291                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 292                       ("make", "make -j", "text/plain"),
 293                       # we currently cannot run a full make test, a limited list of tests could be run
 294                       # via "make test TESTS=sometests"
 295                       ("test", "make test FAIL_IMMEDIATELY=1 "
 296                        "TESTS='--include-env=ktest'", "text/plain"),
 297                       ("install", "make install", "text/plain"),
 298                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 299                       ("clean", "make clean", "text/plain")
 300                       ],
 301
 302     # Test Samba without python still builds.  When this test fails
 303     # due to more use of Python, the expectations is that the newly
 304     # failing part of the code should be disabled when
 305     # --disable-python is set (rather than major work being done to
 306     # support this environment).  The target here is for vendors
 307     # shipping a minimal smbd.
 308     "samba-nopython": [
 309                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 310                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 311                       ("make", "make -j", "text/plain"),
 312                       ("install", "make install", "text/plain"),
 313                       ("test", "make test-nopython", "text/plain"),
 314                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 315                       ("clean", "make clean", "text/plain"),
 316
 317                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 318                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 319                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 320
 321                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 322                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 323                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 324
 325                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 326                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 327                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 328
 329                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 330                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 331                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 332
 333                       # retry against installed library packages
 334                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 335                       ("libs-make", "make -j", "text/plain"),
 336                       ("libs-install", "make install", "text/plain"),
 337                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 338                       ("libs-clean", "make clean", "text/plain")
 339                       ],
 340
 341     # check we can do the same thing using python2
 342     "samba-nopython-py2": [
 343                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 344                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 345                       ("make", "PYTHON=python2 make -j", "text/plain"),
 346                       ("install", "PYTHON=python2 make install", "text/plain"),
 347                       ("test", "make test-nopython", "text/plain"),
 348                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 349                       ("clean", "PYTHON=python2 make clean", "text/plain"),
 350
 351                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 352                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
 353                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
 354
 355                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 356                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
 357                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
 358
 359                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 360                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
 361                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
 362
 363                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 364                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
 365                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
 366
 367                       # retry against installed library packages
 368                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 369                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
 370                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
 371                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 372                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
 373                       ],
 374
 375     "ldb": [
 376               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 377               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 378               ("make", "make", "text/plain"),
 379               ("install", "make install", "text/plain"),
 380               ("test", "make test", "text/plain"),
 381               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
 382               ("make-no-lmdb", "make", "text/plain"),
 383               ("install-no-lmdb", "make install", "text/plain"),
 384               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 385               ("distcheck", "make distcheck", "text/plain"),
 386               ("clean", "make clean", "text/plain")],
 387
 388     "tdb": [
 389               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 390               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 391               ("make", "make", "text/plain"),
 392               ("install", "make install", "text/plain"),
 393               ("test", "make test", "text/plain"),
 394               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 395               ("distcheck", "make distcheck", "text/plain"),
 396               ("clean", "make clean", "text/plain")],
 397
 398     "talloc": [
 399                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 400                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 401                  ("make", "make", "text/plain"),
 402                  ("install", "make install", "text/plain"),
 403                  ("test", "make test", "text/plain"),
 404                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 405                  ("distcheck", "make distcheck", "text/plain"),
 406                  ("clean", "make clean", "text/plain")],
 407
 408     "replace": [
 409                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 410                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 411                   ("make", "make", "text/plain"),
 412                   ("install", "make install", "text/plain"),
 413                   ("test", "make test", "text/plain"),
 414                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 415                   ("distcheck", "make distcheck", "text/plain"),
 416                   ("clean", "make clean", "text/plain")],
 417
 418     "tevent": [
 419                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 420                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 421                  ("make", "make", "text/plain"),
 422                  ("install", "make install", "text/plain"),
 423                  ("test", "make test", "text/plain"),
 424                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 425                  ("distcheck", "make distcheck", "text/plain"),
 426                  ("clean", "make clean", "text/plain")],
 427
 428     "pidl": [
 429         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 430         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 431         ("touch", "touch *.yp", "text/plain"),
 432         ("make", "make", "text/plain"),
 433         ("test", "make test", "text/plain"),
 434         ("install", "make install", "text/plain"),
 435         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 436         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 437         ("clean", "make clean", "text/plain")],
 438
 439
 440     # these are useful for debugging autobuild
 441     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 442     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 443
 444
 445 }
 446
 447
 448 def do_print(msg):
 449     print("%s" % msg)
 450     sys.stdout.flush()
 451     sys.stderr.flush()
 452
 453
 454 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 455     if show is None:
 456         show = options.verbose
 457     if show:
 458         do_print("Running: '%s' in '%s'" % (cmd, dir))
 459     if output:
 460         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 461     elif checkfail:
 462         return check_call(cmd, shell=True, cwd=dir)
 463     else:
 464         return call(cmd, shell=True, cwd=dir)
 465
 466
 467 class builder(object):
 468     '''handle build of one directory'''
 469
 470     def __init__(self, name, sequence, cp=True):
 471         self.name = name
 472         if name in builddirs:
 473             self.dir = builddirs[name]
 474         else:
 475             self.dir = "."
 476
 477         self.tag = self.name.replace('/', '_')
 478         self.sequence = sequence
 479         self.next = 0
 480         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 481         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 482         if options.verbose:
 483             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 484             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 485         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 486         self.stdout = open(self.stdout_path, 'w')
 487         self.stderr = open(self.stderr_path, 'w')
 488         self.stdin  = open("/dev/null", 'r')
 489         self.sdir = "%s/%s" % (testbase, self.tag)
 490         self.prefix = "%s/%s" % (test_prefix, self.tag)
 491         run_cmd("rm -rf %s" % self.sdir)
 492         run_cmd("rm -rf %s" % self.prefix)
 493         if cp:
 494             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 495         else:
 496             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 497         self.start_next()
 498
 499     def start_next(self):
 500         if self.next == len(self.sequence):
 501             if not options.nocleanup:
 502                 run_cmd("rm -rf %s" % self.sdir)
 503                 run_cmd("rm -rf %s" % self.prefix)
 504             do_print('%s: Completed OK' % self.name)
 505             self.done = True
 506             return
 507         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 508         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 509         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 510         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 511         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 512 #        if self.output_mime_type == "text/x-subunit":
 513 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 514         cwd = os.getcwd()
 515         os.chdir("%s/%s" % (self.sdir, self.dir))
 516         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 517         self.proc = Popen(self.cmd, shell=True,
 518                           close_fds=True,
 519                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 520         os.chdir(cwd)
 521         self.next += 1
 522
 523
 524 class buildlist(object):
 525     '''handle build of multiple directories'''
 526
 527     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 528         global tasks
 529         self.tlist = []
 530         self.tail_proc = None
 531         self.retry = None
 532         if tasknames == []:
 533             if options.restrict_tests:
 534                 tasknames = ["samba-test-only"]
 535             else:
 536                 tasknames = defaulttasks
 537         else:
 538             # If we are only running one test,
 539             # do not sleep randomly to wait for it to start
 540             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 541
 542         for n in tasknames:
 543             b = builder(n, tasks[n], cp=n is not "pidl")
 544             self.tlist.append(b)
 545         if options.retry:
 546             rebase_remote = "rebaseon"
 547             retry_task = [("retry",
 548                             '''set -e
 549                             git remote add -t %s %s %s
 550                             git fetch %s
 551                             while :; do
 552                               sleep 60
 553                               git describe %s/%s > old_remote_branch.desc
 554                               git fetch %s
 555                               git describe %s/%s > remote_branch.desc
 556                               diff old_remote_branch.desc remote_branch.desc
 557                             done
 558                            ''' % (
 559                                 rebase_branch, rebase_remote, rebase_url,
 560                                rebase_remote,
 561                                rebase_remote, rebase_branch,
 562                                rebase_remote,
 563                                rebase_remote, rebase_branch
 564                             ),
 565                             "test/plain")]
 566
 567             self.retry = builder('retry', retry_task, cp=False)
 568             self.need_retry = False
 569
 570     def kill_kids(self):
 571         if self.tail_proc is not None:
 572             self.tail_proc.terminate()
 573             self.tail_proc.wait()
 574             self.tail_proc = None
 575         if self.retry is not None:
 576             self.retry.proc.terminate()
 577             self.retry.proc.wait()
 578             self.retry = None
 579         for b in self.tlist:
 580             if b.proc is not None:
 581                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 582                 b.proc.terminate()
 583                 b.proc.wait()
 584                 b.proc = None
 585
 586     def wait_one(self):
 587         while True:
 588             none_running = True
 589             for b in self.tlist:
 590                 if b.proc is None:
 591                     continue
 592                 none_running = False
 593                 b.status = b.proc.poll()
 594                 if b.status is None:
 595                     continue
 596                 b.proc = None
 597                 return b
 598             if options.retry:
 599                 ret = self.retry.proc.poll()
 600                 if ret is not None:
 601                     self.need_retry = True
 602                     self.retry = None
 603                     return None
 604             if none_running:
 605                 return None
 606             time.sleep(0.1)
 607
 608     def run(self):
 609         while True:
 610             b = self.wait_one()
 611             if options.retry and self.need_retry:
 612                 self.kill_kids()
 613                 do_print("retry needed")
 614                 return (0, None, None, None, "retry")
 615             if b is None:
 616                 break
 617             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 618                 self.kill_kids()
 619                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 620             b.start_next()
 621         self.kill_kids()
 622         return (0, None, None, None, "All OK")
 623
 624     def write_system_info(self):
 625         filename = 'system-info.txt'
 626         f = open(filename, 'w')
 627         for cmd in ['uname -a',
 628                     'lsb_release -a',
 629                     'free',
 630                     'mount',
 631                     'cat /proc/cpuinfo',
 632                     'cc --version',
 633                     'df -m .',
 634                     'df -m %s' % testbase]:
 635             out = run_cmd(cmd, output=True, checkfail=False)
 636             print('### %s' % cmd, file=f)
 637             print(out.decode('utf8', 'backslashreplace'), file=f)
 638             print(file=f)
 639         f.close()
 640         return filename
 641
 642     def tarlogs(self, fname):
 643         tar = tarfile.open(fname, "w:gz")
 644         for b in self.tlist:
 645             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 646             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 647         if os.path.exists("autobuild.log"):
 648             tar.add("autobuild.log")
 649         sys_info = self.write_system_info()
 650         tar.add(sys_info)
 651         tar.close()
 652
 653     def remove_logs(self):
 654         for b in self.tlist:
 655             os.unlink(b.stdout_path)
 656             os.unlink(b.stderr_path)
 657
 658     def start_tail(self):
 659         cmd = ["tail", "-f"]
 660         for b in self.tlist:
 661             cmd.append(b.stdout_path)
 662             cmd.append(b.stderr_path)
 663         self.tail_proc = Popen(cmd, close_fds=True)
 664
 665
 666 def cleanup():
 667     if options.nocleanup:
 668         return
 669     run_cmd("stat %s || true" % test_tmpdir, show=True)
 670     run_cmd("stat %s" % testbase, show=True)
 671     do_print("Cleaning up %r" % cleanup_list)
 672     for d in cleanup_list:
 673         run_cmd("rm -rf %s" % d)
 674
 675
 676 def find_git_root():
 677     '''get to the top of the git repo'''
 678     p = os.getcwd()
 679     while p != '/':
 680         if os.path.isdir(os.path.join(p, ".git")):
 681             return p
 682         p = os.path.abspath(os.path.join(p, '..'))
 683     return None
 684
 685
 686 def daemonize(logfile):
 687     pid = os.fork()
 688     if pid == 0:  # Parent
 689         os.setsid()
 690         pid = os.fork()
 691         if pid != 0:  # Actual daemon
 692             os._exit(0)
 693     else:  # Grandparent
 694         os._exit(0)
 695
 696     import resource      # Resource usage information.
 697     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 698     if maxfd == resource.RLIM_INFINITY:
 699         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 700     for fd in range(0, maxfd):
 701         try:
 702             os.close(fd)
 703         except OSError:
 704             pass
 705     os.open(logfile, os.O_RDWR | os.O_CREAT)
 706     os.dup2(0, 1)
 707     os.dup2(0, 2)
 708
 709
 710 def write_pidfile(fname):
 711     '''write a pid file, cleanup on exit'''
 712     f = open(fname, mode='w')
 713     f.write("%u\n" % os.getpid())
 714     f.close()
 715
 716
 717 def rebase_tree(rebase_url, rebase_branch="master"):
 718     rebase_remote = "rebaseon"
 719     do_print("Rebasing on %s" % rebase_url)
 720     run_cmd("git describe HEAD", show=True, dir=test_master)
 721     run_cmd("git remote add -t %s %s %s" %
 722             (rebase_branch, rebase_remote, rebase_url),
 723             show=True, dir=test_master)
 724     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 725     if options.fix_whitespace:
 726         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 727                 (rebase_remote, rebase_branch),
 728                 show=True, dir=test_master)
 729     else:
 730         run_cmd("git rebase --force-rebase %s/%s" %
 731                 (rebase_remote, rebase_branch),
 732                 show=True, dir=test_master)
 733     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 734                    (rebase_remote, rebase_branch),
 735                    dir=test_master, output=True)
 736     if diff == '':
 737         do_print("No differences between HEAD and %s/%s - exiting" %
 738                  (rebase_remote, rebase_branch))
 739         sys.exit(0)
 740     run_cmd("git describe %s/%s" %
 741             (rebase_remote, rebase_branch),
 742             show=True, dir=test_master)
 743     run_cmd("git describe HEAD", show=True, dir=test_master)
 744     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 745             (rebase_remote, rebase_branch),
 746             show=True, dir=test_master)
 747
 748
 749 def push_to(push_url, push_branch="master"):
 750     push_remote = "pushto"
 751     do_print("Pushing to %s" % push_url)
 752     if options.mark:
 753         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 754         run_cmd("git commit --amend -c HEAD", dir=test_master)
 755         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 756         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 757     run_cmd("git remote add -t %s %s %s" %
 758             (push_branch, push_remote, push_url),
 759             show=True, dir=test_master)
 760     run_cmd("git push %s +HEAD:%s" %
 761             (push_remote, push_branch),
 762             show=True, dir=test_master)
 763
 764
 765 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 766
 767 gitroot = find_git_root()
 768 if gitroot is None:
 769     raise Exception("Failed to find git root")
 770
 771 parser = OptionParser()
 772 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 773 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 774 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 775 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 776                   default=def_testbase)
 777 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 778 parser.add_option("", "--verbose", help="show all commands as they are run",
 779                   default=False, action="store_true")
 780 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 781                   default=None, type='str')
 782 parser.add_option("", "--pushto", help="push to a git url on success",
 783                   default=None, type='str')
 784 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 785                   default=False, action="store_true")
 786 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 787                   default=False, action="store_true")
 788 parser.add_option("", "--retry", help="automatically retry if master changes",
 789                   default=False, action="store_true")
 790 parser.add_option("", "--email", help="send email to the given address on failure",
 791                   type='str', default=None)
 792 parser.add_option("", "--email-from", help="send email from the given address",
 793                   type='str', default="autobuild@samba.org")
 794 parser.add_option("", "--email-server", help="send email via the given server",
 795                   type='str', default='localhost')
 796 parser.add_option("", "--always-email", help="always send email, even on success",
 797                   action="store_true")
 798 parser.add_option("", "--daemon", help="daemonize after initial setup",
 799                   action="store_true")
 800 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 801                   default="master", type='str')
 802 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 803                   default=gitroot, type='str')
 804 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 805                   default=False, action="store_true")
 806 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 807                   default='')
 808
 809
 810 def send_email(subject, text, log_tar):
 811     if options.email is None:
 812         do_print("not sending email because the recipient is not set")
 813         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 814                  (subject, text))
 815         return
 816     outer = MIMEMultipart()
 817     outer['Subject'] = subject
 818     outer['To'] = options.email
 819     outer['From'] = options.email_from
 820     outer['Date'] = email.utils.formatdate(localtime=True)
 821     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 822     outer.attach(MIMEText(text, 'plain'))
 823     if options.attach_logs:
 824         fp = open(log_tar, 'rb')
 825         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 826         fp.close()
 827         # Set the filename parameter
 828         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 829         outer.attach(msg)
 830     content = outer.as_string()
 831     s = smtplib.SMTP(options.email_server)
 832     s.sendmail(options.email_from, [options.email], content)
 833     s.set_debuglevel(1)
 834     s.quit()
 835
 836
 837 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 838                   elapsed_time, log_base=None, add_log_tail=True):
 839     '''send an email to options.email about the failure'''
 840     elapsed_minutes = elapsed_time / 60.0
 841     if log_base is None:
 842         log_base = gitroot
 843     text = '''
 844 Dear Developer,
 845
 846 Your autobuild on %s failed after %.1f minutes
 847 when trying to test %s with the following error:
 848
 849    %s
 850
 851 the autobuild has been abandoned. Please fix the error and resubmit.
 852
 853 A summary of the autobuild process is here:
 854
 855   %s/autobuild.log
 856 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 857
 858     if options.restrict_tests:
 859         text += """
 860 The build was restricted to tests matching %s\n""" % options.restrict_tests
 861
 862     if failed_task != 'rebase':
 863         text += '''
 864 You can see logs of the failed task here:
 865
 866   %s/%s.stdout
 867   %s/%s.stderr
 868
 869 or you can get full logs of all tasks in this job here:
 870
 871   %s/logs.tar.gz
 872
 873 The top commit for the tree that was built was:
 874
 875 %s
 876
 877 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 878
 879     if add_log_tail:
 880         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 881         lines = f.readlines()
 882         log_tail = "".join(lines[-50:])
 883         num_lines = len(lines)
 884         if num_lines < 50:
 885             # Also include stderr (compile failures) if < 50 lines of stdout
 886             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 887             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 888
 889         text += '''
 890 The last 50 lines of log messages:
 891
 892 %s
 893     ''' % log_tail
 894         f.close()
 895
 896     logs = os.path.join(gitroot, 'logs.tar.gz')
 897     send_email('autobuild[%s] failure on %s for task %s during %s'
 898                % (options.branch, platform.node(), failed_task, failed_stage),
 899                text, logs)
 900
 901
 902 def email_success(elapsed_time, log_base=None):
 903     '''send an email to options.email about a successful build'''
 904     if log_base is None:
 905         log_base = gitroot
 906     text = '''
 907 Dear Developer,
 908
 909 Your autobuild on %s has succeeded after %.1f minutes.
 910
 911 ''' % (platform.node(), elapsed_time / 60.)
 912
 913     if options.restrict_tests:
 914         text += """
 915 The build was restricted to tests matching %s\n""" % options.restrict_tests
 916
 917     if options.keeplogs:
 918         text += '''
 919
 920 you can get full logs of all tasks in this job here:
 921
 922   %s/logs.tar.gz
 923
 924 ''' % log_base
 925
 926     text += '''
 927 The top commit for the tree that was built was:
 928
 929 %s
 930 ''' % top_commit_msg
 931
 932     logs = os.path.join(gitroot, 'logs.tar.gz')
 933     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 934                text, logs)
 935
 936
 937 (options, args) = parser.parse_args()
 938
 939 if options.retry:
 940     if options.rebase is None:
 941         raise Exception('You can only use --retry if you also rebase')
 942
 943 testbase = "%s/b%u" % (options.testbase, os.getpid())
 944 test_master = "%s/master" % testbase
 945 test_prefix = "%s/prefix" % testbase
 946 test_tmpdir = "%s/tmp" % testbase
 947 os.environ['TMPDIR'] = test_tmpdir
 948
 949 # get the top commit message, for emails
 950 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 951 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
 952
 953 try:
 954     os.makedirs(testbase)
 955 except Exception as reason:
 956     raise Exception("Unable to create %s : %s" % (testbase, reason))
 957 cleanup_list.append(testbase)
 958
 959 if options.daemon:
 960     logfile = os.path.join(testbase, "log")
 961     do_print("Forking into the background, writing progress to %s" % logfile)
 962     daemonize(logfile)
 963
 964 write_pidfile(gitroot + "/autobuild.pid")
 965
 966 start_time = time.time()
 967
 968 while True:
 969     try:
 970         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 971         os.makedirs(test_tmpdir)
 972         # The waf uninstall code removes empty directories all the way
 973         # up the tree.  Creating a file in test_tmpdir stops it from
 974         # being removed.
 975         run_cmd("touch %s" % os.path.join(test_tmpdir,
 976                                           ".directory-is-not-empty"), show=True)
 977         run_cmd("stat %s" % test_tmpdir, show=True)
 978         run_cmd("stat %s" % testbase, show=True)
 979         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 980     except Exception:
 981         cleanup()
 982         raise
 983
 984     try:
 985         try:
 986             if options.rebase is not None:
 987                 rebase_tree(options.rebase, rebase_branch=options.branch)
 988         except Exception:
 989             cleanup_list.append(gitroot + "/autobuild.pid")
 990             cleanup()
 991             elapsed_time = time.time() - start_time
 992             email_failure(-1, 'rebase', 'rebase', 'rebase',
 993                           'rebase on %s failed' % options.branch,
 994                           elapsed_time, log_base=options.log_base)
 995             sys.exit(1)
 996         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 997         if options.tail:
 998             blist.start_tail()
 999         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1000         if status != 0 or errstr != "retry":
1001             break
1002         cleanup()
1003     except Exception:
1004         cleanup()
1005         raise
1006
1007 cleanup_list.append(gitroot + "/autobuild.pid")
1008
1009 do_print(errstr)
1010
1011 blist.kill_kids()
1012 if options.tail:
1013     do_print("waiting for tail to flush")
1014     time.sleep(1)
1015
1016 elapsed_time = time.time() - start_time
1017 if status == 0:
1018     if options.passcmd is not None:
1019         do_print("Running passcmd: %s" % options.passcmd)
1020         run_cmd(options.passcmd, dir=test_master)
1021     if options.pushto is not None:
1022         push_to(options.pushto, push_branch=options.branch)
1023     if options.keeplogs or options.attach_logs:
1024         blist.tarlogs("logs.tar.gz")
1025         do_print("Logs in logs.tar.gz")
1026     if options.always_email:
1027         email_success(elapsed_time, log_base=options.log_base)
1028     blist.remove_logs()
1029     cleanup()
1030     do_print(errstr)
1031     sys.exit(0)
1032
1033 # something failed, gather a tar of the logs
1034 blist.tarlogs("logs.tar.gz")
1035
1036 if options.email is not None:
1037     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1038                   elapsed_time, log_base=options.log_base)
1039 else:
1040     elapsed_minutes = elapsed_time / 60.0
1041     print('''
1042
1043 ####################################################################
1044
1045 AUTOBUILD FAILURE
1046
1047 Your autobuild[%s] on %s failed after %.1f minutes
1048 when trying to test %s with the following error:
1049
1050    %s
1051
1052 the autobuild has been abandoned. Please fix the error and resubmit.
1053
1054 ####################################################################
1055
1056 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1057
1058 cleanup()
1059 do_print(errstr)
1060 do_print("Logs in logs.tar.gz")
1061 sys.exit(status)