script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-py3": ".",
  40     "samba-nt4": ".",
  41     "samba-nt4-py3": ".",
  42     "samba-fileserver": ".",
  43     "samba-xc": ".",
  44     "samba-o3": ".",
  45     "samba-ctdb": ".",
  46     "samba-libs": ".",
  47     "samba-libs-py3": ".",
  48     "samba-static": ".",
  49     "samba-none-env": ".",
  50     "samba-ad-dc": ".",
  51     "samba-ad-dc-py3": ".",
  52     "samba-ad-dc-2": ".",
  53     "samba-ad-dc-2-py3": ".",
  54     "samba-systemkrb5": ".",
  55     "samba-nopython": ".",
  56     "samba-buildpy3-only": ".",
  57     "samba-purepy3-none-env": ".",
  58     "ldb": "lib/ldb",
  59     "tdb": "lib/tdb",
  60     "talloc": "lib/talloc",
  61     "replace": "lib/replace",
  62     "tevent": "lib/tevent",
  63     "pidl": "pidl"
  64 }
  65
  66 defaulttasks = builddirs.keys()
  67
  68 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  69     defaulttasks.remove("samba-o3")
  70
  71 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  72 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  73
  74 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  75 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  76 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  77 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  78 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  79 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  80 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  81
  82 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  83     extra_python = ""
  84 else:
  85     extra_python = "--extra-python=/usr/bin/python3"
  86
  87 tasks = {
  88     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  89                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  90                ("make", "make all", "text/plain"),
  91                ("install", "make install", "text/plain"),
  92                ("test", "make autotest", "text/plain"),
  93                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  94                ("clean", "make clean", "text/plain")],
  95
  96     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
  97     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  98                 ("make", "make -j", "text/plain"),
  99                 ("test", "make test FAIL_IMMEDIATELY=1 "
 100                  "TESTS='${PY3_ONLY}"
 101                  "--exclude-env=none "
 102                  "--exclude-env=nt4_dc "
 103                  "--exclude-env=nt4_member "
 104                  "--exclude-env=ad_dc "
 105                  "--exclude-env=ad_dc_no_nss "
 106                  "--exclude-env=fl2003dc "
 107                  "--exclude-env=fl2008r2dc "
 108                  "--exclude-env=ad_member "
 109                  "--exclude-env=ad_member_idmap_rid "
 110                  "--exclude-env=ad_member_idmap_ad "
 111                  "--exclude-env=chgdcpass "
 112                  "--exclude-env=vampire_2000_dc "
 113                  "--exclude-env=fl2000dc "
 114                  "--exclude-env=fileserver "
 115                  "--exclude-env=backupfromdc "
 116                  "--exclude-env=restoredc "
 117                  "--exclude-env=renamedc "
 118                  "--exclude-env=offlinebackupdc "
 119                  "--exclude-env=labdc "
 120                  "'",
 121                  "text/plain"),
 122                 ("install", "make install", "text/plain"),
 123                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 124                 ("clean", "make clean", "text/plain")],
 125
 126     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 127     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 128                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 129                     ("make", "make -j", "text/plain"),
 130                     ("test", "make test FAIL_IMMEDIATELY=1 "
 131                      "TESTS='${PY3_ONLY}"
 132                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 133                     ("install", "make install", "text/plain"),
 134                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 135                     ("clean", "make clean", "text/plain")],
 136
 137     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 138     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 139                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 140                            ("make", "make -j", "text/plain"),
 141                            ("test", "make test FAIL_IMMEDIATELY=1 "
 142                             "TESTS='${PY3_ONLY}"
 143                             "--include-env=fileserver'", "text/plain"),
 144                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 145
 146     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 147     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 148                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 149                       ("make", "make -j", "text/plain"),
 150                       ("test", "make test FAIL_IMMEDIATELY=1 "
 151                        "TESTS='${PY3_ONLY}"
 152                        "--include-env=ad_dc "
 153                        "--include-env=fl2003dc "
 154                        "--include-env=fl2008r2dc "
 155                        "--include-env=ad_member "
 156                        "--include-env=ad_member_idmap_rid "
 157                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 158                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 159
 160     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 161     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 162                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 163                         ("make", "make -j", "text/plain"),
 164                         ("test", "make test FAIL_IMMEDIATELY=1 "
 165                          "TESTS='${PY3_ONLY}"
 166                          "--include-env=chgdcpass "
 167                          "--include-env=vampire_2000_dc "
 168                          "--include-env=fl2000dc "
 169                          "--include-env=ad_dc_no_nss "
 170                          "--include-env=backupfromdc "
 171                          "--include-env=restoredc "
 172                          "--include-env=renamedc "
 173                          "--include-env=offlinebackupdc "
 174                          "--include-env=labdc "
 175                          "'",
 176                          "text/plain"),
 177                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 178
 179     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 180                           ("make", "make -j", "text/plain"),
 181                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 182
 183     # Test cross-compile infrastructure
 184     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 185                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 186                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 187                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 188                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 189                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 190                    ("compare-results", "script/compare_cc_results.py "
 191                     "./bin/c4che/default{} "
 192                     "./bin-xe/c4che/default{} "
 193                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 194
 195     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 196     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 197                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 198                    ("make", "make -j", "text/plain"),
 199                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 200                     "TESTS='${PY3_ONLY}"
 201                     "--include-env=ad_dc'", "text/plain"),
 202                    ("install", "make install", "text/plain"),
 203                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 204                    ("clean", "make clean", "text/plain")],
 205
 206     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 207
 208                      # make sure we have tdb around:
 209                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 210                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 211                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 212
 213
 214                      # build samba with cluster support (also building ctdb):
 215                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 216                      ("samba-make", "make", "text/plain"),
 217                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 218                      ("samba-install", "make install", "text/plain"),
 219                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 220
 221                      # clean up:
 222                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 223                      ("clean", "make clean", "text/plain"),
 224                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 225
 226     "samba-libs": [
 227                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 228                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 229                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 230                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 231
 232                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 233                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 234                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 235
 236                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 237                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 238                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 239
 240                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 241                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 242                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 243
 244                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 245                       ("nondevel-make", "make -j", "text/plain"),
 246                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 247                       ("nondevel-install", "make install", "text/plain"),
 248                       ("nondevel-dist", "make dist", "text/plain"),
 249
 250                       # retry with all modules shared
 251                       ("allshared-distclean", "make distclean", "text/plain"),
 252                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 253                       ("allshared-make", "make -j", "text/plain")],
 254
 255     "samba-none-env": [
 256                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 257                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 258                       ("make", "make -j", "text/plain"),
 259                       ("test", "make test "
 260                        "FAIL_IMMEDIATELY=1 "
 261                        "TESTS='${PY3_ONLY}"
 262                        "--include-env=none'",
 263                        "text/plain")],
 264
 265     "samba-static": [
 266                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 267                       # build with all modules static
 268                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 269                       ("allstatic-make", "make -j", "text/plain"),
 270                       ("allstatic-test", "make test "
 271                        "FAIL_IMMEDIATELY=1 "
 272                        "TESTS='samba3.smb2.create.*nt4_dc'",
 273                        "text/plain"),
 274
 275                       # retry without any required modules
 276                       ("none-distclean", "make distclean", "text/plain"),
 277                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 278                       ("none-make", "make -j", "text/plain"),
 279
 280                       # retry with nonshared smbd and smbtorture
 281                       ("nonshared-distclean", "make distclean", "text/plain"),
 282                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 283                       ("nonshared-make", "make -j", "text/plain")],
 284
 285     "samba-systemkrb5": [
 286                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 287                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 288                       ("make", "make -j", "text/plain"),
 289                       # we currently cannot run a full make test, a limited list of tests could be run
 290                       # via "make test TESTS=sometests"
 291                       ("test", "make test FAIL_IMMEDIATELY=1 "
 292                        "TESTS='${PY3_ONLY}"
 293                        "--include-env=ktest'", "text/plain"),
 294                       ("install", "make install", "text/plain"),
 295                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 296                       ("clean", "make clean", "text/plain")
 297                       ],
 298
 299     # Test Samba without python still builds.  When this test fails
 300     # due to more use of Python, the expectations is that the newly
 301     # failing part of the code should be disabled when
 302     # --disable-python is set (rather than major work being done to
 303     # support this environment).  The target here is for vendors
 304     # shipping a minimal smbd.
 305     "samba-nopython": [
 306                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 307                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 308                       ("make", "make -j", "text/plain"),
 309                       ("install", "make install", "text/plain"),
 310                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 311                       ("clean", "make clean", "text/plain"),
 312
 313                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 314                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 315                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 316
 317                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 318                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 319                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 320
 321                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 322                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 323                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 324
 325                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 326                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 327                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 328
 329                       # retry against installed library packages
 330                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 331                       ("libs-make", "make -j", "text/plain"),
 332                       ("libs-install", "make install", "text/plain"),
 333                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 334                       ("libs-clean", "make clean", "text/plain")
 335                       ],
 336
 337
 338
 339     "ldb": [
 340               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 341               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 342               ("make", "make", "text/plain"),
 343               ("install", "make install", "text/plain"),
 344               ("test", "make test", "text/plain"),
 345               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 346               ("make-no-lmdb", "make", "text/plain"),
 347               ("install-no-lmdb", "make install", "text/plain"),
 348               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 349               ("distcheck", "make distcheck", "text/plain"),
 350               ("clean", "make clean", "text/plain")],
 351
 352     "tdb": [
 353               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 354               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 355               ("make", "make", "text/plain"),
 356               ("install", "make install", "text/plain"),
 357               ("test", "make test", "text/plain"),
 358               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 359               ("distcheck", "make distcheck", "text/plain"),
 360               ("clean", "make clean", "text/plain")],
 361
 362     "talloc": [
 363                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 364                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 365                  ("make", "make", "text/plain"),
 366                  ("install", "make install", "text/plain"),
 367                  ("test", "make test", "text/plain"),
 368                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 369                  ("distcheck", "make distcheck", "text/plain"),
 370                  ("clean", "make clean", "text/plain")],
 371
 372     "replace": [
 373                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 374                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 375                   ("make", "make", "text/plain"),
 376                   ("install", "make install", "text/plain"),
 377                   ("test", "make test", "text/plain"),
 378                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 379                   ("distcheck", "make distcheck", "text/plain"),
 380                   ("clean", "make clean", "text/plain")],
 381
 382     "tevent": [
 383                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 384                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 385                  ("make", "make", "text/plain"),
 386                  ("install", "make install", "text/plain"),
 387                  ("test", "make test", "text/plain"),
 388                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 389                  ("distcheck", "make distcheck", "text/plain"),
 390                  ("clean", "make clean", "text/plain")],
 391
 392     "pidl": [
 393         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 394         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 395         ("touch", "touch *.yp", "text/plain"),
 396         ("make", "make", "text/plain"),
 397         ("test", "make test", "text/plain"),
 398         ("install", "make install", "text/plain"),
 399         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 400         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 401         ("clean", "make clean", "text/plain")],
 402
 403     "samba-buildpy3-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 404                    ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
 405                    ("make", "PYTHON='python3' make -j", "text/plain"),
 406                    ("install", "PYTHON='python3' make install", "text/plain"),
 407                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 408                    ("clean", "PYTHON='python3' make clean", "text/plain")],
 409
 410     "samba-purepy3-none-env": [
 411                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 412                       ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 413                       ("make", "PYTHON='python3' make -j", "text/plain"),
 414                       ("test", "PYTHON='python3' make test "
 415                        "FAIL_IMMEDIATELY=1 "
 416                        "TESTS='${PY3_ONLY}"
 417                        "--include-env=none'",
 418                        "text/plain")],
 419
 420     # these are useful for debugging autobuild
 421     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 422     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 423
 424
 425 }
 426
 427
 428 def do_print(msg):
 429     print("%s" % msg)
 430     sys.stdout.flush()
 431     sys.stderr.flush()
 432
 433
 434 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 435     if show is None:
 436         show = options.verbose
 437     if show:
 438         do_print("Running: '%s' in '%s'" % (cmd, dir))
 439     if output:
 440         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 441     elif checkfail:
 442         return check_call(cmd, shell=True, cwd=dir)
 443     else:
 444         return call(cmd, shell=True, cwd=dir)
 445
 446
 447 class builder(object):
 448     '''handle build of one directory'''
 449
 450     def __init__(self, name, sequence, cp=True, py3=False):
 451         self.name = name
 452         self.py3 = py3
 453         if name in builddirs:
 454             self.dir = builddirs[name]
 455         else:
 456             self.dir = "."
 457
 458         self.tag = self.name.replace('/', '_')
 459         self.sequence = sequence
 460         self.next = 0
 461         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 462         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 463         if options.verbose:
 464             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 465             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 466         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 467         self.stdout = open(self.stdout_path, 'w')
 468         self.stderr = open(self.stderr_path, 'w')
 469         self.stdin  = open("/dev/null", 'r')
 470         self.sdir = "%s/%s" % (testbase, self.tag)
 471         self.prefix = "%s/%s" % (test_prefix, self.tag)
 472         run_cmd("rm -rf %s" % self.sdir)
 473         run_cmd("rm -rf %s" % self.prefix)
 474         if cp:
 475             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 476         else:
 477             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 478         self.start_next()
 479
 480     def start_next(self):
 481         if self.next == len(self.sequence):
 482             if not options.nocleanup:
 483                 run_cmd("rm -rf %s" % self.sdir)
 484                 run_cmd("rm -rf %s" % self.prefix)
 485             do_print('%s: Completed OK' % self.name)
 486             self.done = True
 487             return
 488         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 489         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 490         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 491         if self.py3:
 492             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 493             # The trailing space is important
 494             self.cmd = self.cmd.replace("${PY3_ONLY}", "python3 ")
 495         else:
 496             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 497             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 498         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 499         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 500 #        if self.output_mime_type == "text/x-subunit":
 501 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 502         cwd = os.getcwd()
 503         os.chdir("%s/%s" % (self.sdir, self.dir))
 504         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 505         self.proc = Popen(self.cmd, shell=True,
 506                           close_fds=True,
 507                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 508         os.chdir(cwd)
 509         self.next += 1
 510
 511
 512 class buildlist(object):
 513     '''handle build of multiple directories'''
 514
 515     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 516         global tasks
 517         self.tlist = []
 518         self.tail_proc = None
 519         self.retry = None
 520         if tasknames == []:
 521             if options.restrict_tests:
 522                 tasknames = ["samba-test-only"]
 523             else:
 524                 tasknames = defaulttasks
 525         else:
 526             # If we are only running one test,
 527             # do not sleep randomly to wait for it to start
 528             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 529
 530         for n in tasknames:
 531             if n not in tasks and n.endswith("-py3"):
 532                 b = builder(n,
 533                             tasks[n[:-4]],
 534                             cp=n is not "pidl",
 535                             py3=True)
 536             else:
 537                 b = builder(n, tasks[n], cp=n is not "pidl")
 538             self.tlist.append(b)
 539         if options.retry:
 540             rebase_remote = "rebaseon"
 541             retry_task = [("retry",
 542                             '''set -e
 543                             git remote add -t %s %s %s
 544                             git fetch %s
 545                             while :; do
 546                               sleep 60
 547                               git describe %s/%s > old_remote_branch.desc
 548                               git fetch %s
 549                               git describe %s/%s > remote_branch.desc
 550                               diff old_remote_branch.desc remote_branch.desc
 551                             done
 552                            ''' % (
 553                                 rebase_branch, rebase_remote, rebase_url,
 554                                rebase_remote,
 555                                rebase_remote, rebase_branch,
 556                                rebase_remote,
 557                                rebase_remote, rebase_branch
 558                             ),
 559                             "test/plain")]
 560
 561             self.retry = builder('retry', retry_task, cp=False)
 562             self.need_retry = False
 563
 564     def kill_kids(self):
 565         if self.tail_proc is not None:
 566             self.tail_proc.terminate()
 567             self.tail_proc.wait()
 568             self.tail_proc = None
 569         if self.retry is not None:
 570             self.retry.proc.terminate()
 571             self.retry.proc.wait()
 572             self.retry = None
 573         for b in self.tlist:
 574             if b.proc is not None:
 575                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 576                 b.proc.terminate()
 577                 b.proc.wait()
 578                 b.proc = None
 579
 580     def wait_one(self):
 581         while True:
 582             none_running = True
 583             for b in self.tlist:
 584                 if b.proc is None:
 585                     continue
 586                 none_running = False
 587                 b.status = b.proc.poll()
 588                 if b.status is None:
 589                     continue
 590                 b.proc = None
 591                 return b
 592             if options.retry:
 593                 ret = self.retry.proc.poll()
 594                 if ret is not None:
 595                     self.need_retry = True
 596                     self.retry = None
 597                     return None
 598             if none_running:
 599                 return None
 600             time.sleep(0.1)
 601
 602     def run(self):
 603         while True:
 604             b = self.wait_one()
 605             if options.retry and self.need_retry:
 606                 self.kill_kids()
 607                 do_print("retry needed")
 608                 return (0, None, None, None, "retry")
 609             if b is None:
 610                 break
 611             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 612                 self.kill_kids()
 613                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 614             b.start_next()
 615         self.kill_kids()
 616         return (0, None, None, None, "All OK")
 617
 618     def write_system_info(self):
 619         filename = 'system-info.txt'
 620         f = open(filename, 'w')
 621         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 622                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 623             print('### %s' % cmd, file=f)
 624             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 625             print(file=f)
 626         f.close()
 627         return filename
 628
 629     def tarlogs(self, fname):
 630         tar = tarfile.open(fname, "w:gz")
 631         for b in self.tlist:
 632             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 633             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 634         if os.path.exists("autobuild.log"):
 635             tar.add("autobuild.log")
 636         sys_info = self.write_system_info()
 637         tar.add(sys_info)
 638         tar.close()
 639
 640     def remove_logs(self):
 641         for b in self.tlist:
 642             os.unlink(b.stdout_path)
 643             os.unlink(b.stderr_path)
 644
 645     def start_tail(self):
 646         cmd = ["tail", "-f"]
 647         for b in self.tlist:
 648             cmd.append(b.stdout_path)
 649             cmd.append(b.stderr_path)
 650         self.tail_proc = Popen(cmd, close_fds=True)
 651
 652
 653 def cleanup():
 654     if options.nocleanup:
 655         return
 656     run_cmd("stat %s || true" % test_tmpdir, show=True)
 657     run_cmd("stat %s" % testbase, show=True)
 658     do_print("Cleaning up %r" % cleanup_list)
 659     for d in cleanup_list:
 660         run_cmd("rm -rf %s" % d)
 661
 662
 663 def find_git_root():
 664     '''get to the top of the git repo'''
 665     p = os.getcwd()
 666     while p != '/':
 667         if os.path.isdir(os.path.join(p, ".git")):
 668             return p
 669         p = os.path.abspath(os.path.join(p, '..'))
 670     return None
 671
 672
 673 def daemonize(logfile):
 674     pid = os.fork()
 675     if pid == 0:  # Parent
 676         os.setsid()
 677         pid = os.fork()
 678         if pid != 0:  # Actual daemon
 679             os._exit(0)
 680     else:  # Grandparent
 681         os._exit(0)
 682
 683     import resource      # Resource usage information.
 684     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 685     if maxfd == resource.RLIM_INFINITY:
 686         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 687     for fd in range(0, maxfd):
 688         try:
 689             os.close(fd)
 690         except OSError:
 691             pass
 692     os.open(logfile, os.O_RDWR | os.O_CREAT)
 693     os.dup2(0, 1)
 694     os.dup2(0, 2)
 695
 696
 697 def write_pidfile(fname):
 698     '''write a pid file, cleanup on exit'''
 699     f = open(fname, mode='w')
 700     f.write("%u\n" % os.getpid())
 701     f.close()
 702
 703
 704 def rebase_tree(rebase_url, rebase_branch="master"):
 705     rebase_remote = "rebaseon"
 706     do_print("Rebasing on %s" % rebase_url)
 707     run_cmd("git describe HEAD", show=True, dir=test_master)
 708     run_cmd("git remote add -t %s %s %s" %
 709             (rebase_branch, rebase_remote, rebase_url),
 710             show=True, dir=test_master)
 711     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 712     if options.fix_whitespace:
 713         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 714                 (rebase_remote, rebase_branch),
 715                 show=True, dir=test_master)
 716     else:
 717         run_cmd("git rebase --force-rebase %s/%s" %
 718                 (rebase_remote, rebase_branch),
 719                 show=True, dir=test_master)
 720     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 721                    (rebase_remote, rebase_branch),
 722                    dir=test_master, output=True)
 723     if diff == '':
 724         do_print("No differences between HEAD and %s/%s - exiting" %
 725                  (rebase_remote, rebase_branch))
 726         sys.exit(0)
 727     run_cmd("git describe %s/%s" %
 728             (rebase_remote, rebase_branch),
 729             show=True, dir=test_master)
 730     run_cmd("git describe HEAD", show=True, dir=test_master)
 731     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 732             (rebase_remote, rebase_branch),
 733             show=True, dir=test_master)
 734
 735
 736 def push_to(push_url, push_branch="master"):
 737     push_remote = "pushto"
 738     do_print("Pushing to %s" % push_url)
 739     if options.mark:
 740         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 741         run_cmd("git commit --amend -c HEAD", dir=test_master)
 742         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 743         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 744     run_cmd("git remote add -t %s %s %s" %
 745             (push_branch, push_remote, push_url),
 746             show=True, dir=test_master)
 747     run_cmd("git push %s +HEAD:%s" %
 748             (push_remote, push_branch),
 749             show=True, dir=test_master)
 750
 751
 752 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 753
 754 gitroot = find_git_root()
 755 if gitroot is None:
 756     raise Exception("Failed to find git root")
 757
 758 parser = OptionParser()
 759 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 760 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 761 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 762 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 763                   default=def_testbase)
 764 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 765 parser.add_option("", "--verbose", help="show all commands as they are run",
 766                   default=False, action="store_true")
 767 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 768                   default=None, type='str')
 769 parser.add_option("", "--pushto", help="push to a git url on success",
 770                   default=None, type='str')
 771 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 772                   default=False, action="store_true")
 773 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 774                   default=False, action="store_true")
 775 parser.add_option("", "--retry", help="automatically retry if master changes",
 776                   default=False, action="store_true")
 777 parser.add_option("", "--email", help="send email to the given address on failure",
 778                   type='str', default=None)
 779 parser.add_option("", "--email-from", help="send email from the given address",
 780                   type='str', default="autobuild@samba.org")
 781 parser.add_option("", "--email-server", help="send email via the given server",
 782                   type='str', default='localhost')
 783 parser.add_option("", "--always-email", help="always send email, even on success",
 784                   action="store_true")
 785 parser.add_option("", "--daemon", help="daemonize after initial setup",
 786                   action="store_true")
 787 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 788                   default="master", type='str')
 789 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 790                   default=gitroot, type='str')
 791 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 792                   default=False, action="store_true")
 793 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 794                   default='')
 795
 796
 797 def send_email(subject, text, log_tar):
 798     if options.email is None:
 799         do_print("not sending email because the recipient is not set")
 800         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 801                  (subject, text))
 802         return
 803     outer = MIMEMultipart()
 804     outer['Subject'] = subject
 805     outer['To'] = options.email
 806     outer['From'] = options.email_from
 807     outer['Date'] = email.utils.formatdate(localtime=True)
 808     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 809     outer.attach(MIMEText(text, 'plain'))
 810     if options.attach_logs:
 811         fp = open(log_tar, 'rb')
 812         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 813         fp.close()
 814         # Set the filename parameter
 815         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 816         outer.attach(msg)
 817     content = outer.as_string()
 818     s = smtplib.SMTP(options.email_server)
 819     s.sendmail(options.email_from, [options.email], content)
 820     s.set_debuglevel(1)
 821     s.quit()
 822
 823
 824 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 825                   elapsed_time, log_base=None, add_log_tail=True):
 826     '''send an email to options.email about the failure'''
 827     elapsed_minutes = elapsed_time / 60.0
 828     user = os.getenv("USER")
 829     if log_base is None:
 830         log_base = gitroot
 831     text = '''
 832 Dear Developer,
 833
 834 Your autobuild on %s failed after %.1f minutes
 835 when trying to test %s with the following error:
 836
 837    %s
 838
 839 the autobuild has been abandoned. Please fix the error and resubmit.
 840
 841 A summary of the autobuild process is here:
 842
 843   %s/autobuild.log
 844 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 845
 846     if options.restrict_tests:
 847         text += """
 848 The build was restricted to tests matching %s\n""" % options.restrict_tests
 849
 850     if failed_task != 'rebase':
 851         text += '''
 852 You can see logs of the failed task here:
 853
 854   %s/%s.stdout
 855   %s/%s.stderr
 856
 857 or you can get full logs of all tasks in this job here:
 858
 859   %s/logs.tar.gz
 860
 861 The top commit for the tree that was built was:
 862
 863 %s
 864
 865 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 866
 867     if add_log_tail:
 868         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 869         lines = f.readlines()
 870         log_tail = "".join(lines[-50:])
 871         num_lines = len(lines)
 872         if num_lines < 50:
 873             # Also include stderr (compile failures) if < 50 lines of stdout
 874             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 875             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 876
 877         text += '''
 878 The last 50 lines of log messages:
 879
 880 %s
 881     ''' % log_tail
 882         f.close()
 883
 884     logs = os.path.join(gitroot, 'logs.tar.gz')
 885     send_email('autobuild[%s] failure on %s for task %s during %s'
 886                % (options.branch, platform.node(), failed_task, failed_stage),
 887                text, logs)
 888
 889
 890 def email_success(elapsed_time, log_base=None):
 891     '''send an email to options.email about a successful build'''
 892     user = os.getenv("USER")
 893     if log_base is None:
 894         log_base = gitroot
 895     text = '''
 896 Dear Developer,
 897
 898 Your autobuild on %s has succeeded after %.1f minutes.
 899
 900 ''' % (platform.node(), elapsed_time / 60.)
 901
 902     if options.restrict_tests:
 903         text += """
 904 The build was restricted to tests matching %s\n""" % options.restrict_tests
 905
 906     if options.keeplogs:
 907         text += '''
 908
 909 you can get full logs of all tasks in this job here:
 910
 911   %s/logs.tar.gz
 912
 913 ''' % log_base
 914
 915     text += '''
 916 The top commit for the tree that was built was:
 917
 918 %s
 919 ''' % top_commit_msg
 920
 921     logs = os.path.join(gitroot, 'logs.tar.gz')
 922     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 923                text, logs)
 924
 925
 926 (options, args) = parser.parse_args()
 927
 928 if options.retry:
 929     if options.rebase is None:
 930         raise Exception('You can only use --retry if you also rebase')
 931
 932 testbase = "%s/b%u" % (options.testbase, os.getpid())
 933 test_master = "%s/master" % testbase
 934 test_prefix = "%s/prefix" % testbase
 935 test_tmpdir = "%s/tmp" % testbase
 936 os.environ['TMPDIR'] = test_tmpdir
 937
 938 # get the top commit message, for emails
 939 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 940
 941 try:
 942     os.makedirs(testbase)
 943 except Exception as reason:
 944     raise Exception("Unable to create %s : %s" % (testbase, reason))
 945 cleanup_list.append(testbase)
 946
 947 if options.daemon:
 948     logfile = os.path.join(testbase, "log")
 949     do_print("Forking into the background, writing progress to %s" % logfile)
 950     daemonize(logfile)
 951
 952 write_pidfile(gitroot + "/autobuild.pid")
 953
 954 start_time = time.time()
 955
 956 while True:
 957     try:
 958         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 959         os.makedirs(test_tmpdir)
 960         # The waf uninstall code removes empty directories all the way
 961         # up the tree.  Creating a file in test_tmpdir stops it from
 962         # being removed.
 963         run_cmd("touch %s" % os.path.join(test_tmpdir,
 964                                           ".directory-is-not-empty"), show=True)
 965         run_cmd("stat %s" % test_tmpdir, show=True)
 966         run_cmd("stat %s" % testbase, show=True)
 967         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 968     except Exception:
 969         cleanup()
 970         raise
 971
 972     try:
 973         try:
 974             if options.rebase is not None:
 975                 rebase_tree(options.rebase, rebase_branch=options.branch)
 976         except Exception:
 977             cleanup_list.append(gitroot + "/autobuild.pid")
 978             cleanup()
 979             elapsed_time = time.time() - start_time
 980             email_failure(-1, 'rebase', 'rebase', 'rebase',
 981                           'rebase on %s failed' % options.branch,
 982                           elapsed_time, log_base=options.log_base)
 983             sys.exit(1)
 984         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 985         if options.tail:
 986             blist.start_tail()
 987         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 988         if status != 0 or errstr != "retry":
 989             break
 990         cleanup()
 991     except Exception:
 992         cleanup()
 993         raise
 994
 995 cleanup_list.append(gitroot + "/autobuild.pid")
 996
 997 do_print(errstr)
 998
 999 blist.kill_kids()
1000 if options.tail:
1001     do_print("waiting for tail to flush")
1002     time.sleep(1)
1003
1004 elapsed_time = time.time() - start_time
1005 if status == 0:
1006     if options.passcmd is not None:
1007         do_print("Running passcmd: %s" % options.passcmd)
1008         run_cmd(options.passcmd, dir=test_master)
1009     if options.pushto is not None:
1010         push_to(options.pushto, push_branch=options.branch)
1011     if options.keeplogs or options.attach_logs:
1012         blist.tarlogs("logs.tar.gz")
1013         do_print("Logs in logs.tar.gz")
1014     if options.always_email:
1015         email_success(elapsed_time, log_base=options.log_base)
1016     blist.remove_logs()
1017     cleanup()
1018     do_print(errstr)
1019     sys.exit(0)
1020
1021 # something failed, gather a tar of the logs
1022 blist.tarlogs("logs.tar.gz")
1023
1024 if options.email is not None:
1025     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1026                   elapsed_time, log_base=options.log_base)
1027 else:
1028     elapsed_minutes = elapsed_time / 60.0
1029     print('''
1030
1031 ####################################################################
1032
1033 AUTOBUILD FAILURE
1034
1035 Your autobuild[%s] on %s failed after %.1f minutes
1036 when trying to test %s with the following error:
1037
1038    %s
1039
1040 the autobuild has been abandoned. Please fix the error and resubmit.
1041
1042 ####################################################################
1043
1044 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1045
1046 cleanup()
1047 do_print(errstr)
1048 do_print("Logs in logs.tar.gz")
1049 sys.exit(status)