script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-py3": ".",
  40     "samba-nt4": ".",
  41     "samba-nt4-py3": ".",
  42     "samba-fileserver": ".",
  43     "samba-xc": ".",
  44     "samba-o3": ".",
  45     "samba-ctdb": ".",
  46     "samba-libs": ".",
  47     "samba-libs-py3": ".",
  48     "samba-static": ".",
  49     "samba-none-env": ".",
  50     "samba-ad-dc": ".",
  51     "samba-ad-dc-py3": ".",
  52     "samba-ad-dc-2": ".",
  53     "samba-ad-dc-2-py3": ".",
  54     "samba-systemkrb5": ".",
  55     "samba-nopython": ".",
  56     "samba-buildpy3-only": ".",
  57     "ldb": "lib/ldb",
  58     "tdb": "lib/tdb",
  59     "talloc": "lib/talloc",
  60     "replace": "lib/replace",
  61     "tevent": "lib/tevent",
  62     "pidl": "pidl"
  63 }
  64
  65 defaulttasks = builddirs.keys()
  66
  67 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  68     defaulttasks.remove("samba-o3")
  69
  70 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  71 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  72
  73 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  74 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  75 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  76 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  77 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  78 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  79 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  80
  81 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  82     extra_python = ""
  83 else:
  84     extra_python = "--extra-python=/usr/bin/python3"
  85
  86 tasks = {
  87     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  88                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  89                ("make", "make all", "text/plain"),
  90                ("install", "make install", "text/plain"),
  91                ("test", "make autotest", "text/plain"),
  92                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  93                ("clean", "make clean", "text/plain")],
  94
  95     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
  96     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  97                 ("make", "make -j", "text/plain"),
  98                 ("test", "make test FAIL_IMMEDIATELY=1 "
  99                  "TESTS='${PY3_ONLY}"
 100                  "--exclude-env=none "
 101                  "--exclude-env=nt4_dc "
 102                  "--exclude-env=nt4_member "
 103                  "--exclude-env=ad_dc "
 104                  "--exclude-env=ad_dc_no_nss "
 105                  "--exclude-env=fl2003dc "
 106                  "--exclude-env=fl2008r2dc "
 107                  "--exclude-env=ad_member "
 108                  "--exclude-env=ad_member_idmap_rid "
 109                  "--exclude-env=ad_member_idmap_ad "
 110                  "--exclude-env=chgdcpass "
 111                  "--exclude-env=vampire_2000_dc "
 112                  "--exclude-env=fl2000dc "
 113                  "--exclude-env=fileserver "
 114                  "--exclude-env=backupfromdc "
 115                  "--exclude-env=restoredc "
 116                  "--exclude-env=renamedc "
 117                  "--exclude-env=offlinebackupdc "
 118                  "--exclude-env=labdc "
 119                  "'",
 120                  "text/plain"),
 121                 ("install", "make install", "text/plain"),
 122                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 123                 ("clean", "make clean", "text/plain")],
 124
 125     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 126     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 127                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 128                     ("make", "make -j", "text/plain"),
 129                     ("test", "make test FAIL_IMMEDIATELY=1 "
 130                      "TESTS='${PY3_ONLY}"
 131                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 132                     ("install", "make install", "text/plain"),
 133                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 134                     ("clean", "make clean", "text/plain")],
 135
 136     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 137     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 138                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 139                            ("make", "make -j", "text/plain"),
 140                            ("test", "make test FAIL_IMMEDIATELY=1 "
 141                             "TESTS='${PY3_ONLY}"
 142                             "--include-env=fileserver'", "text/plain"),
 143                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 144
 145     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 146     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 147                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 148                       ("make", "make -j", "text/plain"),
 149                       ("test", "make test FAIL_IMMEDIATELY=1 "
 150                        "TESTS='${PY3_ONLY}"
 151                        "--include-env=ad_dc "
 152                        "--include-env=fl2003dc "
 153                        "--include-env=fl2008r2dc "
 154                        "--include-env=ad_member "
 155                        "--include-env=ad_member_idmap_rid "
 156                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 157                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 158
 159     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 160     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 161                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 162                         ("make", "make -j", "text/plain"),
 163                         ("test", "make test FAIL_IMMEDIATELY=1 "
 164                          "TESTS='${PY3_ONLY}"
 165                          "--include-env=chgdcpass "
 166                          "--include-env=vampire_2000_dc "
 167                          "--include-env=fl2000dc "
 168                          "--include-env=ad_dc_no_nss "
 169                          "--include-env=backupfromdc "
 170                          "--include-env=restoredc "
 171                          "--include-env=renamedc "
 172                          "--include-env=offlinebackupdc "
 173                          "--include-env=labdc "
 174                          "'",
 175                          "text/plain"),
 176                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 177
 178     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 179                           ("make", "make -j", "text/plain"),
 180                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 181
 182     # Test cross-compile infrastructure
 183     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 184                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 185                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 186                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 187                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 188                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 189                    ("compare-results", "script/compare_cc_results.py "
 190                     "./bin/c4che/default{} "
 191                     "./bin-xe/c4che/default{} "
 192                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 193
 194     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 195     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 196                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 197                    ("make", "make -j", "text/plain"),
 198                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 199                     "TESTS='${PY3_ONLY}"
 200                     "--include-env=ad_dc'", "text/plain"),
 201                    ("install", "make install", "text/plain"),
 202                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 203                    ("clean", "make clean", "text/plain")],
 204
 205     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 206
 207                      # make sure we have tdb around:
 208                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 209                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 210                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 211
 212
 213                      # build samba with cluster support (also building ctdb):
 214                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 215                      ("samba-make", "make", "text/plain"),
 216                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 217                      ("samba-install", "make install", "text/plain"),
 218                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 219
 220                      # clean up:
 221                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 222                      ("clean", "make clean", "text/plain"),
 223                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 224
 225     "samba-libs": [
 226                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 227                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 228                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 229                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 230
 231                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 232                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 233                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 234
 235                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 236                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 237                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 238
 239                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 240                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 241                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 242
 243                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 244                       ("nondevel-make", "make -j", "text/plain"),
 245                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 246                       ("nondevel-install", "make install", "text/plain"),
 247                       ("nondevel-dist", "make dist", "text/plain"),
 248
 249                       # retry with all modules shared
 250                       ("allshared-distclean", "make distclean", "text/plain"),
 251                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 252                       ("allshared-make", "make -j", "text/plain")],
 253
 254     "samba-none-env": [
 255                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 256                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 257                       ("make", "make -j", "text/plain"),
 258                       ("test", "make test "
 259                        "FAIL_IMMEDIATELY=1 "
 260                        "TESTS='${PY3_ONLY}"
 261                        "--include-env=none'",
 262                        "text/plain")],
 263
 264     "samba-static": [
 265                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 266                       # build with all modules static
 267                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 268                       ("allstatic-make", "make -j", "text/plain"),
 269                       ("allstatic-test", "make test "
 270                        "FAIL_IMMEDIATELY=1 "
 271                        "TESTS='samba3.smb2.create.*nt4_dc'",
 272                        "text/plain"),
 273
 274                       # retry without any required modules
 275                       ("none-distclean", "make distclean", "text/plain"),
 276                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 277                       ("none-make", "make -j", "text/plain"),
 278
 279                       # retry with nonshared smbd and smbtorture
 280                       ("nonshared-distclean", "make distclean", "text/plain"),
 281                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 282                       ("nonshared-make", "make -j", "text/plain")],
 283
 284     "samba-systemkrb5": [
 285                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 286                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 287                       ("make", "make -j", "text/plain"),
 288                       # we currently cannot run a full make test, a limited list of tests could be run
 289                       # via "make test TESTS=sometests"
 290                       ("test", "make test FAIL_IMMEDIATELY=1 "
 291                        "TESTS='${PY3_ONLY}"
 292                        "--include-env=ktest'", "text/plain"),
 293                       ("install", "make install", "text/plain"),
 294                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 295                       ("clean", "make clean", "text/plain")
 296                       ],
 297
 298     # Test Samba without python still builds.  When this test fails
 299     # due to more use of Python, the expectations is that the newly
 300     # failing part of the code should be disabled when
 301     # --disable-python is set (rather than major work being done to
 302     # support this environment).  The target here is for vendors
 303     # shipping a minimal smbd.
 304     "samba-nopython": [
 305                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 306                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 307                       ("make", "make -j", "text/plain"),
 308                       ("install", "make install", "text/plain"),
 309                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 310                       ("clean", "make clean", "text/plain"),
 311
 312                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 313                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 314                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 315
 316                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 317                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 318                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 319
 320                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 321                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 322                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 323
 324                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 325                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 326                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 327
 328                       # retry against installed library packages
 329                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 330                       ("libs-make", "make -j", "text/plain"),
 331                       ("libs-install", "make install", "text/plain"),
 332                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 333                       ("libs-clean", "make clean", "text/plain")
 334                       ],
 335
 336
 337
 338     "ldb": [
 339               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 340               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 341               ("make", "make", "text/plain"),
 342               ("install", "make install", "text/plain"),
 343               ("test", "make test", "text/plain"),
 344               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 345               ("make-no-lmdb", "make", "text/plain"),
 346               ("install-no-lmdb", "make install", "text/plain"),
 347               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 348               ("distcheck", "make distcheck", "text/plain"),
 349               ("clean", "make clean", "text/plain")],
 350
 351     "tdb": [
 352               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 353               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 354               ("make", "make", "text/plain"),
 355               ("install", "make install", "text/plain"),
 356               ("test", "make test", "text/plain"),
 357               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 358               ("distcheck", "make distcheck", "text/plain"),
 359               ("clean", "make clean", "text/plain")],
 360
 361     "talloc": [
 362                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 363                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 364                  ("make", "make", "text/plain"),
 365                  ("install", "make install", "text/plain"),
 366                  ("test", "make test", "text/plain"),
 367                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 368                  ("distcheck", "make distcheck", "text/plain"),
 369                  ("clean", "make clean", "text/plain")],
 370
 371     "replace": [
 372                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 373                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 374                   ("make", "make", "text/plain"),
 375                   ("install", "make install", "text/plain"),
 376                   ("test", "make test", "text/plain"),
 377                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 378                   ("distcheck", "make distcheck", "text/plain"),
 379                   ("clean", "make clean", "text/plain")],
 380
 381     "tevent": [
 382                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 383                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 384                  ("make", "make", "text/plain"),
 385                  ("install", "make install", "text/plain"),
 386                  ("test", "make test", "text/plain"),
 387                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 388                  ("distcheck", "make distcheck", "text/plain"),
 389                  ("clean", "make clean", "text/plain")],
 390
 391     "pidl": [
 392         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 393         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 394         ("touch", "touch *.yp", "text/plain"),
 395         ("make", "make", "text/plain"),
 396         ("test", "make test", "text/plain"),
 397         ("install", "make install", "text/plain"),
 398         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 399         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 400         ("clean", "make clean", "text/plain")],
 401
 402     "samba-buildpy3-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 403                    ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
 404                    ("make", "PYTHON='python3' make -j", "text/plain"),
 405                    ("install", "PYTHON='python3' make install", "text/plain"),
 406                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 407                    ("clean", "PYTHON='python3' make clean", "text/plain")],
 408
 409
 410     # these are useful for debugging autobuild
 411     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 412     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 413
 414
 415 }
 416
 417
 418 def do_print(msg):
 419     print("%s" % msg)
 420     sys.stdout.flush()
 421     sys.stderr.flush()
 422
 423
 424 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 425     if show is None:
 426         show = options.verbose
 427     if show:
 428         do_print("Running: '%s' in '%s'" % (cmd, dir))
 429     if output:
 430         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 431     elif checkfail:
 432         return check_call(cmd, shell=True, cwd=dir)
 433     else:
 434         return call(cmd, shell=True, cwd=dir)
 435
 436
 437 class builder(object):
 438     '''handle build of one directory'''
 439
 440     def __init__(self, name, sequence, cp=True, py3=False):
 441         self.name = name
 442         self.py3 = py3
 443         if name in builddirs:
 444             self.dir = builddirs[name]
 445         else:
 446             self.dir = "."
 447
 448         self.tag = self.name.replace('/', '_')
 449         self.sequence = sequence
 450         self.next = 0
 451         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 452         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 453         if options.verbose:
 454             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 455             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 456         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 457         self.stdout = open(self.stdout_path, 'w')
 458         self.stderr = open(self.stderr_path, 'w')
 459         self.stdin  = open("/dev/null", 'r')
 460         self.sdir = "%s/%s" % (testbase, self.tag)
 461         self.prefix = "%s/%s" % (test_prefix, self.tag)
 462         run_cmd("rm -rf %s" % self.sdir)
 463         run_cmd("rm -rf %s" % self.prefix)
 464         if cp:
 465             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 466         else:
 467             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 468         self.start_next()
 469
 470     def start_next(self):
 471         if self.next == len(self.sequence):
 472             if not options.nocleanup:
 473                 run_cmd("rm -rf %s" % self.sdir)
 474                 run_cmd("rm -rf %s" % self.prefix)
 475             do_print('%s: Completed OK' % self.name)
 476             self.done = True
 477             return
 478         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 479         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 480         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 481         if self.py3:
 482             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 483             # The trailing space is important
 484             self.cmd = self.cmd.replace("${PY3_ONLY}", "python3 ")
 485         else:
 486             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 487             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 488         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 489         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 490 #        if self.output_mime_type == "text/x-subunit":
 491 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 492         cwd = os.getcwd()
 493         os.chdir("%s/%s" % (self.sdir, self.dir))
 494         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 495         self.proc = Popen(self.cmd, shell=True,
 496                           close_fds=True,
 497                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 498         os.chdir(cwd)
 499         self.next += 1
 500
 501
 502 class buildlist(object):
 503     '''handle build of multiple directories'''
 504
 505     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 506         global tasks
 507         self.tlist = []
 508         self.tail_proc = None
 509         self.retry = None
 510         if tasknames == []:
 511             if options.restrict_tests:
 512                 tasknames = ["samba-test-only"]
 513             else:
 514                 tasknames = defaulttasks
 515         else:
 516             # If we are only running one test,
 517             # do not sleep randomly to wait for it to start
 518             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 519
 520         for n in tasknames:
 521             if n not in tasks and n.endswith("-py3"):
 522                 b = builder(n,
 523                             tasks[n[:-4]],
 524                             cp=n is not "pidl",
 525                             py3=True)
 526             else:
 527                 b = builder(n, tasks[n], cp=n is not "pidl")
 528             self.tlist.append(b)
 529         if options.retry:
 530             rebase_remote = "rebaseon"
 531             retry_task = [("retry",
 532                             '''set -e
 533                             git remote add -t %s %s %s
 534                             git fetch %s
 535                             while :; do
 536                               sleep 60
 537                               git describe %s/%s > old_remote_branch.desc
 538                               git fetch %s
 539                               git describe %s/%s > remote_branch.desc
 540                               diff old_remote_branch.desc remote_branch.desc
 541                             done
 542                            ''' % (
 543                                 rebase_branch, rebase_remote, rebase_url,
 544                                rebase_remote,
 545                                rebase_remote, rebase_branch,
 546                                rebase_remote,
 547                                rebase_remote, rebase_branch
 548                             ),
 549                             "test/plain")]
 550
 551             self.retry = builder('retry', retry_task, cp=False)
 552             self.need_retry = False
 553
 554     def kill_kids(self):
 555         if self.tail_proc is not None:
 556             self.tail_proc.terminate()
 557             self.tail_proc.wait()
 558             self.tail_proc = None
 559         if self.retry is not None:
 560             self.retry.proc.terminate()
 561             self.retry.proc.wait()
 562             self.retry = None
 563         for b in self.tlist:
 564             if b.proc is not None:
 565                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 566                 b.proc.terminate()
 567                 b.proc.wait()
 568                 b.proc = None
 569
 570     def wait_one(self):
 571         while True:
 572             none_running = True
 573             for b in self.tlist:
 574                 if b.proc is None:
 575                     continue
 576                 none_running = False
 577                 b.status = b.proc.poll()
 578                 if b.status is None:
 579                     continue
 580                 b.proc = None
 581                 return b
 582             if options.retry:
 583                 ret = self.retry.proc.poll()
 584                 if ret is not None:
 585                     self.need_retry = True
 586                     self.retry = None
 587                     return None
 588             if none_running:
 589                 return None
 590             time.sleep(0.1)
 591
 592     def run(self):
 593         while True:
 594             b = self.wait_one()
 595             if options.retry and self.need_retry:
 596                 self.kill_kids()
 597                 do_print("retry needed")
 598                 return (0, None, None, None, "retry")
 599             if b is None:
 600                 break
 601             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 602                 self.kill_kids()
 603                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 604             b.start_next()
 605         self.kill_kids()
 606         return (0, None, None, None, "All OK")
 607
 608     def write_system_info(self):
 609         filename = 'system-info.txt'
 610         f = open(filename, 'w')
 611         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 612                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 613             print('### %s' % cmd, file=f)
 614             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 615             print(file=f)
 616         f.close()
 617         return filename
 618
 619     def tarlogs(self, fname):
 620         tar = tarfile.open(fname, "w:gz")
 621         for b in self.tlist:
 622             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 623             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 624         if os.path.exists("autobuild.log"):
 625             tar.add("autobuild.log")
 626         sys_info = self.write_system_info()
 627         tar.add(sys_info)
 628         tar.close()
 629
 630     def remove_logs(self):
 631         for b in self.tlist:
 632             os.unlink(b.stdout_path)
 633             os.unlink(b.stderr_path)
 634
 635     def start_tail(self):
 636         cmd = ["tail", "-f"]
 637         for b in self.tlist:
 638             cmd.append(b.stdout_path)
 639             cmd.append(b.stderr_path)
 640         self.tail_proc = Popen(cmd, close_fds=True)
 641
 642
 643 def cleanup():
 644     if options.nocleanup:
 645         return
 646     run_cmd("stat %s || true" % test_tmpdir, show=True)
 647     run_cmd("stat %s" % testbase, show=True)
 648     do_print("Cleaning up %r" % cleanup_list)
 649     for d in cleanup_list:
 650         run_cmd("rm -rf %s" % d)
 651
 652
 653 def find_git_root():
 654     '''get to the top of the git repo'''
 655     p = os.getcwd()
 656     while p != '/':
 657         if os.path.isdir(os.path.join(p, ".git")):
 658             return p
 659         p = os.path.abspath(os.path.join(p, '..'))
 660     return None
 661
 662
 663 def daemonize(logfile):
 664     pid = os.fork()
 665     if pid == 0:  # Parent
 666         os.setsid()
 667         pid = os.fork()
 668         if pid != 0:  # Actual daemon
 669             os._exit(0)
 670     else:  # Grandparent
 671         os._exit(0)
 672
 673     import resource      # Resource usage information.
 674     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 675     if maxfd == resource.RLIM_INFINITY:
 676         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 677     for fd in range(0, maxfd):
 678         try:
 679             os.close(fd)
 680         except OSError:
 681             pass
 682     os.open(logfile, os.O_RDWR | os.O_CREAT)
 683     os.dup2(0, 1)
 684     os.dup2(0, 2)
 685
 686
 687 def write_pidfile(fname):
 688     '''write a pid file, cleanup on exit'''
 689     f = open(fname, mode='w')
 690     f.write("%u\n" % os.getpid())
 691     f.close()
 692
 693
 694 def rebase_tree(rebase_url, rebase_branch="master"):
 695     rebase_remote = "rebaseon"
 696     do_print("Rebasing on %s" % rebase_url)
 697     run_cmd("git describe HEAD", show=True, dir=test_master)
 698     run_cmd("git remote add -t %s %s %s" %
 699             (rebase_branch, rebase_remote, rebase_url),
 700             show=True, dir=test_master)
 701     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 702     if options.fix_whitespace:
 703         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 704                 (rebase_remote, rebase_branch),
 705                 show=True, dir=test_master)
 706     else:
 707         run_cmd("git rebase --force-rebase %s/%s" %
 708                 (rebase_remote, rebase_branch),
 709                 show=True, dir=test_master)
 710     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 711                    (rebase_remote, rebase_branch),
 712                    dir=test_master, output=True)
 713     if diff == '':
 714         do_print("No differences between HEAD and %s/%s - exiting" %
 715                  (rebase_remote, rebase_branch))
 716         sys.exit(0)
 717     run_cmd("git describe %s/%s" %
 718             (rebase_remote, rebase_branch),
 719             show=True, dir=test_master)
 720     run_cmd("git describe HEAD", show=True, dir=test_master)
 721     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 722             (rebase_remote, rebase_branch),
 723             show=True, dir=test_master)
 724
 725
 726 def push_to(push_url, push_branch="master"):
 727     push_remote = "pushto"
 728     do_print("Pushing to %s" % push_url)
 729     if options.mark:
 730         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 731         run_cmd("git commit --amend -c HEAD", dir=test_master)
 732         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 733         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 734     run_cmd("git remote add -t %s %s %s" %
 735             (push_branch, push_remote, push_url),
 736             show=True, dir=test_master)
 737     run_cmd("git push %s +HEAD:%s" %
 738             (push_remote, push_branch),
 739             show=True, dir=test_master)
 740
 741
 742 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 743
 744 gitroot = find_git_root()
 745 if gitroot is None:
 746     raise Exception("Failed to find git root")
 747
 748 parser = OptionParser()
 749 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 750 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 751 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 752 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 753                   default=def_testbase)
 754 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 755 parser.add_option("", "--verbose", help="show all commands as they are run",
 756                   default=False, action="store_true")
 757 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 758                   default=None, type='str')
 759 parser.add_option("", "--pushto", help="push to a git url on success",
 760                   default=None, type='str')
 761 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 762                   default=False, action="store_true")
 763 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 764                   default=False, action="store_true")
 765 parser.add_option("", "--retry", help="automatically retry if master changes",
 766                   default=False, action="store_true")
 767 parser.add_option("", "--email", help="send email to the given address on failure",
 768                   type='str', default=None)
 769 parser.add_option("", "--email-from", help="send email from the given address",
 770                   type='str', default="autobuild@samba.org")
 771 parser.add_option("", "--email-server", help="send email via the given server",
 772                   type='str', default='localhost')
 773 parser.add_option("", "--always-email", help="always send email, even on success",
 774                   action="store_true")
 775 parser.add_option("", "--daemon", help="daemonize after initial setup",
 776                   action="store_true")
 777 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 778                   default="master", type='str')
 779 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 780                   default=gitroot, type='str')
 781 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 782                   default=False, action="store_true")
 783 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 784                   default='')
 785
 786
 787 def send_email(subject, text, log_tar):
 788     if options.email is None:
 789         do_print("not sending email because the recipient is not set")
 790         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 791                  (subject, text))
 792         return
 793     outer = MIMEMultipart()
 794     outer['Subject'] = subject
 795     outer['To'] = options.email
 796     outer['From'] = options.email_from
 797     outer['Date'] = email.utils.formatdate(localtime=True)
 798     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 799     outer.attach(MIMEText(text, 'plain'))
 800     if options.attach_logs:
 801         fp = open(log_tar, 'rb')
 802         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 803         fp.close()
 804         # Set the filename parameter
 805         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 806         outer.attach(msg)
 807     content = outer.as_string()
 808     s = smtplib.SMTP(options.email_server)
 809     s.sendmail(options.email_from, [options.email], content)
 810     s.set_debuglevel(1)
 811     s.quit()
 812
 813
 814 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 815                   elapsed_time, log_base=None, add_log_tail=True):
 816     '''send an email to options.email about the failure'''
 817     elapsed_minutes = elapsed_time / 60.0
 818     if log_base is None:
 819         log_base = gitroot
 820     text = '''
 821 Dear Developer,
 822
 823 Your autobuild on %s failed after %.1f minutes
 824 when trying to test %s with the following error:
 825
 826    %s
 827
 828 the autobuild has been abandoned. Please fix the error and resubmit.
 829
 830 A summary of the autobuild process is here:
 831
 832   %s/autobuild.log
 833 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 834
 835     if options.restrict_tests:
 836         text += """
 837 The build was restricted to tests matching %s\n""" % options.restrict_tests
 838
 839     if failed_task != 'rebase':
 840         text += '''
 841 You can see logs of the failed task here:
 842
 843   %s/%s.stdout
 844   %s/%s.stderr
 845
 846 or you can get full logs of all tasks in this job here:
 847
 848   %s/logs.tar.gz
 849
 850 The top commit for the tree that was built was:
 851
 852 %s
 853
 854 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 855
 856     if add_log_tail:
 857         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 858         lines = f.readlines()
 859         log_tail = "".join(lines[-50:])
 860         num_lines = len(lines)
 861         if num_lines < 50:
 862             # Also include stderr (compile failures) if < 50 lines of stdout
 863             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 864             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 865
 866         text += '''
 867 The last 50 lines of log messages:
 868
 869 %s
 870     ''' % log_tail
 871         f.close()
 872
 873     logs = os.path.join(gitroot, 'logs.tar.gz')
 874     send_email('autobuild[%s] failure on %s for task %s during %s'
 875                % (options.branch, platform.node(), failed_task, failed_stage),
 876                text, logs)
 877
 878
 879 def email_success(elapsed_time, log_base=None):
 880     '''send an email to options.email about a successful build'''
 881     if log_base is None:
 882         log_base = gitroot
 883     text = '''
 884 Dear Developer,
 885
 886 Your autobuild on %s has succeeded after %.1f minutes.
 887
 888 ''' % (platform.node(), elapsed_time / 60.)
 889
 890     if options.restrict_tests:
 891         text += """
 892 The build was restricted to tests matching %s\n""" % options.restrict_tests
 893
 894     if options.keeplogs:
 895         text += '''
 896
 897 you can get full logs of all tasks in this job here:
 898
 899   %s/logs.tar.gz
 900
 901 ''' % log_base
 902
 903     text += '''
 904 The top commit for the tree that was built was:
 905
 906 %s
 907 ''' % top_commit_msg
 908
 909     logs = os.path.join(gitroot, 'logs.tar.gz')
 910     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 911                text, logs)
 912
 913
 914 (options, args) = parser.parse_args()
 915
 916 if options.retry:
 917     if options.rebase is None:
 918         raise Exception('You can only use --retry if you also rebase')
 919
 920 testbase = "%s/b%u" % (options.testbase, os.getpid())
 921 test_master = "%s/master" % testbase
 922 test_prefix = "%s/prefix" % testbase
 923 test_tmpdir = "%s/tmp" % testbase
 924 os.environ['TMPDIR'] = test_tmpdir
 925
 926 # get the top commit message, for emails
 927 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 928
 929 try:
 930     os.makedirs(testbase)
 931 except Exception as reason:
 932     raise Exception("Unable to create %s : %s" % (testbase, reason))
 933 cleanup_list.append(testbase)
 934
 935 if options.daemon:
 936     logfile = os.path.join(testbase, "log")
 937     do_print("Forking into the background, writing progress to %s" % logfile)
 938     daemonize(logfile)
 939
 940 write_pidfile(gitroot + "/autobuild.pid")
 941
 942 start_time = time.time()
 943
 944 while True:
 945     try:
 946         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 947         os.makedirs(test_tmpdir)
 948         # The waf uninstall code removes empty directories all the way
 949         # up the tree.  Creating a file in test_tmpdir stops it from
 950         # being removed.
 951         run_cmd("touch %s" % os.path.join(test_tmpdir,
 952                                           ".directory-is-not-empty"), show=True)
 953         run_cmd("stat %s" % test_tmpdir, show=True)
 954         run_cmd("stat %s" % testbase, show=True)
 955         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 956     except Exception:
 957         cleanup()
 958         raise
 959
 960     try:
 961         try:
 962             if options.rebase is not None:
 963                 rebase_tree(options.rebase, rebase_branch=options.branch)
 964         except Exception:
 965             cleanup_list.append(gitroot + "/autobuild.pid")
 966             cleanup()
 967             elapsed_time = time.time() - start_time
 968             email_failure(-1, 'rebase', 'rebase', 'rebase',
 969                           'rebase on %s failed' % options.branch,
 970                           elapsed_time, log_base=options.log_base)
 971             sys.exit(1)
 972         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 973         if options.tail:
 974             blist.start_tail()
 975         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 976         if status != 0 or errstr != "retry":
 977             break
 978         cleanup()
 979     except Exception:
 980         cleanup()
 981         raise
 982
 983 cleanup_list.append(gitroot + "/autobuild.pid")
 984
 985 do_print(errstr)
 986
 987 blist.kill_kids()
 988 if options.tail:
 989     do_print("waiting for tail to flush")
 990     time.sleep(1)
 991
 992 elapsed_time = time.time() - start_time
 993 if status == 0:
 994     if options.passcmd is not None:
 995         do_print("Running passcmd: %s" % options.passcmd)
 996         run_cmd(options.passcmd, dir=test_master)
 997     if options.pushto is not None:
 998         push_to(options.pushto, push_branch=options.branch)
 999     if options.keeplogs or options.attach_logs:
1000         blist.tarlogs("logs.tar.gz")
1001         do_print("Logs in logs.tar.gz")
1002     if options.always_email:
1003         email_success(elapsed_time, log_base=options.log_base)
1004     blist.remove_logs()
1005     cleanup()
1006     do_print(errstr)
1007     sys.exit(0)
1008
1009 # something failed, gather a tar of the logs
1010 blist.tarlogs("logs.tar.gz")
1011
1012 if options.email is not None:
1013     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1014                   elapsed_time, log_base=options.log_base)
1015 else:
1016     elapsed_minutes = elapsed_time / 60.0
1017     print('''
1018
1019 ####################################################################
1020
1021 AUTOBUILD FAILURE
1022
1023 Your autobuild[%s] on %s failed after %.1f minutes
1024 when trying to test %s with the following error:
1025
1026    %s
1027
1028 the autobuild has been abandoned. Please fix the error and resubmit.
1029
1030 ####################################################################
1031
1032 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1033
1034 cleanup()
1035 do_print(errstr)
1036 do_print("Logs in logs.tar.gz")
1037 sys.exit(status)