script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-py3": ".",
  40     "samba-nt4": ".",
  41     "samba-nt4-py3": ".",
  42     "samba-fileserver": ".",
  43     "samba-xc": ".",
  44     "samba-o3": ".",
  45     "samba-ctdb": ".",
  46     "samba-libs": ".",
  47     "samba-libs-py3": ".",
  48     "samba-static": ".",
  49     "samba-none-env": ".",
  50     "samba-none-env-py3": ".",
  51     "samba-ad-dc": ".",
  52     "samba-ad-dc-py3": ".",
  53     "samba-ad-dc-2": ".",
  54     "samba-ad-dc-2-py3": ".",
  55     "samba-systemkrb5": ".",
  56     "samba-nopython": ".",
  57     "samba-buildpy3-only": ".",
  58     "ldb": "lib/ldb",
  59     "tdb": "lib/tdb",
  60     "talloc": "lib/talloc",
  61     "replace": "lib/replace",
  62     "tevent": "lib/tevent",
  63     "pidl": "pidl"
  64 }
  65
  66 defaulttasks = builddirs.keys()
  67
  68 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  69     defaulttasks.remove("samba-o3")
  70
  71 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  72 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  73
  74 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  75 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  76 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  77 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  78 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  79 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  80 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  81
  82 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  83     extra_python = ""
  84 else:
  85     extra_python = "--extra-python=/usr/bin/python3"
  86
  87 tasks = {
  88     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  89                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  90                ("make", "make all", "text/plain"),
  91                ("install", "make install", "text/plain"),
  92                ("test", "make autotest", "text/plain"),
  93                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  94                ("clean", "make clean", "text/plain")],
  95
  96     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
  97     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  98                 ("make", "make -j", "text/plain"),
  99                 ("test", "make test FAIL_IMMEDIATELY=1 "
 100                  "TESTS='${PY3_ONLY}"
 101                  "--exclude-env=none "
 102                  "--exclude-env=nt4_dc "
 103                  "--exclude-env=nt4_member "
 104                  "--exclude-env=ad_dc "
 105                  "--exclude-env=ad_dc_no_nss "
 106                  "--exclude-env=fl2003dc "
 107                  "--exclude-env=fl2008r2dc "
 108                  "--exclude-env=ad_member "
 109                  "--exclude-env=ad_member_idmap_rid "
 110                  "--exclude-env=ad_member_idmap_ad "
 111                  "--exclude-env=chgdcpass "
 112                  "--exclude-env=vampire_2000_dc "
 113                  "--exclude-env=fl2000dc "
 114                  "--exclude-env=fileserver "
 115                  "--exclude-env=backupfromdc "
 116                  "--exclude-env=restoredc "
 117                  "--exclude-env=renamedc "
 118                  "--exclude-env=offlinebackupdc "
 119                  "--exclude-env=labdc "
 120                  "'",
 121                  "text/plain"),
 122                 ("install", "make install", "text/plain"),
 123                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 124                 ("clean", "make clean", "text/plain")],
 125
 126     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 127     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 128                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 129                     ("make", "make -j", "text/plain"),
 130                     ("test", "make test FAIL_IMMEDIATELY=1 "
 131                      "TESTS='${PY3_ONLY}"
 132                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 133                     ("install", "make install", "text/plain"),
 134                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 135                     ("clean", "make clean", "text/plain")],
 136
 137     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 138     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 139                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 140                            ("make", "make -j", "text/plain"),
 141                            ("test", "make test FAIL_IMMEDIATELY=1 "
 142                             "TESTS='${PY3_ONLY}"
 143                             "--include-env=fileserver'", "text/plain"),
 144                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 145
 146     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 147     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 148                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 149                       ("make", "make -j", "text/plain"),
 150                       ("test", "make test FAIL_IMMEDIATELY=1 "
 151                        "TESTS='${PY3_ONLY}"
 152                        "--include-env=ad_dc "
 153                        "--include-env=fl2003dc "
 154                        "--include-env=fl2008r2dc "
 155                        "--include-env=ad_member "
 156                        "--include-env=ad_member_idmap_rid "
 157                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 158                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 159
 160     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 161     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 162                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 163                         ("make", "make -j", "text/plain"),
 164                         ("test", "make test FAIL_IMMEDIATELY=1 "
 165                          "TESTS='${PY3_ONLY}"
 166                          "--include-env=chgdcpass "
 167                          "--include-env=vampire_2000_dc "
 168                          "--include-env=fl2000dc "
 169                          "--include-env=ad_dc_no_nss "
 170                          "--include-env=backupfromdc "
 171                          "--include-env=restoredc "
 172                          "--include-env=renamedc "
 173                          "--include-env=offlinebackupdc "
 174                          "--include-env=labdc "
 175                          "'",
 176                          "text/plain"),
 177                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 178
 179     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 180                           ("make", "make -j", "text/plain"),
 181                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 182
 183     # Test cross-compile infrastructure
 184     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 185                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 186                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 187                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 188                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 189                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 190                    ("compare-results", "script/compare_cc_results.py "
 191                     "./bin/c4che/default{} "
 192                     "./bin-xe/c4che/default{} "
 193                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 194
 195     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 196     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 197                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 198                    ("make", "make -j", "text/plain"),
 199                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 200                     "TESTS='${PY3_ONLY}"
 201                     "--include-env=ad_dc'", "text/plain"),
 202                    ("install", "make install", "text/plain"),
 203                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 204                    ("clean", "make clean", "text/plain")],
 205
 206     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 207
 208                      # make sure we have tdb around:
 209                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 210                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 211                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 212
 213
 214                      # build samba with cluster support (also building ctdb):
 215                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 216                      ("samba-make", "make", "text/plain"),
 217                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 218                      ("samba-install", "make install", "text/plain"),
 219                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 220
 221                      # clean up:
 222                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 223                      ("clean", "make clean", "text/plain"),
 224                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 225
 226     "samba-libs": [
 227                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 228                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 229                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 230                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 231
 232                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 233                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 234                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 235
 236                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 237                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 238                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 239
 240                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 241                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 242                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 243
 244                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 245                       ("nondevel-make", "make -j", "text/plain"),
 246                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 247                       ("nondevel-install", "make install", "text/plain"),
 248                       ("nondevel-dist", "make dist", "text/plain"),
 249
 250                       # retry with all modules shared
 251                       ("allshared-distclean", "make distclean", "text/plain"),
 252                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 253                       ("allshared-make", "make -j", "text/plain")],
 254
 255     "samba-none-env": [
 256                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 257                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 258                       ("make", "make -j", "text/plain"),
 259                       ("test", "make test "
 260                        "FAIL_IMMEDIATELY=1 "
 261                        "TESTS='${PY3_ONLY}"
 262                        "--include-env=none'",
 263                        "text/plain")],
 264
 265     "samba-static": [
 266                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 267                       # build with all modules static
 268                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 269                       ("allstatic-make", "make -j", "text/plain"),
 270                       ("allstatic-test", "make test "
 271                        "FAIL_IMMEDIATELY=1 "
 272                        "TESTS='samba3.smb2.create.*nt4_dc'",
 273                        "text/plain"),
 274
 275                       # retry without any required modules
 276                       ("none-distclean", "make distclean", "text/plain"),
 277                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 278                       ("none-make", "make -j", "text/plain"),
 279
 280                       # retry with nonshared smbd and smbtorture
 281                       ("nonshared-distclean", "make distclean", "text/plain"),
 282                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 283                       ("nonshared-make", "make -j", "text/plain")],
 284
 285     "samba-systemkrb5": [
 286                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 287                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 288                       ("make", "make -j", "text/plain"),
 289                       # we currently cannot run a full make test, a limited list of tests could be run
 290                       # via "make test TESTS=sometests"
 291                       ("test", "make test FAIL_IMMEDIATELY=1 "
 292                        "TESTS='${PY3_ONLY}"
 293                        "--include-env=ktest'", "text/plain"),
 294                       ("install", "make install", "text/plain"),
 295                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 296                       ("clean", "make clean", "text/plain")
 297                       ],
 298
 299     # Test Samba without python still builds.  When this test fails
 300     # due to more use of Python, the expectations is that the newly
 301     # failing part of the code should be disabled when
 302     # --disable-python is set (rather than major work being done to
 303     # support this environment).  The target here is for vendors
 304     # shipping a minimal smbd.
 305     "samba-nopython": [
 306                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 307                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 308                       ("make", "make -j", "text/plain"),
 309                       ("install", "make install", "text/plain"),
 310                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 311                       ("clean", "make clean", "text/plain"),
 312
 313                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 314                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 315                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 316
 317                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 318                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 319                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 320
 321                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 322                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 323                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 324
 325                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 326                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 327                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 328
 329                       # retry against installed library packages
 330                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 331                       ("libs-make", "make -j", "text/plain"),
 332                       ("libs-install", "make install", "text/plain"),
 333                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 334                       ("libs-clean", "make clean", "text/plain")
 335                       ],
 336
 337
 338
 339     "ldb": [
 340               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 341               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 342               ("make", "make", "text/plain"),
 343               ("install", "make install", "text/plain"),
 344               ("test", "make test", "text/plain"),
 345               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 346               ("make-no-lmdb", "make", "text/plain"),
 347               ("install-no-lmdb", "make install", "text/plain"),
 348               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 349               ("distcheck", "make distcheck", "text/plain"),
 350               ("clean", "make clean", "text/plain")],
 351
 352     "tdb": [
 353               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 354               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 355               ("make", "make", "text/plain"),
 356               ("install", "make install", "text/plain"),
 357               ("test", "make test", "text/plain"),
 358               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 359               ("distcheck", "make distcheck", "text/plain"),
 360               ("clean", "make clean", "text/plain")],
 361
 362     "talloc": [
 363                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 364                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 365                  ("make", "make", "text/plain"),
 366                  ("install", "make install", "text/plain"),
 367                  ("test", "make test", "text/plain"),
 368                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 369                  ("distcheck", "make distcheck", "text/plain"),
 370                  ("clean", "make clean", "text/plain")],
 371
 372     "replace": [
 373                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 374                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 375                   ("make", "make", "text/plain"),
 376                   ("install", "make install", "text/plain"),
 377                   ("test", "make test", "text/plain"),
 378                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 379                   ("distcheck", "make distcheck", "text/plain"),
 380                   ("clean", "make clean", "text/plain")],
 381
 382     "tevent": [
 383                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 384                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 385                  ("make", "make", "text/plain"),
 386                  ("install", "make install", "text/plain"),
 387                  ("test", "make test", "text/plain"),
 388                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 389                  ("distcheck", "make distcheck", "text/plain"),
 390                  ("clean", "make clean", "text/plain")],
 391
 392     "pidl": [
 393         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 394         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 395         ("touch", "touch *.yp", "text/plain"),
 396         ("make", "make", "text/plain"),
 397         ("test", "make test", "text/plain"),
 398         ("install", "make install", "text/plain"),
 399         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 400         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 401         ("clean", "make clean", "text/plain")],
 402
 403     "samba-buildpy3-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 404                    ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
 405                    ("make", "PYTHON='python3' make -j", "text/plain"),
 406                    ("install", "PYTHON='python3' make install", "text/plain"),
 407                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 408                    ("clean", "PYTHON='python3' make clean", "text/plain")],
 409
 410     # these are useful for debugging autobuild
 411     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 412     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 413 }
 414
 415
 416 def do_print(msg):
 417     print("%s" % msg)
 418     sys.stdout.flush()
 419     sys.stderr.flush()
 420
 421
 422 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 423     if show is None:
 424         show = options.verbose
 425     if show:
 426         do_print("Running: '%s' in '%s'" % (cmd, dir))
 427     if output:
 428         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 429     elif checkfail:
 430         return check_call(cmd, shell=True, cwd=dir)
 431     else:
 432         return call(cmd, shell=True, cwd=dir)
 433
 434
 435 class builder(object):
 436     '''handle build of one directory'''
 437
 438     def __init__(self, name, sequence, cp=True, py3=False):
 439         self.name = name
 440         self.py3 = py3
 441         if name in builddirs:
 442             self.dir = builddirs[name]
 443         else:
 444             self.dir = "."
 445
 446         self.tag = self.name.replace('/', '_')
 447         self.sequence = sequence
 448         self.next = 0
 449         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 450         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 451         if options.verbose:
 452             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 453             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 454         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 455         self.stdout = open(self.stdout_path, 'w')
 456         self.stderr = open(self.stderr_path, 'w')
 457         self.stdin  = open("/dev/null", 'r')
 458         self.sdir = "%s/%s" % (testbase, self.tag)
 459         self.prefix = "%s/%s" % (test_prefix, self.tag)
 460         run_cmd("rm -rf %s" % self.sdir)
 461         run_cmd("rm -rf %s" % self.prefix)
 462         if cp:
 463             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 464         else:
 465             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 466         self.start_next()
 467
 468     def start_next(self):
 469         if self.next == len(self.sequence):
 470             if not options.nocleanup:
 471                 run_cmd("rm -rf %s" % self.sdir)
 472                 run_cmd("rm -rf %s" % self.prefix)
 473             do_print('%s: Completed OK' % self.name)
 474             self.done = True
 475             return
 476         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 477         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 478         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 479         if self.py3:
 480             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 481             # The trailing space is important
 482             self.cmd = self.cmd.replace("${PY3_ONLY}", "python3 ")
 483         else:
 484             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 485             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 486         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 487         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 488 #        if self.output_mime_type == "text/x-subunit":
 489 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 490         cwd = os.getcwd()
 491         os.chdir("%s/%s" % (self.sdir, self.dir))
 492         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 493         self.proc = Popen(self.cmd, shell=True,
 494                           close_fds=True,
 495                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 496         os.chdir(cwd)
 497         self.next += 1
 498
 499
 500 class buildlist(object):
 501     '''handle build of multiple directories'''
 502
 503     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 504         global tasks
 505         self.tlist = []
 506         self.tail_proc = None
 507         self.retry = None
 508         if tasknames == []:
 509             if options.restrict_tests:
 510                 tasknames = ["samba-test-only"]
 511             else:
 512                 tasknames = defaulttasks
 513         else:
 514             # If we are only running one test,
 515             # do not sleep randomly to wait for it to start
 516             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 517
 518         for n in tasknames:
 519             if n not in tasks and n.endswith("-py3"):
 520                 b = builder(n,
 521                             tasks[n[:-4]],
 522                             cp=n is not "pidl",
 523                             py3=True)
 524             else:
 525                 b = builder(n, tasks[n], cp=n is not "pidl")
 526             self.tlist.append(b)
 527         if options.retry:
 528             rebase_remote = "rebaseon"
 529             retry_task = [("retry",
 530                             '''set -e
 531                             git remote add -t %s %s %s
 532                             git fetch %s
 533                             while :; do
 534                               sleep 60
 535                               git describe %s/%s > old_remote_branch.desc
 536                               git fetch %s
 537                               git describe %s/%s > remote_branch.desc
 538                               diff old_remote_branch.desc remote_branch.desc
 539                             done
 540                            ''' % (
 541                                 rebase_branch, rebase_remote, rebase_url,
 542                                rebase_remote,
 543                                rebase_remote, rebase_branch,
 544                                rebase_remote,
 545                                rebase_remote, rebase_branch
 546                             ),
 547                             "test/plain")]
 548
 549             self.retry = builder('retry', retry_task, cp=False)
 550             self.need_retry = False
 551
 552     def kill_kids(self):
 553         if self.tail_proc is not None:
 554             self.tail_proc.terminate()
 555             self.tail_proc.wait()
 556             self.tail_proc = None
 557         if self.retry is not None:
 558             self.retry.proc.terminate()
 559             self.retry.proc.wait()
 560             self.retry = None
 561         for b in self.tlist:
 562             if b.proc is not None:
 563                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 564                 b.proc.terminate()
 565                 b.proc.wait()
 566                 b.proc = None
 567
 568     def wait_one(self):
 569         while True:
 570             none_running = True
 571             for b in self.tlist:
 572                 if b.proc is None:
 573                     continue
 574                 none_running = False
 575                 b.status = b.proc.poll()
 576                 if b.status is None:
 577                     continue
 578                 b.proc = None
 579                 return b
 580             if options.retry:
 581                 ret = self.retry.proc.poll()
 582                 if ret is not None:
 583                     self.need_retry = True
 584                     self.retry = None
 585                     return None
 586             if none_running:
 587                 return None
 588             time.sleep(0.1)
 589
 590     def run(self):
 591         while True:
 592             b = self.wait_one()
 593             if options.retry and self.need_retry:
 594                 self.kill_kids()
 595                 do_print("retry needed")
 596                 return (0, None, None, None, "retry")
 597             if b is None:
 598                 break
 599             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 600                 self.kill_kids()
 601                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 602             b.start_next()
 603         self.kill_kids()
 604         return (0, None, None, None, "All OK")
 605
 606     def write_system_info(self):
 607         filename = 'system-info.txt'
 608         f = open(filename, 'w')
 609         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 610                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 611             print('### %s' % cmd, file=f)
 612             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 613             print(file=f)
 614         f.close()
 615         return filename
 616
 617     def tarlogs(self, fname):
 618         tar = tarfile.open(fname, "w:gz")
 619         for b in self.tlist:
 620             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 621             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 622         if os.path.exists("autobuild.log"):
 623             tar.add("autobuild.log")
 624         sys_info = self.write_system_info()
 625         tar.add(sys_info)
 626         tar.close()
 627
 628     def remove_logs(self):
 629         for b in self.tlist:
 630             os.unlink(b.stdout_path)
 631             os.unlink(b.stderr_path)
 632
 633     def start_tail(self):
 634         cmd = ["tail", "-f"]
 635         for b in self.tlist:
 636             cmd.append(b.stdout_path)
 637             cmd.append(b.stderr_path)
 638         self.tail_proc = Popen(cmd, close_fds=True)
 639
 640
 641 def cleanup():
 642     if options.nocleanup:
 643         return
 644     run_cmd("stat %s || true" % test_tmpdir, show=True)
 645     run_cmd("stat %s" % testbase, show=True)
 646     do_print("Cleaning up %r" % cleanup_list)
 647     for d in cleanup_list:
 648         run_cmd("rm -rf %s" % d)
 649
 650
 651 def find_git_root():
 652     '''get to the top of the git repo'''
 653     p = os.getcwd()
 654     while p != '/':
 655         if os.path.isdir(os.path.join(p, ".git")):
 656             return p
 657         p = os.path.abspath(os.path.join(p, '..'))
 658     return None
 659
 660
 661 def daemonize(logfile):
 662     pid = os.fork()
 663     if pid == 0:  # Parent
 664         os.setsid()
 665         pid = os.fork()
 666         if pid != 0:  # Actual daemon
 667             os._exit(0)
 668     else:  # Grandparent
 669         os._exit(0)
 670
 671     import resource      # Resource usage information.
 672     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 673     if maxfd == resource.RLIM_INFINITY:
 674         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 675     for fd in range(0, maxfd):
 676         try:
 677             os.close(fd)
 678         except OSError:
 679             pass
 680     os.open(logfile, os.O_RDWR | os.O_CREAT)
 681     os.dup2(0, 1)
 682     os.dup2(0, 2)
 683
 684
 685 def write_pidfile(fname):
 686     '''write a pid file, cleanup on exit'''
 687     f = open(fname, mode='w')
 688     f.write("%u\n" % os.getpid())
 689     f.close()
 690
 691
 692 def rebase_tree(rebase_url, rebase_branch="master"):
 693     rebase_remote = "rebaseon"
 694     do_print("Rebasing on %s" % rebase_url)
 695     run_cmd("git describe HEAD", show=True, dir=test_master)
 696     run_cmd("git remote add -t %s %s %s" %
 697             (rebase_branch, rebase_remote, rebase_url),
 698             show=True, dir=test_master)
 699     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 700     if options.fix_whitespace:
 701         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 702                 (rebase_remote, rebase_branch),
 703                 show=True, dir=test_master)
 704     else:
 705         run_cmd("git rebase --force-rebase %s/%s" %
 706                 (rebase_remote, rebase_branch),
 707                 show=True, dir=test_master)
 708     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 709                    (rebase_remote, rebase_branch),
 710                    dir=test_master, output=True)
 711     if diff == '':
 712         do_print("No differences between HEAD and %s/%s - exiting" %
 713                  (rebase_remote, rebase_branch))
 714         sys.exit(0)
 715     run_cmd("git describe %s/%s" %
 716             (rebase_remote, rebase_branch),
 717             show=True, dir=test_master)
 718     run_cmd("git describe HEAD", show=True, dir=test_master)
 719     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 720             (rebase_remote, rebase_branch),
 721             show=True, dir=test_master)
 722
 723
 724 def push_to(push_url, push_branch="master"):
 725     push_remote = "pushto"
 726     do_print("Pushing to %s" % push_url)
 727     if options.mark:
 728         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 729         run_cmd("git commit --amend -c HEAD", dir=test_master)
 730         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 731         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 732     run_cmd("git remote add -t %s %s %s" %
 733             (push_branch, push_remote, push_url),
 734             show=True, dir=test_master)
 735     run_cmd("git push %s +HEAD:%s" %
 736             (push_remote, push_branch),
 737             show=True, dir=test_master)
 738
 739
 740 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 741
 742 gitroot = find_git_root()
 743 if gitroot is None:
 744     raise Exception("Failed to find git root")
 745
 746 parser = OptionParser()
 747 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 748 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 749 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 750 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 751                   default=def_testbase)
 752 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 753 parser.add_option("", "--verbose", help="show all commands as they are run",
 754                   default=False, action="store_true")
 755 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 756                   default=None, type='str')
 757 parser.add_option("", "--pushto", help="push to a git url on success",
 758                   default=None, type='str')
 759 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 760                   default=False, action="store_true")
 761 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 762                   default=False, action="store_true")
 763 parser.add_option("", "--retry", help="automatically retry if master changes",
 764                   default=False, action="store_true")
 765 parser.add_option("", "--email", help="send email to the given address on failure",
 766                   type='str', default=None)
 767 parser.add_option("", "--email-from", help="send email from the given address",
 768                   type='str', default="autobuild@samba.org")
 769 parser.add_option("", "--email-server", help="send email via the given server",
 770                   type='str', default='localhost')
 771 parser.add_option("", "--always-email", help="always send email, even on success",
 772                   action="store_true")
 773 parser.add_option("", "--daemon", help="daemonize after initial setup",
 774                   action="store_true")
 775 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 776                   default="master", type='str')
 777 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 778                   default=gitroot, type='str')
 779 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 780                   default=False, action="store_true")
 781 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 782                   default='')
 783
 784
 785 def send_email(subject, text, log_tar):
 786     if options.email is None:
 787         do_print("not sending email because the recipient is not set")
 788         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 789                  (subject, text))
 790         return
 791     outer = MIMEMultipart()
 792     outer['Subject'] = subject
 793     outer['To'] = options.email
 794     outer['From'] = options.email_from
 795     outer['Date'] = email.utils.formatdate(localtime=True)
 796     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 797     outer.attach(MIMEText(text, 'plain'))
 798     if options.attach_logs:
 799         fp = open(log_tar, 'rb')
 800         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 801         fp.close()
 802         # Set the filename parameter
 803         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 804         outer.attach(msg)
 805     content = outer.as_string()
 806     s = smtplib.SMTP(options.email_server)
 807     s.sendmail(options.email_from, [options.email], content)
 808     s.set_debuglevel(1)
 809     s.quit()
 810
 811
 812 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 813                   elapsed_time, log_base=None, add_log_tail=True):
 814     '''send an email to options.email about the failure'''
 815     elapsed_minutes = elapsed_time / 60.0
 816     user = os.getenv("USER")
 817     if log_base is None:
 818         log_base = gitroot
 819     text = '''
 820 Dear Developer,
 821
 822 Your autobuild on %s failed after %.1f minutes
 823 when trying to test %s with the following error:
 824
 825    %s
 826
 827 the autobuild has been abandoned. Please fix the error and resubmit.
 828
 829 A summary of the autobuild process is here:
 830
 831   %s/autobuild.log
 832 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 833
 834     if options.restrict_tests:
 835         text += """
 836 The build was restricted to tests matching %s\n""" % options.restrict_tests
 837
 838     if failed_task != 'rebase':
 839         text += '''
 840 You can see logs of the failed task here:
 841
 842   %s/%s.stdout
 843   %s/%s.stderr
 844
 845 or you can get full logs of all tasks in this job here:
 846
 847   %s/logs.tar.gz
 848
 849 The top commit for the tree that was built was:
 850
 851 %s
 852
 853 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 854
 855     if add_log_tail:
 856         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 857         lines = f.readlines()
 858         log_tail = "".join(lines[-50:])
 859         num_lines = len(lines)
 860         if num_lines < 50:
 861             # Also include stderr (compile failures) if < 50 lines of stdout
 862             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 863             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 864
 865         text += '''
 866 The last 50 lines of log messages:
 867
 868 %s
 869     ''' % log_tail
 870         f.close()
 871
 872     logs = os.path.join(gitroot, 'logs.tar.gz')
 873     send_email('autobuild[%s] failure on %s for task %s during %s'
 874                % (options.branch, platform.node(), failed_task, failed_stage),
 875                text, logs)
 876
 877
 878 def email_success(elapsed_time, log_base=None):
 879     '''send an email to options.email about a successful build'''
 880     user = os.getenv("USER")
 881     if log_base is None:
 882         log_base = gitroot
 883     text = '''
 884 Dear Developer,
 885
 886 Your autobuild on %s has succeeded after %.1f minutes.
 887
 888 ''' % (platform.node(), elapsed_time / 60.)
 889
 890     if options.restrict_tests:
 891         text += """
 892 The build was restricted to tests matching %s\n""" % options.restrict_tests
 893
 894     if options.keeplogs:
 895         text += '''
 896
 897 you can get full logs of all tasks in this job here:
 898
 899   %s/logs.tar.gz
 900
 901 ''' % log_base
 902
 903     text += '''
 904 The top commit for the tree that was built was:
 905
 906 %s
 907 ''' % top_commit_msg
 908
 909     logs = os.path.join(gitroot, 'logs.tar.gz')
 910     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 911                text, logs)
 912
 913
 914 (options, args) = parser.parse_args()
 915
 916 if options.retry:
 917     if options.rebase is None:
 918         raise Exception('You can only use --retry if you also rebase')
 919
 920 testbase = "%s/b%u" % (options.testbase, os.getpid())
 921 test_master = "%s/master" % testbase
 922 test_prefix = "%s/prefix" % testbase
 923 test_tmpdir = "%s/tmp" % testbase
 924 os.environ['TMPDIR'] = test_tmpdir
 925
 926 # get the top commit message, for emails
 927 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 928
 929 try:
 930     os.makedirs(testbase)
 931 except Exception as reason:
 932     raise Exception("Unable to create %s : %s" % (testbase, reason))
 933 cleanup_list.append(testbase)
 934
 935 if options.daemon:
 936     logfile = os.path.join(testbase, "log")
 937     do_print("Forking into the background, writing progress to %s" % logfile)
 938     daemonize(logfile)
 939
 940 write_pidfile(gitroot + "/autobuild.pid")
 941
 942 start_time = time.time()
 943
 944 while True:
 945     try:
 946         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 947         os.makedirs(test_tmpdir)
 948         # The waf uninstall code removes empty directories all the way
 949         # up the tree.  Creating a file in test_tmpdir stops it from
 950         # being removed.
 951         run_cmd("touch %s" % os.path.join(test_tmpdir,
 952                                           ".directory-is-not-empty"), show=True)
 953         run_cmd("stat %s" % test_tmpdir, show=True)
 954         run_cmd("stat %s" % testbase, show=True)
 955         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 956     except Exception:
 957         cleanup()
 958         raise
 959
 960     try:
 961         try:
 962             if options.rebase is not None:
 963                 rebase_tree(options.rebase, rebase_branch=options.branch)
 964         except Exception:
 965             cleanup_list.append(gitroot + "/autobuild.pid")
 966             cleanup()
 967             elapsed_time = time.time() - start_time
 968             email_failure(-1, 'rebase', 'rebase', 'rebase',
 969                           'rebase on %s failed' % options.branch,
 970                           elapsed_time, log_base=options.log_base)
 971             sys.exit(1)
 972         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 973         if options.tail:
 974             blist.start_tail()
 975         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 976         if status != 0 or errstr != "retry":
 977             break
 978         cleanup()
 979     except Exception:
 980         cleanup()
 981         raise
 982
 983 cleanup_list.append(gitroot + "/autobuild.pid")
 984
 985 do_print(errstr)
 986
 987 blist.kill_kids()
 988 if options.tail:
 989     do_print("waiting for tail to flush")
 990     time.sleep(1)
 991
 992 elapsed_time = time.time() - start_time
 993 if status == 0:
 994     if options.passcmd is not None:
 995         do_print("Running passcmd: %s" % options.passcmd)
 996         run_cmd(options.passcmd, dir=test_master)
 997     if options.pushto is not None:
 998         push_to(options.pushto, push_branch=options.branch)
 999     if options.keeplogs or options.attach_logs:
1000         blist.tarlogs("logs.tar.gz")
1001         do_print("Logs in logs.tar.gz")
1002     if options.always_email:
1003         email_success(elapsed_time, log_base=options.log_base)
1004     blist.remove_logs()
1005     cleanup()
1006     do_print(errstr)
1007     sys.exit(0)
1008
1009 # something failed, gather a tar of the logs
1010 blist.tarlogs("logs.tar.gz")
1011
1012 if options.email is not None:
1013     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1014                   elapsed_time, log_base=options.log_base)
1015 else:
1016     elapsed_minutes = elapsed_time / 60.0
1017     print('''
1018
1019 ####################################################################
1020
1021 AUTOBUILD FAILURE
1022
1023 Your autobuild[%s] on %s failed after %.1f minutes
1024 when trying to test %s with the following error:
1025
1026    %s
1027
1028 the autobuild has been abandoned. Please fix the error and resubmit.
1029
1030 ####################################################################
1031
1032 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1033
1034 cleanup()
1035 do_print(errstr)
1036 do_print("Logs in logs.tar.gz")
1037 sys.exit(status)