script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-py3": ".",
  40     "samba-nt4": ".",
  41     "samba-nt4-py3": ".",
  42     "samba-fileserver": ".",
  43     "samba-xc": ".",
  44     "samba-o3": ".",
  45     "samba-ctdb": ".",
  46     "samba-libs": ".",
  47     "samba-libs-py3": ".",
  48     "samba-static": ".",
  49     "samba-none-env": ".",
  50     "samba-ad-dc": ".",
  51     "samba-ad-dc-py3": ".",
  52     "samba-ad-dc-2": ".",
  53     "samba-ad-dc-2-py3": ".",
  54     "samba-systemkrb5": ".",
  55     "samba-nopython": ".",
  56     "samba-buildpy3-only": ".",
  57     "samba-purepy3-none-env": ".",
  58     "samba-purepy3-ad-dc-2": ".",
  59     "samba-purepy3-ad-dc": ".",
  60     "ldb": "lib/ldb",
  61     "tdb": "lib/tdb",
  62     "talloc": "lib/talloc",
  63     "replace": "lib/replace",
  64     "tevent": "lib/tevent",
  65     "pidl": "pidl"
  66 }
  67
  68 defaulttasks = builddirs.keys()
  69
  70 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  71     defaulttasks.remove("samba-o3")
  72
  73 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  74 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  75
  76 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  77 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  78 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  79 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  80 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  81 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  82 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  83
  84 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  85     extra_python = ""
  86 else:
  87     extra_python = "--extra-python=/usr/bin/python3"
  88
  89 tasks = {
  90     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  91                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  92                ("make", "make all", "text/plain"),
  93                ("install", "make install", "text/plain"),
  94                ("test", "make autotest", "text/plain"),
  95                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  96                ("clean", "make clean", "text/plain")],
  97
  98     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
  99     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 100                 ("make", "make -j", "text/plain"),
 101                 ("test", "make test FAIL_IMMEDIATELY=1 "
 102                  "TESTS='${PY3_ONLY}"
 103                  "--exclude-env=none "
 104                  "--exclude-env=nt4_dc "
 105                  "--exclude-env=nt4_member "
 106                  "--exclude-env=ad_dc "
 107                  "--exclude-env=ad_dc_no_nss "
 108                  "--exclude-env=fl2003dc "
 109                  "--exclude-env=fl2008r2dc "
 110                  "--exclude-env=ad_member "
 111                  "--exclude-env=ad_member_idmap_rid "
 112                  "--exclude-env=ad_member_idmap_ad "
 113                  "--exclude-env=chgdcpass "
 114                  "--exclude-env=vampire_2000_dc "
 115                  "--exclude-env=fl2000dc "
 116                  "--exclude-env=fileserver "
 117                  "--exclude-env=backupfromdc "
 118                  "--exclude-env=restoredc "
 119                  "--exclude-env=renamedc "
 120                  "--exclude-env=offlinebackupdc "
 121                  "--exclude-env=labdc "
 122                  "'",
 123                  "text/plain"),
 124                 ("install", "make install", "text/plain"),
 125                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 126                 ("clean", "make clean", "text/plain")],
 127
 128     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 129     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 130                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 131                     ("make", "make -j", "text/plain"),
 132                     ("test", "make test FAIL_IMMEDIATELY=1 "
 133                      "TESTS='${PY3_ONLY}"
 134                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 135                     ("install", "make install", "text/plain"),
 136                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 137                     ("clean", "make clean", "text/plain")],
 138
 139     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 140     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 141                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 142                            ("make", "make -j", "text/plain"),
 143                            ("test", "make test FAIL_IMMEDIATELY=1 "
 144                             "TESTS='${PY3_ONLY}"
 145                             "--include-env=fileserver'", "text/plain"),
 146                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 147
 148     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 149     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 150                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 151                       ("make", "make -j", "text/plain"),
 152                       ("test", "make test FAIL_IMMEDIATELY=1 "
 153                        "TESTS='${PY3_ONLY}"
 154                        "--include-env=ad_dc "
 155                        "--include-env=fl2003dc "
 156                        "--include-env=fl2008r2dc "
 157                        "--include-env=ad_member "
 158                        "--include-env=ad_member_idmap_rid "
 159                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 160                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 161
 162     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 163     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 164                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 165                         ("make", "make -j", "text/plain"),
 166                         ("test", "make test FAIL_IMMEDIATELY=1 "
 167                          "TESTS='${PY3_ONLY}"
 168                          "--include-env=chgdcpass "
 169                          "--include-env=vampire_2000_dc "
 170                          "--include-env=fl2000dc "
 171                          "--include-env=ad_dc_no_nss "
 172                          "--include-env=backupfromdc "
 173                          "--include-env=restoredc "
 174                          "--include-env=renamedc "
 175                          "--include-env=offlinebackupdc "
 176                          "--include-env=labdc "
 177                          "'",
 178                          "text/plain"),
 179                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 180
 181     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 182                           ("make", "make -j", "text/plain"),
 183                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 184
 185     # Test cross-compile infrastructure
 186     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 187                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 188                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 189                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 190                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 191                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 192                    ("compare-results", "script/compare_cc_results.py "
 193                     "./bin/c4che/default{} "
 194                     "./bin-xe/c4che/default{} "
 195                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 196
 197     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 198     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 199                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 200                    ("make", "make -j", "text/plain"),
 201                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 202                     "TESTS='${PY3_ONLY}"
 203                     "--include-env=ad_dc'", "text/plain"),
 204                    ("install", "make install", "text/plain"),
 205                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 206                    ("clean", "make clean", "text/plain")],
 207
 208     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 209
 210                      # make sure we have tdb around:
 211                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 212                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 213                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 214
 215
 216                      # build samba with cluster support (also building ctdb):
 217                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 218                      ("samba-make", "make", "text/plain"),
 219                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 220                      ("samba-install", "make install", "text/plain"),
 221                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 222
 223                      # clean up:
 224                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 225                      ("clean", "make clean", "text/plain"),
 226                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 227
 228     "samba-libs": [
 229                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 230                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 231                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 232                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 233
 234                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 235                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 236                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 237
 238                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 239                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 240                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 241
 242                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 243                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 244                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 245
 246                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 247                       ("nondevel-make", "make -j", "text/plain"),
 248                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 249                       ("nondevel-install", "make install", "text/plain"),
 250                       ("nondevel-dist", "make dist", "text/plain"),
 251
 252                       # retry with all modules shared
 253                       ("allshared-distclean", "make distclean", "text/plain"),
 254                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 255                       ("allshared-make", "make -j", "text/plain")],
 256
 257     "samba-none-env": [
 258                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 259                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 260                       ("make", "make -j", "text/plain"),
 261                       ("test", "make test "
 262                        "FAIL_IMMEDIATELY=1 "
 263                        "TESTS='${PY3_ONLY}"
 264                        "--include-env=none'",
 265                        "text/plain")],
 266
 267     "samba-static": [
 268                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 269                       # build with all modules static
 270                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 271                       ("allstatic-make", "make -j", "text/plain"),
 272                       ("allstatic-test", "make test "
 273                        "FAIL_IMMEDIATELY=1 "
 274                        "TESTS='samba3.smb2.create.*nt4_dc'",
 275                        "text/plain"),
 276
 277                       # retry without any required modules
 278                       ("none-distclean", "make distclean", "text/plain"),
 279                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 280                       ("none-make", "make -j", "text/plain"),
 281
 282                       # retry with nonshared smbd and smbtorture
 283                       ("nonshared-distclean", "make distclean", "text/plain"),
 284                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 285                       ("nonshared-make", "make -j", "text/plain")],
 286
 287     "samba-systemkrb5": [
 288                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 289                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 290                       ("make", "make -j", "text/plain"),
 291                       # we currently cannot run a full make test, a limited list of tests could be run
 292                       # via "make test TESTS=sometests"
 293                       ("test", "make test FAIL_IMMEDIATELY=1 "
 294                        "TESTS='${PY3_ONLY}"
 295                        "--include-env=ktest'", "text/plain"),
 296                       ("install", "make install", "text/plain"),
 297                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 298                       ("clean", "make clean", "text/plain")
 299                       ],
 300
 301     # Test Samba without python still builds.  When this test fails
 302     # due to more use of Python, the expectations is that the newly
 303     # failing part of the code should be disabled when
 304     # --disable-python is set (rather than major work being done to
 305     # support this environment).  The target here is for vendors
 306     # shipping a minimal smbd.
 307     "samba-nopython": [
 308                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 309                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 310                       ("make", "make -j", "text/plain"),
 311                       ("install", "make install", "text/plain"),
 312                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 313                       ("clean", "make clean", "text/plain"),
 314
 315                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 316                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 317                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 318
 319                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 320                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 321                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 322
 323                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 324                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 325                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 326
 327                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 328                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 329                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 330
 331                       # retry against installed library packages
 332                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 333                       ("libs-make", "make -j", "text/plain"),
 334                       ("libs-install", "make install", "text/plain"),
 335                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 336                       ("libs-clean", "make clean", "text/plain")
 337                       ],
 338
 339
 340
 341     "ldb": [
 342               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 343               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 344               ("make", "make", "text/plain"),
 345               ("install", "make install", "text/plain"),
 346               ("test", "make test", "text/plain"),
 347               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 348               ("make-no-lmdb", "make", "text/plain"),
 349               ("install-no-lmdb", "make install", "text/plain"),
 350               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 351               ("distcheck", "make distcheck", "text/plain"),
 352               ("clean", "make clean", "text/plain")],
 353
 354     "tdb": [
 355               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 356               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 357               ("make", "make", "text/plain"),
 358               ("install", "make install", "text/plain"),
 359               ("test", "make test", "text/plain"),
 360               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 361               ("distcheck", "make distcheck", "text/plain"),
 362               ("clean", "make clean", "text/plain")],
 363
 364     "talloc": [
 365                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 366                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 367                  ("make", "make", "text/plain"),
 368                  ("install", "make install", "text/plain"),
 369                  ("test", "make test", "text/plain"),
 370                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 371                  ("distcheck", "make distcheck", "text/plain"),
 372                  ("clean", "make clean", "text/plain")],
 373
 374     "replace": [
 375                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 376                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 377                   ("make", "make", "text/plain"),
 378                   ("install", "make install", "text/plain"),
 379                   ("test", "make test", "text/plain"),
 380                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 381                   ("distcheck", "make distcheck", "text/plain"),
 382                   ("clean", "make clean", "text/plain")],
 383
 384     "tevent": [
 385                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 386                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 387                  ("make", "make", "text/plain"),
 388                  ("install", "make install", "text/plain"),
 389                  ("test", "make test", "text/plain"),
 390                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 391                  ("distcheck", "make distcheck", "text/plain"),
 392                  ("clean", "make clean", "text/plain")],
 393
 394     "pidl": [
 395         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 396         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 397         ("touch", "touch *.yp", "text/plain"),
 398         ("make", "make", "text/plain"),
 399         ("test", "make test", "text/plain"),
 400         ("install", "make install", "text/plain"),
 401         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 402         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 403         ("clean", "make clean", "text/plain")],
 404
 405     "samba-buildpy3-only": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 406                    ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab " + samba_configure_params, "text/plain"),
 407                    ("make", "PYTHON='python3' make -j", "text/plain"),
 408                    ("install", "PYTHON='python3' make install", "text/plain"),
 409                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 410                    ("clean", "PYTHON='python3' make clean", "text/plain")],
 411
 412     "samba-purepy3-none-env": [
 413                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 414                       ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 415                       ("make", "PYTHON='python3' make -j", "text/plain"),
 416                       ("test", "PYTHON='python3' make test "
 417                        "FAIL_IMMEDIATELY=1 "
 418                        "TESTS='${PY3_ONLY}"
 419                        "--include-env=none'",
 420                        "text/plain")],
 421     "samba-purepy3-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 422                         ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 423                         ("make", "PYTHON='python3' make -j", "text/plain"),
 424                         ("test", "PYTHON='python3' make test FAIL_IMMEDIATELY=1 "
 425                          "TESTS='${PY3_ONLY}"
 426                          "--include-env=chgdcpass "
 427                          "--include-env=vampire_2000_dc "
 428                          "--include-env=fl2000dc "
 429                          "--include-env=ad_dc_no_nss "
 430                          "--include-env=backupfromdc "
 431                          "--include-env=restoredc "
 432                          "--include-env=renamedc "
 433                          "--include-env=offlinebackupdc "
 434                          "--include-env=labdc "
 435                          "'",
 436                          "text/plain"),
 437                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 438     "samba-purepy3-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 439                       ("configure", "PYTHON='python3' ./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 440                       ("make", "PYTHON='python3' make -j", "text/plain"),
 441                       ("test", "PYTHON='python3' make test FAIL_IMMEDIATELY=1 "
 442                        "TESTS='${PY3_ONLY}"
 443                        "--include-env=ad_dc "
 444                        "--include-env=fl2003dc "
 445                        "--include-env=fl2008r2dc "
 446                        "--include-env=ad_member "
 447                        "--include-env=ad_member_idmap_rid "
 448                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 449                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 450
 451
 452
 453     # these are useful for debugging autobuild
 454     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 455     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 456
 457
 458 }
 459
 460
 461 def do_print(msg):
 462     print("%s" % msg)
 463     sys.stdout.flush()
 464     sys.stderr.flush()
 465
 466
 467 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 468     if show is None:
 469         show = options.verbose
 470     if show:
 471         do_print("Running: '%s' in '%s'" % (cmd, dir))
 472     if output:
 473         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 474     elif checkfail:
 475         return check_call(cmd, shell=True, cwd=dir)
 476     else:
 477         return call(cmd, shell=True, cwd=dir)
 478
 479
 480 class builder(object):
 481     '''handle build of one directory'''
 482
 483     def __init__(self, name, sequence, cp=True, py3=False):
 484         self.name = name
 485         self.py3 = py3
 486         if name in builddirs:
 487             self.dir = builddirs[name]
 488         else:
 489             self.dir = "."
 490
 491         self.tag = self.name.replace('/', '_')
 492         self.sequence = sequence
 493         self.next = 0
 494         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 495         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 496         if options.verbose:
 497             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 498             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 499         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 500         self.stdout = open(self.stdout_path, 'w')
 501         self.stderr = open(self.stderr_path, 'w')
 502         self.stdin  = open("/dev/null", 'r')
 503         self.sdir = "%s/%s" % (testbase, self.tag)
 504         self.prefix = "%s/%s" % (test_prefix, self.tag)
 505         run_cmd("rm -rf %s" % self.sdir)
 506         run_cmd("rm -rf %s" % self.prefix)
 507         if cp:
 508             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 509         else:
 510             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 511         self.start_next()
 512
 513     def start_next(self):
 514         if self.next == len(self.sequence):
 515             if not options.nocleanup:
 516                 run_cmd("rm -rf %s" % self.sdir)
 517                 run_cmd("rm -rf %s" % self.prefix)
 518             do_print('%s: Completed OK' % self.name)
 519             self.done = True
 520             return
 521         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 522         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 523         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 524         if self.py3:
 525             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 526             # The trailing space is important
 527             self.cmd = self.cmd.replace("${PY3_ONLY}", "python3 ")
 528         else:
 529             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 530             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 531         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 532         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 533 #        if self.output_mime_type == "text/x-subunit":
 534 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 535         cwd = os.getcwd()
 536         os.chdir("%s/%s" % (self.sdir, self.dir))
 537         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 538         self.proc = Popen(self.cmd, shell=True,
 539                           close_fds=True,
 540                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 541         os.chdir(cwd)
 542         self.next += 1
 543
 544
 545 class buildlist(object):
 546     '''handle build of multiple directories'''
 547
 548     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 549         global tasks
 550         self.tlist = []
 551         self.tail_proc = None
 552         self.retry = None
 553         if tasknames == []:
 554             if options.restrict_tests:
 555                 tasknames = ["samba-test-only"]
 556             else:
 557                 tasknames = defaulttasks
 558         else:
 559             # If we are only running one test,
 560             # do not sleep randomly to wait for it to start
 561             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 562
 563         for n in tasknames:
 564             if n not in tasks and n.endswith("-py3"):
 565                 b = builder(n,
 566                             tasks[n[:-4]],
 567                             cp=n is not "pidl",
 568                             py3=True)
 569             else:
 570                 b = builder(n, tasks[n], cp=n is not "pidl")
 571             self.tlist.append(b)
 572         if options.retry:
 573             rebase_remote = "rebaseon"
 574             retry_task = [("retry",
 575                             '''set -e
 576                             git remote add -t %s %s %s
 577                             git fetch %s
 578                             while :; do
 579                               sleep 60
 580                               git describe %s/%s > old_remote_branch.desc
 581                               git fetch %s
 582                               git describe %s/%s > remote_branch.desc
 583                               diff old_remote_branch.desc remote_branch.desc
 584                             done
 585                            ''' % (
 586                                 rebase_branch, rebase_remote, rebase_url,
 587                                rebase_remote,
 588                                rebase_remote, rebase_branch,
 589                                rebase_remote,
 590                                rebase_remote, rebase_branch
 591                             ),
 592                             "test/plain")]
 593
 594             self.retry = builder('retry', retry_task, cp=False)
 595             self.need_retry = False
 596
 597     def kill_kids(self):
 598         if self.tail_proc is not None:
 599             self.tail_proc.terminate()
 600             self.tail_proc.wait()
 601             self.tail_proc = None
 602         if self.retry is not None:
 603             self.retry.proc.terminate()
 604             self.retry.proc.wait()
 605             self.retry = None
 606         for b in self.tlist:
 607             if b.proc is not None:
 608                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 609                 b.proc.terminate()
 610                 b.proc.wait()
 611                 b.proc = None
 612
 613     def wait_one(self):
 614         while True:
 615             none_running = True
 616             for b in self.tlist:
 617                 if b.proc is None:
 618                     continue
 619                 none_running = False
 620                 b.status = b.proc.poll()
 621                 if b.status is None:
 622                     continue
 623                 b.proc = None
 624                 return b
 625             if options.retry:
 626                 ret = self.retry.proc.poll()
 627                 if ret is not None:
 628                     self.need_retry = True
 629                     self.retry = None
 630                     return None
 631             if none_running:
 632                 return None
 633             time.sleep(0.1)
 634
 635     def run(self):
 636         while True:
 637             b = self.wait_one()
 638             if options.retry and self.need_retry:
 639                 self.kill_kids()
 640                 do_print("retry needed")
 641                 return (0, None, None, None, "retry")
 642             if b is None:
 643                 break
 644             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 645                 self.kill_kids()
 646                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 647             b.start_next()
 648         self.kill_kids()
 649         return (0, None, None, None, "All OK")
 650
 651     def write_system_info(self):
 652         filename = 'system-info.txt'
 653         f = open(filename, 'w')
 654         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 655                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 656             print('### %s' % cmd, file=f)
 657             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 658             print(file=f)
 659         f.close()
 660         return filename
 661
 662     def tarlogs(self, fname):
 663         tar = tarfile.open(fname, "w:gz")
 664         for b in self.tlist:
 665             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 666             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 667         if os.path.exists("autobuild.log"):
 668             tar.add("autobuild.log")
 669         sys_info = self.write_system_info()
 670         tar.add(sys_info)
 671         tar.close()
 672
 673     def remove_logs(self):
 674         for b in self.tlist:
 675             os.unlink(b.stdout_path)
 676             os.unlink(b.stderr_path)
 677
 678     def start_tail(self):
 679         cmd = ["tail", "-f"]
 680         for b in self.tlist:
 681             cmd.append(b.stdout_path)
 682             cmd.append(b.stderr_path)
 683         self.tail_proc = Popen(cmd, close_fds=True)
 684
 685
 686 def cleanup():
 687     if options.nocleanup:
 688         return
 689     run_cmd("stat %s || true" % test_tmpdir, show=True)
 690     run_cmd("stat %s" % testbase, show=True)
 691     do_print("Cleaning up %r" % cleanup_list)
 692     for d in cleanup_list:
 693         run_cmd("rm -rf %s" % d)
 694
 695
 696 def find_git_root():
 697     '''get to the top of the git repo'''
 698     p = os.getcwd()
 699     while p != '/':
 700         if os.path.isdir(os.path.join(p, ".git")):
 701             return p
 702         p = os.path.abspath(os.path.join(p, '..'))
 703     return None
 704
 705
 706 def daemonize(logfile):
 707     pid = os.fork()
 708     if pid == 0:  # Parent
 709         os.setsid()
 710         pid = os.fork()
 711         if pid != 0:  # Actual daemon
 712             os._exit(0)
 713     else:  # Grandparent
 714         os._exit(0)
 715
 716     import resource      # Resource usage information.
 717     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 718     if maxfd == resource.RLIM_INFINITY:
 719         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 720     for fd in range(0, maxfd):
 721         try:
 722             os.close(fd)
 723         except OSError:
 724             pass
 725     os.open(logfile, os.O_RDWR | os.O_CREAT)
 726     os.dup2(0, 1)
 727     os.dup2(0, 2)
 728
 729
 730 def write_pidfile(fname):
 731     '''write a pid file, cleanup on exit'''
 732     f = open(fname, mode='w')
 733     f.write("%u\n" % os.getpid())
 734     f.close()
 735
 736
 737 def rebase_tree(rebase_url, rebase_branch="master"):
 738     rebase_remote = "rebaseon"
 739     do_print("Rebasing on %s" % rebase_url)
 740     run_cmd("git describe HEAD", show=True, dir=test_master)
 741     run_cmd("git remote add -t %s %s %s" %
 742             (rebase_branch, rebase_remote, rebase_url),
 743             show=True, dir=test_master)
 744     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 745     if options.fix_whitespace:
 746         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 747                 (rebase_remote, rebase_branch),
 748                 show=True, dir=test_master)
 749     else:
 750         run_cmd("git rebase --force-rebase %s/%s" %
 751                 (rebase_remote, rebase_branch),
 752                 show=True, dir=test_master)
 753     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 754                    (rebase_remote, rebase_branch),
 755                    dir=test_master, output=True)
 756     if diff == '':
 757         do_print("No differences between HEAD and %s/%s - exiting" %
 758                  (rebase_remote, rebase_branch))
 759         sys.exit(0)
 760     run_cmd("git describe %s/%s" %
 761             (rebase_remote, rebase_branch),
 762             show=True, dir=test_master)
 763     run_cmd("git describe HEAD", show=True, dir=test_master)
 764     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 765             (rebase_remote, rebase_branch),
 766             show=True, dir=test_master)
 767
 768
 769 def push_to(push_url, push_branch="master"):
 770     push_remote = "pushto"
 771     do_print("Pushing to %s" % push_url)
 772     if options.mark:
 773         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 774         run_cmd("git commit --amend -c HEAD", dir=test_master)
 775         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 776         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 777     run_cmd("git remote add -t %s %s %s" %
 778             (push_branch, push_remote, push_url),
 779             show=True, dir=test_master)
 780     run_cmd("git push %s +HEAD:%s" %
 781             (push_remote, push_branch),
 782             show=True, dir=test_master)
 783
 784
 785 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 786
 787 gitroot = find_git_root()
 788 if gitroot is None:
 789     raise Exception("Failed to find git root")
 790
 791 parser = OptionParser()
 792 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 793 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 794 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 795 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 796                   default=def_testbase)
 797 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 798 parser.add_option("", "--verbose", help="show all commands as they are run",
 799                   default=False, action="store_true")
 800 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 801                   default=None, type='str')
 802 parser.add_option("", "--pushto", help="push to a git url on success",
 803                   default=None, type='str')
 804 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 805                   default=False, action="store_true")
 806 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 807                   default=False, action="store_true")
 808 parser.add_option("", "--retry", help="automatically retry if master changes",
 809                   default=False, action="store_true")
 810 parser.add_option("", "--email", help="send email to the given address on failure",
 811                   type='str', default=None)
 812 parser.add_option("", "--email-from", help="send email from the given address",
 813                   type='str', default="autobuild@samba.org")
 814 parser.add_option("", "--email-server", help="send email via the given server",
 815                   type='str', default='localhost')
 816 parser.add_option("", "--always-email", help="always send email, even on success",
 817                   action="store_true")
 818 parser.add_option("", "--daemon", help="daemonize after initial setup",
 819                   action="store_true")
 820 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 821                   default="master", type='str')
 822 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 823                   default=gitroot, type='str')
 824 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 825                   default=False, action="store_true")
 826 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 827                   default='')
 828
 829
 830 def send_email(subject, text, log_tar):
 831     if options.email is None:
 832         do_print("not sending email because the recipient is not set")
 833         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 834                  (subject, text))
 835         return
 836     outer = MIMEMultipart()
 837     outer['Subject'] = subject
 838     outer['To'] = options.email
 839     outer['From'] = options.email_from
 840     outer['Date'] = email.utils.formatdate(localtime=True)
 841     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 842     outer.attach(MIMEText(text, 'plain'))
 843     if options.attach_logs:
 844         fp = open(log_tar, 'rb')
 845         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 846         fp.close()
 847         # Set the filename parameter
 848         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 849         outer.attach(msg)
 850     content = outer.as_string()
 851     s = smtplib.SMTP(options.email_server)
 852     s.sendmail(options.email_from, [options.email], content)
 853     s.set_debuglevel(1)
 854     s.quit()
 855
 856
 857 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 858                   elapsed_time, log_base=None, add_log_tail=True):
 859     '''send an email to options.email about the failure'''
 860     elapsed_minutes = elapsed_time / 60.0
 861     if log_base is None:
 862         log_base = gitroot
 863     text = '''
 864 Dear Developer,
 865
 866 Your autobuild on %s failed after %.1f minutes
 867 when trying to test %s with the following error:
 868
 869    %s
 870
 871 the autobuild has been abandoned. Please fix the error and resubmit.
 872
 873 A summary of the autobuild process is here:
 874
 875   %s/autobuild.log
 876 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 877
 878     if options.restrict_tests:
 879         text += """
 880 The build was restricted to tests matching %s\n""" % options.restrict_tests
 881
 882     if failed_task != 'rebase':
 883         text += '''
 884 You can see logs of the failed task here:
 885
 886   %s/%s.stdout
 887   %s/%s.stderr
 888
 889 or you can get full logs of all tasks in this job here:
 890
 891   %s/logs.tar.gz
 892
 893 The top commit for the tree that was built was:
 894
 895 %s
 896
 897 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 898
 899     if add_log_tail:
 900         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 901         lines = f.readlines()
 902         log_tail = "".join(lines[-50:])
 903         num_lines = len(lines)
 904         if num_lines < 50:
 905             # Also include stderr (compile failures) if < 50 lines of stdout
 906             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 907             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 908
 909         text += '''
 910 The last 50 lines of log messages:
 911
 912 %s
 913     ''' % log_tail
 914         f.close()
 915
 916     logs = os.path.join(gitroot, 'logs.tar.gz')
 917     send_email('autobuild[%s] failure on %s for task %s during %s'
 918                % (options.branch, platform.node(), failed_task, failed_stage),
 919                text, logs)
 920
 921
 922 def email_success(elapsed_time, log_base=None):
 923     '''send an email to options.email about a successful build'''
 924     if log_base is None:
 925         log_base = gitroot
 926     text = '''
 927 Dear Developer,
 928
 929 Your autobuild on %s has succeeded after %.1f minutes.
 930
 931 ''' % (platform.node(), elapsed_time / 60.)
 932
 933     if options.restrict_tests:
 934         text += """
 935 The build was restricted to tests matching %s\n""" % options.restrict_tests
 936
 937     if options.keeplogs:
 938         text += '''
 939
 940 you can get full logs of all tasks in this job here:
 941
 942   %s/logs.tar.gz
 943
 944 ''' % log_base
 945
 946     text += '''
 947 The top commit for the tree that was built was:
 948
 949 %s
 950 ''' % top_commit_msg
 951
 952     logs = os.path.join(gitroot, 'logs.tar.gz')
 953     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 954                text, logs)
 955
 956
 957 (options, args) = parser.parse_args()
 958
 959 if options.retry:
 960     if options.rebase is None:
 961         raise Exception('You can only use --retry if you also rebase')
 962
 963 testbase = "%s/b%u" % (options.testbase, os.getpid())
 964 test_master = "%s/master" % testbase
 965 test_prefix = "%s/prefix" % testbase
 966 test_tmpdir = "%s/tmp" % testbase
 967 os.environ['TMPDIR'] = test_tmpdir
 968
 969 # get the top commit message, for emails
 970 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 971
 972 try:
 973     os.makedirs(testbase)
 974 except Exception as reason:
 975     raise Exception("Unable to create %s : %s" % (testbase, reason))
 976 cleanup_list.append(testbase)
 977
 978 if options.daemon:
 979     logfile = os.path.join(testbase, "log")
 980     do_print("Forking into the background, writing progress to %s" % logfile)
 981     daemonize(logfile)
 982
 983 write_pidfile(gitroot + "/autobuild.pid")
 984
 985 start_time = time.time()
 986
 987 while True:
 988     try:
 989         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 990         os.makedirs(test_tmpdir)
 991         # The waf uninstall code removes empty directories all the way
 992         # up the tree.  Creating a file in test_tmpdir stops it from
 993         # being removed.
 994         run_cmd("touch %s" % os.path.join(test_tmpdir,
 995                                           ".directory-is-not-empty"), show=True)
 996         run_cmd("stat %s" % test_tmpdir, show=True)
 997         run_cmd("stat %s" % testbase, show=True)
 998         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 999     except Exception:
1000         cleanup()
1001         raise
1002
1003     try:
1004         try:
1005             if options.rebase is not None:
1006                 rebase_tree(options.rebase, rebase_branch=options.branch)
1007         except Exception:
1008             cleanup_list.append(gitroot + "/autobuild.pid")
1009             cleanup()
1010             elapsed_time = time.time() - start_time
1011             email_failure(-1, 'rebase', 'rebase', 'rebase',
1012                           'rebase on %s failed' % options.branch,
1013                           elapsed_time, log_base=options.log_base)
1014             sys.exit(1)
1015         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1016         if options.tail:
1017             blist.start_tail()
1018         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1019         if status != 0 or errstr != "retry":
1020             break
1021         cleanup()
1022     except Exception:
1023         cleanup()
1024         raise
1025
1026 cleanup_list.append(gitroot + "/autobuild.pid")
1027
1028 do_print(errstr)
1029
1030 blist.kill_kids()
1031 if options.tail:
1032     do_print("waiting for tail to flush")
1033     time.sleep(1)
1034
1035 elapsed_time = time.time() - start_time
1036 if status == 0:
1037     if options.passcmd is not None:
1038         do_print("Running passcmd: %s" % options.passcmd)
1039         run_cmd(options.passcmd, dir=test_master)
1040     if options.pushto is not None:
1041         push_to(options.pushto, push_branch=options.branch)
1042     if options.keeplogs or options.attach_logs:
1043         blist.tarlogs("logs.tar.gz")
1044         do_print("Logs in logs.tar.gz")
1045     if options.always_email:
1046         email_success(elapsed_time, log_base=options.log_base)
1047     blist.remove_logs()
1048     cleanup()
1049     do_print(errstr)
1050     sys.exit(0)
1051
1052 # something failed, gather a tar of the logs
1053 blist.tarlogs("logs.tar.gz")
1054
1055 if options.email is not None:
1056     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1057                   elapsed_time, log_base=options.log_base)
1058 else:
1059     elapsed_minutes = elapsed_time / 60.0
1060     print('''
1061
1062 ####################################################################
1063
1064 AUTOBUILD FAILURE
1065
1066 Your autobuild[%s] on %s failed after %.1f minutes
1067 when trying to test %s with the following error:
1068
1069    %s
1070
1071 the autobuild has been abandoned. Please fix the error and resubmit.
1072
1073 ####################################################################
1074
1075 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1076
1077 cleanup()
1078 do_print(errstr)
1079 do_print("Logs in logs.tar.gz")
1080 sys.exit(status)