script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-nt4": ".",
  40     "samba-fileserver": ".",
  41     "samba-ad-member": ".",
  42     "samba-xc": ".",
  43     "samba-o3": ".",
  44     "samba-ctdb": ".",
  45     "samba-libs": ".",
  46     "samba-static": ".",
  47     "samba-none-env": ".",
  48     "samba-ad-dc-1": ".",
  49     "samba-ad-dc-2": ".",
  50     "samba-ad-dc-3": ".",
  51     "samba-ad-dc-4": ".",
  52     "samba-ad-dc-5": ".",
  53     "samba-ad-dc-6": ".",
  54     "samba-ad-dc-ntvfs": ".",
  55     "samba-ad-dc-backup": ".",
  56     "samba-systemkrb5": ".",
  57     "samba-nopython": ".",
  58     "samba-nopython-py2": ".",
  59     "samba-schemaupgrade": ".",
  60     "ldb": "lib/ldb",
  61     "tdb": "lib/tdb",
  62     "talloc": "lib/talloc",
  63     "replace": "lib/replace",
  64     "tevent": "lib/tevent",
  65     "pidl": "pidl"
  66 }
  67
  68 defaulttasks = builddirs.keys()
  69
  70 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  71     defaulttasks.remove("samba-o3")
  72
  73 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  74 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
  75
  76 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  77 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  78 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  79 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  80 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
  81 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  82 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
  83
  84 tasks = {
  85     "ctdb": [("random-sleep", "../script/random-sleep.sh 300 900", "text/plain"),
  86                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  87                ("make", "make all", "text/plain"),
  88                ("install", "make install", "text/plain"),
  89                ("test", "make autotest", "text/plain"),
  90                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  91                ("clean", "make clean", "text/plain")],
  92
  93     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
  94     "samba": [
  95                 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
  96                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  97                 ("make", "make -j", "text/plain"),
  98                 ("test", "make test FAIL_IMMEDIATELY=1 "
  99                  "TESTS='--exclude-env=none "
 100                  "--exclude-env=nt4_dc "
 101                  "--exclude-env=nt4_dc_schannel "
 102                  "--exclude-env=nt4_member "
 103                  "--exclude-env=ad_dc "
 104                  "--exclude-env=ad_dc_backup "
 105                  "--exclude-env=ad_dc_ntvfs "
 106                  "--exclude-env=ad_dc_default "
 107                  "--exclude-env=ad_dc_slowtests "
 108                  "--exclude-env=ad_dc_no_nss "
 109                  "--exclude-env=ad_dc_no_ntlm "
 110                  "--exclude-env=fl2003dc "
 111                  "--exclude-env=fl2008dc "
 112                  "--exclude-env=fl2008r2dc "
 113                  "--exclude-env=ad_member "
 114                  "--exclude-env=ad_member_idmap_rid "
 115                  "--exclude-env=ad_member_idmap_ad "
 116                  "--exclude-env=ad_member_rfc2307 "
 117                  "--exclude-env=chgdcpass "
 118                  "--exclude-env=vampire_2000_dc "
 119                  "--exclude-env=fl2000dc "
 120                  "--exclude-env=fileserver "
 121                  "--exclude-env=maptoguest "
 122                  "--exclude-env=simpleserver "
 123                  "--exclude-env=backupfromdc "
 124                  "--exclude-env=restoredc "
 125                  "--exclude-env=renamedc "
 126                  "--exclude-env=offlinebackupdc "
 127                  "--exclude-env=labdc "
 128                  "--exclude-env=preforkrestartdc "
 129                  "--exclude-env=proclimitdc "
 130                  "--exclude-env=promoted_dc "
 131                  "--exclude-env=vampire_dc "
 132                  "--exclude-env=rodc "
 133                  "--exclude-env=ad_dc_default "
 134                  "--exclude-env=ad_dc_slowtests "
 135                  "--exclude-env=schema_pair_dc "
 136                  "--exclude-env=schema_dc "
 137                  "'",
 138                  "text/plain"),
 139                 ("install", "make install", "text/plain"),
 140                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 141                 ("clean", "make clean", "text/plain")],
 142
 143     "samba-nt4": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 144                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 145                     ("make", "make -j", "text/plain"),
 146                     ("test", "make test FAIL_IMMEDIATELY=1 "
 147                      "TESTS='"
 148                      "--include-env=nt4_dc "
 149                      "--include-env=nt4_dc_schannel "
 150                      "--include-env=nt4_member "
 151                      "'", "text/plain"),
 152                     ("install", "make install", "text/plain"),
 153                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 154                     ("clean", "make clean", "text/plain")],
 155
 156     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 157                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 158                            ("make", "make -j", "text/plain"),
 159                            ("test", "make test FAIL_IMMEDIATELY=1 "
 160                             "TESTS='"
 161                             "--include-env=fileserver "
 162                             "--include-env=maptoguest "
 163                             "--include-env=simpleserver "
 164                             "'", "text/plain"),
 165                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 166
 167     "samba-ad-member": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 168                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 169                         ("make", "make -j", "text/plain"),
 170                         ("test", "make test FAIL_IMMEDIATELY=1 "
 171                          "TESTS='"
 172                          "--include-env=ad_member "
 173                          "--include-env=ad_member_idmap_rid "
 174                          "--include-env=ad_member_idmap_ad "
 175                          "--include-env=ad_member_rfc2307 "
 176                          "'", "text/plain"),
 177                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 178
 179     "samba-ad-dc-1": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 180                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 181                       ("make", "make -j", "text/plain"),
 182                       ("test", "make test FAIL_IMMEDIATELY=1 "
 183                        "TESTS='--include-env=ad_dc "
 184                        "--include-env=ad_dc_no_nss "
 185                        "--include-env=ad_dc_no_ntlm "
 186                        "'", "text/plain"),
 187                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 188
 189     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 190                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 191                         ("make", "make -j", "text/plain"),
 192                         ("test", "make test FAIL_IMMEDIATELY=1 "
 193                          "TESTS='"
 194                          "--include-env=vampire_dc "
 195                          "--include-env=vampire_2000_dc "
 196                          "--include-env=rodc "
 197                          "'", "text/plain"),
 198                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 199
 200     "samba-ad-dc-3": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 201                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 202                         ("make", "make -j", "text/plain"),
 203                         ("test", "make test FAIL_IMMEDIATELY=1 "
 204                          "TESTS='"
 205                          "--include-env=promoted_dc "
 206                          "--include-env=chgdcpass "
 207                          "--include-env=preforkrestartdc "
 208                          "--include-env=proclimitdc "
 209                          "'", "text/plain"),
 210                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 211
 212     "samba-ad-dc-4": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 213                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 214                         ("make", "make -j", "text/plain"),
 215                         ("test", "make test FAIL_IMMEDIATELY=1 "
 216                          "TESTS='"
 217                          "--include-env=fl2000dc "
 218                          "--include-env=fl2003dc "
 219                          "--include-env=fl2008dc "
 220                          "--include-env=fl2008r2dc "
 221                          "'", "text/plain"),
 222                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 223
 224     "samba-ad-dc-5": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 225                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 226                         ("make", "make -j", "text/plain"),
 227                         ("test", "make test FAIL_IMMEDIATELY=1 "
 228                          "TESTS='"
 229                          "--include-env=ad_dc_default "
 230                          "'", "text/plain"),
 231                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 232
 233     "samba-ad-dc-6": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 234                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 235                         ("make", "make -j", "text/plain"),
 236                         ("test", "make test FAIL_IMMEDIATELY=1 "
 237                          "TESTS='"
 238                          "--include-env=ad_dc_slowtests "
 239                          "'", "text/plain"),
 240                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 241
 242
 243     "samba-schemaupgrade": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 244                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 245                         ("make", "make -j", "text/plain"),
 246                         ("test", "make test FAIL_IMMEDIATELY=1 "
 247                          "TESTS='"
 248                          "--include-env=schema_dc "
 249                          "--include-env=schema_pair_dc "
 250                          "'", "text/plain"),
 251                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 252
 253     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 254     # This is currently the longest task, so we don't randomly delay it.
 255     "samba-ad-dc-ntvfs": [
 256                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 257                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 258                       ("make", "make -j", "text/plain"),
 259                       ("test", "make test FAIL_IMMEDIATELY=1 "
 260                        "TESTS='"
 261                        "--include-env=ad_dc_ntvfs "
 262                        "'", "text/plain"),
 263                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 264
 265     # run the backup/restore testenvs separately as they're fairly standalone
 266     # (and CI seems to max out at ~8 different DCs running at once)
 267     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 268                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 269                         ("make", "make -j", "text/plain"),
 270                         ("test", "make test FAIL_IMMEDIATELY=1 "
 271                          "TESTS='--include-env=backupfromdc "
 272                          "--include-env=restoredc "
 273                          "--include-env=renamedc "
 274                          "--include-env=offlinebackupdc "
 275                          "--include-env=labdc "
 276                          "--include-env=ad_dc_backup "
 277                          "'", "text/plain"),
 278                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 279
 280     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 281                           ("make", "make -j", "text/plain"),
 282                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 283
 284     # Test cross-compile infrastructure
 285     "samba-xc": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 286                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 287                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 288                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 289                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 290                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 291                    ("compare-results", "script/compare_cc_results.py "
 292                     "./bin/c4che/default{} "
 293                     "./bin-xe/c4che/default{} "
 294                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 295
 296     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 297     "samba-o3": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 298                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 299                    ("make", "make -j", "text/plain"),
 300                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 301                     "TESTS='--include-env=ad_dc'", "text/plain"),
 302                    ("install", "make install", "text/plain"),
 303                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 304                    ("clean", "make clean", "text/plain")],
 305
 306     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 307
 308                      # make sure we have tdb around:
 309                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 310                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 311                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 312
 313
 314                      # build samba with cluster support (also building ctdb):
 315                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 316                      ("samba-make", "make", "text/plain"),
 317                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 318                      ("samba-install", "make install", "text/plain"),
 319                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 320
 321                      # clean up:
 322                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 323                      ("clean", "make clean", "text/plain"),
 324                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 325
 326     "samba-libs": [
 327                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 328                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 329                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 330                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 331
 332                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 333                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 334                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 335
 336                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 337                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 338                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 339
 340                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 341                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 342                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 343
 344                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 345                       ("nondevel-make", "make -j", "text/plain"),
 346                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 347                       ("nondevel-install", "make install", "text/plain"),
 348                       ("nondevel-dist", "make dist", "text/plain"),
 349
 350                       # retry with all modules shared
 351                       ("allshared-distclean", "make distclean", "text/plain"),
 352                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 353                       ("allshared-make", "make -j", "text/plain")],
 354
 355     "samba-none-env": [
 356                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 357                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 358                       ("make", "make -j", "text/plain"),
 359                       ("test", "make test "
 360                        "FAIL_IMMEDIATELY=1 "
 361                        "TESTS='--include-env=none'",
 362                        "text/plain")],
 363
 364     "samba-static": [
 365                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 366                       # build with all modules static
 367                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 368                       ("allstatic-make", "make -j", "text/plain"),
 369                       ("allstatic-test", "make test "
 370                        "FAIL_IMMEDIATELY=1 "
 371                        "TESTS='samba3.smb2.create.*nt4_dc'",
 372                        "text/plain"),
 373
 374                       # retry without any required modules
 375                       ("none-distclean", "make distclean", "text/plain"),
 376                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 377                       ("none-make", "make -j", "text/plain"),
 378
 379                       # retry with nonshared smbd and smbtorture
 380                       ("nonshared-distclean", "make distclean", "text/plain"),
 381                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 382                       ("nonshared-make", "make -j", "text/plain")],
 383
 384     "samba-systemkrb5": [
 385                       ("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 386                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 387                       ("make", "make -j", "text/plain"),
 388                       # we currently cannot run a full make test, a limited list of tests could be run
 389                       # via "make test TESTS=sometests"
 390                       ("test", "make test FAIL_IMMEDIATELY=1 "
 391                        "TESTS='--include-env=ktest'", "text/plain"),
 392                       ("install", "make install", "text/plain"),
 393                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 394                       ("clean", "make clean", "text/plain")
 395                       ],
 396
 397     # Test Samba without python still builds.  When this test fails
 398     # due to more use of Python, the expectations is that the newly
 399     # failing part of the code should be disabled when
 400     # --disable-python is set (rather than major work being done to
 401     # support this environment).  The target here is for vendors
 402     # shipping a minimal smbd.
 403     "samba-nopython": [
 404                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 405                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 406                       ("make", "make -j", "text/plain"),
 407                       ("install", "make install", "text/plain"),
 408                       ("test", "make test-nopython", "text/plain"),
 409                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 410                       ("clean", "make clean", "text/plain"),
 411
 412                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 413                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 414                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 415
 416                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 417                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 418                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 419
 420                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 421                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 422                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 423
 424                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 425                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 426                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 427
 428                       # retry against installed library packages
 429                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 430                       ("libs-make", "make -j", "text/plain"),
 431                       ("libs-install", "make install", "text/plain"),
 432                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 433                       ("libs-clean", "make clean", "text/plain")
 434                       ],
 435
 436     # check we can do the same thing using python2
 437     "samba-nopython-py2": [
 438                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 439                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 440                       ("make", "PYTHON=python2 make -j", "text/plain"),
 441                       ("install", "PYTHON=python2 make install", "text/plain"),
 442                       ("test", "make test-nopython", "text/plain"),
 443                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 444                       ("clean", "PYTHON=python2 make clean", "text/plain"),
 445
 446                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 447                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
 448                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
 449
 450                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 451                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
 452                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
 453
 454                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 455                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
 456                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
 457
 458                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 459                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
 460                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
 461
 462                       # retry against installed library packages
 463                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 464                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
 465                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
 466                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 467                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
 468                       ],
 469
 470     "ldb": [
 471               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 472               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 473               ("make", "make", "text/plain"),
 474               ("install", "make install", "text/plain"),
 475               ("test", "make test", "text/plain"),
 476               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
 477               ("make-no-lmdb", "make", "text/plain"),
 478               ("install-no-lmdb", "make install", "text/plain"),
 479               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 480               ("distcheck", "make distcheck", "text/plain"),
 481               ("clean", "make clean", "text/plain")],
 482
 483     "tdb": [
 484               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 485               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 486               ("make", "make", "text/plain"),
 487               ("install", "make install", "text/plain"),
 488               ("test", "make test", "text/plain"),
 489               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 490               ("distcheck", "make distcheck", "text/plain"),
 491               ("clean", "make clean", "text/plain")],
 492
 493     "talloc": [
 494                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 495                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 496                  ("make", "make", "text/plain"),
 497                  ("install", "make install", "text/plain"),
 498                  ("test", "make test", "text/plain"),
 499                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 500                  ("distcheck", "make distcheck", "text/plain"),
 501                  ("clean", "make clean", "text/plain")],
 502
 503     "replace": [
 504                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 505                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 506                   ("make", "make", "text/plain"),
 507                   ("install", "make install", "text/plain"),
 508                   ("test", "make test", "text/plain"),
 509                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 510                   ("distcheck", "make distcheck", "text/plain"),
 511                   ("clean", "make clean", "text/plain")],
 512
 513     "tevent": [
 514                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 515                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 516                  ("make", "make", "text/plain"),
 517                  ("install", "make install", "text/plain"),
 518                  ("test", "make test", "text/plain"),
 519                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 520                  ("distcheck", "make distcheck", "text/plain"),
 521                  ("clean", "make clean", "text/plain")],
 522
 523     "pidl": [
 524         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 525         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 526         ("touch", "touch *.yp", "text/plain"),
 527         ("make", "make", "text/plain"),
 528         ("test", "make test", "text/plain"),
 529         ("install", "make install", "text/plain"),
 530         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 531         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 532         ("clean", "make clean", "text/plain")],
 533
 534
 535     # these are useful for debugging autobuild
 536     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 537     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 538
 539
 540 }
 541
 542
 543 def do_print(msg):
 544     print("%s" % msg)
 545     sys.stdout.flush()
 546     sys.stderr.flush()
 547
 548
 549 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 550     if show is None:
 551         show = options.verbose
 552     if show:
 553         do_print("Running: '%s' in '%s'" % (cmd, dir))
 554     if output:
 555         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 556     elif checkfail:
 557         return check_call(cmd, shell=True, cwd=dir)
 558     else:
 559         return call(cmd, shell=True, cwd=dir)
 560
 561
 562 class builder(object):
 563     '''handle build of one directory'''
 564
 565     def __init__(self, name, sequence, cp=True):
 566         self.name = name
 567         if name in builddirs:
 568             self.dir = builddirs[name]
 569         else:
 570             self.dir = "."
 571
 572         self.tag = self.name.replace('/', '_')
 573         self.sequence = sequence
 574         self.next = 0
 575         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 576         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 577         if options.verbose:
 578             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 579             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 580         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 581         self.stdout = open(self.stdout_path, 'w')
 582         self.stderr = open(self.stderr_path, 'w')
 583         self.stdin  = open("/dev/null", 'r')
 584         self.sdir = "%s/%s" % (testbase, self.tag)
 585         self.prefix = "%s/%s" % (test_prefix, self.tag)
 586         run_cmd("rm -rf %s" % self.sdir)
 587         run_cmd("rm -rf %s" % self.prefix)
 588         if cp:
 589             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 590         else:
 591             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 592         self.start_next()
 593
 594     def start_next(self):
 595         if self.next == len(self.sequence):
 596             if not options.nocleanup:
 597                 run_cmd("rm -rf %s" % self.sdir)
 598                 run_cmd("rm -rf %s" % self.prefix)
 599             do_print('%s: Completed OK' % self.name)
 600             self.done = True
 601             return
 602         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 603         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 604         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 605         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 606         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 607 #        if self.output_mime_type == "text/x-subunit":
 608 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 609         cwd = os.getcwd()
 610         os.chdir("%s/%s" % (self.sdir, self.dir))
 611         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 612         self.proc = Popen(self.cmd, shell=True,
 613                           close_fds=True,
 614                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 615         os.chdir(cwd)
 616         self.next += 1
 617
 618
 619 class buildlist(object):
 620     '''handle build of multiple directories'''
 621
 622     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 623         global tasks
 624         self.tlist = []
 625         self.tail_proc = None
 626         self.retry = None
 627         if tasknames == []:
 628             if options.restrict_tests:
 629                 tasknames = ["samba-test-only"]
 630             else:
 631                 tasknames = defaulttasks
 632         else:
 633             # If we are only running one test,
 634             # do not sleep randomly to wait for it to start
 635             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 636
 637         for n in tasknames:
 638             b = builder(n, tasks[n], cp=n is not "pidl")
 639             self.tlist.append(b)
 640         if options.retry:
 641             rebase_remote = "rebaseon"
 642             retry_task = [("retry",
 643                             '''set -e
 644                             git remote add -t %s %s %s
 645                             git fetch %s
 646                             while :; do
 647                               sleep 60
 648                               git describe %s/%s > old_remote_branch.desc
 649                               git fetch %s
 650                               git describe %s/%s > remote_branch.desc
 651                               diff old_remote_branch.desc remote_branch.desc
 652                             done
 653                            ''' % (
 654                                 rebase_branch, rebase_remote, rebase_url,
 655                                rebase_remote,
 656                                rebase_remote, rebase_branch,
 657                                rebase_remote,
 658                                rebase_remote, rebase_branch
 659                             ),
 660                             "test/plain")]
 661
 662             self.retry = builder('retry', retry_task, cp=False)
 663             self.need_retry = False
 664
 665     def kill_kids(self):
 666         if self.tail_proc is not None:
 667             self.tail_proc.terminate()
 668             self.tail_proc.wait()
 669             self.tail_proc = None
 670         if self.retry is not None:
 671             self.retry.proc.terminate()
 672             self.retry.proc.wait()
 673             self.retry = None
 674         for b in self.tlist:
 675             if b.proc is not None:
 676                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 677                 b.proc.terminate()
 678                 b.proc.wait()
 679                 b.proc = None
 680
 681     def wait_one(self):
 682         while True:
 683             none_running = True
 684             for b in self.tlist:
 685                 if b.proc is None:
 686                     continue
 687                 none_running = False
 688                 b.status = b.proc.poll()
 689                 if b.status is None:
 690                     continue
 691                 b.proc = None
 692                 return b
 693             if options.retry:
 694                 ret = self.retry.proc.poll()
 695                 if ret is not None:
 696                     self.need_retry = True
 697                     self.retry = None
 698                     return None
 699             if none_running:
 700                 return None
 701             time.sleep(0.1)
 702
 703     def run(self):
 704         while True:
 705             b = self.wait_one()
 706             if options.retry and self.need_retry:
 707                 self.kill_kids()
 708                 do_print("retry needed")
 709                 return (0, None, None, None, "retry")
 710             if b is None:
 711                 break
 712             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 713                 self.kill_kids()
 714                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 715             b.start_next()
 716         self.kill_kids()
 717         return (0, None, None, None, "All OK")
 718
 719     def write_system_info(self):
 720         filename = 'system-info.txt'
 721         f = open(filename, 'w')
 722         for cmd in ['uname -a',
 723                     'lsb_release -a',
 724                     'free',
 725                     'mount',
 726                     'cat /proc/cpuinfo',
 727                     'cc --version',
 728                     'df -m .',
 729                     'df -m %s' % testbase]:
 730             out = run_cmd(cmd, output=True, checkfail=False)
 731             print('### %s' % cmd, file=f)
 732             print(out.decode('utf8', 'backslashreplace'), file=f)
 733             print(file=f)
 734         f.close()
 735         return filename
 736
 737     def tarlogs(self, fname):
 738         tar = tarfile.open(fname, "w:gz")
 739         for b in self.tlist:
 740             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 741             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 742         if os.path.exists("autobuild.log"):
 743             tar.add("autobuild.log")
 744         sys_info = self.write_system_info()
 745         tar.add(sys_info)
 746         tar.close()
 747
 748     def remove_logs(self):
 749         for b in self.tlist:
 750             os.unlink(b.stdout_path)
 751             os.unlink(b.stderr_path)
 752
 753     def start_tail(self):
 754         cmd = ["tail", "-f"]
 755         for b in self.tlist:
 756             cmd.append(b.stdout_path)
 757             cmd.append(b.stderr_path)
 758         self.tail_proc = Popen(cmd, close_fds=True)
 759
 760
 761 def cleanup():
 762     if options.nocleanup:
 763         return
 764     run_cmd("stat %s || true" % test_tmpdir, show=True)
 765     run_cmd("stat %s" % testbase, show=True)
 766     do_print("Cleaning up %r" % cleanup_list)
 767     for d in cleanup_list:
 768         run_cmd("rm -rf %s" % d)
 769
 770
 771 def find_git_root():
 772     '''get to the top of the git repo'''
 773     p = os.getcwd()
 774     while p != '/':
 775         if os.path.isdir(os.path.join(p, ".git")):
 776             return p
 777         p = os.path.abspath(os.path.join(p, '..'))
 778     return None
 779
 780
 781 def daemonize(logfile):
 782     pid = os.fork()
 783     if pid == 0:  # Parent
 784         os.setsid()
 785         pid = os.fork()
 786         if pid != 0:  # Actual daemon
 787             os._exit(0)
 788     else:  # Grandparent
 789         os._exit(0)
 790
 791     import resource      # Resource usage information.
 792     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 793     if maxfd == resource.RLIM_INFINITY:
 794         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 795     for fd in range(0, maxfd):
 796         try:
 797             os.close(fd)
 798         except OSError:
 799             pass
 800     os.open(logfile, os.O_RDWR | os.O_CREAT)
 801     os.dup2(0, 1)
 802     os.dup2(0, 2)
 803
 804
 805 def write_pidfile(fname):
 806     '''write a pid file, cleanup on exit'''
 807     f = open(fname, mode='w')
 808     f.write("%u\n" % os.getpid())
 809     f.close()
 810
 811
 812 def rebase_tree(rebase_url, rebase_branch="master"):
 813     rebase_remote = "rebaseon"
 814     do_print("Rebasing on %s" % rebase_url)
 815     run_cmd("git describe HEAD", show=True, dir=test_master)
 816     run_cmd("git remote add -t %s %s %s" %
 817             (rebase_branch, rebase_remote, rebase_url),
 818             show=True, dir=test_master)
 819     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 820     if options.fix_whitespace:
 821         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 822                 (rebase_remote, rebase_branch),
 823                 show=True, dir=test_master)
 824     else:
 825         run_cmd("git rebase --force-rebase %s/%s" %
 826                 (rebase_remote, rebase_branch),
 827                 show=True, dir=test_master)
 828     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 829                    (rebase_remote, rebase_branch),
 830                    dir=test_master, output=True)
 831     if diff == '':
 832         do_print("No differences between HEAD and %s/%s - exiting" %
 833                  (rebase_remote, rebase_branch))
 834         sys.exit(0)
 835     run_cmd("git describe %s/%s" %
 836             (rebase_remote, rebase_branch),
 837             show=True, dir=test_master)
 838     run_cmd("git describe HEAD", show=True, dir=test_master)
 839     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 840             (rebase_remote, rebase_branch),
 841             show=True, dir=test_master)
 842
 843
 844 def push_to(push_url, push_branch="master"):
 845     push_remote = "pushto"
 846     do_print("Pushing to %s" % push_url)
 847     if options.mark:
 848         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 849         run_cmd("git commit --amend -c HEAD", dir=test_master)
 850         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 851         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 852     run_cmd("git remote add -t %s %s %s" %
 853             (push_branch, push_remote, push_url),
 854             show=True, dir=test_master)
 855     run_cmd("git push %s +HEAD:%s" %
 856             (push_remote, push_branch),
 857             show=True, dir=test_master)
 858
 859
 860 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 861
 862 gitroot = find_git_root()
 863 if gitroot is None:
 864     raise Exception("Failed to find git root")
 865
 866 parser = OptionParser()
 867 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 868 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 869 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 870 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 871                   default=def_testbase)
 872 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 873 parser.add_option("", "--verbose", help="show all commands as they are run",
 874                   default=False, action="store_true")
 875 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 876                   default=None, type='str')
 877 parser.add_option("", "--pushto", help="push to a git url on success",
 878                   default=None, type='str')
 879 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 880                   default=False, action="store_true")
 881 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 882                   default=False, action="store_true")
 883 parser.add_option("", "--retry", help="automatically retry if master changes",
 884                   default=False, action="store_true")
 885 parser.add_option("", "--email", help="send email to the given address on failure",
 886                   type='str', default=None)
 887 parser.add_option("", "--email-from", help="send email from the given address",
 888                   type='str', default="autobuild@samba.org")
 889 parser.add_option("", "--email-server", help="send email via the given server",
 890                   type='str', default='localhost')
 891 parser.add_option("", "--always-email", help="always send email, even on success",
 892                   action="store_true")
 893 parser.add_option("", "--daemon", help="daemonize after initial setup",
 894                   action="store_true")
 895 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 896                   default="master", type='str')
 897 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 898                   default=gitroot, type='str')
 899 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 900                   default=False, action="store_true")
 901 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 902                   default='')
 903
 904
 905 def send_email(subject, text, log_tar):
 906     if options.email is None:
 907         do_print("not sending email because the recipient is not set")
 908         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 909                  (subject, text))
 910         return
 911     outer = MIMEMultipart()
 912     outer['Subject'] = subject
 913     outer['To'] = options.email
 914     outer['From'] = options.email_from
 915     outer['Date'] = email.utils.formatdate(localtime=True)
 916     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 917     outer.attach(MIMEText(text, 'plain'))
 918     if options.attach_logs:
 919         fp = open(log_tar, 'rb')
 920         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 921         fp.close()
 922         # Set the filename parameter
 923         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 924         outer.attach(msg)
 925     content = outer.as_string()
 926     s = smtplib.SMTP(options.email_server)
 927     email_user = os.getenv('SMTP_USERNAME')
 928     email_password = os.getenv('SMTP_PASSWORD')
 929     if email_user is not None:
 930         s.starttls()
 931         s.login(email_user, email_password)
 932
 933     s.sendmail(options.email_from, [options.email], content)
 934     s.set_debuglevel(1)
 935     s.quit()
 936
 937
 938 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 939                   elapsed_time, log_base=None, add_log_tail=True):
 940     '''send an email to options.email about the failure'''
 941     elapsed_minutes = elapsed_time / 60.0
 942     if log_base is None:
 943         log_base = gitroot
 944     text = '''
 945 Dear Developer,
 946
 947 Your autobuild on %s failed after %.1f minutes
 948 when trying to test %s with the following error:
 949
 950    %s
 951
 952 the autobuild has been abandoned. Please fix the error and resubmit.
 953
 954 A summary of the autobuild process is here:
 955
 956   %s/autobuild.log
 957 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 958
 959     if options.restrict_tests:
 960         text += """
 961 The build was restricted to tests matching %s\n""" % options.restrict_tests
 962
 963     if failed_task != 'rebase':
 964         text += '''
 965 You can see logs of the failed task here:
 966
 967   %s/%s.stdout
 968   %s/%s.stderr
 969
 970 or you can get full logs of all tasks in this job here:
 971
 972   %s/logs.tar.gz
 973
 974 The top commit for the tree that was built was:
 975
 976 %s
 977
 978 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 979
 980     if add_log_tail:
 981         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 982         lines = f.readlines()
 983         log_tail = "".join(lines[-50:])
 984         num_lines = len(lines)
 985         if num_lines < 50:
 986             # Also include stderr (compile failures) if < 50 lines of stdout
 987             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 988             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 989
 990         text += '''
 991 The last 50 lines of log messages:
 992
 993 %s
 994     ''' % log_tail
 995         f.close()
 996
 997     logs = os.path.join(gitroot, 'logs.tar.gz')
 998     send_email('autobuild[%s] failure on %s for task %s during %s'
 999                % (options.branch, platform.node(), failed_task, failed_stage),
1000                text, logs)
1001
1002
1003 def email_success(elapsed_time, log_base=None):
1004     '''send an email to options.email about a successful build'''
1005     if log_base is None:
1006         log_base = gitroot
1007     text = '''
1008 Dear Developer,
1009
1010 Your autobuild on %s has succeeded after %.1f minutes.
1011
1012 ''' % (platform.node(), elapsed_time / 60.)
1013
1014     if options.restrict_tests:
1015         text += """
1016 The build was restricted to tests matching %s\n""" % options.restrict_tests
1017
1018     if options.keeplogs:
1019         text += '''
1020
1021 you can get full logs of all tasks in this job here:
1022
1023   %s/logs.tar.gz
1024
1025 ''' % log_base
1026
1027     text += '''
1028 The top commit for the tree that was built was:
1029
1030 %s
1031 ''' % top_commit_msg
1032
1033     logs = os.path.join(gitroot, 'logs.tar.gz')
1034     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1035                text, logs)
1036
1037
1038 (options, args) = parser.parse_args()
1039
1040 if options.retry:
1041     if options.rebase is None:
1042         raise Exception('You can only use --retry if you also rebase')
1043
1044 testbase = "%s/b%u" % (options.testbase, os.getpid())
1045 test_master = "%s/master" % testbase
1046 test_prefix = "%s/prefix" % testbase
1047 test_tmpdir = "%s/tmp" % testbase
1048 os.environ['TMPDIR'] = test_tmpdir
1049
1050 # get the top commit message, for emails
1051 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1052 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
1053
1054 try:
1055     os.makedirs(testbase)
1056 except Exception as reason:
1057     raise Exception("Unable to create %s : %s" % (testbase, reason))
1058 cleanup_list.append(testbase)
1059
1060 if options.daemon:
1061     logfile = os.path.join(testbase, "log")
1062     do_print("Forking into the background, writing progress to %s" % logfile)
1063     daemonize(logfile)
1064
1065 write_pidfile(gitroot + "/autobuild.pid")
1066
1067 start_time = time.time()
1068
1069 while True:
1070     try:
1071         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1072         os.makedirs(test_tmpdir)
1073         # The waf uninstall code removes empty directories all the way
1074         # up the tree.  Creating a file in test_tmpdir stops it from
1075         # being removed.
1076         run_cmd("touch %s" % os.path.join(test_tmpdir,
1077                                           ".directory-is-not-empty"), show=True)
1078         run_cmd("stat %s" % test_tmpdir, show=True)
1079         run_cmd("stat %s" % testbase, show=True)
1080         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1081     except Exception:
1082         cleanup()
1083         raise
1084
1085     try:
1086         try:
1087             if options.rebase is not None:
1088                 rebase_tree(options.rebase, rebase_branch=options.branch)
1089         except Exception:
1090             cleanup_list.append(gitroot + "/autobuild.pid")
1091             cleanup()
1092             elapsed_time = time.time() - start_time
1093             email_failure(-1, 'rebase', 'rebase', 'rebase',
1094                           'rebase on %s failed' % options.branch,
1095                           elapsed_time, log_base=options.log_base)
1096             sys.exit(1)
1097         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1098         if options.tail:
1099             blist.start_tail()
1100         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1101         if status != 0 or errstr != "retry":
1102             break
1103         cleanup()
1104     except Exception:
1105         cleanup()
1106         raise
1107
1108 cleanup_list.append(gitroot + "/autobuild.pid")
1109
1110 do_print(errstr)
1111
1112 blist.kill_kids()
1113 if options.tail:
1114     do_print("waiting for tail to flush")
1115     time.sleep(1)
1116
1117 elapsed_time = time.time() - start_time
1118 if status == 0:
1119     if options.passcmd is not None:
1120         do_print("Running passcmd: %s" % options.passcmd)
1121         run_cmd(options.passcmd, dir=test_master)
1122     if options.pushto is not None:
1123         push_to(options.pushto, push_branch=options.branch)
1124     if options.keeplogs or options.attach_logs:
1125         blist.tarlogs("logs.tar.gz")
1126         do_print("Logs in logs.tar.gz")
1127     if options.always_email:
1128         email_success(elapsed_time, log_base=options.log_base)
1129     blist.remove_logs()
1130     cleanup()
1131     do_print(errstr)
1132     sys.exit(0)
1133
1134 # something failed, gather a tar of the logs
1135 blist.tarlogs("logs.tar.gz")
1136
1137 if options.email is not None:
1138     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1139                   elapsed_time, log_base=options.log_base)
1140 else:
1141     elapsed_minutes = elapsed_time / 60.0
1142     print('''
1143
1144 ####################################################################
1145
1146 AUTOBUILD FAILURE
1147
1148 Your autobuild[%s] on %s failed after %.1f minutes
1149 when trying to test %s with the following error:
1150
1151    %s
1152
1153 the autobuild has been abandoned. Please fix the error and resubmit.
1154
1155 ####################################################################
1156
1157 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1158
1159 cleanup()
1160 do_print(errstr)
1161 do_print("Logs in logs.tar.gz")
1162 sys.exit(status)