script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 cleanup_list = []
  35
  36 builddirs = {
  37     "ctdb": "ctdb",
  38     "samba": ".",
  39     "samba-nt4": ".",
  40     "samba-fileserver": ".",
  41     "samba-ad-member": ".",
  42     "samba-xc": ".",
  43     "samba-o3": ".",
  44     "samba-ctdb": ".",
  45     "samba-libs": ".",
  46     "samba-static": ".",
  47     "samba-none-env": ".",
  48     "samba-ad-dc-1": ".",
  49     "samba-ad-dc-2": ".",
  50     "samba-ad-dc-3": ".",
  51     "samba-ad-dc-4": ".",
  52     "samba-ad-dc-5": ".",
  53     "samba-ad-dc-6": ".",
  54     "samba-ad-dc-ntvfs": ".",
  55     "samba-ad-dc-backup": ".",
  56     "samba-systemkrb5": ".",
  57     "samba-nopython": ".",
  58     "samba-nopython-py2": ".",
  59     "ldb": "lib/ldb",
  60     "tdb": "lib/tdb",
  61     "talloc": "lib/talloc",
  62     "replace": "lib/replace",
  63     "tevent": "lib/tevent",
  64     "pidl": "pidl"
  65 }
  66
  67 defaulttasks = builddirs.keys()
  68
  69 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  70     defaulttasks.remove("samba-o3")
  71
  72 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  73 samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
  74
  75 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
  76 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  77 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  78 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  79 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
  80 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  81 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
  82
  83 tasks = {
  84     "ctdb": [("random-sleep", "../script/random-sleep.sh 300 900", "text/plain"),
  85                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  86                ("make", "make all", "text/plain"),
  87                ("install", "make install", "text/plain"),
  88                ("test", "make autotest", "text/plain"),
  89                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  90                ("clean", "make clean", "text/plain")],
  91
  92     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
  93     "samba": [
  94                 ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
  95                 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  96                 ("make", "make -j", "text/plain"),
  97                 ("test", "make test FAIL_IMMEDIATELY=1 "
  98                  "TESTS='--exclude-env=none "
  99                  "--exclude-env=nt4_dc "
 100                  "--exclude-env=nt4_dc_schannel "
 101                  "--exclude-env=nt4_member "
 102                  "--exclude-env=ad_dc "
 103                  "--exclude-env=ad_dc_backup "
 104                  "--exclude-env=ad_dc_ntvfs "
 105                  "--exclude-env=ad_dc_default "
 106                  "--exclude-env=ad_dc_slowtests "
 107                  "--exclude-env=ad_dc_no_nss "
 108                  "--exclude-env=ad_dc_no_ntlm "
 109                  "--exclude-env=fl2003dc "
 110                  "--exclude-env=fl2008dc "
 111                  "--exclude-env=fl2008r2dc "
 112                  "--exclude-env=ad_member "
 113                  "--exclude-env=ad_member_idmap_rid "
 114                  "--exclude-env=ad_member_idmap_ad "
 115                  "--exclude-env=ad_member_rfc2307 "
 116                  "--exclude-env=chgdcpass "
 117                  "--exclude-env=vampire_2000_dc "
 118                  "--exclude-env=fl2000dc "
 119                  "--exclude-env=fileserver "
 120                  "--exclude-env=maptoguest "
 121                  "--exclude-env=simpleserver "
 122                  "--exclude-env=backupfromdc "
 123                  "--exclude-env=restoredc "
 124                  "--exclude-env=renamedc "
 125                  "--exclude-env=offlinebackupdc "
 126                  "--exclude-env=labdc "
 127                  "--exclude-env=preforkrestartdc "
 128                  "--exclude-env=proclimitdc "
 129                  "--exclude-env=promoted_dc "
 130                  "--exclude-env=vampire_dc "
 131                  "--exclude-env=rodc "
 132                  "--exclude-env=ad_dc_default "
 133                  "--exclude-env=ad_dc_slowtests "
 134                  "'",
 135                  "text/plain"),
 136                 ("install", "make install", "text/plain"),
 137                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 138                 ("clean", "make clean", "text/plain")],
 139
 140     "samba-nt4": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 141                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 142                     ("make", "make -j", "text/plain"),
 143                     ("test", "make test FAIL_IMMEDIATELY=1 "
 144                      "TESTS='"
 145                      "--include-env=nt4_dc "
 146                      "--include-env=nt4_dc_schannel "
 147                      "--include-env=nt4_member "
 148                      "'", "text/plain"),
 149                     ("install", "make install", "text/plain"),
 150                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 151                     ("clean", "make clean", "text/plain")],
 152
 153     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 154                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 155                            ("make", "make -j", "text/plain"),
 156                            ("test", "make test FAIL_IMMEDIATELY=1 "
 157                             "TESTS='"
 158                             "--include-env=fileserver "
 159                             "--include-env=maptoguest "
 160                             "--include-env=simpleserver "
 161                             "'", "text/plain"),
 162                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 163
 164     "samba-ad-member": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 165                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 166                         ("make", "make -j", "text/plain"),
 167                         ("test", "make test FAIL_IMMEDIATELY=1 "
 168                          "TESTS='"
 169                          "--include-env=ad_member "
 170                          "--include-env=ad_member_idmap_rid "
 171                          "--include-env=ad_member_idmap_ad "
 172                          "--include-env=ad_member_rfc2307 "
 173                          "'", "text/plain"),
 174                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 175
 176     "samba-ad-dc-1": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 177                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 178                       ("make", "make -j", "text/plain"),
 179                       ("test", "make test FAIL_IMMEDIATELY=1 "
 180                        "TESTS='--include-env=ad_dc "
 181                        "--include-env=ad_dc_no_nss "
 182                        "--include-env=ad_dc_no_ntlm "
 183                        "'", "text/plain"),
 184                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 185
 186     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 187                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 188                         ("make", "make -j", "text/plain"),
 189                         ("test", "make test FAIL_IMMEDIATELY=1 "
 190                          "TESTS='"
 191                          "--include-env=vampire_dc "
 192                          "--include-env=vampire_2000_dc "
 193                          "--include-env=rodc "
 194                          "'", "text/plain"),
 195                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 196
 197     "samba-ad-dc-3": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 198                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 199                         ("make", "make -j", "text/plain"),
 200                         ("test", "make test FAIL_IMMEDIATELY=1 "
 201                          "TESTS='"
 202                          "--include-env=promoted_dc "
 203                          "--include-env=chgdcpass "
 204                          "--include-env=preforkrestartdc "
 205                          "--include-env=proclimitdc "
 206                          "'", "text/plain"),
 207                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 208
 209     "samba-ad-dc-4": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 210                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 211                         ("make", "make -j", "text/plain"),
 212                         ("test", "make test FAIL_IMMEDIATELY=1 "
 213                          "TESTS='"
 214                          "--include-env=fl2000dc "
 215                          "--include-env=fl2003dc "
 216                          "--include-env=fl2008dc "
 217                          "--include-env=fl2008r2dc "
 218                          "'", "text/plain"),
 219                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 220
 221     "samba-ad-dc-5": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 222                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 223                         ("make", "make -j", "text/plain"),
 224                         ("test", "make test FAIL_IMMEDIATELY=1 "
 225                          "TESTS='"
 226                          "--include-env=ad_dc_default "
 227                          "'", "text/plain"),
 228                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 229
 230     "samba-ad-dc-6": [("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 231                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 232                         ("make", "make -j", "text/plain"),
 233                         ("test", "make test FAIL_IMMEDIATELY=1 "
 234                          "TESTS='"
 235                          "--include-env=ad_dc_slowtests "
 236                          "'", "text/plain"),
 237                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 238
 239     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 240     # This is currently the longest task, so we don't randomly delay it.
 241     "samba-ad-dc-ntvfs": [
 242                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 243                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 244                       ("make", "make -j", "text/plain"),
 245                       ("test", "make test FAIL_IMMEDIATELY=1 "
 246                        "TESTS='"
 247                        "--include-env=ad_dc_ntvfs "
 248                        "'", "text/plain"),
 249                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 250
 251     # run the backup/restore testenvs separately as they're fairly standalone
 252     # (and CI seems to max out at ~8 different DCs running at once)
 253     "samba-ad-dc-backup": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 254                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 255                         ("make", "make -j", "text/plain"),
 256                         ("test", "make test FAIL_IMMEDIATELY=1 "
 257                          "TESTS='--include-env=backupfromdc "
 258                          "--include-env=restoredc "
 259                          "--include-env=renamedc "
 260                          "--include-env=offlinebackupdc "
 261                          "--include-env=labdc "
 262                          "--include-env=ad_dc_backup "
 263                          "'", "text/plain"),
 264                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 265
 266     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 267                           ("make", "make -j", "text/plain"),
 268                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 269
 270     # Test cross-compile infrastructure
 271     "samba-xc": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 272                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 273                    ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 274                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 275                    ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 276                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 277                    ("compare-results", "script/compare_cc_results.py "
 278                     "./bin/c4che/default{} "
 279                     "./bin-xe/c4che/default{} "
 280                     "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3)), "text/plain")],
 281
 282     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 283     "samba-o3": [("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 284                    ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 285                    ("make", "make -j", "text/plain"),
 286                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 287                     "TESTS='--include-env=ad_dc'", "text/plain"),
 288                    ("install", "make install", "text/plain"),
 289                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 290                    ("clean", "make clean", "text/plain")],
 291
 292     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 293
 294                      # make sure we have tdb around:
 295                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 296                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 297                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 298
 299
 300                      # build samba with cluster support (also building ctdb):
 301                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 302                      ("samba-make", "make", "text/plain"),
 303                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 304                      ("samba-install", "make install", "text/plain"),
 305                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 306
 307                      # clean up:
 308                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 309                      ("clean", "make clean", "text/plain"),
 310                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 311
 312     "samba-libs": [
 313                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 314                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 315                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 316                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 317
 318                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 319                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 320                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 321
 322                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 323                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 324                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 325
 326                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 327                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 328                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 329
 330                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 331                       ("nondevel-make", "make -j", "text/plain"),
 332                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 333                       ("nondevel-install", "make install", "text/plain"),
 334                       ("nondevel-dist", "make dist", "text/plain"),
 335
 336                       # retry with all modules shared
 337                       ("allshared-distclean", "make distclean", "text/plain"),
 338                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 339                       ("allshared-make", "make -j", "text/plain")],
 340
 341     "samba-none-env": [
 342                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 343                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 344                       ("make", "make -j", "text/plain"),
 345                       ("test", "make test "
 346                        "FAIL_IMMEDIATELY=1 "
 347                        "TESTS='--include-env=none'",
 348                        "text/plain")],
 349
 350     "samba-static": [
 351                       ("random-sleep", "script/random-sleep.sh 1 1", "text/plain"),
 352                       # build with all modules static
 353                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 354                       ("allstatic-make", "make -j", "text/plain"),
 355                       ("allstatic-test", "make test "
 356                        "FAIL_IMMEDIATELY=1 "
 357                        "TESTS='samba3.smb2.create.*nt4_dc'",
 358                        "text/plain"),
 359
 360                       # retry without any required modules
 361                       ("none-distclean", "make distclean", "text/plain"),
 362                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 363                       ("none-make", "make -j", "text/plain"),
 364
 365                       # retry with nonshared smbd and smbtorture
 366                       ("nonshared-distclean", "make distclean", "text/plain"),
 367                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 368                       ("nonshared-make", "make -j", "text/plain")],
 369
 370     "samba-systemkrb5": [
 371                       ("random-sleep", "script/random-sleep.sh 900 1500", "text/plain"),
 372                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 373                       ("make", "make -j", "text/plain"),
 374                       # we currently cannot run a full make test, a limited list of tests could be run
 375                       # via "make test TESTS=sometests"
 376                       ("test", "make test FAIL_IMMEDIATELY=1 "
 377                        "TESTS='--include-env=ktest'", "text/plain"),
 378                       ("install", "make install", "text/plain"),
 379                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 380                       ("clean", "make clean", "text/plain")
 381                       ],
 382
 383     # Test Samba without python still builds.  When this test fails
 384     # due to more use of Python, the expectations is that the newly
 385     # failing part of the code should be disabled when
 386     # --disable-python is set (rather than major work being done to
 387     # support this environment).  The target here is for vendors
 388     # shipping a minimal smbd.
 389     "samba-nopython": [
 390                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 391                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 392                       ("make", "make -j", "text/plain"),
 393                       ("install", "make install", "text/plain"),
 394                       ("test", "make test-nopython", "text/plain"),
 395                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 396                       ("clean", "make clean", "text/plain"),
 397
 398                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 399                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 400                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 401
 402                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 403                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 404                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 405
 406                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 407                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 408                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 409
 410                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 411                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 412                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 413
 414                       # retry against installed library packages
 415                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 416                       ("libs-make", "make -j", "text/plain"),
 417                       ("libs-install", "make install", "text/plain"),
 418                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 419                       ("libs-clean", "make clean", "text/plain")
 420                       ],
 421
 422     # check we can do the same thing using python2
 423     "samba-nopython-py2": [
 424                       ("random-sleep", "script/random-sleep.sh 300 900", "text/plain"),
 425                       ("configure", "PYTHON=python2 ./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 426                       ("make", "PYTHON=python2 make -j", "text/plain"),
 427                       ("install", "PYTHON=python2 make install", "text/plain"),
 428                       ("test", "make test-nopython", "text/plain"),
 429                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 430                       ("clean", "PYTHON=python2 make clean", "text/plain"),
 431
 432                       ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 433                       ("talloc-make", "cd lib/talloc && PYTHON=python2 make", "text/plain"),
 434                       ("talloc-install", "cd lib/talloc && PYTHON=python2 make install", "text/plain"),
 435
 436                       ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 437                       ("tdb-make", "cd lib/tdb && PYTHON=python2 make", "text/plain"),
 438                       ("tdb-install", "cd lib/tdb && PYTHON=python2 make install", "text/plain"),
 439
 440                       ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 441                       ("tevent-make", "cd lib/tevent && PYTHON=python2 make", "text/plain"),
 442                       ("tevent-install", "cd lib/tevent && PYTHON=python2 make install", "text/plain"),
 443
 444                       ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 445                       ("ldb-make", "cd lib/ldb && PYTHON=python2 make", "text/plain"),
 446                       ("ldb-install", "cd lib/ldb && PYTHON=python2 make install", "text/plain"),
 447
 448                       # retry against installed library packages
 449                       ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 450                       ("libs-make", "PYTHON=python2 make -j", "text/plain"),
 451                       ("libs-install", "PYTHON=python2 make install", "text/plain"),
 452                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 453                       ("libs-clean", "PYTHON=python2 make clean", "text/plain")
 454                       ],
 455
 456     "ldb": [
 457               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 458               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 459               ("make", "make", "text/plain"),
 460               ("install", "make install", "text/plain"),
 461               ("test", "make test", "text/plain"),
 462               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX}", "text/plain"),
 463               ("make-no-lmdb", "make", "text/plain"),
 464               ("install-no-lmdb", "make install", "text/plain"),
 465               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 466               ("distcheck", "make distcheck", "text/plain"),
 467               ("clean", "make clean", "text/plain")],
 468
 469     "tdb": [
 470               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 471               ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 472               ("make", "make", "text/plain"),
 473               ("install", "make install", "text/plain"),
 474               ("test", "make test", "text/plain"),
 475               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 476               ("distcheck", "make distcheck", "text/plain"),
 477               ("clean", "make clean", "text/plain")],
 478
 479     "talloc": [
 480                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 481                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 482                  ("make", "make", "text/plain"),
 483                  ("install", "make install", "text/plain"),
 484                  ("test", "make test", "text/plain"),
 485                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 486                  ("distcheck", "make distcheck", "text/plain"),
 487                  ("clean", "make clean", "text/plain")],
 488
 489     "replace": [
 490                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 491                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 492                   ("make", "make", "text/plain"),
 493                   ("install", "make install", "text/plain"),
 494                   ("test", "make test", "text/plain"),
 495                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 496                   ("distcheck", "make distcheck", "text/plain"),
 497                   ("clean", "make clean", "text/plain")],
 498
 499     "tevent": [
 500                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 501                  ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 502                  ("make", "make", "text/plain"),
 503                  ("install", "make install", "text/plain"),
 504                  ("test", "make test", "text/plain"),
 505                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 506                  ("distcheck", "make distcheck", "text/plain"),
 507                  ("clean", "make clean", "text/plain")],
 508
 509     "pidl": [
 510         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 511         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 512         ("touch", "touch *.yp", "text/plain"),
 513         ("make", "make", "text/plain"),
 514         ("test", "make test", "text/plain"),
 515         ("install", "make install", "text/plain"),
 516         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 517         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 518         ("clean", "make clean", "text/plain")],
 519
 520
 521     # these are useful for debugging autobuild
 522     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 523     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 524
 525
 526 }
 527
 528
 529 def do_print(msg):
 530     print("%s" % msg)
 531     sys.stdout.flush()
 532     sys.stderr.flush()
 533
 534
 535 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 536     if show is None:
 537         show = options.verbose
 538     if show:
 539         do_print("Running: '%s' in '%s'" % (cmd, dir))
 540     if output:
 541         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 542     elif checkfail:
 543         return check_call(cmd, shell=True, cwd=dir)
 544     else:
 545         return call(cmd, shell=True, cwd=dir)
 546
 547
 548 class builder(object):
 549     '''handle build of one directory'''
 550
 551     def __init__(self, name, sequence, cp=True):
 552         self.name = name
 553         if name in builddirs:
 554             self.dir = builddirs[name]
 555         else:
 556             self.dir = "."
 557
 558         self.tag = self.name.replace('/', '_')
 559         self.sequence = sequence
 560         self.next = 0
 561         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 562         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 563         if options.verbose:
 564             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 565             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 566         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 567         self.stdout = open(self.stdout_path, 'w')
 568         self.stderr = open(self.stderr_path, 'w')
 569         self.stdin  = open("/dev/null", 'r')
 570         self.sdir = "%s/%s" % (testbase, self.tag)
 571         self.prefix = "%s/%s" % (test_prefix, self.tag)
 572         run_cmd("rm -rf %s" % self.sdir)
 573         run_cmd("rm -rf %s" % self.prefix)
 574         if cp:
 575             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 576         else:
 577             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 578         self.start_next()
 579
 580     def start_next(self):
 581         if self.next == len(self.sequence):
 582             if not options.nocleanup:
 583                 run_cmd("rm -rf %s" % self.sdir)
 584                 run_cmd("rm -rf %s" % self.prefix)
 585             do_print('%s: Completed OK' % self.name)
 586             self.done = True
 587             return
 588         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 589         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
 590         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 591         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 592         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 593 #        if self.output_mime_type == "text/x-subunit":
 594 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 595         cwd = os.getcwd()
 596         os.chdir("%s/%s" % (self.sdir, self.dir))
 597         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 598         self.proc = Popen(self.cmd, shell=True,
 599                           close_fds=True,
 600                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 601         os.chdir(cwd)
 602         self.next += 1
 603
 604
 605 class buildlist(object):
 606     '''handle build of multiple directories'''
 607
 608     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 609         global tasks
 610         self.tlist = []
 611         self.tail_proc = None
 612         self.retry = None
 613         if tasknames == []:
 614             if options.restrict_tests:
 615                 tasknames = ["samba-test-only"]
 616             else:
 617                 tasknames = defaulttasks
 618         else:
 619             # If we are only running one test,
 620             # do not sleep randomly to wait for it to start
 621             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 622
 623         for n in tasknames:
 624             b = builder(n, tasks[n], cp=n is not "pidl")
 625             self.tlist.append(b)
 626         if options.retry:
 627             rebase_remote = "rebaseon"
 628             retry_task = [("retry",
 629                             '''set -e
 630                             git remote add -t %s %s %s
 631                             git fetch %s
 632                             while :; do
 633                               sleep 60
 634                               git describe %s/%s > old_remote_branch.desc
 635                               git fetch %s
 636                               git describe %s/%s > remote_branch.desc
 637                               diff old_remote_branch.desc remote_branch.desc
 638                             done
 639                            ''' % (
 640                                 rebase_branch, rebase_remote, rebase_url,
 641                                rebase_remote,
 642                                rebase_remote, rebase_branch,
 643                                rebase_remote,
 644                                rebase_remote, rebase_branch
 645                             ),
 646                             "test/plain")]
 647
 648             self.retry = builder('retry', retry_task, cp=False)
 649             self.need_retry = False
 650
 651     def kill_kids(self):
 652         if self.tail_proc is not None:
 653             self.tail_proc.terminate()
 654             self.tail_proc.wait()
 655             self.tail_proc = None
 656         if self.retry is not None:
 657             self.retry.proc.terminate()
 658             self.retry.proc.wait()
 659             self.retry = None
 660         for b in self.tlist:
 661             if b.proc is not None:
 662                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 663                 b.proc.terminate()
 664                 b.proc.wait()
 665                 b.proc = None
 666
 667     def wait_one(self):
 668         while True:
 669             none_running = True
 670             for b in self.tlist:
 671                 if b.proc is None:
 672                     continue
 673                 none_running = False
 674                 b.status = b.proc.poll()
 675                 if b.status is None:
 676                     continue
 677                 b.proc = None
 678                 return b
 679             if options.retry:
 680                 ret = self.retry.proc.poll()
 681                 if ret is not None:
 682                     self.need_retry = True
 683                     self.retry = None
 684                     return None
 685             if none_running:
 686                 return None
 687             time.sleep(0.1)
 688
 689     def run(self):
 690         while True:
 691             b = self.wait_one()
 692             if options.retry and self.need_retry:
 693                 self.kill_kids()
 694                 do_print("retry needed")
 695                 return (0, None, None, None, "retry")
 696             if b is None:
 697                 break
 698             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 699                 self.kill_kids()
 700                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 701             b.start_next()
 702         self.kill_kids()
 703         return (0, None, None, None, "All OK")
 704
 705     def write_system_info(self):
 706         filename = 'system-info.txt'
 707         f = open(filename, 'w')
 708         for cmd in ['uname -a',
 709                     'lsb_release -a',
 710                     'free',
 711                     'mount',
 712                     'cat /proc/cpuinfo',
 713                     'cc --version',
 714                     'df -m .',
 715                     'df -m %s' % testbase]:
 716             out = run_cmd(cmd, output=True, checkfail=False)
 717             print('### %s' % cmd, file=f)
 718             print(out.decode('utf8', 'backslashreplace'), file=f)
 719             print(file=f)
 720         f.close()
 721         return filename
 722
 723     def tarlogs(self, fname):
 724         tar = tarfile.open(fname, "w:gz")
 725         for b in self.tlist:
 726             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 727             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 728         if os.path.exists("autobuild.log"):
 729             tar.add("autobuild.log")
 730         sys_info = self.write_system_info()
 731         tar.add(sys_info)
 732         tar.close()
 733
 734     def remove_logs(self):
 735         for b in self.tlist:
 736             os.unlink(b.stdout_path)
 737             os.unlink(b.stderr_path)
 738
 739     def start_tail(self):
 740         cmd = ["tail", "-f"]
 741         for b in self.tlist:
 742             cmd.append(b.stdout_path)
 743             cmd.append(b.stderr_path)
 744         self.tail_proc = Popen(cmd, close_fds=True)
 745
 746
 747 def cleanup():
 748     if options.nocleanup:
 749         return
 750     run_cmd("stat %s || true" % test_tmpdir, show=True)
 751     run_cmd("stat %s" % testbase, show=True)
 752     do_print("Cleaning up %r" % cleanup_list)
 753     for d in cleanup_list:
 754         run_cmd("rm -rf %s" % d)
 755
 756
 757 def find_git_root():
 758     '''get to the top of the git repo'''
 759     p = os.getcwd()
 760     while p != '/':
 761         if os.path.isdir(os.path.join(p, ".git")):
 762             return p
 763         p = os.path.abspath(os.path.join(p, '..'))
 764     return None
 765
 766
 767 def daemonize(logfile):
 768     pid = os.fork()
 769     if pid == 0:  # Parent
 770         os.setsid()
 771         pid = os.fork()
 772         if pid != 0:  # Actual daemon
 773             os._exit(0)
 774     else:  # Grandparent
 775         os._exit(0)
 776
 777     import resource      # Resource usage information.
 778     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 779     if maxfd == resource.RLIM_INFINITY:
 780         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 781     for fd in range(0, maxfd):
 782         try:
 783             os.close(fd)
 784         except OSError:
 785             pass
 786     os.open(logfile, os.O_RDWR | os.O_CREAT)
 787     os.dup2(0, 1)
 788     os.dup2(0, 2)
 789
 790
 791 def write_pidfile(fname):
 792     '''write a pid file, cleanup on exit'''
 793     f = open(fname, mode='w')
 794     f.write("%u\n" % os.getpid())
 795     f.close()
 796
 797
 798 def rebase_tree(rebase_url, rebase_branch="master"):
 799     rebase_remote = "rebaseon"
 800     do_print("Rebasing on %s" % rebase_url)
 801     run_cmd("git describe HEAD", show=True, dir=test_master)
 802     run_cmd("git remote add -t %s %s %s" %
 803             (rebase_branch, rebase_remote, rebase_url),
 804             show=True, dir=test_master)
 805     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 806     if options.fix_whitespace:
 807         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 808                 (rebase_remote, rebase_branch),
 809                 show=True, dir=test_master)
 810     else:
 811         run_cmd("git rebase --force-rebase %s/%s" %
 812                 (rebase_remote, rebase_branch),
 813                 show=True, dir=test_master)
 814     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 815                    (rebase_remote, rebase_branch),
 816                    dir=test_master, output=True)
 817     if diff == '':
 818         do_print("No differences between HEAD and %s/%s - exiting" %
 819                  (rebase_remote, rebase_branch))
 820         sys.exit(0)
 821     run_cmd("git describe %s/%s" %
 822             (rebase_remote, rebase_branch),
 823             show=True, dir=test_master)
 824     run_cmd("git describe HEAD", show=True, dir=test_master)
 825     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 826             (rebase_remote, rebase_branch),
 827             show=True, dir=test_master)
 828
 829
 830 def push_to(push_url, push_branch="master"):
 831     push_remote = "pushto"
 832     do_print("Pushing to %s" % push_url)
 833     if options.mark:
 834         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 835         run_cmd("git commit --amend -c HEAD", dir=test_master)
 836         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 837         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 838     run_cmd("git remote add -t %s %s %s" %
 839             (push_branch, push_remote, push_url),
 840             show=True, dir=test_master)
 841     run_cmd("git push %s +HEAD:%s" %
 842             (push_remote, push_branch),
 843             show=True, dir=test_master)
 844
 845
 846 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 847
 848 gitroot = find_git_root()
 849 if gitroot is None:
 850     raise Exception("Failed to find git root")
 851
 852 parser = OptionParser()
 853 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 854 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 855 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 856 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 857                   default=def_testbase)
 858 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 859 parser.add_option("", "--verbose", help="show all commands as they are run",
 860                   default=False, action="store_true")
 861 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 862                   default=None, type='str')
 863 parser.add_option("", "--pushto", help="push to a git url on success",
 864                   default=None, type='str')
 865 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 866                   default=False, action="store_true")
 867 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 868                   default=False, action="store_true")
 869 parser.add_option("", "--retry", help="automatically retry if master changes",
 870                   default=False, action="store_true")
 871 parser.add_option("", "--email", help="send email to the given address on failure",
 872                   type='str', default=None)
 873 parser.add_option("", "--email-from", help="send email from the given address",
 874                   type='str', default="autobuild@samba.org")
 875 parser.add_option("", "--email-server", help="send email via the given server",
 876                   type='str', default='localhost')
 877 parser.add_option("", "--always-email", help="always send email, even on success",
 878                   action="store_true")
 879 parser.add_option("", "--daemon", help="daemonize after initial setup",
 880                   action="store_true")
 881 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 882                   default="master", type='str')
 883 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 884                   default=gitroot, type='str')
 885 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 886                   default=False, action="store_true")
 887 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 888                   default='')
 889
 890
 891 def send_email(subject, text, log_tar):
 892     if options.email is None:
 893         do_print("not sending email because the recipient is not set")
 894         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
 895                  (subject, text))
 896         return
 897     outer = MIMEMultipart()
 898     outer['Subject'] = subject
 899     outer['To'] = options.email
 900     outer['From'] = options.email_from
 901     outer['Date'] = email.utils.formatdate(localtime=True)
 902     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 903     outer.attach(MIMEText(text, 'plain'))
 904     if options.attach_logs:
 905         fp = open(log_tar, 'rb')
 906         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 907         fp.close()
 908         # Set the filename parameter
 909         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 910         outer.attach(msg)
 911     content = outer.as_string()
 912     s = smtplib.SMTP(options.email_server)
 913     email_user = os.getenv('SMTP_USERNAME')
 914     email_password = os.getenv('SMTP_PASSWORD')
 915     if email_user is not None:
 916         s.starttls()
 917         s.login(email_user, email_password)
 918
 919     s.sendmail(options.email_from, [options.email], content)
 920     s.set_debuglevel(1)
 921     s.quit()
 922
 923
 924 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 925                   elapsed_time, log_base=None, add_log_tail=True):
 926     '''send an email to options.email about the failure'''
 927     elapsed_minutes = elapsed_time / 60.0
 928     if log_base is None:
 929         log_base = gitroot
 930     text = '''
 931 Dear Developer,
 932
 933 Your autobuild on %s failed after %.1f minutes
 934 when trying to test %s with the following error:
 935
 936    %s
 937
 938 the autobuild has been abandoned. Please fix the error and resubmit.
 939
 940 A summary of the autobuild process is here:
 941
 942   %s/autobuild.log
 943 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 944
 945     if options.restrict_tests:
 946         text += """
 947 The build was restricted to tests matching %s\n""" % options.restrict_tests
 948
 949     if failed_task != 'rebase':
 950         text += '''
 951 You can see logs of the failed task here:
 952
 953   %s/%s.stdout
 954   %s/%s.stderr
 955
 956 or you can get full logs of all tasks in this job here:
 957
 958   %s/logs.tar.gz
 959
 960 The top commit for the tree that was built was:
 961
 962 %s
 963
 964 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 965
 966     if add_log_tail:
 967         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 968         lines = f.readlines()
 969         log_tail = "".join(lines[-50:])
 970         num_lines = len(lines)
 971         if num_lines < 50:
 972             # Also include stderr (compile failures) if < 50 lines of stdout
 973             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 974             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 975
 976         text += '''
 977 The last 50 lines of log messages:
 978
 979 %s
 980     ''' % log_tail
 981         f.close()
 982
 983     logs = os.path.join(gitroot, 'logs.tar.gz')
 984     send_email('autobuild[%s] failure on %s for task %s during %s'
 985                % (options.branch, platform.node(), failed_task, failed_stage),
 986                text, logs)
 987
 988
 989 def email_success(elapsed_time, log_base=None):
 990     '''send an email to options.email about a successful build'''
 991     if log_base is None:
 992         log_base = gitroot
 993     text = '''
 994 Dear Developer,
 995
 996 Your autobuild on %s has succeeded after %.1f minutes.
 997
 998 ''' % (platform.node(), elapsed_time / 60.)
 999
1000     if options.restrict_tests:
1001         text += """
1002 The build was restricted to tests matching %s\n""" % options.restrict_tests
1003
1004     if options.keeplogs:
1005         text += '''
1006
1007 you can get full logs of all tasks in this job here:
1008
1009   %s/logs.tar.gz
1010
1011 ''' % log_base
1012
1013     text += '''
1014 The top commit for the tree that was built was:
1015
1016 %s
1017 ''' % top_commit_msg
1018
1019     logs = os.path.join(gitroot, 'logs.tar.gz')
1020     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1021                text, logs)
1022
1023
1024 (options, args) = parser.parse_args()
1025
1026 if options.retry:
1027     if options.rebase is None:
1028         raise Exception('You can only use --retry if you also rebase')
1029
1030 testbase = "%s/b%u" % (options.testbase, os.getpid())
1031 test_master = "%s/master" % testbase
1032 test_prefix = "%s/prefix" % testbase
1033 test_tmpdir = "%s/tmp" % testbase
1034 os.environ['TMPDIR'] = test_tmpdir
1035
1036 # get the top commit message, for emails
1037 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1038 top_commit_msg = top_commit_msg.decode('utf-8', 'backslashreplace')
1039
1040 try:
1041     os.makedirs(testbase)
1042 except Exception as reason:
1043     raise Exception("Unable to create %s : %s" % (testbase, reason))
1044 cleanup_list.append(testbase)
1045
1046 if options.daemon:
1047     logfile = os.path.join(testbase, "log")
1048     do_print("Forking into the background, writing progress to %s" % logfile)
1049     daemonize(logfile)
1050
1051 write_pidfile(gitroot + "/autobuild.pid")
1052
1053 start_time = time.time()
1054
1055 while True:
1056     try:
1057         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1058         os.makedirs(test_tmpdir)
1059         # The waf uninstall code removes empty directories all the way
1060         # up the tree.  Creating a file in test_tmpdir stops it from
1061         # being removed.
1062         run_cmd("touch %s" % os.path.join(test_tmpdir,
1063                                           ".directory-is-not-empty"), show=True)
1064         run_cmd("stat %s" % test_tmpdir, show=True)
1065         run_cmd("stat %s" % testbase, show=True)
1066         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1067     except Exception:
1068         cleanup()
1069         raise
1070
1071     try:
1072         try:
1073             if options.rebase is not None:
1074                 rebase_tree(options.rebase, rebase_branch=options.branch)
1075         except Exception:
1076             cleanup_list.append(gitroot + "/autobuild.pid")
1077             cleanup()
1078             elapsed_time = time.time() - start_time
1079             email_failure(-1, 'rebase', 'rebase', 'rebase',
1080                           'rebase on %s failed' % options.branch,
1081                           elapsed_time, log_base=options.log_base)
1082             sys.exit(1)
1083         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1084         if options.tail:
1085             blist.start_tail()
1086         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1087         if status != 0 or errstr != "retry":
1088             break
1089         cleanup()
1090     except Exception:
1091         cleanup()
1092         raise
1093
1094 cleanup_list.append(gitroot + "/autobuild.pid")
1095
1096 do_print(errstr)
1097
1098 blist.kill_kids()
1099 if options.tail:
1100     do_print("waiting for tail to flush")
1101     time.sleep(1)
1102
1103 elapsed_time = time.time() - start_time
1104 if status == 0:
1105     if options.passcmd is not None:
1106         do_print("Running passcmd: %s" % options.passcmd)
1107         run_cmd(options.passcmd, dir=test_master)
1108     if options.pushto is not None:
1109         push_to(options.pushto, push_branch=options.branch)
1110     if options.keeplogs or options.attach_logs:
1111         blist.tarlogs("logs.tar.gz")
1112         do_print("Logs in logs.tar.gz")
1113     if options.always_email:
1114         email_success(elapsed_time, log_base=options.log_base)
1115     blist.remove_logs()
1116     cleanup()
1117     do_print(errstr)
1118     sys.exit(0)
1119
1120 # something failed, gather a tar of the logs
1121 blist.tarlogs("logs.tar.gz")
1122
1123 if options.email is not None:
1124     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1125                   elapsed_time, log_base=options.log_base)
1126 else:
1127     elapsed_minutes = elapsed_time / 60.0
1128     print('''
1129
1130 ####################################################################
1131
1132 AUTOBUILD FAILURE
1133
1134 Your autobuild[%s] on %s failed after %.1f minutes
1135 when trying to test %s with the following error:
1136
1137    %s
1138
1139 the autobuild has been abandoned. Please fix the error and resubmit.
1140
1141 ####################################################################
1142
1143 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1144
1145 cleanup()
1146 do_print(errstr)
1147 do_print("Logs in logs.tar.gz")
1148 sys.exit(status)