script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 os.environ["PYTHONUNBUFFERED"] = "1"
  23
  24 # This speeds up testing remarkably.
  25 os.environ['TDB_NO_FSYNC'] = '1'
  26
  27 cleanup_list = []
  28
  29 builddirs = {
  30     "ctdb": "ctdb",
  31     "samba": ".",
  32     "samba-py3": ".",
  33     "samba-nt4": ".",
  34     "samba-nt4-py3": ".",
  35     "samba-fileserver": ".",
  36     "samba-xc": ".",
  37     "samba-o3": ".",
  38     "samba-ctdb": ".",
  39     "samba-libs": ".",
  40     "samba-libs-py3": ".",
  41     "samba-static": ".",
  42     "samba-none-env": ".",
  43     "samba-none-env-py3": ".",
  44     "samba-ad-dc": ".",
  45     "samba-ad-dc-py3": ".",
  46     "samba-ad-dc-2": ".",
  47     "samba-ad-dc-2-py3": ".",
  48     "samba-systemkrb5": ".",
  49     "samba-nopython": ".",
  50     "ldb": "lib/ldb",
  51     "tdb": "lib/tdb",
  52     "talloc": "lib/talloc",
  53     "replace": "lib/replace",
  54     "tevent": "lib/tevent",
  55     "pidl": "pidl"
  56 }
  57
  58 defaulttasks = builddirs.keys()
  59
  60 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  61     defaulttasks.remove("samba-o3")
  62
  63 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  64 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  65
  66 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  67 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  68 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  69 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  70 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  71 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  72 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  73
  74 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  75     extra_python = ""
  76 else:
  77     extra_python = "--extra-python=/usr/bin/python3"
  78
  79 tasks = {
  80     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  81                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  82                ("make", "make all", "text/plain"),
  83                ("install", "make install", "text/plain"),
  84                ("test", "make autotest", "text/plain"),
  85                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  86                ("clean", "make clean", "text/plain")],
  87
  88     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
  89     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  90                 ("make", "make -j", "text/plain"),
  91                 ("test", "make test FAIL_IMMEDIATELY=1 "
  92                  "TESTS='${PY3_ONLY}"
  93                  "--exclude-env=none "
  94                  "--exclude-env=nt4_dc "
  95                  "--exclude-env=nt4_member "
  96                  "--exclude-env=ad_dc "
  97                  "--exclude-env=ad_dc_no_nss "
  98                  "--exclude-env=fl2003dc "
  99                  "--exclude-env=fl2008r2dc "
 100                  "--exclude-env=ad_member "
 101                  "--exclude-env=ad_member_idmap_rid "
 102                  "--exclude-env=ad_member_idmap_ad "
 103                  "--exclude-env=chgdcpass "
 104                  "--exclude-env=vampire_2000_dc "
 105                  "--exclude-env=fl2000dc "
 106                  "--exclude-env=fileserver "
 107                  "--exclude-env=backupfromdc "
 108                  "--exclude-env=restoredc "
 109                  "--exclude-env=renamedc "
 110                  "--exclude-env=offlinebackupdc "
 111                  "--exclude-env=labdc "
 112                  "'",
 113                  "text/plain"),
 114                 ("install", "make install", "text/plain"),
 115                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 116                 ("clean", "make clean", "text/plain")],
 117
 118     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 119     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 120                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 121                     ("make", "make -j", "text/plain"),
 122                     ("test", "make test FAIL_IMMEDIATELY=1 "
 123                      "TESTS='${PY3_ONLY}"
 124                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 125                     ("install", "make install", "text/plain"),
 126                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 127                     ("clean", "make clean", "text/plain")],
 128
 129     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 130     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 131                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json-audit --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 132                            ("make", "make -j", "text/plain"),
 133                            ("test", "make test FAIL_IMMEDIATELY=1 "
 134                             "TESTS='${PY3_ONLY}"
 135                             "--include-env=fileserver'", "text/plain"),
 136                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 137
 138     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 139     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 140                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 141                       ("make", "make -j", "text/plain"),
 142                       ("test", "make test FAIL_IMMEDIATELY=1 "
 143                        "TESTS='${PY3_ONLY}"
 144                        "--include-env=ad_dc "
 145                        "--include-env=fl2003dc "
 146                        "--include-env=fl2008r2dc "
 147                        "--include-env=ad_member "
 148                        "--include-env=ad_member_idmap_rid "
 149                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 150                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 151
 152     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 153     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 154                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 155                         ("make", "make -j", "text/plain"),
 156                         ("test", "make test FAIL_IMMEDIATELY=1 "
 157                          "TESTS='${PY3_ONLY}"
 158                          "--include-env=chgdcpass "
 159                          "--include-env=vampire_2000_dc "
 160                          "--include-env=fl2000dc "
 161                          "--include-env=ad_dc_no_nss "
 162                          "--include-env=backupfromdc "
 163                          "--include-env=restoredc "
 164                          "--include-env=renamedc "
 165                          "--include-env=offlinebackupdc "
 166                          "--include-env=labdc "
 167                          "'",
 168                          "text/plain"),
 169                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 170
 171     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 172                           ("make", "make -j", "text/plain"),
 173                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 174
 175     # Test cross-compile infrastructure
 176     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 177                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 178                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 179                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 180                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 181                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 182                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 183
 184     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 185     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 186                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 187                    ("make", "make -j", "text/plain"),
 188                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 189                     "TESTS='${PY3_ONLY}"
 190                     "--include-env=ad_dc'", "text/plain"),
 191                    ("install", "make install", "text/plain"),
 192                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 193                    ("clean", "make clean", "text/plain")],
 194
 195     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 196
 197                      # make sure we have tdb around:
 198                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 199                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 200                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 201
 202
 203                      # build samba with cluster support (also building ctdb):
 204                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 205                      ("samba-make", "make", "text/plain"),
 206                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 207                      ("samba-install", "make install", "text/plain"),
 208                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 209
 210                      # clean up:
 211                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 212                      ("clean", "make clean", "text/plain"),
 213                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 214
 215     "samba-libs": [
 216                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 217                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 218                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 219                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 220
 221                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 222                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 223                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 224
 225                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 226                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 227                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 228
 229                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 230                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 231                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 232
 233                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 234                       ("nondevel-make", "make -j", "text/plain"),
 235                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 236                       ("nondevel-install", "make install", "text/plain"),
 237                       ("nondevel-dist", "make dist", "text/plain"),
 238
 239                       # retry with all modules shared
 240                       ("allshared-distclean", "make distclean", "text/plain"),
 241                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 242                       ("allshared-make", "make -j", "text/plain")],
 243
 244     "samba-none-env": [
 245                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 246                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 247                       ("make", "make -j", "text/plain"),
 248                       ("test", "make test "
 249                        "FAIL_IMMEDIATELY=1 "
 250                        "TESTS='${PY3_ONLY}"
 251                        "--include-env=none'",
 252                        "text/plain")],
 253
 254     "samba-static": [
 255                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 256                       # build with all modules static
 257                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 258                       ("allstatic-make", "make -j", "text/plain"),
 259                       ("allstatic-test", "make test "
 260                        "FAIL_IMMEDIATELY=1 "
 261                        "TESTS='samba3.smb2.create.*nt4_dc'",
 262                        "text/plain"),
 263
 264                       # retry without any required modules
 265                       ("none-distclean", "make distclean", "text/plain"),
 266                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 267                       ("none-make", "make -j", "text/plain"),
 268
 269                       # retry with nonshared smbd and smbtorture
 270                       ("nonshared-distclean", "make distclean", "text/plain"),
 271                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 272                       ("nonshared-make", "make -j", "text/plain")],
 273
 274     "samba-systemkrb5": [
 275                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 276                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 277                       ("make", "make -j", "text/plain"),
 278                       # we currently cannot run a full make test, a limited list of tests could be run
 279                       # via "make test TESTS=sometests"
 280                       ("test", "make test FAIL_IMMEDIATELY=1 "
 281                        "TESTS='${PY3_ONLY}"
 282                        "--include-env=ktest'", "text/plain"),
 283                       ("install", "make install", "text/plain"),
 284                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 285                       ("clean", "make clean", "text/plain")
 286                       ],
 287
 288     # Test Samba without python still builds.  When this test fails
 289     # due to more use of Python, the expectations is that the newly
 290     # failing part of the code should be disabled when
 291     # --disable-python is set (rather than major work being done to
 292     # support this environment).  The target here is for vendors
 293     # shipping a minimal smbd.
 294     "samba-nopython": [
 295                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 296                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 297                       ("make", "make -j", "text/plain"),
 298                       ("install", "make install", "text/plain"),
 299                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 300                       ("clean", "make clean", "text/plain"),
 301
 302                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 303                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 304                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 305
 306                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 307                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 308                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 309
 310                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 311                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 312                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 313
 314                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 315                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 316                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 317
 318                       # retry against installed library packages
 319                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 320                       ("libs-make", "make -j", "text/plain"),
 321                       ("libs-install", "make install", "text/plain"),
 322                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 323                       ("libs-clean", "make clean", "text/plain")
 324                       ],
 325
 326
 327
 328     "ldb": [
 329               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 330               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 331               ("make", "make", "text/plain"),
 332               ("install", "make install", "text/plain"),
 333               ("test", "make test", "text/plain"),
 334               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 335               ("make-no-lmdb", "make", "text/plain"),
 336               ("install-no-lmdb", "make install", "text/plain"),
 337               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 338               ("distcheck", "make distcheck", "text/plain"),
 339               ("clean", "make clean", "text/plain")],
 340
 341     "tdb": [
 342               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 343               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 344               ("make", "make", "text/plain"),
 345               ("install", "make install", "text/plain"),
 346               ("test", "make test", "text/plain"),
 347               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 348               ("distcheck", "make distcheck", "text/plain"),
 349               ("clean", "make clean", "text/plain")],
 350
 351     "talloc": [
 352                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 353                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 354                  ("make", "make", "text/plain"),
 355                  ("install", "make install", "text/plain"),
 356                  ("test", "make test", "text/plain"),
 357                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 358                  ("distcheck", "make distcheck", "text/plain"),
 359                  ("clean", "make clean", "text/plain")],
 360
 361     "replace": [
 362                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 363                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 364                   ("make", "make", "text/plain"),
 365                   ("install", "make install", "text/plain"),
 366                   ("test", "make test", "text/plain"),
 367                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 368                   ("distcheck", "make distcheck", "text/plain"),
 369                   ("clean", "make clean", "text/plain")],
 370
 371     "tevent": [
 372                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 373                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 374                  ("make", "make", "text/plain"),
 375                  ("install", "make install", "text/plain"),
 376                  ("test", "make test", "text/plain"),
 377                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 378                  ("distcheck", "make distcheck", "text/plain"),
 379                  ("clean", "make clean", "text/plain")],
 380
 381     "pidl": [
 382         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 383         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 384         ("touch", "touch *.yp", "text/plain"),
 385         ("make", "make", "text/plain"),
 386         ("test", "make test", "text/plain"),
 387         ("install", "make install", "text/plain"),
 388         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 389         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 390         ("clean", "make clean", "text/plain")],
 391
 392     # these are useful for debugging autobuild
 393     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 394     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 395 }
 396
 397
 398 def do_print(msg):
 399     print("%s" % msg)
 400     sys.stdout.flush()
 401     sys.stderr.flush()
 402
 403
 404 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 405     if show is None:
 406         show = options.verbose
 407     if show:
 408         do_print("Running: '%s' in '%s'" % (cmd, dir))
 409     if output:
 410         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir, close_fds=True).communicate()[0]
 411     elif checkfail:
 412         return check_call(cmd, shell=True, cwd=dir)
 413     else:
 414         return call(cmd, shell=True, cwd=dir)
 415
 416
 417 class builder(object):
 418     '''handle build of one directory'''
 419
 420     def __init__(self, name, sequence, cp=True, py3=False):
 421         self.name = name
 422         self.py3 = py3
 423         if name in builddirs:
 424             self.dir = builddirs[name]
 425         else:
 426             self.dir = "."
 427
 428         self.tag = self.name.replace('/', '_')
 429         self.sequence = sequence
 430         self.next = 0
 431         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 432         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 433         if options.verbose:
 434             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 435             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 436         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 437         self.stdout = open(self.stdout_path, 'w')
 438         self.stderr = open(self.stderr_path, 'w')
 439         self.stdin  = open("/dev/null", 'r')
 440         self.sdir = "%s/%s" % (testbase, self.tag)
 441         self.prefix = "%s/%s" % (test_prefix, self.tag)
 442         run_cmd("rm -rf %s" % self.sdir)
 443         run_cmd("rm -rf %s" % self.prefix)
 444         if cp:
 445             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 446         else:
 447             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 448         self.start_next()
 449
 450     def start_next(self):
 451         if self.next == len(self.sequence):
 452             if not options.nocleanup:
 453                 run_cmd("rm -rf %s" % self.sdir)
 454                 run_cmd("rm -rf %s" % self.prefix)
 455             do_print('%s: Completed OK' % self.name)
 456             self.done = True
 457             return
 458         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 459         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 460         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 461         if self.py3:
 462             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 463             # The trailing space is important
 464             self.cmd = self.cmd.replace("${PY3_ONLY}", "python3 ")
 465         else:
 466             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 467             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 468         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 469         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 470 #        if self.output_mime_type == "text/x-subunit":
 471 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 472         cwd = os.getcwd()
 473         os.chdir("%s/%s" % (self.sdir, self.dir))
 474         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, os.getcwd()))
 475         self.proc = Popen(self.cmd, shell=True,
 476                           close_fds=True,
 477                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 478         os.chdir(cwd)
 479         self.next += 1
 480
 481
 482 class buildlist(object):
 483     '''handle build of multiple directories'''
 484
 485     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 486         global tasks
 487         self.tlist = []
 488         self.tail_proc = None
 489         self.retry = None
 490         if tasknames == []:
 491             if options.restrict_tests:
 492                 tasknames = ["samba-test-only"]
 493             else:
 494                 tasknames = defaulttasks
 495         else:
 496             # If we are only running one test,
 497             # do not sleep randomly to wait for it to start
 498             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 499
 500         for n in tasknames:
 501             if n not in tasks and n.endswith("-py3"):
 502                 b = builder(n,
 503                             tasks[n[:-4]],
 504                             cp=n is not "pidl",
 505                             py3=True)
 506             else:
 507                 b = builder(n, tasks[n], cp=n is not "pidl")
 508             self.tlist.append(b)
 509         if options.retry:
 510             rebase_remote = "rebaseon"
 511             retry_task = [("retry",
 512                             '''set -e
 513                             git remote add -t %s %s %s
 514                             git fetch %s
 515                             while :; do
 516                               sleep 60
 517                               git describe %s/%s > old_remote_branch.desc
 518                               git fetch %s
 519                               git describe %s/%s > remote_branch.desc
 520                               diff old_remote_branch.desc remote_branch.desc
 521                             done
 522                            ''' % (
 523                                 rebase_branch, rebase_remote, rebase_url,
 524                                rebase_remote,
 525                                rebase_remote, rebase_branch,
 526                                rebase_remote,
 527                                rebase_remote, rebase_branch
 528                             ),
 529                             "test/plain")]
 530
 531             self.retry = builder('retry', retry_task, cp=False)
 532             self.need_retry = False
 533
 534     def kill_kids(self):
 535         if self.tail_proc is not None:
 536             self.tail_proc.terminate()
 537             self.tail_proc.wait()
 538             self.tail_proc = None
 539         if self.retry is not None:
 540             self.retry.proc.terminate()
 541             self.retry.proc.wait()
 542             self.retry = None
 543         for b in self.tlist:
 544             if b.proc is not None:
 545                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 546                 b.proc.terminate()
 547                 b.proc.wait()
 548                 b.proc = None
 549
 550     def wait_one(self):
 551         while True:
 552             none_running = True
 553             for b in self.tlist:
 554                 if b.proc is None:
 555                     continue
 556                 none_running = False
 557                 b.status = b.proc.poll()
 558                 if b.status is None:
 559                     continue
 560                 b.proc = None
 561                 return b
 562             if options.retry:
 563                 ret = self.retry.proc.poll()
 564                 if ret is not None:
 565                     self.need_retry = True
 566                     self.retry = None
 567                     return None
 568             if none_running:
 569                 return None
 570             time.sleep(0.1)
 571
 572     def run(self):
 573         while True:
 574             b = self.wait_one()
 575             if options.retry and self.need_retry:
 576                 self.kill_kids()
 577                 do_print("retry needed")
 578                 return (0, None, None, None, "retry")
 579             if b is None:
 580                 break
 581             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 582                 self.kill_kids()
 583                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 584             b.start_next()
 585         self.kill_kids()
 586         return (0, None, None, None, "All OK")
 587
 588     def write_system_info(self):
 589         filename = 'system-info.txt'
 590         f = open(filename, 'w')
 591         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 592                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 593             print('### %s' % cmd, file=f)
 594             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 595             print(file=f)
 596         f.close()
 597         return filename
 598
 599     def tarlogs(self, fname):
 600         tar = tarfile.open(fname, "w:gz")
 601         for b in self.tlist:
 602             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 603             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 604         if os.path.exists("autobuild.log"):
 605             tar.add("autobuild.log")
 606         sys_info = self.write_system_info()
 607         tar.add(sys_info)
 608         tar.close()
 609
 610     def remove_logs(self):
 611         for b in self.tlist:
 612             os.unlink(b.stdout_path)
 613             os.unlink(b.stderr_path)
 614
 615     def start_tail(self):
 616         cmd = ["tail", "-f"]
 617         for b in self.tlist:
 618             cmd.append(b.stdout_path)
 619             cmd.append(b.stderr_path)
 620         self.tail_proc = Popen(cmd, close_fds=True)
 621
 622
 623 def cleanup():
 624     if options.nocleanup:
 625         return
 626     run_cmd("stat %s || true" % test_tmpdir, show=True)
 627     run_cmd("stat %s" % testbase, show=True)
 628     do_print("Cleaning up %r" % cleanup_list)
 629     for d in cleanup_list:
 630         run_cmd("rm -rf %s" % d)
 631
 632
 633 def find_git_root():
 634     '''get to the top of the git repo'''
 635     p = os.getcwd()
 636     while p != '/':
 637         if os.path.isdir(os.path.join(p, ".git")):
 638             return p
 639         p = os.path.abspath(os.path.join(p, '..'))
 640     return None
 641
 642
 643 def daemonize(logfile):
 644     pid = os.fork()
 645     if pid == 0:  # Parent
 646         os.setsid()
 647         pid = os.fork()
 648         if pid != 0:  # Actual daemon
 649             os._exit(0)
 650     else:  # Grandparent
 651         os._exit(0)
 652
 653     import resource      # Resource usage information.
 654     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 655     if maxfd == resource.RLIM_INFINITY:
 656         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 657     for fd in range(0, maxfd):
 658         try:
 659             os.close(fd)
 660         except OSError:
 661             pass
 662     os.open(logfile, os.O_RDWR | os.O_CREAT)
 663     os.dup2(0, 1)
 664     os.dup2(0, 2)
 665
 666
 667 def write_pidfile(fname):
 668     '''write a pid file, cleanup on exit'''
 669     f = open(fname, mode='w')
 670     f.write("%u\n" % os.getpid())
 671     f.close()
 672
 673
 674 def rebase_tree(rebase_url, rebase_branch="master"):
 675     rebase_remote = "rebaseon"
 676     do_print("Rebasing on %s" % rebase_url)
 677     run_cmd("git describe HEAD", show=True, dir=test_master)
 678     run_cmd("git remote add -t %s %s %s" %
 679             (rebase_branch, rebase_remote, rebase_url),
 680             show=True, dir=test_master)
 681     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 682     if options.fix_whitespace:
 683         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 684                 (rebase_remote, rebase_branch),
 685                 show=True, dir=test_master)
 686     else:
 687         run_cmd("git rebase --force-rebase %s/%s" %
 688                 (rebase_remote, rebase_branch),
 689                 show=True, dir=test_master)
 690     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 691                    (rebase_remote, rebase_branch),
 692                    dir=test_master, output=True)
 693     if diff == '':
 694         do_print("No differences between HEAD and %s/%s - exiting" %
 695                  (rebase_remote, rebase_branch))
 696         sys.exit(0)
 697     run_cmd("git describe %s/%s" %
 698             (rebase_remote, rebase_branch),
 699             show=True, dir=test_master)
 700     run_cmd("git describe HEAD", show=True, dir=test_master)
 701     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 702             (rebase_remote, rebase_branch),
 703             show=True, dir=test_master)
 704
 705
 706 def push_to(push_url, push_branch="master"):
 707     push_remote = "pushto"
 708     do_print("Pushing to %s" % push_url)
 709     if options.mark:
 710         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 711         run_cmd("git commit --amend -c HEAD", dir=test_master)
 712         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 713         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 714     run_cmd("git remote add -t %s %s %s" %
 715             (push_branch, push_remote, push_url),
 716             show=True, dir=test_master)
 717     run_cmd("git push %s +HEAD:%s" %
 718             (push_remote, push_branch),
 719             show=True, dir=test_master)
 720
 721
 722 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 723
 724 gitroot = find_git_root()
 725 if gitroot is None:
 726     raise Exception("Failed to find git root")
 727
 728 parser = OptionParser()
 729 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 730 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 731 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 732 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 733                   default=def_testbase)
 734 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 735 parser.add_option("", "--verbose", help="show all commands as they are run",
 736                   default=False, action="store_true")
 737 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 738                   default=None, type='str')
 739 parser.add_option("", "--pushto", help="push to a git url on success",
 740                   default=None, type='str')
 741 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 742                   default=False, action="store_true")
 743 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 744                   default=False, action="store_true")
 745 parser.add_option("", "--retry", help="automatically retry if master changes",
 746                   default=False, action="store_true")
 747 parser.add_option("", "--email", help="send email to the given address on failure",
 748                   type='str', default=None)
 749 parser.add_option("", "--email-from", help="send email from the given address",
 750                   type='str', default="autobuild@samba.org")
 751 parser.add_option("", "--email-server", help="send email via the given server",
 752                   type='str', default='localhost')
 753 parser.add_option("", "--always-email", help="always send email, even on success",
 754                   action="store_true")
 755 parser.add_option("", "--daemon", help="daemonize after initial setup",
 756                   action="store_true")
 757 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 758                   default="master", type='str')
 759 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 760                   default=gitroot, type='str')
 761 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 762                   default=False, action="store_true")
 763 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 764                   default='')
 765
 766
 767 def send_email(subject, text, log_tar):
 768     if options.email is None:
 769         do_print("not sending email because the recipient is not set")
 770         do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
 771                  (subject, text))
 772         return
 773     outer = MIMEMultipart()
 774     outer['Subject'] = subject
 775     outer['To'] = options.email
 776     outer['From'] = options.email_from
 777     outer['Date'] = email.utils.formatdate(localtime=True)
 778     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 779     outer.attach(MIMEText(text, 'plain'))
 780     if options.attach_logs:
 781         fp = open(log_tar, 'rb')
 782         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 783         fp.close()
 784         # Set the filename parameter
 785         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 786         outer.attach(msg)
 787     content = outer.as_string()
 788     s = smtplib.SMTP(options.email_server)
 789     s.sendmail(options.email_from, [options.email], content)
 790     s.set_debuglevel(1)
 791     s.quit()
 792
 793
 794 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 795                   elapsed_time, log_base=None, add_log_tail=True):
 796     '''send an email to options.email about the failure'''
 797     elapsed_minutes = elapsed_time / 60.0
 798     user = os.getenv("USER")
 799     if log_base is None:
 800         log_base = gitroot
 801     text = '''
 802 Dear Developer,
 803
 804 Your autobuild on %s failed after %.1f minutes
 805 when trying to test %s with the following error:
 806
 807    %s
 808
 809 the autobuild has been abandoned. Please fix the error and resubmit.
 810
 811 A summary of the autobuild process is here:
 812
 813   %s/autobuild.log
 814 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 815
 816     if options.restrict_tests:
 817         text += """
 818 The build was restricted to tests matching %s\n""" % options.restrict_tests
 819
 820     if failed_task != 'rebase':
 821         text += '''
 822 You can see logs of the failed task here:
 823
 824   %s/%s.stdout
 825   %s/%s.stderr
 826
 827 or you can get full logs of all tasks in this job here:
 828
 829   %s/logs.tar.gz
 830
 831 The top commit for the tree that was built was:
 832
 833 %s
 834
 835 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 836
 837     if add_log_tail:
 838         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 839         lines = f.readlines()
 840         log_tail = "".join(lines[-50:])
 841         num_lines = len(lines)
 842         if num_lines < 50:
 843             # Also include stderr (compile failures) if < 50 lines of stdout
 844             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 845             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 846
 847         text += '''
 848 The last 50 lines of log messages:
 849
 850 %s
 851     ''' % log_tail
 852         f.close()
 853
 854     logs = os.path.join(gitroot, 'logs.tar.gz')
 855     send_email('autobuild[%s] failure on %s for task %s during %s'
 856                % (options.branch, platform.node(), failed_task, failed_stage),
 857                text, logs)
 858
 859
 860 def email_success(elapsed_time, log_base=None):
 861     '''send an email to options.email about a successful build'''
 862     user = os.getenv("USER")
 863     if log_base is None:
 864         log_base = gitroot
 865     text = '''
 866 Dear Developer,
 867
 868 Your autobuild on %s has succeeded after %.1f minutes.
 869
 870 ''' % (platform.node(), elapsed_time / 60.)
 871
 872     if options.restrict_tests:
 873         text += """
 874 The build was restricted to tests matching %s\n""" % options.restrict_tests
 875
 876     if options.keeplogs:
 877         text += '''
 878
 879 you can get full logs of all tasks in this job here:
 880
 881   %s/logs.tar.gz
 882
 883 ''' % log_base
 884
 885     text += '''
 886 The top commit for the tree that was built was:
 887
 888 %s
 889 ''' % top_commit_msg
 890
 891     logs = os.path.join(gitroot, 'logs.tar.gz')
 892     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 893                text, logs)
 894
 895
 896 (options, args) = parser.parse_args()
 897
 898 if options.retry:
 899     if options.rebase is None:
 900         raise Exception('You can only use --retry if you also rebase')
 901
 902 testbase = "%s/b%u" % (options.testbase, os.getpid())
 903 test_master = "%s/master" % testbase
 904 test_prefix = "%s/prefix" % testbase
 905 test_tmpdir = "%s/tmp" % testbase
 906 os.environ['TMPDIR'] = test_tmpdir
 907
 908 # get the top commit message, for emails
 909 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 910
 911 try:
 912     os.makedirs(testbase)
 913 except Exception as reason:
 914     raise Exception("Unable to create %s : %s" % (testbase, reason))
 915 cleanup_list.append(testbase)
 916
 917 if options.daemon:
 918     logfile = os.path.join(testbase, "log")
 919     do_print("Forking into the background, writing progress to %s" % logfile)
 920     daemonize(logfile)
 921
 922 write_pidfile(gitroot + "/autobuild.pid")
 923
 924 start_time = time.time()
 925
 926 while True:
 927     try:
 928         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 929         os.makedirs(test_tmpdir)
 930         # The waf uninstall code removes empty directories all the way
 931         # up the tree.  Creating a file in test_tmpdir stops it from
 932         # being removed.
 933         run_cmd("touch %s" % os.path.join(test_tmpdir,
 934                                           ".directory-is-not-empty"), show=True)
 935         run_cmd("stat %s" % test_tmpdir, show=True)
 936         run_cmd("stat %s" % testbase, show=True)
 937         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 938     except Exception:
 939         cleanup()
 940         raise
 941
 942     try:
 943         try:
 944             if options.rebase is not None:
 945                 rebase_tree(options.rebase, rebase_branch=options.branch)
 946         except Exception:
 947             cleanup_list.append(gitroot + "/autobuild.pid")
 948             cleanup()
 949             elapsed_time = time.time() - start_time
 950             email_failure(-1, 'rebase', 'rebase', 'rebase',
 951                           'rebase on %s failed' % options.branch,
 952                           elapsed_time, log_base=options.log_base)
 953             sys.exit(1)
 954         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 955         if options.tail:
 956             blist.start_tail()
 957         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 958         if status != 0 or errstr != "retry":
 959             break
 960         cleanup()
 961     except Exception:
 962         cleanup()
 963         raise
 964
 965 cleanup_list.append(gitroot + "/autobuild.pid")
 966
 967 do_print(errstr)
 968
 969 blist.kill_kids()
 970 if options.tail:
 971     do_print("waiting for tail to flush")
 972     time.sleep(1)
 973
 974 elapsed_time = time.time() - start_time
 975 if status == 0:
 976     if options.passcmd is not None:
 977         do_print("Running passcmd: %s" % options.passcmd)
 978         run_cmd(options.passcmd, dir=test_master)
 979     if options.pushto is not None:
 980         push_to(options.pushto, push_branch=options.branch)
 981     if options.keeplogs or options.attach_logs:
 982         blist.tarlogs("logs.tar.gz")
 983         do_print("Logs in logs.tar.gz")
 984     if options.always_email:
 985         email_success(elapsed_time, log_base=options.log_base)
 986     blist.remove_logs()
 987     cleanup()
 988     do_print(errstr)
 989     sys.exit(0)
 990
 991 # something failed, gather a tar of the logs
 992 blist.tarlogs("logs.tar.gz")
 993
 994 if options.email is not None:
 995     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 996                   elapsed_time, log_base=options.log_base)
 997 else:
 998     elapsed_minutes = elapsed_time / 60.0
 999     print('''
1000
1001 ####################################################################
1002
1003 AUTOBUILD FAILURE
1004
1005 Your autobuild[%s] on %s failed after %.1f minutes
1006 when trying to test %s with the following error:
1007
1008    %s
1009
1010 the autobuild has been abandoned. Please fix the error and resubmit.
1011
1012 ####################################################################
1013
1014 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1015
1016 cleanup()
1017 do_print(errstr)
1018 do_print("Logs in logs.tar.gz")
1019 sys.exit(status)