script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 os.environ["PYTHONUNBUFFERED"] = "1"
  23
  24 # This speeds up testing remarkably.
  25 os.environ['TDB_NO_FSYNC'] = '1'
  26
  27 cleanup_list = []
  28
  29 builddirs = {
  30     "ctdb": "ctdb",
  31     "samba": ".",
  32     "samba-py3": ".",
  33     "samba-nt4": ".",
  34     "samba-nt4-py3": ".",
  35     "samba-fileserver": ".",
  36     "samba-xc": ".",
  37     "samba-o3": ".",
  38     "samba-ctdb": ".",
  39     "samba-libs": ".",
  40     "samba-libs-py3": ".",
  41     "samba-static": ".",
  42     "samba-none-env": ".",
  43     "samba-none-env-py3": ".",
  44     "samba-ad-dc": ".",
  45     "samba-ad-dc-py3": ".",
  46     "samba-ad-dc-2": ".",
  47     "samba-ad-dc-2-py3": ".",
  48     "samba-systemkrb5": ".",
  49     "samba-nopython": ".",
  50     "ldb": "lib/ldb",
  51     "tdb": "lib/tdb",
  52     "talloc": "lib/talloc",
  53     "replace": "lib/replace",
  54     "tevent": "lib/tevent",
  55     "pidl": "pidl"
  56 }
  57
  58 defaulttasks = builddirs.keys()
  59
  60 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  61     defaulttasks.remove("samba-o3")
  62
  63 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  64 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  65
  66 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  67 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  68 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  69 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  70 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  71 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  72 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  73
  74 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  75     extra_python = ""
  76 else:
  77     extra_python = "--extra-python=/usr/bin/python3"
  78
  79 tasks = {
  80     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  81                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  82                ("make", "make all", "text/plain"),
  83                ("install", "make install", "text/plain"),
  84                ("test", "make autotest", "text/plain"),
  85                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  86                ("clean", "make clean", "text/plain")],
  87
  88     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
  89     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
  90                 ("make", "make -j", "text/plain"),
  91                 ("test", "make test FAIL_IMMEDIATELY=1 "
  92                  "TESTS='${PY3_ONLY}"
  93                  "--exclude-env=none "
  94                  "--exclude-env=nt4_dc "
  95                  "--exclude-env=nt4_member "
  96                  "--exclude-env=ad_dc "
  97                  "--exclude-env=ad_dc_no_nss "
  98                  "--exclude-env=fl2003dc "
  99                  "--exclude-env=fl2008r2dc "
 100                  "--exclude-env=ad_member "
 101                  "--exclude-env=ad_member_idmap_rid "
 102                  "--exclude-env=ad_member_idmap_ad "
 103                  "--exclude-env=chgdcpass "
 104                  "--exclude-env=vampire_2000_dc "
 105                  "--exclude-env=fl2000dc "
 106                  "--exclude-env=fileserver "
 107                  "--exclude-env=backupfromdc "
 108                  "--exclude-env=restoredc "
 109                  "--exclude-env=renamedc "
 110                  "--exclude-env=offlinebackupdc "
 111                  "--exclude-env=labdc "
 112                  "'",
 113                  "text/plain"),
 114                 ("install", "make install", "text/plain"),
 115                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 116                 ("clean", "make clean", "text/plain")],
 117
 118     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 119     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 120                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 121                     ("make", "make -j", "text/plain"),
 122                     ("test", "make test FAIL_IMMEDIATELY=1 "
 123                      "TESTS='${PY3_ONLY}"
 124                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 125                     ("install", "make install", "text/plain"),
 126                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 127                     ("clean", "make clean", "text/plain")],
 128
 129     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 130     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 131                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json-audit --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 132                            ("make", "make -j", "text/plain"),
 133                            ("test", "make test FAIL_IMMEDIATELY=1 "
 134                             "TESTS='${PY3_ONLY}"
 135                             "--include-env=fileserver'", "text/plain"),
 136                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 137
 138     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 139     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 140                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 141                       ("make", "make -j", "text/plain"),
 142                       ("test", "make test FAIL_IMMEDIATELY=1 "
 143                        "TESTS='${PY3_ONLY}"
 144                        "--include-env=ad_dc "
 145                        "--include-env=fl2003dc "
 146                        "--include-env=fl2008r2dc "
 147                        "--include-env=ad_member "
 148                        "--include-env=ad_member_idmap_rid "
 149                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 150                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 151
 152     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 153     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 154                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 155                         ("make", "make -j", "text/plain"),
 156                         ("test", "make test FAIL_IMMEDIATELY=1 "
 157                          "TESTS='${PY3_ONLY}"
 158                          "--include-env=chgdcpass "
 159                          "--include-env=vampire_2000_dc "
 160                          "--include-env=fl2000dc "
 161                          "--include-env=ad_dc_no_nss "
 162                          "--include-env=backupfromdc "
 163                          "--include-env=restoredc "
 164                          "--include-env=renamedc "
 165                          "--include-env=offlinebackupdc "
 166                          "--include-env=labdc "
 167                          "'",
 168                          "text/plain"),
 169                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 170
 171     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 172                           ("make", "make -j", "text/plain"),
 173                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 174
 175     # Test cross-compile infrastructure
 176     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 177                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 178                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 179                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 180                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 181                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 182                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 183
 184     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 185     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 186                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 187                    ("make", "make -j", "text/plain"),
 188                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 189                     "TESTS='${PY3_ONLY}"
 190                     "--include-env=ad_dc'", "text/plain"),
 191                    ("install", "make install", "text/plain"),
 192                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 193                    ("clean", "make clean", "text/plain")],
 194
 195     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 196
 197                      # make sure we have tdb around:
 198                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 199                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 200                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 201
 202
 203                      # build samba with cluster support (also building ctdb):
 204                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 205                      ("samba-make", "make", "text/plain"),
 206                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 207                      ("samba-install", "make install", "text/plain"),
 208                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 209
 210                      # clean up:
 211                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 212                      ("clean", "make clean", "text/plain"),
 213                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 214
 215     "samba-libs": [
 216                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 217                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 218                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 219                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 220
 221                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 222                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 223                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 224
 225                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 226                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 227                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 228
 229                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 230                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 231                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 232
 233                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 234                       ("nondevel-make", "make -j", "text/plain"),
 235                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 236                       ("nondevel-install", "make install", "text/plain"),
 237                       ("nondevel-dist", "make dist", "text/plain"),
 238
 239                       # retry with all modules shared
 240                       ("allshared-distclean", "make distclean", "text/plain"),
 241                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 242                       ("allshared-make", "make -j", "text/plain")],
 243
 244     "samba-none-env": [
 245                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 246                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 247                       ("make", "make -j", "text/plain"),
 248                       ("test", "make test "
 249                        "FAIL_IMMEDIATELY=1 "
 250                        "TESTS='${PY3_ONLY}"
 251                        "--include-env=none'",
 252                        "text/plain")],
 253
 254     "samba-static": [
 255                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 256                       # build with all modules static
 257                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 258                       ("allstatic-make", "make -j", "text/plain"),
 259                       ("allstatic-test", "make test "
 260                        "FAIL_IMMEDIATELY=1 "
 261                        "TESTS='samba3.smb2.create.*nt4_dc'",
 262                        "text/plain"),
 263
 264                       # retry without any required modules
 265                       ("none-distclean", "make distclean", "text/plain"),
 266                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 267                       ("none-make", "make -j", "text/plain"),
 268
 269                       # retry with nonshared smbd and smbtorture
 270                       ("nonshared-distclean", "make distclean", "text/plain"),
 271                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 272                       ("nonshared-make", "make -j", "text/plain")],
 273
 274     "samba-systemkrb5": [
 275                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 276                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 277                       ("make", "make -j", "text/plain"),
 278                       # we currently cannot run a full make test, a limited list of tests could be run
 279                       # via "make test TESTS=sometests"
 280                       ("test", "make test FAIL_IMMEDIATELY=1 "
 281                        "TESTS='${PY3_ONLY}"
 282                        "--include-env=ktest'", "text/plain"),
 283                       ("install", "make install", "text/plain"),
 284                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 285                       ("clean", "make clean", "text/plain")
 286                       ],
 287
 288     # Test Samba without python still builds.  When this test fails
 289     # due to more use of Python, the expectations is that the newly
 290     # failing part of the code should be disabled when
 291     # --disable-python is set (rather than major work being done to
 292     # support this environment).  The target here is for vendors
 293     # shipping a minimal smbd.
 294     "samba-nopython": [
 295                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 296                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 297                       ("make", "make -j", "text/plain"),
 298                       ("install", "make install", "text/plain"),
 299                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 300                       ("clean", "make clean", "text/plain"),
 301
 302                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 303                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 304                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 305
 306                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 307                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 308                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 309
 310                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 311                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 312                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 313
 314                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 315                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 316                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 317
 318                       # retry against installed library packages
 319                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 320                       ("libs-make", "make -j", "text/plain"),
 321                       ("libs-install", "make install", "text/plain"),
 322                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 323                       ("libs-clean", "make clean", "text/plain")
 324                       ],
 325
 326
 327
 328     "ldb": [
 329               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 330               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 331               ("make", "make", "text/plain"),
 332               ("install", "make install", "text/plain"),
 333               ("test", "make test", "text/plain"),
 334               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 335               ("make-no-lmdb", "make", "text/plain"),
 336               ("install-no-lmdb", "make install", "text/plain"),
 337               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 338               ("distcheck", "make distcheck", "text/plain"),
 339               ("clean", "make clean", "text/plain")],
 340
 341     "tdb": [
 342               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 343               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 344               ("make", "make", "text/plain"),
 345               ("install", "make install", "text/plain"),
 346               ("test", "make test", "text/plain"),
 347               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 348               ("distcheck", "make distcheck", "text/plain"),
 349               ("clean", "make clean", "text/plain")],
 350
 351     "talloc": [
 352                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 353                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 354                  ("make", "make", "text/plain"),
 355                  ("install", "make install", "text/plain"),
 356                  ("test", "make test", "text/plain"),
 357                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 358                  ("distcheck", "make distcheck", "text/plain"),
 359                  ("clean", "make clean", "text/plain")],
 360
 361     "replace": [
 362                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 363                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 364                   ("make", "make", "text/plain"),
 365                   ("install", "make install", "text/plain"),
 366                   ("test", "make test", "text/plain"),
 367                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 368                   ("distcheck", "make distcheck", "text/plain"),
 369                   ("clean", "make clean", "text/plain")],
 370
 371     "tevent": [
 372                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 373                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 374                  ("make", "make", "text/plain"),
 375                  ("install", "make install", "text/plain"),
 376                  ("test", "make test", "text/plain"),
 377                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 378                  ("distcheck", "make distcheck", "text/plain"),
 379                  ("clean", "make clean", "text/plain")],
 380
 381     "pidl": [
 382         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 383         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 384         ("touch", "touch *.yp", "text/plain"),
 385         ("make", "make", "text/plain"),
 386         ("test", "make test", "text/plain"),
 387         ("install", "make install", "text/plain"),
 388         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 389         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 390         ("clean", "make clean", "text/plain")],
 391
 392     # these are useful for debugging autobuild
 393     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 394     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 395 }
 396
 397
 398 def do_print(msg):
 399     print("%s" % msg)
 400     sys.stdout.flush()
 401     sys.stderr.flush()
 402
 403
 404 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 405     if show is None:
 406         show = options.verbose
 407     if show:
 408         do_print("Running: '%s' in '%s'" % (cmd, dir))
 409     if output:
 410         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 411     elif checkfail:
 412         return check_call(cmd, shell=True, cwd=dir)
 413     else:
 414         return call(cmd, shell=True, cwd=dir)
 415
 416
 417 class builder(object):
 418     '''handle build of one directory'''
 419
 420     def __init__(self, name, sequence, cp=True, py3=False):
 421         self.name = name
 422         self.py3 = py3
 423         if name in builddirs:
 424             self.dir = builddirs[name]
 425         else:
 426             self.dir = "."
 427
 428         self.tag = self.name.replace('/', '_')
 429         self.sequence = sequence
 430         self.next = 0
 431         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 432         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 433         if options.verbose:
 434             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 435             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 436         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 437         self.stdout = open(self.stdout_path, 'w')
 438         self.stderr = open(self.stderr_path, 'w')
 439         self.stdin  = open("/dev/null", 'r')
 440         self.sdir = "%s/%s" % (testbase, self.tag)
 441         self.prefix = "%s/%s" % (test_prefix, self.tag)
 442         run_cmd("rm -rf %s" % self.sdir)
 443         run_cmd("rm -rf %s" % self.prefix)
 444         if cp:
 445             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 446         else:
 447             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 448         self.start_next()
 449
 450     def start_next(self):
 451         if self.next == len(self.sequence):
 452             if not options.nocleanup:
 453                 run_cmd("rm -rf %s" % self.sdir)
 454                 run_cmd("rm -rf %s" % self.prefix)
 455             do_print('%s: Completed OK' % self.name)
 456             self.done = True
 457             return
 458         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 459         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 460         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 461         if self.py3:
 462             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 463             # The trailing space is important
 464             self.cmd = self.cmd.replace("${PY3_ONLY}", "python3 ")
 465         else:
 466             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 467             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 468         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 469         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 470 #        if self.output_mime_type == "text/x-subunit":
 471 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 472         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 473         cwd = os.getcwd()
 474         os.chdir("%s/%s" % (self.sdir, self.dir))
 475         self.proc = Popen(self.cmd, shell=True,
 476                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 477         os.chdir(cwd)
 478         self.next += 1
 479
 480
 481 class buildlist(object):
 482     '''handle build of multiple directories'''
 483
 484     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 485         global tasks
 486         self.tlist = []
 487         self.tail_proc = None
 488         self.retry = None
 489         if tasknames == []:
 490             if options.restrict_tests:
 491                 tasknames = ["samba-test-only"]
 492             else:
 493                 tasknames = defaulttasks
 494         else:
 495             # If we are only running one test,
 496             # do not sleep randomly to wait for it to start
 497             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 498
 499         for n in tasknames:
 500             if n not in tasks and n.endswith("-py3"):
 501                 b = builder(n,
 502                             tasks[n[:-4]],
 503                             cp=n is not "pidl",
 504                             py3=True)
 505             else:
 506                 b = builder(n, tasks[n], cp=n is not "pidl")
 507             self.tlist.append(b)
 508         if options.retry:
 509             rebase_remote = "rebaseon"
 510             retry_task = [("retry",
 511                             '''set -e
 512                             git remote add -t %s %s %s
 513                             git fetch %s
 514                             while :; do
 515                               sleep 60
 516                               git describe %s/%s > old_remote_branch.desc
 517                               git fetch %s
 518                               git describe %s/%s > remote_branch.desc
 519                               diff old_remote_branch.desc remote_branch.desc
 520                             done
 521                            ''' % (
 522                                 rebase_branch, rebase_remote, rebase_url,
 523                                rebase_remote,
 524                                rebase_remote, rebase_branch,
 525                                rebase_remote,
 526                                rebase_remote, rebase_branch
 527                             ),
 528                             "test/plain")]
 529
 530             self.retry = builder('retry', retry_task, cp=False)
 531             self.need_retry = False
 532
 533     def kill_kids(self):
 534         if self.tail_proc is not None:
 535             self.tail_proc.terminate()
 536             self.tail_proc.wait()
 537             self.tail_proc = None
 538         if self.retry is not None:
 539             self.retry.proc.terminate()
 540             self.retry.proc.wait()
 541             self.retry = None
 542         for b in self.tlist:
 543             if b.proc is not None:
 544                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 545                 b.proc.terminate()
 546                 b.proc.wait()
 547                 b.proc = None
 548
 549     def wait_one(self):
 550         while True:
 551             none_running = True
 552             for b in self.tlist:
 553                 if b.proc is None:
 554                     continue
 555                 none_running = False
 556                 b.status = b.proc.poll()
 557                 if b.status is None:
 558                     continue
 559                 b.proc = None
 560                 return b
 561             if options.retry:
 562                 ret = self.retry.proc.poll()
 563                 if ret is not None:
 564                     self.need_retry = True
 565                     self.retry = None
 566                     return None
 567             if none_running:
 568                 return None
 569             time.sleep(0.1)
 570
 571     def run(self):
 572         while True:
 573             b = self.wait_one()
 574             if options.retry and self.need_retry:
 575                 self.kill_kids()
 576                 do_print("retry needed")
 577                 return (0, None, None, None, "retry")
 578             if b is None:
 579                 break
 580             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 581                 self.kill_kids()
 582                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 583             b.start_next()
 584         self.kill_kids()
 585         return (0, None, None, None, "All OK")
 586
 587     def write_system_info(self):
 588         filename = 'system-info.txt'
 589         f = open(filename, 'w')
 590         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 591                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 592             print('### %s' % cmd, file=f)
 593             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 594             print(file=f)
 595         f.close()
 596         return filename
 597
 598     def tarlogs(self, fname):
 599         tar = tarfile.open(fname, "w:gz")
 600         for b in self.tlist:
 601             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 602             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 603         if os.path.exists("autobuild.log"):
 604             tar.add("autobuild.log")
 605         sys_info = self.write_system_info()
 606         tar.add(sys_info)
 607         tar.close()
 608
 609     def remove_logs(self):
 610         for b in self.tlist:
 611             os.unlink(b.stdout_path)
 612             os.unlink(b.stderr_path)
 613
 614     def start_tail(self):
 615         cwd = os.getcwd()
 616         cmd = "tail -f *.stdout *.stderr"
 617         os.chdir(gitroot)
 618         self.tail_proc = Popen(cmd, shell=True)
 619         os.chdir(cwd)
 620
 621
 622 def cleanup():
 623     if options.nocleanup:
 624         return
 625     run_cmd("stat %s || true" % test_tmpdir, show=True)
 626     run_cmd("stat %s" % testbase, show=True)
 627     do_print("Cleaning up ....")
 628     for d in cleanup_list:
 629         run_cmd("rm -rf %s" % d)
 630
 631
 632 def find_git_root():
 633     '''get to the top of the git repo'''
 634     p = os.getcwd()
 635     while p != '/':
 636         if os.path.isdir(os.path.join(p, ".git")):
 637             return p
 638         p = os.path.abspath(os.path.join(p, '..'))
 639     return None
 640
 641
 642 def daemonize(logfile):
 643     pid = os.fork()
 644     if pid == 0:  # Parent
 645         os.setsid()
 646         pid = os.fork()
 647         if pid != 0:  # Actual daemon
 648             os._exit(0)
 649     else:  # Grandparent
 650         os._exit(0)
 651
 652     import resource      # Resource usage information.
 653     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 654     if maxfd == resource.RLIM_INFINITY:
 655         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 656     for fd in range(0, maxfd):
 657         try:
 658             os.close(fd)
 659         except OSError:
 660             pass
 661     os.open(logfile, os.O_RDWR | os.O_CREAT)
 662     os.dup2(0, 1)
 663     os.dup2(0, 2)
 664
 665
 666 def write_pidfile(fname):
 667     '''write a pid file, cleanup on exit'''
 668     f = open(fname, mode='w')
 669     f.write("%u\n" % os.getpid())
 670     f.close()
 671
 672
 673 def rebase_tree(rebase_url, rebase_branch="master"):
 674     rebase_remote = "rebaseon"
 675     do_print("Rebasing on %s" % rebase_url)
 676     run_cmd("git describe HEAD", show=True, dir=test_master)
 677     run_cmd("git remote add -t %s %s %s" %
 678             (rebase_branch, rebase_remote, rebase_url),
 679             show=True, dir=test_master)
 680     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 681     if options.fix_whitespace:
 682         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 683                 (rebase_remote, rebase_branch),
 684                 show=True, dir=test_master)
 685     else:
 686         run_cmd("git rebase --force-rebase %s/%s" %
 687                 (rebase_remote, rebase_branch),
 688                 show=True, dir=test_master)
 689     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 690                    (rebase_remote, rebase_branch),
 691                    dir=test_master, output=True)
 692     if diff == '':
 693         do_print("No differences between HEAD and %s/%s - exiting" %
 694                  (rebase_remote, rebase_branch))
 695         sys.exit(0)
 696     run_cmd("git describe %s/%s" %
 697             (rebase_remote, rebase_branch),
 698             show=True, dir=test_master)
 699     run_cmd("git describe HEAD", show=True, dir=test_master)
 700     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 701             (rebase_remote, rebase_branch),
 702             show=True, dir=test_master)
 703
 704
 705 def push_to(push_url, push_branch="master"):
 706     push_remote = "pushto"
 707     do_print("Pushing to %s" % push_url)
 708     if options.mark:
 709         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 710         run_cmd("git commit --amend -c HEAD", dir=test_master)
 711         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 712         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 713     run_cmd("git remote add -t %s %s %s" %
 714             (push_branch, push_remote, push_url),
 715             show=True, dir=test_master)
 716     run_cmd("git push %s +HEAD:%s" %
 717             (push_remote, push_branch),
 718             show=True, dir=test_master)
 719
 720
 721 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 722
 723 gitroot = find_git_root()
 724 if gitroot is None:
 725     raise Exception("Failed to find git root")
 726
 727 parser = OptionParser()
 728 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 729 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 730 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 731 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 732                   default=def_testbase)
 733 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 734 parser.add_option("", "--verbose", help="show all commands as they are run",
 735                   default=False, action="store_true")
 736 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 737                   default=None, type='str')
 738 parser.add_option("", "--pushto", help="push to a git url on success",
 739                   default=None, type='str')
 740 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 741                   default=False, action="store_true")
 742 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 743                   default=False, action="store_true")
 744 parser.add_option("", "--retry", help="automatically retry if master changes",
 745                   default=False, action="store_true")
 746 parser.add_option("", "--email", help="send email to the given address on failure",
 747                   type='str', default=None)
 748 parser.add_option("", "--email-from", help="send email from the given address",
 749                   type='str', default="autobuild@samba.org")
 750 parser.add_option("", "--email-server", help="send email via the given server",
 751                   type='str', default='localhost')
 752 parser.add_option("", "--always-email", help="always send email, even on success",
 753                   action="store_true")
 754 parser.add_option("", "--daemon", help="daemonize after initial setup",
 755                   action="store_true")
 756 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 757                   default="master", type='str')
 758 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 759                   default=gitroot, type='str')
 760 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 761                   default=False, action="store_true")
 762 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 763                   default='')
 764
 765
 766 def send_email(subject, text, log_tar):
 767     if options.email is None:
 768         do_print("not sending email because the recipient is not set")
 769         do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
 770                  (subject, text))
 771         return
 772     outer = MIMEMultipart()
 773     outer['Subject'] = subject
 774     outer['To'] = options.email
 775     outer['From'] = options.email_from
 776     outer['Date'] = email.utils.formatdate(localtime=True)
 777     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 778     outer.attach(MIMEText(text, 'plain'))
 779     if options.attach_logs:
 780         fp = open(log_tar, 'rb')
 781         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 782         fp.close()
 783         # Set the filename parameter
 784         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 785         outer.attach(msg)
 786     content = outer.as_string()
 787     s = smtplib.SMTP(options.email_server)
 788     s.sendmail(options.email_from, [options.email], content)
 789     s.set_debuglevel(1)
 790     s.quit()
 791
 792
 793 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 794                   elapsed_time, log_base=None, add_log_tail=True):
 795     '''send an email to options.email about the failure'''
 796     elapsed_minutes = elapsed_time / 60.0
 797     user = os.getenv("USER")
 798     if log_base is None:
 799         log_base = gitroot
 800     text = '''
 801 Dear Developer,
 802
 803 Your autobuild on %s failed after %.1f minutes
 804 when trying to test %s with the following error:
 805
 806    %s
 807
 808 the autobuild has been abandoned. Please fix the error and resubmit.
 809
 810 A summary of the autobuild process is here:
 811
 812   %s/autobuild.log
 813 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 814
 815     if options.restrict_tests:
 816         text += """
 817 The build was restricted to tests matching %s\n""" % options.restrict_tests
 818
 819     if failed_task != 'rebase':
 820         text += '''
 821 You can see logs of the failed task here:
 822
 823   %s/%s.stdout
 824   %s/%s.stderr
 825
 826 or you can get full logs of all tasks in this job here:
 827
 828   %s/logs.tar.gz
 829
 830 The top commit for the tree that was built was:
 831
 832 %s
 833
 834 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 835
 836     if add_log_tail:
 837         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 838         lines = f.readlines()
 839         log_tail = "".join(lines[-50:])
 840         num_lines = len(lines)
 841         if num_lines < 50:
 842             # Also include stderr (compile failures) if < 50 lines of stdout
 843             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 844             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 845
 846         text += '''
 847 The last 50 lines of log messages:
 848
 849 %s
 850     ''' % log_tail
 851         f.close()
 852
 853     logs = os.path.join(gitroot, 'logs.tar.gz')
 854     send_email('autobuild[%s] failure on %s for task %s during %s'
 855                % (options.branch, platform.node(), failed_task, failed_stage),
 856                text, logs)
 857
 858
 859 def email_success(elapsed_time, log_base=None):
 860     '''send an email to options.email about a successful build'''
 861     user = os.getenv("USER")
 862     if log_base is None:
 863         log_base = gitroot
 864     text = '''
 865 Dear Developer,
 866
 867 Your autobuild on %s has succeeded after %.1f minutes.
 868
 869 ''' % (platform.node(), elapsed_time / 60.)
 870
 871     if options.restrict_tests:
 872         text += """
 873 The build was restricted to tests matching %s\n""" % options.restrict_tests
 874
 875     if options.keeplogs:
 876         text += '''
 877
 878 you can get full logs of all tasks in this job here:
 879
 880   %s/logs.tar.gz
 881
 882 ''' % log_base
 883
 884     text += '''
 885 The top commit for the tree that was built was:
 886
 887 %s
 888 ''' % top_commit_msg
 889
 890     logs = os.path.join(gitroot, 'logs.tar.gz')
 891     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 892                text, logs)
 893
 894
 895 (options, args) = parser.parse_args()
 896
 897 if options.retry:
 898     if options.rebase is None:
 899         raise Exception('You can only use --retry if you also rebase')
 900
 901 testbase = "%s/b%u" % (options.testbase, os.getpid())
 902 test_master = "%s/master" % testbase
 903 test_prefix = "%s/prefix" % testbase
 904 test_tmpdir = "%s/tmp" % testbase
 905 os.environ['TMPDIR'] = test_tmpdir
 906
 907 # get the top commit message, for emails
 908 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 909
 910 try:
 911     os.makedirs(testbase)
 912 except Exception as reason:
 913     raise Exception("Unable to create %s : %s" % (testbase, reason))
 914 cleanup_list.append(testbase)
 915
 916 if options.daemon:
 917     logfile = os.path.join(testbase, "log")
 918     do_print("Forking into the background, writing progress to %s" % logfile)
 919     daemonize(logfile)
 920
 921 write_pidfile(gitroot + "/autobuild.pid")
 922
 923 start_time = time.time()
 924
 925 while True:
 926     try:
 927         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 928         os.makedirs(test_tmpdir)
 929         # The waf uninstall code removes empty directories all the way
 930         # up the tree.  Creating a file in test_tmpdir stops it from
 931         # being removed.
 932         run_cmd("touch %s" % os.path.join(test_tmpdir,
 933                                           ".directory-is-not-empty"), show=True)
 934         run_cmd("stat %s" % test_tmpdir, show=True)
 935         run_cmd("stat %s" % testbase, show=True)
 936         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 937     except Exception:
 938         cleanup()
 939         raise
 940
 941     try:
 942         try:
 943             if options.rebase is not None:
 944                 rebase_tree(options.rebase, rebase_branch=options.branch)
 945         except Exception:
 946             cleanup_list.append(gitroot + "/autobuild.pid")
 947             cleanup()
 948             elapsed_time = time.time() - start_time
 949             email_failure(-1, 'rebase', 'rebase', 'rebase',
 950                           'rebase on %s failed' % options.branch,
 951                           elapsed_time, log_base=options.log_base)
 952             sys.exit(1)
 953         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 954         if options.tail:
 955             blist.start_tail()
 956         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 957         if status != 0 or errstr != "retry":
 958             break
 959         cleanup()
 960     except Exception:
 961         cleanup()
 962         raise
 963
 964 cleanup_list.append(gitroot + "/autobuild.pid")
 965
 966 do_print(errstr)
 967
 968 blist.kill_kids()
 969 if options.tail:
 970     do_print("waiting for tail to flush")
 971     time.sleep(1)
 972
 973 elapsed_time = time.time() - start_time
 974 if status == 0:
 975     if options.passcmd is not None:
 976         do_print("Running passcmd: %s" % options.passcmd)
 977         run_cmd(options.passcmd, dir=test_master)
 978     if options.pushto is not None:
 979         push_to(options.pushto, push_branch=options.branch)
 980     if options.keeplogs or options.attach_logs:
 981         blist.tarlogs("logs.tar.gz")
 982         do_print("Logs in logs.tar.gz")
 983     if options.always_email:
 984         email_success(elapsed_time, log_base=options.log_base)
 985     blist.remove_logs()
 986     cleanup()
 987     do_print(errstr)
 988     sys.exit(0)
 989
 990 # something failed, gather a tar of the logs
 991 blist.tarlogs("logs.tar.gz")
 992
 993 if options.email is not None:
 994     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 995                   elapsed_time, log_base=options.log_base)
 996 else:
 997     elapsed_minutes = elapsed_time / 60.0
 998     print('''
 999
1000 ####################################################################
1001
1002 AUTOBUILD FAILURE
1003
1004 Your autobuild[%s] on %s failed after %.1f minutes
1005 when trying to test %s with the following error:
1006
1007    %s
1008
1009 the autobuild has been abandoned. Please fix the error and resubmit.
1010
1011 ####################################################################
1012
1013 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1014
1015 cleanup()
1016 do_print(errstr)
1017 do_print("Logs in logs.tar.gz")
1018 sys.exit(status)