script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 os.environ["PYTHONUNBUFFERED"] = "1"
  23
  24 # This speeds up testing remarkably.
  25 os.environ['TDB_NO_FSYNC'] = '1'
  26
  27 cleanup_list = []
  28
  29 builddirs = {
  30     "ctdb": "ctdb",
  31     "samba": ".",
  32     "samba-py3": ".",
  33     "samba-nt4": ".",
  34     "samba-nt4-py3": ".",
  35     "samba-fileserver": ".",
  36     "samba-xc": ".",
  37     "samba-o3": ".",
  38     "samba-ctdb": ".",
  39     "samba-libs": ".",
  40     "samba-libs-py3": ".",
  41     "samba-static": ".",
  42     "samba-test-only": ".",
  43     "samba-none-env": ".",
  44     "samba-none-env-py3": ".",
  45     "samba-ad-dc": ".",
  46     "samba-ad-dc-py3": ".",
  47     "samba-ad-dc-2": ".",
  48     "samba-ad-dc-2-py3": ".",
  49     "samba-systemkrb5": ".",
  50     "samba-nopython": ".",
  51     "ldb": "lib/ldb",
  52     "tdb": "lib/tdb",
  53     "talloc": "lib/talloc",
  54     "replace": "lib/replace",
  55     "tevent": "lib/tevent",
  56     "pidl": "pidl",
  57     "pass": ".",
  58     "fail": ".",
  59     "retry": "."
  60 }
  61
  62 defaulttasks = ["ctdb",
  63                 "samba",
  64                 "samba-py3",
  65                 "samba-nt4",
  66                 "samba-nt4-py3",
  67                 "samba-fileserver",
  68                 "samba-xc",
  69                 "samba-o3",
  70                 "samba-ctdb",
  71                 "samba-libs",
  72                 "samba-libs-py3",
  73                 "samba-static",
  74                 "samba-none-env",
  75                 "samba-none-env-py3",
  76                 "samba-ad-dc",
  77                 "samba-ad-dc-py3",
  78                 "samba-ad-dc-2",
  79                 "samba-ad-dc-2-py3",
  80                 "samba-systemkrb5",
  81                 "samba-nopython",
  82                 "ldb",
  83                 "tdb",
  84                 "talloc",
  85                 "replace",
  86                 "tevent",
  87                 "pidl"]
  88
  89 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  90     defaulttasks.remove("samba-o3")
  91
  92 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  93 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  94
  95 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  96 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  97 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  98 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  99 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
 100 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 101 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
 102
 103 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
 104     extra_python = ""
 105 else:
 106     extra_python = "--extra-python=/usr/bin/python3"
 107
 108 tasks = {
 109     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 110                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
 111                ("make", "make all", "text/plain"),
 112                ("install", "make install", "text/plain"),
 113                ("test", "make autotest", "text/plain"),
 114                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 115                ("clean", "make clean", "text/plain")],
 116
 117     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
 118     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 119                 ("make", "make -j", "text/plain"),
 120                 ("test", "make test FAIL_IMMEDIATELY=1 "
 121                  "TESTS='${PY3_ONLY}"
 122                  "--exclude-env=none "
 123                  "--exclude-env=nt4_dc "
 124                  "--exclude-env=nt4_member "
 125                  "--exclude-env=ad_dc "
 126                  "--exclude-env=fl2003dc "
 127                  "--exclude-env=fl2008r2dc "
 128                  "--exclude-env=ad_member "
 129                  "--exclude-env=ad_member_idmap_rid "
 130                  "--exclude-env=ad_member_idmap_ad "
 131                  "--exclude-env=chgdcpass "
 132                  "--exclude-env=vampire_2000_dc "
 133                  "--exclude-env=fl2000dc "
 134                  "--exclude-env=fileserver'",
 135                  "text/plain"),
 136                 ("install", "make install", "text/plain"),
 137                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 138                 ("clean", "make clean", "text/plain")],
 139
 140     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 141     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 142                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 143                     ("make", "make -j", "text/plain"),
 144                     ("test", "make test FAIL_IMMEDIATELY=1 "
 145                      "TESTS='${PY3_ONLY}"
 146                      "--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 147                     ("install", "make install", "text/plain"),
 148                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 149                     ("clean", "make clean", "text/plain")],
 150
 151     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 152     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 153                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json-audit --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 154                            ("make", "make -j", "text/plain"),
 155                            ("test", "make test FAIL_IMMEDIATELY=1 "
 156                             "TESTS='${PY3_ONLY}"
 157                             "--include-env=fileserver'", "text/plain"),
 158                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 159
 160     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 161     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 162                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 163                       ("make", "make -j", "text/plain"),
 164                       ("test", "make test FAIL_IMMEDIATELY=1 "
 165                        "TESTS='${PY3_ONLY}"
 166                        "--include-env=ad_dc "
 167                        "--include-env=fl2003dc "
 168                        "--include-env=fl2008r2dc "
 169                        "--include-env=ad_member "
 170                        "--include-env=ad_member_idmap_rid "
 171                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 172                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 173
 174     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 175     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 176                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 177                         ("make", "make -j", "text/plain"),
 178                         ("test", "make test FAIL_IMMEDIATELY=1 "
 179                          "TESTS='${PY3_ONLY}"
 180                          "--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
 181                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 182
 183     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 184                           ("make", "make -j", "text/plain"),
 185                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 186
 187     # Test cross-compile infrastructure
 188     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 189                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 190                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 191                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 192                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 193                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 194                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 195
 196     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 197     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 198                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 199                    ("make", "make -j", "text/plain"),
 200                    ("test", "make quicktest FAIL_IMMEDIATELY=1 "
 201                     "TESTS='${PY3_ONLY}"
 202                     "--include-env=ad_dc'", "text/plain"),
 203                    ("install", "make install", "text/plain"),
 204                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 205                    ("clean", "make clean", "text/plain")],
 206
 207     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 208
 209                      # make sure we have tdb around:
 210                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 211                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 212                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 213
 214
 215                      # build samba with cluster support (also building ctdb):
 216                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 217                      ("samba-make", "make", "text/plain"),
 218                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 219                      ("samba-install", "make install", "text/plain"),
 220                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 221
 222                      # clean up:
 223                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 224                      ("clean", "make clean", "text/plain"),
 225                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 226
 227     "samba-libs": [
 228                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 229                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 230                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 231                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 232
 233                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 234                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 235                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 236
 237                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 238                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 239                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 240
 241                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 242                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 243                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 244
 245                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 246                       ("nondevel-make", "make -j", "text/plain"),
 247                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 248                       ("nondevel-install", "make install", "text/plain"),
 249                       ("nondevel-dist", "make dist", "text/plain"),
 250
 251                       # retry with all modules shared
 252                       ("allshared-distclean", "make distclean", "text/plain"),
 253                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 254                       ("allshared-make", "make -j", "text/plain")],
 255
 256     "samba-none-env": [
 257                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 258                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 259                       ("make", "make -j", "text/plain"),
 260                       ("test", "make test "
 261                        "FAIL_IMMEDIATELY=1 "
 262                        "TESTS='${PY3_ONLY}"
 263                        "--include-env=none'",
 264                        "text/plain")],
 265
 266     "samba-static": [
 267                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 268                       # build with all modules static
 269                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 270                       ("allstatic-make", "make -j", "text/plain"),
 271                       ("allstatic-test", "make test "
 272                        "FAIL_IMMEDIATELY=1 "
 273                        "TESTS='samba3.smb2.create.*nt4_dc'",
 274                        "text/plain"),
 275
 276                       # retry without any required modules
 277                       ("none-distclean", "make distclean", "text/plain"),
 278                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 279                       ("none-make", "make -j", "text/plain"),
 280
 281                       # retry with nonshared smbd and smbtorture
 282                       ("nonshared-distclean", "make distclean", "text/plain"),
 283                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 284                       ("nonshared-make", "make -j", "text/plain")],
 285
 286     "samba-systemkrb5": [
 287                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 288                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 289                       ("make", "make -j", "text/plain"),
 290                       # we currently cannot run a full make test, a limited list of tests could be run
 291                       # via "make test TESTS=sometests"
 292                       ("test", "make test FAIL_IMMEDIATELY=1 "
 293                        "TESTS='${PY3_ONLY}"
 294                        "--include-env=ktest'", "text/plain"),
 295                       ("install", "make install", "text/plain"),
 296                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 297                       ("clean", "make clean", "text/plain")
 298                       ],
 299
 300     # Test Samba without python still builds.  When this test fails
 301     # due to more use of Python, the expectations is that the newly
 302     # failing part of the code should be disabled when
 303     # --disable-python is set (rather than major work being done to
 304     # support this environment).  The target here is for vendors
 305     # shipping a minimal smbd.
 306     "samba-nopython": [
 307                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 308                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 309                       ("make", "make -j", "text/plain"),
 310                       ("install", "make install", "text/plain"),
 311                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 312                       ("clean", "make clean", "text/plain"),
 313
 314                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 315                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 316                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 317
 318                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 319                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 320                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 321
 322                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 323                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 324                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 325
 326                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 327                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 328                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 329
 330                       # retry against installed library packages
 331                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 332                       ("libs-make", "make -j", "text/plain"),
 333                       ("libs-install", "make install", "text/plain"),
 334                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 335                       ("libs-clean", "make clean", "text/plain")
 336                       ],
 337
 338
 339
 340     "ldb": [
 341               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 342               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 343               ("make", "make", "text/plain"),
 344               ("install", "make install", "text/plain"),
 345               ("test", "make test", "text/plain"),
 346               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 347               ("make-no-lmdb", "make", "text/plain"),
 348               ("install-no-lmdb", "make install", "text/plain"),
 349               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 350               ("distcheck", "make distcheck", "text/plain"),
 351               ("clean", "make clean", "text/plain")],
 352
 353     "tdb": [
 354               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 355               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 356               ("make", "make", "text/plain"),
 357               ("install", "make install", "text/plain"),
 358               ("test", "make test", "text/plain"),
 359               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 360               ("distcheck", "make distcheck", "text/plain"),
 361               ("clean", "make clean", "text/plain")],
 362
 363     "talloc": [
 364                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 365                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 366                  ("make", "make", "text/plain"),
 367                  ("install", "make install", "text/plain"),
 368                  ("test", "make test", "text/plain"),
 369                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 370                  ("distcheck", "make distcheck", "text/plain"),
 371                  ("clean", "make clean", "text/plain")],
 372
 373     "replace": [
 374                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 375                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 376                   ("make", "make", "text/plain"),
 377                   ("install", "make install", "text/plain"),
 378                   ("test", "make test", "text/plain"),
 379                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 380                   ("distcheck", "make distcheck", "text/plain"),
 381                   ("clean", "make clean", "text/plain")],
 382
 383     "tevent": [
 384                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 385                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 386                  ("make", "make", "text/plain"),
 387                  ("install", "make install", "text/plain"),
 388                  ("test", "make test", "text/plain"),
 389                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 390                  ("distcheck", "make distcheck", "text/plain"),
 391                  ("clean", "make clean", "text/plain")],
 392
 393     "pidl": [
 394         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 395         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 396         ("touch", "touch *.yp", "text/plain"),
 397         ("make", "make", "text/plain"),
 398         ("test", "make test", "text/plain"),
 399         ("install", "make install", "text/plain"),
 400         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 401         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 402         ("clean", "make clean", "text/plain")],
 403
 404     # these are useful for debugging autobuild
 405     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 406     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 407 }
 408
 409
 410 def do_print(msg):
 411     print("%s" % msg)
 412     sys.stdout.flush()
 413     sys.stderr.flush()
 414
 415
 416 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 417     if show is None:
 418         show = options.verbose
 419     if show:
 420         do_print("Running: '%s' in '%s'" % (cmd, dir))
 421     if output:
 422         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 423     elif checkfail:
 424         return check_call(cmd, shell=True, cwd=dir)
 425     else:
 426         return call(cmd, shell=True, cwd=dir)
 427
 428
 429 class builder(object):
 430     '''handle build of one directory'''
 431
 432     def __init__(self, name, sequence, cp=True, py3=False):
 433         self.name = name
 434         self.py3 = py3
 435         self.dir = builddirs[name]
 436
 437         self.tag = self.name.replace('/', '_')
 438         self.sequence = sequence
 439         self.next = 0
 440         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 441         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 442         if options.verbose:
 443             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 444             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 445         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 446         self.stdout = open(self.stdout_path, 'w')
 447         self.stderr = open(self.stderr_path, 'w')
 448         self.stdin  = open("/dev/null", 'r')
 449         self.sdir = "%s/%s" % (testbase, self.tag)
 450         self.prefix = "%s/%s" % (test_prefix, self.tag)
 451         run_cmd("rm -rf %s" % self.sdir)
 452         run_cmd("rm -rf %s" % self.prefix)
 453         if cp:
 454             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 455         else:
 456             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 457         self.start_next()
 458
 459     def start_next(self):
 460         if self.next == len(self.sequence):
 461             if not options.nocleanup:
 462                 run_cmd("rm -rf %s" % self.sdir)
 463                 run_cmd("rm -rf %s" % self.prefix)
 464             do_print('%s: Completed OK' % self.name)
 465             self.done = True
 466             return
 467         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 468         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 469         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 470         if self.py3:
 471             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 472             # The trailing space is important
 473             self.cmd = self.cmd.replace("${PY3_ONLY}", "python3 ")
 474         else:
 475             self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "")
 476             self.cmd = self.cmd.replace("${PY3_ONLY}", "")
 477         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 478         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 479 #        if self.output_mime_type == "text/x-subunit":
 480 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 481         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 482         cwd = os.getcwd()
 483         os.chdir("%s/%s" % (self.sdir, self.dir))
 484         self.proc = Popen(self.cmd, shell=True,
 485                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 486         os.chdir(cwd)
 487         self.next += 1
 488
 489
 490 class buildlist(object):
 491     '''handle build of multiple directories'''
 492
 493     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 494         global tasks
 495         self.tlist = []
 496         self.tail_proc = None
 497         self.retry = None
 498         if tasknames == []:
 499             if options.restrict_tests:
 500                 tasknames = ["samba-test-only"]
 501             else:
 502                 tasknames = defaulttasks
 503         else:
 504             # If we are only running one test,
 505             # do not sleep randomly to wait for it to start
 506             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 507
 508         for n in tasknames:
 509             if n not in tasks and n.endswith("-py3"):
 510                 b = builder(n,
 511                             tasks[n[:-4]],
 512                             cp=n is not "pidl",
 513                             py3=True)
 514             else:
 515                 b = builder(n, tasks[n], cp=n is not "pidl")
 516             self.tlist.append(b)
 517         if options.retry:
 518             rebase_remote = "rebaseon"
 519             retry_task = [("retry",
 520                             '''set -e
 521                             git remote add -t %s %s %s
 522                             git fetch %s
 523                             while :; do
 524                               sleep 60
 525                               git describe %s/%s > old_remote_branch.desc
 526                               git fetch %s
 527                               git describe %s/%s > remote_branch.desc
 528                               diff old_remote_branch.desc remote_branch.desc
 529                             done
 530                            ''' % (
 531                                 rebase_branch, rebase_remote, rebase_url,
 532                                rebase_remote,
 533                                rebase_remote, rebase_branch,
 534                                rebase_remote,
 535                                rebase_remote, rebase_branch
 536                             ),
 537                             "test/plain")]
 538
 539             self.retry = builder('retry', retry_task, cp=False)
 540             self.need_retry = False
 541
 542     def kill_kids(self):
 543         if self.tail_proc is not None:
 544             self.tail_proc.terminate()
 545             self.tail_proc.wait()
 546             self.tail_proc = None
 547         if self.retry is not None:
 548             self.retry.proc.terminate()
 549             self.retry.proc.wait()
 550             self.retry = None
 551         for b in self.tlist:
 552             if b.proc is not None:
 553                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 554                 b.proc.terminate()
 555                 b.proc.wait()
 556                 b.proc = None
 557
 558     def wait_one(self):
 559         while True:
 560             none_running = True
 561             for b in self.tlist:
 562                 if b.proc is None:
 563                     continue
 564                 none_running = False
 565                 b.status = b.proc.poll()
 566                 if b.status is None:
 567                     continue
 568                 b.proc = None
 569                 return b
 570             if options.retry:
 571                 ret = self.retry.proc.poll()
 572                 if ret is not None:
 573                     self.need_retry = True
 574                     self.retry = None
 575                     return None
 576             if none_running:
 577                 return None
 578             time.sleep(0.1)
 579
 580     def run(self):
 581         while True:
 582             b = self.wait_one()
 583             if options.retry and self.need_retry:
 584                 self.kill_kids()
 585                 do_print("retry needed")
 586                 return (0, None, None, None, "retry")
 587             if b is None:
 588                 break
 589             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 590                 self.kill_kids()
 591                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 592             b.start_next()
 593         self.kill_kids()
 594         return (0, None, None, None, "All OK")
 595
 596     def write_system_info(self):
 597         filename = 'system-info.txt'
 598         f = open(filename, 'w')
 599         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 600                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 601             print('### %s' % cmd, file=f)
 602             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 603             print(file=f)
 604         f.close()
 605         return filename
 606
 607     def tarlogs(self, fname):
 608         tar = tarfile.open(fname, "w:gz")
 609         for b in self.tlist:
 610             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 611             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 612         if os.path.exists("autobuild.log"):
 613             tar.add("autobuild.log")
 614         sys_info = self.write_system_info()
 615         tar.add(sys_info)
 616         tar.close()
 617
 618     def remove_logs(self):
 619         for b in self.tlist:
 620             os.unlink(b.stdout_path)
 621             os.unlink(b.stderr_path)
 622
 623     def start_tail(self):
 624         cwd = os.getcwd()
 625         cmd = "tail -f *.stdout *.stderr"
 626         os.chdir(gitroot)
 627         self.tail_proc = Popen(cmd, shell=True)
 628         os.chdir(cwd)
 629
 630
 631 def cleanup():
 632     if options.nocleanup:
 633         return
 634     run_cmd("stat %s || true" % test_tmpdir, show=True)
 635     run_cmd("stat %s" % testbase, show=True)
 636     do_print("Cleaning up ....")
 637     for d in cleanup_list:
 638         run_cmd("rm -rf %s" % d)
 639
 640
 641 def find_git_root():
 642     '''get to the top of the git repo'''
 643     p = os.getcwd()
 644     while p != '/':
 645         if os.path.isdir(os.path.join(p, ".git")):
 646             return p
 647         p = os.path.abspath(os.path.join(p, '..'))
 648     return None
 649
 650
 651 def daemonize(logfile):
 652     pid = os.fork()
 653     if pid == 0:  # Parent
 654         os.setsid()
 655         pid = os.fork()
 656         if pid != 0:  # Actual daemon
 657             os._exit(0)
 658     else:  # Grandparent
 659         os._exit(0)
 660
 661     import resource      # Resource usage information.
 662     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 663     if maxfd == resource.RLIM_INFINITY:
 664         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 665     for fd in range(0, maxfd):
 666         try:
 667             os.close(fd)
 668         except OSError:
 669             pass
 670     os.open(logfile, os.O_RDWR | os.O_CREAT)
 671     os.dup2(0, 1)
 672     os.dup2(0, 2)
 673
 674
 675 def write_pidfile(fname):
 676     '''write a pid file, cleanup on exit'''
 677     f = open(fname, mode='w')
 678     f.write("%u\n" % os.getpid())
 679     f.close()
 680
 681
 682 def rebase_tree(rebase_url, rebase_branch="master"):
 683     rebase_remote = "rebaseon"
 684     do_print("Rebasing on %s" % rebase_url)
 685     run_cmd("git describe HEAD", show=True, dir=test_master)
 686     run_cmd("git remote add -t %s %s %s" %
 687             (rebase_branch, rebase_remote, rebase_url),
 688             show=True, dir=test_master)
 689     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 690     if options.fix_whitespace:
 691         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 692                 (rebase_remote, rebase_branch),
 693                 show=True, dir=test_master)
 694     else:
 695         run_cmd("git rebase --force-rebase %s/%s" %
 696                 (rebase_remote, rebase_branch),
 697                 show=True, dir=test_master)
 698     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 699                    (rebase_remote, rebase_branch),
 700                    dir=test_master, output=True)
 701     if diff == '':
 702         do_print("No differences between HEAD and %s/%s - exiting" %
 703                  (rebase_remote, rebase_branch))
 704         sys.exit(0)
 705     run_cmd("git describe %s/%s" %
 706             (rebase_remote, rebase_branch),
 707             show=True, dir=test_master)
 708     run_cmd("git describe HEAD", show=True, dir=test_master)
 709     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 710             (rebase_remote, rebase_branch),
 711             show=True, dir=test_master)
 712
 713
 714 def push_to(push_url, push_branch="master"):
 715     push_remote = "pushto"
 716     do_print("Pushing to %s" % push_url)
 717     if options.mark:
 718         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 719         run_cmd("git commit --amend -c HEAD", dir=test_master)
 720         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 721         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 722     run_cmd("git remote add -t %s %s %s" %
 723             (push_branch, push_remote, push_url),
 724             show=True, dir=test_master)
 725     run_cmd("git push %s +HEAD:%s" %
 726             (push_remote, push_branch),
 727             show=True, dir=test_master)
 728
 729
 730 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 731
 732 gitroot = find_git_root()
 733 if gitroot is None:
 734     raise Exception("Failed to find git root")
 735
 736 parser = OptionParser()
 737 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 738 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 739 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 740 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 741                   default=def_testbase)
 742 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 743 parser.add_option("", "--verbose", help="show all commands as they are run",
 744                   default=False, action="store_true")
 745 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 746                   default=None, type='str')
 747 parser.add_option("", "--pushto", help="push to a git url on success",
 748                   default=None, type='str')
 749 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 750                   default=False, action="store_true")
 751 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 752                   default=False, action="store_true")
 753 parser.add_option("", "--retry", help="automatically retry if master changes",
 754                   default=False, action="store_true")
 755 parser.add_option("", "--email", help="send email to the given address on failure",
 756                   type='str', default=None)
 757 parser.add_option("", "--email-from", help="send email from the given address",
 758                   type='str', default="autobuild@samba.org")
 759 parser.add_option("", "--email-server", help="send email via the given server",
 760                   type='str', default='localhost')
 761 parser.add_option("", "--always-email", help="always send email, even on success",
 762                   action="store_true")
 763 parser.add_option("", "--daemon", help="daemonize after initial setup",
 764                   action="store_true")
 765 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 766                   default="master", type='str')
 767 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 768                   default=gitroot, type='str')
 769 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 770                   default=False, action="store_true")
 771 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 772                   default='')
 773
 774
 775 def send_email(subject, text, log_tar):
 776     if options.email is None:
 777         do_print("not sending email because the recipient is not set")
 778         do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
 779                  (subject, text))
 780         return
 781     outer = MIMEMultipart()
 782     outer['Subject'] = subject
 783     outer['To'] = options.email
 784     outer['From'] = options.email_from
 785     outer['Date'] = email.utils.formatdate(localtime=True)
 786     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 787     outer.attach(MIMEText(text, 'plain'))
 788     if options.attach_logs:
 789         fp = open(log_tar, 'rb')
 790         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 791         fp.close()
 792         # Set the filename parameter
 793         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 794         outer.attach(msg)
 795     content = outer.as_string()
 796     s = smtplib.SMTP(options.email_server)
 797     s.sendmail(options.email_from, [options.email], content)
 798     s.set_debuglevel(1)
 799     s.quit()
 800
 801
 802 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 803                   elapsed_time, log_base=None, add_log_tail=True):
 804     '''send an email to options.email about the failure'''
 805     elapsed_minutes = elapsed_time / 60.0
 806     user = os.getenv("USER")
 807     if log_base is None:
 808         log_base = gitroot
 809     text = '''
 810 Dear Developer,
 811
 812 Your autobuild on %s failed after %.1f minutes
 813 when trying to test %s with the following error:
 814
 815    %s
 816
 817 the autobuild has been abandoned. Please fix the error and resubmit.
 818
 819 A summary of the autobuild process is here:
 820
 821   %s/autobuild.log
 822 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 823
 824     if options.restrict_tests:
 825         text += """
 826 The build was restricted to tests matching %s\n""" % options.restrict_tests
 827
 828     if failed_task != 'rebase':
 829         text += '''
 830 You can see logs of the failed task here:
 831
 832   %s/%s.stdout
 833   %s/%s.stderr
 834
 835 or you can get full logs of all tasks in this job here:
 836
 837   %s/logs.tar.gz
 838
 839 The top commit for the tree that was built was:
 840
 841 %s
 842
 843 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 844
 845     if add_log_tail:
 846         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 847         lines = f.readlines()
 848         log_tail = "".join(lines[-50:])
 849         num_lines = len(lines)
 850         if num_lines < 50:
 851             # Also include stderr (compile failures) if < 50 lines of stdout
 852             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 853             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 854
 855         text += '''
 856 The last 50 lines of log messages:
 857
 858 %s
 859     ''' % log_tail
 860         f.close()
 861
 862     logs = os.path.join(gitroot, 'logs.tar.gz')
 863     send_email('autobuild[%s] failure on %s for task %s during %s'
 864                % (options.branch, platform.node(), failed_task, failed_stage),
 865                text, logs)
 866
 867
 868 def email_success(elapsed_time, log_base=None):
 869     '''send an email to options.email about a successful build'''
 870     user = os.getenv("USER")
 871     if log_base is None:
 872         log_base = gitroot
 873     text = '''
 874 Dear Developer,
 875
 876 Your autobuild on %s has succeeded after %.1f minutes.
 877
 878 ''' % (platform.node(), elapsed_time / 60.)
 879
 880     if options.restrict_tests:
 881         text += """
 882 The build was restricted to tests matching %s\n""" % options.restrict_tests
 883
 884     if options.keeplogs:
 885         text += '''
 886
 887 you can get full logs of all tasks in this job here:
 888
 889   %s/logs.tar.gz
 890
 891 ''' % log_base
 892
 893     text += '''
 894 The top commit for the tree that was built was:
 895
 896 %s
 897 ''' % top_commit_msg
 898
 899     logs = os.path.join(gitroot, 'logs.tar.gz')
 900     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 901                text, logs)
 902
 903
 904 (options, args) = parser.parse_args()
 905
 906 if options.retry:
 907     if options.rebase is None:
 908         raise Exception('You can only use --retry if you also rebase')
 909
 910 testbase = "%s/b%u" % (options.testbase, os.getpid())
 911 test_master = "%s/master" % testbase
 912 test_prefix = "%s/prefix" % testbase
 913 test_tmpdir = "%s/tmp" % testbase
 914 os.environ['TMPDIR'] = test_tmpdir
 915
 916 # get the top commit message, for emails
 917 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 918
 919 try:
 920     os.makedirs(testbase)
 921 except Exception as reason:
 922     raise Exception("Unable to create %s : %s" % (testbase, reason))
 923 cleanup_list.append(testbase)
 924
 925 if options.daemon:
 926     logfile = os.path.join(testbase, "log")
 927     do_print("Forking into the background, writing progress to %s" % logfile)
 928     daemonize(logfile)
 929
 930 write_pidfile(gitroot + "/autobuild.pid")
 931
 932 start_time = time.time()
 933
 934 while True:
 935     try:
 936         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 937         os.makedirs(test_tmpdir)
 938         # The waf uninstall code removes empty directories all the way
 939         # up the tree.  Creating a file in test_tmpdir stops it from
 940         # being removed.
 941         run_cmd("touch %s" % os.path.join(test_tmpdir,
 942                                           ".directory-is-not-empty"), show=True)
 943         run_cmd("stat %s" % test_tmpdir, show=True)
 944         run_cmd("stat %s" % testbase, show=True)
 945         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 946     except Exception:
 947         cleanup()
 948         raise
 949
 950     try:
 951         try:
 952             if options.rebase is not None:
 953                 rebase_tree(options.rebase, rebase_branch=options.branch)
 954         except Exception:
 955             cleanup_list.append(gitroot + "/autobuild.pid")
 956             cleanup()
 957             elapsed_time = time.time() - start_time
 958             email_failure(-1, 'rebase', 'rebase', 'rebase',
 959                           'rebase on %s failed' % options.branch,
 960                           elapsed_time, log_base=options.log_base)
 961             sys.exit(1)
 962         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 963         if options.tail:
 964             blist.start_tail()
 965         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 966         if status != 0 or errstr != "retry":
 967             break
 968         cleanup()
 969     except Exception:
 970         cleanup()
 971         raise
 972
 973 cleanup_list.append(gitroot + "/autobuild.pid")
 974
 975 do_print(errstr)
 976
 977 blist.kill_kids()
 978 if options.tail:
 979     do_print("waiting for tail to flush")
 980     time.sleep(1)
 981
 982 elapsed_time = time.time() - start_time
 983 if status == 0:
 984     if options.passcmd is not None:
 985         do_print("Running passcmd: %s" % options.passcmd)
 986         run_cmd(options.passcmd, dir=test_master)
 987     if options.pushto is not None:
 988         push_to(options.pushto, push_branch=options.branch)
 989     if options.keeplogs or options.attach_logs:
 990         blist.tarlogs("logs.tar.gz")
 991         do_print("Logs in logs.tar.gz")
 992     if options.always_email:
 993         email_success(elapsed_time, log_base=options.log_base)
 994     blist.remove_logs()
 995     cleanup()
 996     do_print(errstr)
 997     sys.exit(0)
 998
 999 # something failed, gather a tar of the logs
1000 blist.tarlogs("logs.tar.gz")
1001
1002 if options.email is not None:
1003     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1004                   elapsed_time, log_base=options.log_base)
1005 else:
1006     elapsed_minutes = elapsed_time / 60.0
1007     print('''
1008
1009 ####################################################################
1010
1011 AUTOBUILD FAILURE
1012
1013 Your autobuild[%s] on %s failed after %.1f minutes
1014 when trying to test %s with the following error:
1015
1016    %s
1017
1018 the autobuild has been abandoned. Please fix the error and resubmit.
1019
1020 ####################################################################
1021
1022 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1023
1024 cleanup()
1025 do_print(errstr)
1026 do_print("Logs in logs.tar.gz")
1027 sys.exit(status)