script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call,Popen, PIPE
   8 import os, tarfile, sys, time
   9 from optparse import OptionParser
  10 import smtplib
  11 import email
  12 from email.mime.text import MIMEText
  13 from email.mime.base import MIMEBase
  14 from email.mime.application import MIMEApplication
  15 from email.mime.multipart import MIMEMultipart
  16 from distutils.sysconfig import get_python_lib
  17 import platform
  18
  19 os.environ["PYTHONUNBUFFERED"] = "1"
  20
  21 # This speeds up testing remarkably.
  22 os.environ['TDB_NO_FSYNC'] = '1'
  23
  24 cleanup_list = []
  25
  26 builddirs = {
  27     "ctdb"    : "ctdb",
  28     "samba"  : ".",
  29     "samba-nt4"  : ".",
  30     "samba-fileserver"  : ".",
  31     "samba-xc" : ".",
  32     "samba-o3" : ".",
  33     "samba-ctdb" : ".",
  34     "samba-libs"  : ".",
  35     "samba-static"  : ".",
  36     "samba-test-only"  : ".",
  37     "samba-none-env"  : ".",
  38     "samba-ad-dc"  : ".",
  39     "samba-ad-dc-2"  : ".",
  40     "samba-systemkrb5"  : ".",
  41     "samba-nopython"  : ".",
  42     "ldb"     : "lib/ldb",
  43     "tdb"     : "lib/tdb",
  44     "talloc"  : "lib/talloc",
  45     "replace" : "lib/replace",
  46     "tevent"  : "lib/tevent",
  47     "pidl"    : "pidl",
  48     "pass"    : ".",
  49     "fail"    : ".",
  50     "retry"   : "."
  51     }
  52
  53 defaulttasks = [ "ctdb",
  54                  "samba",
  55                  "samba-nt4",
  56                  "samba-fileserver",
  57                  "samba-xc",
  58                  "samba-o3",
  59                  "samba-ctdb",
  60                  "samba-libs",
  61                  "samba-static",
  62                  "samba-none-env",
  63                  "samba-ad-dc",
  64                  "samba-ad-dc-2",
  65                  "samba-systemkrb5",
  66                  "samba-nopython",
  67                  "ldb",
  68                  "tdb",
  69                  "talloc",
  70                  "replace",
  71                  "tevent",
  72                  "pidl" ]
  73
  74 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  75     defaulttasks.remove("samba-o3")
  76
  77 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  78 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  79
  80 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  81 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  82 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  83 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  84 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE ${EXTRA_PYTHON}"
  85 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
  86 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  87
  88 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  89     extra_python = ""
  90 else:
  91     extra_python = "--extra-python=/usr/bin/python3"
  92
  93 tasks = {
  94     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  95                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  96                ("make", "make all", "text/plain"),
  97                ("install", "make install", "text/plain"),
  98                ("test", "make autotest", "text/plain"),
  99                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 100                ("clean", "make clean", "text/plain") ],
 101
 102     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
 103     "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 104                 ("make", "make -j", "text/plain"),
 105                 ("test", "make test FAIL_IMMEDIATELY=1 "
 106                  "TESTS='--exclude-env=none "
 107                  "--exclude-env=nt4_dc "
 108                  "--exclude-env=nt4_member "
 109                  "--exclude-env=ad_dc "
 110                  "--exclude-env=fl2003dc "
 111                  "--exclude-env=fl2008r2dc "
 112                  "--exclude-env=ad_member "
 113                  "--exclude-env=ad_member_idmap_rid "
 114                  "--exclude-env=ad_member_idmap_ad "
 115                  "--exclude-env=chgdcpass "
 116                  "--exclude-env=vampire_2000_dc "
 117                  "--exclude-env=fl2000dc "
 118                  "--exclude-env=fileserver'",
 119                  "text/plain"),
 120                 ("install", "make install", "text/plain"),
 121                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 122                 ("clean", "make clean", "text/plain") ],
 123
 124     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 125     "samba-nt4" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 126                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 127                     ("make", "make -j", "text/plain"),
 128                     ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 129                     ("install", "make install", "text/plain"),
 130                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 131                     ("clean", "make clean", "text/plain") ],
 132
 133     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 134     "samba-fileserver" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 135                       ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json-audit --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 136                       ("make", "make -j", "text/plain"),
 137                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
 138                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 139
 140     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 141     "samba-ad-dc" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 142                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 143                       ("make", "make -j", "text/plain"),
 144                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='"
 145                        "--include-env=ad_dc "
 146                        "--include-env=fl2003dc "
 147                        "--include-env=fl2008r2dc "
 148                        "--include-env=ad_member "
 149                        "--include-env=ad_member_idmap_rid "
 150                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 151                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 152
 153     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 154     "samba-ad-dc-2" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 155                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 156                       ("make", "make -j", "text/plain"),
 157                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
 158                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 159
 160     "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 161                           ("make", "make -j", "text/plain"),
 162                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
 163
 164     # Test cross-compile infrastructure
 165     "samba-xc" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 166                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 167                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 168                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 169                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 170                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 171                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 172
 173     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 174     "samba-o3" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 175                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 176                    ("make", "make -j", "text/plain"),
 177                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 178                    ("install", "make install", "text/plain"),
 179                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 180                    ("clean", "make clean", "text/plain") ],
 181
 182     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 183
 184                      # make sure we have tdb around:
 185                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 186                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 187                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 188
 189
 190                      # build samba with cluster support (also building ctdb):
 191                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 192                      ("samba-make", "make", "text/plain"),
 193                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 194                      ("samba-install", "make install", "text/plain"),
 195                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 196
 197                      # clean up:
 198                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 199                      ("clean", "make clean", "text/plain"),
 200                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
 201
 202     "samba-libs" : [
 203                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 204                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 205                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 206                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 207
 208                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 209                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 210                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 211
 212                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 213                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 214                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 215
 216                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 217                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 218                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 219
 220                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 221                       ("nondevel-make", "make -j", "text/plain"),
 222                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 223                       ("nondevel-install", "make install", "text/plain"),
 224                       ("nondevel-dist", "make dist", "text/plain"),
 225
 226                       # retry with all modules shared
 227                       ("allshared-distclean", "make distclean", "text/plain"),
 228                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 229                       ("allshared-make", "make -j", "text/plain")],
 230
 231     "samba-none-env" : [
 232                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 233                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 234                       ("make", "make -j", "text/plain"),
 235                       ("test", "make test "
 236                        "FAIL_IMMEDIATELY=1 "
 237                        "TESTS='--include-env=none'",
 238                        "text/plain")],
 239
 240     "samba-static" : [
 241                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 242                       # build with all modules static
 243                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 244                       ("allstatic-make", "make -j", "text/plain"),
 245                       ("allstatic-test", "make test "
 246                        "FAIL_IMMEDIATELY=1 "
 247                        "TESTS='samba3.smb2.create.*nt4_dc'",
 248                        "text/plain"),
 249
 250                       # retry without any required modules
 251                       ("none-distclean", "make distclean", "text/plain"),
 252                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 253                       ("none-make", "make -j", "text/plain"),
 254
 255                       # retry with nonshared smbd and smbtorture
 256                       ("nonshared-distclean", "make distclean", "text/plain"),
 257                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 258                       ("nonshared-make", "make -j", "text/plain")],
 259
 260     "samba-systemkrb5" : [
 261                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 262                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 263                       ("make", "make -j", "text/plain"),
 264                       # we currently cannot run a full make test, a limited list of tests could be run
 265                       # via "make test TESTS=sometests"
 266                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 267                       ("install", "make install", "text/plain"),
 268                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 269                       ("clean", "make clean", "text/plain")
 270                       ],
 271
 272     # Test Samba without python still builds.  When this test fails
 273     # due to more use of Python, the expectations is that the newly
 274     # failing part of the code should be disabled when
 275     # --disable-python is set (rather than major work being done to
 276     # support this environment).  The target here is for vendors
 277     # shipping a minimal smbd.
 278     "samba-nopython" : [
 279                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 280                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 281                       ("make", "make -j", "text/plain"),
 282                       ("install", "make install", "text/plain"),
 283                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 284                       ("clean", "make clean", "text/plain"),
 285
 286                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 287                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 288                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 289
 290                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 291                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 292                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 293
 294                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 295                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 296                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 297
 298                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 299                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 300                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 301
 302                       # retry against installed library packages
 303                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 304                       ("libs-make", "make -j", "text/plain"),
 305                       ("libs-install", "make install", "text/plain"),
 306                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 307                       ("libs-clean", "make clean", "text/plain")
 308                       ],
 309
 310
 311
 312     "ldb" : [
 313               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 314               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 315               ("make", "make", "text/plain"),
 316               ("install", "make install", "text/plain"),
 317               ("test", "make test", "text/plain"),
 318               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 319               ("make-no-lmdb", "make", "text/plain"),
 320               ("install-no-lmdb", "make install", "text/plain"),
 321               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 322               ("distcheck", "make distcheck", "text/plain"),
 323               ("clean", "make clean", "text/plain") ],
 324
 325     "tdb" : [
 326               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 327               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 328               ("make", "make", "text/plain"),
 329               ("install", "make install", "text/plain"),
 330               ("test", "make test", "text/plain"),
 331               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 332               ("distcheck", "make distcheck", "text/plain"),
 333               ("clean", "make clean", "text/plain") ],
 334
 335     "talloc" : [
 336                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 337                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 338                  ("make", "make", "text/plain"),
 339                  ("install", "make install", "text/plain"),
 340                  ("test", "make test", "text/plain"),
 341                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 342                  ("distcheck", "make distcheck", "text/plain"),
 343                  ("clean", "make clean", "text/plain") ],
 344
 345     "replace" : [
 346                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 347                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 348                   ("make", "make", "text/plain"),
 349                   ("install", "make install", "text/plain"),
 350                   ("test", "make test", "text/plain"),
 351                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 352                   ("distcheck", "make distcheck", "text/plain"),
 353                   ("clean", "make clean", "text/plain") ],
 354
 355     "tevent" : [
 356                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 357                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 358                  ("make", "make", "text/plain"),
 359                  ("install", "make install", "text/plain"),
 360                  ("test", "make test", "text/plain"),
 361                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 362                  ("distcheck", "make distcheck", "text/plain"),
 363                  ("clean", "make clean", "text/plain") ],
 364
 365     "pidl" : [
 366                ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 367                ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 368                ("touch", "touch *.yp", "text/plain"),
 369                ("make", "make", "text/plain"),
 370                ("test", "make test", "text/plain"),
 371                ("install", "make install", "text/plain"),
 372                ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 373                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 374                ("clean", "make clean", "text/plain") ],
 375
 376     # these are useful for debugging autobuild
 377     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
 378     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
 379 }
 380
 381 def do_print(msg):
 382     print("%s" % msg)
 383     sys.stdout.flush()
 384     sys.stderr.flush()
 385
 386 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 387     if show is None:
 388         show = options.verbose
 389     if show:
 390         do_print("Running: '%s' in '%s'" % (cmd, dir))
 391     if output:
 392         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 393     elif checkfail:
 394         return check_call(cmd, shell=True, cwd=dir)
 395     else:
 396         return call(cmd, shell=True, cwd=dir)
 397
 398
 399 class builder(object):
 400     '''handle build of one directory'''
 401
 402     def __init__(self, name, sequence, cp=True):
 403         self.name = name
 404         self.dir = builddirs[name]
 405
 406         self.tag = self.name.replace('/', '_')
 407         self.sequence = sequence
 408         self.next = 0
 409         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 410         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 411         if options.verbose:
 412             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 413             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 414         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 415         self.stdout = open(self.stdout_path, 'w')
 416         self.stderr = open(self.stderr_path, 'w')
 417         self.stdin  = open("/dev/null", 'r')
 418         self.sdir = "%s/%s" % (testbase, self.tag)
 419         self.prefix = "%s/%s" % (test_prefix, self.tag)
 420         run_cmd("rm -rf %s" % self.sdir)
 421         run_cmd("rm -rf %s" % self.prefix)
 422         if cp:
 423             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 424         else:
 425             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 426         self.start_next()
 427
 428     def start_next(self):
 429         if self.next == len(self.sequence):
 430             if not options.nocleanup:
 431                 run_cmd("rm -rf %s" % self.sdir)
 432                 run_cmd("rm -rf %s" % self.prefix)
 433             do_print('%s: Completed OK' % self.name)
 434             self.done = True
 435             return
 436         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 437         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 438         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 439         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 440         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 441         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 442 #        if self.output_mime_type == "text/x-subunit":
 443 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 444         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 445         cwd = os.getcwd()
 446         os.chdir("%s/%s" % (self.sdir, self.dir))
 447         self.proc = Popen(self.cmd, shell=True,
 448                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 449         os.chdir(cwd)
 450         self.next += 1
 451
 452
 453 class buildlist(object):
 454     '''handle build of multiple directories'''
 455
 456     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 457         global tasks
 458         self.tlist = []
 459         self.tail_proc = None
 460         self.retry = None
 461         if tasknames == []:
 462             if options.restrict_tests:
 463                 tasknames = ["samba-test-only"]
 464             else:
 465                 tasknames = defaulttasks
 466         else:
 467             # If we are only running one test,
 468             # do not sleep randomly to wait for it to start
 469             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 470
 471         for n in tasknames:
 472             b = builder(n, tasks[n], cp=n is not "pidl")
 473             self.tlist.append(b)
 474         if options.retry:
 475             rebase_remote = "rebaseon"
 476             retry_task = [ ("retry",
 477                             '''set -e
 478                             git remote add -t %s %s %s
 479                             git fetch %s
 480                             while :; do
 481                               sleep 60
 482                               git describe %s/%s > old_remote_branch.desc
 483                               git fetch %s
 484                               git describe %s/%s > remote_branch.desc
 485                               diff old_remote_branch.desc remote_branch.desc
 486                             done
 487                            ''' % (
 488                                rebase_branch, rebase_remote, rebase_url,
 489                                rebase_remote,
 490                                rebase_remote, rebase_branch,
 491                                rebase_remote,
 492                                rebase_remote, rebase_branch
 493                            ),
 494                            "test/plain" ) ]
 495
 496             self.retry = builder('retry', retry_task, cp=False)
 497             self.need_retry = False
 498
 499     def kill_kids(self):
 500         if self.tail_proc is not None:
 501             self.tail_proc.terminate()
 502             self.tail_proc.wait()
 503             self.tail_proc = None
 504         if self.retry is not None:
 505             self.retry.proc.terminate()
 506             self.retry.proc.wait()
 507             self.retry = None
 508         for b in self.tlist:
 509             if b.proc is not None:
 510                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 511                 b.proc.terminate()
 512                 b.proc.wait()
 513                 b.proc = None
 514
 515     def wait_one(self):
 516         while True:
 517             none_running = True
 518             for b in self.tlist:
 519                 if b.proc is None:
 520                     continue
 521                 none_running = False
 522                 b.status = b.proc.poll()
 523                 if b.status is None:
 524                     continue
 525                 b.proc = None
 526                 return b
 527             if options.retry:
 528                 ret = self.retry.proc.poll()
 529                 if ret is not None:
 530                     self.need_retry = True
 531                     self.retry = None
 532                     return None
 533             if none_running:
 534                 return None
 535             time.sleep(0.1)
 536
 537     def run(self):
 538         while True:
 539             b = self.wait_one()
 540             if options.retry and self.need_retry:
 541                 self.kill_kids()
 542                 do_print("retry needed")
 543                 return (0, None, None, None, "retry")
 544             if b is None:
 545                 break
 546             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 547                 self.kill_kids()
 548                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 549             b.start_next()
 550         self.kill_kids()
 551         return (0, None, None, None, "All OK")
 552
 553     def write_system_info(self):
 554         filename = 'system-info.txt'
 555         f = open(filename, 'w')
 556         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 557                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 558             print('### %s' % cmd, file=f)
 559             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 560             print(file=f)
 561         f.close()
 562         return filename
 563
 564     def tarlogs(self, fname):
 565         tar = tarfile.open(fname, "w:gz")
 566         for b in self.tlist:
 567             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 568             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 569         if os.path.exists("autobuild.log"):
 570             tar.add("autobuild.log")
 571         sys_info = self.write_system_info()
 572         tar.add(sys_info)
 573         tar.close()
 574
 575     def remove_logs(self):
 576         for b in self.tlist:
 577             os.unlink(b.stdout_path)
 578             os.unlink(b.stderr_path)
 579
 580     def start_tail(self):
 581         cwd = os.getcwd()
 582         cmd = "tail -f *.stdout *.stderr"
 583         os.chdir(gitroot)
 584         self.tail_proc = Popen(cmd, shell=True)
 585         os.chdir(cwd)
 586
 587
 588 def cleanup():
 589     if options.nocleanup:
 590         return
 591     run_cmd("stat %s || true" % test_tmpdir, show=True)
 592     run_cmd("stat %s" % testbase, show=True)
 593     do_print("Cleaning up ....")
 594     for d in cleanup_list:
 595         run_cmd("rm -rf %s" % d)
 596
 597
 598 def find_git_root():
 599     '''get to the top of the git repo'''
 600     p=os.getcwd()
 601     while p != '/':
 602         if os.path.isdir(os.path.join(p, ".git")):
 603             return p
 604         p = os.path.abspath(os.path.join(p, '..'))
 605     return None
 606
 607
 608 def daemonize(logfile):
 609     pid = os.fork()
 610     if pid == 0: # Parent
 611         os.setsid()
 612         pid = os.fork()
 613         if pid != 0: # Actual daemon
 614             os._exit(0)
 615     else: # Grandparent
 616         os._exit(0)
 617
 618     import resource      # Resource usage information.
 619     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 620     if maxfd == resource.RLIM_INFINITY:
 621         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
 622     for fd in range(0, maxfd):
 623         try:
 624             os.close(fd)
 625         except OSError:
 626             pass
 627     os.open(logfile, os.O_RDWR | os.O_CREAT)
 628     os.dup2(0, 1)
 629     os.dup2(0, 2)
 630
 631 def write_pidfile(fname):
 632     '''write a pid file, cleanup on exit'''
 633     f = open(fname, mode='w')
 634     f.write("%u\n" % os.getpid())
 635     f.close()
 636
 637
 638 def rebase_tree(rebase_url, rebase_branch = "master"):
 639     rebase_remote = "rebaseon"
 640     do_print("Rebasing on %s" % rebase_url)
 641     run_cmd("git describe HEAD", show=True, dir=test_master)
 642     run_cmd("git remote add -t %s %s %s" %
 643             (rebase_branch, rebase_remote, rebase_url),
 644             show=True, dir=test_master)
 645     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 646     if options.fix_whitespace:
 647         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 648                 (rebase_remote, rebase_branch),
 649                 show=True, dir=test_master)
 650     else:
 651         run_cmd("git rebase --force-rebase %s/%s" %
 652                 (rebase_remote, rebase_branch),
 653                 show=True, dir=test_master)
 654     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 655                    (rebase_remote, rebase_branch),
 656                    dir=test_master, output=True)
 657     if diff == '':
 658         do_print("No differences between HEAD and %s/%s - exiting" %
 659               (rebase_remote, rebase_branch))
 660         sys.exit(0)
 661     run_cmd("git describe %s/%s" %
 662             (rebase_remote, rebase_branch),
 663             show=True, dir=test_master)
 664     run_cmd("git describe HEAD", show=True, dir=test_master)
 665     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 666             (rebase_remote, rebase_branch),
 667             show=True, dir=test_master)
 668
 669 def push_to(push_url, push_branch = "master"):
 670     push_remote = "pushto"
 671     do_print("Pushing to %s" % push_url)
 672     if options.mark:
 673         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 674         run_cmd("git commit --amend -c HEAD", dir=test_master)
 675         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 676         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 677     run_cmd("git remote add -t %s %s %s" %
 678             (push_branch, push_remote, push_url),
 679             show=True, dir=test_master)
 680     run_cmd("git push %s +HEAD:%s" %
 681             (push_remote, push_branch),
 682             show=True, dir=test_master)
 683
 684 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 685
 686 gitroot = find_git_root()
 687 if gitroot is None:
 688     raise Exception("Failed to find git root")
 689
 690 parser = OptionParser()
 691 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 692 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 693 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 694 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 695                   default=def_testbase)
 696 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 697 parser.add_option("", "--verbose", help="show all commands as they are run",
 698                   default=False, action="store_true")
 699 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 700                   default=None, type='str')
 701 parser.add_option("", "--pushto", help="push to a git url on success",
 702                   default=None, type='str')
 703 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 704                   default=False, action="store_true")
 705 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 706                   default=False, action="store_true")
 707 parser.add_option("", "--retry", help="automatically retry if master changes",
 708                   default=False, action="store_true")
 709 parser.add_option("", "--email", help="send email to the given address on failure",
 710                   type='str', default=None)
 711 parser.add_option("", "--email-from", help="send email from the given address",
 712                   type='str', default="autobuild@samba.org")
 713 parser.add_option("", "--email-server", help="send email via the given server",
 714                   type='str', default='localhost')
 715 parser.add_option("", "--always-email", help="always send email, even on success",
 716                   action="store_true")
 717 parser.add_option("", "--daemon", help="daemonize after initial setup",
 718                   action="store_true")
 719 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 720                   default="master", type='str')
 721 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 722                   default=gitroot, type='str')
 723 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 724                   default=False, action="store_true")
 725 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 726                   default='')
 727
 728 def send_email(subject, text, log_tar):
 729     if options.email is None:
 730         do_print("not sending email because the recipient is not set")
 731         do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
 732                  (subject, text))
 733         return
 734     outer = MIMEMultipart()
 735     outer['Subject'] = subject
 736     outer['To'] = options.email
 737     outer['From'] = options.email_from
 738     outer['Date'] = email.utils.formatdate(localtime = True)
 739     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 740     outer.attach(MIMEText(text, 'plain'))
 741     if options.attach_logs:
 742         fp = open(log_tar, 'rb')
 743         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 744         fp.close()
 745         # Set the filename parameter
 746         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 747         outer.attach(msg)
 748     content = outer.as_string()
 749     s = smtplib.SMTP(options.email_server)
 750     s.sendmail(options.email_from, [options.email], content)
 751     s.set_debuglevel(1)
 752     s.quit()
 753
 754 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 755                   elapsed_time, log_base=None, add_log_tail=True):
 756     '''send an email to options.email about the failure'''
 757     elapsed_minutes = elapsed_time / 60.0
 758     user = os.getenv("USER")
 759     if log_base is None:
 760         log_base = gitroot
 761     text = '''
 762 Dear Developer,
 763
 764 Your autobuild on %s failed after %.1f minutes
 765 when trying to test %s with the following error:
 766
 767    %s
 768
 769 the autobuild has been abandoned. Please fix the error and resubmit.
 770
 771 A summary of the autobuild process is here:
 772
 773   %s/autobuild.log
 774 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 775
 776     if options.restrict_tests:
 777         text += """
 778 The build was restricted to tests matching %s\n""" % options.restrict_tests
 779
 780     if failed_task != 'rebase':
 781         text += '''
 782 You can see logs of the failed task here:
 783
 784   %s/%s.stdout
 785   %s/%s.stderr
 786
 787 or you can get full logs of all tasks in this job here:
 788
 789   %s/logs.tar.gz
 790
 791 The top commit for the tree that was built was:
 792
 793 %s
 794
 795 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 796
 797     if add_log_tail:
 798         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 799         lines = f.readlines()
 800         log_tail = "".join(lines[-50:])
 801         num_lines = len(lines)
 802         if num_lines < 50:
 803             # Also include stderr (compile failures) if < 50 lines of stdout
 804             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 805             log_tail += "".join(f.readlines()[-(50-num_lines):])
 806
 807         text += '''
 808 The last 50 lines of log messages:
 809
 810 %s
 811     ''' % log_tail
 812         f.close()
 813
 814     logs = os.path.join(gitroot, 'logs.tar.gz')
 815     send_email('autobuild[%s] failure on %s for task %s during %s'
 816                % (options.branch, platform.node(), failed_task, failed_stage),
 817                text, logs)
 818
 819 def email_success(elapsed_time, log_base=None):
 820     '''send an email to options.email about a successful build'''
 821     user = os.getenv("USER")
 822     if log_base is None:
 823         log_base = gitroot
 824     text = '''
 825 Dear Developer,
 826
 827 Your autobuild on %s has succeeded after %.1f minutes.
 828
 829 ''' % (platform.node(), elapsed_time / 60.)
 830
 831     if options.restrict_tests:
 832         text += """
 833 The build was restricted to tests matching %s\n""" % options.restrict_tests
 834
 835     if options.keeplogs:
 836         text += '''
 837
 838 you can get full logs of all tasks in this job here:
 839
 840   %s/logs.tar.gz
 841
 842 ''' % log_base
 843
 844     text += '''
 845 The top commit for the tree that was built was:
 846
 847 %s
 848 ''' % top_commit_msg
 849
 850     logs = os.path.join(gitroot, 'logs.tar.gz')
 851     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 852                text, logs)
 853
 854
 855 (options, args) = parser.parse_args()
 856
 857 if options.retry:
 858     if options.rebase is None:
 859         raise Exception('You can only use --retry if you also rebase')
 860
 861 testbase = "%s/b%u" % (options.testbase, os.getpid())
 862 test_master = "%s/master" % testbase
 863 test_prefix = "%s/prefix" % testbase
 864 test_tmpdir = "%s/tmp" % testbase
 865 os.environ['TMPDIR'] = test_tmpdir
 866
 867 # get the top commit message, for emails
 868 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 869
 870 try:
 871     os.makedirs(testbase)
 872 except Exception as reason:
 873     raise Exception("Unable to create %s : %s" % (testbase, reason))
 874 cleanup_list.append(testbase)
 875
 876 if options.daemon:
 877     logfile = os.path.join(testbase, "log")
 878     do_print("Forking into the background, writing progress to %s" % logfile)
 879     daemonize(logfile)
 880
 881 write_pidfile(gitroot + "/autobuild.pid")
 882
 883 start_time = time.time()
 884
 885 while True:
 886     try:
 887         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 888         os.makedirs(test_tmpdir)
 889         # The waf uninstall code removes empty directories all the way
 890         # up the tree.  Creating a file in test_tmpdir stops it from
 891         # being removed.
 892         run_cmd("touch %s" % os.path.join(test_tmpdir,
 893                                           ".directory-is-not-empty"), show=True)
 894         run_cmd("stat %s" % test_tmpdir, show=True)
 895         run_cmd("stat %s" % testbase, show=True)
 896         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 897     except Exception:
 898         cleanup()
 899         raise
 900
 901     try:
 902         try:
 903             if options.rebase is not None:
 904                 rebase_tree(options.rebase, rebase_branch=options.branch)
 905         except Exception:
 906             cleanup_list.append(gitroot + "/autobuild.pid")
 907             cleanup()
 908             elapsed_time = time.time() - start_time
 909             email_failure(-1, 'rebase', 'rebase', 'rebase',
 910                           'rebase on %s failed' % options.branch,
 911                           elapsed_time, log_base=options.log_base)
 912             sys.exit(1)
 913         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 914         if options.tail:
 915             blist.start_tail()
 916         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 917         if status != 0 or errstr != "retry":
 918             break
 919         cleanup()
 920     except Exception:
 921         cleanup()
 922         raise
 923
 924 cleanup_list.append(gitroot + "/autobuild.pid")
 925
 926 do_print(errstr)
 927
 928 blist.kill_kids()
 929 if options.tail:
 930     do_print("waiting for tail to flush")
 931     time.sleep(1)
 932
 933 elapsed_time = time.time() - start_time
 934 if status == 0:
 935     if options.passcmd is not None:
 936         do_print("Running passcmd: %s" % options.passcmd)
 937         run_cmd(options.passcmd, dir=test_master)
 938     if options.pushto is not None:
 939         push_to(options.pushto, push_branch=options.branch)
 940     if options.keeplogs or options.attach_logs:
 941         blist.tarlogs("logs.tar.gz")
 942         do_print("Logs in logs.tar.gz")
 943     if options.always_email:
 944         email_success(elapsed_time, log_base=options.log_base)
 945     blist.remove_logs()
 946     cleanup()
 947     do_print(errstr)
 948     sys.exit(0)
 949
 950 # something failed, gather a tar of the logs
 951 blist.tarlogs("logs.tar.gz")
 952
 953 if options.email is not None:
 954     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 955                   elapsed_time, log_base=options.log_base)
 956 else:
 957     elapsed_minutes = elapsed_time / 60.0
 958     print('''
 959
 960 ####################################################################
 961
 962 AUTOBUILD FAILURE
 963
 964 Your autobuild[%s] on %s failed after %.1f minutes
 965 when trying to test %s with the following error:
 966
 967    %s
 968
 969 the autobuild has been abandoned. Please fix the error and resubmit.
 970
 971 ####################################################################
 972
 973 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 974
 975 cleanup()
 976 do_print(errstr)
 977 do_print("Logs in logs.tar.gz")
 978 sys.exit(status)