script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call,Popen, PIPE
   8 import os, tarfile, sys, time
   9 from optparse import OptionParser
  10 import smtplib
  11 import email
  12 from email.mime.text import MIMEText
  13 from email.mime.base import MIMEBase
  14 from email.mime.application import MIMEApplication
  15 from email.mime.multipart import MIMEMultipart
  16 from distutils.sysconfig import get_python_lib
  17 import platform
  18
  19 os.environ["PYTHONUNBUFFERED"] = "1"
  20
  21 # This speeds up testing remarkably.
  22 os.environ['TDB_NO_FSYNC'] = '1'
  23
  24 cleanup_list = []
  25
  26 builddirs = {
  27     "ctdb"    : "ctdb",
  28     "samba"  : ".",
  29     "samba-nt4"  : ".",
  30     "samba-fileserver"  : ".",
  31     "samba-xc" : ".",
  32     "samba-o3" : ".",
  33     "samba-ctdb" : ".",
  34     "samba-libs"  : ".",
  35     "samba-static"  : ".",
  36     "samba-test-only"  : ".",
  37     "samba-none-env"  : ".",
  38     "samba-ad-dc"  : ".",
  39     "samba-ad-dc-2"  : ".",
  40     "samba-systemkrb5"  : ".",
  41     "samba-nopython"  : ".",
  42     "ldb"     : "lib/ldb",
  43     "tdb"     : "lib/tdb",
  44     "talloc"  : "lib/talloc",
  45     "replace" : "lib/replace",
  46     "tevent"  : "lib/tevent",
  47     "pidl"    : "pidl",
  48     "pass"    : ".",
  49     "fail"    : ".",
  50     "retry"   : "."
  51     }
  52
  53 defaulttasks = [ "ctdb",
  54                  "samba",
  55                  "samba-nt4",
  56                  "samba-fileserver",
  57                  "samba-xc",
  58                  "samba-o3",
  59                  "samba-ctdb",
  60                  "samba-libs",
  61                  "samba-static",
  62                  "samba-none-env",
  63                  "samba-ad-dc",
  64                  "samba-ad-dc-2",
  65                  "samba-systemkrb5",
  66                  "samba-nopython",
  67                  "ldb",
  68                  "tdb",
  69                  "talloc",
  70                  "replace",
  71                  "tevent",
  72                  "pidl" ]
  73
  74 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  75     defaulttasks.remove("samba-o3")
  76
  77 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  78 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  79
  80 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  81 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  82 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  83 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  84 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE ${EXTRA_PYTHON}"
  85 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
  86 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  87
  88 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  89     extra_python = ""
  90 else:
  91     extra_python = "--extra-python=/usr/bin/python3"
  92
  93 tasks = {
  94     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  95                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  96                ("make", "make all", "text/plain"),
  97                ("install", "make install", "text/plain"),
  98                ("test", "make autotest", "text/plain"),
  99                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 100                ("clean", "make clean", "text/plain") ],
 101
 102     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
 103     "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 104                 ("make", "make -j", "text/plain"),
 105                 ("test", "make test FAIL_IMMEDIATELY=1 "
 106                  "TESTS='--exclude-env=none "
 107                  "--exclude-env=nt4_dc "
 108                  "--exclude-env=nt4_member "
 109                  "--exclude-env=ad_dc "
 110                  "--exclude-env=fl2003dc "
 111                  "--exclude-env=fl2008r2dc "
 112                  "--exclude-env=ad_member "
 113                  "--exclude-env=ad_member_idmap_rid "
 114                  "--exclude-env=ad_member_idmap_ad "
 115                  "--exclude-env=chgdcpass "
 116                  "--exclude-env=vampire_2000_dc "
 117                  "--exclude-env=fl2000dc "
 118                  "--exclude-env=fileserver'",
 119                  "text/plain"),
 120                 ("install", "make install", "text/plain"),
 121                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 122                 ("clean", "make clean", "text/plain") ],
 123
 124     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 125     "samba-nt4" : [ ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 126                        ("make", "make -j", "text/plain"),
 127                        ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 128                        ("install", "make install", "text/plain"),
 129                        ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 130                        ("clean", "make clean", "text/plain") ],
 131
 132     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 133     "samba-fileserver" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 134                       ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 135                       ("make", "make -j", "text/plain"),
 136                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
 137                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 138
 139     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 140     "samba-ad-dc" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 141                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 142                       ("make", "make -j", "text/plain"),
 143                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='"
 144                        "--include-env=ad_dc "
 145                        "--include-env=fl2003dc "
 146                        "--include-env=fl2008r2dc "
 147                        "--include-env=ad_member "
 148                        "--include-env=ad_member_idmap_rid "
 149                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 150                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 151
 152     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 153     "samba-ad-dc-2" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 154                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 155                       ("make", "make -j", "text/plain"),
 156                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
 157                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 158
 159     "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 160                           ("make", "make -j", "text/plain"),
 161                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
 162
 163     # Test cross-compile infrastructure
 164     "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 165                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 166                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 167                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 168                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 169                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 170
 171     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 172     "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 173                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 174                    ("make", "make -j", "text/plain"),
 175                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 176                    ("install", "make install", "text/plain"),
 177                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 178                    ("clean", "make clean", "text/plain") ],
 179
 180     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 181
 182                      # make sure we have tdb around:
 183                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 184                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 185                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 186
 187
 188                      # build samba with cluster support (also building ctdb):
 189                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 190                      ("samba-make", "make", "text/plain"),
 191                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 192                      ("samba-install", "make install", "text/plain"),
 193                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 194
 195                      # clean up:
 196                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 197                      ("clean", "make clean", "text/plain"),
 198                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
 199
 200     "samba-libs" : [
 201                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 202                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 203                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 204                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 205
 206                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 207                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 208                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 209
 210                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 211                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 212                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 213
 214                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 215                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 216                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 217
 218                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 219                       ("nondevel-make", "make -j", "text/plain"),
 220                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 221                       ("nondevel-install", "make install", "text/plain"),
 222                       ("nondevel-dist", "make dist", "text/plain"),
 223
 224                       # retry with all modules shared
 225                       ("allshared-distclean", "make distclean", "text/plain"),
 226                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 227                       ("allshared-make", "make -j", "text/plain")],
 228
 229     "samba-none-env" : [
 230                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 231                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 232                       ("make", "make -j", "text/plain"),
 233                       ("test", "make test "
 234                        "FAIL_IMMEDIATELY=1 "
 235                        "TESTS='--include-env=none'",
 236                        "text/plain")],
 237
 238     "samba-static" : [
 239                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 240                       # build with all modules static
 241                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 242                       ("allstatic-make", "make -j", "text/plain"),
 243
 244                       # retry without any required modules
 245                       ("none-distclean", "make distclean", "text/plain"),
 246                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 247                       ("none-make", "make -j", "text/plain"),
 248
 249                       # retry with nonshared smbd and smbtorture
 250                       ("nonshared-distclean", "make distclean", "text/plain"),
 251                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 252                       ("nonshared-make", "make -j", "text/plain")],
 253
 254     "samba-systemkrb5" : [
 255                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 256                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 257                       ("make", "make -j", "text/plain"),
 258                       # we currently cannot run a full make test, a limited list of tests could be run
 259                       # via "make test TESTS=sometests"
 260                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 261                       ("install", "make install", "text/plain"),
 262                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 263                       ("clean", "make clean", "text/plain")
 264                       ],
 265
 266     # Test Samba without python still builds.  When this test fails
 267     # due to more use of Python, the expectations is that the newly
 268     # failing part of the code should be disabled when
 269     # --disable-python is set (rather than major work being done to
 270     # support this environment).  The target here is for vendors
 271     # shipping a minimal smbd.
 272     "samba-nopython" : [
 273                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 274                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 275                       ("make", "make -j", "text/plain"),
 276                       ("install", "make install", "text/plain"),
 277                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 278                       ("clean", "make clean", "text/plain"),
 279
 280                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 281                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 282                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 283
 284                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 285                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 286                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 287
 288                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 289                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 290                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 291
 292                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 293                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 294                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 295
 296                       # retry against installed library packages
 297                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 298                       ("libs-make", "make -j", "text/plain"),
 299                       ("libs-install", "make install", "text/plain"),
 300                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 301                       ("libs-clean", "make clean", "text/plain")
 302                       ],
 303
 304
 305
 306     "ldb" : [
 307               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 308               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 309               ("make", "make", "text/plain"),
 310               ("install", "make install", "text/plain"),
 311               ("test", "make test", "text/plain"),
 312               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 313               ("make-no-lmdb", "make", "text/plain"),
 314               ("install-no-lmdb", "make install", "text/plain"),
 315               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 316               ("distcheck", "make distcheck", "text/plain"),
 317               ("clean", "make clean", "text/plain") ],
 318
 319     "tdb" : [
 320               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 321               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 322               ("make", "make", "text/plain"),
 323               ("install", "make install", "text/plain"),
 324               ("test", "make test", "text/plain"),
 325               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 326               ("distcheck", "make distcheck", "text/plain"),
 327               ("clean", "make clean", "text/plain") ],
 328
 329     "talloc" : [
 330                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 331                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 332                  ("make", "make", "text/plain"),
 333                  ("install", "make install", "text/plain"),
 334                  ("test", "make test", "text/plain"),
 335                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 336                  ("distcheck", "make distcheck", "text/plain"),
 337                  ("clean", "make clean", "text/plain") ],
 338
 339     "replace" : [
 340                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 341                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 342                   ("make", "make", "text/plain"),
 343                   ("install", "make install", "text/plain"),
 344                   ("test", "make test", "text/plain"),
 345                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 346                   ("distcheck", "make distcheck", "text/plain"),
 347                   ("clean", "make clean", "text/plain") ],
 348
 349     "tevent" : [
 350                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 351                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 352                  ("make", "make", "text/plain"),
 353                  ("install", "make install", "text/plain"),
 354                  ("test", "make test", "text/plain"),
 355                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 356                  ("distcheck", "make distcheck", "text/plain"),
 357                  ("clean", "make clean", "text/plain") ],
 358
 359     "pidl" : [
 360                ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 361                ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 362                ("touch", "touch *.yp", "text/plain"),
 363                ("make", "make", "text/plain"),
 364                ("test", "make test", "text/plain"),
 365                ("install", "make install", "text/plain"),
 366                ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 367                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 368                ("clean", "make clean", "text/plain") ],
 369
 370     # these are useful for debugging autobuild
 371     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
 372     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
 373 }
 374
 375 def do_print(msg):
 376     print("%s" % msg)
 377     sys.stdout.flush()
 378     sys.stderr.flush()
 379
 380 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 381     if show is None:
 382         show = options.verbose
 383     if show:
 384         do_print("Running: '%s' in '%s'" % (cmd, dir))
 385     if output:
 386         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 387     elif checkfail:
 388         return check_call(cmd, shell=True, cwd=dir)
 389     else:
 390         return call(cmd, shell=True, cwd=dir)
 391
 392
 393 class builder(object):
 394     '''handle build of one directory'''
 395
 396     def __init__(self, name, sequence, cp=True):
 397         self.name = name
 398         self.dir = builddirs[name]
 399
 400         self.tag = self.name.replace('/', '_')
 401         self.sequence = sequence
 402         self.next = 0
 403         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 404         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 405         if options.verbose:
 406             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 407             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 408         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 409         self.stdout = open(self.stdout_path, 'w')
 410         self.stderr = open(self.stderr_path, 'w')
 411         self.stdin  = open("/dev/null", 'r')
 412         self.sdir = "%s/%s" % (testbase, self.tag)
 413         self.prefix = "%s/%s" % (test_prefix, self.tag)
 414         run_cmd("rm -rf %s" % self.sdir)
 415         run_cmd("rm -rf %s" % self.prefix)
 416         if cp:
 417             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 418         else:
 419             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 420         self.start_next()
 421
 422     def start_next(self):
 423         if self.next == len(self.sequence):
 424             if not options.nocleanup:
 425                 run_cmd("rm -rf %s" % self.sdir)
 426                 run_cmd("rm -rf %s" % self.prefix)
 427             do_print('%s: Completed OK' % self.name)
 428             self.done = True
 429             return
 430         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 431         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 432         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 433         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 434         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 435         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 436 #        if self.output_mime_type == "text/x-subunit":
 437 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 438         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 439         cwd = os.getcwd()
 440         os.chdir("%s/%s" % (self.sdir, self.dir))
 441         self.proc = Popen(self.cmd, shell=True,
 442                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 443         os.chdir(cwd)
 444         self.next += 1
 445
 446
 447 class buildlist(object):
 448     '''handle build of multiple directories'''
 449
 450     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 451         global tasks
 452         self.tlist = []
 453         self.tail_proc = None
 454         self.retry = None
 455         if tasknames == []:
 456             if options.restrict_tests:
 457                 tasknames = ["samba-test-only"]
 458             else:
 459                 tasknames = defaulttasks
 460         else:
 461             # If we are only running one test,
 462             # do not sleep randomly to wait for it to start
 463             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 464
 465         for n in tasknames:
 466             b = builder(n, tasks[n], cp=n is not "pidl")
 467             self.tlist.append(b)
 468         if options.retry:
 469             rebase_remote = "rebaseon"
 470             retry_task = [ ("retry",
 471                             '''set -e
 472                             git remote add -t %s %s %s
 473                             git fetch %s
 474                             while :; do
 475                               sleep 60
 476                               git describe %s/%s > old_remote_branch.desc
 477                               git fetch %s
 478                               git describe %s/%s > remote_branch.desc
 479                               diff old_remote_branch.desc remote_branch.desc
 480                             done
 481                            ''' % (
 482                                rebase_branch, rebase_remote, rebase_url,
 483                                rebase_remote,
 484                                rebase_remote, rebase_branch,
 485                                rebase_remote,
 486                                rebase_remote, rebase_branch
 487                            ),
 488                            "test/plain" ) ]
 489
 490             self.retry = builder('retry', retry_task, cp=False)
 491             self.need_retry = False
 492
 493     def kill_kids(self):
 494         if self.tail_proc is not None:
 495             self.tail_proc.terminate()
 496             self.tail_proc.wait()
 497             self.tail_proc = None
 498         if self.retry is not None:
 499             self.retry.proc.terminate()
 500             self.retry.proc.wait()
 501             self.retry = None
 502         for b in self.tlist:
 503             if b.proc is not None:
 504                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 505                 b.proc.terminate()
 506                 b.proc.wait()
 507                 b.proc = None
 508
 509     def wait_one(self):
 510         while True:
 511             none_running = True
 512             for b in self.tlist:
 513                 if b.proc is None:
 514                     continue
 515                 none_running = False
 516                 b.status = b.proc.poll()
 517                 if b.status is None:
 518                     continue
 519                 b.proc = None
 520                 return b
 521             if options.retry:
 522                 ret = self.retry.proc.poll()
 523                 if ret is not None:
 524                     self.need_retry = True
 525                     self.retry = None
 526                     return None
 527             if none_running:
 528                 return None
 529             time.sleep(0.1)
 530
 531     def run(self):
 532         while True:
 533             b = self.wait_one()
 534             if options.retry and self.need_retry:
 535                 self.kill_kids()
 536                 do_print("retry needed")
 537                 return (0, None, None, None, "retry")
 538             if b is None:
 539                 break
 540             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 541                 self.kill_kids()
 542                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 543             b.start_next()
 544         self.kill_kids()
 545         return (0, None, None, None, "All OK")
 546
 547     def write_system_info(self):
 548         filename = 'system-info.txt'
 549         f = open(filename, 'w')
 550         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 551                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 552             print('### %s' % cmd, file=f)
 553             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 554             print(file=f)
 555         f.close()
 556         return filename
 557
 558     def tarlogs(self, fname):
 559         tar = tarfile.open(fname, "w:gz")
 560         for b in self.tlist:
 561             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 562             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 563         if os.path.exists("autobuild.log"):
 564             tar.add("autobuild.log")
 565         sys_info = self.write_system_info()
 566         tar.add(sys_info)
 567         tar.close()
 568
 569     def remove_logs(self):
 570         for b in self.tlist:
 571             os.unlink(b.stdout_path)
 572             os.unlink(b.stderr_path)
 573
 574     def start_tail(self):
 575         cwd = os.getcwd()
 576         cmd = "tail -f *.stdout *.stderr"
 577         os.chdir(gitroot)
 578         self.tail_proc = Popen(cmd, shell=True)
 579         os.chdir(cwd)
 580
 581
 582 def cleanup():
 583     if options.nocleanup:
 584         return
 585     run_cmd("stat %s || true" % test_tmpdir, show=True)
 586     run_cmd("stat %s" % testbase, show=True)
 587     do_print("Cleaning up ....")
 588     for d in cleanup_list:
 589         run_cmd("rm -rf %s" % d)
 590
 591
 592 def find_git_root():
 593     '''get to the top of the git repo'''
 594     p=os.getcwd()
 595     while p != '/':
 596         if os.path.isdir(os.path.join(p, ".git")):
 597             return p
 598         p = os.path.abspath(os.path.join(p, '..'))
 599     return None
 600
 601
 602 def daemonize(logfile):
 603     pid = os.fork()
 604     if pid == 0: # Parent
 605         os.setsid()
 606         pid = os.fork()
 607         if pid != 0: # Actual daemon
 608             os._exit(0)
 609     else: # Grandparent
 610         os._exit(0)
 611
 612     import resource      # Resource usage information.
 613     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 614     if maxfd == resource.RLIM_INFINITY:
 615         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
 616     for fd in range(0, maxfd):
 617         try:
 618             os.close(fd)
 619         except OSError:
 620             pass
 621     os.open(logfile, os.O_RDWR | os.O_CREAT)
 622     os.dup2(0, 1)
 623     os.dup2(0, 2)
 624
 625 def write_pidfile(fname):
 626     '''write a pid file, cleanup on exit'''
 627     f = open(fname, mode='w')
 628     f.write("%u\n" % os.getpid())
 629     f.close()
 630
 631
 632 def rebase_tree(rebase_url, rebase_branch = "master"):
 633     rebase_remote = "rebaseon"
 634     do_print("Rebasing on %s" % rebase_url)
 635     run_cmd("git describe HEAD", show=True, dir=test_master)
 636     run_cmd("git remote add -t %s %s %s" %
 637             (rebase_branch, rebase_remote, rebase_url),
 638             show=True, dir=test_master)
 639     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 640     if options.fix_whitespace:
 641         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 642                 (rebase_remote, rebase_branch),
 643                 show=True, dir=test_master)
 644     else:
 645         run_cmd("git rebase --force-rebase %s/%s" %
 646                 (rebase_remote, rebase_branch),
 647                 show=True, dir=test_master)
 648     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 649                    (rebase_remote, rebase_branch),
 650                    dir=test_master, output=True)
 651     if diff == '':
 652         do_print("No differences between HEAD and %s/%s - exiting" %
 653               (rebase_remote, rebase_branch))
 654         sys.exit(0)
 655     run_cmd("git describe %s/%s" %
 656             (rebase_remote, rebase_branch),
 657             show=True, dir=test_master)
 658     run_cmd("git describe HEAD", show=True, dir=test_master)
 659     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 660             (rebase_remote, rebase_branch),
 661             show=True, dir=test_master)
 662
 663 def push_to(push_url, push_branch = "master"):
 664     push_remote = "pushto"
 665     do_print("Pushing to %s" % push_url)
 666     if options.mark:
 667         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 668         run_cmd("git commit --amend -c HEAD", dir=test_master)
 669         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 670         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 671     run_cmd("git remote add -t %s %s %s" %
 672             (push_branch, push_remote, push_url),
 673             show=True, dir=test_master)
 674     run_cmd("git push %s +HEAD:%s" %
 675             (push_remote, push_branch),
 676             show=True, dir=test_master)
 677
 678 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 679
 680 gitroot = find_git_root()
 681 if gitroot is None:
 682     raise Exception("Failed to find git root")
 683
 684 parser = OptionParser()
 685 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 686 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 687 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 688 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 689                   default=def_testbase)
 690 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 691 parser.add_option("", "--verbose", help="show all commands as they are run",
 692                   default=False, action="store_true")
 693 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 694                   default=None, type='str')
 695 parser.add_option("", "--pushto", help="push to a git url on success",
 696                   default=None, type='str')
 697 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 698                   default=False, action="store_true")
 699 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 700                   default=False, action="store_true")
 701 parser.add_option("", "--retry", help="automatically retry if master changes",
 702                   default=False, action="store_true")
 703 parser.add_option("", "--email", help="send email to the given address on failure",
 704                   type='str', default=None)
 705 parser.add_option("", "--email-from", help="send email from the given address",
 706                   type='str', default="autobuild@samba.org")
 707 parser.add_option("", "--email-server", help="send email via the given server",
 708                   type='str', default='localhost')
 709 parser.add_option("", "--always-email", help="always send email, even on success",
 710                   action="store_true")
 711 parser.add_option("", "--daemon", help="daemonize after initial setup",
 712                   action="store_true")
 713 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 714                   default="master", type='str')
 715 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 716                   default=gitroot, type='str')
 717 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 718                   default=False, action="store_true")
 719 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 720                   default='')
 721
 722 def send_email(subject, text, log_tar):
 723     if options.email is None:
 724         do_print("not sending email because the recipient is not set")
 725         do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
 726                  (subject, text))
 727         return
 728     outer = MIMEMultipart()
 729     outer['Subject'] = subject
 730     outer['To'] = options.email
 731     outer['From'] = options.email_from
 732     outer['Date'] = email.utils.formatdate(localtime = True)
 733     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 734     outer.attach(MIMEText(text, 'plain'))
 735     if options.attach_logs:
 736         fp = open(log_tar, 'rb')
 737         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 738         fp.close()
 739         # Set the filename parameter
 740         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 741         outer.attach(msg)
 742     content = outer.as_string()
 743     s = smtplib.SMTP(options.email_server)
 744     s.sendmail(options.email_from, [options.email], content)
 745     s.set_debuglevel(1)
 746     s.quit()
 747
 748 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 749                   elapsed_time, log_base=None, add_log_tail=True):
 750     '''send an email to options.email about the failure'''
 751     elapsed_minutes = elapsed_time / 60.0
 752     user = os.getenv("USER")
 753     if log_base is None:
 754         log_base = gitroot
 755     text = '''
 756 Dear Developer,
 757
 758 Your autobuild on %s failed after %.1f minutes
 759 when trying to test %s with the following error:
 760
 761    %s
 762
 763 the autobuild has been abandoned. Please fix the error and resubmit.
 764
 765 A summary of the autobuild process is here:
 766
 767   %s/autobuild.log
 768 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 769
 770     if options.restrict_tests:
 771         text += """
 772 The build was restricted to tests matching %s\n""" % options.restrict_tests
 773
 774     if failed_task != 'rebase':
 775         text += '''
 776 You can see logs of the failed task here:
 777
 778   %s/%s.stdout
 779   %s/%s.stderr
 780
 781 or you can get full logs of all tasks in this job here:
 782
 783   %s/logs.tar.gz
 784
 785 The top commit for the tree that was built was:
 786
 787 %s
 788
 789 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 790
 791     if add_log_tail:
 792         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 793         lines = f.readlines()
 794         log_tail = "".join(lines[-50:])
 795         num_lines = len(lines)
 796         if num_lines < 50:
 797             # Also include stderr (compile failures) if < 50 lines of stdout
 798             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 799             log_tail += "".join(f.readlines()[-(50-num_lines):])
 800
 801         text += '''
 802 The last 50 lines of log messages:
 803
 804 %s
 805     ''' % log_tail
 806         f.close()
 807
 808     logs = os.path.join(gitroot, 'logs.tar.gz')
 809     send_email('autobuild[%s] failure on %s for task %s during %s'
 810                % (options.branch, platform.node(), failed_task, failed_stage),
 811                text, logs)
 812
 813 def email_success(elapsed_time, log_base=None):
 814     '''send an email to options.email about a successful build'''
 815     user = os.getenv("USER")
 816     if log_base is None:
 817         log_base = gitroot
 818     text = '''
 819 Dear Developer,
 820
 821 Your autobuild on %s has succeeded after %.1f minutes.
 822
 823 ''' % (platform.node(), elapsed_time / 60.)
 824
 825     if options.restrict_tests:
 826         text += """
 827 The build was restricted to tests matching %s\n""" % options.restrict_tests
 828
 829     if options.keeplogs:
 830         text += '''
 831
 832 you can get full logs of all tasks in this job here:
 833
 834   %s/logs.tar.gz
 835
 836 ''' % log_base
 837
 838     text += '''
 839 The top commit for the tree that was built was:
 840
 841 %s
 842 ''' % top_commit_msg
 843
 844     logs = os.path.join(gitroot, 'logs.tar.gz')
 845     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 846                text, logs)
 847
 848
 849 (options, args) = parser.parse_args()
 850
 851 if options.retry:
 852     if options.rebase is None:
 853         raise Exception('You can only use --retry if you also rebase')
 854
 855 testbase = "%s/b%u" % (options.testbase, os.getpid())
 856 test_master = "%s/master" % testbase
 857 test_prefix = "%s/prefix" % testbase
 858 test_tmpdir = "%s/tmp" % testbase
 859 os.environ['TMPDIR'] = test_tmpdir
 860
 861 # get the top commit message, for emails
 862 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 863
 864 try:
 865     os.makedirs(testbase)
 866 except Exception as reason:
 867     raise Exception("Unable to create %s : %s" % (testbase, reason))
 868 cleanup_list.append(testbase)
 869
 870 if options.daemon:
 871     logfile = os.path.join(testbase, "log")
 872     do_print("Forking into the background, writing progress to %s" % logfile)
 873     daemonize(logfile)
 874
 875 write_pidfile(gitroot + "/autobuild.pid")
 876
 877 start_time = time.time()
 878
 879 while True:
 880     try:
 881         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 882         os.makedirs(test_tmpdir)
 883         # The waf uninstall code removes empty directories all the way
 884         # up the tree.  Creating a file in test_tmpdir stops it from
 885         # being removed.
 886         run_cmd("touch %s" % os.path.join(test_tmpdir,
 887                                           ".directory-is-not-empty"), show=True)
 888         run_cmd("stat %s" % test_tmpdir, show=True)
 889         run_cmd("stat %s" % testbase, show=True)
 890         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 891     except Exception:
 892         cleanup()
 893         raise
 894
 895     try:
 896         try:
 897             if options.rebase is not None:
 898                 rebase_tree(options.rebase, rebase_branch=options.branch)
 899         except Exception:
 900             cleanup_list.append(gitroot + "/autobuild.pid")
 901             cleanup()
 902             elapsed_time = time.time() - start_time
 903             email_failure(-1, 'rebase', 'rebase', 'rebase',
 904                           'rebase on %s failed' % options.branch,
 905                           elapsed_time, log_base=options.log_base)
 906             sys.exit(1)
 907         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 908         if options.tail:
 909             blist.start_tail()
 910         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 911         if status != 0 or errstr != "retry":
 912             break
 913         cleanup()
 914     except Exception:
 915         cleanup()
 916         raise
 917
 918 cleanup_list.append(gitroot + "/autobuild.pid")
 919
 920 do_print(errstr)
 921
 922 blist.kill_kids()
 923 if options.tail:
 924     do_print("waiting for tail to flush")
 925     time.sleep(1)
 926
 927 elapsed_time = time.time() - start_time
 928 if status == 0:
 929     if options.passcmd is not None:
 930         do_print("Running passcmd: %s" % options.passcmd)
 931         run_cmd(options.passcmd, dir=test_master)
 932     if options.pushto is not None:
 933         push_to(options.pushto, push_branch=options.branch)
 934     if options.keeplogs or options.attach_logs:
 935         blist.tarlogs("logs.tar.gz")
 936         do_print("Logs in logs.tar.gz")
 937     if options.always_email:
 938         email_success(elapsed_time, log_base=options.log_base)
 939     blist.remove_logs()
 940     cleanup()
 941     do_print(errstr)
 942     sys.exit(0)
 943
 944 # something failed, gather a tar of the logs
 945 blist.tarlogs("logs.tar.gz")
 946
 947 if options.email is not None:
 948     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 949                   elapsed_time, log_base=options.log_base)
 950 else:
 951     elapsed_minutes = elapsed_time / 60.0
 952     print('''
 953
 954 ####################################################################
 955
 956 AUTOBUILD FAILURE
 957
 958 Your autobuild[%s] on %s failed after %.1f minutes
 959 when trying to test %s with the following error:
 960
 961    %s
 962
 963 the autobuild has been abandoned. Please fix the error and resubmit.
 964
 965 ####################################################################
 966
 967 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 968
 969 cleanup()
 970 do_print(errstr)
 971 do_print("Logs in logs.tar.gz")
 972 sys.exit(status)