script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call,Popen, PIPE
   8 import os, tarfile, sys, time
   9 from optparse import OptionParser
  10 import smtplib
  11 import email
  12 from email.mime.text import MIMEText
  13 from email.mime.base import MIMEBase
  14 from email.mime.application import MIMEApplication
  15 from email.mime.multipart import MIMEMultipart
  16 from distutils.sysconfig import get_python_lib
  17 import platform
  18
  19 os.environ["PYTHONUNBUFFERED"] = "1"
  20
  21 # This speeds up testing remarkably.
  22 os.environ['TDB_NO_FSYNC'] = '1'
  23
  24 cleanup_list = []
  25
  26 builddirs = {
  27     "ctdb"    : "ctdb",
  28     "samba"  : ".",
  29     "samba-nt4"  : ".",
  30     "samba-fileserver"  : ".",
  31     "samba-xc" : ".",
  32     "samba-o3" : ".",
  33     "samba-ctdb" : ".",
  34     "samba-libs"  : ".",
  35     "samba-static"  : ".",
  36     "samba-test-only"  : ".",
  37     "samba-none-env"  : ".",
  38     "samba-ad-dc"  : ".",
  39     "samba-ad-dc-2"  : ".",
  40     "samba-systemkrb5"  : ".",
  41     "samba-nopython"  : ".",
  42     "ldb"     : "lib/ldb",
  43     "tdb"     : "lib/tdb",
  44     "talloc"  : "lib/talloc",
  45     "replace" : "lib/replace",
  46     "tevent"  : "lib/tevent",
  47     "pidl"    : "pidl",
  48     "pass"    : ".",
  49     "fail"    : ".",
  50     "retry"   : "."
  51     }
  52
  53 defaulttasks = [ "ctdb",
  54                  "samba",
  55                  "samba-nt4",
  56                  "samba-fileserver",
  57                  "samba-xc",
  58                  "samba-o3",
  59                  "samba-ctdb",
  60                  "samba-libs",
  61                  "samba-static",
  62                  "samba-none-env",
  63                  "samba-ad-dc",
  64                  "samba-ad-dc-2",
  65                  "samba-systemkrb5",
  66                  "samba-nopython",
  67                  "ldb",
  68                  "tdb",
  69                  "talloc",
  70                  "replace",
  71                  "tevent",
  72                  "pidl" ]
  73
  74 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  75     defaulttasks.remove("samba-o3")
  76
  77 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  78 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  79
  80 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  81 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  82 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  83 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX} ${EXTRA_PYTHON}"
  84 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE"
  85 samba_libs_configure_samba = samba_libs_configure_base + " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
  86
  87 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  88     extra_python = ""
  89 else:
  90     extra_python = "--extra-python=/usr/bin/python3"
  91
  92 tasks = {
  93     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  94                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  95                ("make", "make all", "text/plain"),
  96                ("install", "make install", "text/plain"),
  97                ("test", "make autotest", "text/plain"),
  98                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  99                ("clean", "make clean", "text/plain") ],
 100
 101     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
 102     "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 103                 ("make", "make -j", "text/plain"),
 104                 ("test", "make test FAIL_IMMEDIATELY=1 "
 105                  "TESTS='--exclude-env=none "
 106                  "--exclude-env=nt4_dc "
 107                  "--exclude-env=nt4_member "
 108                  "--exclude-env=ad_dc "
 109                  "--exclude-env=fl2003dc "
 110                  "--exclude-env=fl2008r2dc "
 111                  "--exclude-env=ad_member "
 112                  "--exclude-env=ad_member_idmap_rid "
 113                  "--exclude-env=ad_member_idmap_ad "
 114                  "--exclude-env=chgdcpass "
 115                  "--exclude-env=vampire_2000_dc "
 116                  "--exclude-env=fl2000dc "
 117                  "--exclude-env=fileserver'",
 118                  "text/plain"),
 119                 ("install", "make install", "text/plain"),
 120                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 121                 ("clean", "make clean", "text/plain") ],
 122
 123     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 124     "samba-nt4" : [ ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 125                        ("make", "make -j", "text/plain"),
 126                        ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 127                        ("install", "make install", "text/plain"),
 128                        ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 129                        ("clean", "make clean", "text/plain") ],
 130
 131     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 132     "samba-fileserver" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 133                       ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 134                       ("make", "make -j", "text/plain"),
 135                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
 136                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 137
 138     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 139     "samba-ad-dc" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 140                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 141                       ("make", "make -j", "text/plain"),
 142                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='"
 143                        "--include-env=ad_dc "
 144                        "--include-env=fl2003dc "
 145                        "--include-env=fl2008r2dc "
 146                        "--include-env=ad_member "
 147                        "--include-env=ad_member_idmap_rid "
 148                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 149                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 150
 151     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 152     "samba-ad-dc-2" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 153                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 154                       ("make", "make -j", "text/plain"),
 155                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
 156                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 157
 158     "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 159                           ("make", "make -j", "text/plain"),
 160                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
 161
 162     # Test cross-compile infrastructure
 163     "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 164                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 165                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 166                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 167                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 168                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 169
 170     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 171     "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 172                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 173                    ("make", "make -j", "text/plain"),
 174                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 175                    ("install", "make install", "text/plain"),
 176                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 177                    ("clean", "make clean", "text/plain") ],
 178
 179     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 180
 181                      # make sure we have tdb around:
 182                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 183                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 184                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 185
 186
 187                      # build samba with cluster support (also building ctdb):
 188                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 189                      ("samba-make", "make", "text/plain"),
 190                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 191                      ("samba-install", "make install", "text/plain"),
 192                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 193
 194                      # clean up:
 195                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 196                      ("clean", "make clean", "text/plain"),
 197                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
 198
 199     "samba-libs" : [
 200                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 201                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 202                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 203                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 204
 205                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 206                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 207                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 208
 209                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 210                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 211                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 212
 213                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 214                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 215                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 216
 217                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 218                       ("nondevel-make", "make -j", "text/plain"),
 219                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 220                       ("nondevel-install", "make install", "text/plain"),
 221                       ("nondevel-dist", "make dist", "text/plain"),
 222
 223                       # retry with all modules shared
 224                       ("allshared-distclean", "make distclean", "text/plain"),
 225                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 226                       ("allshared-make", "make -j", "text/plain")],
 227
 228     "samba-none-env" : [
 229                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 230                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 231                       ("make", "make -j", "text/plain"),
 232                       ("test", "make test "
 233                        "FAIL_IMMEDIATELY=1 "
 234                        "TESTS='--include-env=none'",
 235                        "text/plain")],
 236
 237     "samba-static" : [
 238                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 239                       # build with all modules static
 240                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 241                       ("allstatic-make", "make -j", "text/plain"),
 242
 243                       # retry without any required modules
 244                       ("none-distclean", "make distclean", "text/plain"),
 245                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 246                       ("none-make", "make -j", "text/plain"),
 247
 248                       # retry with nonshared smbd and smbtorture
 249                       ("nonshared-distclean", "make distclean", "text/plain"),
 250                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 251                       ("nonshared-make", "make -j", "text/plain")],
 252
 253     "samba-systemkrb5" : [
 254                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 255                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 256                       ("make", "make -j", "text/plain"),
 257                       # we currently cannot run a full make test, a limited list of tests could be run
 258                       # via "make test TESTS=sometests"
 259                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 260                       ("install", "make install", "text/plain"),
 261                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 262                       ("clean", "make clean", "text/plain")
 263                       ],
 264
 265     # Test Samba without python still builds.  When this test fails
 266     # due to more use of Python, the expectations is that the newly
 267     # failing part of the code should be disabled when
 268     # --disable-python is set (rather than major work being done to
 269     # support this environment).  The target here is for vendors
 270     # shipping a minimal smbd.
 271     "samba-nopython" : [
 272                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 273                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 274                       ("make", "make -j", "text/plain"),
 275                       ("install", "make install", "text/plain"),
 276                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 277                       ("clean", "make clean", "text/plain")
 278                       ],
 279
 280
 281
 282     "ldb" : [
 283               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 284               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 285               ("make", "make", "text/plain"),
 286               ("install", "make install", "text/plain"),
 287               ("test", "make test", "text/plain"),
 288               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 289               ("make-no-lmdb", "make", "text/plain"),
 290               ("install-no-lmdb", "make install", "text/plain"),
 291               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 292               ("distcheck", "make distcheck", "text/plain"),
 293               ("clean", "make clean", "text/plain") ],
 294
 295     "tdb" : [
 296               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 297               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 298               ("make", "make", "text/plain"),
 299               ("install", "make install", "text/plain"),
 300               ("test", "make test", "text/plain"),
 301               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 302               ("distcheck", "make distcheck", "text/plain"),
 303               ("clean", "make clean", "text/plain") ],
 304
 305     "talloc" : [
 306                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 307                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 308                  ("make", "make", "text/plain"),
 309                  ("install", "make install", "text/plain"),
 310                  ("test", "make test", "text/plain"),
 311                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 312                  ("distcheck", "make distcheck", "text/plain"),
 313                  ("clean", "make clean", "text/plain") ],
 314
 315     "replace" : [
 316                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 317                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 318                   ("make", "make", "text/plain"),
 319                   ("install", "make install", "text/plain"),
 320                   ("test", "make test", "text/plain"),
 321                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 322                   ("distcheck", "make distcheck", "text/plain"),
 323                   ("clean", "make clean", "text/plain") ],
 324
 325     "tevent" : [
 326                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 327                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 328                  ("make", "make", "text/plain"),
 329                  ("install", "make install", "text/plain"),
 330                  ("test", "make test", "text/plain"),
 331                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 332                  ("distcheck", "make distcheck", "text/plain"),
 333                  ("clean", "make clean", "text/plain") ],
 334
 335     "pidl" : [
 336                ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 337                ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 338                ("touch", "touch *.yp", "text/plain"),
 339                ("make", "make", "text/plain"),
 340                ("test", "make test", "text/plain"),
 341                ("install", "make install", "text/plain"),
 342                ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 343                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 344                ("clean", "make clean", "text/plain") ],
 345
 346     # these are useful for debugging autobuild
 347     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
 348     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
 349 }
 350
 351 def do_print(msg):
 352     print("%s" % msg)
 353     sys.stdout.flush()
 354     sys.stderr.flush()
 355
 356 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 357     if show is None:
 358         show = options.verbose
 359     if show:
 360         do_print("Running: '%s' in '%s'" % (cmd, dir))
 361     if output:
 362         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 363     elif checkfail:
 364         return check_call(cmd, shell=True, cwd=dir)
 365     else:
 366         return call(cmd, shell=True, cwd=dir)
 367
 368
 369 class builder(object):
 370     '''handle build of one directory'''
 371
 372     def __init__(self, name, sequence, cp=True):
 373         self.name = name
 374         self.dir = builddirs[name]
 375
 376         self.tag = self.name.replace('/', '_')
 377         self.sequence = sequence
 378         self.next = 0
 379         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 380         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 381         if options.verbose:
 382             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 383             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 384         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 385         self.stdout = open(self.stdout_path, 'w')
 386         self.stderr = open(self.stderr_path, 'w')
 387         self.stdin  = open("/dev/null", 'r')
 388         self.sdir = "%s/%s" % (testbase, self.tag)
 389         self.prefix = "%s/%s" % (test_prefix, self.tag)
 390         run_cmd("rm -rf %s" % self.sdir)
 391         run_cmd("rm -rf %s" % self.prefix)
 392         if cp:
 393             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 394         else:
 395             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 396         self.start_next()
 397
 398     def start_next(self):
 399         if self.next == len(self.sequence):
 400             if not options.nocleanup:
 401                 run_cmd("rm -rf %s" % self.sdir)
 402                 run_cmd("rm -rf %s" % self.prefix)
 403             do_print('%s: Completed OK' % self.name)
 404             self.done = True
 405             return
 406         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 407         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 408         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 409         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 410         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 411         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 412 #        if self.output_mime_type == "text/x-subunit":
 413 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 414         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 415         cwd = os.getcwd()
 416         os.chdir("%s/%s" % (self.sdir, self.dir))
 417         self.proc = Popen(self.cmd, shell=True,
 418                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 419         os.chdir(cwd)
 420         self.next += 1
 421
 422
 423 class buildlist(object):
 424     '''handle build of multiple directories'''
 425
 426     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 427         global tasks
 428         self.tlist = []
 429         self.tail_proc = None
 430         self.retry = None
 431         if tasknames == []:
 432             if options.restrict_tests:
 433                 tasknames = ["samba-test-only"]
 434             else:
 435                 tasknames = defaulttasks
 436         else:
 437             # If we are only running one test,
 438             # do not sleep randomly to wait for it to start
 439             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 440
 441         for n in tasknames:
 442             b = builder(n, tasks[n], cp=n is not "pidl")
 443             self.tlist.append(b)
 444         if options.retry:
 445             rebase_remote = "rebaseon"
 446             retry_task = [ ("retry",
 447                             '''set -e
 448                             git remote add -t %s %s %s
 449                             git fetch %s
 450                             while :; do
 451                               sleep 60
 452                               git describe %s/%s > old_remote_branch.desc
 453                               git fetch %s
 454                               git describe %s/%s > remote_branch.desc
 455                               diff old_remote_branch.desc remote_branch.desc
 456                             done
 457                            ''' % (
 458                                rebase_branch, rebase_remote, rebase_url,
 459                                rebase_remote,
 460                                rebase_remote, rebase_branch,
 461                                rebase_remote,
 462                                rebase_remote, rebase_branch
 463                            ),
 464                            "test/plain" ) ]
 465
 466             self.retry = builder('retry', retry_task, cp=False)
 467             self.need_retry = False
 468
 469     def kill_kids(self):
 470         if self.tail_proc is not None:
 471             self.tail_proc.terminate()
 472             self.tail_proc.wait()
 473             self.tail_proc = None
 474         if self.retry is not None:
 475             self.retry.proc.terminate()
 476             self.retry.proc.wait()
 477             self.retry = None
 478         for b in self.tlist:
 479             if b.proc is not None:
 480                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 481                 b.proc.terminate()
 482                 b.proc.wait()
 483                 b.proc = None
 484
 485     def wait_one(self):
 486         while True:
 487             none_running = True
 488             for b in self.tlist:
 489                 if b.proc is None:
 490                     continue
 491                 none_running = False
 492                 b.status = b.proc.poll()
 493                 if b.status is None:
 494                     continue
 495                 b.proc = None
 496                 return b
 497             if options.retry:
 498                 ret = self.retry.proc.poll()
 499                 if ret is not None:
 500                     self.need_retry = True
 501                     self.retry = None
 502                     return None
 503             if none_running:
 504                 return None
 505             time.sleep(0.1)
 506
 507     def run(self):
 508         while True:
 509             b = self.wait_one()
 510             if options.retry and self.need_retry:
 511                 self.kill_kids()
 512                 do_print("retry needed")
 513                 return (0, None, None, None, "retry")
 514             if b is None:
 515                 break
 516             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 517                 self.kill_kids()
 518                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 519             b.start_next()
 520         self.kill_kids()
 521         return (0, None, None, None, "All OK")
 522
 523     def write_system_info(self):
 524         filename = 'system-info.txt'
 525         f = open(filename, 'w')
 526         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo', 'cc --version']:
 527             print('### %s' % cmd, file=f)
 528             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 529             print(file=f)
 530         f.close()
 531         return filename
 532
 533     def tarlogs(self, fname):
 534         tar = tarfile.open(fname, "w:gz")
 535         for b in self.tlist:
 536             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 537             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 538         if os.path.exists("autobuild.log"):
 539             tar.add("autobuild.log")
 540         sys_info = self.write_system_info()
 541         tar.add(sys_info)
 542         tar.close()
 543
 544     def remove_logs(self):
 545         for b in self.tlist:
 546             os.unlink(b.stdout_path)
 547             os.unlink(b.stderr_path)
 548
 549     def start_tail(self):
 550         cwd = os.getcwd()
 551         cmd = "tail -f *.stdout *.stderr"
 552         os.chdir(gitroot)
 553         self.tail_proc = Popen(cmd, shell=True)
 554         os.chdir(cwd)
 555
 556
 557 def cleanup():
 558     if options.nocleanup:
 559         return
 560     run_cmd("stat %s || true" % test_tmpdir, show=True)
 561     run_cmd("stat %s" % testbase, show=True)
 562     do_print("Cleaning up ....")
 563     for d in cleanup_list:
 564         run_cmd("rm -rf %s" % d)
 565
 566
 567 def find_git_root():
 568     '''get to the top of the git repo'''
 569     p=os.getcwd()
 570     while p != '/':
 571         if os.path.isdir(os.path.join(p, ".git")):
 572             return p
 573         p = os.path.abspath(os.path.join(p, '..'))
 574     return None
 575
 576
 577 def daemonize(logfile):
 578     pid = os.fork()
 579     if pid == 0: # Parent
 580         os.setsid()
 581         pid = os.fork()
 582         if pid != 0: # Actual daemon
 583             os._exit(0)
 584     else: # Grandparent
 585         os._exit(0)
 586
 587     import resource      # Resource usage information.
 588     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 589     if maxfd == resource.RLIM_INFINITY:
 590         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
 591     for fd in range(0, maxfd):
 592         try:
 593             os.close(fd)
 594         except OSError:
 595             pass
 596     os.open(logfile, os.O_RDWR | os.O_CREAT)
 597     os.dup2(0, 1)
 598     os.dup2(0, 2)
 599
 600 def write_pidfile(fname):
 601     '''write a pid file, cleanup on exit'''
 602     f = open(fname, mode='w')
 603     f.write("%u\n" % os.getpid())
 604     f.close()
 605
 606
 607 def rebase_tree(rebase_url, rebase_branch = "master"):
 608     rebase_remote = "rebaseon"
 609     do_print("Rebasing on %s" % rebase_url)
 610     run_cmd("git describe HEAD", show=True, dir=test_master)
 611     run_cmd("git remote add -t %s %s %s" %
 612             (rebase_branch, rebase_remote, rebase_url),
 613             show=True, dir=test_master)
 614     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 615     if options.fix_whitespace:
 616         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 617                 (rebase_remote, rebase_branch),
 618                 show=True, dir=test_master)
 619     else:
 620         run_cmd("git rebase --force-rebase %s/%s" %
 621                 (rebase_remote, rebase_branch),
 622                 show=True, dir=test_master)
 623     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 624                    (rebase_remote, rebase_branch),
 625                    dir=test_master, output=True)
 626     if diff == '':
 627         do_print("No differences between HEAD and %s/%s - exiting" %
 628               (rebase_remote, rebase_branch))
 629         sys.exit(0)
 630     run_cmd("git describe %s/%s" %
 631             (rebase_remote, rebase_branch),
 632             show=True, dir=test_master)
 633     run_cmd("git describe HEAD", show=True, dir=test_master)
 634     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 635             (rebase_remote, rebase_branch),
 636             show=True, dir=test_master)
 637
 638 def push_to(push_url, push_branch = "master"):
 639     push_remote = "pushto"
 640     do_print("Pushing to %s" % push_url)
 641     if options.mark:
 642         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 643         run_cmd("git commit --amend -c HEAD", dir=test_master)
 644         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 645         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 646     run_cmd("git remote add -t %s %s %s" %
 647             (push_branch, push_remote, push_url),
 648             show=True, dir=test_master)
 649     run_cmd("git push %s +HEAD:%s" %
 650             (push_remote, push_branch),
 651             show=True, dir=test_master)
 652
 653 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 654
 655 gitroot = find_git_root()
 656 if gitroot is None:
 657     raise Exception("Failed to find git root")
 658
 659 parser = OptionParser()
 660 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 661 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 662 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 663 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 664                   default=def_testbase)
 665 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 666 parser.add_option("", "--verbose", help="show all commands as they are run",
 667                   default=False, action="store_true")
 668 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 669                   default=None, type='str')
 670 parser.add_option("", "--pushto", help="push to a git url on success",
 671                   default=None, type='str')
 672 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 673                   default=False, action="store_true")
 674 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 675                   default=False, action="store_true")
 676 parser.add_option("", "--retry", help="automatically retry if master changes",
 677                   default=False, action="store_true")
 678 parser.add_option("", "--email", help="send email to the given address on failure",
 679                   type='str', default=None)
 680 parser.add_option("", "--email-from", help="send email from the given address",
 681                   type='str', default="autobuild@samba.org")
 682 parser.add_option("", "--email-server", help="send email via the given server",
 683                   type='str', default='localhost')
 684 parser.add_option("", "--always-email", help="always send email, even on success",
 685                   action="store_true")
 686 parser.add_option("", "--daemon", help="daemonize after initial setup",
 687                   action="store_true")
 688 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 689                   default="master", type='str')
 690 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 691                   default=gitroot, type='str')
 692 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 693                   default=False, action="store_true")
 694 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 695                   default='')
 696
 697 def send_email(subject, text, log_tar):
 698     if options.email is None:
 699         do_print("not sending email because the recipient is not set")
 700         do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
 701                  (subject, text))
 702         return
 703     outer = MIMEMultipart()
 704     outer['Subject'] = subject
 705     outer['To'] = options.email
 706     outer['From'] = options.email_from
 707     outer['Date'] = email.utils.formatdate(localtime = True)
 708     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 709     outer.attach(MIMEText(text, 'plain'))
 710     if options.attach_logs:
 711         fp = open(log_tar, 'rb')
 712         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 713         fp.close()
 714         # Set the filename parameter
 715         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 716         outer.attach(msg)
 717     content = outer.as_string()
 718     s = smtplib.SMTP(options.email_server)
 719     s.sendmail(options.email_from, [options.email], content)
 720     s.set_debuglevel(1)
 721     s.quit()
 722
 723 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 724                   elapsed_time, log_base=None, add_log_tail=True):
 725     '''send an email to options.email about the failure'''
 726     elapsed_minutes = elapsed_time / 60.0
 727     user = os.getenv("USER")
 728     if log_base is None:
 729         log_base = gitroot
 730     text = '''
 731 Dear Developer,
 732
 733 Your autobuild on %s failed after %.1f minutes
 734 when trying to test %s with the following error:
 735
 736    %s
 737
 738 the autobuild has been abandoned. Please fix the error and resubmit.
 739
 740 A summary of the autobuild process is here:
 741
 742   %s/autobuild.log
 743 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 744
 745     if options.restrict_tests:
 746         text += """
 747 The build was restricted to tests matching %s\n""" % options.restrict_tests
 748
 749     if failed_task != 'rebase':
 750         text += '''
 751 You can see logs of the failed task here:
 752
 753   %s/%s.stdout
 754   %s/%s.stderr
 755
 756 or you can get full logs of all tasks in this job here:
 757
 758   %s/logs.tar.gz
 759
 760 The top commit for the tree that was built was:
 761
 762 %s
 763
 764 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 765
 766     if add_log_tail:
 767         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 768         lines = f.readlines()
 769         log_tail = "".join(lines[-50:])
 770         num_lines = len(lines)
 771         if num_lines < 50:
 772             # Also include stderr (compile failures) if < 50 lines of stdout
 773             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 774             log_tail += "".join(f.readlines()[-(50-num_lines):])
 775
 776         text += '''
 777 The last 50 lines of log messages:
 778
 779 %s
 780     ''' % log_tail
 781         f.close()
 782
 783     logs = os.path.join(gitroot, 'logs.tar.gz')
 784     send_email('autobuild[%s] failure on %s for task %s during %s'
 785                % (options.branch, platform.node(), failed_task, failed_stage),
 786                text, logs)
 787
 788 def email_success(elapsed_time, log_base=None):
 789     '''send an email to options.email about a successful build'''
 790     user = os.getenv("USER")
 791     if log_base is None:
 792         log_base = gitroot
 793     text = '''
 794 Dear Developer,
 795
 796 Your autobuild on %s has succeeded after %.1f minutes.
 797
 798 ''' % (platform.node(), elapsed_time / 60.)
 799
 800     if options.restrict_tests:
 801         text += """
 802 The build was restricted to tests matching %s\n""" % options.restrict_tests
 803
 804     if options.keeplogs:
 805         text += '''
 806
 807 you can get full logs of all tasks in this job here:
 808
 809   %s/logs.tar.gz
 810
 811 ''' % log_base
 812
 813     text += '''
 814 The top commit for the tree that was built was:
 815
 816 %s
 817 ''' % top_commit_msg
 818
 819     logs = os.path.join(gitroot, 'logs.tar.gz')
 820     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 821                text, logs)
 822
 823
 824 (options, args) = parser.parse_args()
 825
 826 if options.retry:
 827     if options.rebase is None:
 828         raise Exception('You can only use --retry if you also rebase')
 829
 830 testbase = "%s/b%u" % (options.testbase, os.getpid())
 831 test_master = "%s/master" % testbase
 832 test_prefix = "%s/prefix" % testbase
 833 test_tmpdir = "%s/tmp" % testbase
 834 os.environ['TMPDIR'] = test_tmpdir
 835
 836 # get the top commit message, for emails
 837 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 838
 839 try:
 840     os.makedirs(testbase)
 841 except Exception as reason:
 842     raise Exception("Unable to create %s : %s" % (testbase, reason))
 843 cleanup_list.append(testbase)
 844
 845 if options.daemon:
 846     logfile = os.path.join(testbase, "log")
 847     do_print("Forking into the background, writing progress to %s" % logfile)
 848     daemonize(logfile)
 849
 850 write_pidfile(gitroot + "/autobuild.pid")
 851
 852 start_time = time.time()
 853
 854 while True:
 855     try:
 856         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 857         os.makedirs(test_tmpdir)
 858         # The waf uninstall code removes empty directories all the way
 859         # up the tree.  Creating a file in test_tmpdir stops it from
 860         # being removed.
 861         run_cmd("touch %s" % os.path.join(test_tmpdir,
 862                                           ".directory-is-not-empty"), show=True)
 863         run_cmd("stat %s" % test_tmpdir, show=True)
 864         run_cmd("stat %s" % testbase, show=True)
 865         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 866     except Exception:
 867         cleanup()
 868         raise
 869
 870     try:
 871         try:
 872             if options.rebase is not None:
 873                 rebase_tree(options.rebase, rebase_branch=options.branch)
 874         except Exception:
 875             cleanup_list.append(gitroot + "/autobuild.pid")
 876             cleanup()
 877             elapsed_time = time.time() - start_time
 878             email_failure(-1, 'rebase', 'rebase', 'rebase',
 879                           'rebase on %s failed' % options.branch,
 880                           elapsed_time, log_base=options.log_base)
 881             sys.exit(1)
 882         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 883         if options.tail:
 884             blist.start_tail()
 885         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 886         if status != 0 or errstr != "retry":
 887             break
 888         cleanup()
 889     except Exception:
 890         cleanup()
 891         raise
 892
 893 cleanup_list.append(gitroot + "/autobuild.pid")
 894
 895 do_print(errstr)
 896
 897 blist.kill_kids()
 898 if options.tail:
 899     do_print("waiting for tail to flush")
 900     time.sleep(1)
 901
 902 elapsed_time = time.time() - start_time
 903 if status == 0:
 904     if options.passcmd is not None:
 905         do_print("Running passcmd: %s" % options.passcmd)
 906         run_cmd(options.passcmd, dir=test_master)
 907     if options.pushto is not None:
 908         push_to(options.pushto, push_branch=options.branch)
 909     if options.keeplogs or options.attach_logs:
 910         blist.tarlogs("logs.tar.gz")
 911         do_print("Logs in logs.tar.gz")
 912     if options.always_email:
 913         email_success(elapsed_time, log_base=options.log_base)
 914     blist.remove_logs()
 915     cleanup()
 916     do_print(errstr)
 917     sys.exit(0)
 918
 919 # something failed, gather a tar of the logs
 920 blist.tarlogs("logs.tar.gz")
 921
 922 if options.email is not None:
 923     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 924                   elapsed_time, log_base=options.log_base)
 925 else:
 926     elapsed_minutes = elapsed_time / 60.0
 927     print('''
 928
 929 ####################################################################
 930
 931 AUTOBUILD FAILURE
 932
 933 Your autobuild[%s] on %s failed after %.1f minutes
 934 when trying to test %s with the following error:
 935
 936    %s
 937
 938 the autobuild has been abandoned. Please fix the error and resubmit.
 939
 940 ####################################################################
 941
 942 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 943
 944 cleanup()
 945 do_print(errstr)
 946 do_print("Logs in logs.tar.gz")
 947 sys.exit(status)