script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call,Popen, PIPE
   8 import os, tarfile, sys, time
   9 from optparse import OptionParser
  10 import smtplib
  11 import email
  12 from email.mime.text import MIMEText
  13 from email.mime.base import MIMEBase
  14 from email.mime.application import MIMEApplication
  15 from email.mime.multipart import MIMEMultipart
  16 from distutils.sysconfig import get_python_lib
  17 import platform
  18
  19 os.environ["PYTHONUNBUFFERED"] = "1"
  20
  21 # This speeds up testing remarkably.
  22 os.environ['TDB_NO_FSYNC'] = '1'
  23
  24 cleanup_list = []
  25
  26 builddirs = {
  27     "ctdb"    : "ctdb",
  28     "samba"  : ".",
  29     "samba-nt4"  : ".",
  30     "samba-fileserver"  : ".",
  31     "samba-xc" : ".",
  32     "samba-o3" : ".",
  33     "samba-ctdb" : ".",
  34     "samba-libs"  : ".",
  35     "samba-static"  : ".",
  36     "samba-test-only"  : ".",
  37     "samba-none-env"  : ".",
  38     "samba-ad-dc"  : ".",
  39     "samba-ad-dc-2"  : ".",
  40     "samba-systemkrb5"  : ".",
  41     "samba-nopython"  : ".",
  42     "ldb"     : "lib/ldb",
  43     "tdb"     : "lib/tdb",
  44     "talloc"  : "lib/talloc",
  45     "replace" : "lib/replace",
  46     "tevent"  : "lib/tevent",
  47     "pidl"    : "pidl",
  48     "pass"    : ".",
  49     "fail"    : ".",
  50     "retry"   : "."
  51     }
  52
  53 defaulttasks = [ "ctdb",
  54                  "samba",
  55                  "samba-nt4",
  56                  "samba-fileserver",
  57                  "samba-xc",
  58                  "samba-o3",
  59                  "samba-ctdb",
  60                  "samba-libs",
  61                  "samba-static",
  62                  "samba-none-env",
  63                  "samba-ad-dc",
  64                  "samba-ad-dc-2",
  65                  "samba-systemkrb5",
  66                  "samba-nopython",
  67                  "ldb",
  68                  "tdb",
  69                  "talloc",
  70                  "replace",
  71                  "tevent",
  72                  "pidl" ]
  73
  74 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  75     defaulttasks.remove("samba-o3")
  76
  77 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  78 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  79
  80 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  81 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  82 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  83 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX} ${EXTRA_PYTHON}"
  84 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE"
  85 samba_libs_configure_samba = samba_libs_configure_base + " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
  86
  87 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  88     extra_python = ""
  89 else:
  90     extra_python = "--extra-python=/usr/bin/python3"
  91
  92 tasks = {
  93     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  94                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  95                ("make", "make all", "text/plain"),
  96                ("install", "make install", "text/plain"),
  97                ("test", "make autotest", "text/plain"),
  98                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  99                ("clean", "make clean", "text/plain") ],
 100
 101     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
 102     "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 103                 ("make", "make -j", "text/plain"),
 104                 ("test", "make test FAIL_IMMEDIATELY=1 "
 105                  "TESTS='--exclude-env=none "
 106                  "--exclude-env=nt4_dc "
 107                  "--exclude-env=nt4_member "
 108                  "--exclude-env=ad_dc "
 109                  "--exclude-env=fl2003dc "
 110                  "--exclude-env=fl2008r2dc "
 111                  "--exclude-env=ad_member "
 112                  "--exclude-env=ad_member_idmap_rid "
 113                  "--exclude-env=ad_member_idmap_ad "
 114                  "--exclude-env=chgdcpass "
 115                  "--exclude-env=vampire_2000_dc "
 116                  "--exclude-env=fl2000dc "
 117                  "--exclude-env=fileserver'",
 118                  "text/plain"),
 119                 ("install", "make install", "text/plain"),
 120                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 121                 ("clean", "make clean", "text/plain") ],
 122
 123     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 124     "samba-nt4" : [ ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 125                        ("make", "make -j", "text/plain"),
 126                        ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 127                        ("install", "make install", "text/plain"),
 128                        ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 129                        ("clean", "make clean", "text/plain") ],
 130
 131     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 132     "samba-fileserver" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 133                       ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 134                       ("make", "make -j", "text/plain"),
 135                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
 136                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 137
 138     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 139     "samba-ad-dc" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 140                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 141                       ("make", "make -j", "text/plain"),
 142                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='"
 143                        "--include-env=ad_dc "
 144                        "--include-env=fl2003dc "
 145                        "--include-env=fl2008r2dc "
 146                        "--include-env=ad_member "
 147                        "--include-env=ad_member_idmap_rid "
 148                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 149                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 150
 151     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 152     "samba-ad-dc-2" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 153                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 154                       ("make", "make -j", "text/plain"),
 155                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
 156                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 157
 158     "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 159                           ("make", "make -j", "text/plain"),
 160                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
 161
 162     # Test cross-compile infrastructure
 163     "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 164                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 165                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 166                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 167                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 168                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 169
 170     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 171     "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 172                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 173                    ("make", "make -j", "text/plain"),
 174                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 175                    ("install", "make install", "text/plain"),
 176                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 177                    ("clean", "make clean", "text/plain") ],
 178
 179     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 180
 181                      # make sure we have tdb around:
 182                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 183                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 184                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 185
 186
 187                      # build samba with cluster support (also building ctdb):
 188                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 189                      ("samba-make", "make", "text/plain"),
 190                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 191                      ("samba-install", "make install", "text/plain"),
 192                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 193
 194                      # clean up:
 195                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 196                      ("clean", "make clean", "text/plain"),
 197                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
 198
 199     "samba-libs" : [
 200                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 201                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 202                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 203                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 204
 205                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 206                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 207                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 208
 209                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 210                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 211                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 212
 213                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 214                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 215                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 216
 217                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 218                       ("nondevel-make", "make -j", "text/plain"),
 219                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 220                       ("nondevel-install", "make install", "text/plain"),
 221                       ("nondevel-dist", "make dist", "text/plain"),
 222
 223                       # retry with all modules shared
 224                       ("allshared-distclean", "make distclean", "text/plain"),
 225                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 226                       ("allshared-make", "make -j", "text/plain")],
 227
 228     "samba-none-env" : [
 229                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 230                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 231                       ("make", "make -j", "text/plain"),
 232                       ("test", "make test "
 233                        "FAIL_IMMEDIATELY=1 "
 234                        "TESTS='--include-env=none'",
 235                        "text/plain")],
 236
 237     "samba-static" : [
 238                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 239                       # build with all modules static
 240                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 241                       ("allstatic-make", "make -j", "text/plain"),
 242
 243                       # retry without any required modules
 244                       ("none-distclean", "make distclean", "text/plain"),
 245                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 246                       ("none-make", "make -j", "text/plain"),
 247
 248                       # retry with nonshared smbd and smbtorture
 249                       ("nonshared-distclean", "make distclean", "text/plain"),
 250                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 251                       ("nonshared-make", "make -j", "text/plain")],
 252
 253     "samba-systemkrb5" : [
 254                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 255                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 256                       ("make", "make -j", "text/plain"),
 257                       # we currently cannot run a full make test, a limited list of tests could be run
 258                       # via "make test TESTS=sometests"
 259                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 260                       ("install", "make install", "text/plain"),
 261                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 262                       ("clean", "make clean", "text/plain")
 263                       ],
 264
 265     # Test Samba without python still builds.  When this test fails
 266     # due to more use of Python, the expectations is that the newly
 267     # failing part of the code should be disabled when
 268     # --disable-python is set (rather than major work being done to
 269     # support this environment).  The target here is for vendors
 270     # shipping a minimal smbd.
 271     "samba-nopython" : [
 272                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 273                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 274                       ("make", "make -j", "text/plain"),
 275                       ("install", "make install", "text/plain"),
 276                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 277                       ("clean", "make clean", "text/plain")
 278                       ],
 279
 280
 281
 282     "ldb" : [
 283               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 284               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 285               ("make", "make", "text/plain"),
 286               ("install", "make install", "text/plain"),
 287               ("test", "make test", "text/plain"),
 288               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 289               ("distcheck", "make distcheck", "text/plain"),
 290               ("clean", "make clean", "text/plain") ],
 291
 292     "tdb" : [
 293               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 294               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 295               ("make", "make", "text/plain"),
 296               ("install", "make install", "text/plain"),
 297               ("test", "make test", "text/plain"),
 298               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 299               ("distcheck", "make distcheck", "text/plain"),
 300               ("clean", "make clean", "text/plain") ],
 301
 302     "talloc" : [
 303                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 304                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 305                  ("make", "make", "text/plain"),
 306                  ("install", "make install", "text/plain"),
 307                  ("test", "make test", "text/plain"),
 308                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 309                  ("distcheck", "make distcheck", "text/plain"),
 310                  ("clean", "make clean", "text/plain") ],
 311
 312     "replace" : [
 313                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 314                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 315                   ("make", "make", "text/plain"),
 316                   ("install", "make install", "text/plain"),
 317                   ("test", "make test", "text/plain"),
 318                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 319                   ("distcheck", "make distcheck", "text/plain"),
 320                   ("clean", "make clean", "text/plain") ],
 321
 322     "tevent" : [
 323                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 324                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 325                  ("make", "make", "text/plain"),
 326                  ("install", "make install", "text/plain"),
 327                  ("test", "make test", "text/plain"),
 328                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 329                  ("distcheck", "make distcheck", "text/plain"),
 330                  ("clean", "make clean", "text/plain") ],
 331
 332     "pidl" : [
 333                ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 334                ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 335                ("touch", "touch *.yp", "text/plain"),
 336                ("make", "make", "text/plain"),
 337                ("test", "make test", "text/plain"),
 338                ("install", "make install", "text/plain"),
 339                ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 340                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 341                ("clean", "make clean", "text/plain") ],
 342
 343     # these are useful for debugging autobuild
 344     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
 345     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
 346 }
 347
 348 def do_print(msg):
 349     print("%s" % msg)
 350     sys.stdout.flush()
 351     sys.stderr.flush()
 352
 353 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 354     if show is None:
 355         show = options.verbose
 356     if show:
 357         do_print("Running: '%s' in '%s'" % (cmd, dir))
 358     if output:
 359         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 360     elif checkfail:
 361         return check_call(cmd, shell=True, cwd=dir)
 362     else:
 363         return call(cmd, shell=True, cwd=dir)
 364
 365
 366 class builder(object):
 367     '''handle build of one directory'''
 368
 369     def __init__(self, name, sequence, cp=True):
 370         self.name = name
 371         self.dir = builddirs[name]
 372
 373         self.tag = self.name.replace('/', '_')
 374         self.sequence = sequence
 375         self.next = 0
 376         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 377         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 378         if options.verbose:
 379             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 380             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 381         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 382         self.stdout = open(self.stdout_path, 'w')
 383         self.stderr = open(self.stderr_path, 'w')
 384         self.stdin  = open("/dev/null", 'r')
 385         self.sdir = "%s/%s" % (testbase, self.tag)
 386         self.prefix = "%s/%s" % (test_prefix, self.tag)
 387         run_cmd("rm -rf %s" % self.sdir)
 388         run_cmd("rm -rf %s" % self.prefix)
 389         if cp:
 390             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 391         else:
 392             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 393         self.start_next()
 394
 395     def start_next(self):
 396         if self.next == len(self.sequence):
 397             if not options.nocleanup:
 398                 run_cmd("rm -rf %s" % self.sdir)
 399                 run_cmd("rm -rf %s" % self.prefix)
 400             do_print('%s: Completed OK' % self.name)
 401             self.done = True
 402             return
 403         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 404         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 405         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 406         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 407         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 408         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 409 #        if self.output_mime_type == "text/x-subunit":
 410 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 411         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 412         cwd = os.getcwd()
 413         os.chdir("%s/%s" % (self.sdir, self.dir))
 414         self.proc = Popen(self.cmd, shell=True,
 415                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 416         os.chdir(cwd)
 417         self.next += 1
 418
 419
 420 class buildlist(object):
 421     '''handle build of multiple directories'''
 422
 423     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 424         global tasks
 425         self.tlist = []
 426         self.tail_proc = None
 427         self.retry = None
 428         if tasknames == []:
 429             if options.restrict_tests:
 430                 tasknames = ["samba-test-only"]
 431             else:
 432                 tasknames = defaulttasks
 433         else:
 434             # If we are only running one test,
 435             # do not sleep randomly to wait for it to start
 436             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 437
 438         for n in tasknames:
 439             b = builder(n, tasks[n], cp=n is not "pidl")
 440             self.tlist.append(b)
 441         if options.retry:
 442             rebase_remote = "rebaseon"
 443             retry_task = [ ("retry",
 444                             '''set -e
 445                             git remote add -t %s %s %s
 446                             git fetch %s
 447                             while :; do
 448                               sleep 60
 449                               git describe %s/%s > old_remote_branch.desc
 450                               git fetch %s
 451                               git describe %s/%s > remote_branch.desc
 452                               diff old_remote_branch.desc remote_branch.desc
 453                             done
 454                            ''' % (
 455                                rebase_branch, rebase_remote, rebase_url,
 456                                rebase_remote,
 457                                rebase_remote, rebase_branch,
 458                                rebase_remote,
 459                                rebase_remote, rebase_branch
 460                            ),
 461                            "test/plain" ) ]
 462
 463             self.retry = builder('retry', retry_task, cp=False)
 464             self.need_retry = False
 465
 466     def kill_kids(self):
 467         if self.tail_proc is not None:
 468             self.tail_proc.terminate()
 469             self.tail_proc.wait()
 470             self.tail_proc = None
 471         if self.retry is not None:
 472             self.retry.proc.terminate()
 473             self.retry.proc.wait()
 474             self.retry = None
 475         for b in self.tlist:
 476             if b.proc is not None:
 477                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 478                 b.proc.terminate()
 479                 b.proc.wait()
 480                 b.proc = None
 481
 482     def wait_one(self):
 483         while True:
 484             none_running = True
 485             for b in self.tlist:
 486                 if b.proc is None:
 487                     continue
 488                 none_running = False
 489                 b.status = b.proc.poll()
 490                 if b.status is None:
 491                     continue
 492                 b.proc = None
 493                 return b
 494             if options.retry:
 495                 ret = self.retry.proc.poll()
 496                 if ret is not None:
 497                     self.need_retry = True
 498                     self.retry = None
 499                     return None
 500             if none_running:
 501                 return None
 502             time.sleep(0.1)
 503
 504     def run(self):
 505         while True:
 506             b = self.wait_one()
 507             if options.retry and self.need_retry:
 508                 self.kill_kids()
 509                 do_print("retry needed")
 510                 return (0, None, None, None, "retry")
 511             if b is None:
 512                 break
 513             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 514                 self.kill_kids()
 515                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 516             b.start_next()
 517         self.kill_kids()
 518         return (0, None, None, None, "All OK")
 519
 520     def write_system_info(self):
 521         filename = 'system-info.txt'
 522         f = open(filename, 'w')
 523         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo', 'cc --version']:
 524             print('### %s' % cmd, file=f)
 525             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 526             print(file=f)
 527         f.close()
 528         return filename
 529
 530     def tarlogs(self, fname):
 531         tar = tarfile.open(fname, "w:gz")
 532         for b in self.tlist:
 533             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 534             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 535         if os.path.exists("autobuild.log"):
 536             tar.add("autobuild.log")
 537         sys_info = self.write_system_info()
 538         tar.add(sys_info)
 539         tar.close()
 540
 541     def remove_logs(self):
 542         for b in self.tlist:
 543             os.unlink(b.stdout_path)
 544             os.unlink(b.stderr_path)
 545
 546     def start_tail(self):
 547         cwd = os.getcwd()
 548         cmd = "tail -f *.stdout *.stderr"
 549         os.chdir(gitroot)
 550         self.tail_proc = Popen(cmd, shell=True)
 551         os.chdir(cwd)
 552
 553
 554 def cleanup():
 555     if options.nocleanup:
 556         return
 557     run_cmd("stat %s || true" % test_tmpdir, show=True)
 558     run_cmd("stat %s" % testbase, show=True)
 559     do_print("Cleaning up ....")
 560     for d in cleanup_list:
 561         run_cmd("rm -rf %s" % d)
 562
 563
 564 def find_git_root():
 565     '''get to the top of the git repo'''
 566     p=os.getcwd()
 567     while p != '/':
 568         if os.path.isdir(os.path.join(p, ".git")):
 569             return p
 570         p = os.path.abspath(os.path.join(p, '..'))
 571     return None
 572
 573
 574 def daemonize(logfile):
 575     pid = os.fork()
 576     if pid == 0: # Parent
 577         os.setsid()
 578         pid = os.fork()
 579         if pid != 0: # Actual daemon
 580             os._exit(0)
 581     else: # Grandparent
 582         os._exit(0)
 583
 584     import resource      # Resource usage information.
 585     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 586     if maxfd == resource.RLIM_INFINITY:
 587         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
 588     for fd in range(0, maxfd):
 589         try:
 590             os.close(fd)
 591         except OSError:
 592             pass
 593     os.open(logfile, os.O_RDWR | os.O_CREAT)
 594     os.dup2(0, 1)
 595     os.dup2(0, 2)
 596
 597 def write_pidfile(fname):
 598     '''write a pid file, cleanup on exit'''
 599     f = open(fname, mode='w')
 600     f.write("%u\n" % os.getpid())
 601     f.close()
 602
 603
 604 def rebase_tree(rebase_url, rebase_branch = "master"):
 605     rebase_remote = "rebaseon"
 606     do_print("Rebasing on %s" % rebase_url)
 607     run_cmd("git describe HEAD", show=True, dir=test_master)
 608     run_cmd("git remote add -t %s %s %s" %
 609             (rebase_branch, rebase_remote, rebase_url),
 610             show=True, dir=test_master)
 611     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 612     if options.fix_whitespace:
 613         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 614                 (rebase_remote, rebase_branch),
 615                 show=True, dir=test_master)
 616     else:
 617         run_cmd("git rebase --force-rebase %s/%s" %
 618                 (rebase_remote, rebase_branch),
 619                 show=True, dir=test_master)
 620     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 621                    (rebase_remote, rebase_branch),
 622                    dir=test_master, output=True)
 623     if diff == '':
 624         do_print("No differences between HEAD and %s/%s - exiting" %
 625               (rebase_remote, rebase_branch))
 626         sys.exit(0)
 627     run_cmd("git describe %s/%s" %
 628             (rebase_remote, rebase_branch),
 629             show=True, dir=test_master)
 630     run_cmd("git describe HEAD", show=True, dir=test_master)
 631     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 632             (rebase_remote, rebase_branch),
 633             show=True, dir=test_master)
 634
 635 def push_to(push_url, push_branch = "master"):
 636     push_remote = "pushto"
 637     do_print("Pushing to %s" % push_url)
 638     if options.mark:
 639         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 640         run_cmd("git commit --amend -c HEAD", dir=test_master)
 641         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 642         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 643     run_cmd("git remote add -t %s %s %s" %
 644             (push_branch, push_remote, push_url),
 645             show=True, dir=test_master)
 646     run_cmd("git push %s +HEAD:%s" %
 647             (push_remote, push_branch),
 648             show=True, dir=test_master)
 649
 650 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 651
 652 gitroot = find_git_root()
 653 if gitroot is None:
 654     raise Exception("Failed to find git root")
 655
 656 parser = OptionParser()
 657 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 658 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 659 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 660 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 661                   default=def_testbase)
 662 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 663 parser.add_option("", "--verbose", help="show all commands as they are run",
 664                   default=False, action="store_true")
 665 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 666                   default=None, type='str')
 667 parser.add_option("", "--pushto", help="push to a git url on success",
 668                   default=None, type='str')
 669 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 670                   default=False, action="store_true")
 671 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 672                   default=False, action="store_true")
 673 parser.add_option("", "--retry", help="automatically retry if master changes",
 674                   default=False, action="store_true")
 675 parser.add_option("", "--email", help="send email to the given address on failure",
 676                   type='str', default=None)
 677 parser.add_option("", "--email-from", help="send email from the given address",
 678                   type='str', default="autobuild@samba.org")
 679 parser.add_option("", "--email-server", help="send email via the given server",
 680                   type='str', default='localhost')
 681 parser.add_option("", "--always-email", help="always send email, even on success",
 682                   action="store_true")
 683 parser.add_option("", "--daemon", help="daemonize after initial setup",
 684                   action="store_true")
 685 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 686                   default="master", type='str')
 687 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 688                   default=gitroot, type='str')
 689 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 690                   default=False, action="store_true")
 691 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 692                   default='')
 693
 694 def send_email(subject, text, log_tar):
 695     outer = MIMEMultipart()
 696     outer['Subject'] = subject
 697     outer['To'] = options.email
 698     outer['From'] = options.email_from
 699     outer['Date'] = email.utils.formatdate(localtime = True)
 700     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 701     outer.attach(MIMEText(text, 'plain'))
 702     if options.attach_logs:
 703         fp = open(log_tar, 'rb')
 704         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 705         fp.close()
 706         # Set the filename parameter
 707         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 708         outer.attach(msg)
 709     content = outer.as_string()
 710     s = smtplib.SMTP(options.email_server)
 711     s.sendmail(options.email_from, [options.email], content)
 712     s.set_debuglevel(1)
 713     s.quit()
 714
 715 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 716                   elapsed_time, log_base=None, add_log_tail=True):
 717     '''send an email to options.email about the failure'''
 718     elapsed_minutes = elapsed_time / 60.0
 719     user = os.getenv("USER")
 720     if log_base is None:
 721         log_base = gitroot
 722     text = '''
 723 Dear Developer,
 724
 725 Your autobuild on %s failed after %.1f minutes
 726 when trying to test %s with the following error:
 727
 728    %s
 729
 730 the autobuild has been abandoned. Please fix the error and resubmit.
 731
 732 A summary of the autobuild process is here:
 733
 734   %s/autobuild.log
 735 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 736
 737     if options.restrict_tests:
 738         text += """
 739 The build was restricted to tests matching %s\n""" % options.restrict_tests
 740
 741     if failed_task != 'rebase':
 742         text += '''
 743 You can see logs of the failed task here:
 744
 745   %s/%s.stdout
 746   %s/%s.stderr
 747
 748 or you can get full logs of all tasks in this job here:
 749
 750   %s/logs.tar.gz
 751
 752 The top commit for the tree that was built was:
 753
 754 %s
 755
 756 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 757
 758     if add_log_tail:
 759         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 760         lines = f.readlines()
 761         log_tail = "".join(lines[-50:])
 762         num_lines = len(lines)
 763         if num_lines < 50:
 764             # Also include stderr (compile failures) if < 50 lines of stdout
 765             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 766             log_tail += "".join(f.readlines()[-(50-num_lines):])
 767
 768         text += '''
 769 The last 50 lines of log messages:
 770
 771 %s
 772     ''' % log_tail
 773         f.close()
 774
 775     logs = os.path.join(gitroot, 'logs.tar.gz')
 776     send_email('autobuild[%s] failure on %s for task %s during %s'
 777                % (options.branch, platform.node(), failed_task, failed_stage),
 778                text, logs)
 779
 780 def email_success(elapsed_time, log_base=None):
 781     '''send an email to options.email about a successful build'''
 782     user = os.getenv("USER")
 783     if log_base is None:
 784         log_base = gitroot
 785     text = '''
 786 Dear Developer,
 787
 788 Your autobuild on %s has succeeded after %.1f minutes.
 789
 790 ''' % (platform.node(), elapsed_time / 60.)
 791
 792     if options.restrict_tests:
 793         text += """
 794 The build was restricted to tests matching %s\n""" % options.restrict_tests
 795
 796     if options.keeplogs:
 797         text += '''
 798
 799 you can get full logs of all tasks in this job here:
 800
 801   %s/logs.tar.gz
 802
 803 ''' % log_base
 804
 805     text += '''
 806 The top commit for the tree that was built was:
 807
 808 %s
 809 ''' % top_commit_msg
 810
 811     logs = os.path.join(gitroot, 'logs.tar.gz')
 812     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 813                text, logs)
 814
 815
 816 (options, args) = parser.parse_args()
 817
 818 if options.retry:
 819     if options.rebase is None:
 820         raise Exception('You can only use --retry if you also rebase')
 821
 822 testbase = "%s/b%u" % (options.testbase, os.getpid())
 823 test_master = "%s/master" % testbase
 824 test_prefix = "%s/prefix" % testbase
 825 test_tmpdir = "%s/tmp" % testbase
 826 os.environ['TMPDIR'] = test_tmpdir
 827
 828 # get the top commit message, for emails
 829 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 830
 831 try:
 832     os.makedirs(testbase)
 833 except Exception as reason:
 834     raise Exception("Unable to create %s : %s" % (testbase, reason))
 835 cleanup_list.append(testbase)
 836
 837 if options.daemon:
 838     logfile = os.path.join(testbase, "log")
 839     do_print("Forking into the background, writing progress to %s" % logfile)
 840     daemonize(logfile)
 841
 842 write_pidfile(gitroot + "/autobuild.pid")
 843
 844 start_time = time.time()
 845
 846 while True:
 847     try:
 848         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 849         os.makedirs(test_tmpdir)
 850         # The waf uninstall code removes empty directories all the way
 851         # up the tree.  Creating a file in test_tmpdir stops it from
 852         # being removed.
 853         run_cmd("touch %s" % os.path.join(test_tmpdir,
 854                                           ".directory-is-not-empty"), show=True)
 855         run_cmd("stat %s" % test_tmpdir, show=True)
 856         run_cmd("stat %s" % testbase, show=True)
 857         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 858     except Exception:
 859         cleanup()
 860         raise
 861
 862     try:
 863         try:
 864             if options.rebase is not None:
 865                 rebase_tree(options.rebase, rebase_branch=options.branch)
 866         except Exception:
 867             cleanup_list.append(gitroot + "/autobuild.pid")
 868             cleanup()
 869             elapsed_time = time.time() - start_time
 870             email_failure(-1, 'rebase', 'rebase', 'rebase',
 871                           'rebase on %s failed' % options.branch,
 872                           elapsed_time, log_base=options.log_base)
 873             sys.exit(1)
 874         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 875         if options.tail:
 876             blist.start_tail()
 877         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 878         if status != 0 or errstr != "retry":
 879             break
 880         cleanup()
 881     except Exception:
 882         cleanup()
 883         raise
 884
 885 cleanup_list.append(gitroot + "/autobuild.pid")
 886
 887 do_print(errstr)
 888
 889 blist.kill_kids()
 890 if options.tail:
 891     do_print("waiting for tail to flush")
 892     time.sleep(1)
 893
 894 elapsed_time = time.time() - start_time
 895 if status == 0:
 896     if options.passcmd is not None:
 897         do_print("Running passcmd: %s" % options.passcmd)
 898         run_cmd(options.passcmd, dir=test_master)
 899     if options.pushto is not None:
 900         push_to(options.pushto, push_branch=options.branch)
 901     if options.keeplogs or options.attach_logs:
 902         blist.tarlogs("logs.tar.gz")
 903         do_print("Logs in logs.tar.gz")
 904     if options.always_email:
 905         email_success(elapsed_time, log_base=options.log_base)
 906     blist.remove_logs()
 907     cleanup()
 908     do_print(errstr)
 909     sys.exit(0)
 910
 911 # something failed, gather a tar of the logs
 912 blist.tarlogs("logs.tar.gz")
 913
 914 if options.email is not None:
 915     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 916                   elapsed_time, log_base=options.log_base)
 917 else:
 918     elapsed_minutes = elapsed_time / 60.0
 919     print('''
 920
 921 ####################################################################
 922
 923 AUTOBUILD FAILURE
 924
 925 Your autobuild[%s] on %s failed after %.1f minutes
 926 when trying to test %s with the following error:
 927
 928    %s
 929
 930 the autobuild has been abandoned. Please fix the error and resubmit.
 931
 932 ####################################################################
 933
 934 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 935
 936 cleanup()
 937 do_print(errstr)
 938 do_print("Logs in logs.tar.gz")
 939 sys.exit(status)