script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call,Popen, PIPE
   8 import os, tarfile, sys, time
   9 from optparse import OptionParser
  10 import smtplib
  11 import email
  12 from email.mime.text import MIMEText
  13 from email.mime.base import MIMEBase
  14 from email.mime.application import MIMEApplication
  15 from email.mime.multipart import MIMEMultipart
  16 from distutils.sysconfig import get_python_lib
  17 import platform
  18
  19 os.environ["PYTHONUNBUFFERED"] = "1"
  20
  21 # This speeds up testing remarkably.
  22 os.environ['TDB_NO_FSYNC'] = '1'
  23
  24 cleanup_list = []
  25
  26 builddirs = {
  27     "ctdb"    : "ctdb",
  28     "samba"  : ".",
  29     "samba-nt4"  : ".",
  30     "samba-fileserver"  : ".",
  31     "samba-xc" : ".",
  32     "samba-o3" : ".",
  33     "samba-ctdb" : ".",
  34     "samba-libs"  : ".",
  35     "samba-static"  : ".",
  36     "samba-test-only"  : ".",
  37     "samba-none-env"  : ".",
  38     "samba-ad-dc"  : ".",
  39     "samba-ad-dc-2"  : ".",
  40     "samba-systemkrb5"  : ".",
  41     "samba-nopython"  : ".",
  42     "ldb"     : "lib/ldb",
  43     "tdb"     : "lib/tdb",
  44     "talloc"  : "lib/talloc",
  45     "replace" : "lib/replace",
  46     "tevent"  : "lib/tevent",
  47     "pidl"    : "pidl",
  48     "pass"    : ".",
  49     "fail"    : ".",
  50     "retry"   : "."
  51     }
  52
  53 defaulttasks = [ "ctdb",
  54                  "samba",
  55                  "samba-nt4",
  56                  "samba-fileserver",
  57                  "samba-xc",
  58                  "samba-o3",
  59                  "samba-ctdb",
  60                  "samba-libs",
  61                  "samba-static",
  62                  "samba-none-env",
  63                  "samba-ad-dc",
  64                  "samba-ad-dc-2",
  65                  "samba-systemkrb5",
  66                  "samba-nopython",
  67                  "ldb",
  68                  "tdb",
  69                  "talloc",
  70                  "replace",
  71                  "tevent",
  72                  "pidl" ]
  73
  74 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  75     defaulttasks.remove("samba-o3")
  76
  77 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  78 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  79
  80 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  81 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  82 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  83 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX} ${EXTRA_PYTHON}"
  84 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE"
  85 samba_libs_configure_samba = samba_libs_configure_base + " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
  86
  87 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  88     extra_python = ""
  89 else:
  90     extra_python = "--extra-python=/usr/bin/python3"
  91
  92 tasks = {
  93     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  94                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  95                ("make", "make all", "text/plain"),
  96                ("install", "make install", "text/plain"),
  97                ("test", "make autotest", "text/plain"),
  98                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  99                ("clean", "make clean", "text/plain") ],
 100
 101     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
 102     "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 103                 ("make", "make -j", "text/plain"),
 104                 ("test", "make test FAIL_IMMEDIATELY=1 "
 105                  "TESTS='--exclude-env=none "
 106                  "--exclude-env=nt4_dc "
 107                  "--exclude-env=nt4_member "
 108                  "--exclude-env=ad_dc "
 109                  "--exclude-env=fl2003dc "
 110                  "--exclude-env=fl2008r2dc "
 111                  "--exclude-env=ad_member "
 112                  "--exclude-env=ad_member_idmap_rid "
 113                  "--exclude-env=ad_member_idmap_ad "
 114                  "--exclude-env=chgdcpass "
 115                  "--exclude-env=vampire_2000_dc "
 116                  "--exclude-env=fl2000dc "
 117                  "--exclude-env=fileserver'",
 118                  "text/plain"),
 119                 ("install", "make install", "text/plain"),
 120                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 121                 ("clean", "make clean", "text/plain") ],
 122
 123     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 124     "samba-nt4" : [ ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 125                        ("make", "make -j", "text/plain"),
 126                        ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 127                        ("install", "make install", "text/plain"),
 128                        ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 129                        ("clean", "make clean", "text/plain") ],
 130
 131     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 132     "samba-fileserver" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 133                       ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 134                       ("make", "make -j", "text/plain"),
 135                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
 136                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 137
 138     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 139     "samba-ad-dc" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 140                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 141                       ("make", "make -j", "text/plain"),
 142                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='"
 143                        "--include-env=ad_dc "
 144                        "--include-env=fl2003dc "
 145                        "--include-env=fl2008r2dc "
 146                        "--include-env=ad_member "
 147                        "--include-env=ad_member_idmap_rid "
 148                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 149                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 150
 151     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 152     "samba-ad-dc-2" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 153                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 154                       ("make", "make -j", "text/plain"),
 155                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
 156                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 157
 158     "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 159                           ("make", "make -j", "text/plain"),
 160                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
 161
 162     # Test cross-compile infrastructure
 163     "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 164                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 165                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 166                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 167                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 168                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 169
 170     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 171     "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 172                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 173                    ("make", "make -j", "text/plain"),
 174                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 175                    ("install", "make install", "text/plain"),
 176                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 177                    ("clean", "make clean", "text/plain") ],
 178
 179     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 180
 181                      # make sure we have tdb around:
 182                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 183                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 184                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 185
 186
 187                      # build samba with cluster support (also building ctdb):
 188                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 189                      ("samba-make", "make", "text/plain"),
 190                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 191                      ("samba-install", "make install", "text/plain"),
 192                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 193
 194                      # clean up:
 195                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 196                      ("clean", "make clean", "text/plain"),
 197                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
 198
 199     "samba-libs" : [
 200                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 201                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 202                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 203                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 204
 205                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 206                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 207                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 208
 209                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 210                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 211                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 212
 213                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 214                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 215                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 216
 217                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 218                       ("nondevel-make", "make -j", "text/plain"),
 219                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 220                       ("nondevel-install", "make install", "text/plain"),
 221                       ("nondevel-dist", "make dist", "text/plain"),
 222
 223                       # retry with all modules shared
 224                       ("allshared-distclean", "make distclean", "text/plain"),
 225                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 226                       ("allshared-make", "make -j", "text/plain")],
 227
 228     "samba-none-env" : [
 229                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 230                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 231                       ("make", "make -j", "text/plain"),
 232                       ("test", "make test "
 233                        "FAIL_IMMEDIATELY=1 "
 234                        "TESTS='--include-env=none'",
 235                        "text/plain")],
 236
 237     "samba-static" : [
 238                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 239                       # build with all modules static
 240                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 241                       ("allstatic-make", "make -j", "text/plain"),
 242
 243                       # retry without any required modules
 244                       ("none-distclean", "make distclean", "text/plain"),
 245                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 246                       ("none-make", "make -j", "text/plain"),
 247
 248                       # retry with nonshared smbd and smbtorture
 249                       ("nonshared-distclean", "make distclean", "text/plain"),
 250                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 251                       ("nonshared-make", "make -j", "text/plain")],
 252
 253     "samba-systemkrb5" : [
 254                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 255                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 256                       ("make", "make -j", "text/plain"),
 257                       # we currently cannot run a full make test, a limited list of tests could be run
 258                       # via "make test TESTS=sometests"
 259                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 260                       ("install", "make install", "text/plain"),
 261                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 262                       ("clean", "make clean", "text/plain")
 263                       ],
 264
 265     # Test Samba without python still builds.  When this test fails
 266     # due to more use of Python, the expectations is that the newly
 267     # failing part of the code should be disabled when
 268     # --disable-python is set (rather than major work being done to
 269     # support this environment).  The target here is for vendors
 270     # shipping a minimal smbd.
 271     "samba-nopython" : [
 272                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 273                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 274                       ("make", "make -j", "text/plain"),
 275                       ("install", "make install", "text/plain"),
 276                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 277                       ("clean", "make clean", "text/plain")
 278                       ],
 279
 280
 281
 282     "ldb" : [
 283               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 284               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 285               ("make", "make", "text/plain"),
 286               ("install", "make install", "text/plain"),
 287               ("test", "make test", "text/plain"),
 288               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 289               ("distcheck", "make distcheck", "text/plain"),
 290               ("clean", "make clean", "text/plain") ],
 291
 292     "tdb" : [
 293               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 294               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 295               ("make", "make", "text/plain"),
 296               ("install", "make install", "text/plain"),
 297               ("test", "make test", "text/plain"),
 298               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 299               ("distcheck", "make distcheck", "text/plain"),
 300               ("clean", "make clean", "text/plain") ],
 301
 302     "talloc" : [
 303                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 304                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 305                  ("make", "make", "text/plain"),
 306                  ("install", "make install", "text/plain"),
 307                  ("test", "make test", "text/plain"),
 308                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 309                  ("distcheck", "make distcheck", "text/plain"),
 310                  ("clean", "make clean", "text/plain") ],
 311
 312     "replace" : [
 313                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 314                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 315                   ("make", "make", "text/plain"),
 316                   ("install", "make install", "text/plain"),
 317                   ("test", "make test", "text/plain"),
 318                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 319                   ("distcheck", "make distcheck", "text/plain"),
 320                   ("clean", "make clean", "text/plain") ],
 321
 322     "tevent" : [
 323                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 324                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 325                  ("make", "make", "text/plain"),
 326                  ("install", "make install", "text/plain"),
 327                  ("test", "make test", "text/plain"),
 328                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 329                  ("distcheck", "make distcheck", "text/plain"),
 330                  ("clean", "make clean", "text/plain") ],
 331
 332     "pidl" : [
 333                ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 334                ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 335                ("touch", "touch *.yp", "text/plain"),
 336                ("make", "make", "text/plain"),
 337                ("test", "make test", "text/plain"),
 338                ("install", "make install", "text/plain"),
 339                ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 340                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 341                ("clean", "make clean", "text/plain") ],
 342
 343     # these are useful for debugging autobuild
 344     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
 345     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
 346 }
 347
 348 def do_print(msg):
 349     print("%s" % msg)
 350     sys.stdout.flush()
 351     sys.stderr.flush()
 352
 353 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 354     if show is None:
 355         show = options.verbose
 356     if show:
 357         do_print("Running: '%s' in '%s'" % (cmd, dir))
 358     if output:
 359         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 360     elif checkfail:
 361         return check_call(cmd, shell=True, cwd=dir)
 362     else:
 363         return call(cmd, shell=True, cwd=dir)
 364
 365
 366 class builder(object):
 367     '''handle build of one directory'''
 368
 369     def __init__(self, name, sequence, cp=True):
 370         self.name = name
 371         self.dir = builddirs[name]
 372
 373         self.tag = self.name.replace('/', '_')
 374         self.sequence = sequence
 375         self.next = 0
 376         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 377         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 378         if options.verbose:
 379             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 380             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 381         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 382         self.stdout = open(self.stdout_path, 'w')
 383         self.stderr = open(self.stderr_path, 'w')
 384         self.stdin  = open("/dev/null", 'r')
 385         self.sdir = "%s/%s" % (testbase, self.tag)
 386         self.prefix = "%s/%s" % (test_prefix, self.tag)
 387         run_cmd("rm -rf %s" % self.sdir)
 388         run_cmd("rm -rf %s" % self.prefix)
 389         if cp:
 390             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 391         else:
 392             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 393         self.start_next()
 394
 395     def start_next(self):
 396         if self.next == len(self.sequence):
 397             if not options.nocleanup:
 398                 run_cmd("rm -rf %s" % self.sdir)
 399                 run_cmd("rm -rf %s" % self.prefix)
 400             do_print('%s: Completed OK' % self.name)
 401             self.done = True
 402             return
 403         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 404         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 405         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 406         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 407         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 408         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 409 #        if self.output_mime_type == "text/x-subunit":
 410 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 411         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 412         cwd = os.getcwd()
 413         os.chdir("%s/%s" % (self.sdir, self.dir))
 414         self.proc = Popen(self.cmd, shell=True,
 415                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 416         os.chdir(cwd)
 417         self.next += 1
 418
 419
 420 class buildlist(object):
 421     '''handle build of multiple directories'''
 422
 423     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 424         global tasks
 425         self.tlist = []
 426         self.tail_proc = None
 427         self.retry = None
 428         if tasknames == []:
 429             if options.restrict_tests:
 430                 tasknames = ["samba-test-only"]
 431             else:
 432                 tasknames = defaulttasks
 433         else:
 434             # If we are only running one test,
 435             # do not sleep randomly to wait for it to start
 436             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 437
 438         for n in tasknames:
 439             b = builder(n, tasks[n], cp=n is not "pidl")
 440             self.tlist.append(b)
 441         if options.retry:
 442             rebase_remote = "rebaseon"
 443             retry_task = [ ("retry",
 444                             '''set -e
 445                             git remote add -t %s %s %s
 446                             git fetch %s
 447                             while :; do
 448                               sleep 60
 449                               git describe %s/%s > old_remote_branch.desc
 450                               git fetch %s
 451                               git describe %s/%s > remote_branch.desc
 452                               diff old_remote_branch.desc remote_branch.desc
 453                             done
 454                            ''' % (
 455                                rebase_branch, rebase_remote, rebase_url,
 456                                rebase_remote,
 457                                rebase_remote, rebase_branch,
 458                                rebase_remote,
 459                                rebase_remote, rebase_branch
 460                            ),
 461                            "test/plain" ) ]
 462
 463             self.retry = builder('retry', retry_task, cp=False)
 464             self.need_retry = False
 465
 466     def kill_kids(self):
 467         if self.tail_proc is not None:
 468             self.tail_proc.terminate()
 469             self.tail_proc.wait()
 470             self.tail_proc = None
 471         if self.retry is not None:
 472             self.retry.proc.terminate()
 473             self.retry.proc.wait()
 474             self.retry = None
 475         for b in self.tlist:
 476             if b.proc is not None:
 477                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 478                 b.proc.terminate()
 479                 b.proc.wait()
 480                 b.proc = None
 481
 482     def wait_one(self):
 483         while True:
 484             none_running = True
 485             for b in self.tlist:
 486                 if b.proc is None:
 487                     continue
 488                 none_running = False
 489                 b.status = b.proc.poll()
 490                 if b.status is None:
 491                     continue
 492                 b.proc = None
 493                 return b
 494             if options.retry:
 495                 ret = self.retry.proc.poll()
 496                 if ret is not None:
 497                     self.need_retry = True
 498                     self.retry = None
 499                     return None
 500             if none_running:
 501                 return None
 502             time.sleep(0.1)
 503
 504     def run(self):
 505         while True:
 506             b = self.wait_one()
 507             if options.retry and self.need_retry:
 508                 self.kill_kids()
 509                 do_print("retry needed")
 510                 return (0, None, None, None, "retry")
 511             if b is None:
 512                 break
 513             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 514                 self.kill_kids()
 515                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 516             b.start_next()
 517         self.kill_kids()
 518         return (0, None, None, None, "All OK")
 519
 520     def write_system_info(self):
 521         filename = 'system-info.txt'
 522         f = open(filename, 'w')
 523         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo', 'cc --version']:
 524             print('### %s' % cmd, file=f)
 525             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 526             print(file=f)
 527         f.close()
 528         return filename
 529
 530     def tarlogs(self, fname):
 531         tar = tarfile.open(fname, "w:gz")
 532         for b in self.tlist:
 533             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 534             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 535         if os.path.exists("autobuild.log"):
 536             tar.add("autobuild.log")
 537         sys_info = self.write_system_info()
 538         tar.add(sys_info)
 539         tar.close()
 540
 541     def remove_logs(self):
 542         for b in self.tlist:
 543             os.unlink(b.stdout_path)
 544             os.unlink(b.stderr_path)
 545
 546     def start_tail(self):
 547         cwd = os.getcwd()
 548         cmd = "tail -f *.stdout *.stderr"
 549         os.chdir(gitroot)
 550         self.tail_proc = Popen(cmd, shell=True)
 551         os.chdir(cwd)
 552
 553
 554 def cleanup():
 555     if options.nocleanup:
 556         return
 557     run_cmd("stat %s || true" % test_tmpdir, show=True)
 558     run_cmd("stat %s" % testbase, show=True)
 559     do_print("Cleaning up ....")
 560     for d in cleanup_list:
 561         run_cmd("rm -rf %s" % d)
 562
 563
 564 def find_git_root():
 565     '''get to the top of the git repo'''
 566     p=os.getcwd()
 567     while p != '/':
 568         if os.path.isdir(os.path.join(p, ".git")):
 569             return p
 570         p = os.path.abspath(os.path.join(p, '..'))
 571     return None
 572
 573
 574 def daemonize(logfile):
 575     pid = os.fork()
 576     if pid == 0: # Parent
 577         os.setsid()
 578         pid = os.fork()
 579         if pid != 0: # Actual daemon
 580             os._exit(0)
 581     else: # Grandparent
 582         os._exit(0)
 583
 584     import resource      # Resource usage information.
 585     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 586     if maxfd == resource.RLIM_INFINITY:
 587         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
 588     for fd in range(0, maxfd):
 589         try:
 590             os.close(fd)
 591         except OSError:
 592             pass
 593     os.open(logfile, os.O_RDWR | os.O_CREAT)
 594     os.dup2(0, 1)
 595     os.dup2(0, 2)
 596
 597 def write_pidfile(fname):
 598     '''write a pid file, cleanup on exit'''
 599     f = open(fname, mode='w')
 600     f.write("%u\n" % os.getpid())
 601     f.close()
 602
 603
 604 def rebase_tree(rebase_url, rebase_branch = "master"):
 605     rebase_remote = "rebaseon"
 606     do_print("Rebasing on %s" % rebase_url)
 607     run_cmd("git describe HEAD", show=True, dir=test_master)
 608     run_cmd("git remote add -t %s %s %s" %
 609             (rebase_branch, rebase_remote, rebase_url),
 610             show=True, dir=test_master)
 611     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 612     if options.fix_whitespace:
 613         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 614                 (rebase_remote, rebase_branch),
 615                 show=True, dir=test_master)
 616     else:
 617         run_cmd("git rebase --force-rebase %s/%s" %
 618                 (rebase_remote, rebase_branch),
 619                 show=True, dir=test_master)
 620     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 621                    (rebase_remote, rebase_branch),
 622                    dir=test_master, output=True)
 623     if diff == '':
 624         do_print("No differences between HEAD and %s/%s - exiting" %
 625               (rebase_remote, rebase_branch))
 626         sys.exit(0)
 627     run_cmd("git describe %s/%s" %
 628             (rebase_remote, rebase_branch),
 629             show=True, dir=test_master)
 630     run_cmd("git describe HEAD", show=True, dir=test_master)
 631     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 632             (rebase_remote, rebase_branch),
 633             show=True, dir=test_master)
 634
 635 def push_to(push_url, push_branch = "master"):
 636     push_remote = "pushto"
 637     do_print("Pushing to %s" % push_url)
 638     if options.mark:
 639         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 640         run_cmd("git commit --amend -c HEAD", dir=test_master)
 641         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 642         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 643     run_cmd("git remote add -t %s %s %s" %
 644             (push_branch, push_remote, push_url),
 645             show=True, dir=test_master)
 646     run_cmd("git push %s +HEAD:%s" %
 647             (push_remote, push_branch),
 648             show=True, dir=test_master)
 649
 650 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 651
 652 gitroot = find_git_root()
 653 if gitroot is None:
 654     raise Exception("Failed to find git root")
 655
 656 parser = OptionParser()
 657 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 658 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 659 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 660 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 661                   default=def_testbase)
 662 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 663 parser.add_option("", "--verbose", help="show all commands as they are run",
 664                   default=False, action="store_true")
 665 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 666                   default=None, type='str')
 667 parser.add_option("", "--pushto", help="push to a git url on success",
 668                   default=None, type='str')
 669 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 670                   default=False, action="store_true")
 671 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 672                   default=False, action="store_true")
 673 parser.add_option("", "--retry", help="automatically retry if master changes",
 674                   default=False, action="store_true")
 675 parser.add_option("", "--email", help="send email to the given address on failure",
 676                   type='str', default=None)
 677 parser.add_option("", "--email-from", help="send email from the given address",
 678                   type='str', default="autobuild@samba.org")
 679 parser.add_option("", "--email-server", help="send email via the given server",
 680                   type='str', default='localhost')
 681 parser.add_option("", "--always-email", help="always send email, even on success",
 682                   action="store_true")
 683 parser.add_option("", "--daemon", help="daemonize after initial setup",
 684                   action="store_true")
 685 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 686                   default="master", type='str')
 687 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 688                   default=gitroot, type='str')
 689 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 690                   default=False, action="store_true")
 691 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 692                   default='')
 693
 694 def send_email(subject, text, log_tar):
 695     if options.email is None:
 696         do_print("not sending email because the recipient is not set")
 697         do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
 698                  (subject, text))
 699         return
 700     outer = MIMEMultipart()
 701     outer['Subject'] = subject
 702     outer['To'] = options.email
 703     outer['From'] = options.email_from
 704     outer['Date'] = email.utils.formatdate(localtime = True)
 705     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 706     outer.attach(MIMEText(text, 'plain'))
 707     if options.attach_logs:
 708         fp = open(log_tar, 'rb')
 709         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 710         fp.close()
 711         # Set the filename parameter
 712         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 713         outer.attach(msg)
 714     content = outer.as_string()
 715     s = smtplib.SMTP(options.email_server)
 716     s.sendmail(options.email_from, [options.email], content)
 717     s.set_debuglevel(1)
 718     s.quit()
 719
 720 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 721                   elapsed_time, log_base=None, add_log_tail=True):
 722     '''send an email to options.email about the failure'''
 723     elapsed_minutes = elapsed_time / 60.0
 724     user = os.getenv("USER")
 725     if log_base is None:
 726         log_base = gitroot
 727     text = '''
 728 Dear Developer,
 729
 730 Your autobuild on %s failed after %.1f minutes
 731 when trying to test %s with the following error:
 732
 733    %s
 734
 735 the autobuild has been abandoned. Please fix the error and resubmit.
 736
 737 A summary of the autobuild process is here:
 738
 739   %s/autobuild.log
 740 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 741
 742     if options.restrict_tests:
 743         text += """
 744 The build was restricted to tests matching %s\n""" % options.restrict_tests
 745
 746     if failed_task != 'rebase':
 747         text += '''
 748 You can see logs of the failed task here:
 749
 750   %s/%s.stdout
 751   %s/%s.stderr
 752
 753 or you can get full logs of all tasks in this job here:
 754
 755   %s/logs.tar.gz
 756
 757 The top commit for the tree that was built was:
 758
 759 %s
 760
 761 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 762
 763     if add_log_tail:
 764         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 765         lines = f.readlines()
 766         log_tail = "".join(lines[-50:])
 767         num_lines = len(lines)
 768         if num_lines < 50:
 769             # Also include stderr (compile failures) if < 50 lines of stdout
 770             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 771             log_tail += "".join(f.readlines()[-(50-num_lines):])
 772
 773         text += '''
 774 The last 50 lines of log messages:
 775
 776 %s
 777     ''' % log_tail
 778         f.close()
 779
 780     logs = os.path.join(gitroot, 'logs.tar.gz')
 781     send_email('autobuild[%s] failure on %s for task %s during %s'
 782                % (options.branch, platform.node(), failed_task, failed_stage),
 783                text, logs)
 784
 785 def email_success(elapsed_time, log_base=None):
 786     '''send an email to options.email about a successful build'''
 787     user = os.getenv("USER")
 788     if log_base is None:
 789         log_base = gitroot
 790     text = '''
 791 Dear Developer,
 792
 793 Your autobuild on %s has succeeded after %.1f minutes.
 794
 795 ''' % (platform.node(), elapsed_time / 60.)
 796
 797     if options.restrict_tests:
 798         text += """
 799 The build was restricted to tests matching %s\n""" % options.restrict_tests
 800
 801     if options.keeplogs:
 802         text += '''
 803
 804 you can get full logs of all tasks in this job here:
 805
 806   %s/logs.tar.gz
 807
 808 ''' % log_base
 809
 810     text += '''
 811 The top commit for the tree that was built was:
 812
 813 %s
 814 ''' % top_commit_msg
 815
 816     logs = os.path.join(gitroot, 'logs.tar.gz')
 817     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 818                text, logs)
 819
 820
 821 (options, args) = parser.parse_args()
 822
 823 if options.retry:
 824     if options.rebase is None:
 825         raise Exception('You can only use --retry if you also rebase')
 826
 827 testbase = "%s/b%u" % (options.testbase, os.getpid())
 828 test_master = "%s/master" % testbase
 829 test_prefix = "%s/prefix" % testbase
 830 test_tmpdir = "%s/tmp" % testbase
 831 os.environ['TMPDIR'] = test_tmpdir
 832
 833 # get the top commit message, for emails
 834 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 835
 836 try:
 837     os.makedirs(testbase)
 838 except Exception as reason:
 839     raise Exception("Unable to create %s : %s" % (testbase, reason))
 840 cleanup_list.append(testbase)
 841
 842 if options.daemon:
 843     logfile = os.path.join(testbase, "log")
 844     do_print("Forking into the background, writing progress to %s" % logfile)
 845     daemonize(logfile)
 846
 847 write_pidfile(gitroot + "/autobuild.pid")
 848
 849 start_time = time.time()
 850
 851 while True:
 852     try:
 853         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 854         os.makedirs(test_tmpdir)
 855         # The waf uninstall code removes empty directories all the way
 856         # up the tree.  Creating a file in test_tmpdir stops it from
 857         # being removed.
 858         run_cmd("touch %s" % os.path.join(test_tmpdir,
 859                                           ".directory-is-not-empty"), show=True)
 860         run_cmd("stat %s" % test_tmpdir, show=True)
 861         run_cmd("stat %s" % testbase, show=True)
 862         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 863     except Exception:
 864         cleanup()
 865         raise
 866
 867     try:
 868         try:
 869             if options.rebase is not None:
 870                 rebase_tree(options.rebase, rebase_branch=options.branch)
 871         except Exception:
 872             cleanup_list.append(gitroot + "/autobuild.pid")
 873             cleanup()
 874             elapsed_time = time.time() - start_time
 875             email_failure(-1, 'rebase', 'rebase', 'rebase',
 876                           'rebase on %s failed' % options.branch,
 877                           elapsed_time, log_base=options.log_base)
 878             sys.exit(1)
 879         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 880         if options.tail:
 881             blist.start_tail()
 882         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 883         if status != 0 or errstr != "retry":
 884             break
 885         cleanup()
 886     except Exception:
 887         cleanup()
 888         raise
 889
 890 cleanup_list.append(gitroot + "/autobuild.pid")
 891
 892 do_print(errstr)
 893
 894 blist.kill_kids()
 895 if options.tail:
 896     do_print("waiting for tail to flush")
 897     time.sleep(1)
 898
 899 elapsed_time = time.time() - start_time
 900 if status == 0:
 901     if options.passcmd is not None:
 902         do_print("Running passcmd: %s" % options.passcmd)
 903         run_cmd(options.passcmd, dir=test_master)
 904     if options.pushto is not None:
 905         push_to(options.pushto, push_branch=options.branch)
 906     if options.keeplogs or options.attach_logs:
 907         blist.tarlogs("logs.tar.gz")
 908         do_print("Logs in logs.tar.gz")
 909     if options.always_email:
 910         email_success(elapsed_time, log_base=options.log_base)
 911     blist.remove_logs()
 912     cleanup()
 913     do_print(errstr)
 914     sys.exit(0)
 915
 916 # something failed, gather a tar of the logs
 917 blist.tarlogs("logs.tar.gz")
 918
 919 if options.email is not None:
 920     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 921                   elapsed_time, log_base=options.log_base)
 922 else:
 923     elapsed_minutes = elapsed_time / 60.0
 924     print('''
 925
 926 ####################################################################
 927
 928 AUTOBUILD FAILURE
 929
 930 Your autobuild[%s] on %s failed after %.1f minutes
 931 when trying to test %s with the following error:
 932
 933    %s
 934
 935 the autobuild has been abandoned. Please fix the error and resubmit.
 936
 937 ####################################################################
 938
 939 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 940
 941 cleanup()
 942 do_print(errstr)
 943 do_print("Logs in logs.tar.gz")
 944 sys.exit(status)