script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call,Popen, PIPE
   8 import os, tarfile, sys, time
   9 from optparse import OptionParser
  10 import smtplib
  11 import email
  12 from email.mime.text import MIMEText
  13 from email.mime.base import MIMEBase
  14 from email.mime.application import MIMEApplication
  15 from email.mime.multipart import MIMEMultipart
  16 from distutils.sysconfig import get_python_lib
  17 import platform
  18
  19 os.environ["PYTHONUNBUFFERED"] = "1"
  20
  21 # This speeds up testing remarkably.
  22 os.environ['TDB_NO_FSYNC'] = '1'
  23
  24 cleanup_list = []
  25
  26 builddirs = {
  27     "ctdb"    : "ctdb",
  28     "samba"  : ".",
  29     "samba-nt4"  : ".",
  30     "samba-fileserver"  : ".",
  31     "samba-xc" : ".",
  32     "samba-o3" : ".",
  33     "samba-ctdb" : ".",
  34     "samba-libs"  : ".",
  35     "samba-static"  : ".",
  36     "samba-test-only"  : ".",
  37     "samba-none-env"  : ".",
  38     "samba-ad-dc"  : ".",
  39     "samba-ad-dc-2"  : ".",
  40     "samba-systemkrb5"  : ".",
  41     "samba-nopython"  : ".",
  42     "ldb"     : "lib/ldb",
  43     "tdb"     : "lib/tdb",
  44     "talloc"  : "lib/talloc",
  45     "replace" : "lib/replace",
  46     "tevent"  : "lib/tevent",
  47     "pidl"    : "pidl",
  48     "pass"    : ".",
  49     "fail"    : ".",
  50     "retry"   : "."
  51     }
  52
  53 defaulttasks = [ "ctdb",
  54                  "samba",
  55                  "samba-nt4",
  56                  "samba-fileserver",
  57                  "samba-xc",
  58                  "samba-o3",
  59                  "samba-ctdb",
  60                  "samba-libs",
  61                  "samba-static",
  62                  "samba-none-env",
  63                  "samba-ad-dc",
  64                  "samba-ad-dc-2",
  65                  "samba-systemkrb5",
  66                  "samba-nopython",
  67                  "ldb",
  68                  "tdb",
  69                  "talloc",
  70                  "replace",
  71                  "tevent",
  72                  "pidl" ]
  73
  74 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  75     defaulttasks.remove("samba-o3")
  76
  77 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  78 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  79
  80 samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  81 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  82 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  83 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX} ${EXTRA_PYTHON}"
  84 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE"
  85 samba_libs_configure_samba = samba_libs_configure_base + " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
  86
  87 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  88     extra_python = ""
  89 else:
  90     extra_python = "--extra-python=/usr/bin/python3"
  91
  92 tasks = {
  93     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  94                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  95                ("make", "make all", "text/plain"),
  96                ("install", "make install", "text/plain"),
  97                ("test", "make autotest", "text/plain"),
  98                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
  99                ("clean", "make clean", "text/plain") ],
 100
 101     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
 102     "samba" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 103                 ("make", "make -j", "text/plain"),
 104                 ("test", "make test FAIL_IMMEDIATELY=1 "
 105                  "TESTS='--exclude-env=none "
 106                  "--exclude-env=nt4_dc "
 107                  "--exclude-env=nt4_member "
 108                  "--exclude-env=ad_dc "
 109                  "--exclude-env=chgdcpass "
 110                  "--exclude-env=vampire_2000_dc "
 111                  "--exclude-env=fl2000dc "
 112                  "--exclude-env=fileserver'",
 113                  "text/plain"),
 114                 ("install", "make install", "text/plain"),
 115                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 116                 ("clean", "make clean", "text/plain") ],
 117
 118     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 119     "samba-nt4" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 120                        ("make", "make -j", "text/plain"),
 121                        ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 122                        ("install", "make install", "text/plain"),
 123                        ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 124                        ("clean", "make clean", "text/plain") ],
 125
 126     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 127     "samba-fileserver" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 128                       ("configure", "./configure.developer --without-ad-dc --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 129                       ("make", "make -j", "text/plain"),
 130                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
 131                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 132
 133     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 134     "samba-ad-dc" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 135                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 136                       ("make", "make -j", "text/plain"),
 137                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 138                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 139
 140     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 141     "samba-ad-dc-2" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 142                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 143                       ("make", "make -j", "text/plain"),
 144                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
 145                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 146
 147     "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 148                           ("make", "make -j", "text/plain"),
 149                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"',"text/plain") ],
 150
 151     # Test cross-compile infrastructure
 152     "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 153                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 154                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 155                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 156                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 157                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 158
 159     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 160     "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 161                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 162                    ("make", "make -j", "text/plain"),
 163                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 164                    ("install", "make install", "text/plain"),
 165                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 166                    ("clean", "make clean", "text/plain") ],
 167
 168     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 169
 170                      # make sure we have tdb around:
 171                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 172                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 173                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 174
 175
 176                      # build samba with cluster support (also building ctdb):
 177                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 178                      ("samba-make", "make", "text/plain"),
 179                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 180                      ("samba-install", "make install", "text/plain"),
 181                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 182
 183                      # clean up:
 184                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 185                      ("clean", "make clean", "text/plain"),
 186                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ],
 187
 188     "samba-libs" : [
 189                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 190                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 191                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 192                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 193
 194                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 195                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 196                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 197
 198                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 199                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 200                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 201
 202                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 203                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 204                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 205
 206                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 207                       ("nondevel-make", "make -j", "text/plain"),
 208                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 209                       ("nondevel-install", "make install", "text/plain"),
 210                       ("nondevel-dist", "make dist", "text/plain"),
 211
 212                       # retry with all modules shared
 213                       ("allshared-distclean", "make distclean", "text/plain"),
 214                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 215                       ("allshared-make", "make -j", "text/plain")],
 216
 217     "samba-none-env" : [
 218                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 219                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 220                       ("make", "make -j", "text/plain"),
 221                       ("test", "make test "
 222                        "FAIL_IMMEDIATELY=1 "
 223                        "TESTS='--include-env=none'",
 224                        "text/plain")],
 225
 226     "samba-static" : [
 227                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 228                       # build with all modules static
 229                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 230                       ("allstatic-make", "make -j", "text/plain"),
 231
 232                       # retry without any required modules
 233                       ("none-distclean", "make distclean", "text/plain"),
 234                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 235                       ("none-make", "make -j", "text/plain"),
 236
 237                       # retry with nonshared smbd and smbtorture
 238                       ("nonshared-distclean", "make distclean", "text/plain"),
 239                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 240                       ("nonshared-make", "make -j", "text/plain")],
 241
 242     "samba-systemkrb5" : [
 243                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 244                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 245                       ("make", "make -j", "text/plain"),
 246                       # we currently cannot run a full make test, a limited list of tests could be run
 247                       # via "make test TESTS=sometests"
 248                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 249                       ("install", "make install", "text/plain"),
 250                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 251                       ("clean", "make clean", "text/plain")
 252                       ],
 253
 254     # Test Samba without python still builds.  When this test fails
 255     # due to more use of Python, the expectations is that the newly
 256     # failing part of the code should be disabled when
 257     # --disable-python is set (rather than major work being done to
 258     # support this environment).  The target here is for vendors
 259     # shipping a minimal smbd.
 260     "samba-nopython" : [
 261                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 262                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 263                       ("make", "make -j", "text/plain"),
 264                       ("install", "make install", "text/plain"),
 265                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 266                       ("clean", "make clean", "text/plain")
 267                       ],
 268
 269
 270
 271     "ldb" : [
 272               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 273               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 274               ("make", "make", "text/plain"),
 275               ("install", "make install", "text/plain"),
 276               ("test", "make test", "text/plain"),
 277               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 278               ("distcheck", "make distcheck", "text/plain"),
 279               ("clean", "make clean", "text/plain") ],
 280
 281     "tdb" : [
 282               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 283               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 284               ("make", "make", "text/plain"),
 285               ("install", "make install", "text/plain"),
 286               ("test", "make test", "text/plain"),
 287               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 288               ("distcheck", "make distcheck", "text/plain"),
 289               ("clean", "make clean", "text/plain") ],
 290
 291     "talloc" : [
 292                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 293                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 294                  ("make", "make", "text/plain"),
 295                  ("install", "make install", "text/plain"),
 296                  ("test", "make test", "text/plain"),
 297                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 298                  ("distcheck", "make distcheck", "text/plain"),
 299                  ("clean", "make clean", "text/plain") ],
 300
 301     "replace" : [
 302                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 303                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 304                   ("make", "make", "text/plain"),
 305                   ("install", "make install", "text/plain"),
 306                   ("test", "make test", "text/plain"),
 307                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 308                   ("distcheck", "make distcheck", "text/plain"),
 309                   ("clean", "make clean", "text/plain") ],
 310
 311     "tevent" : [
 312                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 313                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 314                  ("make", "make", "text/plain"),
 315                  ("install", "make install", "text/plain"),
 316                  ("test", "make test", "text/plain"),
 317                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 318                  ("distcheck", "make distcheck", "text/plain"),
 319                  ("clean", "make clean", "text/plain") ],
 320
 321     "pidl" : [
 322                ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 323                ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 324                ("touch", "touch *.yp", "text/plain"),
 325                ("make", "make", "text/plain"),
 326                ("test", "make test", "text/plain"),
 327                ("install", "make install", "text/plain"),
 328                ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 329                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 330                ("clean", "make clean", "text/plain") ],
 331
 332     # these are useful for debugging autobuild
 333     'pass' : [ ("pass", 'echo passing && /bin/true', "text/plain") ],
 334     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
 335 }
 336
 337 def do_print(msg):
 338     print("%s" % msg)
 339     sys.stdout.flush()
 340     sys.stderr.flush()
 341
 342 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 343     if show is None:
 344         show = options.verbose
 345     if show:
 346         do_print("Running: '%s' in '%s'" % (cmd, dir))
 347     if output:
 348         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 349     elif checkfail:
 350         return check_call(cmd, shell=True, cwd=dir)
 351     else:
 352         return call(cmd, shell=True, cwd=dir)
 353
 354
 355 class builder(object):
 356     '''handle build of one directory'''
 357
 358     def __init__(self, name, sequence, cp=True):
 359         self.name = name
 360         self.dir = builddirs[name]
 361
 362         self.tag = self.name.replace('/', '_')
 363         self.sequence = sequence
 364         self.next = 0
 365         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 366         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 367         if options.verbose:
 368             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 369             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 370         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 371         self.stdout = open(self.stdout_path, 'w')
 372         self.stderr = open(self.stderr_path, 'w')
 373         self.stdin  = open("/dev/null", 'r')
 374         self.sdir = "%s/%s" % (testbase, self.tag)
 375         self.prefix = "%s/%s" % (test_prefix, self.tag)
 376         run_cmd("rm -rf %s" % self.sdir)
 377         run_cmd("rm -rf %s" % self.prefix)
 378         if cp:
 379             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 380         else:
 381             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 382         self.start_next()
 383
 384     def start_next(self):
 385         if self.next == len(self.sequence):
 386             if not options.nocleanup:
 387                 run_cmd("rm -rf %s" % self.sdir)
 388                 run_cmd("rm -rf %s" % self.prefix)
 389             do_print('%s: Completed OK' % self.name)
 390             self.done = True
 391             return
 392         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 393         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 394         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 395         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 396         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 397         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 398 #        if self.output_mime_type == "text/x-subunit":
 399 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 400         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 401         cwd = os.getcwd()
 402         os.chdir("%s/%s" % (self.sdir, self.dir))
 403         self.proc = Popen(self.cmd, shell=True,
 404                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 405         os.chdir(cwd)
 406         self.next += 1
 407
 408
 409 class buildlist(object):
 410     '''handle build of multiple directories'''
 411
 412     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 413         global tasks
 414         self.tlist = []
 415         self.tail_proc = None
 416         self.retry = None
 417         if tasknames == []:
 418             if options.restrict_tests:
 419                 tasknames = ["samba-test-only"]
 420             else:
 421                 tasknames = defaulttasks
 422         else:
 423             # If we are only running one test,
 424             # do not sleep randomly to wait for it to start
 425             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 426
 427         for n in tasknames:
 428             b = builder(n, tasks[n], cp=n is not "pidl")
 429             self.tlist.append(b)
 430         if options.retry:
 431             rebase_remote = "rebaseon"
 432             retry_task = [ ("retry",
 433                             '''set -e
 434                             git remote add -t %s %s %s
 435                             git fetch %s
 436                             while :; do
 437                               sleep 60
 438                               git describe %s/%s > old_remote_branch.desc
 439                               git fetch %s
 440                               git describe %s/%s > remote_branch.desc
 441                               diff old_remote_branch.desc remote_branch.desc
 442                             done
 443                            ''' % (
 444                                rebase_branch, rebase_remote, rebase_url,
 445                                rebase_remote,
 446                                rebase_remote, rebase_branch,
 447                                rebase_remote,
 448                                rebase_remote, rebase_branch
 449                            ),
 450                            "test/plain" ) ]
 451
 452             self.retry = builder('retry', retry_task, cp=False)
 453             self.need_retry = False
 454
 455     def kill_kids(self):
 456         if self.tail_proc is not None:
 457             self.tail_proc.terminate()
 458             self.tail_proc.wait()
 459             self.tail_proc = None
 460         if self.retry is not None:
 461             self.retry.proc.terminate()
 462             self.retry.proc.wait()
 463             self.retry = None
 464         for b in self.tlist:
 465             if b.proc is not None:
 466                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 467                 b.proc.terminate()
 468                 b.proc.wait()
 469                 b.proc = None
 470
 471     def wait_one(self):
 472         while True:
 473             none_running = True
 474             for b in self.tlist:
 475                 if b.proc is None:
 476                     continue
 477                 none_running = False
 478                 b.status = b.proc.poll()
 479                 if b.status is None:
 480                     continue
 481                 b.proc = None
 482                 return b
 483             if options.retry:
 484                 ret = self.retry.proc.poll()
 485                 if ret is not None:
 486                     self.need_retry = True
 487                     self.retry = None
 488                     return None
 489             if none_running:
 490                 return None
 491             time.sleep(0.1)
 492
 493     def run(self):
 494         while True:
 495             b = self.wait_one()
 496             if options.retry and self.need_retry:
 497                 self.kill_kids()
 498                 do_print("retry needed")
 499                 return (0, None, None, None, "retry")
 500             if b is None:
 501                 break
 502             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 503                 self.kill_kids()
 504                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 505             b.start_next()
 506         self.kill_kids()
 507         return (0, None, None, None, "All OK")
 508
 509     def write_system_info(self):
 510         filename = 'system-info.txt'
 511         f = open(filename, 'w')
 512         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo']:
 513             print('### %s' % cmd, file=f)
 514             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 515             print(file=f)
 516         f.close()
 517         return filename
 518
 519     def tarlogs(self, fname):
 520         tar = tarfile.open(fname, "w:gz")
 521         for b in self.tlist:
 522             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 523             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 524         if os.path.exists("autobuild.log"):
 525             tar.add("autobuild.log")
 526         sys_info = self.write_system_info()
 527         tar.add(sys_info)
 528         tar.close()
 529
 530     def remove_logs(self):
 531         for b in self.tlist:
 532             os.unlink(b.stdout_path)
 533             os.unlink(b.stderr_path)
 534
 535     def start_tail(self):
 536         cwd = os.getcwd()
 537         cmd = "tail -f *.stdout *.stderr"
 538         os.chdir(gitroot)
 539         self.tail_proc = Popen(cmd, shell=True)
 540         os.chdir(cwd)
 541
 542
 543 def cleanup():
 544     if options.nocleanup:
 545         return
 546     run_cmd("stat %s || true" % test_tmpdir, show=True)
 547     run_cmd("stat %s" % testbase, show=True)
 548     do_print("Cleaning up ....")
 549     for d in cleanup_list:
 550         run_cmd("rm -rf %s" % d)
 551
 552
 553 def find_git_root():
 554     '''get to the top of the git repo'''
 555     p=os.getcwd()
 556     while p != '/':
 557         if os.path.isdir(os.path.join(p, ".git")):
 558             return p
 559         p = os.path.abspath(os.path.join(p, '..'))
 560     return None
 561
 562
 563 def daemonize(logfile):
 564     pid = os.fork()
 565     if pid == 0: # Parent
 566         os.setsid()
 567         pid = os.fork()
 568         if pid != 0: # Actual daemon
 569             os._exit(0)
 570     else: # Grandparent
 571         os._exit(0)
 572
 573     import resource      # Resource usage information.
 574     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 575     if maxfd == resource.RLIM_INFINITY:
 576         maxfd = 1024 # Rough guess at maximum number of open file descriptors.
 577     for fd in range(0, maxfd):
 578         try:
 579             os.close(fd)
 580         except OSError:
 581             pass
 582     os.open(logfile, os.O_RDWR | os.O_CREAT)
 583     os.dup2(0, 1)
 584     os.dup2(0, 2)
 585
 586 def write_pidfile(fname):
 587     '''write a pid file, cleanup on exit'''
 588     f = open(fname, mode='w')
 589     f.write("%u\n" % os.getpid())
 590     f.close()
 591
 592
 593 def rebase_tree(rebase_url, rebase_branch = "master"):
 594     rebase_remote = "rebaseon"
 595     do_print("Rebasing on %s" % rebase_url)
 596     run_cmd("git describe HEAD", show=True, dir=test_master)
 597     run_cmd("git remote add -t %s %s %s" %
 598             (rebase_branch, rebase_remote, rebase_url),
 599             show=True, dir=test_master)
 600     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 601     if options.fix_whitespace:
 602         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 603                 (rebase_remote, rebase_branch),
 604                 show=True, dir=test_master)
 605     else:
 606         run_cmd("git rebase --force-rebase %s/%s" %
 607                 (rebase_remote, rebase_branch),
 608                 show=True, dir=test_master)
 609     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 610                    (rebase_remote, rebase_branch),
 611                    dir=test_master, output=True)
 612     if diff == '':
 613         do_print("No differences between HEAD and %s/%s - exiting" %
 614               (rebase_remote, rebase_branch))
 615         sys.exit(0)
 616     run_cmd("git describe %s/%s" %
 617             (rebase_remote, rebase_branch),
 618             show=True, dir=test_master)
 619     run_cmd("git describe HEAD", show=True, dir=test_master)
 620     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 621             (rebase_remote, rebase_branch),
 622             show=True, dir=test_master)
 623
 624 def push_to(push_url, push_branch = "master"):
 625     push_remote = "pushto"
 626     do_print("Pushing to %s" % push_url)
 627     if options.mark:
 628         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 629         run_cmd("git commit --amend -c HEAD", dir=test_master)
 630         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 631         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 632     run_cmd("git remote add -t %s %s %s" %
 633             (push_branch, push_remote, push_url),
 634             show=True, dir=test_master)
 635     run_cmd("git push %s +HEAD:%s" %
 636             (push_remote, push_branch),
 637             show=True, dir=test_master)
 638
 639 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 640
 641 gitroot = find_git_root()
 642 if gitroot is None:
 643     raise Exception("Failed to find git root")
 644
 645 parser = OptionParser()
 646 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 647 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 648 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 649 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 650                   default=def_testbase)
 651 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 652 parser.add_option("", "--verbose", help="show all commands as they are run",
 653                   default=False, action="store_true")
 654 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 655                   default=None, type='str')
 656 parser.add_option("", "--pushto", help="push to a git url on success",
 657                   default=None, type='str')
 658 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 659                   default=False, action="store_true")
 660 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 661                   default=False, action="store_true")
 662 parser.add_option("", "--retry", help="automatically retry if master changes",
 663                   default=False, action="store_true")
 664 parser.add_option("", "--email", help="send email to the given address on failure",
 665                   type='str', default=None)
 666 parser.add_option("", "--email-from", help="send email from the given address",
 667                   type='str', default="autobuild@samba.org")
 668 parser.add_option("", "--email-server", help="send email via the given server",
 669                   type='str', default='localhost')
 670 parser.add_option("", "--always-email", help="always send email, even on success",
 671                   action="store_true")
 672 parser.add_option("", "--daemon", help="daemonize after initial setup",
 673                   action="store_true")
 674 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 675                   default="master", type='str')
 676 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 677                   default=gitroot, type='str')
 678 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 679                   default=False, action="store_true")
 680 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 681                   default='')
 682
 683 def send_email(subject, text, log_tar):
 684     outer = MIMEMultipart()
 685     outer['Subject'] = subject
 686     outer['To'] = options.email
 687     outer['From'] = options.email_from
 688     outer['Date'] = email.utils.formatdate(localtime = True)
 689     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 690     outer.attach(MIMEText(text, 'plain'))
 691     if options.attach_logs:
 692         fp = open(log_tar, 'rb')
 693         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 694         fp.close()
 695         # Set the filename parameter
 696         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 697         outer.attach(msg)
 698     content = outer.as_string()
 699     s = smtplib.SMTP(options.email_server)
 700     s.sendmail(options.email_from, [options.email], content)
 701     s.set_debuglevel(1)
 702     s.quit()
 703
 704 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 705                   elapsed_time, log_base=None, add_log_tail=True):
 706     '''send an email to options.email about the failure'''
 707     elapsed_minutes = elapsed_time / 60.0
 708     user = os.getenv("USER")
 709     if log_base is None:
 710         log_base = gitroot
 711     text = '''
 712 Dear Developer,
 713
 714 Your autobuild on %s failed after %.1f minutes
 715 when trying to test %s with the following error:
 716
 717    %s
 718
 719 the autobuild has been abandoned. Please fix the error and resubmit.
 720
 721 A summary of the autobuild process is here:
 722
 723   %s/autobuild.log
 724 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 725
 726     if options.restrict_tests:
 727         text += """
 728 The build was restricted to tests matching %s\n""" % options.restrict_tests
 729
 730     if failed_task != 'rebase':
 731         text += '''
 732 You can see logs of the failed task here:
 733
 734   %s/%s.stdout
 735   %s/%s.stderr
 736
 737 or you can get full logs of all tasks in this job here:
 738
 739   %s/logs.tar.gz
 740
 741 The top commit for the tree that was built was:
 742
 743 %s
 744
 745 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 746
 747     if add_log_tail:
 748         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 749         lines = f.readlines()
 750         log_tail = "".join(lines[-50:])
 751         num_lines = len(lines)
 752         if num_lines < 50:
 753             # Also include stderr (compile failures) if < 50 lines of stdout
 754             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 755             log_tail += "".join(f.readlines()[-(50-num_lines):])
 756
 757         text += '''
 758 The last 50 lines of log messages:
 759
 760 %s
 761     ''' % log_tail
 762         f.close()
 763
 764     logs = os.path.join(gitroot, 'logs.tar.gz')
 765     send_email('autobuild[%s] failure on %s for task %s during %s'
 766                % (options.branch, platform.node(), failed_task, failed_stage),
 767                text, logs)
 768
 769 def email_success(elapsed_time, log_base=None):
 770     '''send an email to options.email about a successful build'''
 771     user = os.getenv("USER")
 772     if log_base is None:
 773         log_base = gitroot
 774     text = '''
 775 Dear Developer,
 776
 777 Your autobuild on %s has succeeded after %.1f minutes.
 778
 779 ''' % (platform.node(), elapsed_time / 60.)
 780
 781     if options.restrict_tests:
 782         text += """
 783 The build was restricted to tests matching %s\n""" % options.restrict_tests
 784
 785     if options.keeplogs:
 786         text += '''
 787
 788 you can get full logs of all tasks in this job here:
 789
 790   %s/logs.tar.gz
 791
 792 ''' % log_base
 793
 794     text += '''
 795 The top commit for the tree that was built was:
 796
 797 %s
 798 ''' % top_commit_msg
 799
 800     logs = os.path.join(gitroot, 'logs.tar.gz')
 801     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 802                text, logs)
 803
 804
 805 (options, args) = parser.parse_args()
 806
 807 if options.retry:
 808     if options.rebase is None:
 809         raise Exception('You can only use --retry if you also rebase')
 810
 811 testbase = "%s/b%u" % (options.testbase, os.getpid())
 812 test_master = "%s/master" % testbase
 813 test_prefix = "%s/prefix" % testbase
 814 test_tmpdir = "%s/tmp" % testbase
 815 os.environ['TMPDIR'] = test_tmpdir
 816
 817 # get the top commit message, for emails
 818 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 819
 820 try:
 821     os.makedirs(testbase)
 822 except Exception as reason:
 823     raise Exception("Unable to create %s : %s" % (testbase, reason))
 824 cleanup_list.append(testbase)
 825
 826 if options.daemon:
 827     logfile = os.path.join(testbase, "log")
 828     do_print("Forking into the background, writing progress to %s" % logfile)
 829     daemonize(logfile)
 830
 831 write_pidfile(gitroot + "/autobuild.pid")
 832
 833 start_time = time.time()
 834
 835 while True:
 836     try:
 837         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 838         os.makedirs(test_tmpdir)
 839         # The waf uninstall code removes empty directories all the way
 840         # up the tree.  Creating a file in test_tmpdir stops it from
 841         # being removed.
 842         run_cmd("touch %s" % os.path.join(test_tmpdir,
 843                                           ".directory-is-not-empty"), show=True)
 844         run_cmd("stat %s" % test_tmpdir, show=True)
 845         run_cmd("stat %s" % testbase, show=True)
 846         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 847     except Exception:
 848         cleanup()
 849         raise
 850
 851     try:
 852         try:
 853             if options.rebase is not None:
 854                 rebase_tree(options.rebase, rebase_branch=options.branch)
 855         except Exception:
 856             cleanup_list.append(gitroot + "/autobuild.pid")
 857             cleanup()
 858             elapsed_time = time.time() - start_time
 859             email_failure(-1, 'rebase', 'rebase', 'rebase',
 860                           'rebase on %s failed' % options.branch,
 861                           elapsed_time, log_base=options.log_base)
 862             sys.exit(1)
 863         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 864         if options.tail:
 865             blist.start_tail()
 866         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 867         if status != 0 or errstr != "retry":
 868             break
 869         cleanup()
 870     except Exception:
 871         cleanup()
 872         raise
 873
 874 cleanup_list.append(gitroot + "/autobuild.pid")
 875
 876 do_print(errstr)
 877
 878 blist.kill_kids()
 879 if options.tail:
 880     do_print("waiting for tail to flush")
 881     time.sleep(1)
 882
 883 elapsed_time = time.time() - start_time
 884 if status == 0:
 885     if options.passcmd is not None:
 886         do_print("Running passcmd: %s" % options.passcmd)
 887         run_cmd(options.passcmd, dir=test_master)
 888     if options.pushto is not None:
 889         push_to(options.pushto, push_branch=options.branch)
 890     if options.keeplogs or options.attach_logs:
 891         blist.tarlogs("logs.tar.gz")
 892         do_print("Logs in logs.tar.gz")
 893     if options.always_email:
 894         email_success(elapsed_time, log_base=options.log_base)
 895     blist.remove_logs()
 896     cleanup()
 897     do_print(errstr)
 898     sys.exit(0)
 899
 900 # something failed, gather a tar of the logs
 901 blist.tarlogs("logs.tar.gz")
 902
 903 if options.email is not None:
 904     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 905                   elapsed_time, log_base=options.log_base)
 906 else:
 907     elapsed_minutes = elapsed_time / 60.0
 908     print('''
 909
 910 ####################################################################
 911
 912 AUTOBUILD FAILURE
 913
 914 Your autobuild[%s] on %s failed after %.1f minutes
 915 when trying to test %s with the following error:
 916
 917    %s
 918
 919 the autobuild has been abandoned. Please fix the error and resubmit.
 920
 921 ####################################################################
 922
 923 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 924
 925 cleanup()
 926 do_print(errstr)
 927 do_print("Logs in logs.tar.gz")
 928 sys.exit(status)