script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os
   9 import tarfile
  10 import sys
  11 import time
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 os.environ["PYTHONUNBUFFERED"] = "1"
  23
  24 # This speeds up testing remarkably.
  25 os.environ['TDB_NO_FSYNC'] = '1'
  26
  27 cleanup_list = []
  28
  29 builddirs = {
  30     "ctdb": "ctdb",
  31     "samba": ".",
  32     "samba-nt4": ".",
  33     "samba-fileserver": ".",
  34     "samba-xc": ".",
  35     "samba-o3": ".",
  36     "samba-ctdb": ".",
  37     "samba-libs": ".",
  38     "samba-static": ".",
  39     "samba-test-only": ".",
  40     "samba-none-env": ".",
  41     "samba-ad-dc": ".",
  42     "samba-ad-dc-2": ".",
  43     "samba-systemkrb5": ".",
  44     "samba-nopython": ".",
  45     "ldb": "lib/ldb",
  46     "tdb": "lib/tdb",
  47     "talloc": "lib/talloc",
  48     "replace": "lib/replace",
  49     "tevent": "lib/tevent",
  50     "pidl": "pidl",
  51     "pass": ".",
  52     "fail": ".",
  53     "retry": "."
  54 }
  55
  56 defaulttasks = ["ctdb",
  57                  "samba",
  58                  "samba-nt4",
  59                  "samba-fileserver",
  60                  "samba-xc",
  61                  "samba-o3",
  62                  "samba-ctdb",
  63                  "samba-libs",
  64                  "samba-static",
  65                  "samba-none-env",
  66                  "samba-ad-dc",
  67                  "samba-ad-dc-2",
  68                  "samba-systemkrb5",
  69                  "samba-nopython",
  70                  "ldb",
  71                  "tdb",
  72                  "talloc",
  73                  "replace",
  74                  "tevent",
  75                  "pidl"]
  76
  77 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  78     defaulttasks.remove("samba-o3")
  79
  80 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  81 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  82
  83 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  84 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  85 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  86 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  87 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  88 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  89 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  90
  91 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  92     extra_python = ""
  93 else:
  94     extra_python = "--extra-python=/usr/bin/python3"
  95
  96 tasks = {
  97     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  98                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  99                ("make", "make all", "text/plain"),
 100                ("install", "make install", "text/plain"),
 101                ("test", "make autotest", "text/plain"),
 102                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 103                ("clean", "make clean", "text/plain")],
 104
 105     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
 106     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 107                 ("make", "make -j", "text/plain"),
 108                 ("test", "make test FAIL_IMMEDIATELY=1 "
 109                  "TESTS='--exclude-env=none "
 110                  "--exclude-env=nt4_dc "
 111                  "--exclude-env=nt4_member "
 112                  "--exclude-env=ad_dc "
 113                  "--exclude-env=fl2003dc "
 114                  "--exclude-env=fl2008r2dc "
 115                  "--exclude-env=ad_member "
 116                  "--exclude-env=ad_member_idmap_rid "
 117                  "--exclude-env=ad_member_idmap_ad "
 118                  "--exclude-env=chgdcpass "
 119                  "--exclude-env=vampire_2000_dc "
 120                  "--exclude-env=fl2000dc "
 121                  "--exclude-env=fileserver'",
 122                  "text/plain"),
 123                 ("install", "make install", "text/plain"),
 124                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 125                 ("clean", "make clean", "text/plain")],
 126
 127     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 128     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 129                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 130                     ("make", "make -j", "text/plain"),
 131                     ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 132                     ("install", "make install", "text/plain"),
 133                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 134                     ("clean", "make clean", "text/plain")],
 135
 136     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 137     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 138                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json-audit --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 139                            ("make", "make -j", "text/plain"),
 140                            ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
 141                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 142
 143     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 144     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 145                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 146                       ("make", "make -j", "text/plain"),
 147                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='"
 148                        "--include-env=ad_dc "
 149                        "--include-env=fl2003dc "
 150                        "--include-env=fl2008r2dc "
 151                        "--include-env=ad_member "
 152                        "--include-env=ad_member_idmap_rid "
 153                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 154                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 155
 156     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 157     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 158                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 159                         ("make", "make -j", "text/plain"),
 160                         ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
 161                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 162
 163     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 164                           ("make", "make -j", "text/plain"),
 165                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 166
 167     # Test cross-compile infrastructure
 168     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 169                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 170                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 171                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 172                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 173                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 174                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 175
 176     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 177     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 178                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 179                    ("make", "make -j", "text/plain"),
 180                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 181                    ("install", "make install", "text/plain"),
 182                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 183                    ("clean", "make clean", "text/plain")],
 184
 185     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 186
 187                      # make sure we have tdb around:
 188                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 189                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 190                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 191
 192
 193                      # build samba with cluster support (also building ctdb):
 194                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 195                      ("samba-make", "make", "text/plain"),
 196                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 197                      ("samba-install", "make install", "text/plain"),
 198                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 199
 200                      # clean up:
 201                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 202                      ("clean", "make clean", "text/plain"),
 203                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 204
 205     "samba-libs" : [
 206                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 207                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 208                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 209                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 210
 211                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 212                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 213                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 214
 215                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 216                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 217                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 218
 219                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 220                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 221                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 222
 223                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 224                       ("nondevel-make", "make -j", "text/plain"),
 225                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 226                       ("nondevel-install", "make install", "text/plain"),
 227                       ("nondevel-dist", "make dist", "text/plain"),
 228
 229                       # retry with all modules shared
 230                       ("allshared-distclean", "make distclean", "text/plain"),
 231                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 232                       ("allshared-make", "make -j", "text/plain")],
 233
 234     "samba-none-env" : [
 235                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 236                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 237                       ("make", "make -j", "text/plain"),
 238                       ("test", "make test "
 239                        "FAIL_IMMEDIATELY=1 "
 240                        "TESTS='--include-env=none'",
 241                        "text/plain")],
 242
 243     "samba-static" : [
 244                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 245                       # build with all modules static
 246                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 247                       ("allstatic-make", "make -j", "text/plain"),
 248                       ("allstatic-test", "make test "
 249                        "FAIL_IMMEDIATELY=1 "
 250                        "TESTS='samba3.smb2.create.*nt4_dc'",
 251                        "text/plain"),
 252
 253                       # retry without any required modules
 254                       ("none-distclean", "make distclean", "text/plain"),
 255                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 256                       ("none-make", "make -j", "text/plain"),
 257
 258                       # retry with nonshared smbd and smbtorture
 259                       ("nonshared-distclean", "make distclean", "text/plain"),
 260                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 261                       ("nonshared-make", "make -j", "text/plain")],
 262
 263     "samba-systemkrb5" : [
 264                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 265                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 266                       ("make", "make -j", "text/plain"),
 267                       # we currently cannot run a full make test, a limited list of tests could be run
 268                       # via "make test TESTS=sometests"
 269                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 270                       ("install", "make install", "text/plain"),
 271                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 272                       ("clean", "make clean", "text/plain")
 273                       ],
 274
 275     # Test Samba without python still builds.  When this test fails
 276     # due to more use of Python, the expectations is that the newly
 277     # failing part of the code should be disabled when
 278     # --disable-python is set (rather than major work being done to
 279     # support this environment).  The target here is for vendors
 280     # shipping a minimal smbd.
 281     "samba-nopython" : [
 282                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 283                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 284                       ("make", "make -j", "text/plain"),
 285                       ("install", "make install", "text/plain"),
 286                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 287                       ("clean", "make clean", "text/plain"),
 288
 289                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 290                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 291                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 292
 293                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 294                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 295                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 296
 297                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 298                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 299                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 300
 301                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 302                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 303                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 304
 305                       # retry against installed library packages
 306                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 307                       ("libs-make", "make -j", "text/plain"),
 308                       ("libs-install", "make install", "text/plain"),
 309                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 310                       ("libs-clean", "make clean", "text/plain")
 311                       ],
 312
 313
 314
 315     "ldb" : [
 316               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 317               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 318               ("make", "make", "text/plain"),
 319               ("install", "make install", "text/plain"),
 320               ("test", "make test", "text/plain"),
 321               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 322               ("make-no-lmdb", "make", "text/plain"),
 323               ("install-no-lmdb", "make install", "text/plain"),
 324               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 325               ("distcheck", "make distcheck", "text/plain"),
 326               ("clean", "make clean", "text/plain")],
 327
 328     "tdb" : [
 329               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 330               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 331               ("make", "make", "text/plain"),
 332               ("install", "make install", "text/plain"),
 333               ("test", "make test", "text/plain"),
 334               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 335               ("distcheck", "make distcheck", "text/plain"),
 336               ("clean", "make clean", "text/plain")],
 337
 338     "talloc" : [
 339                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 340                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 341                  ("make", "make", "text/plain"),
 342                  ("install", "make install", "text/plain"),
 343                  ("test", "make test", "text/plain"),
 344                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 345                  ("distcheck", "make distcheck", "text/plain"),
 346                  ("clean", "make clean", "text/plain")],
 347
 348     "replace" : [
 349                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 350                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 351                   ("make", "make", "text/plain"),
 352                   ("install", "make install", "text/plain"),
 353                   ("test", "make test", "text/plain"),
 354                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 355                   ("distcheck", "make distcheck", "text/plain"),
 356                   ("clean", "make clean", "text/plain")],
 357
 358     "tevent" : [
 359                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 360                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 361                  ("make", "make", "text/plain"),
 362                  ("install", "make install", "text/plain"),
 363                  ("test", "make test", "text/plain"),
 364                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 365                  ("distcheck", "make distcheck", "text/plain"),
 366                  ("clean", "make clean", "text/plain")],
 367
 368     "pidl": [
 369         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 370         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 371         ("touch", "touch *.yp", "text/plain"),
 372         ("make", "make", "text/plain"),
 373         ("test", "make test", "text/plain"),
 374         ("install", "make install", "text/plain"),
 375         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 376         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 377         ("clean", "make clean", "text/plain")],
 378
 379     # these are useful for debugging autobuild
 380     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 381     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 382 }
 383
 384
 385 def do_print(msg):
 386     print("%s" % msg)
 387     sys.stdout.flush()
 388     sys.stderr.flush()
 389
 390
 391 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 392     if show is None:
 393         show = options.verbose
 394     if show:
 395         do_print("Running: '%s' in '%s'" % (cmd, dir))
 396     if output:
 397         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 398     elif checkfail:
 399         return check_call(cmd, shell=True, cwd=dir)
 400     else:
 401         return call(cmd, shell=True, cwd=dir)
 402
 403
 404 class builder(object):
 405     '''handle build of one directory'''
 406
 407     def __init__(self, name, sequence, cp=True):
 408         self.name = name
 409         self.dir = builddirs[name]
 410
 411         self.tag = self.name.replace('/', '_')
 412         self.sequence = sequence
 413         self.next = 0
 414         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 415         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 416         if options.verbose:
 417             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 418             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 419         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 420         self.stdout = open(self.stdout_path, 'w')
 421         self.stderr = open(self.stderr_path, 'w')
 422         self.stdin  = open("/dev/null", 'r')
 423         self.sdir = "%s/%s" % (testbase, self.tag)
 424         self.prefix = "%s/%s" % (test_prefix, self.tag)
 425         run_cmd("rm -rf %s" % self.sdir)
 426         run_cmd("rm -rf %s" % self.prefix)
 427         if cp:
 428             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 429         else:
 430             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 431         self.start_next()
 432
 433     def start_next(self):
 434         if self.next == len(self.sequence):
 435             if not options.nocleanup:
 436                 run_cmd("rm -rf %s" % self.sdir)
 437                 run_cmd("rm -rf %s" % self.prefix)
 438             do_print('%s: Completed OK' % self.name)
 439             self.done = True
 440             return
 441         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 442         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 443         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 444         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 445         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 446         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 447 #        if self.output_mime_type == "text/x-subunit":
 448 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 449         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 450         cwd = os.getcwd()
 451         os.chdir("%s/%s" % (self.sdir, self.dir))
 452         self.proc = Popen(self.cmd, shell=True,
 453                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 454         os.chdir(cwd)
 455         self.next += 1
 456
 457
 458 class buildlist(object):
 459     '''handle build of multiple directories'''
 460
 461     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 462         global tasks
 463         self.tlist = []
 464         self.tail_proc = None
 465         self.retry = None
 466         if tasknames == []:
 467             if options.restrict_tests:
 468                 tasknames = ["samba-test-only"]
 469             else:
 470                 tasknames = defaulttasks
 471         else:
 472             # If we are only running one test,
 473             # do not sleep randomly to wait for it to start
 474             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 475
 476         for n in tasknames:
 477             b = builder(n, tasks[n], cp=n is not "pidl")
 478             self.tlist.append(b)
 479         if options.retry:
 480             rebase_remote = "rebaseon"
 481             retry_task = [("retry",
 482                             '''set -e
 483                             git remote add -t %s %s %s
 484                             git fetch %s
 485                             while :; do
 486                               sleep 60
 487                               git describe %s/%s > old_remote_branch.desc
 488                               git fetch %s
 489                               git describe %s/%s > remote_branch.desc
 490                               diff old_remote_branch.desc remote_branch.desc
 491                             done
 492                            ''' % (
 493                                 rebase_branch, rebase_remote, rebase_url,
 494                                rebase_remote,
 495                                rebase_remote, rebase_branch,
 496                                rebase_remote,
 497                                rebase_remote, rebase_branch
 498                             ),
 499                             "test/plain")]
 500
 501             self.retry = builder('retry', retry_task, cp=False)
 502             self.need_retry = False
 503
 504     def kill_kids(self):
 505         if self.tail_proc is not None:
 506             self.tail_proc.terminate()
 507             self.tail_proc.wait()
 508             self.tail_proc = None
 509         if self.retry is not None:
 510             self.retry.proc.terminate()
 511             self.retry.proc.wait()
 512             self.retry = None
 513         for b in self.tlist:
 514             if b.proc is not None:
 515                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 516                 b.proc.terminate()
 517                 b.proc.wait()
 518                 b.proc = None
 519
 520     def wait_one(self):
 521         while True:
 522             none_running = True
 523             for b in self.tlist:
 524                 if b.proc is None:
 525                     continue
 526                 none_running = False
 527                 b.status = b.proc.poll()
 528                 if b.status is None:
 529                     continue
 530                 b.proc = None
 531                 return b
 532             if options.retry:
 533                 ret = self.retry.proc.poll()
 534                 if ret is not None:
 535                     self.need_retry = True
 536                     self.retry = None
 537                     return None
 538             if none_running:
 539                 return None
 540             time.sleep(0.1)
 541
 542     def run(self):
 543         while True:
 544             b = self.wait_one()
 545             if options.retry and self.need_retry:
 546                 self.kill_kids()
 547                 do_print("retry needed")
 548                 return (0, None, None, None, "retry")
 549             if b is None:
 550                 break
 551             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 552                 self.kill_kids()
 553                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 554             b.start_next()
 555         self.kill_kids()
 556         return (0, None, None, None, "All OK")
 557
 558     def write_system_info(self):
 559         filename = 'system-info.txt'
 560         f = open(filename, 'w')
 561         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 562                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 563             print('### %s' % cmd, file=f)
 564             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 565             print(file=f)
 566         f.close()
 567         return filename
 568
 569     def tarlogs(self, fname):
 570         tar = tarfile.open(fname, "w:gz")
 571         for b in self.tlist:
 572             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 573             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 574         if os.path.exists("autobuild.log"):
 575             tar.add("autobuild.log")
 576         sys_info = self.write_system_info()
 577         tar.add(sys_info)
 578         tar.close()
 579
 580     def remove_logs(self):
 581         for b in self.tlist:
 582             os.unlink(b.stdout_path)
 583             os.unlink(b.stderr_path)
 584
 585     def start_tail(self):
 586         cwd = os.getcwd()
 587         cmd = "tail -f *.stdout *.stderr"
 588         os.chdir(gitroot)
 589         self.tail_proc = Popen(cmd, shell=True)
 590         os.chdir(cwd)
 591
 592
 593 def cleanup():
 594     if options.nocleanup:
 595         return
 596     run_cmd("stat %s || true" % test_tmpdir, show=True)
 597     run_cmd("stat %s" % testbase, show=True)
 598     do_print("Cleaning up ....")
 599     for d in cleanup_list:
 600         run_cmd("rm -rf %s" % d)
 601
 602
 603 def find_git_root():
 604     '''get to the top of the git repo'''
 605     p = os.getcwd()
 606     while p != '/':
 607         if os.path.isdir(os.path.join(p, ".git")):
 608             return p
 609         p = os.path.abspath(os.path.join(p, '..'))
 610     return None
 611
 612
 613 def daemonize(logfile):
 614     pid = os.fork()
 615     if pid == 0:  # Parent
 616         os.setsid()
 617         pid = os.fork()
 618         if pid != 0:  # Actual daemon
 619             os._exit(0)
 620     else:  # Grandparent
 621         os._exit(0)
 622
 623     import resource      # Resource usage information.
 624     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 625     if maxfd == resource.RLIM_INFINITY:
 626         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 627     for fd in range(0, maxfd):
 628         try:
 629             os.close(fd)
 630         except OSError:
 631             pass
 632     os.open(logfile, os.O_RDWR | os.O_CREAT)
 633     os.dup2(0, 1)
 634     os.dup2(0, 2)
 635
 636
 637 def write_pidfile(fname):
 638     '''write a pid file, cleanup on exit'''
 639     f = open(fname, mode='w')
 640     f.write("%u\n" % os.getpid())
 641     f.close()
 642
 643
 644 def rebase_tree(rebase_url, rebase_branch="master"):
 645     rebase_remote = "rebaseon"
 646     do_print("Rebasing on %s" % rebase_url)
 647     run_cmd("git describe HEAD", show=True, dir=test_master)
 648     run_cmd("git remote add -t %s %s %s" %
 649             (rebase_branch, rebase_remote, rebase_url),
 650             show=True, dir=test_master)
 651     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 652     if options.fix_whitespace:
 653         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 654                 (rebase_remote, rebase_branch),
 655                 show=True, dir=test_master)
 656     else:
 657         run_cmd("git rebase --force-rebase %s/%s" %
 658                 (rebase_remote, rebase_branch),
 659                 show=True, dir=test_master)
 660     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 661                    (rebase_remote, rebase_branch),
 662                    dir=test_master, output=True)
 663     if diff == '':
 664         do_print("No differences between HEAD and %s/%s - exiting" %
 665                  (rebase_remote, rebase_branch))
 666         sys.exit(0)
 667     run_cmd("git describe %s/%s" %
 668             (rebase_remote, rebase_branch),
 669             show=True, dir=test_master)
 670     run_cmd("git describe HEAD", show=True, dir=test_master)
 671     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 672             (rebase_remote, rebase_branch),
 673             show=True, dir=test_master)
 674
 675
 676 def push_to(push_url, push_branch="master"):
 677     push_remote = "pushto"
 678     do_print("Pushing to %s" % push_url)
 679     if options.mark:
 680         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 681         run_cmd("git commit --amend -c HEAD", dir=test_master)
 682         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 683         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 684     run_cmd("git remote add -t %s %s %s" %
 685             (push_branch, push_remote, push_url),
 686             show=True, dir=test_master)
 687     run_cmd("git push %s +HEAD:%s" %
 688             (push_remote, push_branch),
 689             show=True, dir=test_master)
 690
 691
 692 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 693
 694 gitroot = find_git_root()
 695 if gitroot is None:
 696     raise Exception("Failed to find git root")
 697
 698 parser = OptionParser()
 699 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 700 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 701 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 702 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 703                   default=def_testbase)
 704 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 705 parser.add_option("", "--verbose", help="show all commands as they are run",
 706                   default=False, action="store_true")
 707 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 708                   default=None, type='str')
 709 parser.add_option("", "--pushto", help="push to a git url on success",
 710                   default=None, type='str')
 711 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 712                   default=False, action="store_true")
 713 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 714                   default=False, action="store_true")
 715 parser.add_option("", "--retry", help="automatically retry if master changes",
 716                   default=False, action="store_true")
 717 parser.add_option("", "--email", help="send email to the given address on failure",
 718                   type='str', default=None)
 719 parser.add_option("", "--email-from", help="send email from the given address",
 720                   type='str', default="autobuild@samba.org")
 721 parser.add_option("", "--email-server", help="send email via the given server",
 722                   type='str', default='localhost')
 723 parser.add_option("", "--always-email", help="always send email, even on success",
 724                   action="store_true")
 725 parser.add_option("", "--daemon", help="daemonize after initial setup",
 726                   action="store_true")
 727 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 728                   default="master", type='str')
 729 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 730                   default=gitroot, type='str')
 731 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 732                   default=False, action="store_true")
 733 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 734                   default='')
 735
 736
 737 def send_email(subject, text, log_tar):
 738     if options.email is None:
 739         do_print("not sending email because the recipient is not set")
 740         do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
 741                  (subject, text))
 742         return
 743     outer = MIMEMultipart()
 744     outer['Subject'] = subject
 745     outer['To'] = options.email
 746     outer['From'] = options.email_from
 747     outer['Date'] = email.utils.formatdate(localtime=True)
 748     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 749     outer.attach(MIMEText(text, 'plain'))
 750     if options.attach_logs:
 751         fp = open(log_tar, 'rb')
 752         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 753         fp.close()
 754         # Set the filename parameter
 755         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 756         outer.attach(msg)
 757     content = outer.as_string()
 758     s = smtplib.SMTP(options.email_server)
 759     s.sendmail(options.email_from, [options.email], content)
 760     s.set_debuglevel(1)
 761     s.quit()
 762
 763
 764 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 765                   elapsed_time, log_base=None, add_log_tail=True):
 766     '''send an email to options.email about the failure'''
 767     elapsed_minutes = elapsed_time / 60.0
 768     user = os.getenv("USER")
 769     if log_base is None:
 770         log_base = gitroot
 771     text = '''
 772 Dear Developer,
 773
 774 Your autobuild on %s failed after %.1f minutes
 775 when trying to test %s with the following error:
 776
 777    %s
 778
 779 the autobuild has been abandoned. Please fix the error and resubmit.
 780
 781 A summary of the autobuild process is here:
 782
 783   %s/autobuild.log
 784 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 785
 786     if options.restrict_tests:
 787         text += """
 788 The build was restricted to tests matching %s\n""" % options.restrict_tests
 789
 790     if failed_task != 'rebase':
 791         text += '''
 792 You can see logs of the failed task here:
 793
 794   %s/%s.stdout
 795   %s/%s.stderr
 796
 797 or you can get full logs of all tasks in this job here:
 798
 799   %s/logs.tar.gz
 800
 801 The top commit for the tree that was built was:
 802
 803 %s
 804
 805 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 806
 807     if add_log_tail:
 808         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 809         lines = f.readlines()
 810         log_tail = "".join(lines[-50:])
 811         num_lines = len(lines)
 812         if num_lines < 50:
 813             # Also include stderr (compile failures) if < 50 lines of stdout
 814             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 815             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 816
 817         text += '''
 818 The last 50 lines of log messages:
 819
 820 %s
 821     ''' % log_tail
 822         f.close()
 823
 824     logs = os.path.join(gitroot, 'logs.tar.gz')
 825     send_email('autobuild[%s] failure on %s for task %s during %s'
 826                % (options.branch, platform.node(), failed_task, failed_stage),
 827                text, logs)
 828
 829
 830 def email_success(elapsed_time, log_base=None):
 831     '''send an email to options.email about a successful build'''
 832     user = os.getenv("USER")
 833     if log_base is None:
 834         log_base = gitroot
 835     text = '''
 836 Dear Developer,
 837
 838 Your autobuild on %s has succeeded after %.1f minutes.
 839
 840 ''' % (platform.node(), elapsed_time / 60.)
 841
 842     if options.restrict_tests:
 843         text += """
 844 The build was restricted to tests matching %s\n""" % options.restrict_tests
 845
 846     if options.keeplogs:
 847         text += '''
 848
 849 you can get full logs of all tasks in this job here:
 850
 851   %s/logs.tar.gz
 852
 853 ''' % log_base
 854
 855     text += '''
 856 The top commit for the tree that was built was:
 857
 858 %s
 859 ''' % top_commit_msg
 860
 861     logs = os.path.join(gitroot, 'logs.tar.gz')
 862     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 863                text, logs)
 864
 865
 866 (options, args) = parser.parse_args()
 867
 868 if options.retry:
 869     if options.rebase is None:
 870         raise Exception('You can only use --retry if you also rebase')
 871
 872 testbase = "%s/b%u" % (options.testbase, os.getpid())
 873 test_master = "%s/master" % testbase
 874 test_prefix = "%s/prefix" % testbase
 875 test_tmpdir = "%s/tmp" % testbase
 876 os.environ['TMPDIR'] = test_tmpdir
 877
 878 # get the top commit message, for emails
 879 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 880
 881 try:
 882     os.makedirs(testbase)
 883 except Exception as reason:
 884     raise Exception("Unable to create %s : %s" % (testbase, reason))
 885 cleanup_list.append(testbase)
 886
 887 if options.daemon:
 888     logfile = os.path.join(testbase, "log")
 889     do_print("Forking into the background, writing progress to %s" % logfile)
 890     daemonize(logfile)
 891
 892 write_pidfile(gitroot + "/autobuild.pid")
 893
 894 start_time = time.time()
 895
 896 while True:
 897     try:
 898         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 899         os.makedirs(test_tmpdir)
 900         # The waf uninstall code removes empty directories all the way
 901         # up the tree.  Creating a file in test_tmpdir stops it from
 902         # being removed.
 903         run_cmd("touch %s" % os.path.join(test_tmpdir,
 904                                           ".directory-is-not-empty"), show=True)
 905         run_cmd("stat %s" % test_tmpdir, show=True)
 906         run_cmd("stat %s" % testbase, show=True)
 907         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 908     except Exception:
 909         cleanup()
 910         raise
 911
 912     try:
 913         try:
 914             if options.rebase is not None:
 915                 rebase_tree(options.rebase, rebase_branch=options.branch)
 916         except Exception:
 917             cleanup_list.append(gitroot + "/autobuild.pid")
 918             cleanup()
 919             elapsed_time = time.time() - start_time
 920             email_failure(-1, 'rebase', 'rebase', 'rebase',
 921                           'rebase on %s failed' % options.branch,
 922                           elapsed_time, log_base=options.log_base)
 923             sys.exit(1)
 924         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 925         if options.tail:
 926             blist.start_tail()
 927         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 928         if status != 0 or errstr != "retry":
 929             break
 930         cleanup()
 931     except Exception:
 932         cleanup()
 933         raise
 934
 935 cleanup_list.append(gitroot + "/autobuild.pid")
 936
 937 do_print(errstr)
 938
 939 blist.kill_kids()
 940 if options.tail:
 941     do_print("waiting for tail to flush")
 942     time.sleep(1)
 943
 944 elapsed_time = time.time() - start_time
 945 if status == 0:
 946     if options.passcmd is not None:
 947         do_print("Running passcmd: %s" % options.passcmd)
 948         run_cmd(options.passcmd, dir=test_master)
 949     if options.pushto is not None:
 950         push_to(options.pushto, push_branch=options.branch)
 951     if options.keeplogs or options.attach_logs:
 952         blist.tarlogs("logs.tar.gz")
 953         do_print("Logs in logs.tar.gz")
 954     if options.always_email:
 955         email_success(elapsed_time, log_base=options.log_base)
 956     blist.remove_logs()
 957     cleanup()
 958     do_print(errstr)
 959     sys.exit(0)
 960
 961 # something failed, gather a tar of the logs
 962 blist.tarlogs("logs.tar.gz")
 963
 964 if options.email is not None:
 965     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 966                   elapsed_time, log_base=options.log_base)
 967 else:
 968     elapsed_minutes = elapsed_time / 60.0
 969     print('''
 970
 971 ####################################################################
 972
 973 AUTOBUILD FAILURE
 974
 975 Your autobuild[%s] on %s failed after %.1f minutes
 976 when trying to test %s with the following error:
 977
 978    %s
 979
 980 the autobuild has been abandoned. Please fix the error and resubmit.
 981
 982 ####################################################################
 983
 984 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 985
 986 cleanup()
 987 do_print(errstr)
 988 do_print("Logs in logs.tar.gz")
 989 sys.exit(status)