script/autobuild.py

   1 #!/usr/bin/env python
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from __future__ import print_function
   7 from subprocess import call, check_call, Popen, PIPE
   8 import os, tarfile, sys, time
   9 from optparse import OptionParser
  10 import smtplib
  11 import email
  12 from email.mime.text import MIMEText
  13 from email.mime.base import MIMEBase
  14 from email.mime.application import MIMEApplication
  15 from email.mime.multipart import MIMEMultipart
  16 from distutils.sysconfig import get_python_lib
  17 import platform
  18
  19 os.environ["PYTHONUNBUFFERED"] = "1"
  20
  21 # This speeds up testing remarkably.
  22 os.environ['TDB_NO_FSYNC'] = '1'
  23
  24 cleanup_list = []
  25
  26 builddirs = {
  27     "ctdb": "ctdb",
  28     "samba": ".",
  29     "samba-nt4": ".",
  30     "samba-fileserver": ".",
  31     "samba-xc": ".",
  32     "samba-o3": ".",
  33     "samba-ctdb": ".",
  34     "samba-libs": ".",
  35     "samba-static": ".",
  36     "samba-test-only": ".",
  37     "samba-none-env": ".",
  38     "samba-ad-dc": ".",
  39     "samba-ad-dc-2": ".",
  40     "samba-systemkrb5": ".",
  41     "samba-nopython": ".",
  42     "ldb": "lib/ldb",
  43     "tdb": "lib/tdb",
  44     "talloc": "lib/talloc",
  45     "replace": "lib/replace",
  46     "tevent": "lib/tevent",
  47     "pidl": "pidl",
  48     "pass": ".",
  49     "fail": ".",
  50     "retry": "."
  51 }
  52
  53 defaulttasks = ["ctdb",
  54                  "samba",
  55                  "samba-nt4",
  56                  "samba-fileserver",
  57                  "samba-xc",
  58                  "samba-o3",
  59                  "samba-ctdb",
  60                  "samba-libs",
  61                  "samba-static",
  62                  "samba-none-env",
  63                  "samba-ad-dc",
  64                  "samba-ad-dc-2",
  65                  "samba-systemkrb5",
  66                  "samba-nopython",
  67                  "ldb",
  68                  "tdb",
  69                  "talloc",
  70                  "replace",
  71                  "tevent",
  72                  "pidl"]
  73
  74 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
  75     defaulttasks.remove("samba-o3")
  76
  77 ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
  78 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
  79
  80 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
  81 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
  82 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
  83 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}"
  84 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE ${EXTRA_PYTHON}"
  85 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
  86 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs + " ${EXTRA_PYTHON}"
  87
  88 if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
  89     extra_python = ""
  90 else:
  91     extra_python = "--extra-python=/usr/bin/python3"
  92
  93 tasks = {
  94     "ctdb": [("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
  95                ("configure", "./configure " + ctdb_configure_params, "text/plain"),
  96                ("make", "make all", "text/plain"),
  97                ("install", "make install", "text/plain"),
  98                ("test", "make autotest", "text/plain"),
  99                ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 100                ("clean", "make clean", "text/plain")],
 101
 102     # We have 'test' before 'install' because, 'test' should work without 'install (runs ad_dc_ntvfs and all the other envs)'
 103     "samba": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 104                 ("make", "make -j", "text/plain"),
 105                 ("test", "make test FAIL_IMMEDIATELY=1 "
 106                  "TESTS='--exclude-env=none "
 107                  "--exclude-env=nt4_dc "
 108                  "--exclude-env=nt4_member "
 109                  "--exclude-env=ad_dc "
 110                  "--exclude-env=fl2003dc "
 111                  "--exclude-env=fl2008r2dc "
 112                  "--exclude-env=ad_member "
 113                  "--exclude-env=ad_member_idmap_rid "
 114                  "--exclude-env=ad_member_idmap_ad "
 115                  "--exclude-env=chgdcpass "
 116                  "--exclude-env=vampire_2000_dc "
 117                  "--exclude-env=fl2000dc "
 118                  "--exclude-env=fileserver'",
 119                  "text/plain"),
 120                 ("install", "make install", "text/plain"),
 121                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 122                 ("clean", "make clean", "text/plain")],
 123
 124     # We split out this so the isolated nt4_dc tests do not wait for ad_dc or ad_dc_ntvfs tests (which are long)
 125     "samba-nt4": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 126                     ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 127                     ("make", "make -j", "text/plain"),
 128                     ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=nt4_dc --include-env=nt4_member'", "text/plain"),
 129                     ("install", "make install", "text/plain"),
 130                     ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 131                     ("clean", "make clean", "text/plain")],
 132
 133     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 134     "samba-fileserver": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 135                            ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json-audit --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 136                            ("make", "make -j", "text/plain"),
 137                            ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=fileserver'", "text/plain"),
 138                            ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 139
 140     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 141     "samba-ad-dc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 142                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 143                       ("make", "make -j", "text/plain"),
 144                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='"
 145                        "--include-env=ad_dc "
 146                        "--include-env=fl2003dc "
 147                        "--include-env=fl2008r2dc "
 148                        "--include-env=ad_member "
 149                        "--include-env=ad_member_idmap_rid "
 150                        "--include-env=ad_member_idmap_ad'", "text/plain"),
 151                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 152
 153     # We split out this so the isolated ad_dc tests do not wait for ad_dc_ntvfs tests (which are long)
 154     "samba-ad-dc-2": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 155                         ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 156                         ("make", "make -j", "text/plain"),
 157                         ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=chgdcpass --include-env=vampire_2000_dc --include-env=fl2000dc'", "text/plain"),
 158                         ("check-clean-tree", "script/clean-source-tree.sh", "text/plain")],
 159
 160     "samba-test-only": [("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
 161                           ("make", "make -j", "text/plain"),
 162                           ("test", 'make test FAIL_IMMEDIATELY=1 TESTS="${TESTS}"', "text/plain")],
 163
 164     # Test cross-compile infrastructure
 165     "samba-xc": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 166                    ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 167                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 168                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params, "text/plain"),
 169                    ("configure-cross-answers", "./configure.developer -b ./bin-xa --cross-compile" \
 170                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
 171                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 172
 173     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 174     "samba-o3": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 175                    ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
 176                    ("make", "make -j", "text/plain"),
 177                    ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='--include-env=ad_dc'", "text/plain"),
 178                    ("install", "make install", "text/plain"),
 179                    ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 180                    ("clean", "make clean", "text/plain")],
 181
 182     "samba-ctdb": [("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 183
 184                      # make sure we have tdb around:
 185                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
 186                      ("tdb-make", "cd lib/tdb && make", "text/plain"),
 187                      ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 188
 189
 190                      # build samba with cluster support (also building ctdb):
 191                      ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"),
 192                      ("samba-make", "make", "text/plain"),
 193                      ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"),
 194                      ("samba-install", "make install", "text/plain"),
 195                      ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"),
 196
 197                      # clean up:
 198                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 199                      ("clean", "make clean", "text/plain"),
 200                      ("ctdb-clean", "cd ./ctdb && make clean", "text/plain")],
 201
 202     "samba-libs" : [
 203                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 204                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
 205                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 206                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 207
 208                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
 209                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 210                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 211
 212                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
 213                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 214                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 215
 216                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
 217                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 218                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 219
 220                       ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
 221                       ("nondevel-make", "make -j", "text/plain"),
 222                       ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
 223                       ("nondevel-install", "make install", "text/plain"),
 224                       ("nondevel-dist", "make dist", "text/plain"),
 225
 226                       # retry with all modules shared
 227                       ("allshared-distclean", "make distclean", "text/plain"),
 228                       ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
 229                       ("allshared-make", "make -j", "text/plain")],
 230
 231     "samba-none-env" : [
 232                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 233                       ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
 234                       ("make", "make -j", "text/plain"),
 235                       ("test", "make test "
 236                        "FAIL_IMMEDIATELY=1 "
 237                        "TESTS='--include-env=none'",
 238                        "text/plain")],
 239
 240     "samba-static" : [
 241                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 242                       # build with all modules static
 243                       ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
 244                       ("allstatic-make", "make -j", "text/plain"),
 245                       ("allstatic-test", "make test "
 246                        "FAIL_IMMEDIATELY=1 "
 247                        "TESTS='samba3.smb2.create.*nt4_dc'",
 248                        "text/plain"),
 249
 250                       # retry without any required modules
 251                       ("none-distclean", "make distclean", "text/plain"),
 252                       ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
 253                       ("none-make", "make -j", "text/plain"),
 254
 255                       # retry with nonshared smbd and smbtorture
 256                       ("nonshared-distclean", "make distclean", "text/plain"),
 257                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
 258                       ("nonshared-make", "make -j", "text/plain")],
 259
 260     "samba-systemkrb5" : [
 261                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 262                       ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
 263                       ("make", "make -j", "text/plain"),
 264                       # we currently cannot run a full make test, a limited list of tests could be run
 265                       # via "make test TESTS=sometests"
 266                       ("test", "make test FAIL_IMMEDIATELY=1 TESTS='--include-env=ktest'", "text/plain"),
 267                       ("install", "make install", "text/plain"),
 268                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 269                       ("clean", "make clean", "text/plain")
 270                       ],
 271
 272     # Test Samba without python still builds.  When this test fails
 273     # due to more use of Python, the expectations is that the newly
 274     # failing part of the code should be disabled when
 275     # --disable-python is set (rather than major work being done to
 276     # support this environment).  The target here is for vendors
 277     # shipping a minimal smbd.
 278     "samba-nopython" : [
 279                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 280                       ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
 281                       ("make", "make -j", "text/plain"),
 282                       ("install", "make install", "text/plain"),
 283                       ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 284                       ("clean", "make clean", "text/plain"),
 285
 286                       ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 287                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
 288                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 289
 290                       ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 291                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
 292                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 293
 294                       ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 295                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
 296                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 297
 298                       ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python", "text/plain"),
 299                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
 300                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 301
 302                       # retry against installed library packages
 303                       ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc", "text/plain"),
 304                       ("libs-make", "make -j", "text/plain"),
 305                       ("libs-install", "make install", "text/plain"),
 306                       ("libs-check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
 307                       ("libs-clean", "make clean", "text/plain")
 308                       ],
 309
 310
 311
 312     "ldb" : [
 313               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 314               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 315               ("make", "make", "text/plain"),
 316               ("install", "make install", "text/plain"),
 317               ("test", "make test", "text/plain"),
 318               ("configure-no-lmdb", "./configure --enable-developer --without-ldb-lmdb -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 319               ("make-no-lmdb", "make", "text/plain"),
 320               ("install-no-lmdb", "make install", "text/plain"),
 321               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 322               ("distcheck", "make distcheck", "text/plain"),
 323               ("clean", "make clean", "text/plain")],
 324
 325     "tdb" : [
 326               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 327               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 328               ("make", "make", "text/plain"),
 329               ("install", "make install", "text/plain"),
 330               ("test", "make test", "text/plain"),
 331               ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 332               ("distcheck", "make distcheck", "text/plain"),
 333               ("clean", "make clean", "text/plain")],
 334
 335     "talloc" : [
 336                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 337                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 338                  ("make", "make", "text/plain"),
 339                  ("install", "make install", "text/plain"),
 340                  ("test", "make test", "text/plain"),
 341                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 342                  ("distcheck", "make distcheck", "text/plain"),
 343                  ("clean", "make clean", "text/plain")],
 344
 345     "replace" : [
 346                   ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 347                   ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
 348                   ("make", "make", "text/plain"),
 349                   ("install", "make install", "text/plain"),
 350                   ("test", "make test", "text/plain"),
 351                   ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 352                   ("distcheck", "make distcheck", "text/plain"),
 353                   ("clean", "make clean", "text/plain")],
 354
 355     "tevent" : [
 356                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
 357                  ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
 358                  ("make", "make", "text/plain"),
 359                  ("install", "make install", "text/plain"),
 360                  ("test", "make test", "text/plain"),
 361                  ("check-clean-tree", "../../script/clean-source-tree.sh", "text/plain"),
 362                  ("distcheck", "make distcheck", "text/plain"),
 363                  ("clean", "make clean", "text/plain")],
 364
 365     "pidl": [
 366         ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
 367         ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}", "text/plain"),
 368         ("touch", "touch *.yp", "text/plain"),
 369         ("make", "make", "text/plain"),
 370         ("test", "make test", "text/plain"),
 371         ("install", "make install", "text/plain"),
 372         ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm", "text/plain"),
 373         ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"),
 374         ("clean", "make clean", "text/plain")],
 375
 376     # these are useful for debugging autobuild
 377     'pass': [("pass", 'echo passing && /bin/true', "text/plain")],
 378     'fail': [("fail", 'echo failing && /bin/false', "text/plain")]
 379 }
 380
 381
 382 def do_print(msg):
 383     print("%s" % msg)
 384     sys.stdout.flush()
 385     sys.stderr.flush()
 386
 387
 388 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 389     if show is None:
 390         show = options.verbose
 391     if show:
 392         do_print("Running: '%s' in '%s'" % (cmd, dir))
 393     if output:
 394         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
 395     elif checkfail:
 396         return check_call(cmd, shell=True, cwd=dir)
 397     else:
 398         return call(cmd, shell=True, cwd=dir)
 399
 400
 401 class builder(object):
 402     '''handle build of one directory'''
 403
 404     def __init__(self, name, sequence, cp=True):
 405         self.name = name
 406         self.dir = builddirs[name]
 407
 408         self.tag = self.name.replace('/', '_')
 409         self.sequence = sequence
 410         self.next = 0
 411         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
 412         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
 413         if options.verbose:
 414             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
 415             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
 416         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
 417         self.stdout = open(self.stdout_path, 'w')
 418         self.stderr = open(self.stderr_path, 'w')
 419         self.stdin  = open("/dev/null", 'r')
 420         self.sdir = "%s/%s" % (testbase, self.tag)
 421         self.prefix = "%s/%s" % (test_prefix, self.tag)
 422         run_cmd("rm -rf %s" % self.sdir)
 423         run_cmd("rm -rf %s" % self.prefix)
 424         if cp:
 425             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 426         else:
 427             run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
 428         self.start_next()
 429
 430     def start_next(self):
 431         if self.next == len(self.sequence):
 432             if not options.nocleanup:
 433                 run_cmd("rm -rf %s" % self.sdir)
 434                 run_cmd("rm -rf %s" % self.prefix)
 435             do_print('%s: Completed OK' % self.name)
 436             self.done = True
 437             return
 438         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
 439         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
 440         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
 441         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
 442         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
 443         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 444 #        if self.output_mime_type == "text/x-subunit":
 445 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
 446         do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
 447         cwd = os.getcwd()
 448         os.chdir("%s/%s" % (self.sdir, self.dir))
 449         self.proc = Popen(self.cmd, shell=True,
 450                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
 451         os.chdir(cwd)
 452         self.next += 1
 453
 454
 455 class buildlist(object):
 456     '''handle build of multiple directories'''
 457
 458     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
 459         global tasks
 460         self.tlist = []
 461         self.tail_proc = None
 462         self.retry = None
 463         if tasknames == []:
 464             if options.restrict_tests:
 465                 tasknames = ["samba-test-only"]
 466             else:
 467                 tasknames = defaulttasks
 468         else:
 469             # If we are only running one test,
 470             # do not sleep randomly to wait for it to start
 471             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 472
 473         for n in tasknames:
 474             b = builder(n, tasks[n], cp=n is not "pidl")
 475             self.tlist.append(b)
 476         if options.retry:
 477             rebase_remote = "rebaseon"
 478             retry_task = [("retry",
 479                             '''set -e
 480                             git remote add -t %s %s %s
 481                             git fetch %s
 482                             while :; do
 483                               sleep 60
 484                               git describe %s/%s > old_remote_branch.desc
 485                               git fetch %s
 486                               git describe %s/%s > remote_branch.desc
 487                               diff old_remote_branch.desc remote_branch.desc
 488                             done
 489                            ''' % (
 490                                 rebase_branch, rebase_remote, rebase_url,
 491                                rebase_remote,
 492                                rebase_remote, rebase_branch,
 493                                rebase_remote,
 494                                rebase_remote, rebase_branch
 495                             ),
 496                             "test/plain")]
 497
 498             self.retry = builder('retry', retry_task, cp=False)
 499             self.need_retry = False
 500
 501     def kill_kids(self):
 502         if self.tail_proc is not None:
 503             self.tail_proc.terminate()
 504             self.tail_proc.wait()
 505             self.tail_proc = None
 506         if self.retry is not None:
 507             self.retry.proc.terminate()
 508             self.retry.proc.wait()
 509             self.retry = None
 510         for b in self.tlist:
 511             if b.proc is not None:
 512                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.sdir, checkfail=False)
 513                 b.proc.terminate()
 514                 b.proc.wait()
 515                 b.proc = None
 516
 517     def wait_one(self):
 518         while True:
 519             none_running = True
 520             for b in self.tlist:
 521                 if b.proc is None:
 522                     continue
 523                 none_running = False
 524                 b.status = b.proc.poll()
 525                 if b.status is None:
 526                     continue
 527                 b.proc = None
 528                 return b
 529             if options.retry:
 530                 ret = self.retry.proc.poll()
 531                 if ret is not None:
 532                     self.need_retry = True
 533                     self.retry = None
 534                     return None
 535             if none_running:
 536                 return None
 537             time.sleep(0.1)
 538
 539     def run(self):
 540         while True:
 541             b = self.wait_one()
 542             if options.retry and self.need_retry:
 543                 self.kill_kids()
 544                 do_print("retry needed")
 545                 return (0, None, None, None, "retry")
 546             if b is None:
 547                 break
 548             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
 549                 self.kill_kids()
 550                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
 551             b.start_next()
 552         self.kill_kids()
 553         return (0, None, None, None, "All OK")
 554
 555     def write_system_info(self):
 556         filename = 'system-info.txt'
 557         f = open(filename, 'w')
 558         for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo',
 559                     'cc --version', 'df -m .', 'df -m %s' % testbase]:
 560             print('### %s' % cmd, file=f)
 561             print(run_cmd(cmd, output=True, checkfail=False), file=f)
 562             print(file=f)
 563         f.close()
 564         return filename
 565
 566     def tarlogs(self, fname):
 567         tar = tarfile.open(fname, "w:gz")
 568         for b in self.tlist:
 569             tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
 570             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
 571         if os.path.exists("autobuild.log"):
 572             tar.add("autobuild.log")
 573         sys_info = self.write_system_info()
 574         tar.add(sys_info)
 575         tar.close()
 576
 577     def remove_logs(self):
 578         for b in self.tlist:
 579             os.unlink(b.stdout_path)
 580             os.unlink(b.stderr_path)
 581
 582     def start_tail(self):
 583         cwd = os.getcwd()
 584         cmd = "tail -f *.stdout *.stderr"
 585         os.chdir(gitroot)
 586         self.tail_proc = Popen(cmd, shell=True)
 587         os.chdir(cwd)
 588
 589
 590 def cleanup():
 591     if options.nocleanup:
 592         return
 593     run_cmd("stat %s || true" % test_tmpdir, show=True)
 594     run_cmd("stat %s" % testbase, show=True)
 595     do_print("Cleaning up ....")
 596     for d in cleanup_list:
 597         run_cmd("rm -rf %s" % d)
 598
 599
 600 def find_git_root():
 601     '''get to the top of the git repo'''
 602     p = os.getcwd()
 603     while p != '/':
 604         if os.path.isdir(os.path.join(p, ".git")):
 605             return p
 606         p = os.path.abspath(os.path.join(p, '..'))
 607     return None
 608
 609
 610 def daemonize(logfile):
 611     pid = os.fork()
 612     if pid == 0:  # Parent
 613         os.setsid()
 614         pid = os.fork()
 615         if pid != 0:  # Actual daemon
 616             os._exit(0)
 617     else:  # Grandparent
 618         os._exit(0)
 619
 620     import resource      # Resource usage information.
 621     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
 622     if maxfd == resource.RLIM_INFINITY:
 623         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
 624     for fd in range(0, maxfd):
 625         try:
 626             os.close(fd)
 627         except OSError:
 628             pass
 629     os.open(logfile, os.O_RDWR | os.O_CREAT)
 630     os.dup2(0, 1)
 631     os.dup2(0, 2)
 632
 633
 634 def write_pidfile(fname):
 635     '''write a pid file, cleanup on exit'''
 636     f = open(fname, mode='w')
 637     f.write("%u\n" % os.getpid())
 638     f.close()
 639
 640
 641 def rebase_tree(rebase_url, rebase_branch="master"):
 642     rebase_remote = "rebaseon"
 643     do_print("Rebasing on %s" % rebase_url)
 644     run_cmd("git describe HEAD", show=True, dir=test_master)
 645     run_cmd("git remote add -t %s %s %s" %
 646             (rebase_branch, rebase_remote, rebase_url),
 647             show=True, dir=test_master)
 648     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
 649     if options.fix_whitespace:
 650         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
 651                 (rebase_remote, rebase_branch),
 652                 show=True, dir=test_master)
 653     else:
 654         run_cmd("git rebase --force-rebase %s/%s" %
 655                 (rebase_remote, rebase_branch),
 656                 show=True, dir=test_master)
 657     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
 658                    (rebase_remote, rebase_branch),
 659                    dir=test_master, output=True)
 660     if diff == '':
 661         do_print("No differences between HEAD and %s/%s - exiting" %
 662                  (rebase_remote, rebase_branch))
 663         sys.exit(0)
 664     run_cmd("git describe %s/%s" %
 665             (rebase_remote, rebase_branch),
 666             show=True, dir=test_master)
 667     run_cmd("git describe HEAD", show=True, dir=test_master)
 668     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
 669             (rebase_remote, rebase_branch),
 670             show=True, dir=test_master)
 671
 672
 673 def push_to(push_url, push_branch="master"):
 674     push_remote = "pushto"
 675     do_print("Pushing to %s" % push_url)
 676     if options.mark:
 677         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
 678         run_cmd("git commit --amend -c HEAD", dir=test_master)
 679         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
 680         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
 681     run_cmd("git remote add -t %s %s %s" %
 682             (push_branch, push_remote, push_url),
 683             show=True, dir=test_master)
 684     run_cmd("git push %s +HEAD:%s" %
 685             (push_remote, push_branch),
 686             show=True, dir=test_master)
 687
 688 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
 689
 690 gitroot = find_git_root()
 691 if gitroot is None:
 692     raise Exception("Failed to find git root")
 693
 694 parser = OptionParser()
 695 parser.add_option("", "--tail", help="show output while running", default=False, action="store_true")
 696 parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true")
 697 parser.add_option("", "--nocleanup", help="don't remove test tree", default=False, action="store_true")
 698 parser.add_option("", "--testbase", help="base directory to run tests in (default %s)" % def_testbase,
 699                   default=def_testbase)
 700 parser.add_option("", "--passcmd", help="command to run on success", default=None)
 701 parser.add_option("", "--verbose", help="show all commands as they are run",
 702                   default=False, action="store_true")
 703 parser.add_option("", "--rebase", help="rebase on the given tree before testing",
 704                   default=None, type='str')
 705 parser.add_option("", "--pushto", help="push to a git url on success",
 706                   default=None, type='str')
 707 parser.add_option("", "--mark", help="add a Tested-By signoff before pushing",
 708                   default=False, action="store_true")
 709 parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase",
 710                   default=False, action="store_true")
 711 parser.add_option("", "--retry", help="automatically retry if master changes",
 712                   default=False, action="store_true")
 713 parser.add_option("", "--email", help="send email to the given address on failure",
 714                   type='str', default=None)
 715 parser.add_option("", "--email-from", help="send email from the given address",
 716                   type='str', default="autobuild@samba.org")
 717 parser.add_option("", "--email-server", help="send email via the given server",
 718                   type='str', default='localhost')
 719 parser.add_option("", "--always-email", help="always send email, even on success",
 720                   action="store_true")
 721 parser.add_option("", "--daemon", help="daemonize after initial setup",
 722                   action="store_true")
 723 parser.add_option("", "--branch", help="the branch to work on (default=master)",
 724                   default="master", type='str')
 725 parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)",
 726                   default=gitroot, type='str')
 727 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
 728                   default=False, action="store_true")
 729 parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
 730                   default='')
 731
 732
 733 def send_email(subject, text, log_tar):
 734     if options.email is None:
 735         do_print("not sending email because the recipient is not set")
 736         do_print("the text content would have been:\n\nSubject: %s\n\nTs" %
 737                  (subject, text))
 738         return
 739     outer = MIMEMultipart()
 740     outer['Subject'] = subject
 741     outer['To'] = options.email
 742     outer['From'] = options.email_from
 743     outer['Date'] = email.utils.formatdate(localtime=True)
 744     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
 745     outer.attach(MIMEText(text, 'plain'))
 746     if options.attach_logs:
 747         fp = open(log_tar, 'rb')
 748         msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
 749         fp.close()
 750         # Set the filename parameter
 751         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
 752         outer.attach(msg)
 753     content = outer.as_string()
 754     s = smtplib.SMTP(options.email_server)
 755     s.sendmail(options.email_from, [options.email], content)
 756     s.set_debuglevel(1)
 757     s.quit()
 758
 759
 760 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 761                   elapsed_time, log_base=None, add_log_tail=True):
 762     '''send an email to options.email about the failure'''
 763     elapsed_minutes = elapsed_time / 60.0
 764     user = os.getenv("USER")
 765     if log_base is None:
 766         log_base = gitroot
 767     text = '''
 768 Dear Developer,
 769
 770 Your autobuild on %s failed after %.1f minutes
 771 when trying to test %s with the following error:
 772
 773    %s
 774
 775 the autobuild has been abandoned. Please fix the error and resubmit.
 776
 777 A summary of the autobuild process is here:
 778
 779   %s/autobuild.log
 780 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 781
 782     if options.restrict_tests:
 783         text += """
 784 The build was restricted to tests matching %s\n""" % options.restrict_tests
 785
 786     if failed_task != 'rebase':
 787         text += '''
 788 You can see logs of the failed task here:
 789
 790   %s/%s.stdout
 791   %s/%s.stderr
 792
 793 or you can get full logs of all tasks in this job here:
 794
 795   %s/logs.tar.gz
 796
 797 The top commit for the tree that was built was:
 798
 799 %s
 800
 801 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 802
 803     if add_log_tail:
 804         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
 805         lines = f.readlines()
 806         log_tail = "".join(lines[-50:])
 807         num_lines = len(lines)
 808         if num_lines < 50:
 809             # Also include stderr (compile failures) if < 50 lines of stdout
 810             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
 811             log_tail += "".join(f.readlines()[-(50 - num_lines):])
 812
 813         text += '''
 814 The last 50 lines of log messages:
 815
 816 %s
 817     ''' % log_tail
 818         f.close()
 819
 820     logs = os.path.join(gitroot, 'logs.tar.gz')
 821     send_email('autobuild[%s] failure on %s for task %s during %s'
 822                % (options.branch, platform.node(), failed_task, failed_stage),
 823                text, logs)
 824
 825
 826 def email_success(elapsed_time, log_base=None):
 827     '''send an email to options.email about a successful build'''
 828     user = os.getenv("USER")
 829     if log_base is None:
 830         log_base = gitroot
 831     text = '''
 832 Dear Developer,
 833
 834 Your autobuild on %s has succeeded after %.1f minutes.
 835
 836 ''' % (platform.node(), elapsed_time / 60.)
 837
 838     if options.restrict_tests:
 839         text += """
 840 The build was restricted to tests matching %s\n""" % options.restrict_tests
 841
 842     if options.keeplogs:
 843         text += '''
 844
 845 you can get full logs of all tasks in this job here:
 846
 847   %s/logs.tar.gz
 848
 849 ''' % log_base
 850
 851     text += '''
 852 The top commit for the tree that was built was:
 853
 854 %s
 855 ''' % top_commit_msg
 856
 857     logs = os.path.join(gitroot, 'logs.tar.gz')
 858     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
 859                text, logs)
 860
 861
 862 (options, args) = parser.parse_args()
 863
 864 if options.retry:
 865     if options.rebase is None:
 866         raise Exception('You can only use --retry if you also rebase')
 867
 868 testbase = "%s/b%u" % (options.testbase, os.getpid())
 869 test_master = "%s/master" % testbase
 870 test_prefix = "%s/prefix" % testbase
 871 test_tmpdir = "%s/tmp" % testbase
 872 os.environ['TMPDIR'] = test_tmpdir
 873
 874 # get the top commit message, for emails
 875 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
 876
 877 try:
 878     os.makedirs(testbase)
 879 except Exception as reason:
 880     raise Exception("Unable to create %s : %s" % (testbase, reason))
 881 cleanup_list.append(testbase)
 882
 883 if options.daemon:
 884     logfile = os.path.join(testbase, "log")
 885     do_print("Forking into the background, writing progress to %s" % logfile)
 886     daemonize(logfile)
 887
 888 write_pidfile(gitroot + "/autobuild.pid")
 889
 890 start_time = time.time()
 891
 892 while True:
 893     try:
 894         run_cmd("rm -rf %s" % test_tmpdir, show=True)
 895         os.makedirs(test_tmpdir)
 896         # The waf uninstall code removes empty directories all the way
 897         # up the tree.  Creating a file in test_tmpdir stops it from
 898         # being removed.
 899         run_cmd("touch %s" % os.path.join(test_tmpdir,
 900                                           ".directory-is-not-empty"), show=True)
 901         run_cmd("stat %s" % test_tmpdir, show=True)
 902         run_cmd("stat %s" % testbase, show=True)
 903         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
 904     except Exception:
 905         cleanup()
 906         raise
 907
 908     try:
 909         try:
 910             if options.rebase is not None:
 911                 rebase_tree(options.rebase, rebase_branch=options.branch)
 912         except Exception:
 913             cleanup_list.append(gitroot + "/autobuild.pid")
 914             cleanup()
 915             elapsed_time = time.time() - start_time
 916             email_failure(-1, 'rebase', 'rebase', 'rebase',
 917                           'rebase on %s failed' % options.branch,
 918                           elapsed_time, log_base=options.log_base)
 919             sys.exit(1)
 920         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
 921         if options.tail:
 922             blist.start_tail()
 923         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
 924         if status != 0 or errstr != "retry":
 925             break
 926         cleanup()
 927     except Exception:
 928         cleanup()
 929         raise
 930
 931 cleanup_list.append(gitroot + "/autobuild.pid")
 932
 933 do_print(errstr)
 934
 935 blist.kill_kids()
 936 if options.tail:
 937     do_print("waiting for tail to flush")
 938     time.sleep(1)
 939
 940 elapsed_time = time.time() - start_time
 941 if status == 0:
 942     if options.passcmd is not None:
 943         do_print("Running passcmd: %s" % options.passcmd)
 944         run_cmd(options.passcmd, dir=test_master)
 945     if options.pushto is not None:
 946         push_to(options.pushto, push_branch=options.branch)
 947     if options.keeplogs or options.attach_logs:
 948         blist.tarlogs("logs.tar.gz")
 949         do_print("Logs in logs.tar.gz")
 950     if options.always_email:
 951         email_success(elapsed_time, log_base=options.log_base)
 952     blist.remove_logs()
 953     cleanup()
 954     do_print(errstr)
 955     sys.exit(0)
 956
 957 # something failed, gather a tar of the logs
 958 blist.tarlogs("logs.tar.gz")
 959
 960 if options.email is not None:
 961     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
 962                   elapsed_time, log_base=options.log_base)
 963 else:
 964     elapsed_minutes = elapsed_time / 60.0
 965     print('''
 966
 967 ####################################################################
 968
 969 AUTOBUILD FAILURE
 970
 971 Your autobuild[%s] on %s failed after %.1f minutes
 972 when trying to test %s with the following error:
 973
 974    %s
 975
 976 the autobuild has been abandoned. Please fix the error and resubmit.
 977
 978 ####################################################################
 979
 980 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
 981
 982 cleanup()
 983 do_print(errstr)
 984 do_print("Logs in logs.tar.gz")
 985 sys.exit(status)