s4:auth/ntlm: allow auth_operations to specify check_password_send/recv()
[sfrench/samba-autobuild/.git] / script / autobuild.py
index d2662b95afe4d628a87b5821a44bf68b44300677..e4a52ee7ff52d9d699d10d518eb0d548d8f340a5 100755 (executable)
@@ -15,6 +15,8 @@ from email.mime.multipart import MIMEMultipart
 from distutils.sysconfig import get_python_lib
 import platform
 
+os.environ["PYTHONUNBUFFERED"] = "1"
+
 # This speeds up testing remarkably.
 os.environ['TDB_NO_FSYNC'] = '1'
 
@@ -24,8 +26,13 @@ builddirs = {
     "ctdb"    : "ctdb",
     "samba"  : ".",
     "samba-xc" : ".",
+    "samba-o3" : ".",
     "samba-ctdb" : ".",
     "samba-libs"  : ".",
+    "samba-static"  : ".",
+    "samba-test-only"  : ".",
+    "samba-systemkrb5"  : ".",
+    "samba-nopython"  : ".",
     "ldb"     : "lib/ldb",
     "tdb"     : "lib/tdb",
     "talloc"  : "lib/talloc",
@@ -37,13 +44,29 @@ builddirs = {
     "retry"   : "."
     }
 
-defaulttasks = [ "ctdb", "samba", "samba-xc", "samba-ctdb", "samba-libs", "ldb", "tdb", "talloc", "replace", "tevent", "pidl" ]
+defaulttasks = [ "ctdb", "samba", "samba-xc", "samba-o3", "samba-ctdb", "samba-libs", "samba-static", "samba-systemkrb5", "samba-nopython", "ldb", "tdb", "talloc", "replace", "tevent", "pidl" ]
+
+if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
+    defaulttasks.remove("samba-o3")
+
+ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}"
+samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
 
-samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data"
+samba_libs_envvars =  "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH"
+samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
+samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
+samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX} ${EXTRA_PYTHON}"
+samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE"
+samba_libs_configure_samba = samba_libs_configure_base + " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent"
+
+if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1":
+    extra_python = ""
+else:
+    extra_python = "--extra-python=/usr/bin/python3"
 
 tasks = {
     "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
-               ("configure", "./configure ${PREFIX}", "text/plain"),
+               ("configure", "./configure " + ctdb_configure_params, "text/plain"),
                ("make", "make all", "text/plain"),
                ("install", "make install", "text/plain"),
                ("test", "make autotest", "text/plain"),
@@ -58,6 +81,10 @@ tasks = {
                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
                 ("clean", "make clean", "text/plain") ],
 
+    "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
+                          ("make", "make -j", "text/plain"),
+                          ("test", "make test FAIL_IMMEDIATELY=1 TESTS=${TESTS}", "text/plain") ],
+
     # Test cross-compile infrastructure
     "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
@@ -66,6 +93,14 @@ tasks = {
                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 
+    # test build with -O3 -- catches extra warnings and bugs
+    "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
+                   ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
+                   ("make", "make -j", "text/plain"),
+                   ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='\(ad_dc\)'", "text/plain"),
+                   ("install", "make install", "text/plain"),
+                   ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
+                   ("clean", "make clean", "text/plain") ],
 
     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 
@@ -89,30 +124,81 @@ tasks = {
 
     "samba-libs" : [
                       ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
-                      ("talloc-configure", "cd lib/talloc && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
+                      ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs, "text/plain"),
                       ("talloc-make", "cd lib/talloc && make", "text/plain"),
                       ("talloc-install", "cd lib/talloc && make install", "text/plain"),
 
-                      ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
+                      ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs, "text/plain"),
                       ("tdb-make", "cd lib/tdb && make", "text/plain"),
                       ("tdb-install", "cd lib/tdb && make install", "text/plain"),
 
-                      ("tevent-configure", "cd lib/tevent && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
+                      ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs, "text/plain"),
                       ("tevent-make", "cd lib/tevent && make", "text/plain"),
                       ("tevent-install", "cd lib/tevent && make install", "text/plain"),
 
-                      ("ldb-configure", "cd lib/ldb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
+                      ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs, "text/plain"),
                       ("ldb-make", "cd lib/ldb && make", "text/plain"),
                       ("ldb-install", "cd lib/ldb && make install", "text/plain"),
 
-                      ("configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=!talloc,!tdb,!pytdb,!ldb,!pyldb,!tevent,!pytevent --abi-check --enable-debug -C ${PREFIX}", "text/plain"),
-                      ("make", "make", "text/plain"),
+                      ("nondevel-configure", "./configure ${PREFIX}", "text/plain"),
+                      ("nondevel-make", "make -j", "text/plain"),
+                      ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0", "text/plain"),
+                      ("nondevel-install", "make install", "text/plain"),
+                      ("nondevel-dist", "make dist", "text/plain"),
+
+                      # retry with all modules shared
+                      ("allshared-distclean", "make distclean", "text/plain"),
+                      ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"),
+                      ("allshared-make", "make -j", "text/plain")],
+
+    "samba-static" : [
+                      ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+                      # build with all modules static
+                      ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"),
+                      ("allstatic-make", "make -j", "text/plain"),
+
+                      # retry without any required modules
+                      ("none-distclean", "make distclean", "text/plain"),
+                      ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"),
+                      ("none-make", "make -j", "text/plain"),
+
+                      # retry with nonshared smbd and smbtorture
+                      ("nonshared-distclean", "make distclean", "text/plain"),
+                      ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
+                      ("nonshared-make", "make -j", "text/plain")],
+
+    "samba-systemkrb5" : [
+                      ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+                      ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
+                      ("make", "make -j", "text/plain"),
+                      # we currently cannot run a full make test, a limited list of tests could be run
+                      # via "make test TESTS=sometests"
+                      ("test", "make test FAIL_IMMEDIATELY=1 TESTS='samba3.*ktest'", "text/plain"),
                       ("install", "make install", "text/plain"),
-                      ("dist", "make dist", "text/plain")],
+                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
+                      ("clean", "make clean", "text/plain")
+                      ],
+
+    # Test Samba without python still builds.  When this test fails
+    # due to more use of Python, the expectations is that the newly
+    # failing part of the code should be disabled when
+    # --disable-python is set (rather than major work being done to
+    # support this environment).  The target here is for vendors
+    # shipping a minimal smbd.
+    "samba-nopython" : [
+                      ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+                      ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"),
+                      ("make", "make -j", "text/plain"),
+                      ("install", "make install", "text/plain"),
+                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
+                      ("clean", "make clean", "text/plain")
+                      ],
+
+
 
     "ldb" : [
               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
-              ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
+              ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
               ("make", "make", "text/plain"),
               ("install", "make install", "text/plain"),
               ("test", "make test", "text/plain"),
@@ -122,7 +208,7 @@ tasks = {
 
     "tdb" : [
               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
-              ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
+              ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
               ("make", "make", "text/plain"),
               ("install", "make install", "text/plain"),
               ("test", "make test", "text/plain"),
@@ -132,7 +218,7 @@ tasks = {
 
     "talloc" : [
                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
-                 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
+                 ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
                  ("make", "make", "text/plain"),
                  ("install", "make install", "text/plain"),
                  ("test", "make test", "text/plain"),
@@ -152,7 +238,7 @@ tasks = {
 
     "tevent" : [
                  ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
-                 ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"),
+                 ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
                  ("make", "make", "text/plain"),
                  ("install", "make install", "text/plain"),
                  ("test", "make test", "text/plain"),
@@ -176,11 +262,16 @@ tasks = {
     'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ]
 }
 
+def do_print(msg):
+    print "%s" % msg
+    sys.stdout.flush()
+    sys.stderr.flush()
+
 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
     if show is None:
         show = options.verbose
     if show:
-        print("Running: '%s' in '%s'" % (cmd, dir))
+        do_print("Running: '%s' in '%s'" % (cmd, dir))
     if output:
         return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0]
     elif checkfail:
@@ -192,7 +283,7 @@ def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
 class builder(object):
     '''handle build of one directory'''
 
-    def __init__(self, name, sequence):
+    def __init__(self, name, sequence, cp=True):
         self.name = name
         self.dir = builddirs[name]
 
@@ -202,34 +293,39 @@ class builder(object):
         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
         if options.verbose:
-            print("stdout for %s in %s" % (self.name, self.stdout_path))
-            print("stderr for %s in %s" % (self.name, self.stderr_path))
+            do_print("stdout for %s in %s" % (self.name, self.stdout_path))
+            do_print("stderr for %s in %s" % (self.name, self.stderr_path))
         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
         self.stdout = open(self.stdout_path, 'w')
         self.stderr = open(self.stderr_path, 'w')
         self.stdin  = open("/dev/null", 'r')
         self.sdir = "%s/%s" % (testbase, self.tag)
-        self.prefix = "%s/prefix/%s" % (testbase, self.tag)
-        run_cmd("rm -rf %s" % self.sdir)
-        cleanup_list.append(self.sdir)
-        cleanup_list.append(self.prefix)
-        os.makedirs(self.sdir)
+        self.prefix = "%s/%s" % (test_prefix, self.tag)
         run_cmd("rm -rf %s" % self.sdir)
-        run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
+        run_cmd("rm -rf %s" % self.prefix)
+        if cp:
+            run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
+        else:
+            run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True)
         self.start_next()
 
     def start_next(self):
         if self.next == len(self.sequence):
-            print '%s: Completed OK' % self.name
+            if not options.nocleanup:
+                run_cmd("rm -rf %s" % self.sdir)
+                run_cmd("rm -rf %s" % self.prefix)
+            do_print('%s: Completed OK' % self.name)
             self.done = True
             return
         (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next]
         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix))
         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
+        self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
+        self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 #        if self.output_mime_type == "text/x-subunit":
 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
-        print '%s: [%s] Running %s' % (self.name, self.stage, self.cmd)
+        do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd))
         cwd = os.getcwd()
         os.chdir("%s/%s" % (self.sdir, self.dir))
         self.proc = Popen(self.cmd, shell=True,
@@ -241,20 +337,23 @@ class builder(object):
 class buildlist(object):
     '''handle build of multiple directories'''
 
-    def __init__(self, tasklist, tasknames, rebase_url, rebase_branch="master"):
+    def __init__(self, tasknames, rebase_url, rebase_branch="master"):
         global tasks
         self.tlist = []
         self.tail_proc = None
         self.retry = None
         if tasknames == []:
-            tasknames = defaulttasks
+            if options.restrict_tests:
+                tasknames = ["samba-test-only"]
+            else:
+                tasknames = defaulttasks
         else:
             # If we are only running one test,
             # do not sleep randomly to wait for it to start
             os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1'
 
         for n in tasknames:
-            b = builder(n, tasks[n])
+            b = builder(n, tasks[n], cp=n is not "pidl")
             self.tlist.append(b)
         if options.retry:
             rebase_remote = "rebaseon"
@@ -278,7 +377,7 @@ class buildlist(object):
                            ),
                            "test/plain" ) ]
 
-            self.retry = builder('retry', retry_task)
+            self.retry = builder('retry', retry_task, cp=False)
             self.need_retry = False
 
     def kill_kids(self):
@@ -324,7 +423,7 @@ class buildlist(object):
             b = self.wait_one()
             if options.retry and self.need_retry:
                 self.kill_kids()
-                print("retry needed")
+                do_print("retry needed")
                 return (0, None, None, None, "retry")
             if b is None:
                 break
@@ -335,6 +434,16 @@ class buildlist(object):
         self.kill_kids()
         return (0, None, None, None, "All OK")
 
+    def write_system_info(self):
+        filename = 'system-info.txt'
+        f = open(filename, 'w')
+        for cmd in ['uname -a', 'free', 'cat /proc/cpuinfo']:
+            print >>f, '### %s' % cmd
+            print >>f, run_cmd(cmd, output=True, checkfail=False)
+            print >>f
+        f.close()
+        return filename
+
     def tarlogs(self, fname):
         tar = tarfile.open(fname, "w:gz")
         for b in self.tlist:
@@ -342,6 +451,8 @@ class buildlist(object):
             tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
         if os.path.exists("autobuild.log"):
             tar.add("autobuild.log")
+        sys_info = self.write_system_info()
+        tar.add(sys_info)
         tar.close()
 
     def remove_logs(self):
@@ -360,7 +471,9 @@ class buildlist(object):
 def cleanup():
     if options.nocleanup:
         return
-    print("Cleaning up ....")
+    run_cmd("stat %s || true" % test_tmpdir, show=True)
+    run_cmd("stat %s" % testbase, show=True)
+    do_print("Cleaning up ....")
     for d in cleanup_list:
         run_cmd("rm -rf %s" % d)
 
@@ -407,7 +520,7 @@ def write_pidfile(fname):
 
 def rebase_tree(rebase_url, rebase_branch = "master"):
     rebase_remote = "rebaseon"
-    print("Rebasing on %s" % rebase_url)
+    do_print("Rebasing on %s" % rebase_url)
     run_cmd("git describe HEAD", show=True, dir=test_master)
     run_cmd("git remote add -t %s %s %s" %
             (rebase_branch, rebase_remote, rebase_url),
@@ -425,7 +538,7 @@ def rebase_tree(rebase_url, rebase_branch = "master"):
                    (rebase_remote, rebase_branch),
                    dir=test_master, output=True)
     if diff == '':
-        print("No differences between HEAD and %s/%s - exiting" %
+        do_print("No differences between HEAD and %s/%s - exiting" %
               (rebase_remote, rebase_branch))
         sys.exit(0)
     run_cmd("git describe %s/%s" %
@@ -438,7 +551,7 @@ def rebase_tree(rebase_url, rebase_branch = "master"):
 
 def push_to(push_url, push_branch = "master"):
     push_remote = "pushto"
-    print("Pushing to %s" % push_url)
+    do_print("Pushing to %s" % push_url)
     if options.mark:
         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
         run_cmd("git commit --amend -c HEAD", dir=test_master)
@@ -492,6 +605,8 @@ parser.add_option("", "--log-base", help="location where the logs can be found (
                   default=gitroot, type='str')
 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
                   default=False, action="store_true")
+parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
+                  default='')
 
 def send_email(subject, text, log_tar):
     outer = MIMEMultipart()
@@ -515,7 +630,7 @@ def send_email(subject, text, log_tar):
     s.quit()
 
 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
-                  elapsed_time, log_base=None):
+                  elapsed_time, log_base=None, add_log_tail=True):
     '''send an email to options.email about the failure'''
     elapsed_minutes = elapsed_time / 60.0
     user = os.getenv("USER")
@@ -536,6 +651,10 @@ A summary of the autobuild process is here:
   %s/autobuild.log
 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 
+    if options.restrict_tests:
+        text += """
+The build was restricted to tests matching %s\n""" % options.restrict_tests
+
     if failed_task != 'rebase':
         text += '''
 You can see logs of the failed task here:
@@ -553,9 +672,26 @@ The top commit for the tree that was built was:
 
 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 
+    if add_log_tail:
+        f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
+        lines = f.readlines()
+        log_tail = "".join(lines[-50:])
+        num_lines = len(lines)
+        if num_lines < 50:
+            # Also include stderr (compile failures) if < 50 lines of stdout
+            f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
+            log_tail += "".join(f.readlines()[-(50-num_lines):])
+
+        text += '''
+The last 50 lines of log messages:
+
+%s
+    ''' % log_tail
+        f.close()
+
     logs = os.path.join(gitroot, 'logs.tar.gz')
-    send_email('autobuild failure on %s for task %s during %s'
-               % (platform.node(), failed_task, failed_stage),
+    send_email('autobuild[%s] failure on %s for task %s during %s'
+               % (options.branch, platform.node(), failed_task, failed_stage),
                text, logs)
 
 def email_success(elapsed_time, log_base=None):
@@ -570,6 +706,10 @@ Your autobuild on %s has succeeded after %.1f minutes.
 
 ''' % (platform.node(), elapsed_time / 60.)
 
+    if options.restrict_tests:
+        text += """
+The build was restricted to tests matching %s\n""" % options.restrict_tests
+
     if options.keeplogs:
         text += '''
 
@@ -586,7 +726,7 @@ The top commit for the tree that was built was:
 ''' % top_commit_msg
 
     logs = os.path.join(gitroot, 'logs.tar.gz')
-    send_email('autobuild sucess on %s ' % platform.node(),
+    send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
                text, logs)
 
 
@@ -598,6 +738,9 @@ if options.retry:
 
 testbase = "%s/b%u" % (options.testbase, os.getpid())
 test_master = "%s/master" % testbase
+test_prefix = "%s/prefix" % testbase
+test_tmpdir = "%s/tmp" % testbase
+os.environ['TMPDIR'] = test_tmpdir
 
 # get the top commit message, for emails
 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
@@ -610,7 +753,7 @@ cleanup_list.append(testbase)
 
 if options.daemon:
     logfile = os.path.join(testbase, "log")
-    print "Forking into the background, writing progress to %s" % logfile
+    do_print("Forking into the background, writing progress to %s" % logfile)
     daemonize(logfile)
 
 write_pidfile(gitroot + "/autobuild.pid")
@@ -619,8 +762,15 @@ start_time = time.time()
 
 while True:
     try:
-        run_cmd("rm -rf %s" % test_master)
-        cleanup_list.append(test_master)
+        run_cmd("rm -rf %s" % test_tmpdir, show=True)
+        os.makedirs(test_tmpdir)
+        # The waf uninstall code removes empty directories all the way
+        # up the tree.  Creating a file in test_tmpdir stops it from
+        # being removed.
+        run_cmd("touch %s" % os.path.join(test_tmpdir,
+                                          ".directory-is-not-empty"), show=True)
+        run_cmd("stat %s" % test_tmpdir, show=True)
+        run_cmd("stat %s" % testbase, show=True)
         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
     except Exception:
         cleanup()
@@ -638,7 +788,7 @@ while True:
                           'rebase on %s failed' % options.branch,
                           elapsed_time, log_base=options.log_base)
             sys.exit(1)
-        blist = buildlist(tasks, args, options.rebase, rebase_branch=options.branch)
+        blist = buildlist(args, options.rebase, rebase_branch=options.branch)
         if options.tail:
             blist.start_tail()
         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
@@ -651,27 +801,28 @@ while True:
 
 cleanup_list.append(gitroot + "/autobuild.pid")
 
+do_print(errstr)
+
 blist.kill_kids()
 if options.tail:
-    print("waiting for tail to flush")
+    do_print("waiting for tail to flush")
     time.sleep(1)
 
 elapsed_time = time.time() - start_time
 if status == 0:
-    print errstr
     if options.passcmd is not None:
-        print("Running passcmd: %s" % options.passcmd)
+        do_print("Running passcmd: %s" % options.passcmd)
         run_cmd(options.passcmd, dir=test_master)
     if options.pushto is not None:
         push_to(options.pushto, push_branch=options.branch)
     if options.keeplogs or options.attach_logs:
         blist.tarlogs("logs.tar.gz")
-        print("Logs in logs.tar.gz")
+        do_print("Logs in logs.tar.gz")
     if options.always_email:
         email_success(elapsed_time, log_base=options.log_base)
     blist.remove_logs()
     cleanup()
-    print(errstr)
+    do_print(errstr)
     sys.exit(0)
 
 # something failed, gather a tar of the logs
@@ -680,8 +831,26 @@ blist.tarlogs("logs.tar.gz")
 if options.email is not None:
     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
                   elapsed_time, log_base=options.log_base)
+else:
+    elapsed_minutes = elapsed_time / 60.0
+    print '''
+
+####################################################################
+
+AUTOBUILD FAILURE
+
+Your autobuild[%s] on %s failed after %.1f minutes
+when trying to test %s with the following error:
+
+   %s
+
+the autobuild has been abandoned. Please fix the error and resubmit.
+
+####################################################################
+
+''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr)
 
 cleanup()
-print(errstr)
-print("Logs in logs.tar.gz")
+do_print(errstr)
+do_print("Logs in logs.tar.gz")
 sys.exit(status)