ctdb-daemon: Refactor check for valid events during recovery

[sfrench/samba-autobuild/.git] / script / autobuild.py

diff --git a/script/autobuild.py b/script/autobuild.py
index 13be1b6a856a88f1aa1833f273259f180600228d..3d76130b57d0f616dfea55534f7fd9c882c4f6a9 100755 (executable)
--- a/script/autobuild.py
+++ b/script/autobuild.py
@@ -24,9 +24,12 @@ builddirs = {
     "ctdb"    : "ctdb",
     "samba"  : ".",
     "samba-xc" : ".",
+    "samba-o3" : ".",
     "samba-ctdb" : ".",
     "samba-libs"  : ".",
     "samba-static"  : ".",
+    "samba-test-only"  : ".",
+    "samba-systemkrb5"  : ".",
     "ldb"     : "lib/ldb",
     "tdb"     : "lib/tdb",
     "talloc"  : "lib/talloc",
@@ -38,7 +41,10 @@ builddirs = {
     "retry"   : "."
     }
 
-defaulttasks = [ "ctdb", "samba", "samba-xc", "samba-ctdb", "samba-libs", "samba-static", "ldb", "tdb", "talloc", "replace", "tevent", "pidl" ]
+defaulttasks = [ "ctdb", "samba", "samba-xc", "samba-o3", "samba-ctdb", "samba-libs", "samba-static", "samba-systemkrb5", "ldb", "tdb", "talloc", "replace", "tevent", "pidl" ]
+
+if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
+    defaulttasks.remove("samba-o3")
 
 samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data"
 
@@ -71,6 +77,10 @@ tasks = {
                 ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
                 ("clean", "make clean", "text/plain") ],
 
+    "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab  --abi-check-disable" + samba_configure_params, "text/plain"),
+                          ("make", "make -j", "text/plain"),
+                          ("test", "make test FAIL_IMMEDIATELY=1 TESTS=${TESTS}", "text/plain") ],
+
     # Test cross-compile infrastructure
     "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"),
                    ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
@@ -79,6 +89,14 @@ tasks = {
                     " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"),
                    ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")],
 
+    # test build with -O3 -- catches extra warnings and bugs
+    "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"),
+                   ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"),
+                   ("make", "make -j", "text/plain"),
+                   ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='\(ad_dc\)'", "text/plain"),
+                   ("install", "make install", "text/plain"),
+                   ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
+                   ("clean", "make clean", "text/plain") ],
 
     "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
 
@@ -145,6 +163,20 @@ tasks = {
                       ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"),
                       ("nonshared-make", "make -j", "text/plain")],
 
+    "samba-systemkrb5" : [
+                      ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"),
+                      ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"),
+                      ("make", "make -j", "text/plain"),
+                      # we currently cannot run a full make test, a limited list of tests could be run
+                      # via "make test TESTS=sometests"
+                      # ("test", "make test FAIL_IMMEDIATELY=1", "text/plain"),
+                      ("install", "make install", "text/plain"),
+                      ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"),
+                      ("clean", "make clean", "text/plain")
+                      ],
+
+
+
     "ldb" : [
               ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"),
               ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"),
@@ -244,11 +276,7 @@ class builder(object):
         self.stderr = open(self.stderr_path, 'w')
         self.stdin  = open("/dev/null", 'r')
         self.sdir = "%s/%s" % (testbase, self.tag)
-        self.prefix = "%s/prefix/%s" % (testbase, self.tag)
-        run_cmd("rm -rf %s" % self.sdir)
-        cleanup_list.append(self.sdir)
-        cleanup_list.append(self.prefix)
-        os.makedirs(self.sdir)
+        self.prefix = "%s/%s" % (test_prefix, self.tag)
         run_cmd("rm -rf %s" % self.sdir)
         if cp:
             run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True)
@@ -266,6 +294,7 @@ class builder(object):
         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
         self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python)
         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
+        self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
 #        if self.output_mime_type == "text/x-subunit":
 #            self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit"))
         print '%s: [%s] Running %s' % (self.name, self.stage, self.cmd)
@@ -280,13 +309,16 @@ class builder(object):
 class buildlist(object):
     '''handle build of multiple directories'''
 
-    def __init__(self, tasklist, tasknames, rebase_url, rebase_branch="master"):
+    def __init__(self, tasknames, rebase_url, rebase_branch="master"):
         global tasks
         self.tlist = []
         self.tail_proc = None
         self.retry = None
         if tasknames == []:
-            tasknames = defaulttasks
+            if options.restrict_tests:
+                tasknames = ["samba-test-only"]
+            else:
+                tasknames = defaulttasks
         else:
             # If we are only running one test,
             # do not sleep randomly to wait for it to start
@@ -543,6 +575,8 @@ parser.add_option("", "--log-base", help="location where the logs can be found (
                   default=gitroot, type='str')
 parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?",
                   default=False, action="store_true")
+parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex",
+                  default='')
 
 def send_email(subject, text, log_tar):
     outer = MIMEMultipart()
@@ -587,6 +621,10 @@ A summary of the autobuild process is here:
   %s/autobuild.log
 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
 
+    if options.restrict_tests:
+        text += """
+The build was restricted to tests matching %s\n""" % options.restrict_tests
+
     if failed_task != 'rebase':
         text += '''
 You can see logs of the failed task here:
@@ -605,8 +643,14 @@ The top commit for the tree that was built was:
 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
 
     if add_log_tail:
-        f = open("%s/%s.stdout" % (log_base, failed_tag), 'r')
-        log_tail = "".join(f.readlines()[-50:])
+        f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
+        lines = f.readlines()
+        log_tail = "".join(lines[-50:])
+        num_lines = len(lines)
+        if num_lines < 50:
+            # Also include stderr (compile failures) if < 50 lines of stdout
+            f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
+            log_tail += "".join(f.readlines()[-(50-num_lines):])
 
         text += '''
 The last 50 lines of log messages:
@@ -616,8 +660,8 @@ The last 50 lines of log messages:
         f.close()
 
     logs = os.path.join(gitroot, 'logs.tar.gz')
-    send_email('autobuild failure on %s for task %s during %s'
-               % (platform.node(), failed_task, failed_stage),
+    send_email('autobuild[%s] failure on %s for task %s during %s'
+               % (options.branch, platform.node(), failed_task, failed_stage),
                text, logs)
 
 def email_success(elapsed_time, log_base=None):
@@ -632,6 +676,10 @@ Your autobuild on %s has succeeded after %.1f minutes.
 
 ''' % (platform.node(), elapsed_time / 60.)
 
+    if options.restrict_tests:
+        text += """
+The build was restricted to tests matching %s\n""" % options.restrict_tests
+
     if options.keeplogs:
         text += '''
 
@@ -648,7 +696,7 @@ The top commit for the tree that was built was:
 ''' % top_commit_msg
 
     logs = os.path.join(gitroot, 'logs.tar.gz')
-    send_email('autobuild sucess on %s ' % platform.node(),
+    send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
                text, logs)
 
 
@@ -660,6 +708,9 @@ if options.retry:
 
 testbase = "%s/b%u" % (options.testbase, os.getpid())
 test_master = "%s/master" % testbase
+test_prefix = "%s/prefix" % testbase
+test_tmpdir = "%s/tmp" % testbase
+os.environ['TMPDIR'] = test_tmpdir
 
 # get the top commit message, for emails
 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
@@ -682,7 +733,9 @@ start_time = time.time()
 while True:
     try:
         run_cmd("rm -rf %s" % test_master)
-        cleanup_list.append(test_master)
+        run_cmd("rm -rf %s" % test_prefix)
+        run_cmd("rm -rf %s" % test_tmpdir)
+        os.makedirs(test_tmpdir)
         run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
     except Exception:
         cleanup()
@@ -700,7 +753,7 @@ while True:
                           'rebase on %s failed' % options.branch,
                           elapsed_time, log_base=options.log_base)
             sys.exit(1)
-        blist = buildlist(tasks, args, options.rebase, rebase_branch=options.branch)
+        blist = buildlist(args, options.rebase, rebase_branch=options.branch)
         if options.tail:
             blist.start_tail()
         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
@@ -750,7 +803,7 @@ else:
 
 AUTOBUILD FAILURE
 
-Your autobuild on %s failed after %.1f minutes
+Your autobuild[%s] on %s failed after %.1f minutes
 when trying to test %s with the following error:
 
    %s
@@ -759,7 +812,7 @@ the autobuild has been abandoned. Please fix the error and resubmit.
 
 ####################################################################
 
-''' % (platform.node(), elapsed_minutes, failed_task, errstr)
+''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr)
 
 cleanup()
 print(errstr)