X-Git-Url: http://git.samba.org/samba.git/?p=sfrench%2Fsamba-autobuild%2F.git;a=blobdiff_plain;f=script%2Fautobuild.py;h=e4a52ee7ff52d9d699d10d518eb0d548d8f340a5;hp=2d3af2a7db28e1e72bc6057769ae100bfd8d33ce;hb=c29db055a7052c25c9d4d1adfe1afff5c32f65ec;hpb=db04752f3ecefc19fbf199930931fa31dde1632c diff --git a/script/autobuild.py b/script/autobuild.py index 2d3af2a7db2..e4a52ee7ff5 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -15,6 +15,8 @@ from email.mime.multipart import MIMEMultipart from distutils.sysconfig import get_python_lib import platform +os.environ["PYTHONUNBUFFERED"] = "1" + # This speeds up testing remarkably. os.environ['TDB_NO_FSYNC'] = '1' @@ -24,8 +26,13 @@ builddirs = { "ctdb" : "ctdb", "samba" : ".", "samba-xc" : ".", + "samba-o3" : ".", "samba-ctdb" : ".", "samba-libs" : ".", + "samba-static" : ".", + "samba-test-only" : ".", + "samba-systemkrb5" : ".", + "samba-nopython" : ".", "ldb" : "lib/ldb", "tdb" : "lib/tdb", "talloc" : "lib/talloc", @@ -37,20 +44,29 @@ builddirs = { "retry" : "." } -defaulttasks = [ "ctdb", "samba", "samba-xc", "samba-ctdb", "samba-libs", "ldb", "tdb", "talloc", "replace", "tevent", "pidl" ] +defaulttasks = [ "ctdb", "samba", "samba-xc", "samba-o3", "samba-ctdb", "samba-libs", "samba-static", "samba-systemkrb5", "samba-nopython", "ldb", "tdb", "talloc", "replace", "tevent", "pidl" ] + +if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1": + defaulttasks.remove("samba-o3") -samba_configure_params = " --picky-developer ${PREFIX} --with-profiling-data" +ctdb_configure_params = " --enable-developer --picky-developer ${PREFIX}" +samba_configure_params = " --picky-developer ${PREFIX} ${EXTRA_PYTHON} --with-profiling-data" samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH" samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig" samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'" -samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX}" -samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=NONE" +samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check --enable-debug --picky-developer -C ${PREFIX} ${EXTRA_PYTHON}" +samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,NONE" samba_libs_configure_samba = samba_libs_configure_base + " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent" +if os.environ.get("AUTOBUILD_NO_EXTRA_PYTHON", "0") == "1": + extra_python = "" +else: + extra_python = "--extra-python=/usr/bin/python3" + tasks = { "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"), - ("configure", "./configure ${PREFIX}", "text/plain"), + ("configure", "./configure " + ctdb_configure_params, "text/plain"), ("make", "make all", "text/plain"), ("install", "make install", "text/plain"), ("test", "make autotest", "text/plain"), @@ -65,6 +81,10 @@ tasks = { ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"), ("clean", "make clean", "text/plain") ], + "samba-test-only" : [ ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"), + ("make", "make -j", "text/plain"), + ("test", "make test FAIL_IMMEDIATELY=1 TESTS=${TESTS}", "text/plain") ], + # Test cross-compile infrastructure "samba-xc" : [ ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params, "text/plain"), ("configure-cross-execute", "./configure.developer -b ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \ @@ -73,6 +93,14 @@ tasks = { " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params, "text/plain"), ("compare-results", "script/compare_cc_results.py ./bin/c4che/default.cache.py ./bin-xe/c4che/default.cache.py ./bin-xa/c4che/default.cache.py", "text/plain")], + # test build with -O3 -- catches extra warnings and bugs + "samba-o3" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"), + ("configure", "ADDITIONAL_CFLAGS='-O3' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params, "text/plain"), + ("make", "make -j", "text/plain"), + ("test", "make quicktest FAIL_IMMEDIATELY=1 TESTS='\(ad_dc\)'", "text/plain"), + ("install", "make install", "text/plain"), + ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"), + ("clean", "make clean", "text/plain") ], "samba-ctdb" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"), @@ -121,26 +149,56 @@ tasks = { # retry with all modules shared ("allshared-distclean", "make distclean", "text/plain"), ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL", "text/plain"), - ("allshared-make", "make", "text/plain"), + ("allshared-make", "make -j", "text/plain")], - # retry with all modules static - ("allstatic-distclean", "make distclean", "text/plain"), - ("allstatic-configure", samba_libs_configure_samba + " --with-static-modules=ALL", "text/plain"), - ("allstatic-make", "make", "text/plain"), + "samba-static" : [ + ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"), + # build with all modules static + ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL", "text/plain"), + ("allstatic-make", "make -j", "text/plain"), # retry without any required modules ("none-distclean", "make distclean", "text/plain"), - ("none-configure", samba_libs_configure_samba + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"), - ("none-make", "make", "text/plain"), + ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT", "text/plain"), + ("none-make", "make -j", "text/plain"), # retry with nonshared smbd and smbtorture ("nonshared-distclean", "make distclean", "text/plain"), - ("nonshared-configure", samba_libs_configure_base + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"), - ("nonshared-make", "make", "text/plain")], + ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd", "text/plain"), + ("nonshared-make", "make -j", "text/plain")], + + "samba-systemkrb5" : [ + ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"), + ("configure", "./configure.developer " + samba_configure_params + " --with-system-mitkrb5 --without-ad-dc", "text/plain"), + ("make", "make -j", "text/plain"), + # we currently cannot run a full make test, a limited list of tests could be run + # via "make test TESTS=sometests" + ("test", "make test FAIL_IMMEDIATELY=1 TESTS='samba3.*ktest'", "text/plain"), + ("install", "make install", "text/plain"), + ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"), + ("clean", "make clean", "text/plain") + ], + + # Test Samba without python still builds. When this test fails + # due to more use of Python, the expectations is that the newly + # failing part of the code should be disabled when + # --disable-python is set (rather than major work being done to + # support this environment). The target here is for vendors + # shipping a minimal smbd. + "samba-nopython" : [ + ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"), + ("configure", "./configure.developer --picky-developer ${PREFIX} --with-profiling-data --disable-python --without-ad-dc", "text/plain"), + ("make", "make -j", "text/plain"), + ("install", "make install", "text/plain"), + ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"), + ("clean", "make clean", "text/plain") + ], + + "ldb" : [ ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"), - ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"), + ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"), ("make", "make", "text/plain"), ("install", "make install", "text/plain"), ("test", "make test", "text/plain"), @@ -150,7 +208,7 @@ tasks = { "tdb" : [ ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"), - ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"), + ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"), ("make", "make", "text/plain"), ("install", "make install", "text/plain"), ("test", "make test", "text/plain"), @@ -160,7 +218,7 @@ tasks = { "talloc" : [ ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"), - ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"), + ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"), ("make", "make", "text/plain"), ("install", "make install", "text/plain"), ("test", "make test", "text/plain"), @@ -180,7 +238,7 @@ tasks = { "tevent" : [ ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"), - ("configure", "./configure --enable-developer -C ${PREFIX}", "text/plain"), + ("configure", "./configure --enable-developer -C ${PREFIX} ${EXTRA_PYTHON}", "text/plain"), ("make", "make", "text/plain"), ("install", "make install", "text/plain"), ("test", "make test", "text/plain"), @@ -204,11 +262,16 @@ tasks = { 'fail' : [ ("fail", 'echo failing && /bin/false', "text/plain") ] } +def do_print(msg): + print "%s" % msg + sys.stdout.flush() + sys.stderr.flush() + def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True): if show is None: show = options.verbose if show: - print("Running: '%s' in '%s'" % (cmd, dir)) + do_print("Running: '%s' in '%s'" % (cmd, dir)) if output: return Popen([cmd], shell=True, stdout=PIPE, cwd=dir).communicate()[0] elif checkfail: @@ -220,7 +283,7 @@ def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True): class builder(object): '''handle build of one directory''' - def __init__(self, name, sequence): + def __init__(self, name, sequence, cp=True): self.name = name self.dir = builddirs[name] @@ -230,34 +293,39 @@ class builder(object): self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag) self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag) if options.verbose: - print("stdout for %s in %s" % (self.name, self.stdout_path)) - print("stderr for %s in %s" % (self.name, self.stderr_path)) + do_print("stdout for %s in %s" % (self.name, self.stdout_path)) + do_print("stderr for %s in %s" % (self.name, self.stderr_path)) run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path)) self.stdout = open(self.stdout_path, 'w') self.stderr = open(self.stderr_path, 'w') self.stdin = open("/dev/null", 'r') self.sdir = "%s/%s" % (testbase, self.tag) - self.prefix = "%s/prefix/%s" % (testbase, self.tag) + self.prefix = "%s/%s" % (test_prefix, self.tag) run_cmd("rm -rf %s" % self.sdir) - cleanup_list.append(self.sdir) - cleanup_list.append(self.prefix) - os.makedirs(self.sdir) - run_cmd("rm -rf %s" % self.sdir) - run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True) + run_cmd("rm -rf %s" % self.prefix) + if cp: + run_cmd("cp --recursive --link --archive %s %s" % (test_master, self.sdir), dir=test_master, show=True) + else: + run_cmd("git clone --recursive --shared %s %s" % (test_master, self.sdir), dir=test_master, show=True) self.start_next() def start_next(self): if self.next == len(self.sequence): - print '%s: Completed OK' % self.name + if not options.nocleanup: + run_cmd("rm -rf %s" % self.sdir) + run_cmd("rm -rf %s" % self.prefix) + do_print('%s: Completed OK' % self.name) self.done = True return (self.stage, self.cmd, self.output_mime_type) = self.sequence[self.next] self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(standard_lib=1, prefix=self.prefix)) self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix) + self.cmd = self.cmd.replace("${EXTRA_PYTHON}", "%s" % extra_python) self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix) + self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests) # if self.output_mime_type == "text/x-subunit": # self.cmd += " | %s --immediate" % (os.path.join(os.path.dirname(__file__), "selftest/format-subunit")) - print '%s: [%s] Running %s' % (self.name, self.stage, self.cmd) + do_print('%s: [%s] Running %s' % (self.name, self.stage, self.cmd)) cwd = os.getcwd() os.chdir("%s/%s" % (self.sdir, self.dir)) self.proc = Popen(self.cmd, shell=True, @@ -269,20 +337,23 @@ class builder(object): class buildlist(object): '''handle build of multiple directories''' - def __init__(self, tasklist, tasknames, rebase_url, rebase_branch="master"): + def __init__(self, tasknames, rebase_url, rebase_branch="master"): global tasks self.tlist = [] self.tail_proc = None self.retry = None if tasknames == []: - tasknames = defaulttasks + if options.restrict_tests: + tasknames = ["samba-test-only"] + else: + tasknames = defaulttasks else: # If we are only running one test, # do not sleep randomly to wait for it to start os.environ['AUTOBUILD_RANDOM_SLEEP_OVERRIDE'] = '1' for n in tasknames: - b = builder(n, tasks[n]) + b = builder(n, tasks[n], cp=n is not "pidl") self.tlist.append(b) if options.retry: rebase_remote = "rebaseon" @@ -306,7 +377,7 @@ class buildlist(object): ), "test/plain" ) ] - self.retry = builder('retry', retry_task) + self.retry = builder('retry', retry_task, cp=False) self.need_retry = False def kill_kids(self): @@ -352,7 +423,7 @@ class buildlist(object): b = self.wait_one() if options.retry and self.need_retry: self.kill_kids() - print("retry needed") + do_print("retry needed") return (0, None, None, None, "retry") if b is None: break @@ -400,7 +471,9 @@ class buildlist(object): def cleanup(): if options.nocleanup: return - print("Cleaning up ....") + run_cmd("stat %s || true" % test_tmpdir, show=True) + run_cmd("stat %s" % testbase, show=True) + do_print("Cleaning up ....") for d in cleanup_list: run_cmd("rm -rf %s" % d) @@ -447,7 +520,7 @@ def write_pidfile(fname): def rebase_tree(rebase_url, rebase_branch = "master"): rebase_remote = "rebaseon" - print("Rebasing on %s" % rebase_url) + do_print("Rebasing on %s" % rebase_url) run_cmd("git describe HEAD", show=True, dir=test_master) run_cmd("git remote add -t %s %s %s" % (rebase_branch, rebase_remote, rebase_url), @@ -465,7 +538,7 @@ def rebase_tree(rebase_url, rebase_branch = "master"): (rebase_remote, rebase_branch), dir=test_master, output=True) if diff == '': - print("No differences between HEAD and %s/%s - exiting" % + do_print("No differences between HEAD and %s/%s - exiting" % (rebase_remote, rebase_branch)) sys.exit(0) run_cmd("git describe %s/%s" % @@ -478,7 +551,7 @@ def rebase_tree(rebase_url, rebase_branch = "master"): def push_to(push_url, push_branch = "master"): push_remote = "pushto" - print("Pushing to %s" % push_url) + do_print("Pushing to %s" % push_url) if options.mark: run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master) run_cmd("git commit --amend -c HEAD", dir=test_master) @@ -532,6 +605,8 @@ parser.add_option("", "--log-base", help="location where the logs can be found ( default=gitroot, type='str') parser.add_option("", "--attach-logs", help="Attach logs to mails sent on success/failure?", default=False, action="store_true") +parser.add_option("", "--restrict-tests", help="run as make test with this TESTS= regex", + default='') def send_email(subject, text, log_tar): outer = MIMEMultipart() @@ -555,7 +630,7 @@ def send_email(subject, text, log_tar): s.quit() def email_failure(status, failed_task, failed_stage, failed_tag, errstr, - elapsed_time, log_base=None): + elapsed_time, log_base=None, add_log_tail=True): '''send an email to options.email about the failure''' elapsed_minutes = elapsed_time / 60.0 user = os.getenv("USER") @@ -576,6 +651,10 @@ A summary of the autobuild process is here: %s/autobuild.log ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base) + if options.restrict_tests: + text += """ +The build was restricted to tests matching %s\n""" % options.restrict_tests + if failed_task != 'rebase': text += ''' You can see logs of the failed task here: @@ -593,9 +672,26 @@ The top commit for the tree that was built was: ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg) + if add_log_tail: + f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r') + lines = f.readlines() + log_tail = "".join(lines[-50:]) + num_lines = len(lines) + if num_lines < 50: + # Also include stderr (compile failures) if < 50 lines of stdout + f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r') + log_tail += "".join(f.readlines()[-(50-num_lines):]) + + text += ''' +The last 50 lines of log messages: + +%s + ''' % log_tail + f.close() + logs = os.path.join(gitroot, 'logs.tar.gz') - send_email('autobuild failure on %s for task %s during %s' - % (platform.node(), failed_task, failed_stage), + send_email('autobuild[%s] failure on %s for task %s during %s' + % (options.branch, platform.node(), failed_task, failed_stage), text, logs) def email_success(elapsed_time, log_base=None): @@ -610,6 +706,10 @@ Your autobuild on %s has succeeded after %.1f minutes. ''' % (platform.node(), elapsed_time / 60.) + if options.restrict_tests: + text += """ +The build was restricted to tests matching %s\n""" % options.restrict_tests + if options.keeplogs: text += ''' @@ -626,7 +726,7 @@ The top commit for the tree that was built was: ''' % top_commit_msg logs = os.path.join(gitroot, 'logs.tar.gz') - send_email('autobuild sucess on %s ' % platform.node(), + send_email('autobuild[%s] success on %s' % (options.branch, platform.node()), text, logs) @@ -638,6 +738,9 @@ if options.retry: testbase = "%s/b%u" % (options.testbase, os.getpid()) test_master = "%s/master" % testbase +test_prefix = "%s/prefix" % testbase +test_tmpdir = "%s/tmp" % testbase +os.environ['TMPDIR'] = test_tmpdir # get the top commit message, for emails top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True) @@ -650,7 +753,7 @@ cleanup_list.append(testbase) if options.daemon: logfile = os.path.join(testbase, "log") - print "Forking into the background, writing progress to %s" % logfile + do_print("Forking into the background, writing progress to %s" % logfile) daemonize(logfile) write_pidfile(gitroot + "/autobuild.pid") @@ -659,8 +762,15 @@ start_time = time.time() while True: try: - run_cmd("rm -rf %s" % test_master) - cleanup_list.append(test_master) + run_cmd("rm -rf %s" % test_tmpdir, show=True) + os.makedirs(test_tmpdir) + # The waf uninstall code removes empty directories all the way + # up the tree. Creating a file in test_tmpdir stops it from + # being removed. + run_cmd("touch %s" % os.path.join(test_tmpdir, + ".directory-is-not-empty"), show=True) + run_cmd("stat %s" % test_tmpdir, show=True) + run_cmd("stat %s" % testbase, show=True) run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot) except Exception: cleanup() @@ -678,7 +788,7 @@ while True: 'rebase on %s failed' % options.branch, elapsed_time, log_base=options.log_base) sys.exit(1) - blist = buildlist(tasks, args, options.rebase, rebase_branch=options.branch) + blist = buildlist(args, options.rebase, rebase_branch=options.branch) if options.tail: blist.start_tail() (status, failed_task, failed_stage, failed_tag, errstr) = blist.run() @@ -691,27 +801,28 @@ while True: cleanup_list.append(gitroot + "/autobuild.pid") +do_print(errstr) + blist.kill_kids() if options.tail: - print("waiting for tail to flush") + do_print("waiting for tail to flush") time.sleep(1) elapsed_time = time.time() - start_time if status == 0: - print errstr if options.passcmd is not None: - print("Running passcmd: %s" % options.passcmd) + do_print("Running passcmd: %s" % options.passcmd) run_cmd(options.passcmd, dir=test_master) if options.pushto is not None: push_to(options.pushto, push_branch=options.branch) if options.keeplogs or options.attach_logs: blist.tarlogs("logs.tar.gz") - print("Logs in logs.tar.gz") + do_print("Logs in logs.tar.gz") if options.always_email: email_success(elapsed_time, log_base=options.log_base) blist.remove_logs() cleanup() - print(errstr) + do_print(errstr) sys.exit(0) # something failed, gather a tar of the logs @@ -720,8 +831,26 @@ blist.tarlogs("logs.tar.gz") if options.email is not None: email_failure(status, failed_task, failed_stage, failed_tag, errstr, elapsed_time, log_base=options.log_base) +else: + elapsed_minutes = elapsed_time / 60.0 + print ''' + +#################################################################### + +AUTOBUILD FAILURE + +Your autobuild[%s] on %s failed after %.1f minutes +when trying to test %s with the following error: + + %s + +the autobuild has been abandoned. Please fix the error and resubmit. + +#################################################################### + +''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr) cleanup() -print(errstr) -print("Logs in logs.tar.gz") +do_print(errstr) +do_print("Logs in logs.tar.gz") sys.exit(status)