X-Git-Url: http://git.samba.org/samba.git/?p=sfrench%2Fsamba-autobuild%2F.git;a=blobdiff_plain;f=script%2Fautobuild.py;h=e776df8a0bd06549d6b9bdfa7b9df11bee1978a0;hp=6e1de9e7281db8d9fb4890bcc21e3a95686512a5;hb=770f222f333dfe51d253038e706c195623df0c0c;hpb=a68ab0e97e8dedc8298ca6502b1efc6ca2e32d66 diff --git a/script/autobuild.py b/script/autobuild.py index 6e1de9e7281..e776df8a0bd 100755 --- a/script/autobuild.py +++ b/script/autobuild.py @@ -10,17 +10,13 @@ import smtplib from email.mime.text import MIMEText from distutils.sysconfig import get_python_lib -samba_master = os.getenv('SAMBA_MASTER', 'git://git.samba.org/samba.git') -samba_master_ssh = os.getenv('SAMBA_MASTER_SSH', 'git+ssh://git.samba.org/data/git/samba.git') - # This speeds up testing remarkably. os.environ['TDB_NO_FSYNC'] = '1' cleanup_list = [] builddirs = { - "samba3" : "source3", - "samba3-ctdb" : "source3", + "ctdb" : "ctdb", "samba" : ".", "samba-ctdb" : ".", "samba-libs" : ".", @@ -36,33 +32,19 @@ builddirs = { "retry" : "." } -defaulttasks = [ "samba3", "samba3-ctdb", "samba", "samba-ctdb", "samba-libs", "ldb", "tdb", "ntdb", "talloc", "replace", "tevent", "pidl" ] +defaulttasks = [ "ctdb", "samba", "samba-ctdb", "samba-libs", "ldb", "tdb", "ntdb", "talloc", "replace", "tevent", "pidl" ] tasks = { - "samba3" : [ ("autogen", "./autogen.sh", "text/plain"), - ("configure", "./configure.developer ${PREFIX}", "text/plain"), - ("make basics", "make basics", "text/plain"), - # we split 'make -j 4', 'make bin/smbtorture4' and 'make -j 4 everything' - # because it makes it much easier to find errors. - ("make", "make -j 4", "text/plain"), # don't use too many processes - ("make bin/smbtorture4", "make bin/smbtorture4", "text/plain"), - ("make everything", "make -j 4 everything", "text/plain"), - ("install", "make install", "text/plain"), - ("test", "make test FAIL_IMMEDIATELY=1", "text/plain"), - ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"), - ("clean", "make clean", "text/plain") ], - - "samba3-ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"), - ("autogen", "./autogen.sh", "text/plain"), - ("configure", "./configure.developer ${PREFIX} --with-cluster-support --with-ctdb=../ctdb", "text/plain"), - ("make basics", "make basics", "text/plain"), - ("make", "make all", "text/plain"), # don't use too many processes - ("check", "LD_LIBRARY_PATH=./bin ./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"), - ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"), - ("clean", "make clean", "text/plain") ], + "ctdb" : [ ("random-sleep", "../script/random-sleep.sh 60 600", "text/plain"), + ("configure", "./configure ${PREFIX}", "text/plain"), + ("make", "make all", "text/plain"), + ("install", "make install", "text/plain"), + ("test", "make autotest", "text/plain"), + ("check-clean-tree", "../script/clean-source-tree.sh", "text/plain"), + ("clean", "make clean", "text/plain") ], # We have 'test' before 'install' because, 'test' should work without 'install' - "samba" : [ ("configure", "./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab", "text/plain"), + "samba" : [ ("configure", "./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab", "text/plain"), ("make", "make -j", "text/plain"), ("test", "make test FAIL_IMMEDIATELY=1", "text/plain"), ("install", "make install", "text/plain"), @@ -76,16 +58,18 @@ tasks = { ("tdb-make", "cd lib/tdb && make", "text/plain"), ("tdb-install", "cd lib/tdb && make install", "text/plain"), - # install the ctdb headers under the prefix: - ("ctdb-header-install", "cp ./ctdb/include/* ${PREFIX_DIR}/include", "text/plain"), - ("ctdb-header-ls", "ls ${PREFIX_DIR}/include/ctdb.h", "text/plain"), - ("configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --with-ctdb-dir=${PREFIX_DIR} --bundled-libraries=!tdb", "text/plain"), - ("make", "make", "text/plain"), - ("check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"), - ("install", "make install", "text/plain"), + # build samba with cluster support against this ctdb: + ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer --picky-developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb", "text/plain"), + ("samba-make", "make", "text/plain"), + ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT", "text/plain"), + ("samba-install", "make install", "text/plain"), + ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd", "text/plain"), + + # clean up: ("check-clean-tree", "script/clean-source-tree.sh", "text/plain"), - ("clean", "make clean", "text/plain") ], + ("clean", "make clean", "text/plain"), + ("ctdb-clean", "cd ./ctdb && make clean", "text/plain") ], "samba-libs" : [ ("random-sleep", "script/random-sleep.sh 60 600", "text/plain"), @@ -97,6 +81,10 @@ tasks = { ("tdb-make", "cd lib/tdb && make", "text/plain"), ("tdb-install", "cd lib/tdb && make install", "text/plain"), + ("ntdb-configure", "cd lib/ntdb && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"), + ("ntdb-make", "cd lib/ntdb && make", "text/plain"), + ("ntdb-install", "cd lib/ntdb && make install", "text/plain"), + ("tevent-configure", "cd lib/tevent && PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}", "text/plain"), ("tevent-make", "cd lib/tevent && make", "text/plain"), ("tevent-install", "cd lib/tevent && make install", "text/plain"), @@ -105,9 +93,10 @@ tasks = { ("ldb-make", "cd lib/ldb && make", "text/plain"), ("ldb-install", "cd lib/ldb && make install", "text/plain"), - ("configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=!talloc,!tdb,!pytdb,!ldb,!pyldb,!tevent,!pytevent --abi-check --enable-debug -C ${PREFIX}", "text/plain"), + ("configure", "PYTHONPATH=${PYTHON_PREFIX}/site-packages:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=!talloc,!tdb,!pytdb,!ntdb,!pyntdb,!ldb,!pyldb,!tevent,!pytevent --abi-check --enable-debug -C ${PREFIX}", "text/plain"), ("make", "make", "text/plain"), - ("install", "make install", "text/plain")], + ("install", "make install", "text/plain"), + ("dist", "make dist", "text/plain")], "ldb" : [ ("random-sleep", "../../script/random-sleep.sh 60 600", "text/plain"), @@ -417,11 +406,11 @@ def rebase_tree(rebase_url, rebase_branch = "master"): show=True, dir=test_master) run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master) if options.fix_whitespace: - run_cmd("git rebase --whitespace=fix %s/%s" % + run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" % (rebase_remote, rebase_branch), show=True, dir=test_master) else: - run_cmd("git rebase %s/%s" % + run_cmd("git rebase --force-rebase %s/%s" % (rebase_remote, rebase_branch), show=True, dir=test_master) diff = run_cmd("git --no-pager diff HEAD %s/%s" % @@ -439,18 +428,27 @@ def rebase_tree(rebase_url, rebase_branch = "master"): (rebase_remote, rebase_branch), show=True, dir=test_master) -def push_to(url): - print("Pushing to %s" % url) +def push_to(push_url, push_branch = "master"): + push_remote = "pushto" + print("Pushing to %s" % push_url) if options.mark: run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master) run_cmd("git commit --amend -c HEAD", dir=test_master) # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master) - run_cmd("git remote add -t master pushto %s" % url, show=True, dir=test_master) - run_cmd("git push pushto +HEAD:master", show=True, dir=test_master) + run_cmd("git remote add -t %s %s %s" % + (push_branch, push_remote, push_url), + show=True, dir=test_master) + run_cmd("git push %s +HEAD:%s" % + (push_remote, push_branch), + show=True, dir=test_master) def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER')) +gitroot = find_git_root() +if gitroot is None: + raise Exception("Failed to find git root") + parser = OptionParser() parser.add_option("", "--tail", help="show output while running", default=False, action="store_true") parser.add_option("", "--keeplogs", help="keep logs", default=False, action="store_true") @@ -462,12 +460,8 @@ parser.add_option("", "--verbose", help="show all commands as they are run", default=False, action="store_true") parser.add_option("", "--rebase", help="rebase on the given tree before testing", default=None, type='str') -parser.add_option("", "--rebase-master", help="rebase on %s before testing" % samba_master, - default=False, action='store_true') parser.add_option("", "--pushto", help="push to a git url on success", default=None, type='str') -parser.add_option("", "--push-master", help="push to %s on success" % samba_master_ssh, - default=False, action='store_true') parser.add_option("", "--mark", help="add a Tested-By signoff before pushing", default=False, action="store_true") parser.add_option("", "--fix-whitespace", help="fix whitespace on rebase", @@ -480,11 +474,16 @@ parser.add_option("", "--always-email", help="always send email, even on success action="store_true") parser.add_option("", "--daemon", help="daemonize after initial setup", action="store_true") +parser.add_option("", "--branch", help="the branch to work on (default=master)", + default="master", type='str') +parser.add_option("", "--log-base", help="location where the logs can be found (default=cwd)", + default=gitroot, type='str') - -def email_failure(status, failed_task, failed_stage, failed_tag, errstr): +def email_failure(status, failed_task, failed_stage, failed_tag, errstr, log_base=None): '''send an email to options.email about the failure''' user = os.getenv("USER") + if log_base is None: + log_base = gitroot text = ''' Dear Developer, @@ -495,25 +494,25 @@ the autobuild has been abandoned. Please fix the error and resubmit. A summary of the autobuild process is here: - http://git.samba.org/%s/samba-autobuild/autobuild.log -''' % (failed_task, errstr, user) + %s/autobuild.log +''' % (failed_task, errstr, log_base) if failed_task != 'rebase': text += ''' You can see logs of the failed task here: - http://git.samba.org/%s/samba-autobuild/%s.stdout - http://git.samba.org/%s/samba-autobuild/%s.stderr + %s/%s.stdout + %s/%s.stderr or you can get full logs of all tasks in this job here: - http://git.samba.org/%s/samba-autobuild/logs.tar.gz + %s/logs.tar.gz The top commit for the tree that was built was: %s -''' % (user, failed_tag, user, failed_tag, user, top_commit_msg) +''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg) msg = MIMEText(text) msg['Subject'] = 'autobuild failure for task %s during %s' % (failed_task, failed_stage) msg['From'] = 'autobuild@samba.org' @@ -524,9 +523,11 @@ The top commit for the tree that was built was: s.sendmail(msg['From'], [msg['To']], msg.as_string()) s.quit() -def email_success(): +def email_success(log_base=None): '''send an email to options.email about a successful build''' user = os.getenv("USER") + if log_base is None: + log_base = gitroot text = ''' Dear Developer, @@ -539,9 +540,9 @@ Your autobuild has succeeded. you can get full logs of all tasks in this job here: - http://git.samba.org/%s/samba-autobuild/logs.tar.gz + %s/logs.tar.gz -''' % user +''' % log_base text += ''' The top commit for the tree that was built was: @@ -563,16 +564,12 @@ The top commit for the tree that was built was: (options, args) = parser.parse_args() if options.retry: - if not options.rebase_master and options.rebase is None: + if options.rebase is None: raise Exception('You can only use --retry if you also rebase') testbase = "%s/b%u" % (options.testbase, os.getpid()) test_master = "%s/master" % testbase -gitroot = find_git_root() -if gitroot is None: - raise Exception("Failed to find git root") - # get the top commit message, for emails top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True) @@ -601,19 +598,15 @@ while True: try: try: if options.rebase is not None: - rebase_url = options.rebase - elif options.rebase_master: - rebase_url = samba_master - else: - rebase_url = None - if rebase_url is not None: - rebase_tree(rebase_url) + rebase_tree(options.rebase, rebase_branch=options.branch) except Exception: cleanup_list.append(gitroot + "/autobuild.pid") cleanup() - email_failure(-1, 'rebase', 'rebase', 'rebase', 'rebase on master failed') + email_failure(-1, 'rebase', 'rebase', 'rebase', + 'rebase on %s failed' % options.branch, + log_base=options.log_base) sys.exit(1) - blist = buildlist(tasks, args, rebase_url) + blist = buildlist(tasks, args, options.rebase, rebase_branch=options.branch) if options.tail: blist.start_tail() (status, failed_task, failed_stage, failed_tag, errstr) = blist.run() @@ -637,14 +630,12 @@ if status == 0: print("Running passcmd: %s" % options.passcmd) run_cmd(options.passcmd, dir=test_master) if options.pushto is not None: - push_to(options.pushto) - elif options.push_master: - push_to(samba_master_ssh) + push_to(options.pushto, push_branch=options.branch) if options.keeplogs: blist.tarlogs("logs.tar.gz") print("Logs in logs.tar.gz") if options.always_email: - email_success() + email_success(log_base=options.log_base) blist.remove_logs() cleanup() print(errstr) @@ -654,7 +645,7 @@ if status == 0: blist.tarlogs("logs.tar.gz") if options.email is not None: - email_failure(status, failed_task, failed_stage, failed_tag, errstr) + email_failure(status, failed_task, failed_stage, failed_tag, errstr, log_base=options.log_base) cleanup() print(errstr)