-# -*- mode: shell-script; sh-indentation: 8; indent-tabs-mode: t; -*-
+#!/bin/sh -*- mode: shell-script; -*-
# build_farm -- distributed build/test architecture for samba, rsync, etc
# Copyright (C) 2001, 2003 by Martin Pool <mbp@samba.org>
# default maximum runtime for any command
-MAXTIME=7200
+MAXTIME=25200 # 7 hours
+SMBD_MAXTIME=18000 # 5 hours for a samba process ..
+# default maximum memory size (100M) for any command
+MAXMEM=100000
RUN_FROM_BUILD_FARM=yes
export RUN_FROM_BUILD_FARM
+export MAXTIME SMBD_MAXTIME
+
+deptrees="";
+
+build_test_fns_id='$Id$'
+
+copy_dir() {
+ Tsrc=$1
+ Tdst=$2
+ pwd
+ echo rsync -a --delete $Tsrc/ $Tdst
+ rsync -a --delete $Tsrc/ $Tdst || return 1
+ return 0
+}
#############################
# build a signature of a tree, used to see if we
# need to rebuild
+#############################
+
sum_tree() {
- sum_tree_test_root=$1
+ sum_tree_test_root=$1
sum_tree_tree=$2
sum_tree_sum=$3
- find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v '.svn' | grep -v version.h | sort | xargs sum > $sum_tree_sum
+ sum_tree_scm=$4
+ find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v version.h | sort | xargs sum > $sum_tree_sum
sum build_test build_test.fns >> $sum_tree_sum
if [ -f "$host.fns" ]; then
- sum $host.fns >> $sum_tree_sum
+ sum $host.fns >> $sum_tree_sum
else
- sum generic.fns >> $sum_tree_sum
+ sum generic.fns >> $sum_tree_sum
fi
+
+ if [ -f "$test_root/$tree.$scm" ]; then
+ sum "$test_root/$tree.$scm" >> $sum_tree_sum
+ fi
+
+ for d in $deptrees; do
+ dscm=`choose_scm "$d"`
+ if [ -f "$test_root/$d.$dscm" ]; then
+ sum "$test_root/$d.$dscm" >> $sum_tree_sum
+ fi
+ done
}
#############################
# send the logs to the master site
+#############################
+
send_logs() {
if [ "$nologreturn" = "yes" ]; then
echo "skipping log transfer"
else
log="$1"
err="$2"
+ shift
+ shift
chmod 0644 "$log" "$err"
- rsync -q --password-file=.password -cz --timeout=200 \
- "$log" "$err" $host@samba.org::build_farm_data/
+
+ # xargs -i is implemented differently or not at all.
+ # GNU xargs did not implement "-I" until 4.2.9:
+ xargs --version 2>&1 | grep "^GNU xargs" > /dev/null
+ status=$?
+ if [ x"$status" = x"0" ]; then
+ XARGS_IS_GNU=yes
+ fi
+
+ if [ x"$XARGS_IS_GNU" = x"yes" ]; then
+ XARGS_I="xargs -i"
+ else
+ XARGS_I="xargs -I '{}'"
+ fi
+
+ find $log -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} && echo "\n***LOG TRUNCATED***" >> {}'
+ find $err -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} && echo "\n***LOG TRUNCATED***" >> {}'
+
+ rsync $* -c -q --password-file=.password -z --timeout=200 \
+ "$log" "$err" $host@build.samba.org::build_farm_data/
fi
}
+#############################
+# send the logs when they haven't changed
+# the aim is to just update the servers timestamp.
+# sending with a very large rsync block size does this
+# with minimal network traffic
+#############################
+
+send_logs_skip() {
+ touch "$1" "$2"
+ send_logs "$1" "$2" -B 10000000
+}
+
############################
# fetch the latest copy of the tree
+############################
+
fetch_tree() {
if [ "$norsync" = "yes" ]; then
echo "skipping tree transfer"
else
- tree=$1
- if rsync --exclude=autom4te.cache/ --exclude=.svn/ --delete-excluded -q --partial --timeout=200 -crlpz --delete --ignore-errors \
- samba.org::ftp/unpacked/$tree/ $test_root/$tree; then
- echo "transferred $tree OK"
+ fetchtree=$1
+ if rsync --exclude=autom4te.cache/ --exclude=.svn/ --exclude=.git/ \
+ --delete-excluded -q --partial --timeout=200 -ctrlpz --delete --ignore-errors \
+ samba.org::ftp/unpacked/$fetchtree/ $test_root/$fetchtree; then
+ echo "transferred $fetchtree OK"
else
- echo "transfer of $tree failed code $?"
+ echo "transfer of $fetchtree failed code $?"
return 1
fi
fi
}
############################
-# fetch the latest copy of the svn entries file
-fetch_svn() {
- tree=$1
-# skip products still in CVS.
- case "$tree" in
- ccache | distcc | rsync)
- return 1
- ;;
- *)
- ;;
- esac
- if [ "$norsync" = "yes" ]; then
- echo "skipping svn transfer"
- else
- if [ -r $test_root/$tree.svn ]; then
- rm -f $test_root/$tree.svn.old
- mv $test_root/$tree.svn $test_root/$tree.svn.old
+# fetch the latest copy of the rev meta info
+############################
+
+fetch_revinfo() {
+ tree=$1
+ scm=$2
+
+ test -z "$scm" && return 1
+ test x"$scm" = x"unknown" && return 1
+ test x"$scm" = x"cvs" && return 1
+
+ if [ "$norsync" = "yes" ]; then
+ echo "skipping .revinfo.$scm transfer"
+ else
+ if [ -r $test_root/$tree.$scm ]; then
+ [ -f $test_root/$tree.$scm.old ] && rm -f $test_root/$tree.$scm.old
+ [ -f $test_root/$tree.$scm ] && mv $test_root/$tree.$scm $test_root/$tree.$scm.old
+ fi
+ rsync -q --timeout=200 -clz --ignore-errors \
+ samba.org::ftp/unpacked/$tree/.revinfo.$scm $test_root/$tree.$scm
+ fi
+ if [ -r $test_root/$tree.$scm ]; then
+ return 0;
fi
- rsync -q --timeout=200 -clz --ignore-errors \
- samba.org::ftp/unpacked/$tree/.svn/entries $test_root/$tree.svn
- fi
- if [ -r $test_root/$tree.svn ]; then
- return 0;
- fi
- return 1
+ return 1
+}
+
+############################
+# choose the scm that is used for the given project
+############################
+
+choose_scm() {
+ tree=$1
+
+ case "$tree" in
+ samba* | rsync | libreplace | talloc | tdb* | ldb | pidl | ccache* | waf*)
+ echo "git"
+ return 0
+ ;;
+ esac
+
+ echo "svn"
+ return 0
}
+locknesting=0
+
############################
# grab a lock file. Not atomic, but close :)
# tries to cope with NFS
+############################
+
lock_file() {
- lck="$1"
- machine=`cat "$lck" 2> /dev/null | cut -d: -f1`
- pid=`cat "$lck" 2> /dev/null | cut -d: -f2`
+ if [ -z "$lock_root" ]; then
+ lock_root=`pwd`;
+ fi
- if [ -f "$lck" ] &&
- ( [ $machine != $host ] || kill -0 $pid ) 2> /dev/null; then
- echo "lock file $lck is valid for $machine:$pid"
- return 1
- fi
- /bin/rm -f "$lck"
- echo "$host:$$" > "$lck"
- return 0
+ lckf="$lock_root/$1"
+ machine=`cat "$lckf" 2> /dev/null | cut -d: -f1`
+ pid=`cat "$lckf" 2> /dev/null | cut -d: -f2`
+
+ if [ "$pid" = "$$" ]; then
+ locknesting=`expr $locknesting + 1`
+ echo "lock nesting now $locknesting"
+ return 0
+ fi
+
+ # We need to assert that the file is > 0 size, as otherwise we never
+ # recover from disk full situations
+ if test -f "$lckf" && test -s "$lckf"; then
+ test x$machine = x$host || {
+ echo "lock file $lckf is valid for other machine $machine"
+ return 1
+ }
+
+ kill -0 $pid && {
+ echo "lock file $lckf is valid for process $pid"
+ return 1
+ }
+
+ echo "stale lock file $lckf for $machine:$pid"
+ cat "$lckf"
+ /bin/rm -f "$lckf"
+ fi
+ echo "$host:$$" > "$lckf"
+ return 0
}
############################
# unlock a lock file
+############################
+
unlock_file() {
- lck="$1"
- /bin/rm -f "$lck"
+ if [ -z "$lock_root" ]; then
+ lock_root=`pwd`;
+ fi
+ if [ "$locknesting" != "0" ]; then
+ locknesting=`expr $locknesting - 1`
+ echo "lock nesting now $locknesting"
+ else
+ lckf="$lock_root/$1"
+ /bin/rm -f "$lckf"
+ fi
}
############################
# run make, and print trace
+############################
+
do_make() {
+ # work out correct make command
+ case "$tree" in
+ waf*)
+ MAKECOMMAND="./waf"
+ ;;
+ *)
+ MAKECOMMAND="$MAKE"
+ if [ x"$MAKECOMMAND" = x ]; then
+ MAKECOMMAND=make
+ fi
+ ;;
+ esac
+
+ MMTIME=$MAXTIME
+ # some trees don't need as much time
+ case "$tree" in
+ rsync | tdb* | talloc | libreplace | ccache* | waf*)
+ if [ "$compiler" != "checker" ]; then
+ MMTIME=`expr $MMTIME / 5`
+ fi
+ ;;
+ esac
+
+ # special build for some trees
+ case "$tree" in
+ waf*)
+ ./waf distclean && ./waf configure build
+ ;;
+ esac
+
+
+ for t in $*; do
+ if [ x"$BUILD_FARM_NUM_JOBS" = x ]; then
+ echo "$MAKECOMMAND $t"
+ $builddir/timelimit $MMTIME "$MAKECOMMAND" "$t"
+ status=$?
+ else
+ # we can parallelize everything and all targets
+ if [ x"$t" = xeverything ] || [ x"$t" = xall]; then
+ echo "$MAKECOMMAND" "-j$BUILD_FARM_NUM_JOBS" "$t"
+ $builddir/timelimit $MMTIME "$MAKECOMMAND" "-j$BUILD_FARM_NUM_JOBS" "$t"
+ status=$?
+ else
+ echo "$MAKECOMMAND $t"
+ $builddir/timelimit $MMTIME "$MAKECOMMAND" "$t"
+ status=$?
+ fi
+ fi
+
+ if [ $status != 0 ]; then
+ case "$t" in
+ test|check|installcheck)
+ ;;
+ *)
+ #run again with V=1, so we see failed commands
+ $builddir/timelimit $MMTIME "$MAKECOMMAND" "$t" V=1
+ status=$?
+ ;;
+ esac
+ fi
+
+ if [ $status != 0 ]; then
+ return $status;
+ fi
+
+ done
+
+ return 0
+}
+
+
+############################
+# do the coverage report
+############################
+
+action_lcovreport() {
+ if [ "$LCOV_REPORT" = "yes" ]; then
+ case "$tree" in
+ lorikeet-heimdal*)
+ lcov --directory $builddir --capture --output-file $builddir/$tree.lcov.info
+ ;;
+ samba_3_master*)
+ lcov --base-directory $builddir --directory $builddir/.. --capture --output-file $builddir/$tree.lcov.info
+ ;;
+ samba_4*|tdb*|talloc|ldb|libreplace)
+ lcov --base-directory $builddir/bin --directory $builddir/bin --capture --output-file $builddir/$tree.lcov.info
+ ;;
+ waf)
+ lcov --base-directory $builddir/demos --directory $builddir/demos --capture --output-file $builddir/$tree.lcov.info
+ ;;
+ *)
+ lcov --base-directory $builddir --directory $builddir --capture --output-file $builddir/$tree.lcov.info
+ ;;
+ esac
+ genhtml -o $builddir/coverage $builddir/$tree.lcov.info
+ rc=$?
+ echo "return code: $rc"
+ else
+ echo "LCOV_REPORT not set and lcovreport asked"
+ echo "Most probably an error please fix !"
+ return 1
+ fi
+}
+
+action_callcatcherreport() {
+ if [ "$CALLCATCHER_REPORT" = "yes" ]; then
+ case "$tree" in
+ tdb*|talloc|ldb)
+ callanalyse `find $builddir/bin -name \*.so*` $builddir/bin/* > $builddir/coverage/unused-fns.txt
+ ;;
+ samba_3_master|samba_4*)
+ callanalyse `find $builddir/bin -name \*.so*` $builddir/bin/* > $builddir/coverage/all-unused-fns.txt
+ grep -v -f $srcdir/callcatcher-exceptions.grep $builddir/coverage/all-unused-fns.txt > $builddir/coverage/unused-fns.txt
+ ;;
+ esac
+ rc=$?
+ echo "return code: $rc"
+ else
+ echo "CALLCATCHER_REPORT not set and callcatcher asked"
+ echo "Most probably an error please fix !"
+ return 1
+ fi
+}
- if [ x"$MAKE" = x ]
- then
- MAKE=make
- fi
-
- for t in $*; do
- if [ x"$BUILD_FARM_NUM_JOBS" = x ]; then
- echo "$MAKE $t"
- ./timelimit $MAXTIME "$MAKE" "$t"
- status=$?
- else
- # we can parallelize everything and all targets
- if [ x"$t" = xeverything ] || [ x"$t" = xall]; then
- echo "$MAKE" "-j$BUILD_FARM_NUM_JOBS" "$t"
- ./timelimit $MAXTIME "$MAKE" "-j$BUILD_FARM_NUM_JOBS" "$t"
- status=$?
- else
- echo "$MAKE $t"
- ./timelimit $MAXTIME "$MAKE" "$t"
- status=$?
- fi
- fi
-
- if [ $status != 0 ]; then
- return $status;
- fi
-
- done
-
- return 0
-}
############################
# configure the tree
+############################
+
action_configure() {
- if [ ! -x configure ]; then
- ls -l configure
- ./autogen.sh
+ # special handling for some trees
+ case "$tree" in
+ waf*)
+ $builddir/timelimit $MAXTIME ./waf configure
+ cstatus=$?
+ echo "CONFIGURE STATUS: $cstatus"
+ return $cstatus
+ ;;
+ esac
+
+ if [ ! -x $srcdir/configure -a -r $srcdir/Makefile.PL ]; then
+ perl $srcdir/Makefile.PL PREFIX="$prefix"
+ cstatus=$?
+ echo "CONFIGURE STATUS: $cstatus"
+ return $cstatus;
+ fi
+
+ if [ ! -x $srcdir/configure ]; then
+ ls -l $srcdir/configure
+ echo "$srcdir/configure is missing"
+ cstatus=255
+ echo "CONFIGURE STATUS: $cstatus"
+ return $cstatus;
fi
+
echo "CFLAGS=$CFLAGS"
echo configure options: $config_and_prefix
echo CC="$CCACHE $compiler" $srcdir/configure $config_and_prefix
+
CC="$CCACHE $compiler"
export CC
- ./timelimit $MAXTIME $srcdir/configure $config_and_prefix
+ $builddir/timelimit $MAXTIME $srcdir/configure $config_and_prefix
cstatus=$?
- echo "CONFIGURE STATUS: $cstatus"
- if [ $cstatus != 0 ]; then
- echo "contents of config.log:"
- cat config.log
+
+ if [ x"$cstatus" != x"0" ]; then
+ if [ -f config.log ]; then
+ echo "contents of config.log:"
+ cat config.log
+ fi
+
+ # Waf style
+ if [ -f bin/config.log ]; then
+ echo "contents of config.log:"
+ cat bin/config.log
+ fi
fi
- return $cstatus;
+ echo "CONFIGURE STATUS: $cstatus"
+ return $cstatus
+}
+
+############################
+# show the configure log
+############################
+
+action_config_log() {
+
+ log_files="config.log bin/config.log"
+ for f in $log_files; do
+ if [ -f $f ]; then
+ echo "contents of config.log:"
+ cat $f
+ return 0
+ fi
+ done
+ return 0
+}
+
+############################
+# show the config.h
+############################
+
+action_config_header() {
+ hdr_files="config.h include/config.h include/autoconf/config.h bin/default/config.h bin/default/include/config.h bin/default/source3/include/config.h"
+ for h in $hdr_files; do
+ if [ -f $h ]; then
+ echo "contents of $h:"
+ cat $h
+ return 0
+ fi
+ done
+
+ return 0
}
+
+
############################
# build the tree
+############################
action_build() {
case "$tree" in
- samba4)
- # the 2nd 'make everything' is to work around a bug
- # in netbsd make.
- do_make proto showflags everything
+ samba_4*)
do_make everything
+ bstatus=$?
+ ;;
+ samba_3*)
+ do_make everything torture
+ bstatus=$?
+ ;;
+ waf*)
+ do_make build
+ bstatus=$?
;;
- samba|samba_3_0)
- do_make proto everything torture
- ;;
*)
- do_make all
- ;;
+ do_make all
+ bstatus=$?
+ ;;
esac
- bstatus=$?
echo "BUILD STATUS: $bstatus"
- if [ "$tree" = "samba4" ] && [ -n "$smbtorture4" ] && [ -f bin/smbtorture ]; then
- rm -f $smbtorture4
- cp bin/smbtorture $smbtorture4
- fi
return $bstatus
}
############################
# show static analysis results
+############################
+
action_cc_checker() {
# default to passing the cc_checker
- cccstatus=0
+ cccstatus=0
if [ -f ibm_checker.out ]; then
cat ibm_checker.out
############################
# install the tree
+############################
+
action_install() {
if [ -d $prefix ]; then
if [ "$noclean" != "yes" ]; then
- rm -rf $prefix
+ rm -rf $prefix
fi
fi
action_test_samba() {
do_make test
totalstatus=$?
+
+ # if we produced a test summary then show it
+ [ -f st/summary ] && {
+ echo "TEST SUMMARY"
+ cat st/summary
+ }
+
return "$totalstatus"
}
action_test_generic() {
- CC="$compiler"
+ CC="$compiler"
export CC
do_make installcheck
totalstatus=$?
}
action_test_lorikeet_heimdal() {
- CC="$compiler"
+ CC="$compiler"
export CC
+ SOCKET_WRAPPER_DIR=`pwd`/sw
+ mkdir $SOCKET_WRAPPER_DIR
+ export SOCKET_WRAPPER_DIR
do_make check
totalstatus=$?
+ SOCKET_WRAPPER_DIR=
+ export SOCKET_WRAPPER_DIR
echo "TEST STATUS: $totalstatus"
return "$totalstatus"
}
#############################
# attempt some basic tests of functionaility
# starting as basic as possible, and getting incresingly complex
+#############################
action_test() {
# Samba needs crufty code of its own for backward
# compatiblity. I think a better way to do this in the future
# is to just call 'make installcheck'.
case "$tree" in
- samba*|smb-build)
- action_test_samba
- ;;
+ samba*|smb-build|pidl)
+ action_test_samba
+ ;;
lorikeet-heimdal*)
- action_test_lorikeet_heimdal
- ;;
+ action_test_lorikeet_heimdal
+ ;;
*)
- action_test_generic
- ;;
+ action_test_generic
+ ;;
esac
}
###########################
# do a test build of a particular tree
+# This is the master function called by generic.fns or
+# host.fns
+###########################
+
test_tree() {
tree=$1
source=$2
shift
shift
shift
- if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && ccache -V > /dev/null; then
- CCACHE="ccache"
- export CCACHE
+ echo "Starting to deal with tree $tree with compiler $compiler"
+ if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && [ "$tree" != "ccache-maint" ] && ccache -V > /dev/null 2>/dev/null; then
+ CCACHE="ccache"
+ export CCACHE
else
- CCACHE=""
+ CCACHE=""
fi
# limit our resource usage
- ulimit -t $MAXTIME
- ulimit -m 100000
+ ulimit -t $MAXTIME 2> /dev/null
+
+ # max mem size 100M
+ ulimit -m $MAXMEM 2> /dev/null
+
+ # max file size 100M
+ # darn, this affects sparse files too! disable it
+ # ulimit -f 100000 2> /dev/null
+
+ # try and limit the number of open files to 250. That means we'll discover
+ # fd leaks faster
+ ulimit -n 250 2> /dev/null
# Keep stuff private
umask 077
log="build.$tree.$host.$compiler.log"
err="build.$tree.$host.$compiler.err"
sum="build.$tree.$host.$compiler.sum"
- lck="$test_root/build.$tree.lck"
- srcdir="$test_root/$tree/$source"
+ lck="build.$tree.lck"
+ srcdir="$test_root/$tree/$source"
+ lock_file "$lck" || {
+ return
+ }
+
+ # work out what other trees this package depends on
+ deptrees=""
case "$tree" in
- tdb | talloc | ldb)
- builddir=$srcdir/tmpbuild
- export builddir
- ;;
- *)
- builddir=$srcdir
- export builddir
- ;;
+ samba-gtk)
+ deptrees="samba_4_0_test"
+ ;;
esac
- if ! lock_file "$lck"; then
- return
+ scm=`choose_scm "$tree"`
+
+ # pull the entries, if any
+ # Remove old .svn or .git files
+ # Move the current .svn org .git to .svn.old or
+ # .git.old then fetch the new from rsync
+ if fetch_revinfo "$tree" "$scm"; then
+ for d in $deptrees; do
+ # If there is dependency substree(s) we add info
+ # from the dependency tree so that we
+ # can rebuild in case one of them has changed
+ dscm=`choose_scm "$d"`
+ if [ -f "$test_root/$d.$dscm" ]; then
+ if [ "$d" != "$tree" ]; then
+ cat "$test_root/$d.$dscm" >> $test_root/$tree.$scm
+ fi
+ fi
+ done
+ [ -f $test_root/$tree.$compiler.$scm.old ] && rm -f $test_root/$tree.$compiler.$scm.old
+ [ -f $test_root/$tree.$compiler.$scm ] && mv $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old
+ [ -f $test_root/$tree.$scm ] && cp $test_root/$tree.$scm $test_root/$tree.$compiler.$scm
+
+ if [ -f $test_root/$tree.$compiler.$scm.old ] && \
+ cmp $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old > /dev/null; then
+
+ echo "skip: $tree.$compiler nothing changed in $scm"
+ cd $test_root
+ send_logs_skip "$log" "$err"
+ unlock_file "$lck"
+ return
+ fi
fi
- # pull the svn entries, if any
- if fetch_svn "$tree"; then
- rm -f $test_root/$tree.$compiler.svn.old
- mv $test_root/$tree.$compiler.svn $test_root/$tree.$compiler.svn.old
- cp $test_root/$tree.svn $test_root/$tree.$compiler.svn
- if cmp $test_root/$tree.$compiler.svn $test_root/$tree.$compiler.svn.old > /dev/null; then
- echo "skip: $tree.$compiler nothing changed in svn"
- touch "$log" "$err"
- send_logs "$log" "$err"
+ # pull the tree
+ fetch_tree "$tree" || {
+ cd $test_root
unlock_file "$lck"
return
- fi
- fi
+ }
- # pull the tree
- if ! fetch_tree "$tree"; then
- unlock_file "$lck"
- return
- fi
+ # check for essential files
+ case "$tree" in
+ pidl)
+ # no generated files
+ ;;
+ waf*)
+ if [ ! -x $srcdir/waf ]; then
+ echo "skip: $tree.$compiler waf not present, try again next time!"
+ cd $test_root
+ unlock_file "$lck"
+ return
+ fi
+ ;;
+ *)
+ if [ ! -x $srcdir/configure ]; then
+ echo "skip: $tree.$compiler configure not present, try again next time!"
+ cd $test_root
+ unlock_file "$lck"
+ return
+ fi
+ ;;
+ esac
- if [ ! -x $srcdir/configure ]; then
- echo "skip: $tree.$compiler configure not present, try again next time!"
- unlock_file "$lck"
- return
- fi
+ echo "Starting build of $tree.$compiler in process $$ at `date`"
+
+
+ # Parameters for the build depending on the tree
+ case "$tree" in
+ *)
+ builddir=$srcdir
+ export builddir
+ ;;
+ esac
- if [ ! $USER = "" ]; then
- whoami=$USER
+ #Fix the user
+ if [ ! x$USER = x"" ]; then
+ whoami=$USER
else
- if [ ! $LOGNAME = "" ]; then
- whoami=$LOGNAME
- else
- whoami=build
- fi
+ if [ ! x$LOGNAME = x"" ]; then
+ whoami=$LOGNAME
+ else
+ whoami=build
+ fi
fi
- prefix="$test_root/prefix"
- if [ ! -d $prefix ]; then
- mkdir $prefix
- fi
- prefix="$prefix/$tree"
+ # build the timelimit utility
+ echo "Building timelimit"
+ mkdir -p $builddir
+ echo $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c
+ $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1
+
+ # build the killbysubdir utility
+ echo "Building killbysubdir"
+ echo $compiler -o $builddir/killbysubdir $test_root/killbysubdir.c
+ $compiler -o $builddir/killbysubdir $test_root/killbysubdir.c
+
+ prefix="$test_root/prefix/$tree.$compiler"
+ mkdir -p "$prefix"
- smbtorture4=$test_root/smbtorture4
- export smbtorture4
+ # This can be defined in <host>.fns files
+ sw_config=$config
case "$tree" in
- samba4)
- config="$config --enable-socket-wrapper"
- ;;
- samba|samba_3_0)
- config="$config --enable-socket-wrapper"
- if [ -f $smbtorture4 ]; then
- # we create a local copy to make sure the same binary is used for all tests
- cp $smbtorture4 $smbtorture4.$tree
- config="$config --with-smbtorture4-path=$smbtorture4.$tree"
- fi
- ;;
- ldb)
- # try to avoid having extra copies of popt lying around, or
- # depending on libpopt-dev on every system
- config="$config --with-popt-src=../../popt"
- fetch_tree popt
- ;;
+ lorikeet-heimdal)
+ sw_config="$config --enable-socket-wrapper"
+ ;;
+ samba_4*)
+ sw_config="$config --enable-socket-wrapper"
+ sw_config="$sw_config --enable-nss-wrapper"
+ sw_config="$sw_config --enable-uid-wrapper"
+ ;;
+ samba_3*)
+ sw_config="$config --enable-socket-wrapper"
+ sw_config="$sw_config --enable-nss-wrapper"
+ ;;
+ samba-gtk)
+ PKG_CONFIG_PATH="$test_root/prefix/samba_4_0_test.$compiler/lib/pkgconfig"
+ export PKG_CONFIG_PATH
+ ;;
*)
- testsuite=testsuite
- ;;
+ testsuite=testsuite
+ ;;
esac
- config_and_prefix="$config --prefix=$prefix"
+ if [ "$LCOV_REPORT" = "yes" ]; then
+ PRE_GCOV_CFLAGS=$CFLAGS
+ PRE_GCOV_LDFLAGS=$LDFLAGS
+ GCOV_FLAGS="--coverage"
+ CFLAGS="$CFLAGS $GCOV_FLAGS"
+ LDFLAGS="$LDFLAGS $GCOV_FLAGS"
+ export CFLAGS LDFLAGS
+ fi
+
+ config_and_prefix="$sw_config --prefix=$prefix"
# see if we need to rebuild
- sum_tree $test_root $tree $sum
+ sum_tree $test_root $tree $sum $scm
echo "CFLAGS=$CFLAGS $config_and_prefix" >> $sum
- if cmp "$sum" "$sum.old" > /dev/null; then
+ if [ -f "$sum.old" ] && cmp "$sum" "$sum.old" > /dev/null; then
echo "skip: $tree.$compiler nothing changed"
- touch "$log" "$err"
- send_logs "$log" "$err"
+ cd $test_root
+ send_logs_skip "$log" "$err"
unlock_file "$lck"
+ echo "Ending build of $tree.$compiler in process $$ at `date`"
+ if [ "$LCOV_REPORT" = "yes" ]; then
+ CFLAGS=$PRE_GCOV_CFLAGS
+ LDFLAGS=$PRE_GCOV_LDFLAGS
+ export CFLAGS LDFLAGS
+ fi
return
fi
# we do need to rebuild - save the old sum
- /bin/rm -f $sum.old
+ [ -f $sum.old ] && /bin/rm -f $sum.old
mv $sum $sum.old
+ #Action == what to do ie. configure config_log ...
actions="$*"
-
+
if [ "$actions" = "" ]; then
- actions="configure build install test"
+ actions="configure config_log config_header build install test $EXTRA_ACTIONS"
fi
# start the build
(
+ {
+ # we all want to be able to read the output...
+ LANG=C
+ export LANG
+
uname -a
- # we need to be able to see if a build farm machine is accumulating
- # stuck processes. We do this in two ways, as we don't know what style
- # of ps it will have
+ echo ""
+ echo "build_test : $build_test_id"
+ echo "build_test.fns : $build_test_fns_id"
+ echo "local settings file : $build_test_settings_local_file"
+ echo "local functions file: $build_test_fns_local_file"
+ echo "used .fns file : $build_test_used_fns_file"
+ echo ""
+
+ # we need to be able to see if a build farm machine is accumulating
+ # stuck processes. We do this in two ways, as we don't know what style
+ # of ps it will have
ps xfuw 2> /dev/null
ps -fu $USER 2> /dev/null
echo "building $tree with CC=$compiler on $host at "`date`
-
- # build the timelimit utility
- echo "Building timelimit"
- mkdir -p $builddir
- $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1
-
- if [ -r $test_root/$tree.svn ]; then
- h_rev=`grep revision= $test_root/$tree.svn | head -1 | cut -d'"' -f2`
- if [ -n "$h_rev" ]; then
- echo "HIGHEST SVN REVISION: $h_rev"
- fi
- rev=`grep committed-rev= $test_root/$tree.svn | head -1 | cut -d'"' -f2`
- if [ -n "$rev" ]; then
+ echo "builddir=$builddir"
+ echo "prefix=$prefix"
+
+ echo "Showing limits"
+ ulimit -a 2> /dev/null
+
+ # the following is for non-samba builds only
+ if [ "$scm" = "svn" -a -r $test_root/$tree.svn ]; then
+ h_rev=`grep 'Revision: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
+ if [ -n "$h_rev" ]; then
+ echo "HIGHEST SVN REVISION: $h_rev"
+ fi
+ rev=`grep 'Last Changed Rev: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
+ if [ -n "$rev" ]; then
echo "BUILD REVISION: $rev"
+ fi
+ elif [ "$scm" = "git" -a -r $test_root/$tree.git ]; then
+ csha1=`cat $test_root/$tree.git |head -3 | tail -1`
+ if [ -n "$csha1" ]; then
+ echo "BUILD COMMIT REVISION: $csha1"
+ fi
+ cdate=`cat $test_root/$tree.git |head -4 | tail -1`
+ if [ -n "$cdate" ]; then
+ echo "BUILD COMMIT DATE: $cdate"
+ fi
+ ctime=`cat $test_root/$tree.git |head -2 | tail -1`
+ if [ -n "$ctime" ]; then
+ echo "BUILD COMMIT TIME: $ctime"
fi
fi
- for action in $actions; do
+ if [ -x $builddir/killbysubdir ]; then
+ echo "$builddir/killbysubdir $builddir in `pwd`"
+ $builddir/killbysubdir $builddir
+ fi
- echo Running action $action
-
- date
-
- cd $builddir || exit 1
- export srcdir
- df .
- mount
- vmstat
-
- ( action_$action )
- action_status=$?
-
- if [ $action_status != 0 ]; then
- echo "ACTION FAILED: $action";
- else
- echo "ACTION PASSED: $action";
- fi
-
- if [ $action_status != 0 ]; then
- break;
- fi
+ for action in $actions; do
+ echo Running action $action
+
+ date
+
+ cd $builddir || exit 1
+ export srcdir
+ df .
+ mount
+ vmstat
+
+ if [ "x$PREHOOKS" != "x" ]; then
+ for hooks in $PREHOOKS; do
+ if [ "x$hooks" = "x$action" ]; then
+ ( prehook_$action )
+ fi
+ done
+ fi
+
+ ( action_$action )
+ action_status=$?
+
+ if [ "x$POSTHOOKS" != "x" ]; then
+ for hooks in $POSTHOOKS; do
+ if [ "x$hooks" = "x$action" ]; then
+ ( posthook_$action )
+ fi
+ done
+ fi
+
+ df .
+
+ if [ $action_status != 0 ]; then
+ echo "ACTION FAILED: $action";
+ echo " return code $action_status $action";
+ else
+ echo "ACTION PASSED: $action";
+ fi
+
+ if [ $action_status != 0 ]; then
+ break;
+ fi
done
+
if [ "$noclean" = "yes" ]; then
- echo cleanup skipped!
+ echo cleanup skipped!
else
- echo cleaning up
- do_make clean
- /bin/rm -rf $prefix
+ echo cleaning up
+ do_make clean
fi
date
- ) > "$log" 2> "$err"
-
+ } 3>&2 2>&1 1>&3 | tee "$err"
+ ) > "$log" 2>&1
+ # be aware the above channel swap may sometimes result in unordered
+ # stdout/stderr merge
+
+ if [ "$LCOV_REPORT" = "yes" ]; then
+ chmod u=rwX,g=rX,o=rX -R $builddir/coverage
+ rsync -rct -q --password-file=.password -z --timeout=200 \
+ $builddir/coverage/ $host@build.samba.org::lcov_data/$host/$tree/
+ CFLAGS=$PRE_GCOV_CFLAGS
+ LDFLAGS=$PRE_GCOV_LDFLAGS
+ export CFLAGS LDFLAGS
+ fi
+
+ cd $test_root
+
+ /bin/rm -rf $prefix
# send the logs to the master site
send_logs "$log" "$err"
# cleanup
+ echo "Ending build of $tree.$compiler in process $$ at `date`"
unlock_file "$lck"
}
-
#########################################################
-# enable this on a per host basis only when needed please
-# (at least for the moment)
-kill_old_processes() {
- # this should work on systems with linux like ps
- (ps uxfw | grep /build | grep -v grep | egrep 'AM|PM|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
- # and this should work on sysv style ps
- (ps -fu $USER | grep /build | grep -v grep | egrep 'AM|PM|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
+# if you want to build only one project at a time
+# add 'global_lock' after 'per_run_hook' and
+# 'global_unlock' to the end of the file
+#########################################################
+
+global_lock() {
+ lock_file "global.lck" || {
+ exit 0
+ }
+}
+
+global_unlock() {
+ unlock_file "global.lck"
}
+delete_old_tree() {
+ otree=$1
+ test -z "$otree" && return 0;
+
+ rm -rf $otree
+ rm -rf $otree.svn
+ rm -rf $otree.*.svn
+ rm -rf $otree.git
+ rm -rf $otree.*.git
+ rm -rf build.$otree.*
+}
# this is a special fn that allows us to add a "special" hook to the build
# farm that we want to do to the build farm. never leave it empty. instead,
# use ":" as the fn body.
per_run_hook() {
- # kill old processes on systems with a known problem
- case $host in
+ # kill old processes on systems with a known problem
+ case $host in
nohost)
- echo "just a placeholder";
- ;;
- esac
+ echo "just a placeholder";
+ ;;
+ deckchair)
+ rm -f deckchair.fns
+ ;;
+ esac
+
+ # trim the log if too large
+ if [ "`wc -c < build.log`" -gt 2000000 ]; then
+ rm -f build.log
+ fi
+
+ old_trees="web popt distcc samba-gtk smb-build lorikeet-heimdal samba_3_2"
+ old_trees="$old_tree samba_3_2_test samba4 samba_4_0_waf samba_4_0_waf.metze"
+ old_trees="$old_tree samba_3_X_test samba_3_X_devel samba_3_X_devel samba_3_waf"
+ for d in $old_trees; do
+ delete_old_tree $d
+ done
}
######################################################
# main code that is run on each call to the build code
-rsync --timeout=200 -q -az samba.org::build_farm/*.c .
+######################################################
+rsync --timeout=200 -q -az build.samba.org::build_farm/*.c .
# build.log can grow to an excessive size, trim it beyond 50M
if [ -f build.log ]; then
- find build.log -size +100000 -exec /bin/rm '{}' \;
+ find build.log -size +100000 -exec /bin/rm '{}' \;
fi
-