#!/bin/sh -*- mode: shell-script; -*-

# build_farm -- distributed build/test architecture for samba, rsync, etc

# Copyright (C) 2001 by Andrew Tridgell <tridge@samba.org>
# Copyright (C) 2001 by Andrew Bartlett <abartlet@samba.org>
# Copyright (C) 2001, 2003 by Martin Pool <mbp@samba.org>

# default maximum runtime for any command
MAXTIME=25200 # 7 hours
SMBD_MAXTIME=18000 # 5 hours for a samba process ..
# default maximum memory size (100M) for any command
MAXMEM=100000
RUN_FROM_BUILD_FARM=yes
export RUN_FROM_BUILD_FARM
export MAXTIME SMBD_MAXTIME

deptrees="";

build_test_fns_id='$Id$'

copy_dir() {
	Tsrc=$1
	Tdst=$2
	pwd
	echo rsync -a --delete $Tsrc/ $Tdst
	rsync -a --delete $Tsrc/ $Tdst || return 1
	return 0
}

#############################
# build a signature of a tree, used to see if we
# need to rebuild 
#############################

sum_tree() {
	sum_tree_test_root=$1
	sum_tree_tree=$2
	sum_tree_sum=$3
	sum_tree_scm=$4
	find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v version.h | sort | xargs sum > $sum_tree_sum
	sum build_test build_test.fns >> $sum_tree_sum

	if [ -f "$host.fns" ]; then
		sum $host.fns >> $sum_tree_sum
	else
		sum generic.fns >> $sum_tree_sum
	fi

	if [ -f "$test_root/$tree.$scm" ]; then
		sum "$test_root/$tree.$scm" >> $sum_tree_sum
	fi

	for d in $deptrees; do
		dscm=`choose_scm "$d"`
		if [ -f "$test_root/$d.$dscm" ]; then
		sum "$test_root/$d.$dscm" >> $sum_tree_sum
		fi
	done
}

#############################
# send the logs to the master site
#############################

send_logs() {
	if [ "$nologreturn" = "yes" ]; then
		echo "skipping log transfer"
	else
		log="$1"
		err="$2"
		shift
		shift
		chmod 0644 "$log" "$err"

		# xargs -i is implemented differently or not at all.
		# GNU xargs did not implement "-I" until 4.2.9:
		xargs --version 2>&1 | grep "^GNU xargs" > /dev/null
		status=$?
		if [ x"$status" = x"0" ]; then
			XARGS_IS_GNU=yes
		fi

		if [ x"$XARGS_IS_GNU" = x"yes" ]; then
			XARGS_I="xargs -i"
		else
			XARGS_I="xargs -I '{}'"
		fi

		find $log -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'
		find $err -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'

		rsync $* -c -q --password-file=.password -z --timeout=200 \
			"$log" "$err" $host@build.samba.org::build_farm_data/
	fi
}

#############################
# send the logs when they haven't changed
# the aim is to just update the servers timestamp.
# sending with a very large rsync block size does this
# with minimal network traffic
#############################

send_logs_skip() {
	touch "$1" "$2"
	send_logs "$1" "$2" -B 10000000
}

############################
# fetch the latest copy of the tree
############################

fetch_tree() {
	if [ "$norsync" = "yes" ]; then
		echo "skipping tree transfer"
	else
		fetchtree=$1
		if rsync --exclude=autom4te.cache/ --exclude=.svn/ --exclude=.git/ \
			--delete-excluded -q --partial --timeout=200 -ctrlpz --delete --ignore-errors \
			samba.org::ftp/unpacked/$fetchtree/ $test_root/$fetchtree; then
			echo "transferred $fetchtree OK"
		else
			echo "transfer of $fetchtree failed code $?"
			return 1
		fi
	fi
	return 0
}

############################
# fetch the latest copy of the rev meta info
############################

fetch_revinfo() {
	tree=$1
	scm=$2

	test -z "$scm" && return 1
	test x"$scm" = x"unknown" && return 1
	test x"$scm" = x"cvs" && return 1

	if [ "$norsync" = "yes" ]; then
		echo "skipping .revinfo.$scm transfer"
	else
		if [ -r $test_root/$tree.$scm ]; then
			[ -f $test_root/$tree.$scm.old ] && rm -f $test_root/$tree.$scm.old
			[ -f $test_root/$tree.$scm ] && mv $test_root/$tree.$scm $test_root/$tree.$scm.old
		fi
		rsync -q --timeout=200 -clz --ignore-errors \
			samba.org::ftp/unpacked/$tree/.revinfo.$scm $test_root/$tree.$scm
	fi
	if [ -r $test_root/$tree.$scm ]; then
		return 0;
	fi
	return 1
}

############################
# choose the scm that is used for the given project
############################

choose_scm() {
	tree=$1

	case "$tree" in
			samba* | rsync | libreplace | talloc | tdb | ldb | pidl | ccache* | waf*)
			echo "git"
				return 0
			;;
	esac

	echo "svn"
	return 0
}

locknesting=0

############################
# grab a lock file. Not atomic, but close :)
# tries to cope with NFS
############################

lock_file() {
		if [ -z "$lock_root" ]; then
			lock_root=`pwd`;
		fi

		lckf="$lock_root/$1"
		machine=`cat "$lckf" 2> /dev/null | cut -d: -f1`
		pid=`cat "$lckf" 2> /dev/null | cut -d: -f2`

		if [ "$pid" = "$$" ]; then
			locknesting=`expr $locknesting + 1`
			echo "lock nesting now $locknesting"
			return 0
		fi

		if test -f "$lckf"; then
			test x$machine = x$host || {
				echo "lock file $lckf is valid for other machine $machine"
				return 1
			}

			kill -0 $pid && {
				echo "lock file $lckf is valid for process $pid"
				return 1
			}

			echo "stale lock file $lckf for $machine:$pid"
			cat "$lckf"
			/bin/rm -f "$lckf"
		fi
		echo "$host:$$" > "$lckf"
		return 0
}

############################
# unlock a lock file
############################

unlock_file() {
	if [ -z "$lock_root" ]; then
		lock_root=`pwd`;
	fi
	if [ "$locknesting" != "0" ]; then
		locknesting=`expr $locknesting - 1`
		echo "lock nesting now $locknesting"
	else 
		lckf="$lock_root/$1"
		/bin/rm -f "$lckf"
	fi
}

############################
# run make, and print trace
############################

do_make() {
        # work out correct make command
	case "$tree" in
	    waf*)
		MAKECOMMAND="./waf"
		;;
	    *)
		MAKECOMMAND="$MAKE"
		if [ x"$MAKECOMMAND" = x ]; then
		    MAKECOMMAND=make
		fi
		;;
	esac

	MMTIME=$MAXTIME
	# some trees don't need as much time
	case "$tree" in
		rsync | tdb | talloc | libreplace | ccache* | waf*)
			if [ "$compiler" != "checker" ]; then
				MMTIME=`expr $MMTIME / 5`
			fi
		;;
	esac

	# special build for some trees
	case "$tree" in
	    waf*)
		./waf distclean && ./waf configure build
		;;
	esac


	for t in $*; do
		if [ x"$BUILD_FARM_NUM_JOBS" = x ]; then
			echo "$MAKECOMMAND $t"
			$builddir/timelimit $MMTIME "$MAKECOMMAND" "$t"
			status=$?
		else
			# we can parallelize everything and all targets
			if [ x"$t" = xeverything ] || [ x"$t" = xall]; then
				echo "$MAKECOMMAND" "-j$BUILD_FARM_NUM_JOBS"  "$t"
				$builddir/timelimit $MMTIME "$MAKECOMMAND" "-j$BUILD_FARM_NUM_JOBS"  "$t"
				status=$?
			else
				echo "$MAKECOMMAND $t"
				$builddir/timelimit $MMTIME "$MAKECOMMAND" "$t"
				status=$?
			fi
		fi

		if [ $status != 0 ]; then
			case "$t" in
				test|check|installcheck)
					;;
				*)
					#run again with V=1, so we see failed commands
					$builddir/timelimit $MMTIME "$MAKECOMMAND" "$t" V=1
					status=$?
					;;
			esac
		fi

		if [ $status != 0 ]; then
			return $status;
		fi

	done

	return 0
}


############################
# do the coverage report
############################

action_lcovreport() {
	if [ "$LCOV_REPORT" = "yes" ]; then
		case "$tree" in
		lorikeet-heimdal*)
			lcov --directory $builddir --capture --output-file $builddir/$tree.lcov.info
			;;
		samba_3_master*)
			lcov --base-directory $builddir --directory $builddir/.. --capture --output-file $builddir/$tree.lcov.info
			;;
		samba_4*|tdb|talloc|ldb|libreplace)
			lcov --base-directory $builddir/bin --directory $builddir/bin --capture --output-file $builddir/$tree.lcov.info
			;;
		waf)
			lcov --base-directory $builddir/demos --directory $builddir/demos --capture --output-file $builddir/$tree.lcov.info
			;;
		*)
			lcov --base-directory $builddir --directory $builddir --capture --output-file $builddir/$tree.lcov.info
			;;
		esac
		genhtml -o $builddir/coverage $builddir/$tree.lcov.info
		rc=$?
		echo "return code: $rc"
	else
		echo "LCOV_REPORT not set and lcovreport asked"
		echo "Most probably an error please fix !"
		return 1
	fi
}

action_callcatcherreport() {
	if [ "$CALLCATCHER_REPORT" = "yes" ]; then
		case "$tree" in
		tdb|talloc|ldb)
			callanalyse `find $builddir/bin -name \*.so*` $builddir/bin/* > $builddir/coverage/unused-fns.txt
			;;
		samba_3_master|samba_4*)
			callanalyse `find $builddir/bin -name \*.so*` $builddir/bin/* > $builddir/coverage/all-unused-fns.txt 
			grep -v -f $srcdir/callcatcher-exceptions.grep $builddir/coverage/all-unused-fns.txt > $builddir/coverage/unused-fns.txt
			;;
		esac
		rc=$?
		echo "return code: $rc"
	else
		echo "CALLCATCHER_REPORT not set and callcatcher asked"
		echo "Most probably an error please fix !"
		return 1
	fi
}


############################
# configure the tree
############################

action_configure() {
        # special handling for some trees
	case "$tree" in
	    waf*)
		$builddir/timelimit $MAXTIME ./waf configure
		cstatus=$?
		echo "CONFIGURE STATUS: $cstatus"
		return $cstatus
		;;
	esac

	if [ ! -x $srcdir/configure -a -r $srcdir/Makefile.PL ]; then
		perl $srcdir/Makefile.PL PREFIX="$prefix"
		cstatus=$?
		echo "CONFIGURE STATUS: $cstatus"
		return $cstatus;
	fi

	if [ ! -x $srcdir/configure ]; then
		ls -l $srcdir/configure
		echo "$srcdir/configure is missing"
		cstatus=255
		echo "CONFIGURE STATUS: $cstatus"
		return $cstatus;
	fi

	echo "CFLAGS=$CFLAGS"
	echo configure options: $config_and_prefix
	echo CC="$CCACHE $compiler" $srcdir/configure $config_and_prefix

	CC="$CCACHE $compiler"
	export CC
	$builddir/timelimit $MAXTIME $srcdir/configure $config_and_prefix
	cstatus=$?

	if [ x"$cstatus" != x"0" ]; then
		if [ -f config.log ]; then
			echo "contents of config.log:"
			cat config.log
		fi

		# Waf style
		if [ -f bin/config.log ]; then
			echo "contents of config.log:"
			cat bin/config.log
		fi
	fi
	echo "CONFIGURE STATUS: $cstatus"
	return $cstatus
}

############################
# show the configure log
############################

action_config_log() {

	log_files="config.log bin/config.log"
	for f in $log_files; do
		if [ -f $f ]; then
			echo "contents of config.log:"
			cat $f
			return 0
		fi
	done
	return 0
}

############################
# show the config.h
############################

action_config_header() {
	hdr_files="config.h include/config.h include/autoconf/config.h bin/default/config.h bin/default/include/config.h bin/default/source3/include/config.h"
	for h in $hdr_files; do
		if [ -f $h ]; then
			echo "contents of $h:"
			cat $h
			return 0
		fi
	done

	return 0
}



############################
# build the tree
############################
action_build() {
	case "$tree" in
	samba_4*)
		do_make everything
		bstatus=$?
		;;
	samba_3*)
		do_make everything torture
		bstatus=$?
		;;
	waf*)
		do_make build
		bstatus=$?
		;;
	*)
		do_make all
		bstatus=$?
		;;
	esac

	echo "BUILD STATUS: $bstatus"

	return $bstatus
}

############################
# show static analysis results
############################

action_cc_checker() {

	# default to passing the cc_checker
	cccstatus=0

	if [ -f ibm_checker.out ]; then
		cat ibm_checker.out
		cccstatus=`cat ibm_checker.out | grep '^\-\- ' | wc -l`
	fi

	echo "CC_CHECKER STATUS: $cccstatus"
	return $cccstatus;	
}

############################
# install the tree
############################

action_install() {
	if [ -d $prefix ]; then
		if [ "$noclean" != "yes" ]; then
			rm -rf $prefix
		fi
	fi

	do_make install
	istatus=$?
	echo "INSTALL STATUS: $istatus"
	return $istatus;
}

############################
# test the tree
action_test_samba() {
	do_make test
	totalstatus=$?

	# if we produced a test summary then show it
	[ -f st/summary ] && {
		echo "TEST SUMMARY"
		cat st/summary
	}

	return "$totalstatus"
}

action_test_generic() {
	CC="$compiler"
	export CC
	do_make installcheck
	totalstatus=$?
	echo "TEST STATUS: $totalstatus"
	return "$totalstatus"
}

action_test_lorikeet_heimdal() {
	CC="$compiler"
	export CC
	SOCKET_WRAPPER_DIR=`pwd`/sw
	mkdir $SOCKET_WRAPPER_DIR
	export SOCKET_WRAPPER_DIR
	do_make check
	totalstatus=$?
	SOCKET_WRAPPER_DIR=
	export SOCKET_WRAPPER_DIR
	echo "TEST STATUS: $totalstatus"
	return "$totalstatus"
}


#############################
# attempt some basic tests of functionaility
# starting as basic as possible, and getting incresingly complex
#############################

action_test() {
	# Samba needs crufty code of its own for backward
	# compatiblity.  I think a better way to do this in the future
	# is to just call 'make installcheck'.
	case "$tree" in
	samba*|smb-build|pidl)
		action_test_samba
		;;
	lorikeet-heimdal*)
		action_test_lorikeet_heimdal
		;;
	*)
		action_test_generic
		;;
	esac
}

###########################
# do a test build of a particular tree
# This is the master function called by generic.fns or
# host.fns
###########################

test_tree() {
	tree=$1
	source=$2
	compiler="$3"
	shift
	shift
	shift
	echo "Starting to deal with tree $tree with compiler $compiler"
	if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && [ "$tree" != "ccache-maint" ] && ccache -V > /dev/null 2>/dev/null; then
		CCACHE="ccache"
		export CCACHE
	else
		CCACHE=""
	fi

	# limit our resource usage
	ulimit -t $MAXTIME 2> /dev/null

	# max mem size 100M
	ulimit -m $MAXMEM 2> /dev/null

	# max file size 100M
	# darn, this affects sparse files too! disable it
	# ulimit -f 100000 2> /dev/null

	# try and limit the number of open files to 250. That means we'll discover
	# fd leaks faster
	ulimit -n 250 2> /dev/null

	# Keep stuff private
	umask 077

	if [ -z "$test_root" ]; then
		test_root=`pwd`
	fi

	log="build.$tree.$host.$compiler.log"
	err="build.$tree.$host.$compiler.err"
	sum="build.$tree.$host.$compiler.sum"
	lck="build.$tree.lck"
		srcdir="$test_root/$tree/$source"

	lock_file "$lck" || {
		return
	}

	# work out what other trees this package depends on
	deptrees=""
	case "$tree" in
		samba-gtk)
		deptrees="samba_4_0_test"
		;;
	esac

	scm=`choose_scm "$tree"`

	# pull the entries, if any
	# Remove old .svn or .git files
	# Move the current .svn org .git to .svn.old or
	# .git.old then fetch the new from rsync
	if fetch_revinfo "$tree" "$scm"; then
		for d in $deptrees; do
			# If there is dependency substree(s) we add info
			# from the dependency tree so that we
			# can rebuild in case one of them has changed
			dscm=`choose_scm "$d"`
			if [ -f "$test_root/$d.$dscm" ]; then
				if [ "$d" != "$tree" ]; then
					cat "$test_root/$d.$dscm" >> $test_root/$tree.$scm
				fi
			fi
		done
		[ -f $test_root/$tree.$compiler.$scm.old ] && rm -f $test_root/$tree.$compiler.$scm.old
		[ -f $test_root/$tree.$compiler.$scm ] && mv $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old
		[ -f $test_root/$tree.$scm ] && cp $test_root/$tree.$scm $test_root/$tree.$compiler.$scm

		if [ -f $test_root/$tree.$compiler.$scm.old ] && \
				cmp $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old > /dev/null; then

			echo "skip: $tree.$compiler nothing changed in $scm"
			cd $test_root
			send_logs_skip "$log" "$err"
			unlock_file "$lck"
			return
		fi
	fi

	# pull the tree
	fetch_tree "$tree" || {
		cd $test_root
		unlock_file "$lck"
		return
	}

	# check for essential files
	case "$tree" in
	    pidl)
		# no generated files
		;;
	    waf*)
		if [ ! -x $srcdir/waf ]; then
		    echo "skip: $tree.$compiler waf not present, try again next time!"
		    cd $test_root
		    unlock_file "$lck"
		    return
		fi
		;;
	    *)
		if [ ! -x $srcdir/configure ]; then
		    echo "skip: $tree.$compiler configure not present, try again next time!"
		    cd $test_root
		    unlock_file "$lck"
		    return
		fi
		;;
	esac

	echo "Starting build of $tree.$compiler in process $$ at `date`"


	# Parameters for the build depending on the tree
	case "$tree" in
		*)
			builddir=$srcdir
			export builddir
			;;
	esac

	#Fix the user
	if [ ! x$USER = x"" ]; then
		whoami=$USER
	else 
		if [ ! x$LOGNAME = x"" ]; then
			whoami=$LOGNAME
		else
			whoami=build
		fi
	fi

	# build the timelimit utility
	echo "Building timelimit"
	mkdir -p $builddir
	echo $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c
	$compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1

	# build the killbysubdir utility
	echo "Building killbysubdir"
	echo $compiler -o $builddir/killbysubdir $test_root/killbysubdir.c
	$compiler -o $builddir/killbysubdir $test_root/killbysubdir.c

	prefix="$test_root/prefix/$tree.$compiler"
	mkdir -p "$prefix"

	# This can be defined in <host>.fns files
	sw_config=$config

	case "$tree" in
	lorikeet-heimdal)
		sw_config="$config --enable-socket-wrapper"
		;;
	samba_4*)
		sw_config="$config --enable-socket-wrapper"
		sw_config="$sw_config --enable-nss-wrapper"
		sw_config="$sw_config --enable-uid-wrapper"
		;;
	samba_3*)
		sw_config="$config --enable-socket-wrapper"
		sw_config="$sw_config --enable-nss-wrapper"
		;;
	samba-gtk)
		PKG_CONFIG_PATH="$test_root/prefix/samba_4_0_test.$compiler/lib/pkgconfig"
		export PKG_CONFIG_PATH
		;;
	*)
		testsuite=testsuite
		;;
	esac

	if [ "$LCOV_REPORT" = "yes" ]; then
	        PRE_GCOV_CFLAGS=$CFLAGS
	        PRE_GCOV_LDFLAGS=$LDFLAGS
		GCOV_FLAGS="--coverage"
		CFLAGS="$CFLAGS $GCOV_FLAGS" 
		LDFLAGS="$LDFLAGS $GCOV_FLAGS" 
		export CFLAGS LDFLAGS
	fi

	config_and_prefix="$sw_config --prefix=$prefix"

	# see if we need to rebuild
	sum_tree $test_root $tree $sum $scm
	echo "CFLAGS=$CFLAGS $config_and_prefix" >> $sum

	if [ -f "$sum.old" ] && cmp "$sum" "$sum.old" > /dev/null; then
		echo "skip: $tree.$compiler nothing changed"
		cd $test_root
		send_logs_skip "$log" "$err"
		unlock_file "$lck"
		echo "Ending build of $tree.$compiler in process $$ at `date`"
		if [ "$LCOV_REPORT" = "yes" ]; then
	            CFLAGS=$PRE_GCOV_CFLAGS
	            LDFLAGS=$PRE_GCOV_LDFLAGS
		    export CFLAGS LDFLAGS
		fi
		return
	fi

	# we do need to rebuild - save the old sum
	[ -f $sum.old ] && /bin/rm -f $sum.old
	mv $sum $sum.old

	#Action == what to do ie. configure config_log ...
	actions="$*"

	if [ "$actions" = "" ]; then
		actions="configure config_log config_header build install test $EXTRA_ACTIONS"
	fi

	# start the build
	(
	{
		# we all want to be able to read the output...
		LANG=C
		export LANG

		uname -a

		echo ""
		echo "build_test		  : $build_test_id"
		echo "build_test.fns	  : $build_test_fns_id"
		echo "local settings file : $build_test_settings_local_file"
		echo "local functions file: $build_test_fns_local_file"
		echo "used .fns file	  : $build_test_used_fns_file"
		echo ""

		# we need to be able to see if a build farm machine is accumulating
		# stuck processes. We do this in two ways, as we don't know what style
		# of ps it will have
		ps xfuw 2> /dev/null
		ps -fu $USER 2> /dev/null

		echo "building $tree with CC=$compiler on $host at "`date`
		echo "builddir=$builddir"
		echo "prefix=$prefix"

		echo "Showing limits"
		ulimit -a 2> /dev/null

		# the following is for non-samba builds only
		if [ "$scm" = "svn" -a -r $test_root/$tree.svn ]; then
			h_rev=`grep 'Revision: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
			if [ -n "$h_rev" ]; then
				echo "HIGHEST SVN REVISION: $h_rev"
			fi
			rev=`grep 'Last Changed Rev: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
			if [ -n "$rev" ]; then
			echo "BUILD REVISION: $rev"
			fi
		elif [ "$scm" = "git" -a -r $test_root/$tree.git ]; then
			csha1=`cat $test_root/$tree.git |head -3 | tail -1`
			if [ -n "$csha1" ]; then
				echo "BUILD COMMIT REVISION: $csha1"
			fi
			cdate=`cat $test_root/$tree.git |head -4 | tail -1`
			if [ -n "$cdate" ]; then
				echo "BUILD COMMIT DATE: $cdate"
			fi
			ctime=`cat $test_root/$tree.git |head -2 | tail -1`
			if [ -n "$ctime" ]; then
				echo "BUILD COMMIT TIME: $ctime"
		  fi
		fi

		if [ -x $builddir/killbysubdir ]; then
			echo "$builddir/killbysubdir $builddir in `pwd`"
			$builddir/killbysubdir $builddir
		fi

		for action in $actions; do

			echo Running action $action

			date

			cd $builddir || exit 1
			export srcdir
			df .
			mount
			vmstat

			if [ "x$PREHOOKS" != "x" ]; then
				for hooks in $PREHOOKS; do
					if [ "x$hooks" = "x$action" ]; then
						( prehook_$action )
					fi
				done
			fi

			( action_$action )
			action_status=$?

			if [ "x$POSTHOOKS" != "x" ]; then
				for hooks in $POSTHOOKS; do
					if [ "x$hooks" = "x$action" ]; then
						( posthook_$action )
					fi
				done
			fi

			df .

			if [ $action_status != 0 ]; then
				echo "ACTION FAILED: $action";
				echo " return code $action_status $action";
			else
				echo "ACTION PASSED: $action";
			fi

			if [ $action_status != 0 ]; then 
				break;
			fi
		done


		if [ "$noclean" = "yes" ]; then
			echo cleanup skipped!
		else
			echo cleaning up
			do_make clean
		fi
		date
	} 3>&2 2>&1 1>&3 | tee "$err"
	) > "$log" 2>&1
	# be aware the above channel swap may sometimes result in unordered
	# stdout/stderr merge

	if [ "$LCOV_REPORT" = "yes" ]; then
		chmod u=rwX,g=rX,o=rX -R $builddir/coverage
		rsync -rct -q --password-file=.password -z --timeout=200 \
			$builddir/coverage/ $host@build.samba.org::lcov_data/$host/$tree/
	        CFLAGS=$PRE_GCOV_CFLAGS
	        LDFLAGS=$PRE_GCOV_LDFLAGS
		export CFLAGS LDFLAGS
	fi

	cd $test_root

	/bin/rm -rf $prefix
	# send the logs to the master site
	send_logs "$log" "$err"

	# cleanup
	echo "Ending build of $tree.$compiler in process $$ at `date`"
	unlock_file "$lck"
}

#########################################################
# if you want to build only one project at a time
# add 'global_lock' after 'per_run_hook' and
# 'global_unlock' to the end of the file
#########################################################

global_lock() {
	lock_file "global.lck" || {
		exit 0
	}
}

global_unlock() {
	unlock_file "global.lck"
}

delete_old_tree() {
	otree=$1
	test -z "$otree" && return 0;

	rm -rf $otree
	rm -rf $otree.svn
	rm -rf $otree.*.svn
	rm -rf $otree.git
	rm -rf $otree.*.git
	rm -rf build.$otree.*
}

# this is a special fn that allows us to add a "special" hook to the build
# farm that we want to do to the build farm. never leave it empty. instead,
# use ":" as the fn body.
per_run_hook() {
	# kill old processes on systems with a known problem
	case $host in
	nohost)
		echo "just a placeholder";
		;;
	deckchair)
		rm -f deckchair.fns
		;;
	esac

	# trim the log if too large
	if [ "`wc -c < build.log`" -gt 2000000 ]; then
	rm -f build.log
	fi

	old_trees="web popt distcc samba-gtk smb-build lorikeet-heimdal samba_3_2"
	old_trees="$old_tree samba_3_2_test samba4 samba_4_0_waf samba_4_0_waf.metze"
	old_trees="$old_tree samba_3_X_test samba_3_X_devel samba_3_X_devel samba_3_waf"
	for d in $old_trees; do
		delete_old_tree $d
	done
}


######################################################
# main code that is run on each call to the build code
######################################################
rsync --timeout=200 -q -az build.samba.org::build_farm/*.c .


# build.log can grow to an excessive size, trim it beyond 50M
if [ -f build.log ]; then
	find build.log -size +100000 -exec /bin/rm '{}' \;
fi