#!/bin/sh -*- mode: shell-script; sh-indentation: 8; indent-tabs-mode: t; -*-

# build_farm -- distributed build/test architecture for samba, rsync, etc

# Copyright (C) 2001 by Andrew Tridgell <tridge@samba.org>
# Copyright (C) 2001 by Andrew Bartlett <abartlet@samba.org>
# Copyright (C) 2001, 2003 by Martin Pool <mbp@samba.org>

# default maximum runtime for any command
MAXTIME=7200
# default maximum memory size (100M) for any command
MAXMEM=100000
RUN_FROM_BUILD_FARM=yes
export RUN_FROM_BUILD_FARM

deptrees="";

build_test_fns_id='$Id$'

#############################
# build a signature of a tree, used to see if we
# need to rebuild 
sum_tree() {
        sum_tree_test_root=$1
	sum_tree_tree=$2
	sum_tree_sum=$3
	sum_tree_scm=$4
	find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v version.h | sort | xargs sum > $sum_tree_sum
	sum build_test build_test.fns >> $sum_tree_sum

	if [ -f "$host.fns" ]; then
	    sum $host.fns >> $sum_tree_sum
	else
            sum generic.fns >> $sum_tree_sum
	fi

	if [ -f "$test_root/$tree.$scm" ]; then
	    sum "$test_root/$tree.$scm" >> $sum_tree_sum
	fi

	for d in $deptrees; do
	    dscm=`choose_scm "$d"`
	    if [ -f "$test_root/$d.$dscm" ]; then
		sum "$test_root/$d.$dscm" >> $sum_tree_sum
	    fi
	done
}

#############################
# send the logs to the master site
send_logs() {
	if [ "$nologreturn" = "yes" ]; then
		echo "skipping log transfer"
	else
		log="$1"
		err="$2"
		shift
		shift
		chmod 0644 "$log" "$err"

		XARGS_I="xargs -i"
		if [ "`uname`" = "FreeBSD" ]; then
			XARGS_I="xargs -I '{}' -R -1"
		fi
		if [ "`uname`" = "Darwin" ]; then
			XARGS_I="xargs -I '{}' -R -1"
		fi
		find $log -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'
		find $err -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'

		rsync $* -ct -q --password-file=.password -z --timeout=200 \
		    "$log" "$err" $host@build.samba.org::build_farm_data/
	fi
}

#############################
# send the logs when they haven't changed
# the aim is to just update the servers timestamp.
# sending with a very large rsync block size does this
# with minimal network traffic
send_logs_skip() {
    touch "$1" "$2"
    send_logs "$1" "$2" -B 10000000
}

############################
# fetch the latest copy of the tree
fetch_tree() {
	if [ "$norsync" = "yes" ]; then
		echo "skipping tree transfer"
	else
		fetchtree=$1
		if rsync --exclude=autom4te.cache/ --exclude=.svn/ --exclude=.git/ \
			--delete-excluded -q --partial --timeout=200 -crlpz --delete --ignore-errors \
			samba.org::ftp/unpacked/$fetchtree/ $test_root/$fetchtree; then
			echo "transferred $fetchtree OK"
		else
			echo "transfer of $fetchtree failed code $?"
			return 1
		fi
	fi
	return 0
}

############################
# fetch the latest copy of the rev meta info
fetch_revinfo() {
    tree=$1
    scm=$2

    test -z "$scm" && return 1
    test x"$scm" = x"unknown" && return 1
    test x"$scm" = x"cvs" && return 1

    if [ "$norsync" = "yes" ]; then
	echo "skipping .revinfo.$scm transfer"
    else
	if [ -r $test_root/$tree.$scm ]; then
		rm -f $test_root/$tree.$scm.old
	    mv $test_root/$tree.$scm $test_root/$tree.$scm.old
	fi
	rsync -q --timeout=200 -clz --ignore-errors \
	    samba.org::ftp/unpacked/$tree/.revinfo.$scm $test_root/$tree.$scm
    fi
    if [ -r $test_root/$tree.$scm ]; then
	return 0;
    fi
    return 1
}

############################
# choose the scm that is used for the given project
choose_scm() {
	tree=$1

	case "$tree" in
    		ccache | distcc | rsync)
			echo "cvs"
        		return 0
        	;;
    		samba_3_2_test)
			echo "git"
        		return 0
        	;;
	esac

	echo "svn"
	return 0
}

locknesting=0

############################
# grab a lock file. Not atomic, but close :)
# tries to cope with NFS
lock_file() {
        if [ -z "$lock_root" ]; then
	  lock_root=`pwd`;
        fi
        lckf="$lock_root/$1"
        machine=`cat "$lckf" 2> /dev/null | cut -d: -f1`
        pid=`cat "$lckf" 2> /dev/null | cut -d: -f2`

        if [ "$pid" = "$$" ]; then
            locknesting=`expr $locknesting + 1`
            echo "lock nesting now $locknesting"
            return 0
        fi

        if test -f "$lckf"; then
            test $machine = $host || {
                echo "lock file $lckf is valid for other machine $machine"
                return 1		 
	    }
	    kill -0 $pid && {
                echo "lock file $lckf is valid for process $pid"
                return 1
	    }
            echo "stale lock file $lckf for $machine:$pid"
            cat "$lckf"
            /bin/rm -f "$lckf"
        fi
        echo "$host:$$" > "$lckf"
        return 0
}

############################
# unlock a lock file
unlock_file() {
        if [ -z "$lock_root" ]; then
	  lock_root=`pwd`;
        fi
	if [ "$locknesting" != "0" ]; then
	    locknesting=`expr $locknesting - 1`
	    echo "lock nesting now $locknesting"
	else 
	    lckf="$lock_root/$1"
	    /bin/rm -f "$lckf"
	fi
}

############################
# run make, and print trace
do_make() {

  if [ x"$MAKE" = x ] 
  then
    MAKE=make
  fi 

  MMTIME=$MAXTIME
  # some trees don't need as much time
  case "$tree" in
	rsync | tdb | talloc | libreplace | ccache | distcc)
	  if [ "$compiler" != "checker" ]; then
	      MMTIME=`expr $MMTIME / 5`
	  fi
	  ;;
  esac
  
    
  for t in $*; do
    if [ x"$BUILD_FARM_NUM_JOBS" = x ]; then
      echo "$MAKE $t"
      ./timelimit $MMTIME "$MAKE" "$t"
      status=$?
    else
      # we can parallelize everything and all targets
      if [ x"$t" = xeverything ] || [ x"$t" = xall]; then
        echo "$MAKE" "-j$BUILD_FARM_NUM_JOBS"  "$t"
        ./timelimit $MMTIME "$MAKE" "-j$BUILD_FARM_NUM_JOBS"  "$t"
        status=$?
      else
        echo "$MAKE $t"
        ./timelimit $MMTIME "$MAKE" "$t"
        status=$?
      fi
    fi

    if [ $status != 0 ]; then
      return $status;
    fi

  done

  return 0
}      

############################
# configure the tree
action_configure() {
        if [ ! -x $srcdir/configure ]; then
	    ls -l $srcdir/configure
	    echo "$srcdir/configure is missing"
	    cstatus=255
	    echo "CONFIGURE STATUS: $cstatus"
	    return $cstatus;
	fi
	echo "CFLAGS=$CFLAGS"
	echo configure options: $config_and_prefix
	echo CC="$CCACHE $compiler" $srcdir/configure $config_and_prefix
	CC="$CCACHE $compiler"
	export CC
	./timelimit $MAXTIME $srcdir/configure $config_and_prefix
	cstatus=$?
	echo "CONFIGURE STATUS: $cstatus"
	if [ -f config.h ]; then
	    echo "contents of config.h:"
	    cat config.h
	fi
	if [ -f include/config.h ]; then
	    echo "contents of include/config.h:"
	    cat include/config.h
	fi
	return $cstatus;
}

############################
# show the configure log
action_config_log() {
        if [ ! -f config.log ]; then
	    return 0;
	fi
	echo "contents of config.log:"
	cat config.log
	return 0;
}

copy_dir() {
	Tsrc=$1
	Tdst=$2
	rsync -a --delete $Tsrc/ $Tdst
}

s4selftest_create() {
	lock_file "s4selftest.lck" || {
		return 1;
	}

	rm -rf $s4selftest/
	mkdir -p $s4selftest/source

	copy_dir $builddir/bin $s4selftest/source/bin || {
		rm -rf $s4selftest/;
		unlock_file "s4selftest.lck";
		return 1;
	}

	copy_dir $srcdir/setup $s4selftest/source/setup || {
		rm -rf $s4selftest/;
		unlock_file "s4selftest.lck";
		return 1;
	}

	copy_dir $srcdir/../testprogs $s4selftest/testprogs || {
		rm -rf $s4selftest/;
		unlock_file "s4selftest.lck";
		return 1;
	}

	copy_dir $srcdir/selftest $s4selftest/source/selftest || {
		rm -rf $s4selftest/;
		unlock_file "s4selftest.lck";
		return 1;
	}

	copy_dir $srcdir/script $s4selftest/source/script || {
		rm -rf $s4selftest/;
		unlock_file "s4selftest.lck";
		return 1;
	}

	mkdir -p $s4selftest/source/scripting || {
		rm -rf $s4selftest/;
		unlock_file "s4selftest.lck";
		return 1;
	}

	copy_dir $srcdir/scripting/libjs $s4selftest/source/scripting/libjs || {
		rm -rf $s4selftest/;
		unlock_file "s4selftest.lck";
		return 1;
	}
	
	unlock_file "s4selftest.lck"
}

s4selftest_update() {
	lock_file "s4selftest.lck" || {
		return 1;
	}

	copy_dir $s4selftest $s4selftest.$tree.$compiler.$$ || {
		rm -rf $s4selftest.$tree.$compiler.$$;
		unlock_file "s4selftest.lck";
		return 1;
	}

	rm -rf $s4selftest.$tree.$compiler
	mv $s4selftest.$tree.$compiler.$$ $s4selftest.$tree.$compiler

	unlock_file "s4selftest.lck"
}

############################
# build the tree
action_build() {
	case "$tree" in
	samba4)
		do_make everything
		bstatus=$?
		if test x"$bstatus" != x"0"; then
			# the 2nd 'make everything' is to work around a bug
			# in netbsd make. 
			do_make everything
			bstatus=$?
		fi

		if test x"$bstatus" != x"0"; then
			do_make testsuite
			bstatus=$?
		fi

		if test x"$bstatus" = x"0"; then
			s4selftest_create
		fi

		;;
	samba_3_2*)
		do_make proto everything torture
		bstatus=$?
		;;
	*)
		do_make all
		bstatus=$?
		;;
	esac

	echo "BUILD STATUS: $bstatus"

	return $bstatus
}

############################
# show static analysis results
action_cc_checker() {

	# default to passing the cc_checker
        cccstatus=0

	if [ -f ibm_checker.out ]; then
		cat ibm_checker.out
		cccstatus=`cat ibm_checker.out | grep '^\-\- ' | wc -l`
	fi

	echo "CC_CHECKER STATUS: $cccstatus"
	return $cccstatus;	
}

############################
# install the tree
action_install() {
	if [ -d $prefix ]; then
		if [ "$noclean" != "yes" ]; then
		    rm -rf $prefix
		fi
	fi

	do_make install
	istatus=$?
	echo "INSTALL STATUS: $istatus"
	return $istatus;
}

############################
# test the tree
action_test_samba() {
	do_make test
	totalstatus=$?
	return "$totalstatus"
}

action_test_generic() {
        CC="$compiler"
	export CC
	do_make installcheck
	totalstatus=$?
	echo "TEST STATUS: $totalstatus"
	return "$totalstatus"
}

action_test_lorikeet_heimdal() {
        CC="$compiler"
	export CC
	SOCKET_WRAPPER_DIR=`pwd`/sw
	mkdir $SOCKET_WRAPPER_DIR
 	export SOCKET_WRAPPER_DIR
	do_make check
	totalstatus=$?
	SOCKET_WRAPPER_DIR=
 	export SOCKET_WRAPPER_DIR
	echo "TEST STATUS: $totalstatus"
	return "$totalstatus"
}


#############################
# attempt some basic tests of functionaility
# starting as basic as possible, and getting incresingly complex

action_test() {
	# Samba needs crufty code of its own for backward
	# compatiblity.  I think a better way to do this in the future
	# is to just call 'make installcheck'.
	case "$tree" in
	samba*|smb-build|pidl)
	    action_test_samba
	    ;;
	lorikeet-heimdal*)
	    action_test_lorikeet_heimdal
	    ;;
	*)
	    action_test_generic
	    ;;
	esac
}

###########################
# do a test build of a particular tree
test_tree() {
	tree=$1
	source=$2
	compiler="$3"
	shift
	shift
	shift
	if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && ccache -V > /dev/null; then
	    CCACHE="ccache"
	    export CCACHE
	else
	    CCACHE=""
	fi

	# limit our resource usage
	ulimit -t $MAXTIME 2> /dev/null

	# max mem size 100M
	ulimit -m $MAXMEM 2> /dev/null

	# max file size 100M
	# darn, this affects sparse files too! disable it
	# ulimit -f 100000 2> /dev/null

	# try and limit the number of open files to 150. That means we'll discover
	# fd leaks faster
	ulimit -n 150 2> /dev/null

	# Keep stuff private
	umask 077

	if [ -z "$test_root" ]; then
		test_root=`pwd`
	fi

	log="build.$tree.$host.$compiler.log"
	err="build.$tree.$host.$compiler.err"
	sum="build.$tree.$host.$compiler.sum"
	lck="build.$tree.lck"
        srcdir="$test_root/$tree/$source"

	lock_file "$lck" || {
		return
	}

	# work out what other trees this package depends on
	deptrees=""
	case "$tree" in
	    talloc | tdb)
		deptrees="libreplace";
	    ;;
	    ldb)
		deptrees="libreplace talloc tdb";
	    ;;
		samba-gtk)
		deptrees="samba4"
		;;
	esac

	scm=`choose_scm "$tree"`

	# pull the entries, if any
	if fetch_revinfo "$tree" "$scm"; then
	    for d in $deptrees; do
	    	dscm=`choose_scm "$d"`
		if [ -f "$test_root/$d.$dscm" ]; then
		    if [ "$d" != "$tree" ]; then
			cat "$test_root/$d.$dscm" >> $test_root/$tree.$scm
		    fi
		fi
	    done
	    rm -f $test_root/$tree.$compiler.$scm.old
	    mv $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old
	    cp $test_root/$tree.$scm $test_root/$tree.$compiler.$scm
	    if cmp $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old > /dev/null; then
		echo "skip: $tree.$compiler nothing changed in $scm"
		cd $test_root
		send_logs_skip "$log" "$err"
		unlock_file "$lck"
		return
	    fi
	fi

	# pull the tree
	fetch_tree "$tree" || {
	    cd $test_root
	    unlock_file "$lck"
	    return
	}

        if [ ! -x $srcdir/configure ] && [ "$tree" != "pidl" ]; then
		echo "skip: $tree.$compiler configure not present, try again next time!"
		cd $test_root
		unlock_file "$lck"
		return
	fi

	echo "Starting build of $tree.$compiler in process $$ at `date`"

	case "$tree" in
	    tdb | talloc | ldb | libreplace)
		builddir="$test_root/tmp.$tree.$compiler"
		usingtmpbuild=1
		if [ -d $builddir ]; then
		    rm -rf $builddir
		fi
		mkdir -p $builddir
		export builddir
	    ;;
	    *)
		builddir=$srcdir
		usingtmpbuild=0
		export builddir
	    ;;
	esac
	
	if [ ! x$USER = x"" ]; then
	    whoami=$USER
	else 
	    if [ ! x$LOGNAME = x"" ]; then
		whoami=$LOGNAME
	    else
		whoami=build
	    fi
	fi

	prefix="$test_root/prefix/$tree.$compiler"
	mkdir -p "$prefix"

	s4selftest=$test_root/s4selftest
	export s4selftest

	sw_config=$config

	case "$tree" in
	samba4|lorikeet-heimdal)
		sw_config="$config --enable-socket-wrapper"
		;;
	samba_3_2)
		sw_config="$config --enable-socket-wrapper"
		s4selftest_update "$tree" "$compiler" && {
			t="$s4selftest.$tree.$compiler/source"
			sw_config="$sw_config --with-samba4srcdir=$t"
		}
	samba_3_2_test)
		sw_config="$config --enable-socket-wrapper"
		s4selftest_update "$tree" "$compiler" && {
			t="$s4selftest.$tree.$compiler/source"
			t="$t/bin/smbtorture"
			sw_config="$sw_config --with-smbtorture4-path=$t"
		}
		;;
	samba-gtk)
		PKG_CONFIG_PATH="$test_root/prefix/samba4.$compiler/lib/pkgconfig"
		export PKG_CONFIG_PATH
		;;
	ldb)
		fetch_tree popt
		;;
	talloc)
		fetch_tree libreplace
		;;
	*)
		testsuite=testsuite
		;;
	esac

	if [ "$LCOV_REPORT" = "yes" ]; then
	    GCOV_FLAGS="-ftest-coverage -fprofile-arcs"
	    GCOV_LIBS="-lgcov"
	    HOSTCC_CFLAGS="$HOSTCC_CFLAGS $GCOV_FLAGS" 
	    CFLAGS="$CFLAGS $GCOV_FLAGS" 
	    LDFLAGS="$LDFLAGS $GCOV_FLAGS $GCOV_LIBS" 
	    SHLD_FLAGS="$SHLD_FLAGS $GCOV_FLAGS $GCOV_LIBS"
	    export HOSTCC_CFLAGS CFLAGS LDFLAGS SHLD_FLAGS
	fi

	config_and_prefix="$sw_config --prefix=$prefix"

	# see if we need to rebuild
	sum_tree $test_root $tree $sum $scm
	echo "CFLAGS=$CFLAGS $config_and_prefix" >> $sum

	if cmp "$sum" "$sum.old" > /dev/null; then
		echo "skip: $tree.$compiler nothing changed"
		cd $test_root
		send_logs_skip "$log" "$err"
		unlock_file "$lck"
		echo "Ending build of $tree.$compiler in process $$ at `date`"
		return
	fi

	# we do need to rebuild - save the old sum
	/bin/rm -f $sum.old
	mv $sum $sum.old

	actions="$*"
	
	if [ "$actions" = "" ]; then
	    actions="configure config_log build install test"
	fi

	# start the build
	(
		# we all want to be able to read the output...
		LANG=C
		export LANG

		uname -a

		echo ""
		echo "build_test          : $build_test_id"
		echo "build_test.fns      : $build_test_fns_id"
		echo "local settings file : $build_test_settings_local_file"
		echo "local functions file: $build_test_fns_local_file"
		echo "used .fns file      : $build_test_used_fns_file"
		echo ""

                # we need to be able to see if a build farm machine is accumulating
                # stuck processes. We do this in two ways, as we don't know what style
                # of ps it will have
		ps xfuw 2> /dev/null
		ps -fu $USER 2> /dev/null

		echo "building $tree with CC=$compiler on $host at "`date`
		echo "builddir=$builddir"
		echo "prefix=$prefix"

		echo "Showing limits"
		ulimit -a 2> /dev/null

           	# build the timelimit utility
		echo "Building timelimit"
		mkdir -p $builddir
		$compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1

		# the following is for non-samba builds only
		if [ -r $test_root/$tree.svn ]; then
		  h_rev=`grep 'Revision: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
		  if [ -n "$h_rev" ]; then
			echo "HIGHEST SVN REVISION: $h_rev"
		  fi
		  rev=`grep 'Last Changed Rev: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
		  if [ -n "$rev" ]; then
			echo "BUILD REVISION: $rev"
		  fi
		fi


		if [ "$tree" = "pidl" ] 
		then
			cd $builddir
			perl ./Makefile.PL "$prefix"
		fi

		for action in $actions; do

		    echo Running action $action

		    date

		    cd $builddir || exit 1
		    export srcdir
		    df .
		    mount
		    vmstat

		    ( action_$action )
		    action_status=$?
		    
		    if [ $action_status != 0 ]; then
			echo "ACTION FAILED: $action";
		    else
			echo "ACTION PASSED: $action";
		    fi
		    
		    if [ $action_status != 0 ]; then 
			break;
		    fi

		done

		if [ "$LCOV_REPORT" = "yes" ]; then
		    case "$tree" in
			lorikeet-heimdal*)
			    lcov --directory $builddir --capture --output-file $builddir/$tree.lcov.info
			    ;;
			*)
			    # ugly hack for s4, as lcov is otherwise not able to find 
			    # these files
			    rm -f heimdal/lib/*/{lex,parse}.{gcda,gcno}
			    lcov --base-directory $builddir --directory $builddir --capture --output-file $builddir/$tree.lcov.info
			    ;;
		    esac
		    genhtml -o $builddir/coverage $builddir/$tree.lcov.info
		fi

		if [ "$noclean" = "yes" ]; then
		    echo cleanup skipped!
		else
		    echo cleaning up
		    do_make clean
		fi
		date
	) > "$log" 2> "$err"

	if [ "$LCOV_REPORT" = "yes" ]; then
	    chmod u=rwX,g=rX,o=rX -R $builddir/coverage
	    rsync -rct -q --password-file=.password -z --timeout=200 \
		$builddir/coverage/ $host@build.samba.org::lcov_data/$host/$tree/
	fi

	cd $test_root

        /bin/rm -rf $prefix
	if [ "$usingtmpbuild" = "1" ]; then
	    if [ "$noclean" = "yes" ]; then
		echo builddir cleanup skipped!
	    else
		/bin/rm -rf $builddir
	    fi
	fi
	# send the logs to the master site
	send_logs "$log" "$err"

	# cleanup
	echo "Ending build of $tree.$compiler in process $$ at `date`"
	unlock_file "$lck"
}

#########################################################
# if you want to build only one project at a time
# add 'global_lock' after 'per_run_hook' and
# 'global_unlock' to the end of the file
global_lock() {
    lock_file "global.lck" || {
        exit 0
    }
}
global_unlock() {
    unlock_file "global.lck"
}

#########################################################
# enable this on a per host basis only when needed please
# (at least for the moment)
kill_old_processes() {
    # this should work on systems with linux like ps
    (ps uxfw | grep /build | grep -v grep | egrep 'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
    # and this should work on sysv style ps
    (ps -fu $USER | grep /build | grep -v grep | egrep 'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
}

delete_old_svn_tree() {
	otree=$1

	test -z "$otree" && return 0;

	rm -rf $otree
	rm -rf $otree.svn
	rm -rf $otree.*.svn
	rm -rf s4selftest.$otree.*
	rm -rf build.$otree.*
}

# this is a special fn that allows us to add a "special" hook to the build
# farm that we want to do to the build farm. never leave it empty. instead,
# use ":" as the fn body.
per_run_hook() {
    # kill old processes on systems with a known problem
    case $host in
	nohost)
	    echo "just a placeholder";
	    ;;
	tridge)
	    kill_old_processes
	    ;;
	deckchair)
	    rm -f deckchair.fns
	    ;;
    esac
    # trim the log if too large
    if [ "`wc -c < build.log`" -gt 2000000 ]; then
	rm -f build.log
    fi
}


######################################################
# main code that is run on each call to the build code
rsync --timeout=200 -q -az build.samba.org::build_farm/*.c .


# build.log can grow to an excessive size, trim it beyond 50M
if [ -f build.log ]; then
  find build.log -size +100000 -exec /bin/rm '{}' \;
fi