-# -*- mode: shell-script; sh-indentation: 8; indent-tabs-mode: t; -*-
+#!/bin/sh -*- mode: shell-script; sh-indentation: 8; indent-tabs-mode: t; -*-
# build_farm -- distributed build/test architecture for samba, rsync, etc
# Copyright (C) 2001, 2003 by Martin Pool <mbp@samba.org>
# default maximum runtime for any command
-MAXTIME=7200
+MAXTIME=12000
+# default maximum memory size (100M) for any command
+MAXMEM=100000
RUN_FROM_BUILD_FARM=yes
export RUN_FROM_BUILD_FARM
sum_tree_test_root=$1
sum_tree_tree=$2
sum_tree_sum=$3
- find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v '.svn' | grep -v version.h | sort | xargs sum > $sum_tree_sum
+ sum_tree_scm=$4
+ find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v version.h | sort | xargs sum > $sum_tree_sum
sum build_test build_test.fns >> $sum_tree_sum
if [ -f "$host.fns" ]; then
else
sum generic.fns >> $sum_tree_sum
fi
- if [ -f "$test_root/$tree.svn" ]; then
- sum "$test_root/$tree.svn" >> $sum_tree_sum
+
+ if [ -f "$test_root/$tree.$scm" ]; then
+ sum "$test_root/$tree.$scm" >> $sum_tree_sum
fi
+
for d in $deptrees; do
- if [ -f "$test_root/$d.svn" ]; then
- sum "$test_root/$d.svn" >> $sum_tree_sum
+ dscm=`choose_scm "$d"`
+ if [ -f "$test_root/$d.$dscm" ]; then
+ sum "$test_root/$d.$dscm" >> $sum_tree_sum
fi
done
}
shift
chmod 0644 "$log" "$err"
- XARGS_I="xargs -i"
- if [ "`uname`" = "FreeBSD" ]; then
- XARGS_I="xargs -I '{}' -R -1"
+ # xargs -i is implemented differently or not at all.
+ # GNU xargs did not implement "-I" until 4.2.9:
+ xargs --version 2>&1 | grep "^GNU xargs" > /dev/null
+ status=$?
+ if [ x"$status" = x"0" ]; then
+ XARGS_IS_GNU=yes
+ fi
+
+ if [ x"$XARGS_IS_GNU" = x"yes" ]; then
+ XARGS_I="xargs -i"
+ else
+ XARGS_I="xargs -I '{}'"
fi
+
find $log -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} && echo "\n***LOG TRUNCATED***" >> {}'
find $err -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} && echo "\n***LOG TRUNCATED***" >> {}'
- rsync $* -ct -q --password-file=.password -z --timeout=200 \
- "$log" "$err" $host@samba.org::build_farm_data/
+ rsync $* -c -q --password-file=.password -z --timeout=200 \
+ "$log" "$err" $host@build.samba.org::build_farm_data/
fi
}
echo "skipping tree transfer"
else
fetchtree=$1
- if rsync --exclude=autom4te.cache/ --exclude=.svn/ --delete-excluded -q --partial --timeout=200 -crlpz --delete --ignore-errors \
+ if rsync --exclude=autom4te.cache/ --exclude=.svn/ --exclude=.git/ \
+ --delete-excluded -q --partial --timeout=200 -ctrlpz --delete --ignore-errors \
samba.org::ftp/unpacked/$fetchtree/ $test_root/$fetchtree; then
echo "transferred $fetchtree OK"
else
}
############################
-# fetch the latest copy of the svn entries file
-fetch_svn() {
+# fetch the latest copy of the rev meta info
+fetch_revinfo() {
tree=$1
-# skip products still in CVS.
- case "$tree" in
- ccache | distcc | rsync)
- return 1
- ;;
- *)
- ;;
- esac
+ scm=$2
+
+ test -z "$scm" && return 1
+ test x"$scm" = x"unknown" && return 1
+ test x"$scm" = x"cvs" && return 1
+
if [ "$norsync" = "yes" ]; then
- echo "skipping svn transfer"
+ echo "skipping .revinfo.$scm transfer"
else
- if [ -r $test_root/$tree.svn ]; then
- rm -f $test_root/$tree.svn.old
- mv $test_root/$tree.svn $test_root/$tree.svn.old
+ if [ -r $test_root/$tree.$scm ]; then
+ rm -f $test_root/$tree.$scm.old
+ mv $test_root/$tree.$scm $test_root/$tree.$scm.old
fi
rsync -q --timeout=200 -clz --ignore-errors \
- samba.org::ftp/unpacked/$tree/.svn/entries $test_root/$tree.svn.tmp
- chmod u+w $test_root/$tree.svn.tmp
- sort -u < $test_root/$tree.svn.tmp > $test_root/$tree.svn
- rm -f $test_root/$tree.svn.tmp
+ samba.org::ftp/unpacked/$tree/.revinfo.$scm $test_root/$tree.$scm
fi
- if [ -r $test_root/$tree.svn ]; then
+ if [ -r $test_root/$tree.$scm ]; then
return 0;
fi
return 1
}
+############################
+# choose the scm that is used for the given project
+choose_scm() {
+ tree=$1
+
+ case "$tree" in
+ samba* | rsync | libreplace | talloc | tdb | ldb | pidl | ccache*)
+ echo "git"
+ return 0
+ ;;
+ esac
+
+ echo "svn"
+ return 0
+}
+
locknesting=0
############################
MMTIME=$MAXTIME
# some trees don't need as much time
case "$tree" in
- rsync | tdb | talloc | libreplace | ccache | distcc)
+ rsync | tdb | talloc | libreplace | ccache*)
if [ "$compiler" != "checker" ]; then
MMTIME=`expr $MMTIME / 5`
fi
for t in $*; do
if [ x"$BUILD_FARM_NUM_JOBS" = x ]; then
echo "$MAKE $t"
- ./timelimit $MMTIME "$MAKE" "$t"
+ $builddir/timelimit $MMTIME "$MAKE" "$t"
status=$?
else
# we can parallelize everything and all targets
if [ x"$t" = xeverything ] || [ x"$t" = xall]; then
echo "$MAKE" "-j$BUILD_FARM_NUM_JOBS" "$t"
- ./timelimit $MMTIME "$MAKE" "-j$BUILD_FARM_NUM_JOBS" "$t"
+ $builddir/timelimit $MMTIME "$MAKE" "-j$BUILD_FARM_NUM_JOBS" "$t"
status=$?
else
echo "$MAKE $t"
- ./timelimit $MMTIME "$MAKE" "$t"
+ $builddir/timelimit $MMTIME "$MAKE" "$t"
status=$?
fi
fi
+ if [ $status != 0 ]; then
+ case "$t" in
+ test | check | installcheck)
+ ;;
+ *)
+ # run again with V=1, so we see failed commands
+ $builddir/timelimit $MMTIME "$MAKE" "$t" V=1
+ status=$?
+ ;;
+ esac
+ fi
+
if [ $status != 0 ]; then
return $status;
fi
############################
# configure the tree
action_configure() {
+ if [ ! -x $srcdir/configure -a -r $srcdir/Makefile.PL ]; then
+ perl $srcdir/Makefile.PL PREFIX="$prefix"
+ cstatus=$?
+ echo "CONFIGURE STATUS: $cstatus"
+ return $cstatus;
+ fi
if [ ! -x $srcdir/configure ]; then
ls -l $srcdir/configure
echo "$srcdir/configure is missing"
echo CC="$CCACHE $compiler" $srcdir/configure $config_and_prefix
CC="$CCACHE $compiler"
export CC
- ./timelimit $MAXTIME $srcdir/configure $config_and_prefix
+ $builddir/timelimit $MAXTIME $srcdir/configure $config_and_prefix
cstatus=$?
- echo "CONFIGURE STATUS: $cstatus"
- if [ -f config.h ]; then
- echo "contents of config.h:"
- cat config.h
- fi
- if [ -f include/config.h ]; then
- echo "contents of include/config.h:"
- cat include/config.h
+ if [ x"$cstatus" != x"0" ]; then
+ if [ -f config.log ]; then
+ echo "contents of config.log:"
+ cat config.log
+ fi
+ if [ -f bin/config.log ]; then
+ echo "contents of config.log:"
+ cat bin/config.log
+ fi
fi
+ echo "CONFIGURE STATUS: $cstatus"
return $cstatus;
}
############################
# show the configure log
action_config_log() {
- if [ ! -f config.log ]; then
+ log_files="config.log bin/config.log"
+ for f in $log_files; do
+ if [ -f $f ]; then
+ echo "contents of config.log:"
+ cat $f
return 0;
fi
- echo "contents of config.log:"
- cat config.log
- return 0;
+ done
+ return 0
}
+############################
+# show the config.h
+action_config_header() {
+ hdr_files="config.h include/config.h bin/default/config.h bin/default/source4/include/config.h bin/default/source3/include/config.h"
+ for h in $hdr_files; do
+ if [ -f $h ]; then
+ echo "contents of $h:"
+ cat $h
+ return 0;
+ fi
+ done
+ return 0;
+}
+
+copy_dir() {
+ Tsrc=$1
+ Tdst=$2
+ pwd
+ echo rsync -a --delete $Tsrc/ $Tdst
+ rsync -a --delete $Tsrc/ $Tdst || return 1
+ return 0
+}
+
+
############################
# build the tree
action_build() {
case "$tree" in
- samba4)
- rm -f bin/smbtorture
+ samba_4*)
do_make everything
bstatus=$?
- if test x"$bstatus" != x"0"; then
- # the 2nd 'make everything' is to work around a bug
- # in netbsd make.
- do_make everything
- bstatus=$?
- fi
-
- if [ -n "$smbtorture4" ]; then
- if [ ! -f bin/smbtorture ]; then
- do_make bin/smbtorture
- fi
- if [ -f bin/smbtorture ]; then
- cp bin/smbtorture $smbtorture4
- fi
- fi
-
;;
- samba|samba_3_0)
- do_make proto everything torture
+ samba_3*)
+ do_make everything torture
bstatus=$?
;;
*)
action_test_samba() {
do_make test
totalstatus=$?
+ # if we produced a test summary then show it
+ [ -f st/summary ] && {
+ echo "TEST SUMMARY"
+ cat st/summary
+ }
return "$totalstatus"
}
# compatiblity. I think a better way to do this in the future
# is to just call 'make installcheck'.
case "$tree" in
-# samba_3_0)
-# echo "testing of samba_3_0 disabled until cause of runaway processes found (tridge - 7th sep 2006)"
-# ;;
samba*|smb-build|pidl)
action_test_samba
;;
shift
shift
shift
- if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && ccache -V > /dev/null; then
+ if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && [ "$tree" != "ccache-maint" ] && ccache -V > /dev/null; then
CCACHE="ccache"
export CCACHE
else
# limit our resource usage
ulimit -t $MAXTIME 2> /dev/null
-
+
# max mem size 100M
- ulimit -m 100000 2> /dev/null
+ ulimit -m $MAXMEM 2> /dev/null
# max file size 100M
# darn, this affects sparse files too! disable it
# ulimit -f 100000 2> /dev/null
- # try and limit the number of open files to 150. That means we'll discover
+ # try and limit the number of open files to 250. That means we'll discover
# fd leaks faster
- ulimit -n 150 2> /dev/null
+ ulimit -n 250 2> /dev/null
# Keep stuff private
umask 077
# work out what other trees this package depends on
deptrees=""
case "$tree" in
- talloc | tdb)
- deptrees="libreplace";
- ;;
- ldb)
- deptrees="libreplace talloc tdb";
- ;;
- samba-gtk)
- deptrees="samba4"
+ samba-gtk)
+ deptrees="samba_4_0_test"
;;
esac
- # pull the svn entries, if any
- if fetch_svn "$tree"; then
+ scm=`choose_scm "$tree"`
+
+ # pull the entries, if any
+ if fetch_revinfo "$tree" "$scm"; then
for d in $deptrees; do
- if [ -f "$test_root/$d.svn" ]; then
+ dscm=`choose_scm "$d"`
+ if [ -f "$test_root/$d.$dscm" ]; then
if [ "$d" != "$tree" ]; then
- cat "$test_root/$d.svn" >> $test_root/$tree.svn
+ cat "$test_root/$d.$dscm" >> $test_root/$tree.$scm
fi
fi
done
- rm -f $test_root/$tree.$compiler.svn.old
- mv $test_root/$tree.$compiler.svn $test_root/$tree.$compiler.svn.old
- cp $test_root/$tree.svn $test_root/$tree.$compiler.svn
- if cmp $test_root/$tree.$compiler.svn $test_root/$tree.$compiler.svn.old > /dev/null; then
- echo "skip: $tree.$compiler nothing changed in svn"
+ rm -f $test_root/$tree.$compiler.$scm.old
+ mv $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old
+ cp $test_root/$tree.$scm $test_root/$tree.$compiler.$scm
+ if cmp $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old > /dev/null; then
+ echo "skip: $tree.$compiler nothing changed in $scm"
cd $test_root
send_logs_skip "$log" "$err"
unlock_file "$lck"
echo "Starting build of $tree.$compiler in process $$ at `date`"
case "$tree" in
- tdb | talloc | ldb | libreplace)
+ libreplace)
builddir="$test_root/tmp.$tree.$compiler"
usingtmpbuild=1
if [ -d $builddir ]; then
;;
esac
- if [ ! $USER = "" ]; then
+ if [ ! x$USER = x"" ]; then
whoami=$USER
else
- if [ ! $LOGNAME = "" ]; then
+ if [ ! x$LOGNAME = x"" ]; then
whoami=$LOGNAME
else
whoami=build
fi
fi
+ # build the timelimit utility
+ echo "Building timelimit"
+ mkdir -p $builddir
+ echo $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c
+ $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1
+
+ # build the killbysubdir utility
+ echo "Building killbysubdir"
+ echo $compiler -o $builddir/killbysubdir $test_root/killbysubdir.c
+ $compiler -o $builddir/killbysubdir $test_root/killbysubdir.c
+
prefix="$test_root/prefix/$tree.$compiler"
mkdir -p "$prefix"
- smbtorture4=$test_root/smbtorture4
- export smbtorture4
-
sw_config=$config
case "$tree" in
- samba4|lorikeet-heimdal)
- sw_config="$config --enable-socket-wrapper"
- ;;
- samba|samba_3_0)
- sw_config="$config --enable-socket-wrapper"
- if [ -f $smbtorture4 ]; then
- # we create a local copy to make sure the same binary is used for all tests
- cp $smbtorture4 $smbtorture4.$tree
- sw_config="$sw_config --with-smbtorture4-path=$smbtorture4.$tree"
- fi
- ;;
+ lorikeet-heimdal)
+ sw_config="$config --enable-socket-wrapper"
+ ;;
+ samba_4*)
+ sw_config="$config --enable-socket-wrapper"
+ sw_config="$sw_config --enable-nss-wrapper"
+ sw_config="$sw_config --enable-uid-wrapper"
+ ;;
+ samba_3*)
+ sw_config="$config --enable-socket-wrapper"
+ sw_config="$sw_config --enable-nss-wrapper"
+ ;;
samba-gtk)
- PKG_CONFIG_PATH="$test_root/prefix/samba4.$compiler/lib/pkgconfig"
+ PKG_CONFIG_PATH="$test_root/prefix/samba_4_0_test.$compiler/lib/pkgconfig"
export PKG_CONFIG_PATH
;;
- ldb)
- fetch_tree popt
- ;;
- talloc)
- fetch_tree libreplace
- ;;
*)
- testsuite=testsuite
- ;;
+ testsuite=testsuite
+ ;;
esac
if [ "$LCOV_REPORT" = "yes" ]; then
- GCOV_FLAGS="-ftest-coverage -fprofile-arcs"
- GCOV_LIBS="-lgcov"
- HOSTCC_CFLAGS="$HOSTCC_CFLAGS $GCOV_FLAGS"
+ GCOV_FLAGS="--coverage"
CFLAGS="$CFLAGS $GCOV_FLAGS"
- LDFLAGS="$LDFLAGS $GCOV_FLAGS $GCOV_LIBS"
- SHLD_FLAGS="$SHLD_FLAGS $GCOV_FLAGS $GCOV_LIBS"
- export HOSTCC_CFLAGS CFLAGS LDFLAGS SHLD_FLAGS
+ LDFLAGS="$LDFLAGS $GCOV_FLAGS"
+ export CFLAGS LDFLAGS
fi
config_and_prefix="$sw_config --prefix=$prefix"
# see if we need to rebuild
- sum_tree $test_root $tree $sum
+ sum_tree $test_root $tree $sum $scm
echo "CFLAGS=$CFLAGS $config_and_prefix" >> $sum
if cmp "$sum" "$sum.old" > /dev/null; then
actions="$*"
if [ "$actions" = "" ]; then
- actions="configure config_log build install test"
+ actions="configure config_log config_header build install test"
fi
# start the build
(
+ {
# we all want to be able to read the output...
LANG=C
export LANG
- echo $build_test_id
- echo $build_test_fns_id
- echo $build_test_used_fns_file
-
uname -a
+ echo ""
+ echo "build_test : $build_test_id"
+ echo "build_test.fns : $build_test_fns_id"
+ echo "local settings file : $build_test_settings_local_file"
+ echo "local functions file: $build_test_fns_local_file"
+ echo "used .fns file : $build_test_used_fns_file"
+ echo ""
+
# we need to be able to see if a build farm machine is accumulating
# stuck processes. We do this in two ways, as we don't know what style
# of ps it will have
echo "Showing limits"
ulimit -a 2> /dev/null
- # build the timelimit utility
- echo "Building timelimit"
- mkdir -p $builddir
- $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1
-
- if [ -r $test_root/$tree.svn ]; then
- h_rev=`grep revision= $test_root/$tree.svn | cut -d'"' -f2 | sort -n | tail -1`
+ # the following is for non-samba builds only
+ if [ "$scm" = "svn" -a -r $test_root/$tree.svn ]; then
+ h_rev=`grep 'Revision: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
if [ -n "$h_rev" ]; then
echo "HIGHEST SVN REVISION: $h_rev"
fi
- rev=`grep committed-rev= $test_root/$tree.svn | cut -d'"' -f2 | sort -n | tail -1`
+ rev=`grep 'Last Changed Rev: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
if [ -n "$rev" ]; then
echo "BUILD REVISION: $rev"
fi
+ elif [ "$scm" = "git" -a -r $test_root/$tree.git ]; then
+ csha1=`cat $test_root/$tree.git |head -3 | tail -1`
+ if [ -n "$csha1" ]; then
+ echo "BUILD COMMIT REVISION: $csha1"
+ fi
+ cdate=`cat $test_root/$tree.git |head -4 | tail -1`
+ if [ -n "$cdate" ]; then
+ echo "BUILD COMMIT DATE: $cdate"
+ fi
+ ctime=`cat $test_root/$tree.git |head -2 | tail -1`
+ if [ -n "$ctime" ]; then
+ echo "BUILD COMMIT TIME: $ctime"
+ fi
fi
-
- if [ "$tree" = "pidl" ]
- then
- cd $builddir
- perl ./Makefile.PL "$prefix"
+ if [ -x $builddir/killbysubdir ]; then
+ echo "$builddir/killbysubdir $builddir in `pwd`"
+ $builddir/killbysubdir $builddir
fi
for action in $actions; do
( action_$action )
action_status=$?
-
+
+ df .
+
if [ $action_status != 0 ]; then
echo "ACTION FAILED: $action";
else
lorikeet-heimdal*)
lcov --directory $builddir --capture --output-file $builddir/$tree.lcov.info
;;
- *)
+ samba_3_master*)
+ lcov --base-directory $builddir --directory $builddir/.. --capture --output-file $builddir/$tree.lcov.info
+ ;;
+ samba_4*)
# ugly hack for s4, as lcov is otherwise not able to find
# these files
- rm -f heimdal/lib/*/{lex,parse}.{gcda,gcno}
+ rm -f heimdal/lib/*/{lex,parse,sel-lex}.{gcda,gcno}
+ lcov --base-directory $builddir --directory $builddir/.. --capture --output-file $builddir/$tree.lcov.info
+ ;;
+ *)
lcov --base-directory $builddir --directory $builddir --capture --output-file $builddir/$tree.lcov.info
;;
esac
do_make clean
fi
date
- ) > "$log" 2> "$err"
+ } 3>&2 2>&1 1>&3 | tee "$err"
+ ) > "$log" 2>&1
+ # be aware the above channel swap may sometimes result in unordered
+ # stdout/stderr merge
if [ "$LCOV_REPORT" = "yes" ]; then
chmod u=rwX,g=rX,o=rX -R $builddir/coverage
rsync -rct -q --password-file=.password -z --timeout=200 \
- $builddir/coverage/ $host@samba.org::lcov_data/$host/$tree/
+ $builddir/coverage/ $host@build.samba.org::lcov_data/$host/$tree/
fi
cd $test_root
unlock_file "global.lck"
}
-#########################################################
-# enable this on a per host basis only when needed please
-# (at least for the moment)
-kill_old_processes() {
- # this should work on systems with linux like ps
- (ps uxfw | grep /build | grep -v grep | egrep 'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
- # and this should work on sysv style ps
- (ps -fu $USER | grep /build | grep -v grep | egrep 'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
-}
+delete_old_tree() {
+ otree=$1
+
+ test -z "$otree" && return 0;
+ rm -rf $otree
+ rm -rf $otree.svn
+ rm -rf $otree.*.svn
+ rm -rf $otree.git
+ rm -rf $otree.*.git
+ rm -rf build.$otree.*
+}
# this is a special fn that allows us to add a "special" hook to the build
# farm that we want to do to the build farm. never leave it empty. instead,
nohost)
echo "just a placeholder";
;;
- tridge)
- kill_old_processes
- ;;
deckchair)
rm -f deckchair.fns
;;
if [ "`wc -c < build.log`" -gt 2000000 ]; then
rm -f build.log
fi
+
+ old_trees="web popt distcc samba-gtk smb-build lorikeet-heimdal samba_3_2 samba_3_2_test samba4 samba_4_0_waf samba_4_0_waf.metze samba_3_X_test samba_3_X_devel samba_3_X_devel"
+ for d in $old_trees; do
+ delete_old_tree $d
+ done
}
######################################################
# main code that is run on each call to the build code
-rsync --timeout=200 -q -az samba.org::build_farm/*.c .
+rsync --timeout=200 -q -az build.samba.org::build_farm/*.c .
# build.log can grow to an excessive size, trim it beyond 50M
git.samba.org / build-farm.git / blobdiff