added waf trunk to the build farm
[amitay/build-farm.git] / build_test.fns
1 #!/bin/sh -*- mode: shell-script; -*-
2
3 # build_farm -- distributed build/test architecture for samba, rsync, etc
4
5 # Copyright (C) 2001 by Andrew Tridgell <tridge@samba.org>
6 # Copyright (C) 2001 by Andrew Bartlett <abartlet@samba.org>
7 # Copyright (C) 2001, 2003 by Martin Pool <mbp@samba.org>
8
9 # default maximum runtime for any command
10 MAXTIME=25200 # 7 hours
11 SMBD_MAXTIME=18000 # 5 hours for a samba process ..
12 # default maximum memory size (100M) for any command
13 MAXMEM=100000
14 RUN_FROM_BUILD_FARM=yes
15 export RUN_FROM_BUILD_FARM
16 export MAXTIME SMBD_MAXTIME
17
18 deptrees="";
19
20 build_test_fns_id='$Id$'
21
22 copy_dir() {
23         Tsrc=$1
24         Tdst=$2
25         pwd
26         echo rsync -a --delete $Tsrc/ $Tdst
27         rsync -a --delete $Tsrc/ $Tdst || return 1
28         return 0
29 }
30
31 #############################
32 # build a signature of a tree, used to see if we
33 # need to rebuild 
34 #############################
35
36 sum_tree() {
37         sum_tree_test_root=$1
38         sum_tree_tree=$2
39         sum_tree_sum=$3
40         sum_tree_scm=$4
41         find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v version.h | sort | xargs sum > $sum_tree_sum
42         sum build_test build_test.fns >> $sum_tree_sum
43
44         if [ -f "$host.fns" ]; then
45                 sum $host.fns >> $sum_tree_sum
46         else
47                 sum generic.fns >> $sum_tree_sum
48         fi
49
50         if [ -f "$test_root/$tree.$scm" ]; then
51                 sum "$test_root/$tree.$scm" >> $sum_tree_sum
52         fi
53
54         for d in $deptrees; do
55                 dscm=`choose_scm "$d"`
56                 if [ -f "$test_root/$d.$dscm" ]; then
57                 sum "$test_root/$d.$dscm" >> $sum_tree_sum
58                 fi
59         done
60 }
61
62 #############################
63 # send the logs to the master site
64 #############################
65
66 send_logs() {
67         if [ "$nologreturn" = "yes" ]; then
68                 echo "skipping log transfer"
69         else
70                 log="$1"
71                 err="$2"
72                 shift
73                 shift
74                 chmod 0644 "$log" "$err"
75
76                 # xargs -i is implemented differently or not at all.
77                 # GNU xargs did not implement "-I" until 4.2.9:
78                 xargs --version 2>&1 | grep "^GNU xargs" > /dev/null
79                 status=$?
80                 if [ x"$status" = x"0" ]; then
81                         XARGS_IS_GNU=yes
82                 fi
83
84                 if [ x"$XARGS_IS_GNU" = x"yes" ]; then
85                         XARGS_I="xargs -i"
86                 else
87                         XARGS_I="xargs -I '{}'"
88                 fi
89
90                 find $log -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'
91                 find $err -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'
92
93                 rsync $* -c -q --password-file=.password -z --timeout=200 \
94                         "$log" "$err" $host@build.samba.org::build_farm_data/
95         fi
96 }
97
98 #############################
99 # send the logs when they haven't changed
100 # the aim is to just update the servers timestamp.
101 # sending with a very large rsync block size does this
102 # with minimal network traffic
103 #############################
104
105 send_logs_skip() {
106         touch "$1" "$2"
107         send_logs "$1" "$2" -B 10000000
108 }
109
110 ############################
111 # fetch the latest copy of the tree
112 ############################
113
114 fetch_tree() {
115         if [ "$norsync" = "yes" ]; then
116                 echo "skipping tree transfer"
117         else
118                 fetchtree=$1
119                 if rsync --exclude=autom4te.cache/ --exclude=.svn/ --exclude=.git/ \
120                         --delete-excluded -q --partial --timeout=200 -ctrlpz --delete --ignore-errors \
121                         samba.org::ftp/unpacked/$fetchtree/ $test_root/$fetchtree; then
122                         echo "transferred $fetchtree OK"
123                 else
124                         echo "transfer of $fetchtree failed code $?"
125                         return 1
126                 fi
127         fi
128         return 0
129 }
130
131 ############################
132 # fetch the latest copy of the rev meta info
133 ############################
134
135 fetch_revinfo() {
136         tree=$1
137         scm=$2
138
139         test -z "$scm" && return 1
140         test x"$scm" = x"unknown" && return 1
141         test x"$scm" = x"cvs" && return 1
142
143         if [ "$norsync" = "yes" ]; then
144                 echo "skipping .revinfo.$scm transfer"
145         else
146                 if [ -r $test_root/$tree.$scm ]; then
147                         [ -f $test_root/$tree.$scm.old ] && rm -f $test_root/$tree.$scm.old
148                         [ -f $test_root/$tree.$scm ] && mv $test_root/$tree.$scm $test_root/$tree.$scm.old
149                 fi
150                 rsync -q --timeout=200 -clz --ignore-errors \
151                         samba.org::ftp/unpacked/$tree/.revinfo.$scm $test_root/$tree.$scm
152         fi
153         if [ -r $test_root/$tree.$scm ]; then
154                 return 0;
155         fi
156         return 1
157 }
158
159 ############################
160 # choose the scm that is used for the given project
161 ############################
162
163 choose_scm() {
164         tree=$1
165
166         case "$tree" in
167                         samba* | rsync | libreplace | talloc | tdb | ldb | pidl | ccache* | waf*)
168                         echo "git"
169                                 return 0
170                         ;;
171         esac
172
173         echo "svn"
174         return 0
175 }
176
177 locknesting=0
178
179 ############################
180 # grab a lock file. Not atomic, but close :)
181 # tries to cope with NFS
182 ############################
183
184 lock_file() {
185                 if [ -z "$lock_root" ]; then
186                         lock_root=`pwd`;
187                 fi
188
189                 lckf="$lock_root/$1"
190                 machine=`cat "$lckf" 2> /dev/null | cut -d: -f1`
191                 pid=`cat "$lckf" 2> /dev/null | cut -d: -f2`
192
193                 if [ "$pid" = "$$" ]; then
194                         locknesting=`expr $locknesting + 1`
195                         echo "lock nesting now $locknesting"
196                         return 0
197                 fi
198
199                 if test -f "$lckf"; then
200                         test x$machine = x$host || {
201                                 echo "lock file $lckf is valid for other machine $machine"
202                                 return 1
203                         }
204
205                         kill -0 $pid && {
206                                 echo "lock file $lckf is valid for process $pid"
207                                 return 1
208                         }
209
210                         echo "stale lock file $lckf for $machine:$pid"
211                         cat "$lckf"
212                         /bin/rm -f "$lckf"
213                 fi
214                 echo "$host:$$" > "$lckf"
215                 return 0
216 }
217
218 ############################
219 # unlock a lock file
220 ############################
221
222 unlock_file() {
223         if [ -z "$lock_root" ]; then
224                 lock_root=`pwd`;
225         fi
226         if [ "$locknesting" != "0" ]; then
227                 locknesting=`expr $locknesting - 1`
228                 echo "lock nesting now $locknesting"
229         else 
230                 lckf="$lock_root/$1"
231                 /bin/rm -f "$lckf"
232         fi
233 }
234
235 ############################
236 # run make, and print trace
237 ############################
238
239 do_make() {
240         # work out correct make command
241         case "$tree" in
242             waf*)
243                 MAKECOMMAND="./waf"
244                 ;;
245             *)
246                 MAKECOMMAND="$MAKE"
247                 if [ x"$MAKECOMMAND" = x ]; then
248                     MAKECOMMAND=make
249                 fi
250                 ;;
251         esac
252
253         MMTIME=$MAXTIME
254         # some trees don't need as much time
255         case "$tree" in
256                 rsync | tdb | talloc | libreplace | ccache* | waf*)
257                         if [ "$compiler" != "checker" ]; then
258                                 MMTIME=`expr $MMTIME / 5`
259                         fi
260                 ;;
261         esac
262
263         # special build for some trees
264         case "$tree" in
265             waf*)
266                 ./waf distclean && ./waf configure build
267                 ;;
268         esac
269
270
271         for t in $*; do
272                 if [ x"$BUILD_FARM_NUM_JOBS" = x ]; then
273                         echo "$MAKECOMMAND $t"
274                         $builddir/timelimit $MMTIME "$MAKECOMMAND" "$t"
275                         status=$?
276                 else
277                         # we can parallelize everything and all targets
278                         if [ x"$t" = xeverything ] || [ x"$t" = xall]; then
279                                 echo "$MAKECOMMAND" "-j$BUILD_FARM_NUM_JOBS"  "$t"
280                                 $builddir/timelimit $MMTIME "$MAKECOMMAND" "-j$BUILD_FARM_NUM_JOBS"  "$t"
281                                 status=$?
282                         else
283                                 echo "$MAKECOMMAND $t"
284                                 $builddir/timelimit $MMTIME "$MAKECOMMAND" "$t"
285                                 status=$?
286                         fi
287                 fi
288
289                 if [ $status != 0 ]; then
290                         case "$t" in
291                                 test|check|installcheck)
292                                         ;;
293                                 *)
294                                         #run again with V=1, so we see failed commands
295                                         $builddir/timelimit $MMTIME "$MAKECOMMAND" "$t" V=1
296                                         status=$?
297                                         ;;
298                         esac
299                 fi
300
301                 if [ $status != 0 ]; then
302                         return $status;
303                 fi
304
305         done
306
307         return 0
308 }
309
310
311 ############################
312 # do the coverage report
313 ############################
314
315 action_lcovreport() {
316         if [ "$LCOV_REPORT" = "yes" ]; then
317                 case "$tree" in
318                 lorikeet-heimdal*)
319                         lcov --directory $builddir --capture --output-file $builddir/$tree.lcov.info
320                         ;;
321                 samba_3_master*)
322                         lcov --base-directory $builddir --directory $builddir/.. --capture --output-file $builddir/$tree.lcov.info
323                         ;;
324                 samba_4*)
325                         # rm -f heimdal/lib/*/{lex,parse,sel-lex}.{gcda,gcno}
326                         lcov --base-directory $builddir --directory $builddir/.. --capture --output-file $builddir/$tree.lcov.info
327                         ;;
328                 *)
329                         lcov --base-directory $builddir --directory $builddir --capture --output-file $builddir/$tree.lcov.info
330                         ;;
331                 esac
332                 genhtml -o $builddir/coverage $builddir/$tree.lcov.info
333                 rc=$?
334                 echo "return code: $rc"
335         else
336                 echo "LCOV_REPORT not set and lcovreport asked"
337                 echo "Most probably an error please fix !"
338                 return 1
339         fi
340 }
341
342
343 ############################
344 # configure the tree
345 ############################
346
347 action_configure() {
348
349         case "$tree" in
350             waf*)
351                 $builddir/timelimit $MAXTIME ./waf configure
352                 cstatus=$?
353                 echo "CONFIGURE STATUS: $cstatus"
354                 return $cstatus
355                 ;;
356         esac
357
358         if [ ! -x $srcdir/configure -a -r $srcdir/Makefile.PL ]; then
359                 perl $srcdir/Makefile.PL PREFIX="$prefix"
360                 cstatus=$?
361                 echo "CONFIGURE STATUS: $cstatus"
362                 return $cstatus;
363         fi
364
365         if [ ! -x $srcdir/configure ]; then
366                 ls -l $srcdir/configure
367                 echo "$srcdir/configure is missing"
368                 cstatus=255
369                 echo "CONFIGURE STATUS: $cstatus"
370                 return $cstatus;
371         fi
372
373         echo "CFLAGS=$CFLAGS"
374         echo configure options: $config_and_prefix
375         echo CC="$CCACHE $compiler" $srcdir/configure $config_and_prefix
376
377         CC="$CCACHE $compiler"
378         export CC
379         $builddir/timelimit $MAXTIME $srcdir/configure $config_and_prefix
380         cstatus=$?
381
382         if [ x"$cstatus" != x"0" ]; then
383                 if [ -f config.log ]; then
384                         echo "contents of config.log:"
385                         cat config.log
386                 fi
387
388                 # Waf style
389                 if [ -f bin/config.log ]; then
390                         echo "contents of config.log:"
391                         cat bin/config.log
392                 fi
393         fi
394         echo "CONFIGURE STATUS: $cstatus"
395         return $cstatus
396 }
397
398 ############################
399 # show the configure log
400 ############################
401
402 action_config_log() {
403
404         log_files="config.log bin/config.log"
405         for f in $log_files; do
406                 if [ -f $f ]; then
407                         echo "contents of config.log:"
408                         cat $f
409                         return 0
410                 fi
411         done
412         return 0
413 }
414
415 ############################
416 # show the config.h
417 ############################
418
419 action_config_header() {
420         hdr_files="config.h include/config.h bin/default/config.h bin/default/source4/include/config.h bin/default/source3/include/config.h"
421         for h in $hdr_files; do
422                 if [ -f $h ]; then
423                         echo "contents of $h:"
424                         cat $h
425                         return 0
426                 fi
427         done
428
429         return 0
430 }
431
432
433
434 ############################
435 # build the tree
436 ############################
437 action_build() {
438         case "$tree" in
439         samba_4*)
440                 do_make everything
441                 bstatus=$?
442                 ;;
443         samba_3*)
444                 do_make everything torture
445                 bstatus=$?
446                 ;;
447         waf*)
448                 do_make build
449                 ;;
450         *)
451                 do_make all
452                 bstatus=$?
453                 ;;
454         esac
455
456         echo "BUILD STATUS: $bstatus"
457
458         return $bstatus
459 }
460
461 ############################
462 # show static analysis results
463 ############################
464
465 action_cc_checker() {
466
467         # default to passing the cc_checker
468         cccstatus=0
469
470         if [ -f ibm_checker.out ]; then
471                 cat ibm_checker.out
472                 cccstatus=`cat ibm_checker.out | grep '^\-\- ' | wc -l`
473         fi
474
475         echo "CC_CHECKER STATUS: $cccstatus"
476         return $cccstatus;      
477 }
478
479 ############################
480 # install the tree
481 ############################
482
483 action_install() {
484         if [ -d $prefix ]; then
485                 if [ "$noclean" != "yes" ]; then
486                         rm -rf $prefix
487                 fi
488         fi
489
490         do_make install
491         istatus=$?
492         echo "INSTALL STATUS: $istatus"
493         return $istatus;
494 }
495
496 ############################
497 # test the tree
498 action_test_samba() {
499         do_make test
500         totalstatus=$?
501
502         # if we produced a test summary then show it
503         [ -f st/summary ] && {
504                 echo "TEST SUMMARY"
505                 cat st/summary
506         }
507
508         return "$totalstatus"
509 }
510
511 action_test_generic() {
512         CC="$compiler"
513         export CC
514         do_make installcheck
515         totalstatus=$?
516         echo "TEST STATUS: $totalstatus"
517         return "$totalstatus"
518 }
519
520 action_test_lorikeet_heimdal() {
521         CC="$compiler"
522         export CC
523         SOCKET_WRAPPER_DIR=`pwd`/sw
524         mkdir $SOCKET_WRAPPER_DIR
525         export SOCKET_WRAPPER_DIR
526         do_make check
527         totalstatus=$?
528         SOCKET_WRAPPER_DIR=
529         export SOCKET_WRAPPER_DIR
530         echo "TEST STATUS: $totalstatus"
531         return "$totalstatus"
532 }
533
534
535 #############################
536 # attempt some basic tests of functionaility
537 # starting as basic as possible, and getting incresingly complex
538 #############################
539
540 action_test() {
541         # Samba needs crufty code of its own for backward
542         # compatiblity.  I think a better way to do this in the future
543         # is to just call 'make installcheck'.
544         case "$tree" in
545         samba*|smb-build|pidl)
546                 action_test_samba
547                 ;;
548         lorikeet-heimdal*)
549                 action_test_lorikeet_heimdal
550                 ;;
551         *)
552                 action_test_generic
553                 ;;
554         esac
555 }
556
557 ###########################
558 # do a test build of a particular tree
559 # This is the master function called by generic.fns or
560 # host.fns
561 ###########################
562
563 test_tree() {
564         tree=$1
565         source=$2
566         compiler="$3"
567         shift
568         shift
569         shift
570         echo "Starting to deal with tree $tree with compiler $compiler"
571         if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && [ "$tree" != "ccache-maint" ] && ccache -V > /dev/null 2>/dev/null; then
572                 CCACHE="ccache"
573                 export CCACHE
574         else
575                 CCACHE=""
576         fi
577
578         # limit our resource usage
579         ulimit -t $MAXTIME 2> /dev/null
580
581         # max mem size 100M
582         ulimit -m $MAXMEM 2> /dev/null
583
584         # max file size 100M
585         # darn, this affects sparse files too! disable it
586         # ulimit -f 100000 2> /dev/null
587
588         # try and limit the number of open files to 250. That means we'll discover
589         # fd leaks faster
590         ulimit -n 250 2> /dev/null
591
592         # Keep stuff private
593         umask 077
594
595         if [ -z "$test_root" ]; then
596                 test_root=`pwd`
597         fi
598
599         log="build.$tree.$host.$compiler.log"
600         err="build.$tree.$host.$compiler.err"
601         sum="build.$tree.$host.$compiler.sum"
602         lck="build.$tree.lck"
603                 srcdir="$test_root/$tree/$source"
604
605         lock_file "$lck" || {
606                 return
607         }
608
609         # work out what other trees this package depends on
610         deptrees=""
611         case "$tree" in
612                 samba-gtk)
613                 deptrees="samba_4_0_test"
614                 ;;
615         esac
616
617         scm=`choose_scm "$tree"`
618
619         # pull the entries, if any
620         # Remove old .svn or .git files
621         # Move the current .svn org .git to .svn.old or
622         # .git.old then fetch the new from rsync
623         if fetch_revinfo "$tree" "$scm"; then
624                 for d in $deptrees; do
625                         # If there is dependency substree(s) we add info
626                         # from the dependency tree so that we
627                         # can rebuild in case one of them has changed
628                         dscm=`choose_scm "$d"`
629                         if [ -f "$test_root/$d.$dscm" ]; then
630                                 if [ "$d" != "$tree" ]; then
631                                         cat "$test_root/$d.$dscm" >> $test_root/$tree.$scm
632                                 fi
633                         fi
634                 done
635                 [ -f $test_root/$tree.$compiler.$scm.old ] && rm -f $test_root/$tree.$compiler.$scm.old
636                 [ -f $test_root/$tree.$compiler.$scm ] && mv $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old
637                 [ -f $test_root/$tree.$scm ] && cp $test_root/$tree.$scm $test_root/$tree.$compiler.$scm
638
639                 if [ -f $test_root/$tree.$compiler.$scm.old ] && \
640                                 cmp $test_root/$tree.$compiler.$scm $test_root/$tree.$compiler.$scm.old > /dev/null; then
641
642                         echo "skip: $tree.$compiler nothing changed in $scm"
643                         cd $test_root
644                         send_logs_skip "$log" "$err"
645                         unlock_file "$lck"
646                         return
647                 fi
648         fi
649
650         # pull the tree
651         fetch_tree "$tree" || {
652                 cd $test_root
653                 unlock_file "$lck"
654                 return
655         }
656
657         # check for essential files
658         case "$tree" in
659             pidl)
660                 # no generated files
661                 ;;
662             waf*)
663                 if [ ! -x $srcdir/waf ]; then
664                     echo "skip: $tree.$compiler waf not present, try again next time!"
665                     cd $test_root
666                     unlock_file "$lck"
667                     return
668                 fi
669                 ;;
670             *)
671                 if [ ! -x $srcdir/configure ]; then
672                     echo "skip: $tree.$compiler configure not present, try again next time!"
673                     cd $test_root
674                     unlock_file "$lck"
675                     return
676                 fi
677                 ;;
678         esac
679
680         echo "Starting build of $tree.$compiler in process $$ at `date`"
681
682
683         # Parameters for the build depending on the tree
684         case "$tree" in
685                 *)
686                         builddir=$srcdir
687                         export builddir
688                         ;;
689         esac
690
691         #Fix the user
692         if [ ! x$USER = x"" ]; then
693                 whoami=$USER
694         else 
695                 if [ ! x$LOGNAME = x"" ]; then
696                         whoami=$LOGNAME
697                 else
698                         whoami=build
699                 fi
700         fi
701
702         # build the timelimit utility
703         echo "Building timelimit"
704         mkdir -p $builddir
705         echo $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c
706         $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1
707
708         # build the killbysubdir utility
709         echo "Building killbysubdir"
710         echo $compiler -o $builddir/killbysubdir $test_root/killbysubdir.c
711         $compiler -o $builddir/killbysubdir $test_root/killbysubdir.c
712
713         prefix="$test_root/prefix/$tree.$compiler"
714         mkdir -p "$prefix"
715
716         # This can be defined in <host>.fns files
717         sw_config=$config
718
719         case "$tree" in
720         lorikeet-heimdal)
721                 sw_config="$config --enable-socket-wrapper"
722                 ;;
723         samba_4*)
724                 sw_config="$config --enable-socket-wrapper"
725                 sw_config="$sw_config --enable-nss-wrapper"
726                 sw_config="$sw_config --enable-uid-wrapper"
727                 ;;
728         samba_3*)
729                 sw_config="$config --enable-socket-wrapper"
730                 sw_config="$sw_config --enable-nss-wrapper"
731                 ;;
732         samba-gtk)
733                 PKG_CONFIG_PATH="$test_root/prefix/samba_4_0_test.$compiler/lib/pkgconfig"
734                 export PKG_CONFIG_PATH
735                 ;;
736         *)
737                 testsuite=testsuite
738                 ;;
739         esac
740
741         if [ "$LCOV_REPORT" = "yes" ]; then
742                 GCOV_FLAGS="--coverage"
743                 CFLAGS="$CFLAGS $GCOV_FLAGS" 
744                 LDFLAGS="$LDFLAGS $GCOV_FLAGS" 
745                 export CFLAGS LDFLAGS
746         fi
747
748         config_and_prefix="$sw_config --prefix=$prefix"
749
750         # see if we need to rebuild
751         sum_tree $test_root $tree $sum $scm
752         echo "CFLAGS=$CFLAGS $config_and_prefix" >> $sum
753
754         if [ -f "$sum.old" ] && cmp "$sum" "$sum.old" > /dev/null; then
755                 echo "skip: $tree.$compiler nothing changed"
756                 cd $test_root
757                 send_logs_skip "$log" "$err"
758                 unlock_file "$lck"
759                 echo "Ending build of $tree.$compiler in process $$ at `date`"
760                 return
761         fi
762
763         # we do need to rebuild - save the old sum
764         [ -f $sum.old ] && /bin/rm -f $sum.old
765         mv $sum $sum.old
766
767         #Action == what to do ie. configure config_log ...
768         actions="$*"
769
770         if [ "$actions" = "" ]; then
771                 actions="configure config_log config_header build install test"
772         fi
773
774         # start the build
775         (
776         {
777                 # we all want to be able to read the output...
778                 LANG=C
779                 export LANG
780
781                 uname -a
782
783                 echo ""
784                 echo "build_test                  : $build_test_id"
785                 echo "build_test.fns      : $build_test_fns_id"
786                 echo "local settings file : $build_test_settings_local_file"
787                 echo "local functions file: $build_test_fns_local_file"
788                 echo "used .fns file      : $build_test_used_fns_file"
789                 echo ""
790
791                 # we need to be able to see if a build farm machine is accumulating
792                 # stuck processes. We do this in two ways, as we don't know what style
793                 # of ps it will have
794                 ps xfuw 2> /dev/null
795                 ps -fu $USER 2> /dev/null
796
797                 echo "building $tree with CC=$compiler on $host at "`date`
798                 echo "builddir=$builddir"
799                 echo "prefix=$prefix"
800
801                 echo "Showing limits"
802                 ulimit -a 2> /dev/null
803
804                 # the following is for non-samba builds only
805                 if [ "$scm" = "svn" -a -r $test_root/$tree.svn ]; then
806                         h_rev=`grep 'Revision: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
807                         if [ -n "$h_rev" ]; then
808                                 echo "HIGHEST SVN REVISION: $h_rev"
809                         fi
810                         rev=`grep 'Last Changed Rev: ' $test_root/$tree.svn | cut -d ':' -f2 | cut -d ' ' -f2 | sed 1q`
811                         if [ -n "$rev" ]; then
812                         echo "BUILD REVISION: $rev"
813                         fi
814                 elif [ "$scm" = "git" -a -r $test_root/$tree.git ]; then
815                         csha1=`cat $test_root/$tree.git |head -3 | tail -1`
816                         if [ -n "$csha1" ]; then
817                                 echo "BUILD COMMIT REVISION: $csha1"
818                         fi
819                         cdate=`cat $test_root/$tree.git |head -4 | tail -1`
820                         if [ -n "$cdate" ]; then
821                                 echo "BUILD COMMIT DATE: $cdate"
822                         fi
823                         ctime=`cat $test_root/$tree.git |head -2 | tail -1`
824                         if [ -n "$ctime" ]; then
825                                 echo "BUILD COMMIT TIME: $ctime"
826                   fi
827                 fi
828
829                 if [ -x $builddir/killbysubdir ]; then
830                         echo "$builddir/killbysubdir $builddir in `pwd`"
831                         $builddir/killbysubdir $builddir
832                 fi
833
834                 for action in $actions; do
835
836                         echo Running action $action
837
838                         date
839
840                         cd $builddir || exit 1
841                         export srcdir
842                         df .
843                         mount
844                         vmstat
845
846                         if [ "x$PREHOOKS" != "x" ]; then
847                                 for hooks in $PREHOOKS; do
848                                         if [ "x$hooks" = "x$action" ]; then
849                                                 ( prehook_$action )
850                                         fi
851                                 done
852                         fi
853
854                         ( action_$action )
855                         action_status=$?
856
857                         if [ "x$POSTHOOKS" != "x" ]; then
858                                 for hooks in $POSTHOOKS; do
859                                         if [ "x$hooks" = "x$action" ]; then
860                                                 ( posthook_$action )
861                                         fi
862                                 done
863                         fi
864
865                         df .
866
867                         if [ $action_status != 0 ]; then
868                                 echo "ACTION FAILED: $action";
869                                 echo " return code $action_status $action";
870                         else
871                                 echo "ACTION PASSED: $action";
872                         fi
873
874                         if [ $action_status != 0 ]; then 
875                                 break;
876                         fi
877                 done
878
879
880                 if [ "$noclean" = "yes" ]; then
881                         echo cleanup skipped!
882                 else
883                         echo cleaning up
884                         do_make clean
885                 fi
886                 date
887         } 3>&2 2>&1 1>&3 | tee "$err"
888         ) > "$log" 2>&1
889         # be aware the above channel swap may sometimes result in unordered
890         # stdout/stderr merge
891
892         if [ "$LCOV_REPORT" = "yes" ]; then
893                 chmod u=rwX,g=rX,o=rX -R $builddir/coverage
894                 rsync -rct -q --password-file=.password -z --timeout=200 \
895                         $builddir/coverage/ $host@build.samba.org::lcov_data/$host/$tree/
896         fi
897
898         cd $test_root
899
900         /bin/rm -rf $prefix
901         # send the logs to the master site
902         send_logs "$log" "$err"
903
904         # cleanup
905         echo "Ending build of $tree.$compiler in process $$ at `date`"
906         unlock_file "$lck"
907 }
908
909 #########################################################
910 # if you want to build only one project at a time
911 # add 'global_lock' after 'per_run_hook' and
912 # 'global_unlock' to the end of the file
913 #########################################################
914
915 global_lock() {
916         lock_file "global.lck" || {
917                 exit 0
918         }
919 }
920
921 global_unlock() {
922         unlock_file "global.lck"
923 }
924
925 delete_old_tree() {
926         otree=$1
927         test -z "$otree" && return 0;
928
929         rm -rf $otree
930         rm -rf $otree.svn
931         rm -rf $otree.*.svn
932         rm -rf $otree.git
933         rm -rf $otree.*.git
934         rm -rf build.$otree.*
935 }
936
937 # this is a special fn that allows us to add a "special" hook to the build
938 # farm that we want to do to the build farm. never leave it empty. instead,
939 # use ":" as the fn body.
940 per_run_hook() {
941         # kill old processes on systems with a known problem
942         case $host in
943         nohost)
944                 echo "just a placeholder";
945                 ;;
946         deckchair)
947                 rm -f deckchair.fns
948                 ;;
949         esac
950
951         # trim the log if too large
952         if [ "`wc -c < build.log`" -gt 2000000 ]; then
953         rm -f build.log
954         fi
955
956         old_trees="web popt distcc samba-gtk smb-build lorikeet-heimdal samba_3_2"
957         old_trees="$old_tree samba_3_2_test samba4 samba_4_0_waf samba_4_0_waf.metze"
958         old_trees="$old_tree samba_3_X_test samba_3_X_devel samba_3_X_devel"
959         for d in $old_trees; do
960                 delete_old_tree $d
961         done
962 }
963
964
965 ######################################################
966 # main code that is run on each call to the build code
967 ######################################################
968 rsync --timeout=200 -q -az build.samba.org::build_farm/*.c .
969
970
971 # build.log can grow to an excessive size, trim it beyond 50M
972 if [ -f build.log ]; then
973         find build.log -size +100000 -exec /bin/rm '{}' \;
974 fi