patch from Ronald Klop <ronald@klop.ws>
[build-farm.git] / build_test.fns
1 # -*- mode: shell-script; sh-indentation: 8; indent-tabs-mode: t; -*-
2
3 # build_farm -- distributed build/test architecture for samba, rsync, etc
4
5 # Copyright (C) 2001 by Andrew Tridgell <tridge@samba.org>
6 # Copyright (C) 2001 by Andrew Bartlett <abartlet@samba.org>
7 # Copyright (C) 2001, 2003 by Martin Pool <mbp@samba.org>
8
9 # default maximum runtime for any command
10 MAXTIME=7200
11 RUN_FROM_BUILD_FARM=yes
12 export RUN_FROM_BUILD_FARM
13
14 deptrees="";
15
16 #############################
17 # build a signature of a tree, used to see if we
18 # need to rebuild 
19 sum_tree() {
20         sum_tree_test_root=$1
21         sum_tree_tree=$2
22         sum_tree_sum=$3
23         find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v '.svn' | grep -v version.h | sort | xargs sum > $sum_tree_sum
24         sum build_test build_test.fns >> $sum_tree_sum
25
26         if [ -f "$host.fns" ]; then
27             sum $host.fns >> $sum_tree_sum
28         else
29             sum generic.fns >> $sum_tree_sum
30         fi
31         if [ -f "$test_root/$tree.svn" ]; then
32             sum "$test_root/$tree.svn" >> $sum_tree_sum
33         fi
34         for d in $deptrees; do
35             if [ -f "$test_root/$d.svn" ]; then
36                 sum "$test_root/$d.svn" >> $sum_tree_sum
37             fi
38         done
39 }
40
41 #############################
42 # send the logs to the master site
43 send_logs() {
44         if [ "$nologreturn" = "yes" ]; then
45                 echo "skipping log transfer"
46         else
47                 log="$1"
48                 err="$2"
49                 shift
50                 shift
51                 chmod 0644 "$log" "$err"
52
53                 XARGS_I="xargs -i"
54                 if [ "`uname`" = "FreeBSD" ]; then
55                         XARGS_I="xargs -I '{}' -R -1"
56                 fi
57                 find $log -size +20000k | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'
58                 find $err -size +20000k | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'
59
60                 rsync $* -ct -q --password-file=.password -z --timeout=200 \
61                     "$log" "$err" $host@samba.org::build_farm_data/
62         fi
63 }
64
65 #############################
66 # send the logs when they haven't changed
67 # the aim is to just update the servers timestamp.
68 # sending with a very large rsync block size does this
69 # with minimal network traffic
70 send_logs_skip() {
71     touch "$1" "$2"
72     send_logs "$1" "$2" -B 10000000
73 }
74
75 ############################
76 # fetch the latest copy of the tree
77 fetch_tree() {
78         if [ "$norsync" = "yes" ]; then
79                 echo "skipping tree transfer"
80         else
81                 fetchtree=$1
82                 if rsync --exclude=autom4te.cache/ --exclude=.svn/ --delete-excluded -q --partial --timeout=200 -crlpz --delete --ignore-errors \
83                         samba.org::ftp/unpacked/$fetchtree/ $test_root/$fetchtree; then
84                         echo "transferred $fetchtree OK"
85                 else
86                         echo "transfer of $fetchtree failed code $?"
87                         return 1
88                 fi
89         fi
90         return 0
91 }
92
93 ############################
94 # fetch the latest copy of the svn entries file
95 fetch_svn() {
96     tree=$1
97 # skip products still in CVS.
98     case "$tree" in
99     ccache | distcc | rsync)
100         return 1
101         ;;
102     *)
103         ;;
104     esac
105     if [ "$norsync" = "yes" ]; then
106         echo "skipping svn transfer"
107     else
108         if [ -r $test_root/$tree.svn ]; then
109                 rm -f $test_root/$tree.svn.old
110             mv $test_root/$tree.svn $test_root/$tree.svn.old
111         fi
112         rsync -q --timeout=200 -clz --ignore-errors \
113             samba.org::ftp/unpacked/$tree/.svn/entries $test_root/$tree.svn.tmp
114         chmod u+w $test_root/$tree.svn.tmp
115         sort -u < $test_root/$tree.svn.tmp > $test_root/$tree.svn
116         rm -f $test_root/$tree.svn.tmp
117     fi
118     if [ -r $test_root/$tree.svn ]; then
119         return 0;
120     fi
121     return 1
122 }
123
124 locknesting=0
125
126 ############################
127 # grab a lock file. Not atomic, but close :)
128 # tries to cope with NFS
129 lock_file() {
130         if [ -z "$lock_root" ]; then
131           lock_root=`pwd`;
132         fi
133         lckf="$lock_root/$1"
134         machine=`cat "$lckf" 2> /dev/null | cut -d: -f1`
135         pid=`cat "$lckf" 2> /dev/null | cut -d: -f2`
136
137         if [ "$pid" = "$$" ]; then
138             locknesting=`expr $locknesting + 1`
139             echo "lock nesting now $locknesting"
140             return 0
141         fi
142
143         if test -f "$lckf"; then
144             test $machine = $host || {
145                 echo "lock file $lckf is valid for other machine $machine"
146                 return 1                 
147             }
148             kill -0 $pid && {
149                 echo "lock file $lckf is valid for process $pid"
150                 return 1
151             }
152             echo "stale lock file $lckf for $machine:$pid"
153             cat "$lckf"
154             /bin/rm -f "$lckf"
155         fi
156         echo "$host:$$" > "$lckf"
157         return 0
158 }
159
160 ############################
161 # unlock a lock file
162 unlock_file() {
163         if [ -z "$lock_root" ]; then
164           lock_root=`pwd`;
165         fi
166         if [ "$locknesting" != "0" ]; then
167             locknesting=`expr $locknesting - 1`
168             echo "lock nesting now $locknesting"
169         else 
170             lckf="$lock_root/$1"
171             /bin/rm -f "$lckf"
172         fi
173 }
174
175 ############################
176 # run make, and print trace
177 do_make() {
178
179   if [ x"$MAKE" = x ] 
180   then
181     MAKE=make
182   fi 
183
184   MMTIME=$MAXTIME
185   # some trees don't need as much time
186   case "$tree" in
187         rsync | tdb | talloc | libreplace | ccache | distcc)
188           if [ "$compiler" != "checker" ]; then
189               MMTIME=`expr $MMTIME / 5`
190           fi
191           ;;
192   esac
193   
194     
195   for t in $*; do
196     if [ x"$BUILD_FARM_NUM_JOBS" = x ]; then
197       echo "$MAKE $t"
198       ./timelimit $MMTIME "$MAKE" "$t"
199       status=$?
200     else
201       # we can parallelize everything and all targets
202       if [ x"$t" = xeverything ] || [ x"$t" = xall]; then
203         echo "$MAKE" "-j$BUILD_FARM_NUM_JOBS"  "$t"
204         ./timelimit $MMTIME "$MAKE" "-j$BUILD_FARM_NUM_JOBS"  "$t"
205         status=$?
206       else
207         echo "$MAKE $t"
208         ./timelimit $MMTIME "$MAKE" "$t"
209         status=$?
210       fi
211     fi
212
213     if [ $status != 0 ]; then
214       return $status;
215     fi
216
217   done
218
219   return 0
220 }      
221
222 ############################
223 # configure the tree
224 action_configure() {
225         if [ ! -x configure ]; then
226             ls -l configure
227             ./autogen.sh
228         fi
229         echo "CFLAGS=$CFLAGS"
230         echo configure options: $config_and_prefix
231         echo CC="$CCACHE $compiler" $srcdir/configure $config_and_prefix
232         CC="$CCACHE $compiler"
233         export CC
234         ./timelimit $MAXTIME $srcdir/configure $config_and_prefix
235         cstatus=$?
236         echo "CONFIGURE STATUS: $cstatus"
237         if [ -f config.h ]; then
238             echo "contents of config.h:"
239             cat config.h
240         fi
241         if [ -f include/config.h ]; then
242             echo "contents of include/config.h:"
243             cat include/config.h
244         fi
245         return $cstatus;
246 }
247
248 ############################
249 # show the configure log
250 action_config_log() {
251         if [ ! -f config.log ]; then
252             return 0;
253         fi
254         echo "contents of config.log:"
255         cat config.log
256         return 0;
257 }
258
259 ############################
260 # build the tree
261 action_build() {
262         case "$tree" in
263         samba4)
264                 # the 2nd 'make everything' is to work around a bug
265                 # in netbsd make. 
266                 do_make proto showflags everything
267                 do_make everything
268                 do_make bin/smbtorture
269                 ;;
270         samba|samba_3_0)
271             do_make proto everything torture
272             ;;
273         *)
274             do_make all
275             ;;
276         esac
277
278         bstatus=$?
279         echo "BUILD STATUS: $bstatus"
280
281         if [ "$tree" = "samba4" ] && [ -n "$smbtorture4" ] && [ -f bin/smbtorture ]; then
282             rm -f $smbtorture4
283             cp bin/smbtorture $smbtorture4
284         fi
285         return $bstatus
286 }
287
288 ############################
289 # show static analysis results
290 action_cc_checker() {
291
292         # default to passing the cc_checker
293         cccstatus=0
294
295         if [ -f ibm_checker.out ]; then
296                 cat ibm_checker.out
297                 cccstatus=`cat ibm_checker.out | grep '^\-\- ' | wc -l`
298         fi
299
300         echo "CC_CHECKER STATUS: $cccstatus"
301         return $cccstatus;      
302 }
303
304 ############################
305 # install the tree
306 action_install() {
307         if [ -d $prefix ]; then
308                 if [ "$noclean" != "yes" ]; then
309                     rm -rf $prefix
310                 fi
311         fi
312
313         do_make install
314         istatus=$?
315         echo "INSTALL STATUS: $istatus"
316         return $istatus;
317 }
318
319 ############################
320 # test the tree
321 action_test_samba() {
322         do_make test
323         totalstatus=$?
324         return "$totalstatus"
325 }
326
327 action_test_generic() {
328         CC="$compiler"
329         export CC
330         do_make installcheck
331         totalstatus=$?
332         echo "TEST STATUS: $totalstatus"
333         return "$totalstatus"
334 }
335
336 action_test_lorikeet_heimdal() {
337         CC="$compiler"
338         export CC
339         SOCKET_WRAPPER_DIR=`pwd`/sw
340         mkdir $SOCKET_WRAPPER_DIR
341         export SOCKET_WRAPPER_DIR
342         do_make check
343         totalstatus=$?
344         SOCKET_WRAPPER_DIR=
345         export SOCKET_WRAPPER_DIR
346         echo "TEST STATUS: $totalstatus"
347         return "$totalstatus"
348 }
349
350
351 #############################
352 # attempt some basic tests of functionaility
353 # starting as basic as possible, and getting incresingly complex
354
355 action_test() {
356         # Samba needs crufty code of its own for backward
357         # compatiblity.  I think a better way to do this in the future
358         # is to just call 'make installcheck'.
359         case "$tree" in
360 #        samba_3_0)
361 #               echo "testing of samba_3_0 disabled until cause of runaway processes found (tridge - 7th sep 2006)"
362 #               ;;
363         samba*|smb-build|pidl)
364             action_test_samba
365             ;;
366         lorikeet-heimdal*)
367             action_test_lorikeet_heimdal
368             ;;
369         *)
370             action_test_generic
371             ;;
372         esac
373 }
374
375 ###########################
376 # do a test build of a particular tree
377 test_tree() {
378         tree=$1
379         source=$2
380         compiler="$3"
381         shift
382         shift
383         shift
384         if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && ccache -V > /dev/null; then
385             CCACHE="ccache"
386             export CCACHE
387         else
388             CCACHE=""
389         fi
390
391         # limit our resource usage
392         ulimit -t $MAXTIME 2> /dev/null
393         
394         # max mem size 100M
395         ulimit -m 100000 2> /dev/null
396
397         # max file size 100M
398         # darn, this affects sparse files too! disable it
399         # ulimit -f 100000 2> /dev/null
400
401         # try and limit the number of open files to 150. That means we'll discover
402         # fd leaks faster
403         ulimit -n 150 2> /dev/null
404
405         # Keep stuff private
406         umask 077
407
408         if [ -z "$test_root" ]; then
409                 test_root=`pwd`
410         fi
411
412         log="build.$tree.$host.$compiler.log"
413         err="build.$tree.$host.$compiler.err"
414         sum="build.$tree.$host.$compiler.sum"
415         lck="build.$tree.lck"
416         srcdir="$test_root/$tree/$source"
417
418         lock_file "$lck" || {
419                 return
420         }
421
422         # work out what other trees this package depends on
423         deptrees=""
424         case "$tree" in
425             talloc | tdb)
426                 deptrees="libreplace";
427             ;;
428             ldb)
429                 deptrees="libreplace talloc tdb";
430             ;;
431                 samba-gtk)
432                 deptrees="samba4"
433                 ;;
434         esac
435
436         # pull the svn entries, if any
437         if fetch_svn "$tree"; then
438             for d in $deptrees; do
439                 if [ -f "$test_root/$d.svn" ]; then
440                     if [ "$d" != "$tree" ]; then
441                         cat "$test_root/$d.svn" >> $test_root/$tree.svn
442                     fi
443                 fi
444             done
445             rm -f $test_root/$tree.$compiler.svn.old
446             mv $test_root/$tree.$compiler.svn $test_root/$tree.$compiler.svn.old
447             cp $test_root/$tree.svn $test_root/$tree.$compiler.svn
448             if cmp $test_root/$tree.$compiler.svn $test_root/$tree.$compiler.svn.old > /dev/null; then
449                 echo "skip: $tree.$compiler nothing changed in svn"
450                 cd $test_root
451                 send_logs_skip "$log" "$err"
452                 unlock_file "$lck"
453                 return
454             fi
455         fi
456
457         # pull the tree
458         fetch_tree "$tree" || {
459             cd $test_root
460             unlock_file "$lck"
461             return
462         }
463
464         if [ ! -x $srcdir/configure ] && [ "$tree" != "pidl" ]; then
465                 echo "skip: $tree.$compiler configure not present, try again next time!"
466                 cd $test_root
467                 unlock_file "$lck"
468                 return
469         fi
470
471         echo "Starting build of $tree.$compiler in process $$ at `date`"
472
473         case "$tree" in
474             tdb | talloc | ldb | libreplace)
475                 builddir="$test_root/tmp.$tree.$compiler"
476                 usingtmpbuild=1
477                 if [ -d $builddir ]; then
478                     rm -rf $builddir
479                 fi
480                 mkdir -p $builddir
481                 export builddir
482             ;;
483             *)
484                 builddir=$srcdir
485                 usingtmpbuild=0
486                 export builddir
487             ;;
488         esac
489         
490         if [ ! $USER = "" ]; then
491             whoami=$USER
492         else 
493             if [ ! $LOGNAME = "" ]; then
494                 whoami=$LOGNAME
495             else
496                 whoami=build
497             fi
498         fi
499
500         prefix="$test_root/prefix/$tree.$compiler"
501         mkdir -p "$prefix"
502
503         smbtorture4=$test_root/smbtorture4
504         export smbtorture4
505
506         sw_config=$config
507
508         case "$tree" in
509         samba4|lorikeet-heimdal)
510             sw_config="$config --enable-socket-wrapper"
511             ;;
512         samba|samba_3_0)
513             sw_config="$config --enable-socket-wrapper"
514             if [ -f $smbtorture4 ]; then
515                 # we create a local copy to make sure the same binary is used for all tests
516                 cp $smbtorture4 $smbtorture4.$tree 
517                 sw_config="$sw_config --with-smbtorture4-path=$smbtorture4.$tree"
518             fi
519             ;;
520         samba-gtk)
521                 PKG_CONFIG_PATH="$test_root/prefix/samba4.$compiler/lib/pkgconfig"
522                 export PKG_CONFIG_PATH
523                 ;;
524         ldb)
525             fetch_tree popt
526             ;;
527         talloc)
528             fetch_tree libreplace
529             ;;
530         *)
531             testsuite=testsuite
532             ;;
533         esac
534
535         if [ "$LCOV_REPORT" = "yes" ]; then
536             GCOV_FLAGS="-ftest-coverage -fprofile-arcs"
537             GCOV_LIBS="-lgcov"
538             HOSTCC_CFLAGS="$HOSTCC_CFLAGS $GCOV_FLAGS" 
539             CFLAGS="$CFLAGS $GCOV_FLAGS" 
540             LDFLAGS="$LDFLAGS $GCOV_FLAGS $GCOV_LIBS" 
541             SHLD_FLAGS="$SHLD_FLAGS $GCOV_FLAGS $GCOV_LIBS"
542             export HOSTCC_CFLAGS CFLAGS LDFLAGS SHLD_FLAGS
543         fi
544
545         config_and_prefix="$sw_config --prefix=$prefix"
546
547         # see if we need to rebuild
548         sum_tree $test_root $tree $sum
549         echo "CFLAGS=$CFLAGS $config_and_prefix" >> $sum
550
551         if cmp "$sum" "$sum.old" > /dev/null; then
552                 echo "skip: $tree.$compiler nothing changed"
553                 cd $test_root
554                 send_logs_skip "$log" "$err"
555                 unlock_file "$lck"
556                 echo "Ending build of $tree.$compiler in process $$ at `date`"
557                 return
558         fi
559
560         # we do need to rebuild - save the old sum
561         /bin/rm -f $sum.old
562         mv $sum $sum.old
563
564         actions="$*"
565         
566         if [ "$actions" = "" ]; then
567             actions="configure config_log build install test"
568         fi
569
570         # start the build
571         (
572                 uname -a
573
574                 # we need to be able to see if a build farm machine is accumulating
575                 # stuck processes. We do this in two ways, as we don't know what style
576                 # of ps it will have
577                 ps xfuw 2> /dev/null
578                 ps -fu $USER 2> /dev/null
579
580                 echo "building $tree with CC=$compiler on $host at "`date`
581                 echo "builddir=$builddir"
582                 echo "prefix=$prefix"
583
584                 echo "Showing limits"
585                 ulimit -a 2> /dev/null
586
587                 # build the timelimit utility
588                 echo "Building timelimit"
589                 mkdir -p $builddir
590                 $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1
591
592                 if [ -r $test_root/$tree.svn ]; then
593                   h_rev=`grep revision= $test_root/$tree.svn | cut -d'"' -f2 | sort -n | tail -1`
594                   if [ -n "$h_rev" ]; then
595                         echo "HIGHEST SVN REVISION: $h_rev"
596                   fi
597                   rev=`grep committed-rev= $test_root/$tree.svn | cut -d'"' -f2 | sort -n | tail -1`
598                   if [ -n "$rev" ]; then
599                         echo "BUILD REVISION: $rev"
600                   fi
601                 fi
602
603
604                 if [ "$tree" = "pidl" ] 
605                 then
606                         cd $builddir
607                         perl ./Makefile.PL "$prefix"
608                 fi
609
610                 for action in $actions; do
611
612                     echo Running action $action
613
614                     date
615
616                     cd $builddir || exit 1
617                     export srcdir
618                     df .
619                     mount
620                     vmstat
621
622                     ( action_$action )
623                     action_status=$?
624                     
625                     if [ $action_status != 0 ]; then
626                         echo "ACTION FAILED: $action";
627                     else
628                         echo "ACTION PASSED: $action";
629                     fi
630                     
631                     if [ $action_status != 0 ]; then 
632                         break;
633                     fi
634
635                 done
636
637                 if [ "$LCOV_REPORT" = "yes" ]; then
638                     case "$tree" in
639                         lorikeet-heimdal*)
640                             lcov --directory $builddir --capture --output-file $builddir/$tree.lcov.info
641                             ;;
642                         *)
643                             # ugly hack for s4, as lcov is otherwise not able to find 
644                             # these files
645                             rm -f heimdal/lib/*/{lex,parse}.{gcda,gcno}
646                             lcov --base-directory $builddir --directory $builddir --capture --output-file $builddir/$tree.lcov.info
647                             ;;
648                     esac
649                     genhtml -o $builddir/coverage $builddir/$tree.lcov.info
650                 fi
651
652                 if [ "$noclean" = "yes" ]; then
653                     echo cleanup skipped!
654                 else
655                     echo cleaning up
656                     do_make clean
657                 fi
658                 date
659         ) > "$log" 2> "$err"
660
661         if [ "$LCOV_REPORT" = "yes" ]; then
662             chmod u=rwX,g=rX,o=rX -R $builddir/coverage
663             rsync -rct -q --password-file=.password -z --timeout=200 \
664                 $builddir/coverage/ $host@samba.org::lcov_data/$host/$tree/
665         fi
666
667         cd $test_root
668
669         /bin/rm -rf $prefix
670         if [ "$usingtmpbuild" = "1" ]; then
671             if [ "$noclean" = "yes" ]; then
672                 echo builddir cleanup skipped!
673             else
674                 /bin/rm -rf $builddir
675             fi
676         fi
677         # send the logs to the master site
678         send_logs "$log" "$err"
679
680         # cleanup
681         echo "Ending build of $tree.$compiler in process $$ at `date`"
682         unlock_file "$lck"
683 }
684
685 #########################################################
686 # if you want to build only one project at a time
687 # add 'global_lock' after 'per_run_hook' and
688 # 'global_unlock' to the end of the file
689 global_lock() {
690     lock_file "global.lck" || {
691         exit 0
692     }
693 }
694 global_unlock() {
695     unlock_file "global.lck"
696 }
697
698 #########################################################
699 # enable this on a per host basis only when needed please
700 # (at least for the moment)
701 kill_old_processes() {
702     # this should work on systems with linux like ps
703     (ps uxfw | grep /build | grep -v grep | egrep 'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
704     # and this should work on sysv style ps
705     (ps -fu $USER | grep /build | grep -v grep | egrep 'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
706 }
707
708
709 # this is a special fn that allows us to add a "special" hook to the build
710 # farm that we want to do to the build farm. never leave it empty. instead,
711 # use ":" as the fn body.
712 per_run_hook() {
713     # kill old processes on systems with a known problem
714     case $host in
715         nohost)
716             echo "just a placeholder";
717             ;;
718         tridge)
719             kill_old_processes
720             ;;
721         deckchair)
722             rm -f deckchair.fns
723             ;;
724     esac
725     # trim the log if too large
726     if [ "`wc -c < build.log`" -gt 2000000 ]; then
727         rm -f build.log
728     fi
729 }
730
731
732 ######################################################
733 # main code that is run on each call to the build code
734 rsync --timeout=200 -q -az samba.org::build_farm/*.c .
735
736
737 # build.log can grow to an excessive size, trim it beyond 50M
738 if [ -f build.log ]; then
739   find build.log -size +100000 -exec /bin/rm '{}' \;
740 fi
741