make it possible for admins to configure proxy config...
[amitay/build-farm.git] / build_test.fns
1 # -*- mode: shell-script; sh-indentation: 8; indent-tabs-mode: t; -*-
2
3 # build_farm -- distributed build/test architecture for samba, rsync, etc
4
5 # Copyright (C) 2001 by Andrew Tridgell <tridge@samba.org>
6 # Copyright (C) 2001 by Andrew Bartlett <abartlet@samba.org>
7 # Copyright (C) 2001, 2003 by Martin Pool <mbp@samba.org>
8
9 # default maximum runtime for any command
10 MAXTIME=7200
11 RUN_FROM_BUILD_FARM=yes
12 export RUN_FROM_BUILD_FARM
13
14 deptrees="";
15
16 build_test_fns_id='$Id$'
17
18 #############################
19 # build a signature of a tree, used to see if we
20 # need to rebuild 
21 sum_tree() {
22         sum_tree_test_root=$1
23         sum_tree_tree=$2
24         sum_tree_sum=$3
25         find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v '.svn' | grep -v version.h | sort | xargs sum > $sum_tree_sum
26         sum build_test build_test.fns >> $sum_tree_sum
27
28         if [ -f "$host.fns" ]; then
29             sum $host.fns >> $sum_tree_sum
30         else
31             sum generic.fns >> $sum_tree_sum
32         fi
33         if [ -f "$test_root/$tree.svn" ]; then
34             sum "$test_root/$tree.svn" >> $sum_tree_sum
35         fi
36         for d in $deptrees; do
37             if [ -f "$test_root/$d.svn" ]; then
38                 sum "$test_root/$d.svn" >> $sum_tree_sum
39             fi
40         done
41 }
42
43 #############################
44 # send the logs to the master site
45 send_logs() {
46         if [ "$nologreturn" = "yes" ]; then
47                 echo "skipping log transfer"
48         else
49                 log="$1"
50                 err="$2"
51                 shift
52                 shift
53                 chmod 0644 "$log" "$err"
54
55                 XARGS_I="xargs -i"
56                 if [ "`uname`" = "FreeBSD" ]; then
57                         XARGS_I="xargs -I '{}' -R -1"
58                 fi
59                 find $log -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'
60                 find $err -size +40000 | $XARGS_I sh -c 'dd if={} bs=1024 count=20000 of={}.tmp && mv {}.tmp {} &&  echo "\n***LOG TRUNCATED***" >> {}'
61
62                 rsync $* -ct -q --password-file=.password -z --timeout=200 \
63                     "$log" "$err" $host@samba.org::build_farm_data/
64         fi
65 }
66
67 #############################
68 # send the logs when they haven't changed
69 # the aim is to just update the servers timestamp.
70 # sending with a very large rsync block size does this
71 # with minimal network traffic
72 send_logs_skip() {
73     touch "$1" "$2"
74     send_logs "$1" "$2" -B 10000000
75 }
76
77 ############################
78 # fetch the latest copy of the tree
79 fetch_tree() {
80         if [ "$norsync" = "yes" ]; then
81                 echo "skipping tree transfer"
82         else
83                 fetchtree=$1
84                 if rsync --exclude=autom4te.cache/ --exclude=.svn/ --delete-excluded -q --partial --timeout=200 -crlpz --delete --ignore-errors \
85                         samba.org::ftp/unpacked/$fetchtree/ $test_root/$fetchtree; then
86                         echo "transferred $fetchtree OK"
87                 else
88                         echo "transfer of $fetchtree failed code $?"
89                         return 1
90                 fi
91         fi
92         return 0
93 }
94
95 ############################
96 # fetch the latest copy of the svn entries file
97 fetch_svn() {
98     tree=$1
99 # skip products still in CVS.
100     case "$tree" in
101     ccache | distcc | rsync)
102         return 1
103         ;;
104     *)
105         ;;
106     esac
107     if [ "$norsync" = "yes" ]; then
108         echo "skipping svn transfer"
109     else
110         if [ -r $test_root/$tree.svn ]; then
111                 rm -f $test_root/$tree.svn.old
112             mv $test_root/$tree.svn $test_root/$tree.svn.old
113         fi
114         rsync -q --timeout=200 -clz --ignore-errors \
115             samba.org::ftp/unpacked/$tree/.svn/entries $test_root/$tree.svn.tmp
116         chmod u+w $test_root/$tree.svn.tmp
117         sort -u < $test_root/$tree.svn.tmp > $test_root/$tree.svn
118         rm -f $test_root/$tree.svn.tmp
119     fi
120     if [ -r $test_root/$tree.svn ]; then
121         return 0;
122     fi
123     return 1
124 }
125
126 locknesting=0
127
128 ############################
129 # grab a lock file. Not atomic, but close :)
130 # tries to cope with NFS
131 lock_file() {
132         if [ -z "$lock_root" ]; then
133           lock_root=`pwd`;
134         fi
135         lckf="$lock_root/$1"
136         machine=`cat "$lckf" 2> /dev/null | cut -d: -f1`
137         pid=`cat "$lckf" 2> /dev/null | cut -d: -f2`
138
139         if [ "$pid" = "$$" ]; then
140             locknesting=`expr $locknesting + 1`
141             echo "lock nesting now $locknesting"
142             return 0
143         fi
144
145         if test -f "$lckf"; then
146             test $machine = $host || {
147                 echo "lock file $lckf is valid for other machine $machine"
148                 return 1                 
149             }
150             kill -0 $pid && {
151                 echo "lock file $lckf is valid for process $pid"
152                 return 1
153             }
154             echo "stale lock file $lckf for $machine:$pid"
155             cat "$lckf"
156             /bin/rm -f "$lckf"
157         fi
158         echo "$host:$$" > "$lckf"
159         return 0
160 }
161
162 ############################
163 # unlock a lock file
164 unlock_file() {
165         if [ -z "$lock_root" ]; then
166           lock_root=`pwd`;
167         fi
168         if [ "$locknesting" != "0" ]; then
169             locknesting=`expr $locknesting - 1`
170             echo "lock nesting now $locknesting"
171         else 
172             lckf="$lock_root/$1"
173             /bin/rm -f "$lckf"
174         fi
175 }
176
177 ############################
178 # run make, and print trace
179 do_make() {
180
181   if [ x"$MAKE" = x ] 
182   then
183     MAKE=make
184   fi 
185
186   MMTIME=$MAXTIME
187   # some trees don't need as much time
188   case "$tree" in
189         rsync | tdb | talloc | libreplace | ccache | distcc)
190           if [ "$compiler" != "checker" ]; then
191               MMTIME=`expr $MMTIME / 5`
192           fi
193           ;;
194   esac
195   
196     
197   for t in $*; do
198     if [ x"$BUILD_FARM_NUM_JOBS" = x ]; then
199       echo "$MAKE $t"
200       ./timelimit $MMTIME "$MAKE" "$t"
201       status=$?
202     else
203       # we can parallelize everything and all targets
204       if [ x"$t" = xeverything ] || [ x"$t" = xall]; then
205         echo "$MAKE" "-j$BUILD_FARM_NUM_JOBS"  "$t"
206         ./timelimit $MMTIME "$MAKE" "-j$BUILD_FARM_NUM_JOBS"  "$t"
207         status=$?
208       else
209         echo "$MAKE $t"
210         ./timelimit $MMTIME "$MAKE" "$t"
211         status=$?
212       fi
213     fi
214
215     if [ $status != 0 ]; then
216       return $status;
217     fi
218
219   done
220
221   return 0
222 }      
223
224 ############################
225 # configure the tree
226 action_configure() {
227         if [ ! -x $srcdir/configure ]; then
228             ls -l $srcdir/configure
229             echo "$srcdir/configure is missing"
230             cstatus=255
231             echo "CONFIGURE STATUS: $cstatus"
232             return $cstatus;
233         fi
234         echo "CFLAGS=$CFLAGS"
235         echo configure options: $config_and_prefix
236         echo CC="$CCACHE $compiler" $srcdir/configure $config_and_prefix
237         CC="$CCACHE $compiler"
238         export CC
239         ./timelimit $MAXTIME $srcdir/configure $config_and_prefix
240         cstatus=$?
241         echo "CONFIGURE STATUS: $cstatus"
242         if [ -f config.h ]; then
243             echo "contents of config.h:"
244             cat config.h
245         fi
246         if [ -f include/config.h ]; then
247             echo "contents of include/config.h:"
248             cat include/config.h
249         fi
250         return $cstatus;
251 }
252
253 ############################
254 # show the configure log
255 action_config_log() {
256         if [ ! -f config.log ]; then
257             return 0;
258         fi
259         echo "contents of config.log:"
260         cat config.log
261         return 0;
262 }
263
264 ############################
265 # build the tree
266 action_build() {
267         case "$tree" in
268         samba4)
269                 rm -f bin/smbtorture
270                 do_make everything
271                 bstatus=$?
272                 if test x"$bstatus" != x"0"; then
273                         # the 2nd 'make everything' is to work around a bug
274                         # in netbsd make. 
275                         do_make everything
276                         bstatus=$?
277                 fi
278
279                 if [ -n "$smbtorture4" ]; then
280                         if [ ! -f bin/smbtorture ]; then
281                                 do_make bin/smbtorture
282                         fi
283                         if [ -f bin/smbtorture ]; then
284                                 cp bin/smbtorture $smbtorture4
285                         fi
286                 fi
287
288                 ;;
289         samba|samba_3_0)
290                 do_make proto everything torture
291                 bstatus=$?
292                 ;;
293         *)
294                 do_make all
295                 bstatus=$?
296                 ;;
297         esac
298
299         echo "BUILD STATUS: $bstatus"
300
301         return $bstatus
302 }
303
304 ############################
305 # show static analysis results
306 action_cc_checker() {
307
308         # default to passing the cc_checker
309         cccstatus=0
310
311         if [ -f ibm_checker.out ]; then
312                 cat ibm_checker.out
313                 cccstatus=`cat ibm_checker.out | grep '^\-\- ' | wc -l`
314         fi
315
316         echo "CC_CHECKER STATUS: $cccstatus"
317         return $cccstatus;      
318 }
319
320 ############################
321 # install the tree
322 action_install() {
323         if [ -d $prefix ]; then
324                 if [ "$noclean" != "yes" ]; then
325                     rm -rf $prefix
326                 fi
327         fi
328
329         do_make install
330         istatus=$?
331         echo "INSTALL STATUS: $istatus"
332         return $istatus;
333 }
334
335 ############################
336 # test the tree
337 action_test_samba() {
338         do_make test
339         totalstatus=$?
340         return "$totalstatus"
341 }
342
343 action_test_generic() {
344         CC="$compiler"
345         export CC
346         do_make installcheck
347         totalstatus=$?
348         echo "TEST STATUS: $totalstatus"
349         return "$totalstatus"
350 }
351
352 action_test_lorikeet_heimdal() {
353         CC="$compiler"
354         export CC
355         SOCKET_WRAPPER_DIR=`pwd`/sw
356         mkdir $SOCKET_WRAPPER_DIR
357         export SOCKET_WRAPPER_DIR
358         do_make check
359         totalstatus=$?
360         SOCKET_WRAPPER_DIR=
361         export SOCKET_WRAPPER_DIR
362         echo "TEST STATUS: $totalstatus"
363         return "$totalstatus"
364 }
365
366
367 #############################
368 # attempt some basic tests of functionaility
369 # starting as basic as possible, and getting incresingly complex
370
371 action_test() {
372         # Samba needs crufty code of its own for backward
373         # compatiblity.  I think a better way to do this in the future
374         # is to just call 'make installcheck'.
375         case "$tree" in
376 #        samba_3_0)
377 #               echo "testing of samba_3_0 disabled until cause of runaway processes found (tridge - 7th sep 2006)"
378 #               ;;
379         samba*|smb-build|pidl)
380             action_test_samba
381             ;;
382         lorikeet-heimdal*)
383             action_test_lorikeet_heimdal
384             ;;
385         *)
386             action_test_generic
387             ;;
388         esac
389 }
390
391 ###########################
392 # do a test build of a particular tree
393 test_tree() {
394         tree=$1
395         source=$2
396         compiler="$3"
397         shift
398         shift
399         shift
400         if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && ccache -V > /dev/null; then
401             CCACHE="ccache"
402             export CCACHE
403         else
404             CCACHE=""
405         fi
406
407         # limit our resource usage
408         ulimit -t $MAXTIME 2> /dev/null
409         
410         # max mem size 100M
411         ulimit -m 100000 2> /dev/null
412
413         # max file size 100M
414         # darn, this affects sparse files too! disable it
415         # ulimit -f 100000 2> /dev/null
416
417         # try and limit the number of open files to 150. That means we'll discover
418         # fd leaks faster
419         ulimit -n 150 2> /dev/null
420
421         # Keep stuff private
422         umask 077
423
424         if [ -z "$test_root" ]; then
425                 test_root=`pwd`
426         fi
427
428         log="build.$tree.$host.$compiler.log"
429         err="build.$tree.$host.$compiler.err"
430         sum="build.$tree.$host.$compiler.sum"
431         lck="build.$tree.lck"
432         srcdir="$test_root/$tree/$source"
433
434         lock_file "$lck" || {
435                 return
436         }
437
438         # work out what other trees this package depends on
439         deptrees=""
440         case "$tree" in
441             talloc | tdb)
442                 deptrees="libreplace";
443             ;;
444             ldb)
445                 deptrees="libreplace talloc tdb";
446             ;;
447                 samba-gtk)
448                 deptrees="samba4"
449                 ;;
450         esac
451
452         # pull the svn entries, if any
453         if fetch_svn "$tree"; then
454             for d in $deptrees; do
455                 if [ -f "$test_root/$d.svn" ]; then
456                     if [ "$d" != "$tree" ]; then
457                         cat "$test_root/$d.svn" >> $test_root/$tree.svn
458                     fi
459                 fi
460             done
461             rm -f $test_root/$tree.$compiler.svn.old
462             mv $test_root/$tree.$compiler.svn $test_root/$tree.$compiler.svn.old
463             cp $test_root/$tree.svn $test_root/$tree.$compiler.svn
464             if cmp $test_root/$tree.$compiler.svn $test_root/$tree.$compiler.svn.old > /dev/null; then
465                 echo "skip: $tree.$compiler nothing changed in svn"
466                 cd $test_root
467                 send_logs_skip "$log" "$err"
468                 unlock_file "$lck"
469                 return
470             fi
471         fi
472
473         # pull the tree
474         fetch_tree "$tree" || {
475             cd $test_root
476             unlock_file "$lck"
477             return
478         }
479
480         if [ ! -x $srcdir/configure ] && [ "$tree" != "pidl" ]; then
481                 echo "skip: $tree.$compiler configure not present, try again next time!"
482                 cd $test_root
483                 unlock_file "$lck"
484                 return
485         fi
486
487         echo "Starting build of $tree.$compiler in process $$ at `date`"
488
489         case "$tree" in
490             tdb | talloc | ldb | libreplace)
491                 builddir="$test_root/tmp.$tree.$compiler"
492                 usingtmpbuild=1
493                 if [ -d $builddir ]; then
494                     rm -rf $builddir
495                 fi
496                 mkdir -p $builddir
497                 export builddir
498             ;;
499             *)
500                 builddir=$srcdir
501                 usingtmpbuild=0
502                 export builddir
503             ;;
504         esac
505         
506         if [ ! $USER = "" ]; then
507             whoami=$USER
508         else 
509             if [ ! $LOGNAME = "" ]; then
510                 whoami=$LOGNAME
511             else
512                 whoami=build
513             fi
514         fi
515
516         prefix="$test_root/prefix/$tree.$compiler"
517         mkdir -p "$prefix"
518
519         smbtorture4=$test_root/smbtorture4
520         export smbtorture4
521
522         sw_config=$config
523
524         case "$tree" in
525         samba4|lorikeet-heimdal)
526             sw_config="$config --enable-socket-wrapper"
527             ;;
528         samba|samba_3_0)
529             sw_config="$config --enable-socket-wrapper"
530             if [ -f $smbtorture4 ]; then
531                 # we create a local copy to make sure the same binary is used for all tests
532                 cp $smbtorture4 $smbtorture4.$tree 
533                 sw_config="$sw_config --with-smbtorture4-path=$smbtorture4.$tree"
534             fi
535             ;;
536         samba-gtk)
537                 PKG_CONFIG_PATH="$test_root/prefix/samba4.$compiler/lib/pkgconfig"
538                 export PKG_CONFIG_PATH
539                 ;;
540         ldb)
541             fetch_tree popt
542             ;;
543         talloc)
544             fetch_tree libreplace
545             ;;
546         *)
547             testsuite=testsuite
548             ;;
549         esac
550
551         if [ "$LCOV_REPORT" = "yes" ]; then
552             GCOV_FLAGS="-ftest-coverage -fprofile-arcs"
553             GCOV_LIBS="-lgcov"
554             HOSTCC_CFLAGS="$HOSTCC_CFLAGS $GCOV_FLAGS" 
555             CFLAGS="$CFLAGS $GCOV_FLAGS" 
556             LDFLAGS="$LDFLAGS $GCOV_FLAGS $GCOV_LIBS" 
557             SHLD_FLAGS="$SHLD_FLAGS $GCOV_FLAGS $GCOV_LIBS"
558             export HOSTCC_CFLAGS CFLAGS LDFLAGS SHLD_FLAGS
559         fi
560
561         config_and_prefix="$sw_config --prefix=$prefix"
562
563         # see if we need to rebuild
564         sum_tree $test_root $tree $sum
565         echo "CFLAGS=$CFLAGS $config_and_prefix" >> $sum
566
567         if cmp "$sum" "$sum.old" > /dev/null; then
568                 echo "skip: $tree.$compiler nothing changed"
569                 cd $test_root
570                 send_logs_skip "$log" "$err"
571                 unlock_file "$lck"
572                 echo "Ending build of $tree.$compiler in process $$ at `date`"
573                 return
574         fi
575
576         # we do need to rebuild - save the old sum
577         /bin/rm -f $sum.old
578         mv $sum $sum.old
579
580         actions="$*"
581         
582         if [ "$actions" = "" ]; then
583             actions="configure config_log build install test"
584         fi
585
586         # start the build
587         (
588                 # we all want to be able to read the output...
589                 LANG=C
590                 export LANG
591
592                 echo "build_test          : $build_test_id"
593                 echo "build_test.fns      : $build_test_fns_id"
594                 echo "local settings file : $build_test_settings_local_file"
595                 echo "local functions file: $build_test_fns_local_file"
596                 echo "used .fns file      : $build_test_used_fns_file"
597
598                 uname -a
599
600                 # we need to be able to see if a build farm machine is accumulating
601                 # stuck processes. We do this in two ways, as we don't know what style
602                 # of ps it will have
603                 ps xfuw 2> /dev/null
604                 ps -fu $USER 2> /dev/null
605
606                 echo "building $tree with CC=$compiler on $host at "`date`
607                 echo "builddir=$builddir"
608                 echo "prefix=$prefix"
609
610                 echo "Showing limits"
611                 ulimit -a 2> /dev/null
612
613                 # build the timelimit utility
614                 echo "Building timelimit"
615                 mkdir -p $builddir
616                 $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1
617
618                 if [ -r $test_root/$tree.svn ]; then
619                   h_rev=`grep revision= $test_root/$tree.svn | cut -d'"' -f2 | sort -n | tail -1`
620                   if [ -n "$h_rev" ]; then
621                         echo "HIGHEST SVN REVISION: $h_rev"
622                   fi
623                   rev=`grep committed-rev= $test_root/$tree.svn | cut -d'"' -f2 | sort -n | tail -1`
624                   if [ -n "$rev" ]; then
625                         echo "BUILD REVISION: $rev"
626                   fi
627                 fi
628
629
630                 if [ "$tree" = "pidl" ] 
631                 then
632                         cd $builddir
633                         perl ./Makefile.PL "$prefix"
634                 fi
635
636                 for action in $actions; do
637
638                     echo Running action $action
639
640                     date
641
642                     cd $builddir || exit 1
643                     export srcdir
644                     df .
645                     mount
646                     vmstat
647
648                     ( action_$action )
649                     action_status=$?
650                     
651                     if [ $action_status != 0 ]; then
652                         echo "ACTION FAILED: $action";
653                     else
654                         echo "ACTION PASSED: $action";
655                     fi
656                     
657                     if [ $action_status != 0 ]; then 
658                         break;
659                     fi
660
661                 done
662
663                 if [ "$LCOV_REPORT" = "yes" ]; then
664                     case "$tree" in
665                         lorikeet-heimdal*)
666                             lcov --directory $builddir --capture --output-file $builddir/$tree.lcov.info
667                             ;;
668                         *)
669                             # ugly hack for s4, as lcov is otherwise not able to find 
670                             # these files
671                             rm -f heimdal/lib/*/{lex,parse}.{gcda,gcno}
672                             lcov --base-directory $builddir --directory $builddir --capture --output-file $builddir/$tree.lcov.info
673                             ;;
674                     esac
675                     genhtml -o $builddir/coverage $builddir/$tree.lcov.info
676                 fi
677
678                 if [ "$noclean" = "yes" ]; then
679                     echo cleanup skipped!
680                 else
681                     echo cleaning up
682                     do_make clean
683                 fi
684                 date
685         ) > "$log" 2> "$err"
686
687         if [ "$LCOV_REPORT" = "yes" ]; then
688             chmod u=rwX,g=rX,o=rX -R $builddir/coverage
689             rsync -rct -q --password-file=.password -z --timeout=200 \
690                 $builddir/coverage/ $host@samba.org::lcov_data/$host/$tree/
691         fi
692
693         cd $test_root
694
695         /bin/rm -rf $prefix
696         if [ "$usingtmpbuild" = "1" ]; then
697             if [ "$noclean" = "yes" ]; then
698                 echo builddir cleanup skipped!
699             else
700                 /bin/rm -rf $builddir
701             fi
702         fi
703         # send the logs to the master site
704         send_logs "$log" "$err"
705
706         # cleanup
707         echo "Ending build of $tree.$compiler in process $$ at `date`"
708         unlock_file "$lck"
709 }
710
711 #########################################################
712 # if you want to build only one project at a time
713 # add 'global_lock' after 'per_run_hook' and
714 # 'global_unlock' to the end of the file
715 global_lock() {
716     lock_file "global.lck" || {
717         exit 0
718     }
719 }
720 global_unlock() {
721     unlock_file "global.lck"
722 }
723
724 #########################################################
725 # enable this on a per host basis only when needed please
726 # (at least for the moment)
727 kill_old_processes() {
728     # this should work on systems with linux like ps
729     (ps uxfw | grep /build | grep -v grep | egrep 'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
730     # and this should work on sysv style ps
731     (ps -fu $USER | grep /build | grep -v grep | egrep 'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
732 }
733
734
735 # this is a special fn that allows us to add a "special" hook to the build
736 # farm that we want to do to the build farm. never leave it empty. instead,
737 # use ":" as the fn body.
738 per_run_hook() {
739     # kill old processes on systems with a known problem
740     case $host in
741         nohost)
742             echo "just a placeholder";
743             ;;
744         tridge)
745             kill_old_processes
746             ;;
747         deckchair)
748             rm -f deckchair.fns
749             ;;
750     esac
751     # trim the log if too large
752     if [ "`wc -c < build.log`" -gt 2000000 ]; then
753         rm -f build.log
754     fi
755 }
756
757
758 ######################################################
759 # main code that is run on each call to the build code
760 rsync --timeout=200 -q -az samba.org::build_farm/*.c .
761
762
763 # build.log can grow to an excessive size, trim it beyond 50M
764 if [ -f build.log ]; then
765   find build.log -size +100000 -exec /bin/rm '{}' \;
766 fi
767