build_test.fns

   1 # -*- mode: shell-script; sh-indentation: 8; indent-tabs-mode: t; -*-
   2
   3 # build_farm -- distributed build/test architecture for samba, rsync, etc
   4
   5 # Copyright (C) 2001 by Andrew Tridgell <tridge@samba.org>
   6 # Copyright (C) 2001 by Andrew Bartlett <abartlet@samba.org>
   7 # Copyright (C) 2001, 2003 by Martin Pool <mbp@samba.org>
   8
   9 # default maximum runtime for any command
  10 MAXTIME=7200
  11 RUN_FROM_BUILD_FARM=yes
  12 export RUN_FROM_BUILD_FARM
  13
  14 #############################
  15 # build a signature of a tree, used to see if we
  16 # need to rebuild
  17 sum_tree() {
  18         sum_tree_test_root=$1
  19         sum_tree_tree=$2
  20         sum_tree_sum=$3
  21         find $sum_tree_test_root/$sum_tree_tree -type f -print | grep -v '.svn' | grep -v version.h | sort | xargs sum > $sum_tree_sum
  22         sum build_test build_test.fns >> $sum_tree_sum
  23
  24         if [ -f "$host.fns" ]; then
  25             sum $host.fns >> $sum_tree_sum
  26         else
  27             sum generic.fns >> $sum_tree_sum
  28         fi
  29 }
  30
  31 #############################
  32 # send the logs to the master site
  33 send_logs() {
  34         if [ "$nologreturn" = "yes" ]; then
  35                 echo "skipping log transfer"
  36         else
  37                 log="$1"
  38                 err="$2"
  39                 chmod 0644 "$log" "$err"
  40                 rsync -q --password-file=.password -cz --timeout=200 \
  41                         "$log" "$err" $host@samba.org::build_farm_data/
  42         fi
  43 }
  44
  45 ############################
  46 # fetch the latest copy of the tree
  47 fetch_tree() {
  48         if [ "$norsync" = "yes" ]; then
  49                 echo "skipping tree transfer"
  50         else
  51                 tree=$1
  52                 if rsync --exclude=autom4te.cache/ --exclude=.svn/ --delete-excluded -q --partial --timeout=200 -crlpz --delete --ignore-errors \
  53                         samba.org::ftp/unpacked/$tree/ $test_root/$tree; then
  54                         echo "transferred $tree OK"
  55                 else
  56                         echo "transfer of $tree failed code $?"
  57                         return 1
  58                 fi
  59         fi
  60         return 0
  61 }
  62
  63 ############################
  64 # fetch the latest copy of the svn entries file
  65 fetch_svn() {
  66     tree=$1
  67 # skip products still in CVS.
  68     case "$tree" in
  69     ccache | distcc | rsync)
  70         return 1
  71         ;;
  72     *)
  73         ;;
  74     esac
  75     if [ "$norsync" = "yes" ]; then
  76         echo "skipping svn transfer"
  77     else
  78         if [ -r $test_root/$tree.svn ]; then
  79                 rm -f $test_root/$tree.svn.old
  80             mv $test_root/$tree.svn $test_root/$tree.svn.old
  81         fi
  82         rsync -q --timeout=200 -clz --ignore-errors \
  83             samba.org::ftp/unpacked/$tree/.svn/entries $test_root/$tree.svn
  84     fi
  85     if [ -r $test_root/$tree.svn ]; then
  86         return 0;
  87     fi
  88     return 1
  89 }
  90
  91 ############################
  92 # grab a lock file. Not atomic, but close :)
  93 # tries to cope with NFS
  94 lock_file() {
  95         lck="$1"
  96         machine=`cat "$lck" 2> /dev/null | cut -d: -f1`
  97         pid=`cat "$lck" 2> /dev/null | cut -d: -f2`
  98
  99         if [ -f "$lck" ] &&
 100             ( [ $machine != $host ] || kill -0 $pid ) 2> /dev/null; then
 101                 echo "lock file $lck is valid for $machine:$pid"
 102                 return 1
 103         fi
 104         /bin/rm -f "$lck"
 105         echo "$host:$$" > "$lck"
 106         return 0
 107 }
 108
 109 ############################
 110 # unlock a lock file
 111 unlock_file() {
 112         lck="$1"
 113         /bin/rm -f "$lck"
 114 }
 115
 116 ############################
 117 # run make, and print trace
 118 do_make() {
 119
 120   if [ x"$MAKE" = x ]
 121   then
 122     MAKE=make
 123   fi
 124
 125   for t in $*; do
 126     if [ x"$BUILD_FARM_NUM_JOBS" = x ]; then
 127       echo "$MAKE $t"
 128       ./timelimit $MAXTIME "$MAKE" "$t"
 129       status=$?
 130     else
 131       # we can parallelize everything and all targets
 132       if [ x"$t" = xeverything ] || [ x"$t" = xall]; then
 133         echo "$MAKE" "-j$BUILD_FARM_NUM_JOBS"  "$t"
 134         ./timelimit $MAXTIME "$MAKE" "-j$BUILD_FARM_NUM_JOBS"  "$t"
 135         status=$?
 136       else
 137         echo "$MAKE $t"
 138         ./timelimit $MAXTIME "$MAKE" "$t"
 139         status=$?
 140       fi
 141     fi
 142
 143     if [ $status != 0 ]; then
 144       return $status;
 145     fi
 146
 147   done
 148
 149   return 0
 150 }
 151
 152 ############################
 153 # configure the tree
 154 action_configure() {
 155         if [ ! -x configure ]; then
 156             ls -l configure
 157             ./autogen.sh
 158         fi
 159         echo "CFLAGS=$CFLAGS"
 160         echo configure options: $config_and_prefix
 161         echo CC="$CCACHE $compiler" $srcdir/configure $config_and_prefix
 162         CC="$CCACHE $compiler"
 163         export CC
 164         ./timelimit $MAXTIME $srcdir/configure $config_and_prefix
 165         cstatus=$?
 166         echo "CONFIGURE STATUS: $cstatus"
 167         if [ $cstatus != 0 ]; then
 168             echo "contents of config.log:"
 169             cat config.log
 170         fi
 171         return $cstatus;
 172 }
 173
 174 ############################
 175 # build the tree
 176 action_build() {
 177         case "$tree" in
 178         samba4)
 179                 # the 2nd 'make everything' is to work around a bug
 180                 # in netbsd make.
 181                 do_make proto showflags everything
 182                 do_make everything
 183                 if [ -n "$smbtorture4" ]; then
 184                     if [ -f bin/smbtorture ]; then
 185                         if [ -f $smbtorture4 ]; then
 186                             rm $smbtorture4
 187                         fi
 188                         cp bin/smbtorture $smbtorture4
 189                     fi
 190                 fi
 191                 ;;
 192         samba|samba_3_0)
 193             do_make proto everything torture
 194             ;;
 195         *)
 196             do_make all
 197             ;;
 198         esac
 199
 200         bstatus=$?
 201         echo "BUILD STATUS: $bstatus"
 202         return $bstatus
 203 }
 204
 205 ############################
 206 # show static analysis results
 207 action_cc_checker() {
 208
 209         # default to passing the cc_checker
 210         cccstatus=0
 211
 212         if [ -f ibm_checker.out ]; then
 213                 cat ibm_checker.out
 214                 cccstatus=`cat ibm_checker.out | grep '^\-\- ' | wc -l`
 215         fi
 216
 217         echo "CC_CHECKER STATUS: $cccstatus"
 218         return $cccstatus;
 219 }
 220
 221 ############################
 222 # install the tree
 223 action_install() {
 224         if [ -d $prefix ]; then
 225                 if [ "$noclean" != "yes" ]; then
 226                     rm -rf $prefix
 227                 fi
 228         fi
 229
 230         do_make install
 231         istatus=$?
 232         echo "INSTALL STATUS: $istatus"
 233         return $istatus;
 234 }
 235
 236 ############################
 237 # test the tree
 238 action_test_samba() {
 239         do_make test
 240         totalstatus=$?
 241         return "$totalstatus"
 242 }
 243
 244 action_test_generic() {
 245         CC="$compiler"
 246         export CC
 247         do_make installcheck
 248         totalstatus=$?
 249         echo "TEST STATUS: $totalstatus"
 250         return "$totalstatus"
 251 }
 252
 253 action_test_lorikeet_heimdal() {
 254         CC="$compiler"
 255         export CC
 256         do_make check
 257         totalstatus=$?
 258         echo "TEST STATUS: $totalstatus"
 259         return "$totalstatus"
 260 }
 261
 262
 263 #############################
 264 # attempt some basic tests of functionaility
 265 # starting as basic as possible, and getting incresingly complex
 266
 267 action_test() {
 268         # Samba needs crufty code of its own for backward
 269         # compatiblity.  I think a better way to do this in the future
 270         # is to just call 'make installcheck'.
 271         case "$tree" in
 272         samba*|smb-build)
 273             action_test_samba
 274             ;;
 275         lorikeet-heimdal*)
 276             action_test_lorikeet_heimdal
 277             ;;
 278         *)
 279             action_test_generic
 280             ;;
 281         esac
 282 }
 283
 284 ###########################
 285 # do a test build of a particular tree
 286 test_tree() {
 287         tree=$1
 288         source=$2
 289         compiler="$3"
 290         shift
 291         shift
 292         shift
 293         if [ "$compiler" = "gcc" ] && [ "$tree" != "ccache" ] && ccache -V > /dev/null; then
 294             CCACHE="ccache"
 295             export CCACHE
 296         else
 297             CCACHE=""
 298         fi
 299
 300         # limit our resource usage
 301         ulimit -t $MAXTIME
 302         ulimit -m 100000
 303
 304         # Keep stuff private
 305         umask 077
 306
 307         if [ -z "$test_root" ]; then
 308                 test_root=`pwd`
 309         fi
 310
 311         log="build.$tree.$host.$compiler.log"
 312         err="build.$tree.$host.$compiler.err"
 313         sum="build.$tree.$host.$compiler.sum"
 314         lck="$test_root/build.$tree.lck"
 315         srcdir="$test_root/$tree/$source"
 316
 317         case "$tree" in
 318             tdb | talloc | ldb)
 319                 builddir=$srcdir/tmpbuild
 320                 export builddir
 321             ;;
 322             *)
 323                 builddir=$srcdir
 324                 export builddir
 325             ;;
 326         esac
 327
 328         if ! lock_file "$lck"; then
 329                 return
 330         fi
 331
 332         # pull the svn entries, if any
 333         if fetch_svn "$tree"; then
 334             rm -f $test_root/$tree.$compiler.svn.old
 335             mv $test_root/$tree.$compiler.svn $test_root/$tree.$compiler.svn.old
 336             cp $test_root/$tree.svn $test_root/$tree.$compiler.svn
 337             if cmp $test_root/$tree.$compiler.svn $test_root/$tree.$compiler.svn.old > /dev/null; then
 338                 echo "skip: $tree.$compiler nothing changed in svn"
 339                 touch "$log" "$err"
 340                 send_logs "$log" "$err"
 341                 unlock_file "$lck"
 342                 return
 343             fi
 344         fi
 345
 346         # pull the tree
 347         if ! fetch_tree "$tree"; then
 348             unlock_file "$lck"
 349             return
 350         fi
 351
 352         if [ ! -x $srcdir/configure ]; then
 353                 echo "skip: $tree.$compiler configure not present, try again next time!"
 354                 unlock_file "$lck"
 355                 return
 356         fi
 357
 358         if [ ! $USER = "" ]; then
 359             whoami=$USER
 360         else
 361             if [ ! $LOGNAME = "" ]; then
 362                 whoami=$LOGNAME
 363             else
 364                 whoami=build
 365             fi
 366         fi
 367
 368         prefix="$test_root/prefix"
 369         if [ ! -d $prefix ]; then
 370                 mkdir $prefix
 371         fi
 372         prefix="$prefix/$tree"
 373
 374         smbtorture4=$test_root/smbtorture4
 375         export smbtorture4
 376
 377         case "$tree" in
 378         samba4)
 379             config="$config --enable-socket-wrapper"
 380             ;;
 381         samba|samba_3_0)
 382             config="$config --enable-socket-wrapper"
 383             if [ -f $smbtorture4 ]; then
 384                 # we create a local copy to make sure the same binary is used for all tests
 385                 cp $smbtorture4 $smbtorture4.$tree
 386                 config="$config --with-smbtorture4-path=$smbtorture4.$tree"
 387             fi
 388             ;;
 389         ldb)
 390             # try to avoid having extra copies of popt lying around, or
 391             # depending on libpopt-dev on every system
 392             config="$config --with-popt-src=../../samba4/source/lib/popt"
 393             ;;
 394         *)
 395             testsuite=testsuite
 396             ;;
 397         esac
 398
 399         config_and_prefix="$config --prefix=$prefix"
 400
 401         # see if we need to rebuild
 402         sum_tree $test_root $tree $sum
 403         echo "CFLAGS=$CFLAGS $config_and_prefix" >> $sum
 404
 405         if cmp "$sum" "$sum.old" > /dev/null; then
 406                 echo "skip: $tree.$compiler nothing changed"
 407                 touch "$log" "$err"
 408                 send_logs "$log" "$err"
 409                 unlock_file "$lck"
 410                 return
 411         fi
 412
 413         # we do need to rebuild - save the old sum
 414         /bin/rm -f $sum.old
 415         mv $sum $sum.old
 416
 417         actions="$*"
 418
 419         if [ "$actions" = "" ]; then
 420             actions="configure build install test"
 421         fi
 422
 423         # start the build
 424         (
 425                 uname -a
 426
 427                 # we need to be able to see if a build farm machine is accumulating
 428                 # stuck processes. We do this in two ways, as we don't know what style
 429                 # of ps it will have
 430                 ps xfuw 2> /dev/null
 431                 ps -fu $USER 2> /dev/null
 432
 433                 echo "building $tree with CC=$compiler on $host at "`date`
 434
 435                 # build the timelimit utility
 436                 echo "Building timelimit"
 437                 mkdir -p $builddir
 438                 $compiler $TIMELIMIT_FLAGS -o $builddir/timelimit $test_root/timelimit.c || exit 1
 439
 440                 if [ -r $test_root/$tree.svn ]; then
 441                   h_rev=`grep revision= $test_root/$tree.svn | head -1 | cut -d'"' -f2`
 442                   if [ -n "$h_rev" ]; then
 443                         echo "HIGHEST SVN REVISION: $h_rev"
 444                   fi
 445                   rev=`grep committed-rev= $test_root/$tree.svn | head -1 | cut -d'"' -f2`
 446                   if [ -n "$rev" ]; then
 447                         echo "BUILD REVISION: $rev"
 448                   fi
 449                 fi
 450
 451                 for action in $actions; do
 452
 453                     echo Running action $action
 454
 455                     date
 456
 457                     cd $builddir || exit 1
 458                     export srcdir
 459                     df .
 460                     mount
 461                     vmstat
 462
 463                     ( action_$action )
 464                     action_status=$?
 465
 466                     if [ $action_status != 0 ]; then
 467                         echo "ACTION FAILED: $action";
 468                     else
 469                         echo "ACTION PASSED: $action";
 470                     fi
 471
 472                     if [ $action_status != 0 ]; then
 473                         break;
 474                     fi
 475
 476                 done
 477
 478                 if [ "$noclean" = "yes" ]; then
 479                     echo cleanup skipped!
 480                 else
 481                     echo cleaning up
 482                     do_make clean
 483                     /bin/rm -rf $prefix
 484                 fi
 485                 date
 486         ) > "$log" 2> "$err"
 487
 488         # send the logs to the master site
 489         send_logs "$log" "$err"
 490
 491         # cleanup
 492         unlock_file "$lck"
 493 }
 494
 495
 496 #########################################################
 497 # enable this on a per host basis only when needed please
 498 # (at least for the moment)
 499 kill_old_processes() {
 500     # this should work on systems with linux like ps
 501     (ps uxfw | grep /build | grep -v grep | egrep 'AM|PM|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
 502     # and this should work on sysv style ps
 503     (ps -fu $USER | grep /build | grep -v grep | egrep 'AM|PM|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' | awk '{print $2}' | xargs kill -9) 2> /dev/null
 504 }
 505
 506
 507 # this is a special fn that allows us to add a "special" hook to the build
 508 # farm that we want to do to the build farm. never leave it empty. instead,
 509 # use ":" as the fn body.
 510 per_run_hook() {
 511     # kill old processes on systems with a known problem
 512     case $host in
 513         nohost)
 514             echo "just a placeholder";
 515             ;;
 516     esac
 517 }
 518
 519
 520 ######################################################
 521 # main code that is run on each call to the build code
 522 rsync --timeout=200 -q -az samba.org::build_farm/*.c .
 523
 524
 525 # build.log can grow to an excessive size, trim it beyond 50M
 526 if [ -f build.log ]; then
 527   find build.log -size +100000 -exec /bin/rm '{}' \;
 528 fi
 529