buildfarm/build.py

   1 #!/usr/bin/python
   2 # Simple database query script for the buildfarm
   3 #
   4 # Copyright (C) Andrew Tridgell <tridge@samba.org>     2001-2005
   5 # Copyright (C) Andrew Bartlett <abartlet@samba.org>   2001
   6 # Copyright (C) Vance Lankhaar  <vance@samba.org>      2002-2005
   7 # Copyright (C) Martin Pool <mbp@samba.org>            2001
   8 # Copyright (C) Jelmer Vernooij <jelmer@samba.org>         2007-2010
   9 #
  10 #   This program is free software; you can redistribute it and/or modify
  11 #   it under the terms of the GNU General Public License as published by
  12 #   the Free Software Foundation; either version 2 of the License, or
  13 #   (at your option) any later version.
  14 #
  15 #   This program is distributed in the hope that it will be useful,
  16 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 #   GNU General Public License for more details.
  19 #
  20 #   You should have received a copy of the GNU General Public License
  21 #   along with this program; if not, write to the Free Software
  22 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  23
  24 import bz2
  25 from cStringIO import StringIO
  26 import collections
  27 import hashlib
  28 import os
  29 import re
  30 from storm.locals import Int, RawStr
  31 from storm.store import Store
  32 from storm.expr import Desc
  33 import time
  34
  35
  36 def open_opt_compressed_file(path):
  37     try:
  38         return bz2.BZ2File(path+".bz2", 'r')
  39     except IOError:
  40         return open(path, 'r')
  41
  42
  43 class Test(object):
  44
  45     def __init__(self, name):
  46         self.name = name
  47
  48
  49
  50 class TestResult(object):
  51
  52     def __init__(self, build, test, result):
  53         self.build = build
  54         self.test = test
  55         self.result = result
  56
  57
  58 class BuildSummary(object):
  59
  60     def __init__(self, host, tree, compiler, revision, status):
  61         self.host = host
  62         self.tree = tree
  63         self.compiler = compiler
  64         self.revision = revision
  65         self.status = status
  66
  67
  68 BuildStageResult = collections.namedtuple("BuildStageResult", "name result")
  69
  70
  71 class MissingRevisionInfo(Exception):
  72     """Revision info could not be found in the build log."""
  73
  74     def __init__(self, build=None):
  75         self.build = build
  76
  77
  78 class LogFileMissing(Exception):
  79     """Log file missing."""
  80
  81
  82 class BuildStatus(object):
  83
  84     def __init__(self, stages=None, other_failures=None):
  85         if stages is not None:
  86             self.stages = [BuildStageResult(n, r) for (n, r) in stages]
  87         else:
  88             self.stages = []
  89         if other_failures is not None:
  90             self.other_failures = other_failures
  91         else:
  92             self.other_failures = set()
  93
  94     @property
  95     def failed(self):
  96         if self.other_failures:
  97             return True
  98         return not all([x.result == 0 for x in self.stages])
  99
 100     def __serialize__(self):
 101         return repr(self)
 102
 103     @classmethod
 104     def __deserialize__(cls, text):
 105         return eval(text)
 106
 107     def __str__(self):
 108         if self.other_failures:
 109             return ",".join(self.other_failures)
 110         return "/".join([str(x.result) for x in self.stages])
 111
 112     def broken_host(self):
 113         if "disk full" in self.other_failures:
 114             return True
 115         return False
 116
 117     def regressed_since(self, older):
 118         """Check if this build has regressed since another build."""
 119         if "disk full" in self.other_failures:
 120             return False
 121         if ("timeout" in self.other_failures and
 122             "timeout" in older.other_failures):
 123             # When the timeout happens exactly can differ slightly, so it's
 124             # okay if the numbers are a bit different..
 125             return False
 126         if ("panic" in self.other_failures and
 127             not "panic" in older.other_failures):
 128             # If this build introduced panics, then that's always worse.
 129             return True
 130         if len(self.stages) < len(older.stages):
 131             # Less stages completed
 132             return True
 133         old_stages = dict(older.stages)
 134         new_stages = dict(self.stages)
 135         for name, new_result in new_stages.iteritems():
 136             try:
 137                 old_result = old_stages[name]
 138             except KeyError:
 139                 continue
 140             if new_result == old_result:
 141                 continue
 142             if new_result < 0 and old_result >= 0:
 143                 return True
 144             elif new_result >= 0 and old_result < 0:
 145                 return False
 146             else:
 147                 return (abs(new_result) > abs(old_result))
 148         return False
 149
 150     def __cmp__(self, other):
 151         other_extra = other.other_failures - self.other_failures
 152         self_extra = self.other_failures - other.other_failures
 153         # Give more importance to other failures
 154         if other_extra:
 155             return 1
 156         if self_extra:
 157             return -1
 158
 159         la = len(self.stages)
 160         lb = len(other.stages)
 161         if la > lb:
 162             return 1
 163         elif lb > la:
 164             return -1
 165         else:
 166             return cmp(other.stages, self.stages)
 167
 168     def __repr__(self):
 169         return "%s(%r, %r)" % (self.__class__.__name__, self.stages, self.other_failures)
 170
 171
 172 def check_dir_exists(kind, path):
 173     if not os.path.isdir(path):
 174         raise Exception("%s directory %s does not exist" % (kind, path))
 175
 176
 177 def extract_phase_output(f):
 178     name = None
 179     output = None
 180     re_action = re.compile("^ACTION (PASSED|FAILED):\s+(.*)$")
 181     for l in f:
 182         if l.startswith("Running action "):
 183             name = l[len("Running action "):].strip()
 184             output = []
 185             continue
 186         m = re_action.match(l)
 187         if m:
 188             assert name == m.group(2).strip(), "%r != %r" % (name, m.group(2))
 189             yield name, output
 190             name = None
 191             output = []
 192         elif output is not None:
 193             output.append(l)
 194
 195
 196 def extract_test_output(f):
 197     for name, output in extract_phase_output(f):
 198         if name == "test":
 199             return output
 200     raise NoTestOutput()
 201
 202
 203 def build_status_from_logs(log, err):
 204     """get status of build"""
 205     # FIXME: Perhaps also extract revision here?
 206
 207     test_failures = 0
 208     test_successes = 0
 209     test_seen = 0
 210     ret = BuildStatus()
 211
 212     stages = []
 213     re_status = re.compile("^([A-Z_]+) STATUS:(\s*\d+)$")
 214     re_action = re.compile("^ACTION (PASSED|FAILED):\s+test$")
 215
 216     for l in log:
 217         if l.startswith("No space left on device"):
 218             ret.other_failures.add("disk full")
 219             continue
 220         if "Maximum time expired in timelimit" in l: # Ugh.
 221             ret.other_failures.add("timeout")
 222             continue
 223         if "maximum runtime exceeded" in l: # Ugh.
 224             ret.other_failures.add("timeout")
 225             continue
 226         if l.startswith("PANIC:") or l.startswith("INTERNAL ERROR:"):
 227             ret.other_failures.add("panic")
 228             continue
 229         if l.startswith("testsuite-failure: ") or l.startswith("testsuite-error: "):
 230             test_failures += 1
 231             continue
 232         if l.startswith("testsuite-success: "):
 233             test_successes += 1
 234             continue
 235         m = re_status.match(l)
 236         if m:
 237             stages.append(BuildStageResult(m.group(1), int(m.group(2).strip())))
 238             if m.group(1) == "TEST":
 239                 test_seen = 1
 240             continue
 241         m = re_action.match(l)
 242         if m and not test_seen:
 243             if m.group(1) == "PASSED":
 244                 stages.append(BuildStageResult("TEST", 0))
 245             else:
 246                 stages.append(BuildStageResult("TEST", 1))
 247             continue
 248
 249     # Scan err file for specific errors
 250     for l in err:
 251         if "No space left on device" in l:
 252             ret.other_failures.add("disk full")
 253
 254     def map_stage(sr):
 255         if sr.name != "TEST":
 256             return sr
 257         # TEST is special
 258         if test_successes + test_failures == 0:
 259             # No granular test output
 260             return BuildStageResult("TEST", sr.result)
 261         if sr.result == 1 and test_failures == 0:
 262             ret.other_failures.add("inconsistent test result")
 263             return BuildStageResult("TEST", -1)
 264         return BuildStageResult("TEST", test_failures)
 265
 266     ret.stages = map(map_stage, stages)
 267     return ret
 268
 269
 270 def revision_from_log(log):
 271     revid = None
 272     for l in log:
 273         if l.startswith("BUILD COMMIT REVISION: "):
 274             revid = l.split(":", 1)[1].strip()
 275     if revid is None:
 276         raise MissingRevisionInfo()
 277     return revid
 278
 279
 280 class NoSuchBuildError(Exception):
 281     """The build with the specified name does not exist."""
 282
 283     def __init__(self, tree, host, compiler, rev=None):
 284         self.tree = tree
 285         self.host = host
 286         self.compiler = compiler
 287         self.rev = rev
 288
 289
 290 class NoTestOutput(Exception):
 291     """The build did not have any associated test output."""
 292
 293
 294 class Build(object):
 295     """A single build of a tree on a particular host using a particular compiler.
 296     """
 297
 298     def __init__(self, basename, tree, host, compiler, rev=None):
 299         self.basename = basename
 300         self.tree = tree
 301         self.host = host
 302         self.compiler = compiler
 303         self.revision = rev
 304
 305     def __cmp__(self, other):
 306         return cmp(
 307             (self.upload_time, self.revision, self.host, self.tree, self.compiler),
 308             (other.upload_time, other.revision, other.host, other.tree, other.compiler))
 309
 310     def __eq__(self, other):
 311         return (isinstance(other, Build) and
 312                 self.log_checksum() == other.log_checksum())
 313
 314     def __repr__(self):
 315         if self.revision is not None:
 316             return "<%s: revision %s of %s on %s using %s>" % (self.__class__.__name__, self.revision, self.tree, self.host, self.compiler)
 317         else:
 318             return "<%s: %s on %s using %s>" % (self.__class__.__name__, self.tree, self.host, self.compiler)
 319
 320     def remove_logs(self):
 321         # In general, basename.log should *always* exist.
 322         if os.path.exists(self.basename+".log"):
 323             os.unlink(self.basename + ".log")
 324         if os.path.exists(self.basename+".err"):
 325             os.unlink(self.basename+".err")
 326
 327     def remove(self):
 328         self.remove_logs()
 329
 330     @property
 331     def upload_time(self):
 332         """get timestamp of build"""
 333         st = os.stat("%s.log" % self.basename)
 334         return st.st_mtime
 335
 336     @property
 337     def age(self):
 338         """get the age of build"""
 339         return time.time() - self.upload_time
 340
 341     def read_subunit(self):
 342         """read the test output as subunit"""
 343         return StringIO("".join(extract_test_output(self.read_log())))
 344
 345     def read_log(self):
 346         """read full log file"""
 347         try:
 348             return open_opt_compressed_file(self.basename+".log")
 349         except IOError:
 350             raise LogFileMissing()
 351
 352     def has_log(self):
 353         try:
 354             f = self.read_log()
 355         except LogFileMissing:
 356             return False
 357         else:
 358             f.close()
 359             return True
 360
 361     def read_err(self):
 362         """read full err file"""
 363         try:
 364             return open_opt_compressed_file(self.basename+".err")
 365         except IOError:
 366             # No such file
 367             return StringIO()
 368
 369     def log_checksum(self):
 370         f = self.read_log()
 371         try:
 372             return hashlib.sha1(f.read()).hexdigest()
 373         finally:
 374             f.close()
 375
 376     def summary(self):
 377         revid = self.revision_details()
 378         status = self.status()
 379         return BuildSummary(self.host, self.tree, self.compiler, revid, status)
 380
 381     def revision_details(self):
 382         """get the revision of build
 383
 384         :return: revision id
 385         """
 386         f = self.read_log()
 387         try:
 388             return revision_from_log(f)
 389         finally:
 390             f.close()
 391
 392     def status(self):
 393         """get status of build
 394
 395         :return: tuple with build status
 396         """
 397         log = self.read_log()
 398         try:
 399             err = self.read_err()
 400             try:
 401                 return build_status_from_logs(log, err)
 402             finally:
 403                 err.close()
 404         finally:
 405             log.close()
 406
 407     def err_count(self):
 408         """get status of build"""
 409         file = self.read_err()
 410         return len(file.readlines())
 411
 412
 413 class UploadBuildResultStore(object):
 414
 415     def __init__(self, path):
 416         """Open the database.
 417
 418         :param path: Build result base directory
 419         """
 420         self.path = path
 421
 422     def get_all_builds(self):
 423         for name in os.listdir(self.path):
 424             try:
 425                 (build, tree, host, compiler, extension) = name.split(".")
 426             except ValueError:
 427                 continue
 428             if build != "build" or extension != "log":
 429                 continue
 430             yield self.get_build(tree, host, compiler)
 431
 432     def build_fname(self, tree, host, compiler):
 433         return os.path.join(self.path, "build.%s.%s.%s" % (tree, host, compiler))
 434
 435     def has_host(self, host):
 436         for name in os.listdir(self.path):
 437             try:
 438                 if name.split(".")[2] == host:
 439                     return True
 440             except IndexError:
 441                 pass
 442         return False
 443
 444     def get_build(self, tree, host, compiler):
 445         basename = self.build_fname(tree, host, compiler)
 446         logf = "%s.log" % basename
 447         if not os.path.exists(logf):
 448             raise NoSuchBuildError(tree, host, compiler)
 449         return Build(basename, tree, host, compiler)
 450
 451
 452 class StormBuild(Build):
 453     __storm_table__ = "build"
 454
 455     id = Int(primary=True)
 456     tree = RawStr()
 457     revision = RawStr()
 458     host = RawStr()
 459     compiler = RawStr()
 460     checksum = RawStr()
 461     upload_time = Int(name="age")
 462     status_str = RawStr(name="status")
 463     basename = RawStr()
 464     host_id = Int()
 465     tree_id = Int()
 466     compiler_id = Int()
 467
 468     def status(self):
 469         return BuildStatus.__deserialize__(self.status_str)
 470
 471     def revision_details(self):
 472         return self.revision
 473
 474     def log_checksum(self):
 475         return self.checksum
 476
 477     def remove(self):
 478         super(StormBuild, self).remove()
 479         Store.of(self).remove(self)
 480
 481     def remove_logs(self):
 482         super(StormBuild, self).remove_logs()
 483         self.basename = None
 484
 485
 486 class BuildResultStore(object):
 487     """The build farm build result database."""
 488
 489     def __init__(self, basedir, store=None):
 490         from buildfarm.sqldb import memory_store
 491         if store is None:
 492             store = memory_store()
 493
 494         self.store = store
 495         self.path = basedir
 496
 497     def __contains__(self, build):
 498         try:
 499             self.get_by_checksum(build.log_checksum())
 500             return True
 501         except NoSuchBuildError:
 502             return False
 503
 504     def get_build(self, tree, host, compiler, revision=None, checksum=None):
 505         from buildfarm.sqldb import Cast
 506         expr = [
 507             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 508             Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
 509             Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
 510             ]
 511         if revision is not None:
 512             expr.append(Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
 513         if checksum is not None:
 514             expr.append(Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT"))
 515         result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
 516         ret = result.first()
 517         if ret is None:
 518             raise NoSuchBuildError(tree, host, compiler, revision)
 519         return ret
 520
 521     def build_fname(self, tree, host, compiler, rev):
 522         """get the name of the build file"""
 523         return os.path.join(self.path, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
 524
 525     def get_all_builds(self):
 526         for l in os.listdir(self.path):
 527             m = re.match("^build\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)-([0-9A-Fa-f]+).log$", l)
 528             if not m:
 529                 continue
 530             tree = m.group(1)
 531             host = m.group(2)
 532             compiler = m.group(3)
 533             rev = m.group(4)
 534             stat = os.stat(os.path.join(self.path, l))
 535             # skip the current build
 536             if stat.st_nlink == 2:
 537                 continue
 538             yield self.get_build(tree, host, compiler, rev)
 539
 540     def get_old_builds(self, tree, host, compiler):
 541         result = self.store.find(StormBuild,
 542             StormBuild.tree == tree,
 543             StormBuild.host == host,
 544             StormBuild.compiler == compiler)
 545         return result.order_by(Desc(StormBuild.upload_time))
 546
 547     def upload_build(self, build):
 548         from buildfarm.sqldb import Cast, StormHost
 549         try:
 550             existing_build = self.get_by_checksum(build.log_checksum())
 551         except NoSuchBuildError:
 552             pass
 553         else:
 554             # Already present
 555             assert build.tree == existing_build.tree
 556             assert build.host == existing_build.host
 557             assert build.compiler == existing_build.compiler
 558             return existing_build
 559         rev = build.revision_details()
 560
 561         new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
 562         for name in os.listdir(self.path):
 563             p = os.path.join(self.path, name)
 564             if p.startswith(new_basename+"."):
 565                 os.remove(p)
 566         os.link(build.basename+".log", new_basename+".log")
 567         if os.path.exists(build.basename+".err"):
 568             os.link(build.basename+".err", new_basename+".err")
 569         new_build = StormBuild(new_basename, build.tree, build.host,
 570             build.compiler, rev)
 571         new_build.checksum = build.log_checksum()
 572         new_build.upload_time = build.upload_time
 573         new_build.status_str = build.status().__serialize__()
 574         new_build.basename = new_basename
 575         host = self.store.find(StormHost,
 576             Cast(StormHost.name, "TEXT") == Cast(build.host, "TEXT")).one()
 577         assert host is not None, "Unable to find host %r" % build.host
 578         new_build.host_id = host.id
 579         self.store.add(new_build)
 580         return new_build
 581
 582     def get_by_checksum(self, checksum):
 583         from buildfarm.sqldb import Cast
 584         result = self.store.find(StormBuild,
 585             Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT")).order_by(Desc(StormBuild.upload_time))
 586         ret = result.first()
 587         if ret is None:
 588             raise NoSuchBuildError(None, None, None, None)
 589         return ret
 590
 591     def get_previous_build(self, tree, host, compiler, revision):
 592         from buildfarm.sqldb import Cast
 593         cur_build = self.get_build(tree, host, compiler, revision)
 594
 595         result = self.store.find(StormBuild,
 596             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 597             Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
 598             Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
 599             Cast(StormBuild.revision, "TEXT") != Cast(revision, "TEXT"),
 600             StormBuild.id < cur_build.id)
 601         result = result.order_by(Desc(StormBuild.id))
 602         prev_build = result.first()
 603         if prev_build is None:
 604             raise NoSuchBuildError(tree, host, compiler, revision)
 605         return prev_build
 606
 607     def get_latest_build(self, tree, host, compiler):
 608         result = self.store.find(StormBuild,
 609             StormBuild.tree == tree,
 610             StormBuild.host == host,
 611             StormBuild.compiler == compiler)
 612         result = result.order_by(Desc(StormBuild.id))
 613         build = result.first()
 614         if build is None:
 615             raise NoSuchBuildError(tree, host, compiler)
 616         return build
 617
 618
 619 class BuildDiff(object):
 620     """Represents the difference between two builds."""
 621
 622     def __init__(self, tree, old, new):
 623         self.tree = tree
 624         self.old = old
 625         self.new = new
 626         self.new_rev = new.revision_details()
 627         self.new_status = new.status()
 628
 629         self.old_rev = old.revision_details()
 630         self.old_status = old.status()
 631
 632     def is_regression(self):
 633         """Is there a regression in new build since old build?"""
 634         return self.new_status.regressed_since(self.old_status)
 635
 636     def revisions(self):
 637         """Returns the revisions introduced since old in new."""
 638         branch = self.tree.get_branch()
 639         return branch.log(from_rev=self.new.revision, exclude_revs=set([self.old.revision]))