buildfarm/build.py

   1 #!/usr/bin/python
   2 # Simple database query script for the buildfarm
   3 #
   4 # Copyright (C) Andrew Tridgell <tridge@samba.org>     2001-2005
   5 # Copyright (C) Andrew Bartlett <abartlet@samba.org>   2001
   6 # Copyright (C) Vance Lankhaar  <vance@samba.org>      2002-2005
   7 # Copyright (C) Martin Pool <mbp@samba.org>            2001
   8 # Copyright (C) Jelmer Vernooij <jelmer@samba.org>         2007-2010
   9 #
  10 #   This program is free software; you can redistribute it and/or modify
  11 #   it under the terms of the GNU General Public License as published by
  12 #   the Free Software Foundation; either version 2 of the License, or
  13 #   (at your option) any later version.
  14 #
  15 #   This program is distributed in the hope that it will be useful,
  16 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 #   GNU General Public License for more details.
  19 #
  20 #   You should have received a copy of the GNU General Public License
  21 #   along with this program; if not, write to the Free Software
  22 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  23
  24 from cStringIO import StringIO
  25 import collections
  26 import hashlib
  27 import os
  28 import re
  29 from storm.locals import Int, RawStr
  30 from storm.store import Store
  31 from storm.expr import Desc
  32 import time
  33
  34
  35 class Test(object):
  36
  37     def __init__(self, name):
  38         self.name = name
  39
  40
  41
  42 class TestResult(object):
  43
  44     def __init__(self, build, test, result):
  45         self.build = build
  46         self.test = test
  47         self.result = result
  48
  49
  50 class BuildSummary(object):
  51
  52     def __init__(self, host, tree, compiler, revision, status):
  53         self.host = host
  54         self.tree = tree
  55         self.compiler = compiler
  56         self.revision = revision
  57         self.status = status
  58
  59
  60 BuildStageResult = collections.namedtuple("BuildStageResult", "name result")
  61
  62
  63 class MissingRevisionInfo(Exception):
  64     """Revision info could not be found in the build log."""
  65
  66     def __init__(self, build=None):
  67         self.build = build
  68
  69
  70 class LogFileMissing(Exception):
  71     """Log file missing."""
  72
  73
  74 class BuildStatus(object):
  75
  76     def __init__(self, stages=None, other_failures=None):
  77         if stages is not None:
  78             self.stages = [BuildStageResult(n, r) for (n, r) in stages]
  79         else:
  80             self.stages = []
  81         if other_failures is not None:
  82             self.other_failures = other_failures
  83         else:
  84             self.other_failures = set()
  85
  86     @property
  87     def failed(self):
  88         if self.other_failures:
  89             return True
  90         return not all([x.result == 0 for x in self.stages])
  91
  92     def __serialize__(self):
  93         return repr(self)
  94
  95     @classmethod
  96     def __deserialize__(cls, text):
  97         return eval(text)
  98
  99     def __str__(self):
 100         if self.other_failures:
 101             return ",".join(self.other_failures)
 102         return "/".join([str(x.result) for x in self.stages])
 103
 104     def broken_host(self):
 105         if "disk full" in self.other_failures:
 106             return True
 107         return False
 108
 109     def regressed_since(self, older):
 110         """Check if this build has regressed since another build."""
 111         if "disk full" in self.other_failures:
 112             return False
 113         if ("timeout" in self.other_failures and
 114             "timeout" in older.other_failures):
 115             # When the timeout happens exactly can differ slightly, so it's
 116             # okay if the numbers are a bit different..
 117             return False
 118         if ("panic" in self.other_failures and
 119             not "panic" in older.other_failures):
 120             return True
 121         if len(self.stages) < len(older.stages):
 122             # Less stages completed
 123             return True
 124         for ((old_name, old_result), (new_name, new_result)) in zip(
 125             older.stages, self.stages):
 126             assert old_name == new_name
 127             if new_result > old_result:
 128                 return True
 129         return False
 130
 131     def __cmp__(self, other):
 132         other_extra = other.other_failures - self.other_failures
 133         self_extra = self.other_failures - other.other_failures
 134         # Give more importance to other failures
 135         if other_extra:
 136             return 1
 137         if self_extra:
 138             return -1
 139
 140         la = len(self.stages)
 141         lb = len(other.stages)
 142         if la > lb:
 143             return 1
 144         elif lb > la:
 145             return -1
 146         else:
 147             return cmp(other.stages, self.stages)
 148
 149     def __repr__(self):
 150         return "%s(%r, %r)" % (self.__class__.__name__, self.stages, self.other_failures)
 151
 152
 153 def check_dir_exists(kind, path):
 154     if not os.path.isdir(path):
 155         raise Exception("%s directory %s does not exist" % (kind, path))
 156
 157
 158 def extract_phase_output(f):
 159     name = None
 160     output = None
 161     for l in f:
 162
 163
 164 def extract_test_output(f):
 165     raise NotImplementedError
 166
 167
 168 def build_status_from_logs(log, err):
 169     """get status of build"""
 170     # FIXME: Perhaps also extract revision here?
 171
 172     test_failures = 0
 173     test_successes = 0
 174     test_seen = 0
 175     ret = BuildStatus()
 176
 177     stages = []
 178     re_status = re.compile("^([A-Z_]+) STATUS:(\s*\d+)$")
 179     re_action = re.compile("^ACTION (PASSED|FAILED):\s+test$")
 180
 181     for l in log:
 182         if l.startswith("No space left on device"):
 183             ret.other_failures.add("disk full")
 184             continue
 185         if "Maximum time expired in timelimit" in l: # Ugh.
 186             ret.other_failures.add("timeout")
 187             continue
 188         if "maximum runtime exceeded" in l: # Ugh.
 189             ret.other_failures.add("timeout")
 190             continue
 191         if l.startswith("PANIC:") or l.startswith("INTERNAL ERROR:"):
 192             ret.other_failures.add("panic")
 193             continue
 194         if l.startswith("testsuite-failure: ") or l.startswith("testsuite-error: "):
 195             test_failures += 1
 196             continue
 197         if l.startswith("testsuite-success: "):
 198             test_successes += 1
 199             continue
 200         m = re_status.match(l)
 201         if m:
 202             stages.append(BuildStageResult(m.group(1), int(m.group(2).strip())))
 203             if m.group(1) == "TEST":
 204                 test_seen = 1
 205             continue
 206         m = re_action.match(l)
 207         if m and not test_seen:
 208             if m.group(1) == "PASSED":
 209                 stages.append(BuildStageResult("TEST", 0))
 210             else:
 211                 stages.append(BuildStageResult("TEST", 1))
 212             continue
 213
 214     # Scan err file for specific errors
 215     for l in err:
 216         if "No space left on device" in l:
 217             ret.other_failures.add("disk full")
 218
 219     def map_stage(sr):
 220         if sr.name != "TEST":
 221             return sr
 222         # TEST is special
 223         if test_successes + test_failures == 0:
 224             # No granular test output
 225             return BuildStageResult("TEST", sr.result)
 226         if sr.result == 1 and test_failures == 0:
 227             ret.other_failures.add("inconsistent test result")
 228             return BuildStageResult("TEST", -1)
 229         return BuildStageResult("TEST", test_failures)
 230
 231     ret.stages = map(map_stage, stages)
 232     return ret
 233
 234
 235 def revision_from_log(log):
 236     revid = None
 237     for l in log:
 238         if l.startswith("BUILD COMMIT REVISION: "):
 239             revid = l.split(":", 1)[1].strip()
 240     if revid is None:
 241         raise MissingRevisionInfo()
 242     return revid
 243
 244
 245 class NoSuchBuildError(Exception):
 246     """The build with the specified name does not exist."""
 247
 248     def __init__(self, tree, host, compiler, rev=None):
 249         self.tree = tree
 250         self.host = host
 251         self.compiler = compiler
 252         self.rev = rev
 253
 254
 255 class Build(object):
 256     """A single build of a tree on a particular host using a particular compiler.
 257     """
 258
 259     def __init__(self, basename, tree, host, compiler, rev=None):
 260         self.basename = basename
 261         self.tree = tree
 262         self.host = host
 263         self.compiler = compiler
 264         self.revision = rev
 265
 266     def __cmp__(self, other):
 267         return cmp(
 268             (self.upload_time, self.revision, self.host, self.tree, self.compiler),
 269             (other.upload_time, other.revision, other.host, other.tree, other.compiler))
 270
 271     def __eq__(self, other):
 272         return (isinstance(other, Build) and
 273                 self.log_checksum() == other.log_checksum())
 274
 275     def __repr__(self):
 276         if self.revision is not None:
 277             return "<%s: revision %s of %s on %s using %s>" % (self.__class__.__name__, self.revision, self.tree, self.host, self.compiler)
 278         else:
 279             return "<%s: %s on %s using %s>" % (self.__class__.__name__, self.tree, self.host, self.compiler)
 280
 281     def remove_logs(self):
 282         # In general, basename.log should *always* exist.
 283         if os.path.exists(self.basename+".log"):
 284             os.unlink(self.basename + ".log")
 285         if os.path.exists(self.basename+".err"):
 286             os.unlink(self.basename+".err")
 287
 288     def remove(self):
 289         self.remove_logs()
 290
 291     @property
 292     def upload_time(self):
 293         """get timestamp of build"""
 294         st = os.stat("%s.log" % self.basename)
 295         return st.st_mtime
 296
 297     @property
 298     def age(self):
 299         """get the age of build"""
 300         return time.time() - self.upload_time
 301
 302     def read_subunit(self):
 303         """read the test output as subunit"""
 304         return StringIO("".join(extract_test_output(self.read_log())))
 305
 306     def read_log(self):
 307         """read full log file"""
 308         try:
 309             return open(self.basename+".log", "r")
 310         except IOError:
 311             raise LogFileMissing()
 312
 313     def read_err(self):
 314         """read full err file"""
 315         try:
 316             return open(self.basename+".err", 'r')
 317         except IOError:
 318             # No such file
 319             return StringIO()
 320
 321     def log_checksum(self):
 322         f = self.read_log()
 323         try:
 324             return hashlib.sha1(f.read()).hexdigest()
 325         finally:
 326             f.close()
 327
 328     def summary(self):
 329         revid = self.revision_details()
 330         status = self.status()
 331         return BuildSummary(self.host, self.tree, self.compiler, revid, status)
 332
 333     def revision_details(self):
 334         """get the revision of build
 335
 336         :return: revision id
 337         """
 338         f = self.read_log()
 339         try:
 340             return revision_from_log(f)
 341         finally:
 342             f.close()
 343
 344     def status(self):
 345         """get status of build
 346
 347         :return: tuple with build status
 348         """
 349         log = self.read_log()
 350         try:
 351             err = self.read_err()
 352             try:
 353                 return build_status_from_logs(log, err)
 354             finally:
 355                 err.close()
 356         finally:
 357             log.close()
 358
 359     def err_count(self):
 360         """get status of build"""
 361         file = self.read_err()
 362         return len(file.readlines())
 363
 364
 365 class UploadBuildResultStore(object):
 366
 367     def __init__(self, path):
 368         """Open the database.
 369
 370         :param path: Build result base directory
 371         """
 372         self.path = path
 373
 374     def get_all_builds(self):
 375         for name in os.listdir(self.path):
 376             try:
 377                 (build, tree, host, compiler, extension) = name.split(".")
 378             except ValueError:
 379                 continue
 380             if build != "build" or extension != "log":
 381                 continue
 382             yield self.get_build(tree, host, compiler)
 383
 384     def build_fname(self, tree, host, compiler):
 385         return os.path.join(self.path, "build.%s.%s.%s" % (tree, host, compiler))
 386
 387     def has_host(self, host):
 388         for name in os.listdir(self.path):
 389             try:
 390                 if name.split(".")[2] == host:
 391                     return True
 392             except IndexError:
 393                 pass
 394         return False
 395
 396     def get_build(self, tree, host, compiler):
 397         basename = self.build_fname(tree, host, compiler)
 398         logf = "%s.log" % basename
 399         if not os.path.exists(logf):
 400             raise NoSuchBuildError(tree, host, compiler)
 401         return Build(basename, tree, host, compiler)
 402
 403
 404 class StormBuild(Build):
 405     __storm_table__ = "build"
 406
 407     id = Int(primary=True)
 408     tree = RawStr()
 409     revision = RawStr()
 410     host = RawStr()
 411     compiler = RawStr()
 412     checksum = RawStr()
 413     upload_time = Int(name="age")
 414     status_str = RawStr(name="status")
 415     basename = RawStr()
 416     host_id = Int()
 417     tree_id = Int()
 418     compiler_id = Int()
 419
 420     def status(self):
 421         return BuildStatus.__deserialize__(self.status_str)
 422
 423     def revision_details(self):
 424         return self.revision
 425
 426     def log_checksum(self):
 427         return self.checksum
 428
 429     def remove(self):
 430         super(StormBuild, self).remove()
 431         Store.of(self).remove(self)
 432
 433     def remove_logs(self):
 434         super(StormBuild, self).remove_logs()
 435         self.basename = None
 436
 437
 438 class BuildResultStore(object):
 439     """The build farm build result database."""
 440
 441     def __init__(self, basedir, store=None):
 442         from buildfarm.sqldb import memory_store
 443         if store is None:
 444             store = memory_store()
 445
 446         self.store = store
 447         self.path = basedir
 448
 449     def __contains__(self, build):
 450         try:
 451             self.get_by_checksum(build.log_checksum())
 452             return True
 453         except NoSuchBuildError:
 454             return False
 455
 456     def get_build(self, tree, host, compiler, revision=None, checksum=None):
 457         from buildfarm.sqldb import Cast
 458         expr = [
 459             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 460             Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
 461             Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
 462             ]
 463         if revision is not None:
 464             expr.append(Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
 465         if checksum is not None:
 466             expr.append(Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT"))
 467         result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
 468         ret = result.first()
 469         if ret is None:
 470             raise NoSuchBuildError(tree, host, compiler, revision)
 471         return ret
 472
 473     def build_fname(self, tree, host, compiler, rev):
 474         """get the name of the build file"""
 475         return os.path.join(self.path, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
 476
 477     def get_all_builds(self):
 478         for l in os.listdir(self.path):
 479             m = re.match("^build\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)\.([0-9A-Za-z]+)-([0-9A-Fa-f]+).log$", l)
 480             if not m:
 481                 continue
 482             tree = m.group(1)
 483             host = m.group(2)
 484             compiler = m.group(3)
 485             rev = m.group(4)
 486             stat = os.stat(os.path.join(self.path, l))
 487             # skip the current build
 488             if stat.st_nlink == 2:
 489                 continue
 490             yield self.get_build(tree, host, compiler, rev)
 491
 492     def get_old_builds(self, tree, host, compiler):
 493         result = self.store.find(StormBuild,
 494             StormBuild.tree == tree,
 495             StormBuild.host == host,
 496             StormBuild.compiler == compiler)
 497         return result.order_by(Desc(StormBuild.upload_time))
 498
 499     def upload_build(self, build):
 500         from buildfarm.sqldb import Cast, StormHost
 501         try:
 502             existing_build = self.get_by_checksum(build.log_checksum())
 503         except NoSuchBuildError:
 504             pass
 505         else:
 506             # Already present
 507             assert build.tree == existing_build.tree
 508             assert build.host == existing_build.host
 509             assert build.compiler == existing_build.compiler
 510             return existing_build
 511         rev = build.revision_details()
 512
 513         new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
 514         try:
 515             existing_build = self.get_build(build.tree, build.host, build.compiler, rev)
 516         except NoSuchBuildError:
 517             if os.path.exists(new_basename+".log"):
 518                 os.remove(new_basename+".log")
 519             if os.path.exists(new_basename+".err"):
 520                 os.remove(new_basename+".err")
 521         else:
 522             existing_build.remove_logs()
 523         os.link(build.basename+".log", new_basename+".log")
 524         if os.path.exists(build.basename+".err"):
 525             os.link(build.basename+".err", new_basename+".err")
 526         new_basename = self.build_fname(build.tree, build.host, build.compiler,
 527                 rev)
 528         new_build = StormBuild(new_basename, build.tree, build.host,
 529             build.compiler, rev)
 530         new_build.checksum = build.log_checksum()
 531         new_build.upload_time = build.upload_time
 532         new_build.status_str = build.status().__serialize__()
 533         new_build.basename = new_basename
 534         host = self.store.find(StormHost,
 535             Cast(StormHost.name, "TEXT") == Cast(build.host, "TEXT")).one()
 536         assert host is not None, "Unable to find host %r" % build.host
 537         new_build.host_id = host.id
 538         self.store.add(new_build)
 539         return new_build
 540
 541     def get_by_checksum(self, checksum):
 542         from buildfarm.sqldb import Cast
 543         result = self.store.find(StormBuild,
 544             Cast(StormBuild.checksum, "TEXT") == checksum)
 545         ret = result.one()
 546         if ret is None:
 547             raise NoSuchBuildError(None, None, None, None)
 548         return ret
 549
 550     def get_previous_revision(self, tree, host, compiler, revision):
 551         from buildfarm.sqldb import Cast
 552         cur_build = self.get_build(tree, host, compiler, revision)
 553
 554         result = self.store.find(StormBuild,
 555             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 556             Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
 557             Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
 558             Cast(StormBuild.revision, "TEXT") != Cast(revision, "TEXT"),
 559             StormBuild.id < cur_build.id)
 560         result = result.order_by(Desc(StormBuild.id))
 561         prev_build = result.first()
 562         if prev_build is None:
 563             raise NoSuchBuildError(tree, host, compiler, revision)
 564         return prev_build.revision
 565
 566     def get_latest_revision(self, tree, host, compiler):
 567         result = self.store.find(StormBuild,
 568             StormBuild.tree == tree,
 569             StormBuild.host == host,
 570             StormBuild.compiler == compiler)
 571         result = result.order_by(Desc(StormBuild.id))
 572         build = result.first()
 573         if build is None:
 574             raise NoSuchBuildError(tree, host, compiler)
 575         return build.revision