2 # Simple database query script for the buildfarm
4 # Copyright (C) Andrew Tridgell <tridge@samba.org> 2001-2005
5 # Copyright (C) Andrew Bartlett <abartlet@samba.org> 2001
6 # Copyright (C) Vance Lankhaar <vance@samba.org> 2002-2005
7 # Copyright (C) Martin Pool <mbp@samba.org> 2001
8 # Copyright (C) Jelmer Vernooij <jelmer@samba.org> 2007-2010
10 # This program is free software; you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation; either version 2 of the License, or
13 # (at your option) any later version.
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License
21 # along with this program; if not, write to the Free Software
22 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26 from cStringIO import StringIO
34 class BuildStatus(object):
36 def __init__(self, stages=None, other_failures=None):
37 if stages is not None:
41 if other_failures is not None:
42 self.other_failures = other_failures
44 self.other_failures = set()
46 def broken_host(self):
47 if "disk full" in self.other_failures:
51 def _status_tuple(self):
52 return [v for (k, v) in self.stages]
54 def regressed_since(self, other):
55 """Check if this build has regressed since another build."""
56 if "disk full" in self.other_failures:
58 return cmp(self._status_tuple(), other._status_tuple())
61 return repr((self.stages, self.other_failures))
64 def check_dir_exists(kind, path):
65 if not os.path.isdir(path):
66 raise Exception("%s directory %s does not exist" % (kind, path))
69 def build_status_from_logs(log, err):
70 """get status of build"""
79 m = re.match("^([A-Z_]+) STATUS:(\s*\d+)$", l)
81 stages.append((m.group(1), int(m.group(2).strip())))
82 if m.group(1) == "TEST":
85 m = re.match("^ACTION (PASSED|FAILED):\s+test$", l)
86 if m and not test_seen:
87 if m.group(1) == "PASSED":
88 stages.append(("TEST", 0))
90 stages.append(("TEST", 1))
93 if l.startswith("No space left on device"):
94 ret.other_failures.add("disk full")
96 if l.startswith("maximum runtime exceeded"):
97 ret.other_failures.add("timeout")
99 m = re.match("^(PANIC|INTERNAL ERROR):.*$", l)
101 ret.other_failures.add("panic")
103 if l.startswith("testsuite-failure: ") or l.startswith("testsuite-error: "):
106 if l.startswith("testsuite-success: "):
110 # Scan err file for specific errors
112 if "No space left on device" in l:
113 ret.other_failures.add("disk full")
115 stage_results = dict(stages)
116 def map_stage(name, result):
118 return (name, result)
120 if test_successes + test_failures == 0:
121 # No granular test output
122 return ("TEST", result)
123 if result == 1 and test_failures == 0:
124 ret.other_failures.add("inconsistent test result")
126 return ("TEST", test_failures)
128 ret.stages = [map_stage(name, result) for (name, result) in stages]
132 def lcov_extract_percentage(text):
133 m = re.search('\<td class="headerItem".*?\>Code\ \;covered\:\<\/td\>.*?\n.*?\<td class="headerValue".*?\>([0-9.]+) \%', text)
140 class NoSuchBuildError(Exception):
141 """The build with the specified name does not exist."""
143 def __init__(self, tree, host, compiler, rev=None):
146 self.compiler = compiler
151 """A tree to build."""
153 def __init__(self, name, scm, repo, branch, subdir="", srcdir=""):
163 return "<%s %r>" % (self.__class__.__name__, self.name)
167 """A single build of a tree on a particular host using a particular compiler.
170 def __init__(self, store, tree, host, compiler, rev=None):
174 self.compiler = compiler
178 # the mtime age is used to determine if builds are still happening
180 # the ctime age is used to determine when the last real build happened
183 """get the age of build from mtime"""
184 file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
186 st = os.stat("%s.log" % file)
187 return time.time() - st.st_mtime
190 """get the age of build from ctime"""
191 file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
193 st = os.stat("%s.log" % file)
194 return time.time() - st.st_ctime
197 """read full log file"""
198 return open(self._store.build_fname(self.tree, self.host, self.compiler, self.rev)+".log", "r")
201 """read full err file"""
203 return open(self._store.build_fname(self.tree, self.host, self.compiler, self.rev)+".err", 'r')
209 def log_checksum(self):
212 return hashlib.sha1(f.read()).hexdigest()
216 def revision_details(self):
217 """get the revision of build
219 :return: Tuple with revision id and timestamp (if available)
221 file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
226 f = open("%s.log" % file, 'r')
228 for l in f.readlines():
229 if l.startswith("BUILD COMMIT REVISION: "):
230 commit_revid = l.split(":", 1)[1].strip()
231 elif l.startswith("BUILD REVISION: "):
232 revid = l.split(":", 1)[1].strip()
233 elif l.startswith("BUILD COMMIT TIME"):
234 timestamp = l.split(":", 1)[1].strip()
238 return (revid, commit_revid, timestamp)
241 """get status of build
243 :return: tuple with build status
245 log = self.read_log()
247 err = self.read_err()
249 return build_status_from_logs(log, err)
256 """get status of build"""
257 file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
260 err = util.FileLoad("%s.err" % file)
262 # File does not exist
265 return util.count_lines(err)
268 class CachingBuild(Build):
269 """Build subclass that caches some of the results that are expensive
272 def revision_details(self):
273 file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
274 cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)
275 st1 = os.stat("%s.log" % file)
278 st2 = os.stat("%s.revision" % cachef)
280 # File does not exist
283 # the ctime/mtime asymmetry is needed so we don't get fooled by
284 # the mtime update from rsync
285 if st2 and st1.st_ctime <= st2.st_mtime:
286 (revid, commit_revid, timestamp) = util.FileLoad("%s.revision" % cachef).split(":", 2)
291 if commit_revid == "":
293 return (revid, commit_revid, timestamp)
294 (revid, commit_revid, timestamp) = super(CachingBuild, self).revision_details()
295 if not self._store.readonly:
296 util.FileSave("%s.revision" % cachef, "%s:%s:%s" % (revid, commit_revid or "", timestamp or ""))
297 return (revid, commit_revid, timestamp)
300 file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
301 cachef = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)
302 st1 = os.stat("%s.err" % file)
305 st2 = os.stat("%s.errcount" % cachef)
307 # File does not exist
310 if st2 and st1.st_ctime <= st2.st_mtime:
311 return util.FileLoad("%s.errcount" % cachef)
313 ret = super(CachingBuild, self).err_count()
315 if not self._store.readonly:
316 util.FileSave("%s.errcount" % cachef, str(ret))
321 file = self._store.build_fname(self.tree, self.host, self.compiler, self.rev)
322 cachefile = self._store.cache_fname(self.tree, self.host, self.compiler, self.rev)+".status"
324 st1 = os.stat("%s.log" % file)
327 st2 = os.stat(cachefile)
332 if st2 and st1.st_ctime <= st2.st_mtime:
333 return BuildStatus(*eval(util.FileLoad(cachefile)))
335 ret = super(CachingBuild, self).status()
337 if not self._store.readonly:
338 util.FileSave(cachefile, str(ret))
343 def read_trees_from_conf(path):
344 """Read trees from a configuration file."""
346 cfp = ConfigParser.ConfigParser()
347 cfp.readfp(open(path))
348 for s in cfp.sections():
349 ret[s] = Tree(name=s, **dict(cfp.items(s)))
353 class BuildResultStore(object):
354 """The build farm build result database."""
360 def __init__(self, basedir, readonly=False):
361 """Open the database.
363 :param basedir: Build result base directory
364 :param readonly: Whether to avoid saving cache files
366 self.basedir = basedir
367 check_dir_exists("base", self.basedir)
368 self.readonly = readonly
370 self.webdir = os.path.join(basedir, "web")
371 check_dir_exists("web", self.webdir)
373 self.datadir = os.path.join(basedir, "data")
374 check_dir_exists("data", self.datadir)
376 self.cachedir = os.path.join(basedir, "cache")
377 check_dir_exists("cache", self.cachedir)
379 self.lcovdir = os.path.join(basedir, "lcov/data")
380 check_dir_exists("lcov", self.lcovdir)
382 self.compilers = util.load_list(os.path.join(self.webdir, "compilers.list"))
384 self.trees = read_trees_from_conf(os.path.join(self.webdir, "trees.conf"))
386 def get_build(self, tree, host, compiler, rev=None):
387 logf = self.build_fname(tree, host, compiler, rev) + ".log"
388 if not os.path.exists(logf):
389 raise NoSuchBuildError(tree, host, compiler, rev)
390 return CachingBuild(self, tree, host, compiler, rev)
392 def cache_fname(self, tree, host, compiler, rev=None):
394 return os.path.join(self.cachedir, "build.%s.%s.%s-%s" % (tree, host, compiler, rev))
396 return os.path.join(self.cachedir, "build.%s.%s.%s" % (tree, host, compiler))
398 def build_fname(self, tree, host, compiler, rev=None):
399 """get the name of the build file"""
401 return os.path.join(self.datadir, "oldrevs/build.%s.%s.%s-%s" % (tree, host, compiler, rev))
402 return os.path.join(self.datadir, "upload/build.%s.%s.%s" % (tree, host, compiler))
404 def lcov_status(self, tree):
405 """get status of build"""
406 cachefile = os.path.join(self.cachedir, "lcov.%s.%s.status" % (
407 self.LCOVHOST, tree))
408 file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "index.html")
412 # File does not exist
413 raise NoSuchBuildError(tree, self.LCOVHOST, "lcov")
415 st2 = os.stat(cachefile)
417 # file does not exist
420 if st2 and st1.st_ctime <= st2.st_mtime:
421 ret = util.FileLoad(cachefile)
426 lcov_html = util.FileLoad(file)
427 perc = lcov_extract_percentage(lcov_html)
433 util.FileSave(cachefile, ret)
436 def get_old_revs(self, tree, host, compiler):
437 """get a list of old builds and their status."""
439 directory = os.path.join(self.datadir, "oldrevs")
440 logfiles = [d for d in os.listdir(directory) if d.startswith("build.%s.%s.%s-" % (tree, host, compiler)) and d.endswith(".log")]
442 m = re.match(".*-([0-9A-Fa-f]+).log$", l)
445 stat = os.stat(os.path.join(directory, l))
446 # skip the current build
447 if stat.st_nlink == 2:
449 build = self.get_build(tree, host, compiler, rev)
451 "STATUS": build.status(),
453 "TIMESTAMP": build.age_ctime(),
457 ret.sort(lambda a, b: cmp(a["TIMESTAMP"], b["TIMESTAMP"]))
461 def has_host(self, host):
462 for name in os.listdir(os.path.join(self.datadir, "upload")):
464 if name.split(".")[2] == host:
470 def host_age(self, host):
471 """get the overall age of a host"""
472 # FIXME: Turn this into a simple SQL query, or use something in hostdb ?
474 for compiler in self.compilers:
475 for tree in self.trees:
477 build = self.get_build(tree, host, compiler)
478 except NoSuchBuildError:
481 ret = min(ret, build.age_mtime())