buildfarm/__init__.py

   1 #!/usr/bin/python
   2 # Simple database query script for the buildfarm
   3 #
   4 # Copyright (C) Jelmer Vernooij <jelmer@samba.org>         2010
   5 #
   6 #   This program is free software; you can redistribute it and/or modify
   7 #   it under the terms of the GNU General Public License as published by
   8 #   the Free Software Foundation; either version 2 of the License, or
   9 #   (at your option) any later version.
  10 #
  11 #   This program is distributed in the hope that it will be useful,
  12 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 #   GNU General Public License for more details.
  15 #
  16 #   You should have received a copy of the GNU General Public License
  17 #   along with this program; if not, write to the Free Software
  18 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19
  20 from buildfarm.build import BuildStatus
  21 from buildfarm.sqldb import distinct_builds, Cast, StormBuild, setup_schema, StormHostDatabase
  22 from buildfarm.tree import Tree
  23 from storm.database import create_database
  24 from storm.expr import Desc
  25 from storm.store import Store
  26
  27 import ConfigParser
  28 import os
  29 import re
  30
  31 def read_trees_from_conf(path):
  32     """Read trees from a configuration file.
  33
  34     :param path: tree path
  35     :return: Dictionary with trees
  36     """
  37     ret = {}
  38     cfp = ConfigParser.ConfigParser()
  39     cfp.read(path)
  40     for s in cfp.sections():
  41         ret[s] = Tree(name=s, **dict(cfp.items(s)))
  42     return ret
  43
  44
  45 def lcov_extract_percentage(f):
  46     """Extract the coverage percentage from the lcov file."""
  47     m = re.search('\<td class="headerCovTableEntryLo".*?\>([0-9.]+) \%', f.read())
  48     if m:
  49         return m.group(1)
  50     else:
  51         return None
  52
  53
  54 class BuildFarm(object):
  55
  56     LCOVHOST = "coverage"
  57     OLDAGE = 60*60*4,
  58     DEADAGE = 60*60*24*4
  59
  60     def __init__(self, path=None, store=None, timeout=0.5):
  61         self.timeout = timeout
  62         self.store = store
  63         if path is None:
  64             path = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
  65         self.path = path
  66         self.webdir = os.path.join(self.path, "web")
  67         if not os.path.isdir(path):
  68             raise Exception("web directory %s does not exist" % self.webdir)
  69         self.trees = read_trees_from_conf(os.path.join(self.webdir, "trees.conf"))
  70         self.builds = self._open_build_results()
  71         self.upload_builds = self._open_upload_build_results()
  72         self.hostdb = self._open_hostdb()
  73         self.compilers = self._load_compilers()
  74         self.lcovdir = os.path.join(self.path, "lcov/data")
  75
  76     def __repr__(self):
  77         return "%s(%r)" % (self.__class__.__name__, self.path)
  78
  79     def _open_build_results(self):
  80         path = os.path.join(self.path, "data", "oldrevs")
  81         from buildfarm.build import BuildResultStore
  82         return BuildResultStore(path, self._get_store())
  83
  84     def _open_upload_build_results(self):
  85         from buildfarm.build import UploadBuildResultStore
  86         path = os.path.join(self.path, "data", "upload")
  87         return UploadBuildResultStore(path)
  88
  89     def _open_hostdb(self):
  90         return StormHostDatabase(self._get_store())
  91
  92     def _load_compilers(self):
  93         from buildfarm import util
  94         return set(util.load_list(os.path.join(self.webdir, "compilers.list")))
  95
  96     def commit(self):
  97         if self.store is not None:
  98             self.store.commit()
  99
 100     def lcov_status(self, tree):
 101         """get status of build"""
 102         from buildfarm.build import NoSuchBuildError
 103         file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "index.html")
 104         try:
 105             lcov_html = open(file, 'r')
 106         except (OSError, IOError):
 107             # File does not exist
 108             raise NoSuchBuildError(tree, self.LCOVHOST, "lcov")
 109         try:
 110             return lcov_extract_percentage(lcov_html)
 111         finally:
 112             lcov_html.close()
 113
 114     def unused_fns(self, tree):
 115         """get status of build"""
 116         from buildfarm.build import NoSuchBuildError
 117         file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "unused-fns.txt")
 118         try:
 119             unused_fns_file = open(file, 'r')
 120         except (OSError, IOError):
 121             # File does not exist
 122             raise NoSuchBuildError(tree, self.LCOVHOST, "unused_fns")
 123         try:
 124             return "unused-fns.txt"
 125         finally:
 126             unused_fns_file.close()
 127
 128     def get_build(self, tree, host, compiler, rev=None, checksum=None):
 129         if rev is not None:
 130             return self.builds.get_build(tree, host, compiler, rev,
 131                 checksum=checksum)
 132         else:
 133             return self.upload_builds.get_build(tree, host, compiler)
 134
 135     def get_new_builds(self):
 136         hostnames = set([host.name for host in self.hostdb.hosts()])
 137         for build in self.upload_builds.get_all_builds():
 138             if (build.tree in self.trees and
 139                 build.compiler in self.compilers and
 140                 build.host in hostnames):
 141                 yield build
 142
 143     def get_last_builds(self):
 144         result = self._get_store().find(StormBuild)
 145         return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
 146
 147     def get_summary_builds(self, min_age=0):
 148         """Return last build age, status for each tree/host/compiler.
 149
 150         :param min_age: Minimum timestamp of builds to report
 151         :return: iterator over tree, status
 152         """
 153         store = self._get_store()
 154         return ((tree, BuildStatus.__deserialize__(status_str))
 155                 for (tree, status_str) in store.execute("""
 156 SELECT obd.tree, obd.status AS status_str
 157 FROM build obd
 158 INNER JOIN(
 159     SELECT MAX(age) age, tree, host, compiler
 160     FROM build
 161     WHERE age > ?
 162     GROUP BY tree, host, compiler
 163 ) ibd ON obd.age = ibd.age AND
 164          obd.tree = ibd.tree AND
 165          obd.host = ibd.host AND
 166          obd.compiler = ibd.compiler;
 167 """, min_age))
 168
 169     def get_tree_builds(self, tree):
 170         result = self._get_store().find(StormBuild,
 171             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"))
 172         return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
 173
 174     def host_last_build(self, host):
 175         return max([build.upload_time for build in self.get_host_builds(host)])
 176
 177     def get_host_builds(self, host):
 178         result = self._get_store().find(StormBuild, StormBuild.host == host)
 179         return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
 180
 181     def _get_store(self):
 182         if self.store is not None:
 183             return self.store
 184         db_dir_path = os.path.join(self.path, "db")
 185         if not os.path.isdir(db_dir_path):
 186             os.mkdir(db_dir_path)
 187         db_path = os.path.join(db_dir_path, "hostdb.sqlite")
 188         db = create_database("sqlite:%s?timeout=%f" % (db_path, self.timeout))
 189         self.store = Store(db)
 190         setup_schema(self.store)
 191         return self.store
 192
 193     def get_revision_builds(self, tree, revision=None):
 194         return self._get_store().find(StormBuild,
 195             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 196             Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))