X-Git-Url: http://git.samba.org/?a=blobdiff_plain;f=buildfarm%2F__init__.py;h=7c4428e80bd4aa374ce425f68f8a9b3c0d42a615;hb=326c4af20b34726da9382673d4018e6ae58f18c1;hp=8b137891791fe96927ad78e64b0aad7bded08bdc;hpb=1bfc40bcadef4d572ee3d36680a4dc168cd76178;p=build-farm.git diff --git a/buildfarm/__init__.py b/buildfarm/__init__.py index 8b137891..7c4428e8 100644 --- a/buildfarm/__init__.py +++ b/buildfarm/__init__.py @@ -1 +1,173 @@ +#!/usr/bin/python +# Simple database query script for the buildfarm +# +# Copyright (C) Jelmer Vernooij 2010 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +from buildfarm.sqldb import distinct_builds, Cast, StormBuild, setup_schema, StormHostDatabase +from buildfarm.tree import Tree +from storm.database import create_database +from storm.expr import Desc +from storm.store import Store + +import ConfigParser +import os +import re + +def read_trees_from_conf(path): + """Read trees from a configuration file. + + :param path: tree path + :return: Dictionary with trees + """ + ret = {} + cfp = ConfigParser.ConfigParser() + cfp.read(path) + for s in cfp.sections(): + ret[s] = Tree(name=s, **dict(cfp.items(s))) + return ret + + +def lcov_extract_percentage(f): + """Extract the coverage percentage from the lcov file.""" + m = re.search('\([0-9.]+) \%', f.read()) + if m: + return m.group(1) + else: + return None + + +class BuildFarm(object): + + LCOVHOST = "coverage" + OLDAGE = 60*60*4, + DEADAGE = 60*60*24*4 + + def __init__(self, path=None, store=None, timeout=0.5): + self.timeout = timeout + self.store = store + if path is None: + path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + self.path = path + self.webdir = os.path.join(self.path, "web") + if not os.path.isdir(path): + raise Exception("web directory %s does not exist" % self.webdir) + self.trees = read_trees_from_conf(os.path.join(self.webdir, "trees.conf")) + self.builds = self._open_build_results() + self.upload_builds = self._open_upload_build_results() + self.hostdb = self._open_hostdb() + self.compilers = self._load_compilers() + self.lcovdir = os.path.join(self.path, "lcov/data") + + def __repr__(self): + return "%s(%r)" % (self.__class__.__name__, self.path) + + def _open_build_results(self): + path = os.path.join(self.path, "data", "oldrevs") + from buildfarm.build import BuildResultStore + return BuildResultStore(path, self._get_store()) + + def _open_upload_build_results(self): + from buildfarm.build import UploadBuildResultStore + path = os.path.join(self.path, "data", "upload") + return UploadBuildResultStore(path) + + def _open_hostdb(self): + return StormHostDatabase(self._get_store()) + + def _load_compilers(self): + from buildfarm import util + return set(util.load_list(os.path.join(self.webdir, "compilers.list"))) + + def commit(self): + if self.store is not None: + self.store.commit() + + def lcov_status(self, tree): + """get status of build""" + from buildfarm.build import NoSuchBuildError + file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "index.html") + try: + lcov_html = open(file, 'r') + except (OSError, IOError): + # File does not exist + raise NoSuchBuildError(tree, self.LCOVHOST, "lcov") + try: + return lcov_extract_percentage(lcov_html) + finally: + lcov_html.close() + + def unused_fns(self, tree): + """get status of build""" + from buildfarm.build import NoSuchBuildError + file = os.path.join(self.lcovdir, self.LCOVHOST, tree, "unused-fns.txt") + try: + unused_fns_file = open(file, 'r') + except (OSError, IOError): + # File does not exist + raise NoSuchBuildError(tree, self.LCOVHOST, "unused_fns") + try: + return "unused-fns.txt" + finally: + unused_fns_file.close() + + def get_build(self, tree, host, compiler, rev=None, checksum=None): + if rev is not None: + return self.builds.get_build(tree, host, compiler, rev, + checksum=checksum) + else: + return self.upload_builds.get_build(tree, host, compiler) + + def get_new_builds(self): + hostnames = set([host.name for host in self.hostdb.hosts()]) + for build in self.upload_builds.get_all_builds(): + if (build.tree in self.trees and + build.compiler in self.compilers and + build.host in hostnames): + yield build + + def get_last_builds(self): + result = self._get_store().find(StormBuild) + return distinct_builds(result.order_by(Desc(StormBuild.upload_time))) + + def get_tree_builds(self, tree): + result = self._get_store().find(StormBuild, + Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT")) + return distinct_builds(result.order_by(Desc(StormBuild.upload_time))) + + def host_last_build(self, host): + return max([build.upload_time for build in self.get_host_builds(host)]) + + def get_host_builds(self, host): + result = self._get_store().find(StormBuild, StormBuild.host == host) + return distinct_builds(result.order_by(Desc(StormBuild.upload_time))) + + def _get_store(self): + if self.store is not None: + return self.store + db_dir_path = os.path.join(self.path, "db") + if not os.path.isdir(db_dir_path): + os.mkdir(db_dir_path) + db_path = os.path.join(db_dir_path, "hostdb.sqlite") + db = create_database("sqlite:%s?timeout=%f" % (db_path, self.timeout)) + self.store = Store(db) + setup_schema(self.store) + return self.store + + def get_revision_builds(self, tree, revision=None): + return self._get_store().find(StormBuild, + Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"), + Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))