buildfarm/sqldb.py

   1 #!/usr/bin/python
   2
   3 # Samba.org buildfarm
   4 # Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18 #
  19
  20 from buildfarm import (
  21     BuildFarm,
  22     )
  23 from buildfarm.data import (
  24     Build,
  25     BuildResultStore,
  26     BuildStatus,
  27     NoSuchBuildError,
  28     )
  29 from buildfarm.hostdb import (
  30     Host,
  31     HostDatabase,
  32     HostAlreadyExists,
  33     NoSuchHost,
  34     )
  35
  36 import os
  37 try:
  38     from pysqlite2 import dbapi2 as sqlite3
  39 except ImportError:
  40     import sqlite3
  41 from storm.database import create_database
  42 from storm.locals import Bool, Desc, Int, Unicode, RawStr
  43 from storm.store import Store
  44
  45
  46 class StormBuild(Build):
  47     __storm_table__ = "build"
  48
  49     id = Int(primary=True)
  50     tree = Unicode()
  51     revision = RawStr()
  52     host = Unicode()
  53     compiler = Unicode()
  54     checksum = RawStr()
  55     age = Int()
  56     status_str = Unicode(name="status")
  57     commit_revision = RawStr()
  58
  59     def status(self):
  60         return BuildStatus.__deserialize__(self.status_str)
  61
  62     def revision_details(self):
  63         return (self.commit_revision, None)
  64
  65     def log_checksum(self):
  66         return self.checksum
  67
  68     def remove(self):
  69         super(StormBuild, self).remove()
  70         Store.of(self).remove(self)
  71
  72
  73 class StormHost(Host):
  74     __storm_table__ = "host"
  75
  76     name = Unicode(primary=True)
  77     owner_name = Unicode(name="owner")
  78     owner_email = Unicode()
  79     password = Unicode()
  80     ssh_access = Bool()
  81     fqdn = RawStr()
  82     platform = Unicode()
  83     permission = Unicode()
  84     last_dead_mail = Int()
  85     join_time = Int()
  86
  87     def _set_owner(self, value):
  88         if value is None:
  89             self.owner_name = None
  90             self.owner_email = None
  91         else:
  92             (self.owner_name, self.owner_email) = value
  93
  94     def _get_owner(self):
  95         if self.owner_name is None:
  96             return None
  97         else:
  98             return (self.owner_name, self.owner_email)
  99
 100     owner = property(_get_owner, _set_owner)
 101
 102
 103 class StormHostDatabase(HostDatabase):
 104
 105     def __init__(self, store=None):
 106         if store is None:
 107             self.store = memory_store()
 108         else:
 109             self.store = store
 110
 111     def createhost(self, name, platform=None, owner=None, owner_email=None,
 112             password=None, permission=None):
 113         """See `HostDatabase.createhost`."""
 114         newhost = StormHost(unicode(name), owner=owner, owner_email=owner_email, password=password, permission=permission, platform=platform)
 115         try:
 116             self.store.add(newhost)
 117             self.store.flush()
 118         except sqlite3.IntegrityError:
 119             raise HostAlreadyExists(name)
 120         return newhost
 121
 122     def deletehost(self, name):
 123         """Remove a host."""
 124         host = self.host(name)
 125         self.store.remove(host)
 126
 127     def hosts(self):
 128         """Retrieve an iterable over all hosts."""
 129         return self.store.find(StormHost).order_by(StormHost.name)
 130
 131     def host(self, name):
 132         ret = self.store.find(StormHost, StormHost.name==unicode(name)).one()
 133         if ret is None:
 134             raise NoSuchHost(name)
 135         return ret
 136
 137     def commit(self):
 138         self.store.commit()
 139
 140
 141 class StormCachingBuildResultStore(BuildResultStore):
 142
 143     def __init__(self, basedir, store=None):
 144         super(StormCachingBuildResultStore, self).__init__(basedir)
 145
 146         if store is None:
 147             store = memory_store()
 148
 149         self.store = store
 150
 151     def __contains__(self, build):
 152         return (self._get_by_checksum(build) is not None)
 153
 154     def get_previous_revision(self, tree, host, compiler, revision):
 155         result = self.store.find(StormBuild,
 156             StormBuild.tree == unicode(tree),
 157             StormBuild.host == unicode(host),
 158             StormBuild.compiler == unicode(compiler),
 159             StormBuild.commit_revision == revision)
 160         cur_build = result.any()
 161         if cur_build is None:
 162             raise NoSuchBuildError(tree, host, compiler, revision)
 163
 164         result = self.store.find(StormBuild,
 165             StormBuild.tree == unicode(tree),
 166             StormBuild.host == unicode(host),
 167             StormBuild.compiler == unicode(compiler),
 168             StormBuild.commit_revision != revision,
 169             StormBuild.id < cur_build.id)
 170         result = result.order_by(Desc(StormBuild.id))
 171         prev_build = result.first()
 172         if prev_build is None:
 173             raise NoSuchBuildError(tree, host, compiler, revision)
 174         return prev_build.commit_revision
 175
 176     def get_latest_revision(self, tree, host, compiler):
 177         result = self.store.find(StormBuild,
 178             StormBuild.tree == unicode(tree),
 179             StormBuild.host == unicode(host),
 180             StormBuild.compiler == unicode(compiler))
 181         result = result.order_by(Desc(StormBuild.id))
 182         build = result.first()
 183         if build is None:
 184             raise NoSuchBuildError(tree, host, compiler)
 185         return build.revision
 186
 187     def _get_by_checksum(self, build):
 188         result = self.store.find(StormBuild,
 189             StormBuild.checksum == build.log_checksum())
 190         return result.one()
 191
 192     def upload_build(self, build):
 193         existing_build = self._get_by_checksum(build)
 194         if existing_build is not None:
 195             # Already present
 196             assert build.tree == existing_build.tree
 197             assert build.host == existing_build.host
 198             assert build.compiler == existing_build.compiler
 199             return existing_build
 200         rev, timestamp = build.revision_details()
 201         super(StormCachingBuildResultStore, self).upload_build(build)
 202         new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
 203         new_build = StormBuild(new_basename, unicode(build.tree), unicode(build.host),
 204             unicode(build.compiler), rev)
 205         new_build.checksum = build.log_checksum()
 206         new_build.age = build.age_mtime()
 207         new_build.status_str = unicode(build.status().__serialize__())
 208         self.store.add(new_build)
 209         return new_build
 210
 211     def get_old_revs(self, tree, host, compiler):
 212         return self.store.find(StormBuild,
 213             StormBuild.tree == unicode(tree),
 214             StormBuild.host == unicode(host),
 215             StormBuild.compiler == unicode(compiler)).order_by(Desc(StormBuild.age))
 216
 217
 218 class StormCachingBuildFarm(BuildFarm):
 219
 220     def __init__(self, path=None, store=None, timeout=0.5):
 221         self.timeout = timeout
 222         self.store = store
 223         super(StormCachingBuildFarm, self).__init__(path)
 224
 225     def _get_store(self):
 226         if self.store is not None:
 227             return self.store
 228         db_path = os.path.join(self.path, "db", "hostdb.sqlite")
 229         umask = os.umask(0664)
 230         try:
 231             db = create_database("sqlite:%s?timeout=%f" % (db_path, self.timeout))
 232             self.store = Store(db)
 233             setup_schema(self.store)
 234         finally:
 235             os.umask(umask)
 236         return self.store
 237
 238     def _open_hostdb(self):
 239         return StormHostDatabase(self._get_store())
 240
 241     def _open_build_results(self):
 242         return StormCachingBuildResultStore(os.path.join(self.path, "data", "oldrevs"),
 243             self._get_store())
 244
 245     def get_host_builds(self, host):
 246         return self._get_store().find(StormBuild,
 247             StormBuild.host == host).group_by(StormBuild.compiler, StormBuild.tree)
 248
 249     def commit(self):
 250         self.store.commit()
 251
 252
 253 def setup_schema(db):
 254     db.execute("CREATE TABLE IF NOT EXISTS host (name text, owner text, owner_email text, password text, ssh_access int, fqdn text, platform text, permission text, last_dead_mail int, join_time int);", noresult=True)
 255     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_hostname ON host (name);", noresult=True)
 256     db.execute("CREATE TABLE IF NOT EXISTS build (id integer primary key autoincrement, tree text, revision text, host text, compiler text, checksum text, age int, status text, commit_revision text);", noresult=True)
 257     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_checksum ON build (checksum);", noresult=True)
 258
 259
 260 def memory_store():
 261     db = create_database("sqlite:")
 262     store = Store(db)
 263     setup_schema(store)
 264     return store