buildfarm/sqldb.py

   1 #!/usr/bin/python
   2
   3 # Samba.org buildfarm
   4 # Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18 #
  19
  20 from buildfarm import (
  21     BuildFarm,
  22     )
  23 from buildfarm.data import (
  24     Build,
  25     BuildResultStore,
  26     NoSuchBuildError,
  27     )
  28 from buildfarm.hostdb import (
  29     Host,
  30     HostDatabase,
  31     HostAlreadyExists,
  32     NoSuchHost,
  33     )
  34
  35 import os
  36 try:
  37     import sqlite3
  38 except ImportError:
  39     from pysqlite2 import dbapi2 as sqlite3
  40 from storm.database import create_database
  41 from storm.locals import Bool, Desc, Int, Unicode, RawStr
  42 from storm.store import Store
  43
  44
  45 class StormBuild(Build):
  46     __storm_table__ = "build"
  47
  48     id = Int(primary=True)
  49     tree = Unicode()
  50     revision = RawStr()
  51     host = Unicode()
  52     compiler = Unicode()
  53     checksum = RawStr()
  54     age = Int()
  55     status_str = Unicode(name="status")
  56     commit_revision = RawStr()
  57
  58     def log_checksum(self):
  59         return self.checksum
  60
  61     def remove(self):
  62         super(StormBuild, self).remove()
  63         Store.of(self).remove(self)
  64
  65
  66 class StormHost(Host):
  67     __storm_table__ = "host"
  68
  69     name = Unicode(primary=True)
  70     owner_name = Unicode(name="owner")
  71     owner_email = Unicode()
  72     password = Unicode()
  73     ssh_access = Bool()
  74     fqdn = RawStr()
  75     platform = Unicode()
  76     permission = Unicode()
  77     last_dead_mail = Int()
  78     join_time = Int()
  79
  80     def _set_owner(self, value):
  81         if value is None:
  82             self.owner_name = None
  83             self.owner_email = None
  84         else:
  85             (self.owner_name, self.owner_email) = value
  86
  87     def _get_owner(self):
  88         if self.owner_name is None:
  89             return None
  90         else:
  91             return (self.owner_name, self.owner_email)
  92
  93     owner = property(_get_owner, _set_owner)
  94
  95
  96 class StormHostDatabase(HostDatabase):
  97
  98     def __init__(self, store=None):
  99         if store is None:
 100             self.store = memory_store()
 101         else:
 102             self.store = store
 103
 104     def createhost(self, name, platform=None, owner=None, owner_email=None, password=None, permission=None):
 105         newhost = StormHost(unicode(name), owner=owner, owner_email=owner_email, password=password, permission=permission, platform=platform)
 106         try:
 107             self.store.add(newhost)
 108             self.store.flush()
 109         except sqlite3.IntegrityError:
 110             raise HostAlreadyExists(name)
 111         return newhost
 112
 113     def deletehost(self, name):
 114         """Remove a host."""
 115         host = self.host(name)
 116         self.store.remove(host)
 117
 118     def hosts(self):
 119         """Retrieve an iterable over all hosts."""
 120         return self.store.find(StormHost).order_by(StormHost.name)
 121
 122     def host(self, name):
 123         ret = self.hosts().find(StormHost.name==name).one()
 124         if ret is None:
 125             raise NoSuchHost(name)
 126         return ret
 127
 128     def commit(self):
 129         self.store.commit()
 130
 131
 132 class StormCachingBuildResultStore(BuildResultStore):
 133
 134     def __init__(self, basedir, store=None):
 135         super(StormCachingBuildResultStore, self).__init__(basedir)
 136
 137         if store is None:
 138             store = memory_store()
 139
 140         self.store = store
 141
 142     def __contains__(self, build):
 143         return (self._get_by_checksum(build) is not None)
 144
 145     def get_previous_revision(self, tree, host, compiler, revision):
 146         result = self.store.find(StormBuild,
 147             StormBuild.tree == unicode(tree),
 148             StormBuild.host == unicode(host),
 149             StormBuild.compiler == unicode(compiler),
 150             StormBuild.commit_revision == revision)
 151         cur_build = result.any()
 152         if cur_build is None:
 153             raise NoSuchBuildError(tree, host, compiler, revision)
 154
 155         result = self.store.find(StormBuild,
 156             StormBuild.tree == unicode(tree),
 157             StormBuild.host == unicode(host),
 158             StormBuild.compiler == unicode(compiler),
 159             StormBuild.commit_revision != revision,
 160             StormBuild.id < cur_build.id)
 161         result = result.order_by(Desc(StormBuild.id))
 162         prev_build = result.first()
 163         if prev_build is None:
 164             raise NoSuchBuildError(tree, host, compiler, revision)
 165         return prev_build.commit_revision
 166
 167     def get_latest_revision(self, tree, host, compiler):
 168         result = self.store.find(StormBuild,
 169             StormBuild.tree == unicode(tree),
 170             StormBuild.host == unicode(host),
 171             StormBuild.compiler == unicode(compiler))
 172         result = result.order_by(Desc(StormBuild.id))
 173         build = result.first()
 174         if build is None:
 175             raise NoSuchBuildError(tree, host, compiler)
 176         return build.revision
 177
 178     def _get_by_checksum(self, build):
 179         return self.store.find(StormBuild, StormBuild.checksum == build.log_checksum()).one()
 180
 181     def upload_build(self, build):
 182         existing_build = self._get_by_checksum(build)
 183         if existing_build is not None:
 184             # Already present
 185             assert build.tree == existing_build.tree
 186             assert build.host == existing_build.host
 187             assert build.compiler == existing_build.compiler
 188             return existing_build
 189         rev, timestamp = build.revision_details()
 190         super(StormCachingBuildResultStore, self).upload_build(build)
 191         new_basename = self.build_fname(build.tree, build.host, build.compiler, rev)
 192         new_build = StormBuild(new_basename, unicode(build.tree), unicode(build.host), unicode(build.compiler), rev)
 193         new_build.checksum = build.log_checksum()
 194         new_build.age = build.age_mtime()
 195         new_build.status_str = unicode(str(build.status()))
 196         self.store.add(new_build)
 197         return new_build
 198
 199
 200 class StormCachingBuildFarm(BuildFarm):
 201
 202     def __init__(self, path=None, store=None):
 203         self.store = store
 204         super(StormCachingBuildFarm, self).__init__(path)
 205
 206     def _get_store(self):
 207         if self.store is not None:
 208             return self.store
 209         else:
 210             db = create_database("sqlite:" + os.path.join(self.path, "hostdb.sqlite"))
 211             self.store = Store(db)
 212             setup_schema(self.store)
 213             return self.store
 214
 215     def _open_hostdb(self):
 216         return StormHostDatabase(self._get_store())
 217
 218     def _open_build_results(self):
 219         return StormCachingBuildResultStore(os.path.join(self.path, "data", "oldrevs"),
 220             self._get_store())
 221
 222     def commit(self):
 223         self.store.commit()
 224
 225
 226 def setup_schema(db):
 227     db.execute("CREATE TABLE IF NOT EXISTS host (name text, owner text, owner_email text, password text, ssh_access int, fqdn text, platform text, permission text, last_dead_mail int, join_time int);", noresult=True)
 228     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_hostname ON host (name);", noresult=True)
 229     db.execute("CREATE TABLE IF NOT EXISTS build (id integer primary key autoincrement, tree text, revision text, host text, compiler text, checksum text, age int, status text, commit_revision text);", noresult=True)
 230     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_checksum ON build (checksum);", noresult=True)
 231
 232
 233 def memory_store():
 234     db = create_database("sqlite:")
 235     store = Store(db)
 236     setup_schema(store)
 237     return store