buildfarm/sqldb.py

   1 #!/usr/bin/python
   2
   3 # Samba.org buildfarm
   4 # Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18 #
  19
  20 from buildfarm import (
  21     BuildFarm,
  22     Tree,
  23     )
  24 from buildfarm.build import (
  25     Build,
  26     BuildResultStore,
  27     BuildStatus,
  28     NoSuchBuildError,
  29     Test,
  30     TestResult,
  31     )
  32 from buildfarm.hostdb import (
  33     Host,
  34     HostDatabase,
  35     HostAlreadyExists,
  36     NoSuchHost,
  37     )
  38
  39 import os
  40 try:
  41     from pysqlite2 import dbapi2 as sqlite3
  42 except ImportError:
  43     import sqlite3
  44 from storm.database import create_database
  45 from storm.expr import EXPR, FuncExpr, compile
  46 from storm.locals import Bool, Desc, Int, RawStr, Reference, Unicode
  47 from storm.store import Store
  48
  49
  50 class Cast(FuncExpr):
  51     __slots__ = ("column", "type")
  52     name = "CAST"
  53
  54     def __init__(self, column, type):
  55         self.column = column
  56         self.type = type
  57
  58 @compile.when(Cast)
  59 def compile_count(compile, cast, state):
  60     state.push("context", EXPR)
  61     column = compile(cast.column, state)
  62     state.pop()
  63     return "CAST(%s AS %s)" % (column, cast.type)
  64
  65
  66 class StormBuild(Build):
  67     __storm_table__ = "build"
  68
  69     id = Int(primary=True)
  70     tree = RawStr()
  71     revision = RawStr()
  72     host = RawStr()
  73     compiler = RawStr()
  74     checksum = RawStr()
  75     upload_time = Int(name="age")
  76     status_str = RawStr(name="status")
  77     basename = RawStr()
  78     host_id = Int()
  79     tree_id = Int()
  80     compiler_id = Int()
  81
  82     def status(self):
  83         return BuildStatus.__deserialize__(self.status_str)
  84
  85     def revision_details(self):
  86         return self.revision
  87
  88     def log_checksum(self):
  89         return self.checksum
  90
  91     def remove(self):
  92         super(StormBuild, self).remove()
  93         Store.of(self).remove(self)
  94
  95     def remove_logs(self):
  96         super(StormBuild, self).remove_logs()
  97         self.basename = None
  98
  99
 100 class StormHost(Host):
 101     __storm_table__ = "host"
 102
 103     id = Int(primary=True)
 104     name = RawStr()
 105     owner_name = Unicode(name="owner")
 106     owner_email = Unicode()
 107     password = Unicode()
 108     ssh_access = Bool()
 109     fqdn = RawStr()
 110     platform = Unicode()
 111     permission = Unicode()
 112     last_dead_mail = Int()
 113     join_time = Int()
 114
 115     def _set_owner(self, value):
 116         if value is None:
 117             self.owner_name = None
 118             self.owner_email = None
 119         else:
 120             (self.owner_name, self.owner_email) = value
 121
 122     def _get_owner(self):
 123         if self.owner_name is None:
 124             return None
 125         else:
 126             return (self.owner_name, self.owner_email)
 127
 128     owner = property(_get_owner, _set_owner)
 129
 130
 131 class StormHostDatabase(HostDatabase):
 132
 133     def __init__(self, store=None):
 134         if store is None:
 135             self.store = memory_store()
 136         else:
 137             self.store = store
 138
 139     def createhost(self, name, platform=None, owner=None, owner_email=None,
 140             password=None, permission=None):
 141         """See `HostDatabase.createhost`."""
 142         newhost = StormHost(name, owner=owner, owner_email=owner_email,
 143                 password=password, permission=permission, platform=platform)
 144         try:
 145             self.store.add(newhost)
 146             self.store.flush()
 147         except sqlite3.IntegrityError:
 148             raise HostAlreadyExists(name)
 149         return newhost
 150
 151     def deletehost(self, name):
 152         """Remove a host."""
 153         self.store.remove(self[name])
 154
 155     def hosts(self):
 156         """Retrieve an iterable over all hosts."""
 157         return self.store.find(StormHost).order_by(StormHost.name)
 158
 159     def __getitem__(self, name):
 160         result = self.store.find(StormHost,
 161             Cast(StormHost.name, "TEXT") == Cast(name, "TEXT"))
 162         ret = result.one()
 163         if ret is None:
 164             raise NoSuchHost(name)
 165         return ret
 166
 167     def commit(self):
 168         self.store.commit()
 169
 170
 171 class StormCachingBuildResultStore(BuildResultStore):
 172
 173     def __init__(self, basedir, store=None):
 174         super(StormCachingBuildResultStore, self).__init__(basedir)
 175
 176         if store is None:
 177             store = memory_store()
 178
 179         self.store = store
 180
 181     def get_by_checksum(self, checksum):
 182         result = self.store.find(StormBuild,
 183             Cast(StormBuild.checksum, "TEXT") == checksum)
 184         ret = result.one()
 185         if ret is None:
 186             raise NoSuchBuildError(None, None, None, None)
 187         return ret
 188
 189     def __contains__(self, build):
 190         try:
 191             self.get_by_checksum(build.log_checksum())
 192             return True
 193         except NoSuchBuildError:
 194             return False
 195
 196     def get_previous_revision(self, tree, host, compiler, revision):
 197         cur_build = self.get_build(tree, host, compiler, revision)
 198
 199         result = self.store.find(StormBuild,
 200             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 201             Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
 202             Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
 203             Cast(StormBuild.revision, "TEXT") != Cast(revision, "TEXT"),
 204             StormBuild.id < cur_build.id)
 205         result = result.order_by(Desc(StormBuild.id))
 206         prev_build = result.first()
 207         if prev_build is None:
 208             raise NoSuchBuildError(tree, host, compiler, revision)
 209         return prev_build.revision
 210
 211     def get_latest_revision(self, tree, host, compiler):
 212         result = self.store.find(StormBuild,
 213             StormBuild.tree == tree,
 214             StormBuild.host == host,
 215             StormBuild.compiler == compiler)
 216         result = result.order_by(Desc(StormBuild.id))
 217         build = result.first()
 218         if build is None:
 219             raise NoSuchBuildError(tree, host, compiler)
 220         return build.revision
 221
 222     def upload_build(self, build):
 223         try:
 224             existing_build = self.get_by_checksum(build.log_checksum())
 225         except NoSuchBuildError:
 226             pass
 227         else:
 228             # Already present
 229             assert build.tree == existing_build.tree
 230             assert build.host == existing_build.host
 231             assert build.compiler == existing_build.compiler
 232             return existing_build
 233         rev = build.revision_details()
 234         super(StormCachingBuildResultStore, self).upload_build(build)
 235         new_basename = self.build_fname(build.tree, build.host, build.compiler,
 236                 rev)
 237         new_build = StormBuild(new_basename, build.tree, build.host,
 238             build.compiler, rev)
 239         new_build.checksum = build.log_checksum()
 240         new_build.upload_time = build.upload_time
 241         new_build.status_str = build.status().__serialize__()
 242         new_build.basename = new_basename
 243         host = self.store.find(StormHost,
 244             Cast(StormHost.name, "TEXT") == Cast(build.host, "TEXT")).one()
 245         assert host is not None, "Unable to find host %r" % build.host
 246         new_build.host_id = host.id
 247         self.store.add(new_build)
 248         return new_build
 249
 250     def get_old_builds(self, tree, host, compiler):
 251         result = self.store.find(StormBuild,
 252             StormBuild.tree == tree,
 253             StormBuild.host == host,
 254             StormBuild.compiler == compiler)
 255         return result.order_by(Desc(StormBuild.upload_time))
 256
 257     def get_build(self, tree, host, compiler, revision=None, checksum=None):
 258         expr = [
 259             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 260             Cast(StormBuild.host, "TEXT") == Cast(host, "TEXT"),
 261             Cast(StormBuild.compiler, "TEXT") == Cast(compiler, "TEXT"),
 262             ]
 263         if revision is not None:
 264             expr.append(Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
 265         if checksum is not None:
 266             expr.append(Cast(StormBuild.checksum, "TEXT") == Cast(checksum, "TEXT"))
 267         result = self.store.find(StormBuild, *expr).order_by(Desc(StormBuild.upload_time))
 268         ret = result.first()
 269         if ret is None:
 270             raise NoSuchBuildError(tree, host, compiler, revision)
 271         return ret
 272
 273
 274 def distinct_builds(builds):
 275     done = set()
 276     for build in builds:
 277         key = (build.tree, build.compiler, build.host)
 278         if key in done:
 279             continue
 280         done.add(key)
 281         yield build
 282
 283
 284 class StormCachingBuildFarm(BuildFarm):
 285
 286     def __init__(self, path=None, store=None, timeout=0.5):
 287         self.timeout = timeout
 288         self.store = store
 289         super(StormCachingBuildFarm, self).__init__(path)
 290
 291     def _get_store(self):
 292         if self.store is not None:
 293             return self.store
 294         db_path = os.path.join(self.path, "db", "hostdb.sqlite")
 295         db = create_database("sqlite:%s?timeout=%f" % (db_path, self.timeout))
 296         self.store = Store(db)
 297         setup_schema(self.store)
 298         return self.store
 299
 300     def _open_hostdb(self):
 301         return StormHostDatabase(self._get_store())
 302
 303     def _open_build_results(self):
 304         path = os.path.join(self.path, "data", "oldrevs")
 305         return StormCachingBuildResultStore(path, self._get_store())
 306
 307     def get_host_builds(self, host):
 308         result = self._get_store().find(StormBuild, StormBuild.host == host)
 309         return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
 310
 311     def get_tree_builds(self, tree):
 312         result = self._get_store().find(StormBuild,
 313             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"))
 314         return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
 315
 316     def get_last_builds(self):
 317         result = self._get_store().find(StormBuild)
 318         return distinct_builds(result.order_by(Desc(StormBuild.upload_time)))
 319
 320     def get_revision_builds(self, tree, revision=None):
 321         return self._get_store().find(StormBuild,
 322             Cast(StormBuild.tree, "TEXT") == Cast(tree, "TEXT"),
 323             Cast(StormBuild.revision, "TEXT") == Cast(revision, "TEXT"))
 324
 325     def commit(self):
 326         self.store.commit()
 327
 328
 329 class StormTree(Tree):
 330     __storm_table__ = "tree"
 331
 332     id = Int(primary=True)
 333     name = RawStr()
 334     scm = Int()
 335     branch = RawStr()
 336     subdir = RawStr()
 337     repo = RawStr()
 338     scm = RawStr()
 339
 340
 341 class StormTest(Test):
 342     __storm_table__ = "test"
 343
 344     id = Int(primary=True)
 345     name = RawStr()
 346
 347
 348 class StormTestResult(TestResult):
 349     __storm_table__ = "test_result"
 350
 351     id = Int(primary=True)
 352     build_id = Int(name="build")
 353     build = Reference(build_id, StormBuild)
 354
 355     test_id = Int(name="test")
 356     test = Reference(test_id, StormTest)
 357
 358
 359 def setup_schema(db):
 360     db.execute("PRAGMA foreign_keys = 1;", noresult=True)
 361     db.execute("""
 362 CREATE TABLE IF NOT EXISTS host (
 363     id integer primary key autoincrement,
 364     name blob not null,
 365     owner text,
 366     owner_email text,
 367     password text,
 368     ssh_access int,
 369     fqdn text,
 370     platform text,
 371     permission text,
 372     last_dead_mail int,
 373     join_time int
 374 );""", noresult=True)
 375     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_hostname ON host (name);", noresult=True)
 376     db.execute("""
 377 CREATE TABLE IF NOT EXISTS build (
 378     id integer primary key autoincrement,
 379     tree blob not null,
 380     tree_id int,
 381     revision blob,
 382     host blob not null,
 383     host_id integer,
 384     compiler blob not null,
 385     compiler_id int,
 386     checksum blob,
 387     age int,
 388     status blob,
 389     basename blob,
 390     FOREIGN KEY (host_id) REFERENCES host (id),
 391     FOREIGN KEY (tree_id) REFERENCES tree (id),
 392     FOREIGN KEY (compiler_id) REFERENCES compiler (id)
 393 );""", noresult=True)
 394     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS unique_checksum ON build (checksum);", noresult=True)
 395     db.execute("""
 396 CREATE TABLE IF NOT EXISTS tree (
 397     id integer primary key autoincrement,
 398     name blob not null,
 399     scm int,
 400     branch blob,
 401     subdir blob,
 402     repo blob
 403     );
 404     """, noresult=True)
 405     db.execute("""
 406 CREATE UNIQUE INDEX IF NOT EXISTS unique_tree_name ON tree(name);
 407 """, noresult=True)
 408     db.execute("""
 409 CREATE TABLE IF NOT EXISTS compiler (
 410     id integer primary key autoincrement,
 411     name blob not null
 412     );
 413     """, noresult=True)
 414     db.execute("""
 415 CREATE UNIQUE INDEX IF NOT EXISTS unique_compiler_name ON compiler(name);
 416 """, noresult=True)
 417     db.execute("""
 418 CREATE TABLE IF NOT EXISTS test (
 419     id integer primary key autoincrement,
 420     name text not null);
 421     """, noresult=True)
 422     db.execute("CREATE UNIQUE INDEX IF NOT EXISTS test_name ON test(name);",
 423         noresult=True)
 424     db.execute("""CREATE TABLE IF NOT EXISTS test_result (
 425         build int,
 426         test int,
 427         result int
 428         );""", noresult=True)
 429     db.execute("""CREATE UNIQUE INDEX IF NOT EXISTS build_test_result ON test_result(build, test);""", noresult=True)
 430
 431
 432 def memory_store():
 433     db = create_database("sqlite:")
 434     store = Store(db)
 435     setup_schema(store)
 436     return store