buildfarm/tests/test_build.py

   1 #!/usr/bin/python
   2 # Copyright (C) Jelmer Vernooij <jelmer@samba.org> 2010
   3 #
   4 #   This program is free software; you can redistribute it and/or modify
   5 #   it under the terms of the GNU General Public License as published by
   6 #   the Free Software Foundation; either version 3 of the License, or
   7 #   (at your option) any later version.
   8 #
   9 #   This program is distributed in the hope that it will be useful,
  10 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 #   GNU General Public License for more details.
  13 #
  14 #   You should have received a copy of the GNU General Public License
  15 #   along with this program; if not, write to the Free Software
  16 #   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 from cStringIO import StringIO
  19 import os
  20 import testtools
  21 import unittest
  22
  23 from buildfarm.build import (
  24     Build,
  25     BuildResultStore,
  26     BuildStatus,
  27     NoSuchBuildError,
  28     UploadBuildResultStore,
  29     build_status_from_logs,
  30     )
  31
  32 from buildfarm import BuildFarm
  33 from buildfarm.tests import BuildFarmTestCase
  34
  35
  36 class NonexistantTests(unittest.TestCase):
  37
  38     def test_nonexistant(self):
  39         self.assertRaises(
  40             Exception, BuildResultStore, "somedirthatdoesn'texist", None)
  41
  42
  43 class BuildResultStoreTestBase(object):
  44
  45     def setUp(self):
  46         self.write_compilers(["cc", "gcc"])
  47         self.write_hosts({"charis": "Some machine",
  48                           "myhost": "Another host"})
  49
  50     def test_build_fname(self):
  51         self.assertEquals(
  52             self.x.build_fname("mytree", "myhost", "cc", 123),
  53             "%s/data/oldrevs/build.mytree.myhost.cc-123" % self.path)
  54
  55     def test_build_remove(self):
  56         path = self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
  57                 "BUILD COMMIT REVISION: 12\n")
  58         build = self.x.get_build("tdb", "charis", "cc", "12")
  59         logname = build.basename + ".log"
  60         build.remove()
  61         self.assertFalse(os.path.exists(logname))
  62         self.assertRaises(NoSuchBuildError, self.x.get_build, "tdb", "charis", "cc", "12")
  63
  64     def test_build_repr(self):
  65         path = self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
  66             "BUILD COMMIT REVISION: 12\n")
  67         build = self.x.get_build("tdb", "charis", "cc", "12")
  68         self.assertEquals("<%s: revision 12 of tdb on charis using cc>" % build.__class__.__name__, repr(build))
  69
  70     def test_get_build_nonexistant(self):
  71         self.assertRaises(NoSuchBuildError, self.x.get_build, "tdb",
  72             "charis", "cc", "12")
  73
  74     def test_build_upload_time(self):
  75         path = self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
  76                 "BUILD COMMIT REVISION: 12\n", mtime=5)
  77         build = self.x.get_build("tdb", "charis", "cc", "12")
  78         self.assertEquals(5, build.upload_time)
  79
  80     def test_read_log(self):
  81         path = self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
  82             stdout_contents="This is what a log file looks like.\n"
  83             "BUILD COMMIT REVISION: 12\n")
  84         build = self.x.get_build("tdb", "charis", "cc", "12")
  85         self.assertEquals("This is what a log file looks like.\n"
  86                           "BUILD COMMIT REVISION: 12\n",
  87                           build.read_log().read())
  88
  89     def test_read_err(self):
  90         self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
  91             stdout_contents="BUILD COMMIT REVISION: 12\n",
  92             stderr_contents="This is what an stderr file looks like.")
  93         build = self.x.get_build("tdb", "charis", "cc", "12")
  94         self.assertEquals("This is what an stderr file looks like.",
  95                 build.read_err().read())
  96
  97     def test_read_err_nofile(self):
  98         self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
  99                 stdout_contents="BUILD COMMIT REVISION: 12\n")
 100         build = self.x.get_build("tdb", "charis", "cc", "12")
 101         self.assertEquals("", build.read_err().read())
 102
 103     def test_revision_details(self):
 104         self.upload_mock_logfile(self.x, "tdb", "charis", "cc", stdout_contents="""
 105 BUILD COMMIT REVISION: 43
 106 bla
 107 BUILD COMMIT TIME: 3 August 2010
 108 """)
 109         build = self.x.get_build("tdb", "charis", "cc", "43")
 110         rev = build.revision_details()
 111         self.assertIsInstance(rev, str)
 112         self.assertEquals("43", rev)
 113
 114     def test_revision_details_no_timestamp(self):
 115         self.upload_mock_logfile(self.x, "tdb", "charis", "cc", stdout_contents="""
 116 BUILD COMMIT REVISION: 43
 117 BUILD REVISION: 42
 118 BLA
 119 """)
 120         build = self.x.get_build("tdb", "charis", "cc", "43")
 121         self.assertEquals("43", build.revision_details())
 122
 123     def test_err_count(self):
 124         self.upload_mock_logfile(self.x, "tdb", "charis", "cc",
 125             stdout_contents="BUILD COMMIT REVISION: 12\n",
 126             stderr_contents="""error1
 127 error2
 128 error3""")
 129         build = self.x.get_build("tdb", "charis", "cc", "12")
 130         self.assertEquals(3, build.err_count())
 131
 132     def test_upload_build(self):
 133         path = self.create_mock_logfile("tdb", "charis", "cc", contents="""
 134 BUILD COMMIT REVISION: myrev
 135 """)
 136         build = Build(path[:-4], "tdb", "charis", "cc")
 137         self.x.upload_build(build)
 138         uploaded_build = self.x.get_build("tdb", "charis", "cc", "myrev")
 139         self.assertEquals(uploaded_build.log_checksum(), build.log_checksum())
 140
 141     def test_upload_build_no_rev(self):
 142         path = self.create_mock_logfile("tdb", "charis", "cc", contents="""
 143 """)
 144         build = Build(path[:-4], "tdb", "charis", "cc")
 145         self.assertRaises(Exception, self.x.upload_build, build)
 146
 147     def test_get_previous_revision(self):
 148         self.assertRaises(NoSuchBuildError, self.x.get_previous_revision, "tdb", "charis", "cc", "12")
 149
 150     def test_get_latest_revision_none(self):
 151         self.assertRaises(NoSuchBuildError, self.x.get_latest_revision, "tdb", "charis", "cc")
 152
 153     def test_get_old_builds_none(self):
 154         self.assertEquals([],
 155             list(self.x.get_old_builds("tdb", "charis", "gcc")))
 156
 157     def test_get_old_builds(self):
 158         path = self.create_mock_logfile("tdb", "charis", "cc",
 159             contents="""
 160 BUILD COMMIT REVISION: 12
 161 """)
 162         build = Build(path[:-4], "tdb", "charis", "cc")
 163         b1 = self.x.upload_build(build)
 164         path = self.create_mock_logfile("tdb", "charis", "cc",
 165             contents="""
 166 BUILD COMMIT REVISION: 15
 167 """)
 168         build = Build(path[:-4], "tdb", "charis", "cc")
 169         b2 = self.x.upload_build(build)
 170         path = self.create_mock_logfile("tdb", "charis", "cc",
 171             contents="""
 172 BUILD COMMIT REVISION: 15
 173 """)
 174         self.assertEquals([b1, b2],
 175             list(self.x.get_old_builds("tdb", "charis", "cc")))
 176
 177
 178 class BuildResultStoreTests(BuildFarmTestCase,BuildResultStoreTestBase):
 179
 180     def setUp(self):
 181         BuildFarmTestCase.setUp(self)
 182         BuildResultStoreTestBase.setUp(self)
 183
 184         self.buildfarm = BuildFarm(self.path)
 185
 186         self.x = BuildResultStore(
 187             os.path.join(self.path, "data", "oldrevs"))
 188
 189
 190 class BuildStatusFromLogs(testtools.TestCase):
 191
 192     def parse_logs(self, log, err):
 193         return build_status_from_logs(StringIO(log), StringIO(err))
 194
 195     def test_nothing(self):
 196         s = self.parse_logs("", "")
 197         self.assertEquals([], s.stages)
 198         self.assertEquals(set(), s.other_failures)
 199
 200     def test_disk_full(self):
 201         self.assertEquals(set(["disk full"]),
 202             self.parse_logs("foo\nbar\nNo space left on device\nla\n",
 203                 "").other_failures)
 204         self.assertEquals(set(["disk full"]),
 205             self.parse_logs(
 206                 "", "foo\nbar\nNo space left on device\nla\n").other_failures)
 207
 208     def test_timeout(self):
 209         self.assertEquals(set(["timeout"]),
 210             self.parse_logs("foo\nbar\nmaximum runtime exceeded\nla\n",
 211                 "").other_failures)
 212
 213     def test_failed_test(self):
 214         log = """
 215 TEST STATUS:1
 216 """
 217         res = self.parse_logs(log, "")
 218         self.assertEquals(res.stages, [
 219             ("TEST", 1)])
 220
 221     def test_failed_test_whitespace(self):
 222         log = """
 223 TEST STATUS:  1
 224 """
 225         res = self.parse_logs(log, "")
 226         self.assertEquals(res.stages,
 227             [("TEST", 1)])
 228
 229     def test_failed_test_noise(self):
 230         log = """
 231 CONFIGURE STATUS: 2
 232 TEST STATUS:  1
 233 CC_CHECKER STATUS:      2
 234 """
 235         res = self.parse_logs(log, "")
 236         self.assertEquals(res.stages,
 237             [("CONFIGURE", 2), ("TEST", 1), ("CC_CHECKER", 2)])
 238
 239     def test_no_test_output(self):
 240         log = """
 241 CONFIGURE STATUS: 2
 242 TEST STATUS: 0
 243 CC_CHECKER STATUS:      2
 244 """
 245         res = self.parse_logs(log, "")
 246         self.assertEquals(res.stages,
 247             [("CONFIGURE", 2), ("TEST", 0), ("CC_CHECKER", 2)])
 248
 249     def test_granular_test(self):
 250         log = """
 251 CONFIGURE STATUS: 2
 252 testsuite-success: toto
 253 testsuite-failure: foo
 254 testsuite-failure: bar
 255 testsuite-failure: biz
 256 TEST STATUS: 1
 257 CC_CHECKER STATUS:      2
 258 """
 259         res = self.parse_logs(log, "")
 260         self.assertEquals(res.stages,
 261             [("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)])
 262
 263
 264 class BuildStatusTest(testtools.TestCase):
 265
 266     def test_cmp_equal(self):
 267         a = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)])
 268         b = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)])
 269
 270         self.assertEquals(cmp(a, b), 0)
 271
 272     def test_cmp_empty(self):
 273         self.assertEquals(cmp(BuildStatus(), BuildStatus()), 0)
 274
 275     def test_cmp_other_failures(self):
 276         self.assertEquals(cmp(
 277             BuildStatus((), set(["foo"])), BuildStatus((), set(["foo"]))),
 278             0)
 279
 280     def test_cmp_intermediate_errors(self):
 281         a = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 3)])
 282         b = BuildStatus([("CONFIGURE", 2), ("TEST", 7), ("CC_CHECKER", 3)])
 283         self.assertEquals(cmp(a, b), 1)
 284
 285     def test_cmp_bigger(self):
 286         a = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 3)])
 287         b = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)])
 288         c = BuildStatus([("CONFIGURE", 2), ("TEST", 3)])
 289         d = BuildStatus([], set(["super error"]))
 290         e = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)], set(["super error"]))
 291
 292         # less stage means smaller, more error/higher error code means smaller as well
 293         self.assertEquals(cmp(b, a), 1)
 294
 295         self.assertEquals(cmp(a, c), 1)
 296
 297         self.assertEquals(cmp(a, d), 1)
 298
 299         self.assertEquals(cmp(b, e), 1)
 300
 301     def test_cmp_smaller(self):
 302         a = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 2)])
 303         b = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)])
 304         c = BuildStatus([("CONFIGURE", 2), ("TEST", 3)])
 305         d = BuildStatus([], set(["super error"]))
 306         e = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)], set(["super error"]))
 307
 308         # less stage means smaller, more error/higher error code means smaller as well
 309         self.assertEquals(cmp(a, b), -1)
 310
 311         self.assertEquals(cmp(c, b), -1)
 312
 313         self.assertEquals(cmp(d, c), -1)
 314
 315         self.assertEquals(cmp(e, c), -1)
 316
 317     def test_cmp_with_other_failures(self):
 318         d = BuildStatus([], set(["super error"]))
 319         e = BuildStatus([("CONFIGURE", 2), ("TEST", 3), ("CC_CHECKER", 1)], set(["super error"]))
 320         self.assertEquals(cmp(d, e), -1)
 321
 322     def test_str(self):
 323         a = BuildStatus([("CONFIGURE", 3), ("BUILD", 2)])
 324         self.assertEquals("3/2", str(a))
 325
 326     def test_str_other_failures(self):
 327         a = BuildStatus([("CONFIGURE", 3), ("BUILD", 2)], set(["panic"]))
 328         self.assertEquals("panic", str(a))
 329
 330
 331 class BuildStatusRegressedSinceTests(testtools.TestCase):
 332
 333     def assertRegressedSince(self, expected, old_status, new_status):
 334         (stages1, other_failures1) = old_status
 335         (stages2, other_failures2) = new_status
 336         a = BuildStatus(stages1, set(other_failures1))
 337         b = BuildStatus(stages2, set(other_failures2))
 338         self.assertEquals(expected, b.regressed_since(a))
 339
 340     def test_same(self):
 341         self.assertRegressedSince(
 342             False,
 343             ([("CONFIGURE", 2)], []),
 344             ([("CONFIGURE", 2)], []))
 345
 346     def test_same_panic(self):
 347         self.assertRegressedSince(
 348             False,
 349             ([("CONFIGURE", 2)], ["panic"]),
 350             ([("CONFIGURE", 2)], ["panic"]))
 351
 352     def test_other_failures_gone(self):
 353         self.assertRegressedSince(
 354             True,
 355             ([("CONFIGURE", 0)], ["panic"]),
 356             ([("CONFIGURE", 2)], ["panic"]))
 357
 358     def test_more_stages_completed(self):
 359         self.assertRegressedSince(
 360             False,
 361             ([("CONFIGURE", 0)], []),
 362             ([("CONFIGURE", 0), ("BUILD", 0)], []))
 363
 364     def test_less_errors(self):
 365         self.assertRegressedSince(
 366             False,
 367             ([("CONFIGURE", 0), ("BUILD", 0), ("TEST", 0), ("INSTALL", 1)], []),
 368             ([("CONFIGURE", 0), ("BUILD", 0), ("TEST", 0), ("INSTALL", 0)], []))
 369
 370     def test_no_longer_inconsistent(self):
 371         self.assertRegressedSince(
 372             False,
 373             ([("CONFIGURE", 0)], ["inconsistent test result"]),
 374             ([("CONFIGURE", 0)], []))
 375
 376
 377 class UploadBuildResultStoreTestBase(object):
 378
 379     def test_build_fname(self):
 380         self.assertEquals(
 381             self.x.build_fname("mytree", "myhost", "cc"),
 382             "%s/data/upload/build.mytree.myhost.cc" % self.path)
 383
 384     def test_get_all_builds(self):
 385         self.assertEquals([], list(self.x.get_all_builds()))
 386         path = self.create_mock_logfile("tdb", "charis", "cc")
 387         new_builds = list(self.x.get_all_builds())
 388         self.assertEquals(1, len(new_builds))
 389         self.assertEquals("tdb", new_builds[0].tree)
 390         self.assertEquals("charis", new_builds[0].host)
 391         self.assertEquals("cc", new_builds[0].compiler)
 392
 393
 394 class UploadBuildResultStoreTests(UploadBuildResultStoreTestBase,BuildFarmTestCase):
 395
 396     def setUp(self):
 397         super(UploadBuildResultStoreTests, self).setUp()
 398
 399         self.x = UploadBuildResultStore(
 400             os.path.join(self.path, "data", "upload"))
 401
 402
 403