diff options
| -rw-r--r-- | meta/classes/testimage.bbclass | 6 | ||||
| -rw-r--r-- | meta/classes/testsdk.bbclass | 6 | ||||
| -rw-r--r-- | meta/lib/oeqa/oetest.py | 197 |
3 files changed, 103 insertions, 106 deletions
diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass index 7f9c918273..53c6174ac5 100644 --- a/meta/classes/testimage.bbclass +++ b/meta/classes/testimage.bbclass | |||
| @@ -195,7 +195,7 @@ def testimage_main(d): | |||
| 195 | import oeqa.runtime | 195 | import oeqa.runtime |
| 196 | import time | 196 | import time |
| 197 | import signal | 197 | import signal |
| 198 | from oeqa.oetest import loadTests, runTests, ImageTestContext | 198 | from oeqa.oetest import ImageTestContext |
| 199 | from oeqa.targetcontrol import get_target_controller | 199 | from oeqa.targetcontrol import get_target_controller |
| 200 | from oeqa.utils.dump import get_host_dumper | 200 | from oeqa.utils.dump import get_host_dumper |
| 201 | 201 | ||
| @@ -219,7 +219,7 @@ def testimage_main(d): | |||
| 219 | # we are doing that to find compile errors in the tests themselves | 219 | # we are doing that to find compile errors in the tests themselves |
| 220 | # before booting the image | 220 | # before booting the image |
| 221 | try: | 221 | try: |
| 222 | loadTests(tc) | 222 | tc.loadTests() |
| 223 | except Exception as e: | 223 | except Exception as e: |
| 224 | import traceback | 224 | import traceback |
| 225 | bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) | 225 | bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) |
| @@ -233,7 +233,7 @@ def testimage_main(d): | |||
| 233 | try: | 233 | try: |
| 234 | target.start() | 234 | target.start() |
| 235 | starttime = time.time() | 235 | starttime = time.time() |
| 236 | result = runTests(tc) | 236 | result = tc.runTests() |
| 237 | stoptime = time.time() | 237 | stoptime = time.time() |
| 238 | if result.wasSuccessful(): | 238 | if result.wasSuccessful(): |
| 239 | bb.plain("%s - Ran %d test%s in %.3fs" % (pn, result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime)) | 239 | bb.plain("%s - Ran %d test%s in %.3fs" % (pn, result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime)) |
diff --git a/meta/classes/testsdk.bbclass b/meta/classes/testsdk.bbclass index 47bad29096..ba8897e5ea 100644 --- a/meta/classes/testsdk.bbclass +++ b/meta/classes/testsdk.bbclass | |||
| @@ -13,7 +13,7 @@ def testsdk_main(d): | |||
| 13 | import oeqa.sdk | 13 | import oeqa.sdk |
| 14 | import time | 14 | import time |
| 15 | import subprocess | 15 | import subprocess |
| 16 | from oeqa.oetest import loadTests, runTests, SDKTestContext | 16 | from oeqa.oetest import SDKTestContext |
| 17 | 17 | ||
| 18 | pn = d.getVar("PN", True) | 18 | pn = d.getVar("PN", True) |
| 19 | bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True)) | 19 | bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True)) |
| @@ -40,13 +40,13 @@ def testsdk_main(d): | |||
| 40 | # we are doing that to find compile errors in the tests themselves | 40 | # we are doing that to find compile errors in the tests themselves |
| 41 | # before booting the image | 41 | # before booting the image |
| 42 | try: | 42 | try: |
| 43 | loadTests(tc, "sdk") | 43 | tc.loadTests() |
| 44 | except Exception as e: | 44 | except Exception as e: |
| 45 | import traceback | 45 | import traceback |
| 46 | bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) | 46 | bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) |
| 47 | 47 | ||
| 48 | starttime = time.time() | 48 | starttime = time.time() |
| 49 | result = runTests(tc, "sdk") | 49 | result = tc.runTests() |
| 50 | stoptime = time.time() | 50 | stoptime = time.time() |
| 51 | if result.wasSuccessful(): | 51 | if result.wasSuccessful(): |
| 52 | bb.plain("%s SDK(%s):%s - Ran %d test%s in %.3fs" % (pn, os.path.basename(tcname), os.path.basename(sdkenv),result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime)) | 52 | bb.plain("%s SDK(%s):%s - Ran %d test%s in %.3fs" % (pn, os.path.basename(tcname), os.path.basename(sdkenv),result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime)) |
diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py index 7f07037a1f..16705cc3fe 100644 --- a/meta/lib/oeqa/oetest.py +++ b/meta/lib/oeqa/oetest.py | |||
| @@ -45,106 +45,6 @@ def filterByTagExp(testsuite, tagexp): | |||
| 45 | caseList.append(filterByTagExp(each, tagexp)) | 45 | caseList.append(filterByTagExp(each, tagexp)) |
| 46 | return testsuite.__class__(caseList) | 46 | return testsuite.__class__(caseList) |
| 47 | 47 | ||
| 48 | def loadTests(tc, type="runtime"): | ||
| 49 | if type == "runtime": | ||
| 50 | # set the context object passed from the test class | ||
| 51 | setattr(oeTest, "tc", tc) | ||
| 52 | # set ps command to use | ||
| 53 | setattr(oeRuntimeTest, "pscmd", "ps -ef" if oeTest.hasPackage("procps") else "ps") | ||
| 54 | # prepare test suite, loader and runner | ||
| 55 | suite = unittest.TestSuite() | ||
| 56 | elif type == "sdk": | ||
| 57 | # set the context object passed from the test class | ||
| 58 | setattr(oeTest, "tc", tc) | ||
| 59 | testloader = unittest.TestLoader() | ||
| 60 | testloader.sortTestMethodsUsing = None | ||
| 61 | suites = [testloader.loadTestsFromName(name) for name in tc.testslist] | ||
| 62 | suites = filterByTagExp(suites, getattr(tc, "tagexp", None)) | ||
| 63 | |||
| 64 | def getTests(test): | ||
| 65 | '''Return all individual tests executed when running the suite.''' | ||
| 66 | # Unfortunately unittest does not have an API for this, so we have | ||
| 67 | # to rely on implementation details. This only needs to work | ||
| 68 | # for TestSuite containing TestCase. | ||
| 69 | method = getattr(test, '_testMethodName', None) | ||
| 70 | if method: | ||
| 71 | # leaf case: a TestCase | ||
| 72 | yield test | ||
| 73 | else: | ||
| 74 | # Look into TestSuite. | ||
| 75 | tests = getattr(test, '_tests', []) | ||
| 76 | for t1 in tests: | ||
| 77 | for t2 in getTests(t1): | ||
| 78 | yield t2 | ||
| 79 | |||
| 80 | # Determine dependencies between suites by looking for @skipUnlessPassed | ||
| 81 | # method annotations. Suite A depends on suite B if any method in A | ||
| 82 | # depends on a method on B. | ||
| 83 | for suite in suites: | ||
| 84 | suite.dependencies = [] | ||
| 85 | suite.depth = 0 | ||
| 86 | for test in getTests(suite): | ||
| 87 | methodname = getattr(test, '_testMethodName', None) | ||
| 88 | if methodname: | ||
| 89 | method = getattr(test, methodname) | ||
| 90 | depends_on = getattr(method, '_depends_on', None) | ||
| 91 | if depends_on: | ||
| 92 | for dep_suite in suites: | ||
| 93 | if depends_on in [getattr(t, '_testMethodName', None) for t in getTests(dep_suite)]: | ||
| 94 | if dep_suite not in suite.dependencies and \ | ||
| 95 | dep_suite is not suite: | ||
| 96 | suite.dependencies.append(dep_suite) | ||
| 97 | break | ||
| 98 | else: | ||
| 99 | logger.warning("Test %s was declared as @skipUnlessPassed('%s') but that test is either not defined or not active. Will run the test anyway." % | ||
| 100 | (test, depends_on)) | ||
| 101 | # Use brute-force topological sort to determine ordering. Sort by | ||
| 102 | # depth (higher depth = must run later), with original ordering to | ||
| 103 | # break ties. | ||
| 104 | def set_suite_depth(suite): | ||
| 105 | for dep in suite.dependencies: | ||
| 106 | new_depth = set_suite_depth(dep) + 1 | ||
| 107 | if new_depth > suite.depth: | ||
| 108 | suite.depth = new_depth | ||
| 109 | return suite.depth | ||
| 110 | for index, suite in enumerate(suites): | ||
| 111 | set_suite_depth(suite) | ||
| 112 | suite.index = index | ||
| 113 | suites.sort(cmp=lambda a,b: cmp((a.depth, a.index), (b.depth, b.index))) | ||
| 114 | return testloader.suiteClass(suites) | ||
| 115 | |||
| 116 | _buffer = "" | ||
| 117 | |||
| 118 | def custom_verbose(msg, *args, **kwargs): | ||
| 119 | global _buffer | ||
| 120 | if msg[-1] != "\n": | ||
| 121 | _buffer += msg | ||
| 122 | else: | ||
| 123 | _buffer += msg | ||
| 124 | try: | ||
| 125 | bb.plain(_buffer.rstrip("\n"), *args, **kwargs) | ||
| 126 | except NameError: | ||
| 127 | logger.info(_buffer.rstrip("\n"), *args, **kwargs) | ||
| 128 | _buffer = "" | ||
| 129 | |||
| 130 | def runTests(tc, type="runtime"): | ||
| 131 | |||
| 132 | suite = loadTests(tc, type) | ||
| 133 | logger.info("Test modules %s" % tc.testslist) | ||
| 134 | if hasattr(tc, "tagexp") and tc.tagexp: | ||
| 135 | logger.info("Filter test cases by tags: %s" % tc.tagexp) | ||
| 136 | logger.info("Found %s tests" % suite.countTestCases()) | ||
| 137 | runner = unittest.TextTestRunner(verbosity=2) | ||
| 138 | try: | ||
| 139 | if bb.msg.loggerDefaultVerbose: | ||
| 140 | runner.stream.write = custom_verbose | ||
| 141 | except NameError: | ||
| 142 | # Not in bb environment? | ||
| 143 | pass | ||
| 144 | result = runner.run(suite) | ||
| 145 | |||
| 146 | return result | ||
| 147 | |||
| 148 | @LogResults | 48 | @LogResults |
| 149 | class oeTest(unittest.TestCase): | 49 | class oeTest(unittest.TestCase): |
| 150 | 50 | ||
| @@ -253,6 +153,19 @@ def skipModuleUnless(cond, reason): | |||
| 253 | if not cond: | 153 | if not cond: |
| 254 | skipModule(reason, 3) | 154 | skipModule(reason, 3) |
| 255 | 155 | ||
| 156 | _buffer_logger = "" | ||
| 157 | def custom_verbose(msg, *args, **kwargs): | ||
| 158 | global _buffer_logger | ||
| 159 | if msg[-1] != "\n": | ||
| 160 | _buffer_logger += msg | ||
| 161 | else: | ||
| 162 | _buffer_logger += msg | ||
| 163 | try: | ||
| 164 | bb.plain(_buffer_logger.rstrip("\n"), *args, **kwargs) | ||
| 165 | except NameError: | ||
| 166 | logger.info(_buffer_logger.rstrip("\n"), *args, **kwargs) | ||
| 167 | _buffer_logger = "" | ||
| 168 | |||
| 256 | class TestContext(object): | 169 | class TestContext(object): |
| 257 | def __init__(self, d): | 170 | def __init__(self, d): |
| 258 | self.d = d | 171 | self.d = d |
| @@ -324,6 +237,86 @@ class TestContext(object): | |||
| 324 | 237 | ||
| 325 | return testslist | 238 | return testslist |
| 326 | 239 | ||
| 240 | def loadTests(self): | ||
| 241 | setattr(oeTest, "tc", self) | ||
| 242 | |||
| 243 | testloader = unittest.TestLoader() | ||
| 244 | testloader.sortTestMethodsUsing = None | ||
| 245 | suites = [testloader.loadTestsFromName(name) for name in self.testslist] | ||
| 246 | suites = filterByTagExp(suites, getattr(self, "tagexp", None)) | ||
| 247 | |||
| 248 | def getTests(test): | ||
| 249 | '''Return all individual tests executed when running the suite.''' | ||
| 250 | # Unfortunately unittest does not have an API for this, so we have | ||
| 251 | # to rely on implementation details. This only needs to work | ||
| 252 | # for TestSuite containing TestCase. | ||
| 253 | method = getattr(test, '_testMethodName', None) | ||
| 254 | if method: | ||
| 255 | # leaf case: a TestCase | ||
| 256 | yield test | ||
| 257 | else: | ||
| 258 | # Look into TestSuite. | ||
| 259 | tests = getattr(test, '_tests', []) | ||
| 260 | for t1 in tests: | ||
| 261 | for t2 in getTests(t1): | ||
| 262 | yield t2 | ||
| 263 | |||
| 264 | # Determine dependencies between suites by looking for @skipUnlessPassed | ||
| 265 | # method annotations. Suite A depends on suite B if any method in A | ||
| 266 | # depends on a method on B. | ||
| 267 | for suite in suites: | ||
| 268 | suite.dependencies = [] | ||
| 269 | suite.depth = 0 | ||
| 270 | for test in getTests(suite): | ||
| 271 | methodname = getattr(test, '_testMethodName', None) | ||
| 272 | if methodname: | ||
| 273 | method = getattr(test, methodname) | ||
| 274 | depends_on = getattr(method, '_depends_on', None) | ||
| 275 | if depends_on: | ||
| 276 | for dep_suite in suites: | ||
| 277 | if depends_on in [getattr(t, '_testMethodName', None) for t in getTests(dep_suite)]: | ||
| 278 | if dep_suite not in suite.dependencies and \ | ||
| 279 | dep_suite is not suite: | ||
| 280 | suite.dependencies.append(dep_suite) | ||
| 281 | break | ||
| 282 | else: | ||
| 283 | logger.warning("Test %s was declared as @skipUnlessPassed('%s') but that test is either not defined or not active. Will run the test anyway." % | ||
| 284 | (test, depends_on)) | ||
| 285 | |||
| 286 | # Use brute-force topological sort to determine ordering. Sort by | ||
| 287 | # depth (higher depth = must run later), with original ordering to | ||
| 288 | # break ties. | ||
| 289 | def set_suite_depth(suite): | ||
| 290 | for dep in suite.dependencies: | ||
| 291 | new_depth = set_suite_depth(dep) + 1 | ||
| 292 | if new_depth > suite.depth: | ||
| 293 | suite.depth = new_depth | ||
| 294 | return suite.depth | ||
| 295 | |||
| 296 | for index, suite in enumerate(suites): | ||
| 297 | set_suite_depth(suite) | ||
| 298 | suite.index = index | ||
| 299 | suites.sort(cmp=lambda a,b: cmp((a.depth, a.index), (b.depth, b.index))) | ||
| 300 | |||
| 301 | self.suite = testloader.suiteClass(suites) | ||
| 302 | |||
| 303 | return self.suite | ||
| 304 | |||
| 305 | def runTests(self): | ||
| 306 | logger.info("Test modules %s" % self.testslist) | ||
| 307 | if hasattr(self, "tagexp") and self.tagexp: | ||
| 308 | logger.info("Filter test cases by tags: %s" % self.tagexp) | ||
| 309 | logger.info("Found %s tests" % self.suite.countTestCases()) | ||
| 310 | runner = unittest.TextTestRunner(verbosity=2) | ||
| 311 | try: | ||
| 312 | if bb.msg.loggerDefaultVerbose: | ||
| 313 | runner.stream.write = custom_verbose | ||
| 314 | except NameError: | ||
| 315 | # Not in bb environment? | ||
| 316 | pass | ||
| 317 | |||
| 318 | return runner.run(self.suite) | ||
| 319 | |||
| 327 | class ImageTestContext(TestContext): | 320 | class ImageTestContext(TestContext): |
| 328 | def __init__(self, d, target, host_dumper): | 321 | def __init__(self, d, target, host_dumper): |
| 329 | super(ImageTestContext, self).__init__(d) | 322 | super(ImageTestContext, self).__init__(d) |
| @@ -374,6 +367,10 @@ class ImageTestContext(TestContext): | |||
| 374 | def _get_test_suites_required(self): | 367 | def _get_test_suites_required(self): |
| 375 | return [t for t in self.d.getVar("TEST_SUITES", True).split() if t != "auto"] | 368 | return [t for t in self.d.getVar("TEST_SUITES", True).split() if t != "auto"] |
| 376 | 369 | ||
| 370 | def loadTests(self): | ||
| 371 | super(ImageTestContext, self).loadTests() | ||
| 372 | setattr(oeRuntimeTest, "pscmd", "ps -ef" if oeTest.hasPackage("procps") else "ps") | ||
| 373 | |||
| 377 | class SDKTestContext(TestContext): | 374 | class SDKTestContext(TestContext): |
| 378 | def __init__(self, d, sdktestdir, sdkenv): | 375 | def __init__(self, d, sdktestdir, sdkenv): |
| 379 | super(SDKTestContext, self).__init__(d) | 376 | super(SDKTestContext, self).__init__(d) |
