diff options
| -rw-r--r-- | meta/classes/testimage.bbclass | 11 | ||||
| -rw-r--r-- | meta/classes/testsdk.bbclass | 19 | ||||
| -rw-r--r-- | meta/lib/oeqa/oetest.py | 192 |
3 files changed, 112 insertions, 110 deletions
diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass index bdd6c9de6f..7f9c918273 100644 --- a/meta/classes/testimage.bbclass +++ b/meta/classes/testimage.bbclass | |||
| @@ -195,8 +195,7 @@ def testimage_main(d): | |||
| 195 | import oeqa.runtime | 195 | import oeqa.runtime |
| 196 | import time | 196 | import time |
| 197 | import signal | 197 | import signal |
| 198 | from oeqa.oetest import loadTests, runTests, \ | 198 | from oeqa.oetest import loadTests, runTests, ImageTestContext |
| 199 | get_test_suites, get_tests_list, ImageTestContext | ||
| 200 | from oeqa.targetcontrol import get_target_controller | 199 | from oeqa.targetcontrol import get_target_controller |
| 201 | from oeqa.utils.dump import get_host_dumper | 200 | from oeqa.utils.dump import get_host_dumper |
| 202 | 201 | ||
| @@ -207,12 +206,6 @@ def testimage_main(d): | |||
| 207 | bb.utils.remove(d.getVar("TEST_EXPORT_DIR", True), recurse=True) | 206 | bb.utils.remove(d.getVar("TEST_EXPORT_DIR", True), recurse=True) |
| 208 | bb.utils.mkdirhier(d.getVar("TEST_EXPORT_DIR", True)) | 207 | bb.utils.mkdirhier(d.getVar("TEST_EXPORT_DIR", True)) |
| 209 | 208 | ||
| 210 | # tests in TEST_SUITES become required tests | ||
| 211 | # they won't be skipped even if they aren't suitable for a image (like xorg for minimal) | ||
| 212 | # testslist is what we'll actually pass to the unittest loader | ||
| 213 | testslist = get_tests_list(get_test_suites(d), d.getVar("BBPATH", True).split(':')) | ||
| 214 | testsrequired = [t for t in d.getVar("TEST_SUITES", True).split() if t != "auto"] | ||
| 215 | |||
| 216 | # we need the host dumper in test context | 209 | # we need the host dumper in test context |
| 217 | host_dumper = get_host_dumper(d) | 210 | host_dumper = get_host_dumper(d) |
| 218 | 211 | ||
| @@ -220,7 +213,7 @@ def testimage_main(d): | |||
| 220 | target = get_target_controller(d) | 213 | target = get_target_controller(d) |
| 221 | 214 | ||
| 222 | # test context | 215 | # test context |
| 223 | tc = ImageTestContext(d, testslist, testsrequired, target, host_dumper) | 216 | tc = ImageTestContext(d, target, host_dumper) |
| 224 | 217 | ||
| 225 | # this is a dummy load of tests | 218 | # this is a dummy load of tests |
| 226 | # we are doing that to find compile errors in the tests themselves | 219 | # we are doing that to find compile errors in the tests themselves |
diff --git a/meta/classes/testsdk.bbclass b/meta/classes/testsdk.bbclass index 06e7a77398..47bad29096 100644 --- a/meta/classes/testsdk.bbclass +++ b/meta/classes/testsdk.bbclass | |||
| @@ -13,18 +13,11 @@ def testsdk_main(d): | |||
| 13 | import oeqa.sdk | 13 | import oeqa.sdk |
| 14 | import time | 14 | import time |
| 15 | import subprocess | 15 | import subprocess |
| 16 | from oeqa.oetest import loadTests, runTests, \ | 16 | from oeqa.oetest import loadTests, runTests, SDKTestContext |
| 17 | get_test_suites, get_tests_list, SDKTestContext | ||
| 18 | 17 | ||
| 19 | pn = d.getVar("PN", True) | 18 | pn = d.getVar("PN", True) |
| 20 | bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True)) | 19 | bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True)) |
| 21 | 20 | ||
| 22 | # tests in TEST_SUITES become required tests | ||
| 23 | # they won't be skipped even if they aren't suitable. | ||
| 24 | # testslist is what we'll actually pass to the unittest loader | ||
| 25 | testslist = get_tests_list(get_test_suites(d, "sdk"), d.getVar("BBPATH", True).split(':'), "sdk") | ||
| 26 | testsrequired = [t for t in (d.getVar("TEST_SUITES_SDK", True) or "auto").split() if t != "auto"] | ||
| 27 | |||
| 28 | tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh") | 21 | tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh") |
| 29 | if not os.path.exists(tcname): | 22 | if not os.path.exists(tcname): |
| 30 | bb.fatal("The toolchain is not built. Build it before running the tests: 'bitbake <image> -c populate_sdk' .") | 23 | bb.fatal("The toolchain is not built. Build it before running the tests: 'bitbake <image> -c populate_sdk' .") |
| @@ -41,7 +34,7 @@ def testsdk_main(d): | |||
| 41 | targets = glob.glob(d.expand(sdktestdir + "/tc/environment-setup-*")) | 34 | targets = glob.glob(d.expand(sdktestdir + "/tc/environment-setup-*")) |
| 42 | for sdkenv in targets: | 35 | for sdkenv in targets: |
| 43 | bb.plain("Testing %s" % sdkenv) | 36 | bb.plain("Testing %s" % sdkenv) |
| 44 | tc = SDKTestContext(d, testslist, testsrequired, sdktestdir, sdkenv) | 37 | tc = SDKTestContext(d, sdktestdir, sdkenv) |
| 45 | 38 | ||
| 46 | # this is a dummy load of tests | 39 | # this is a dummy load of tests |
| 47 | # we are doing that to find compile errors in the tests themselves | 40 | # we are doing that to find compile errors in the tests themselves |
| @@ -94,14 +87,6 @@ def testsdkext_main(d): | |||
| 94 | pn = d.getVar("PN", True) | 87 | pn = d.getVar("PN", True) |
| 95 | bb.utils.mkdirhier(d.getVar("TEST_LOG_SDKEXT_DIR", True)) | 88 | bb.utils.mkdirhier(d.getVar("TEST_LOG_SDKEXT_DIR", True)) |
| 96 | 89 | ||
| 97 | # tests in TEST_SUITES become required tests | ||
| 98 | # they won't be skipped even if they aren't suitable. | ||
| 99 | # testslist is what we'll actually pass to the unittest loader | ||
| 100 | testslist = get_tests_list(get_test_suites(d, "sdkext"), | ||
| 101 | d.getVar("BBPATH", True).split(':'), "sdkext") | ||
| 102 | testsrequired = [t for t in (d.getVar("TEST_SUITES_SDKEXT", True) or \ | ||
| 103 | "auto").split() if t != "auto"] | ||
| 104 | |||
| 105 | tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.sh") | 90 | tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.sh") |
| 106 | if not os.path.exists(tcname): | 91 | if not os.path.exists(tcname): |
| 107 | bb.fatal("The toolchain ext is not built. Build it before running the" \ | 92 | bb.fatal("The toolchain ext is not built. Build it before running the" \ |
diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py index 9951a6f40e..7f07037a1f 100644 --- a/meta/lib/oeqa/oetest.py +++ b/meta/lib/oeqa/oetest.py | |||
| @@ -33,7 +33,6 @@ def getVar(obj): | |||
| 33 | def checkTags(tc, tagexp): | 33 | def checkTags(tc, tagexp): |
| 34 | return eval(tagexp, None, getVar(tc)) | 34 | return eval(tagexp, None, getVar(tc)) |
| 35 | 35 | ||
| 36 | |||
| 37 | def filterByTagExp(testsuite, tagexp): | 36 | def filterByTagExp(testsuite, tagexp): |
| 38 | if not tagexp: | 37 | if not tagexp: |
| 39 | return testsuite | 38 | return testsuite |
| @@ -254,96 +253,80 @@ def skipModuleUnless(cond, reason): | |||
| 254 | if not cond: | 253 | if not cond: |
| 255 | skipModule(reason, 3) | 254 | skipModule(reason, 3) |
| 256 | 255 | ||
| 257 | # get testcase list from specified file | ||
| 258 | # if path is a relative path, then relative to build/conf/ | ||
| 259 | def read_testlist(fpath, builddir): | ||
| 260 | if not os.path.isabs(fpath): | ||
| 261 | fpath = os.path.join(builddir, "conf", fpath) | ||
| 262 | if not os.path.exists(fpath): | ||
| 263 | bb.fatal("No such manifest file: ", fpath) | ||
| 264 | tcs = [] | ||
| 265 | for line in open(fpath).readlines(): | ||
| 266 | line = line.strip() | ||
| 267 | if line and not line.startswith("#"): | ||
| 268 | tcs.append(line) | ||
| 269 | return " ".join(tcs) | ||
| 270 | |||
| 271 | # get test suites, returns test suites based on d variables | ||
| 272 | def get_test_suites(d, type='runtime'): | ||
| 273 | testsuites = [] | ||
| 274 | |||
| 275 | if type == "sdk": | ||
| 276 | testsuites = (d.getVar("TEST_SUITES_SDK", True) or "auto").split() | ||
| 277 | elif type == "sdkext": | ||
| 278 | testsuites = (d.getVar("TEST_SUITES_SDKEXT", True) or "auto").split() | ||
| 279 | else: | ||
| 280 | manifests = (d.getVar("TEST_SUITES_MANIFEST", True) or '').split() | ||
| 281 | if manifests: | ||
| 282 | for manifest in manifests: | ||
| 283 | testsuites.extend(read_testlist(manifest, | ||
| 284 | d.getVar("TOPDIR", True)).split()) | ||
| 285 | |||
| 286 | else: | ||
| 287 | testsuites = d.getVar("TEST_SUITES", True).split() | ||
| 288 | |||
| 289 | return testsuites | ||
| 290 | |||
| 291 | # return test list by type also filter if TEST_SUITES is specified | ||
| 292 | def get_tests_list(testsuites, bbpath, type="runtime"): | ||
| 293 | testslist = [] | ||
| 294 | |||
| 295 | # This relies on lib/ under each directory in BBPATH being added to sys.path | ||
| 296 | # (as done by default in base.bbclass) | ||
| 297 | for testname in testsuites: | ||
| 298 | if testname != "auto": | ||
| 299 | if testname.startswith("oeqa."): | ||
| 300 | testslist.append(testname) | ||
| 301 | continue | ||
| 302 | found = False | ||
| 303 | for p in bbpath: | ||
| 304 | if os.path.exists(os.path.join(p, 'lib', 'oeqa', type, testname + '.py')): | ||
| 305 | testslist.append("oeqa." + type + "." + testname) | ||
| 306 | found = True | ||
| 307 | break | ||
| 308 | elif os.path.exists(os.path.join(p, 'lib', 'oeqa', type, testname.split(".")[0] + '.py')): | ||
| 309 | testslist.append("oeqa." + type + "." + testname) | ||
| 310 | found = True | ||
| 311 | break | ||
| 312 | if not found: | ||
| 313 | bb.fatal('Test %s specified in TEST_SUITES could not be found in lib/oeqa/runtime under BBPATH' % testname) | ||
| 314 | |||
| 315 | if "auto" in testsuites: | ||
| 316 | def add_auto_list(path): | ||
| 317 | if not os.path.exists(os.path.join(path, '__init__.py')): | ||
| 318 | bb.fatal('Tests directory %s exists but is missing __init__.py' % path) | ||
| 319 | files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')]) | ||
| 320 | for f in files: | ||
| 321 | module = 'oeqa.' + type + '.' + f[:-3] | ||
| 322 | if module not in testslist: | ||
| 323 | testslist.append(module) | ||
| 324 | |||
| 325 | for p in bbpath: | ||
| 326 | testpath = os.path.join(p, 'lib', 'oeqa', type) | ||
| 327 | bb.debug(2, 'Searching for tests in %s' % testpath) | ||
| 328 | if os.path.exists(testpath): | ||
| 329 | add_auto_list(testpath) | ||
| 330 | |||
| 331 | return testslist | ||
| 332 | |||
| 333 | class TestContext(object): | 256 | class TestContext(object): |
| 334 | def __init__(self, d, testslist, testsrequired): | 257 | def __init__(self, d): |
| 335 | self.d = d | 258 | self.d = d |
| 336 | self.testslist = testslist | 259 | |
| 337 | self.testsrequired = testsrequired | 260 | self.testsuites = self._get_test_suites() |
| 261 | self.testslist = self._get_tests_list(d.getVar("BBPATH", True).split(':')) | ||
| 262 | self.testsrequired = self._get_test_suites_required() | ||
| 338 | 263 | ||
| 339 | self.filesdir = os.path.join(os.path.dirname(os.path.abspath( | 264 | self.filesdir = os.path.join(os.path.dirname(os.path.abspath( |
| 340 | oeqa.runtime.__file__)), "files") | 265 | oeqa.runtime.__file__)), "files") |
| 341 | self.imagefeatures = d.getVar("IMAGE_FEATURES", True).split() | 266 | self.imagefeatures = d.getVar("IMAGE_FEATURES", True).split() |
| 342 | self.distrofeatures = d.getVar("DISTRO_FEATURES", True).split() | 267 | self.distrofeatures = d.getVar("DISTRO_FEATURES", True).split() |
| 343 | 268 | ||
| 269 | # get testcase list from specified file | ||
| 270 | # if path is a relative path, then relative to build/conf/ | ||
| 271 | def _read_testlist(self, fpath, builddir): | ||
| 272 | if not os.path.isabs(fpath): | ||
| 273 | fpath = os.path.join(builddir, "conf", fpath) | ||
| 274 | if not os.path.exists(fpath): | ||
| 275 | bb.fatal("No such manifest file: ", fpath) | ||
| 276 | tcs = [] | ||
| 277 | for line in open(fpath).readlines(): | ||
| 278 | line = line.strip() | ||
| 279 | if line and not line.startswith("#"): | ||
| 280 | tcs.append(line) | ||
| 281 | return " ".join(tcs) | ||
| 282 | |||
| 283 | # return test list by type also filter if TEST_SUITES is specified | ||
| 284 | def _get_tests_list(self, bbpath): | ||
| 285 | testslist = [] | ||
| 286 | |||
| 287 | type = self._get_test_namespace() | ||
| 288 | |||
| 289 | # This relies on lib/ under each directory in BBPATH being added to sys.path | ||
| 290 | # (as done by default in base.bbclass) | ||
| 291 | for testname in self.testsuites: | ||
| 292 | if testname != "auto": | ||
| 293 | if testname.startswith("oeqa."): | ||
| 294 | testslist.append(testname) | ||
| 295 | continue | ||
| 296 | found = False | ||
| 297 | for p in bbpath: | ||
| 298 | if os.path.exists(os.path.join(p, 'lib', 'oeqa', type, testname + '.py')): | ||
| 299 | testslist.append("oeqa." + type + "." + testname) | ||
| 300 | found = True | ||
| 301 | break | ||
| 302 | elif os.path.exists(os.path.join(p, 'lib', 'oeqa', type, testname.split(".")[0] + '.py')): | ||
| 303 | testslist.append("oeqa." + type + "." + testname) | ||
| 304 | found = True | ||
| 305 | break | ||
| 306 | if not found: | ||
| 307 | bb.fatal('Test %s specified in TEST_SUITES could not be found in lib/oeqa/runtime under BBPATH' % testname) | ||
| 308 | |||
| 309 | if "auto" in self.testsuites: | ||
| 310 | def add_auto_list(path): | ||
| 311 | if not os.path.exists(os.path.join(path, '__init__.py')): | ||
| 312 | bb.fatal('Tests directory %s exists but is missing __init__.py' % path) | ||
| 313 | files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')]) | ||
| 314 | for f in files: | ||
| 315 | module = 'oeqa.' + type + '.' + f[:-3] | ||
| 316 | if module not in testslist: | ||
| 317 | testslist.append(module) | ||
| 318 | |||
| 319 | for p in bbpath: | ||
| 320 | testpath = os.path.join(p, 'lib', 'oeqa', type) | ||
| 321 | bb.debug(2, 'Searching for tests in %s' % testpath) | ||
| 322 | if os.path.exists(testpath): | ||
| 323 | add_auto_list(testpath) | ||
| 324 | |||
| 325 | return testslist | ||
| 326 | |||
| 344 | class ImageTestContext(TestContext): | 327 | class ImageTestContext(TestContext): |
| 345 | def __init__(self, d, testslist, testsrequired, target, host_dumper): | 328 | def __init__(self, d, target, host_dumper): |
| 346 | super(ImageTestContext, self).__init__(d, testslist, testsrequired) | 329 | super(ImageTestContext, self).__init__(d) |
| 347 | 330 | ||
| 348 | self.tagexp = d.getVar("TEST_SUITES_TAGS", True) | 331 | self.tagexp = d.getVar("TEST_SUITES_TAGS", True) |
| 349 | 332 | ||
| @@ -371,9 +354,29 @@ class ImageTestContext(TestContext): | |||
| 371 | self.sigterm = True | 354 | self.sigterm = True |
| 372 | self.target.stop() | 355 | self.target.stop() |
| 373 | 356 | ||
| 357 | def _get_test_namespace(self): | ||
| 358 | return "runtime" | ||
| 359 | |||
| 360 | def _get_test_suites(self): | ||
| 361 | testsuites = [] | ||
| 362 | |||
| 363 | manifests = (self.d.getVar("TEST_SUITES_MANIFEST", True) or '').split() | ||
| 364 | if manifests: | ||
| 365 | for manifest in manifests: | ||
| 366 | testsuites.extend(self._read_testlist(manifest, | ||
| 367 | self.d.getVar("TOPDIR", True)).split()) | ||
| 368 | |||
| 369 | else: | ||
| 370 | testsuites = self.d.getVar("TEST_SUITES", True).split() | ||
| 371 | |||
| 372 | return testsuites | ||
| 373 | |||
| 374 | def _get_test_suites_required(self): | ||
| 375 | return [t for t in self.d.getVar("TEST_SUITES", True).split() if t != "auto"] | ||
| 376 | |||
| 374 | class SDKTestContext(TestContext): | 377 | class SDKTestContext(TestContext): |
| 375 | def __init__(self, d, testslist, testsrequired, sdktestdir, sdkenv): | 378 | def __init__(self, d, sdktestdir, sdkenv): |
| 376 | super(SDKTestContext, self).__init__(d, testslist, testsrequired) | 379 | super(SDKTestContext, self).__init__(d) |
| 377 | 380 | ||
| 378 | self.sdktestdir = sdktestdir | 381 | self.sdktestdir = sdktestdir |
| 379 | self.sdkenv = sdkenv | 382 | self.sdkenv = sdkenv |
| @@ -389,3 +392,24 @@ class SDKTestContext(TestContext): | |||
| 389 | self.hostpkgmanifest = f.read() | 392 | self.hostpkgmanifest = f.read() |
| 390 | except IOError as e: | 393 | except IOError as e: |
| 391 | bb.fatal("No host package manifest file found. Did you build the sdk image?\n%s" % e) | 394 | bb.fatal("No host package manifest file found. Did you build the sdk image?\n%s" % e) |
| 395 | |||
| 396 | def _get_test_namespace(self): | ||
| 397 | return "sdk" | ||
| 398 | |||
| 399 | def _get_test_suites(self): | ||
| 400 | return (self.d.getVar("TEST_SUITES_SDK", True) or "auto").split() | ||
| 401 | |||
| 402 | def _get_test_suites_required(self): | ||
| 403 | return [t for t in (self.d.getVar("TEST_SUITES_SDK", True) or \ | ||
| 404 | "auto").split() if t != "auto"] | ||
| 405 | |||
| 406 | class SDKExtTestContext(TestContext): | ||
| 407 | def _get_test_namespace(self): | ||
| 408 | return "sdkext" | ||
| 409 | |||
| 410 | def _get_test_suites(self): | ||
| 411 | return (self.d.getVar("TEST_SUITES_SDK_EXT", True) or "auto").split() | ||
| 412 | |||
| 413 | def _get_test_suites_required(self): | ||
| 414 | return [t for t in (self.d.getVar("TEST_SUITES_SDK_EXT", True) or \ | ||
| 415 | "auto").split() if t != "auto"] | ||
