From 1069c364170ffbf1bff276fd965aeb85efbc22f8 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Thu, 11 Jul 2019 17:05:19 +0100 Subject: bitbake: runqueue: Optimise multiconfig with overlapping setscene Currently if a multiconfig build contains different configurations which have overlapping sstate artefacts, it will build them multiple times. This is clearly suboptimal and not what users want/expect. This adds code to detect this and stall all but one of the setscne tasks so that once its built, it can be found by the other tasks. We take care to iterate the multiconfigs in order so try and avoid dependency loops. We also match on PN+taskname+taskhash since this is what we know sstate in OE-Core would use. There are some tasks even within a multiconfig which match hashes (mostly do_populate_lic tasks) but those have a much higher chance of circular dependency so aren't work attempting to optimise. If a deadlock does occur the build will be slower but there is code to unbreak such a deadlock so it hopefully doens't break anything. Comments are injected into the test tasks so they have different task hashes and a new test for this optimisation is added. (Bitbake rev: a75c5fd6d4ec56836de0be2fe679c81297a080ad) Signed-off-by: Richard Purdie --- .../lib/bb/tests/runqueue-tests/classes/base.bbclass | 19 ++++++++++++++++++- bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf | 9 +++++++-- .../bb/tests/runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../bb/tests/runqueue-tests/conf/multiconfig/mc2.conf | 1 + 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf create mode 100644 bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf (limited to 'bitbake/lib/bb/tests/runqueue-tests') diff --git a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass index e174c02dd6..cf38d09224 100644 --- a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass +++ b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass @@ -4,7 +4,9 @@ SSTATEVALID ??= "" def stamptask(d): import time - thistask = d.expand("${PN}:${BB_CURRENTTASK}") + thistask = d.expand("${PN}:${BB_CURRENTTASK}") + if d.getVar("BB_CURRENT_MC") != "default": + thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}") if thistask in d.getVar("SLOWTASKS").split(): bb.note("Slowing task %s" % thistask) time.sleep(0.5) @@ -13,48 +15,63 @@ def stamptask(d): f.write(thistask + "\n") python do_fetch() { + # fetch stamptask(d) } python do_unpack() { + # unpack stamptask(d) } python do_patch() { + # patch stamptask(d) } python do_populate_lic() { + # populate_lic stamptask(d) } python do_prepare_recipe_sysroot() { + # prepare_recipe_sysroot stamptask(d) } python do_configure() { + # configure stamptask(d) } python do_compile() { + # compile stamptask(d) } python do_install() { + # install stamptask(d) } python do_populate_sysroot() { + # populate_sysroot stamptask(d) } python do_package() { + # package stamptask(d) } python do_package_write_ipk() { + # package_write_ipk stamptask(d) } python do_package_write_rpm() { + # package_write_rpm stamptask(d) } python do_packagedata() { + # packagedata stamptask(d) } python do_package_qa() { + # package_qa stamptask(d) } python do_build() { + # build stamptask(d) } do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot" diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf index 8c7b754dab..96ee1cd5ec 100644 --- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf @@ -6,6 +6,11 @@ PROVIDES = "${PN}" PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}" PF = "${BB_CURRENT_MC}:${PN}" export PATH -STAMP = "${TOPDIR}/stamps/${PN}" -T = "${TOPDIR}/workdir/${PN}/temp" +TMPDIR ??= "${TOPDIR}" +STAMP = "${TMPDIR}/stamps/${PN}" +T = "${TMPDIR}/workdir/${PN}/temp" BB_NUMBER_THREADS = "4" + +BB_HASHBASE_WHITELIST = "BB_CURRENT_MC" + +include conf/multiconfig/${BB_CURRENT_MC}.conf diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf new file mode 100644 index 0000000000..ecf23e1c73 --- /dev/null +++ b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf @@ -0,0 +1 @@ +TMPDIR = "${TOPDIR}/mc1/" diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf new file mode 100644 index 0000000000..eef338e4cc --- /dev/null +++ b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf @@ -0,0 +1 @@ +TMPDIR = "${TOPDIR}/mc2/" -- cgit v1.2.3-54-g00ecf