From 05b02e3d57c0cac37694bd3dcf3c8e04c63d75bd Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 29 Nov 2024 12:51:23 +0000 Subject: bitbake: runqueue: Fix performance of multiconfigs with large overlap There have been complaints about the performance of large multiconfig builds for a while. The key missing data point was that the builds needed to have large overlaps in sstate objects. This can be simulated by building the same things with just different TMPDIRs. In runqueue/bitbake terms this equates to large numbers of deferred tasks. The issue is that the expensive checks in the setscene loop were hit every time through runqueue's execute function before the check on deferred tasks. This leads to task execution starvation as that only happens once per iteration. Move the skip check earlier in the function which speeds things up enormously and should improve performance of such builds for users. (Bitbake rev: 9c6c506757f2b3e28c8b20513b45da6b4659c95f) Signed-off-by: Richard Purdie --- bitbake/lib/bb/runqueue.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'bitbake/lib') diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 61608ac603..2179ee1302 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py @@ -2210,6 +2210,9 @@ class RunQueueExecute: # Find the next setscene to run for nexttask in self.sorted_setscene_tids: if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values() and nexttask not in self.sq_harddep_deferred: + if nexttask in self.sq_deferred and self.sq_deferred[nexttask] not in self.runq_complete: + # Skip deferred tasks quickly before the 'expensive' tests below - this is key to performant multiconfig builds + continue if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and \ nexttask not in self.sq_needed_harddeps and \ self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and \ @@ -2239,8 +2242,7 @@ class RunQueueExecute: if t in self.runq_running and t not in self.runq_complete: continue if nexttask in self.sq_deferred: - if self.sq_deferred[nexttask] not in self.runq_complete: - continue + # Deferred tasks that were still deferred were skipped above so we now need to process logger.debug("Task %s no longer deferred" % nexttask) del self.sq_deferred[nexttask] valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, 0, False, summary=False) -- cgit v1.2.3-54-g00ecf