summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/cooker.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/cooker.py')
-rw-r--r--bitbake/lib/bb/cooker.py30
1 files changed, 18 insertions, 12 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 2bb80e330d..dc131939ed 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -26,6 +26,7 @@ import json
26import pickle 26import pickle
27import codecs 27import codecs
28import hashserv 28import hashserv
29import ctypes
29 30
30logger = logging.getLogger("BitBake") 31logger = logging.getLogger("BitBake")
31collectlog = logging.getLogger("BitBake.Collection") 32collectlog = logging.getLogger("BitBake.Collection")
@@ -1998,8 +1999,9 @@ class ParsingFailure(Exception):
1998 Exception.__init__(self, realexception, recipe) 1999 Exception.__init__(self, realexception, recipe)
1999 2000
2000class Parser(multiprocessing.Process): 2001class Parser(multiprocessing.Process):
2001 def __init__(self, jobs, results, quit, profile): 2002 def __init__(self, jobs, next_job_id, results, quit, profile):
2002 self.jobs = jobs 2003 self.jobs = jobs
2004 self.next_job_id = next_job_id
2003 self.results = results 2005 self.results = results
2004 self.quit = quit 2006 self.quit = quit
2005 multiprocessing.Process.__init__(self) 2007 multiprocessing.Process.__init__(self)
@@ -2065,10 +2067,14 @@ class Parser(multiprocessing.Process):
2065 break 2067 break
2066 2068
2067 job = None 2069 job = None
2068 try: 2070 if havejobs:
2069 job = self.jobs.pop() 2071 with self.next_job_id.get_lock():
2070 except IndexError: 2072 if self.next_job_id.value < len(self.jobs):
2071 havejobs = False 2073 job = self.jobs[self.next_job_id.value]
2074 self.next_job_id.value += 1
2075 else:
2076 havejobs = False
2077
2072 if job: 2078 if job:
2073 result = self.parse(*job) 2079 result = self.parse(*job)
2074 # Clear the siggen cache after parsing to control memory usage, its huge 2080 # Clear the siggen cache after parsing to control memory usage, its huge
@@ -2134,13 +2140,13 @@ class CookerParser(object):
2134 2140
2135 self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array) 2141 self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array)
2136 self.fromcache = set() 2142 self.fromcache = set()
2137 self.willparse = set() 2143 self.willparse = []
2138 for mc in self.cooker.multiconfigs: 2144 for mc in self.cooker.multiconfigs:
2139 for filename in self.mcfilelist[mc]: 2145 for filename in self.mcfilelist[mc]:
2140 appends = self.cooker.collections[mc].get_file_appends(filename) 2146 appends = self.cooker.collections[mc].get_file_appends(filename)
2141 layername = self.cooker.collections[mc].calc_bbfile_priority(filename)[2] 2147 layername = self.cooker.collections[mc].calc_bbfile_priority(filename)[2]
2142 if not self.bb_caches[mc].cacheValid(filename, appends): 2148 if not self.bb_caches[mc].cacheValid(filename, appends):
2143 self.willparse.add((mc, self.bb_caches[mc], filename, appends, layername)) 2149 self.willparse.append((mc, self.bb_caches[mc], filename, appends, layername))
2144 else: 2150 else:
2145 self.fromcache.add((mc, self.bb_caches[mc], filename, appends, layername)) 2151 self.fromcache.add((mc, self.bb_caches[mc], filename, appends, layername))
2146 2152
@@ -2159,18 +2165,18 @@ class CookerParser(object):
2159 def start(self): 2165 def start(self):
2160 self.results = self.load_cached() 2166 self.results = self.load_cached()
2161 self.processes = [] 2167 self.processes = []
2168
2162 if self.toparse: 2169 if self.toparse:
2163 bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) 2170 bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
2164 2171
2172 next_job_id = multiprocessing.Value(ctypes.c_int, 0)
2165 self.parser_quit = multiprocessing.Event() 2173 self.parser_quit = multiprocessing.Event()
2166 self.result_queue = multiprocessing.Queue() 2174 self.result_queue = multiprocessing.Queue()
2167 2175
2168 def chunkify(lst,n): 2176 # Have to pass in willparse at fork time so all parsing processes have the unpickleable data
2169 return [lst[i::n] for i in range(n)] 2177 # then access it by index from the parse queue.
2170 self.jobs = chunkify(list(self.willparse), self.num_processes)
2171
2172 for i in range(0, self.num_processes): 2178 for i in range(0, self.num_processes):
2173 parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, self.cooker.configuration.profile) 2179 parser = Parser(self.willparse, next_job_id, self.result_queue, self.parser_quit, self.cooker.configuration.profile)
2174 parser.start() 2180 parser.start()
2175 self.process_names.append(parser.name) 2181 self.process_names.append(parser.name)
2176 self.processes.append(parser) 2182 self.processes.append(parser)