diff options
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
| -rw-r--r-- | bitbake/lib/bb/runqueue.py | 93 |
1 files changed, 46 insertions, 47 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 9a368b8622..2ecfd09469 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
| @@ -30,7 +30,7 @@ import fcntl | |||
| 30 | 30 | ||
| 31 | class TaskFailure(Exception): | 31 | class TaskFailure(Exception): |
| 32 | """Exception raised when a task in a runqueue fails""" | 32 | """Exception raised when a task in a runqueue fails""" |
| 33 | def __init__(self, x): | 33 | def __init__(self, x): |
| 34 | self.args = x | 34 | self.args = x |
| 35 | 35 | ||
| 36 | 36 | ||
| @@ -60,7 +60,7 @@ class RunQueueStats: | |||
| 60 | def taskActive(self): | 60 | def taskActive(self): |
| 61 | self.active = self.active + 1 | 61 | self.active = self.active + 1 |
| 62 | 62 | ||
| 63 | # These values indicate the next step due to be run in the | 63 | # These values indicate the next step due to be run in the |
| 64 | # runQueue state machine | 64 | # runQueue state machine |
| 65 | runQueuePrepare = 2 | 65 | runQueuePrepare = 2 |
| 66 | runQueueRunInit = 3 | 66 | runQueueRunInit = 3 |
| @@ -76,7 +76,7 @@ class RunQueueScheduler: | |||
| 76 | """ | 76 | """ |
| 77 | def __init__(self, runqueue): | 77 | def __init__(self, runqueue): |
| 78 | """ | 78 | """ |
| 79 | The default scheduler just returns the first buildable task (the | 79 | The default scheduler just returns the first buildable task (the |
| 80 | priority map is sorted by task numer) | 80 | priority map is sorted by task numer) |
| 81 | """ | 81 | """ |
| 82 | self.rq = runqueue | 82 | self.rq = runqueue |
| @@ -123,10 +123,10 @@ class RunQueueSchedulerSpeed(RunQueueScheduler): | |||
| 123 | 123 | ||
| 124 | class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed): | 124 | class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed): |
| 125 | """ | 125 | """ |
| 126 | A scheduler optimised to complete .bb files are quickly as possible. The | 126 | A scheduler optimised to complete .bb files are quickly as possible. The |
| 127 | priority map is sorted by task weight, but then reordered so once a given | 127 | priority map is sorted by task weight, but then reordered so once a given |
| 128 | .bb file starts to build, its completed as quickly as possible. This works | 128 | .bb file starts to build, its completed as quickly as possible. This works |
| 129 | well where disk space is at a premium and classes like OE's rm_work are in | 129 | well where disk space is at a premium and classes like OE's rm_work are in |
| 130 | force. | 130 | force. |
| 131 | """ | 131 | """ |
| 132 | def __init__(self, runqueue): | 132 | def __init__(self, runqueue): |
| @@ -135,7 +135,7 @@ class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed): | |||
| 135 | 135 | ||
| 136 | #FIXME - whilst this groups all fnids together it does not reorder the | 136 | #FIXME - whilst this groups all fnids together it does not reorder the |
| 137 | #fnid groups optimally. | 137 | #fnid groups optimally. |
| 138 | 138 | ||
| 139 | basemap = deepcopy(self.prio_map) | 139 | basemap = deepcopy(self.prio_map) |
| 140 | self.prio_map = [] | 140 | self.prio_map = [] |
| 141 | while (len(basemap) > 0): | 141 | while (len(basemap) > 0): |
| @@ -231,7 +231,7 @@ class RunQueue: | |||
| 231 | if chain1[index] != chain2[index]: | 231 | if chain1[index] != chain2[index]: |
| 232 | return False | 232 | return False |
| 233 | return True | 233 | return True |
| 234 | 234 | ||
| 235 | def chain_array_contains(chain, chain_array): | 235 | def chain_array_contains(chain, chain_array): |
| 236 | """ | 236 | """ |
| 237 | Return True if chain_array contains chain | 237 | Return True if chain_array contains chain |
| @@ -286,7 +286,7 @@ class RunQueue: | |||
| 286 | 286 | ||
| 287 | def calculate_task_weights(self, endpoints): | 287 | def calculate_task_weights(self, endpoints): |
| 288 | """ | 288 | """ |
| 289 | Calculate a number representing the "weight" of each task. Heavier weighted tasks | 289 | Calculate a number representing the "weight" of each task. Heavier weighted tasks |
| 290 | have more dependencies and hence should be executed sooner for maximum speed. | 290 | have more dependencies and hence should be executed sooner for maximum speed. |
| 291 | 291 | ||
| 292 | This function also sanity checks the task list finding tasks that its not | 292 | This function also sanity checks the task list finding tasks that its not |
| @@ -318,7 +318,7 @@ class RunQueue: | |||
| 318 | task_done[revdep] = True | 318 | task_done[revdep] = True |
| 319 | endpoints = next_points | 319 | endpoints = next_points |
| 320 | if len(next_points) == 0: | 320 | if len(next_points) == 0: |
| 321 | break | 321 | break |
| 322 | 322 | ||
| 323 | # Circular dependency sanity check | 323 | # Circular dependency sanity check |
| 324 | problem_tasks = [] | 324 | problem_tasks = [] |
| @@ -345,7 +345,7 @@ class RunQueue: | |||
| 345 | 345 | ||
| 346 | def prepare_runqueue(self): | 346 | def prepare_runqueue(self): |
| 347 | """ | 347 | """ |
| 348 | Turn a set of taskData into a RunQueue and compute data needed | 348 | Turn a set of taskData into a RunQueue and compute data needed |
| 349 | to optimise the execution order. | 349 | to optimise the execution order. |
| 350 | """ | 350 | """ |
| 351 | 351 | ||
| @@ -365,12 +365,12 @@ class RunQueue: | |||
| 365 | # Step A - Work out a list of tasks to run | 365 | # Step A - Work out a list of tasks to run |
| 366 | # | 366 | # |
| 367 | # Taskdata gives us a list of possible providers for every build and run | 367 | # Taskdata gives us a list of possible providers for every build and run |
| 368 | # target ordered by priority. It also gives information on each of those | 368 | # target ordered by priority. It also gives information on each of those |
| 369 | # providers. | 369 | # providers. |
| 370 | # | 370 | # |
| 371 | # To create the actual list of tasks to execute we fix the list of | 371 | # To create the actual list of tasks to execute we fix the list of |
| 372 | # providers and then resolve the dependencies into task IDs. This | 372 | # providers and then resolve the dependencies into task IDs. This |
| 373 | # process is repeated for each type of dependency (tdepends, deptask, | 373 | # process is repeated for each type of dependency (tdepends, deptask, |
| 374 | # rdeptast, recrdeptask, idepends). | 374 | # rdeptast, recrdeptask, idepends). |
| 375 | 375 | ||
| 376 | def add_build_dependencies(depids, tasknames, depends): | 376 | def add_build_dependencies(depids, tasknames, depends): |
| @@ -411,12 +411,12 @@ class RunQueue: | |||
| 411 | 411 | ||
| 412 | if fnid not in taskData.failed_fnids: | 412 | if fnid not in taskData.failed_fnids: |
| 413 | 413 | ||
| 414 | # Resolve task internal dependencies | 414 | # Resolve task internal dependencies |
| 415 | # | 415 | # |
| 416 | # e.g. addtask before X after Y | 416 | # e.g. addtask before X after Y |
| 417 | depends = taskData.tasks_tdepends[task] | 417 | depends = taskData.tasks_tdepends[task] |
| 418 | 418 | ||
| 419 | # Resolve 'deptask' dependencies | 419 | # Resolve 'deptask' dependencies |
| 420 | # | 420 | # |
| 421 | # e.g. do_sometask[deptask] = "do_someothertask" | 421 | # e.g. do_sometask[deptask] = "do_someothertask" |
| 422 | # (makes sure sometask runs after someothertask of all DEPENDS) | 422 | # (makes sure sometask runs after someothertask of all DEPENDS) |
| @@ -424,7 +424,7 @@ class RunQueue: | |||
| 424 | tasknames = task_deps['deptask'][taskData.tasks_name[task]].split() | 424 | tasknames = task_deps['deptask'][taskData.tasks_name[task]].split() |
| 425 | add_build_dependencies(taskData.depids[fnid], tasknames, depends) | 425 | add_build_dependencies(taskData.depids[fnid], tasknames, depends) |
| 426 | 426 | ||
| 427 | # Resolve 'rdeptask' dependencies | 427 | # Resolve 'rdeptask' dependencies |
| 428 | # | 428 | # |
| 429 | # e.g. do_sometask[rdeptask] = "do_someothertask" | 429 | # e.g. do_sometask[rdeptask] = "do_someothertask" |
| 430 | # (makes sure sometask runs after someothertask of all RDEPENDS) | 430 | # (makes sure sometask runs after someothertask of all RDEPENDS) |
| @@ -432,7 +432,7 @@ class RunQueue: | |||
| 432 | taskname = task_deps['rdeptask'][taskData.tasks_name[task]] | 432 | taskname = task_deps['rdeptask'][taskData.tasks_name[task]] |
| 433 | add_runtime_dependencies(taskData.rdepids[fnid], [taskname], depends) | 433 | add_runtime_dependencies(taskData.rdepids[fnid], [taskname], depends) |
| 434 | 434 | ||
| 435 | # Resolve inter-task dependencies | 435 | # Resolve inter-task dependencies |
| 436 | # | 436 | # |
| 437 | # e.g. do_sometask[depends] = "targetname:do_someothertask" | 437 | # e.g. do_sometask[depends] = "targetname:do_someothertask" |
| 438 | # (makes sure sometask runs after targetname's someothertask) | 438 | # (makes sure sometask runs after targetname's someothertask) |
| @@ -467,8 +467,8 @@ class RunQueue: | |||
| 467 | newdep = [] | 467 | newdep = [] |
| 468 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Task %s (%s %s) contains self reference! %s" % (task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], depends)) | 468 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Task %s (%s %s) contains self reference! %s" % (task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], depends)) |
| 469 | for dep in depends: | 469 | for dep in depends: |
| 470 | if task != dep: | 470 | if task != dep: |
| 471 | newdep.append(dep) | 471 | newdep.append(dep) |
| 472 | depends = newdep | 472 | depends = newdep |
| 473 | 473 | ||
| 474 | self.runq_fnid.append(taskData.tasks_fnid[task]) | 474 | self.runq_fnid.append(taskData.tasks_fnid[task]) |
| @@ -482,7 +482,7 @@ class RunQueue: | |||
| 482 | # | 482 | # |
| 483 | # Build a list of recursive cumulative dependencies for each fnid | 483 | # Build a list of recursive cumulative dependencies for each fnid |
| 484 | # We do this by fnid, since if A depends on some task in B | 484 | # We do this by fnid, since if A depends on some task in B |
| 485 | # we're interested in later tasks B's fnid might have but B itself | 485 | # we're interested in later tasks B's fnid might have but B itself |
| 486 | # doesn't depend on | 486 | # doesn't depend on |
| 487 | # | 487 | # |
| 488 | # Algorithm is O(tasks) + O(tasks)*O(fnids) | 488 | # Algorithm is O(tasks) + O(tasks)*O(fnids) |
| @@ -513,7 +513,7 @@ class RunQueue: | |||
| 513 | if len(runq_recrdepends[task]) > 0: | 513 | if len(runq_recrdepends[task]) > 0: |
| 514 | taskfnid = self.runq_fnid[task] | 514 | taskfnid = self.runq_fnid[task] |
| 515 | for dep in reccumdepends[taskfnid]: | 515 | for dep in reccumdepends[taskfnid]: |
| 516 | # Ignore self references | 516 | # Ignore self references |
| 517 | if dep == task: | 517 | if dep == task: |
| 518 | continue | 518 | continue |
| 519 | for taskname in runq_recrdepends[task]: | 519 | for taskname in runq_recrdepends[task]: |
| @@ -635,7 +635,7 @@ class RunQueue: | |||
| 635 | 635 | ||
| 636 | bb.msg.note(2, bb.msg.domain.RunQueue, "Compute totals (have %s endpoint(s))" % len(endpoints)) | 636 | bb.msg.note(2, bb.msg.domain.RunQueue, "Compute totals (have %s endpoint(s))" % len(endpoints)) |
| 637 | 637 | ||
| 638 | # Calculate task weights | 638 | # Calculate task weights |
| 639 | # Check of higher length circular dependencies | 639 | # Check of higher length circular dependencies |
| 640 | self.runq_weight = self.calculate_task_weights(endpoints) | 640 | self.runq_weight = self.calculate_task_weights(endpoints) |
| 641 | 641 | ||
| @@ -657,7 +657,7 @@ class RunQueue: | |||
| 657 | for prov in self.dataCache.fn_provides[fn]: | 657 | for prov in self.dataCache.fn_provides[fn]: |
| 658 | if prov not in prov_list: | 658 | if prov not in prov_list: |
| 659 | prov_list[prov] = [fn] | 659 | prov_list[prov] = [fn] |
| 660 | elif fn not in prov_list[prov]: | 660 | elif fn not in prov_list[prov]: |
| 661 | prov_list[prov].append(fn) | 661 | prov_list[prov].append(fn) |
| 662 | error = False | 662 | error = False |
| 663 | for prov in prov_list: | 663 | for prov in prov_list: |
| @@ -703,7 +703,7 @@ class RunQueue: | |||
| 703 | buildable.append(task) | 703 | buildable.append(task) |
| 704 | 704 | ||
| 705 | def check_buildable(self, task, buildable): | 705 | def check_buildable(self, task, buildable): |
| 706 | for revdep in self.runq_revdeps[task]: | 706 | for revdep in self.runq_revdeps[task]: |
| 707 | alldeps = 1 | 707 | alldeps = 1 |
| 708 | for dep in self.runq_depends[revdep]: | 708 | for dep in self.runq_depends[revdep]: |
| 709 | if dep in unchecked: | 709 | if dep in unchecked: |
| @@ -811,10 +811,10 @@ class RunQueue: | |||
| 811 | try: | 811 | try: |
| 812 | t2 = os.stat(stampfile2)[stat.ST_MTIME] | 812 | t2 = os.stat(stampfile2)[stat.ST_MTIME] |
| 813 | if t1 < t2: | 813 | if t1 < t2: |
| 814 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Stampfile %s < %s" % (stampfile,stampfile2)) | 814 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Stampfile %s < %s" % (stampfile, stampfile2)) |
| 815 | iscurrent = False | 815 | iscurrent = False |
| 816 | except: | 816 | except: |
| 817 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Exception reading %s for %s" % (stampfile2 ,stampfile)) | 817 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Exception reading %s for %s" % (stampfile2 , stampfile)) |
| 818 | iscurrent = False | 818 | iscurrent = False |
| 819 | 819 | ||
| 820 | return iscurrent | 820 | return iscurrent |
| @@ -885,7 +885,7 @@ class RunQueue: | |||
| 885 | def task_complete(self, task): | 885 | def task_complete(self, task): |
| 886 | """ | 886 | """ |
| 887 | Mark a task as completed | 887 | Mark a task as completed |
| 888 | Look at the reverse dependencies and mark any task with | 888 | Look at the reverse dependencies and mark any task with |
| 889 | completed dependencies as buildable | 889 | completed dependencies as buildable |
| 890 | """ | 890 | """ |
| 891 | self.runq_complete[task] = 1 | 891 | self.runq_complete[task] = 1 |
| @@ -1033,10 +1033,10 @@ class RunQueue: | |||
| 1033 | def finish_runqueue_now(self): | 1033 | def finish_runqueue_now(self): |
| 1034 | bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % self.stats.active) | 1034 | bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % self.stats.active) |
| 1035 | for k, v in self.build_pids.iteritems(): | 1035 | for k, v in self.build_pids.iteritems(): |
| 1036 | try: | 1036 | try: |
| 1037 | os.kill(-k, signal.SIGINT) | 1037 | os.kill(-k, signal.SIGINT) |
| 1038 | except: | 1038 | except: |
| 1039 | pass | 1039 | pass |
| 1040 | for pipe in self.build_pipes: | 1040 | for pipe in self.build_pipes: |
| 1041 | self.build_pipes[pipe].read() | 1041 | self.build_pipes[pipe].read() |
| 1042 | 1042 | ||
| @@ -1085,30 +1085,30 @@ class RunQueue: | |||
| 1085 | """ | 1085 | """ |
| 1086 | bb.msg.debug(3, bb.msg.domain.RunQueue, "run_tasks:") | 1086 | bb.msg.debug(3, bb.msg.domain.RunQueue, "run_tasks:") |
| 1087 | for task in range(len(self.runq_task)): | 1087 | for task in range(len(self.runq_task)): |
| 1088 | bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, | 1088 | bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, |
| 1089 | taskQueue.fn_index[self.runq_fnid[task]], | 1089 | taskQueue.fn_index[self.runq_fnid[task]], |
| 1090 | self.runq_task[task], | 1090 | self.runq_task[task], |
| 1091 | self.runq_weight[task], | 1091 | self.runq_weight[task], |
| 1092 | self.runq_depends[task], | 1092 | self.runq_depends[task], |
| 1093 | self.runq_revdeps[task])) | 1093 | self.runq_revdeps[task])) |
| 1094 | 1094 | ||
| 1095 | bb.msg.debug(3, bb.msg.domain.RunQueue, "sorted_tasks:") | 1095 | bb.msg.debug(3, bb.msg.domain.RunQueue, "sorted_tasks:") |
| 1096 | for task1 in range(len(self.runq_task)): | 1096 | for task1 in range(len(self.runq_task)): |
| 1097 | if task1 in self.prio_map: | 1097 | if task1 in self.prio_map: |
| 1098 | task = self.prio_map[task1] | 1098 | task = self.prio_map[task1] |
| 1099 | bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, | 1099 | bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, |
| 1100 | taskQueue.fn_index[self.runq_fnid[task]], | 1100 | taskQueue.fn_index[self.runq_fnid[task]], |
| 1101 | self.runq_task[task], | 1101 | self.runq_task[task], |
| 1102 | self.runq_weight[task], | 1102 | self.runq_weight[task], |
| 1103 | self.runq_depends[task], | 1103 | self.runq_depends[task], |
| 1104 | self.runq_revdeps[task])) | 1104 | self.runq_revdeps[task])) |
| 1105 | 1105 | ||
| 1106 | 1106 | ||
| 1107 | class TaskFailure(Exception): | 1107 | class TaskFailure(Exception): |
| 1108 | """ | 1108 | """ |
| 1109 | Exception raised when a task in a runqueue fails | 1109 | Exception raised when a task in a runqueue fails |
| 1110 | """ | 1110 | """ |
| 1111 | def __init__(self, x): | 1111 | def __init__(self, x): |
| 1112 | self.args = x | 1112 | self.args = x |
| 1113 | 1113 | ||
| 1114 | 1114 | ||
| @@ -1196,4 +1196,3 @@ class runQueuePipe(): | |||
| 1196 | if len(self.queue) > 0: | 1196 | if len(self.queue) > 0: |
| 1197 | print "Warning, worker left partial message" | 1197 | print "Warning, worker left partial message" |
| 1198 | os.close(self.fd) | 1198 | os.close(self.fd) |
| 1199 | |||
