diff options
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
| -rw-r--r-- | bitbake/lib/bb/runqueue.py | 130 |
1 files changed, 94 insertions, 36 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 2765343a3e..7b3defd343 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
| @@ -317,6 +317,7 @@ class RunQueue: | |||
| 317 | 317 | ||
| 318 | depends = [] | 318 | depends = [] |
| 319 | runq_build = [] | 319 | runq_build = [] |
| 320 | recursive_tdepends = {} | ||
| 320 | 321 | ||
| 321 | taskData = self.taskData | 322 | taskData = self.taskData |
| 322 | 323 | ||
| @@ -382,14 +383,45 @@ class RunQueue: | |||
| 382 | # e.g. do_sometask[depends] = "targetname:do_someothertask" | 383 | # e.g. do_sometask[depends] = "targetname:do_someothertask" |
| 383 | # (makes sure sometask runs after targetname's someothertask) | 384 | # (makes sure sometask runs after targetname's someothertask) |
| 384 | idepends = taskData.tasks_idepends[task] | 385 | idepends = taskData.tasks_idepends[task] |
| 385 | for idepend in idepends: | 386 | for (depid, idependtask) in idepends: |
| 386 | depid = int(idepend.split(":")[0]) | ||
| 387 | if depid in taskData.build_targets: | 387 | if depid in taskData.build_targets: |
| 388 | # Won't be in build_targets if ASSUME_PROVIDED | 388 | # Won't be in build_targets if ASSUME_PROVIDED |
| 389 | depdata = taskData.build_targets[depid][0] | 389 | depdata = taskData.build_targets[depid][0] |
| 390 | if depdata is not None: | 390 | if depdata is not None: |
| 391 | dep = taskData.fn_index[depdata] | 391 | dep = taskData.fn_index[depdata] |
| 392 | depends.append(taskData.gettask_id(dep, idepend.split(":")[1])) | 392 | depends.append(taskData.gettask_id(dep, idependtask)) |
| 393 | |||
| 394 | # Create a list of recursive dependent tasks (from tdepends) and cache | ||
| 395 | def get_recursive_tdepends(task): | ||
| 396 | if not task: | ||
| 397 | return [] | ||
| 398 | if task in recursive_tdepends: | ||
| 399 | return recursive_tdepends[task] | ||
| 400 | rectdepends = [task] | ||
| 401 | nextdeps = [task] | ||
| 402 | while len(nextdeps) != 0: | ||
| 403 | newdeps = [] | ||
| 404 | for nextdep in nextdeps: | ||
| 405 | for tdepend in taskData.tasks_tdepends[nextdep]: | ||
| 406 | if tdepend not in rectdepends: | ||
| 407 | rectdepends.append(tdepend) | ||
| 408 | newdeps.append(tdepend) | ||
| 409 | nextdeps = newdeps | ||
| 410 | recursive_tdepends[task] = rectdepends | ||
| 411 | return rectdepends | ||
| 412 | |||
| 413 | # Using the list of tdepends for this task create a list of | ||
| 414 | # the recursive idepends we have | ||
| 415 | def get_recursive_idepends(task): | ||
| 416 | if not task: | ||
| 417 | return [] | ||
| 418 | rectdepends = get_recursive_tdepends(task) | ||
| 419 | |||
| 420 | recidepends = [] | ||
| 421 | for tdepend in rectdepends: | ||
| 422 | for idepend in taskData.tasks_idepends[tdepend]: | ||
| 423 | recidepends.append(idepend) | ||
| 424 | return recidepends | ||
| 393 | 425 | ||
| 394 | def add_recursive_build(depid, depfnid): | 426 | def add_recursive_build(depid, depfnid): |
| 395 | """ | 427 | """ |
| @@ -404,13 +436,11 @@ class RunQueue: | |||
| 404 | depdata = taskData.build_targets[depid][0] | 436 | depdata = taskData.build_targets[depid][0] |
| 405 | if depdata is not None: | 437 | if depdata is not None: |
| 406 | dep = taskData.fn_index[depdata] | 438 | dep = taskData.fn_index[depdata] |
| 407 | idepends = [] | ||
| 408 | # Need to avoid creating new tasks here | 439 | # Need to avoid creating new tasks here |
| 409 | taskid = taskData.gettask_id(dep, taskname, False) | 440 | taskid = taskData.gettask_id(dep, taskname, False) |
| 410 | if taskid is not None: | 441 | if taskid is not None: |
| 411 | depends.append(taskid) | 442 | depends.append(taskid) |
| 412 | fnid = taskData.tasks_fnid[taskid] | 443 | fnid = taskData.tasks_fnid[taskid] |
| 413 | idepends = taskData.tasks_idepends[taskid] | ||
| 414 | #print "Added %s (%s) due to %s" % (taskid, taskData.fn_index[fnid], taskData.fn_index[depfnid]) | 444 | #print "Added %s (%s) due to %s" % (taskid, taskData.fn_index[fnid], taskData.fn_index[depfnid]) |
| 415 | else: | 445 | else: |
| 416 | fnid = taskData.getfn_id(dep) | 446 | fnid = taskData.getfn_id(dep) |
| @@ -420,10 +450,9 @@ class RunQueue: | |||
| 420 | for nextdepid in taskData.rdepids[fnid]: | 450 | for nextdepid in taskData.rdepids[fnid]: |
| 421 | if nextdepid not in rdep_seen: | 451 | if nextdepid not in rdep_seen: |
| 422 | add_recursive_run(nextdepid, fnid) | 452 | add_recursive_run(nextdepid, fnid) |
| 423 | for idepend in idepends: | 453 | for (idependid, idependtask) in get_recursive_idepends(taskid): |
| 424 | nextdepid = int(idepend.split(":")[0]) | 454 | if idependid not in dep_seen: |
| 425 | if nextdepid not in dep_seen: | 455 | add_recursive_build(idependid, fnid) |
| 426 | add_recursive_build(nextdepid, fnid) | ||
| 427 | 456 | ||
| 428 | def add_recursive_run(rdepid, depfnid): | 457 | def add_recursive_run(rdepid, depfnid): |
| 429 | """ | 458 | """ |
| @@ -438,13 +467,11 @@ class RunQueue: | |||
| 438 | depdata = taskData.run_targets[rdepid][0] | 467 | depdata = taskData.run_targets[rdepid][0] |
| 439 | if depdata is not None: | 468 | if depdata is not None: |
| 440 | dep = taskData.fn_index[depdata] | 469 | dep = taskData.fn_index[depdata] |
| 441 | idepends = [] | ||
| 442 | # Need to avoid creating new tasks here | 470 | # Need to avoid creating new tasks here |
| 443 | taskid = taskData.gettask_id(dep, taskname, False) | 471 | taskid = taskData.gettask_id(dep, taskname, False) |
| 444 | if taskid is not None: | 472 | if taskid is not None: |
| 445 | depends.append(taskid) | 473 | depends.append(taskid) |
| 446 | fnid = taskData.tasks_fnid[taskid] | 474 | fnid = taskData.tasks_fnid[taskid] |
| 447 | idepends = taskData.tasks_idepends[taskid] | ||
| 448 | #print "Added %s (%s) due to %s" % (taskid, taskData.fn_index[fnid], taskData.fn_index[depfnid]) | 475 | #print "Added %s (%s) due to %s" % (taskid, taskData.fn_index[fnid], taskData.fn_index[depfnid]) |
| 449 | else: | 476 | else: |
| 450 | fnid = taskData.getfn_id(dep) | 477 | fnid = taskData.getfn_id(dep) |
| @@ -454,10 +481,9 @@ class RunQueue: | |||
| 454 | for nextdepid in taskData.rdepids[fnid]: | 481 | for nextdepid in taskData.rdepids[fnid]: |
| 455 | if nextdepid not in rdep_seen: | 482 | if nextdepid not in rdep_seen: |
| 456 | add_recursive_run(nextdepid, fnid) | 483 | add_recursive_run(nextdepid, fnid) |
| 457 | for idepend in idepends: | 484 | for (idependid, idependtask) in get_recursive_idepends(taskid): |
| 458 | nextdepid = int(idepend.split(":")[0]) | 485 | if idependid not in dep_seen: |
| 459 | if nextdepid not in dep_seen: | 486 | add_recursive_build(idependid, fnid) |
| 460 | add_recursive_build(nextdepid, fnid) | ||
| 461 | 487 | ||
| 462 | # Resolve recursive 'recrdeptask' dependencies | 488 | # Resolve recursive 'recrdeptask' dependencies |
| 463 | # | 489 | # |
| @@ -472,9 +498,9 @@ class RunQueue: | |||
| 472 | add_recursive_build(depid, fnid) | 498 | add_recursive_build(depid, fnid) |
| 473 | for rdepid in taskData.rdepids[fnid]: | 499 | for rdepid in taskData.rdepids[fnid]: |
| 474 | add_recursive_run(rdepid, fnid) | 500 | add_recursive_run(rdepid, fnid) |
| 475 | for idepend in idepends: | 501 | deptaskid = taskData.gettask_id(fn, taskname, False) |
| 476 | depid = int(idepend.split(":")[0]) | 502 | for (idependid, idependtask) in get_recursive_idepends(deptaskid): |
| 477 | add_recursive_build(depid, fnid) | 503 | add_recursive_build(idependid, fnid) |
| 478 | 504 | ||
| 479 | # Rmove all self references | 505 | # Rmove all self references |
| 480 | if task in depends: | 506 | if task in depends: |
| @@ -659,6 +685,16 @@ class RunQueue: | |||
| 659 | if len(self.runq_depends[task]) == 0: | 685 | if len(self.runq_depends[task]) == 0: |
| 660 | buildable.append(task) | 686 | buildable.append(task) |
| 661 | 687 | ||
| 688 | def check_buildable(self, task, buildable): | ||
| 689 | for revdep in self.runq_revdeps[task]: | ||
| 690 | alldeps = 1 | ||
| 691 | for dep in self.runq_depends[revdep]: | ||
| 692 | if dep in unchecked: | ||
| 693 | alldeps = 0 | ||
| 694 | if alldeps == 1: | ||
| 695 | if revdep in unchecked: | ||
| 696 | buildable.append(revdep) | ||
| 697 | |||
| 662 | for task in range(len(self.runq_fnid)): | 698 | for task in range(len(self.runq_fnid)): |
| 663 | if task not in unchecked: | 699 | if task not in unchecked: |
| 664 | continue | 700 | continue |
| @@ -669,12 +705,14 @@ class RunQueue: | |||
| 669 | if not os.access(stampfile, os.F_OK): | 705 | if not os.access(stampfile, os.F_OK): |
| 670 | del unchecked[task] | 706 | del unchecked[task] |
| 671 | notcurrent.append(task) | 707 | notcurrent.append(task) |
| 708 | check_buildable(self, task, buildable) | ||
| 672 | continue | 709 | continue |
| 673 | # If its a 'nostamp' task, it's not current | 710 | # If its a 'nostamp' task, it's not current |
| 674 | taskdep = self.dataCache.task_deps[fn] | 711 | taskdep = self.dataCache.task_deps[fn] |
| 675 | if 'nostamp' in taskdep and task in taskdep['nostamp']: | 712 | if 'nostamp' in taskdep and task in taskdep['nostamp']: |
| 676 | del unchecked[task] | 713 | del unchecked[task] |
| 677 | notcurrent.append(task) | 714 | notcurrent.append(task) |
| 715 | check_buildable(self, task, buildable) | ||
| 678 | continue | 716 | continue |
| 679 | 717 | ||
| 680 | while (len(buildable) > 0): | 718 | while (len(buildable) > 0): |
| @@ -705,14 +743,7 @@ class RunQueue: | |||
| 705 | else: | 743 | else: |
| 706 | notcurrent.append(task) | 744 | notcurrent.append(task) |
| 707 | 745 | ||
| 708 | for revdep in self.runq_revdeps[task]: | 746 | check_buildable(self, task, nextbuildable) |
| 709 | alldeps = 1 | ||
| 710 | for dep in self.runq_depends[revdep]: | ||
| 711 | if dep in unchecked: | ||
| 712 | alldeps = 0 | ||
| 713 | if alldeps == 1: | ||
| 714 | if revdep in unchecked: | ||
| 715 | nextbuildable.append(revdep) | ||
| 716 | 747 | ||
| 717 | buildable = nextbuildable | 748 | buildable = nextbuildable |
| 718 | 749 | ||
| @@ -729,6 +760,40 @@ class RunQueue: | |||
| 729 | bb.fatal("check_stamps fatal internal error") | 760 | bb.fatal("check_stamps fatal internal error") |
| 730 | return current | 761 | return current |
| 731 | 762 | ||
| 763 | def check_stamp(self, task): | ||
| 764 | |||
| 765 | if self.stamppolicy == "perfile": | ||
| 766 | fulldeptree = False | ||
| 767 | else: | ||
| 768 | fulldeptree = True | ||
| 769 | |||
| 770 | fn = self.taskData.fn_index[self.runq_fnid[task]] | ||
| 771 | taskname = self.runq_task[task] | ||
| 772 | stampfile = "%s.%s" % (self.dataCache.stamp[fn], taskname) | ||
| 773 | # If the stamp is missing its not current | ||
| 774 | if not os.access(stampfile, os.F_OK): | ||
| 775 | return False | ||
| 776 | # If its a 'nostamp' task, it's not current | ||
| 777 | taskdep = self.dataCache.task_deps[fn] | ||
| 778 | if 'nostamp' in taskdep and task in taskdep['nostamp']: | ||
| 779 | return False | ||
| 780 | |||
| 781 | iscurrent = True | ||
| 782 | t1 = os.stat(stampfile)[stat.ST_MTIME] | ||
| 783 | for dep in self.runq_depends[task]: | ||
| 784 | if iscurrent: | ||
| 785 | fn2 = self.taskData.fn_index[self.runq_fnid[dep]] | ||
| 786 | taskname2 = self.runq_task[dep] | ||
| 787 | stampfile2 = "%s.%s" % (self.dataCache.stamp[fn2], taskname2) | ||
| 788 | if fulldeptree or fn == fn2: | ||
| 789 | try: | ||
| 790 | t2 = os.stat(stampfile2)[stat.ST_MTIME] | ||
| 791 | if t1 < t2: | ||
| 792 | iscurrent = False | ||
| 793 | except: | ||
| 794 | iscurrent = False | ||
| 795 | |||
| 796 | return iscurrent | ||
| 732 | 797 | ||
| 733 | def execute_runqueue(self): | 798 | def execute_runqueue(self): |
| 734 | """ | 799 | """ |
| @@ -817,25 +882,18 @@ class RunQueue: | |||
| 817 | 882 | ||
| 818 | event.fire(bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp, self.cfgdata)) | 883 | event.fire(bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp, self.cfgdata)) |
| 819 | 884 | ||
| 820 | # Find out which tasks have current stamps which we can skip when the | ||
| 821 | # time comes | ||
| 822 | currentstamps = self.check_stamps() | ||
| 823 | self.stats.taskSkipped(len(currentstamps)) | ||
| 824 | self.stats.taskCompleted(len(currentstamps)) | ||
| 825 | |||
| 826 | while True: | 885 | while True: |
| 827 | task = self.sched.next() | 886 | task = self.sched.next() |
| 828 | if task is not None: | 887 | if task is not None: |
| 829 | fn = self.taskData.fn_index[self.runq_fnid[task]] | 888 | fn = self.taskData.fn_index[self.runq_fnid[task]] |
| 830 | 889 | ||
| 831 | taskname = self.runq_task[task] | 890 | taskname = self.runq_task[task] |
| 832 | if task in currentstamps: | 891 | if self.check_stamp(task): |
| 833 | #if bb.build.stamp_is_current(taskname, self.dataCache, fn): | ||
| 834 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task))) | 892 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task))) |
| 835 | self.runq_running[task] = 1 | 893 | self.runq_running[task] = 1 |
| 836 | self.task_complete(task) | 894 | self.task_complete(task) |
| 837 | #self.stats.taskCompleted() | 895 | self.stats.taskCompleted() |
| 838 | #self.stats.taskSkipped() | 896 | self.stats.taskSkipped() |
| 839 | continue | 897 | continue |
| 840 | 898 | ||
| 841 | bb.msg.note(1, bb.msg.domain.RunQueue, "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.active_builds + 1, len(self.runq_fnid), task, self.get_user_idstring(task))) | 899 | bb.msg.note(1, bb.msg.domain.RunQueue, "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.active_builds + 1, len(self.runq_fnid), task, self.get_user_idstring(task))) |
