bitbake/sstate: Implement a lookup function to speed up setscene processing

Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>

4 files changed, 55 insertions, 2 deletions

diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 646fdecbeb..c6f3794d5e 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -453,6 +453,8 @@ class Cache:
         if not self.getVar('BROKEN', file_name, True) and not self.getVar('EXCLUDE_FROM_WORLD', file_name, True):
             cacheData.possible_world.append(file_name)
 
+        cacheData.hashfn[file_name] = self.getVar('BB_HASHFILENAME', file_name, True)
+
         # Touch this to make sure its in the cache
         self.getVar('__BB_DONT_CACHE', file_name, True)
         self.getVar('__VARIANTS', file_name, True)
@@ -545,6 +547,7 @@ class CacheData:
         self.preferred = {}
         self.tasks = {}
         self.basetaskhash = {}
+        self.hashfn = {}
 
         """
         Indirect Cache variables
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index dff4ff7f3e..b5167126ee 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -757,7 +757,8 @@ class RunQueue:
         self.cfgData = cfgData
         self.rqdata = RunQueueData(self, cooker, cfgData, dataCache, taskData, targets)
 
-        self.stamppolicy = bb.data.getVar("BB_STAMP_POLICY", cfgData, 1) or "perfile"
+        self.stamppolicy = bb.data.getVar("BB_STAMP_POLICY", cfgData, True) or "perfile"
+        self.hashvalidate = bb.data.getVar("BB_HASHCHECK_FUNCTION", cfgData, True) or None
 
         self.state = runQueuePrepare
 
@@ -1326,6 +1327,29 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             if len(self.sq_revdeps[task]) == 0:
                 self.runq_buildable[task] = 1
 
+        if self.rq.hashvalidate:
+            sq_hash = []
+            sq_hashfn = []
+            sq_fn = []
+            sq_task = []
+            for task in range(len(self.sq_revdeps)):
+                realtask = self.rqdata.runq_setscene[task]
+                fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]]
+                sq_fn.append(fn)
+                sq_hashfn.append(self.rqdata.dataCache.hashfn[fn])
+                sq_hash.append(self.rqdata.runq_hash[realtask])
+                sq_task.append(self.rqdata.runq_task[realtask])
+
+            call = self.rq.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)"
+            locs = { "sq_fn" : sq_fn, "sq_task" : sq_task, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.configuration.data }
+            valid = bb.utils.better_eval(call, locs)
+            for task in range(len(self.sq_revdeps)):
+                if task not in valid:
+                    bb.msg.debug(2, bb.msg.domain.RunQueue, "No package found so skipping setscene task %s" % (self.rqdata.get_user_idstring(task)))
+                    self.task_failoutright(task)
+
+            #print(str(valid))
+
         bb.msg.note(1, bb.msg.domain.RunQueue, "Executing setscene Tasks")
 
         self.rq.state = runQueueSceneRun
diff --git a/meta/classes/deploy.bbclass b/meta/classes/deploy.bbclass
index f697e70aa4..c3371421d8 100644
--- a/meta/classes/deploy.bbclass
+++ b/meta/classes/deploy.bbclass
@@ -1,6 +1,6 @@
 DEPLOYDIR = "${WORKDIR}/deploy-${PN}"
 SSTATETASKS += "do_deploy"
-do_deploy[sstate-name] = "deploy-${PN}"
+do_deploy[sstate-name] = "deploy"
 do_deploy[sstate-inputdirs] = "${DEPLOYDIR}"
 do_deploy[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
 
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 855f5a45a7..7f897ae54d 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -9,6 +9,8 @@ SSTATE_PKG        = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
 
 SSTATE_SCAN_CMD ?= "find ${SSTATE_BUILDDIR} \( -name "*.la" -o -name "*-config" \) -type f"
 
+BB_HASHFILENAME = "${SSTATE_PKGNAME}"
+
 python () {
     if bb.data.inherits_class('native', d):
         bb.data.setVar('SSTATE_PKGARCH', bb.data.getVar('BUILD_ARCH', d), d)
@@ -313,3 +315,27 @@ sstate_unpack_package () {
         cd ${SSTATE_INSTDIR}
         tar -xvzf ${SSTATE_PKG}
 }
+
+BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
+
+def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d):
+    ret = []
+    # This needs to go away, FIXME
+    mapping = {
+        "do_populate_sysroot" : "populate-sysroot",
+        "do_package_write_ipk" : "deploy-ipk",
+        "do_package_write_deb" : "deploy-deb",
+        "do_package_write_rpm" : "deploy-rpm",
+        "do_package" : "package",
+        "do_deploy" : "deploy",
+    }
+
+    for task in range(len(sq_fn)):
+        sstatefile = bb.data.expand("${SSTATE_DIR}/" + sq_hashfn[task] + "_" + mapping[sq_task[task]] + ".tgz", d)
+        sstatefile= sstatefile.replace("${BB_TASKHASH}", sq_hash[task])
+        #print("Checking for %s" % sstatefile)
+        if os.path.exists(sstatefile):
+            ret.append(task)
+
+    return ret
+