1 files changed, 219 insertions, 37 deletions

diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index 5fdfeee2c7..ddf98e6a2e 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -28,7 +28,7 @@ BitBake build tools.
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import copy, re
+import copy, re, sys, traceback
 from collections import MutableMapping
 import logging
 import hashlib
@@ -43,6 +43,42 @@ __setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend)(_(?P<
 __expand_var_regexp__ = re.compile(r"\${[^{}]+}")
 __expand_python_regexp__ = re.compile(r"\${@.+?}")
 
+def infer_caller_details(loginfo, parent = False, varval = True):
+    """Save the caller the trouble of specifying everything."""
+    # Save effort.
+    if 'ignore' in loginfo and loginfo['ignore']:
+        return
+    # If nothing was provided, mark this as possibly unneeded.
+    if not loginfo:
+        loginfo['ignore'] = True
+        return
+    # Infer caller's likely values for variable (var) and value (value), 
+    # to reduce clutter in the rest of the code.
+    if varval and ('variable' not in loginfo or 'detail' not in loginfo):
+        try:
+            raise Exception
+        except Exception:
+            tb = sys.exc_info()[2]
+            if parent:
+                above = tb.tb_frame.f_back.f_back
+            else:
+                above = tb.tb_frame.f_back
+            lcls = above.f_locals.items()
+        for k, v in lcls:
+            if k == 'value' and 'detail' not in loginfo:
+                loginfo['detail'] = v
+            if k == 'var' and 'variable' not in loginfo:
+                loginfo['variable'] = v
+    # Infer file/line/function from traceback
+    if 'file' not in loginfo:
+        depth = 3    
+        if parent:
+            depth = 4
+        file, line, func, text = traceback.extract_stack(limit = depth)[0]
+        loginfo['file'] = file
+        loginfo['line'] = line
+        if func not in loginfo:
+            loginfo['func'] = func
 
 class VariableParse:
     def __init__(self, varname, d, val = None):
@@ -157,11 +193,80 @@ class IncludeHistory(object):
             o.write("\n")
             child.emit(o, level)
 
+class VariableHistory(object):
+    def __init__(self, dataroot):
+        self.dataroot = dataroot
+        self.variables = COWDictBase.copy()
+
+    def copy(self):
+        new = VariableHistory(self.dataroot)
+        new.variables = self.variables.copy()
+        return new
+
+    def record(self, *kwonly, **loginfo):
+        if not self.dataroot._tracking:
+            return
+        if len(kwonly) > 0:
+            raise TypeError
+        infer_caller_details(loginfo, parent = True)
+        if 'ignore' in loginfo and loginfo['ignore']:
+            return
+        if 'op' not in loginfo or not loginfo['op']:
+            loginfo['op'] = 'set'
+        if 'detail' in loginfo:
+            loginfo['detail'] = str(loginfo['detail'])
+        if 'variable' not in loginfo or 'file' not in loginfo:
+            raise ValueError("record() missing variable or file.")
+        var = loginfo['variable']
+
+        if var not in self.variables:
+            self.variables[var] = []
+        self.variables[var].append(loginfo.copy())
+
+    def variable(self, var):
+        if var in self.variables:
+            return self.variables[var]
+        else:
+            return []
+
+    def emit(self, var, oval, val, o):
+        history = self.variable(var)
+        commentVal = re.sub('\n', '\n#', str(oval))
+        if history:
+            if len(history) == 1:
+                o.write("#\n# $%s\n" % var)
+            else:
+                o.write("#\n# $%s [%d operations]\n" % (var, len(history)))
+            for event in history:
+                # o.write("# %s\n" % str(event))
+                if 'func' in event:
+                    # If we have a function listed, this is internal
+                    # code, not an operation in a config file, and the
+                    # full path is distracting.
+                    event['file'] = re.sub('.*/', '', event['file'])
+                    display_func = ' [%s]' % event['func']
+                else:
+                    display_func = ''
+                if 'flag' in event:
+                    flag = '[%s] ' % (event['flag'])
+                else:
+                    flag = ''
+                o.write("#   %s %s:%s%s\n#     %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n#     ', event['detail'])))
+            if len(history) > 1:
+                o.write("# computed:\n")
+                o.write('#   "%s"\n' % (commentVal))
+        else:
+            o.write("#\n# $%s\n#   [no history recorded]\n#\n" % var)
+            o.write('#   "%s"\n' % (commentVal))
+
+
 class DataSmart(MutableMapping):
     def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ):
         self.dict = {}
 
         self.inchistory = IncludeHistory()
+        self.varhistory = VariableHistory(self)
+        self._tracking = False
 
         # cookie monster tribute
         self._special_values = special
@@ -169,6 +274,12 @@ class DataSmart(MutableMapping):
 
         self.expand_cache = {}
 
+    def enableTracking(self):
+        self._tracking = True
+
+    def disableTracking(self):
+        self._tracking = False
+
     def expandWithRefs(self, s, varname):
 
         if not isinstance(s, basestring): # sanity check
@@ -204,10 +315,14 @@ class DataSmart(MutableMapping):
         return self.expandWithRefs(s, varname).value
 
 
-    def finalize(self):
+    def finalize(self, parent = False):
         """Performs final steps upon the datastore, including application of overrides"""
 
         overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+        finalize_caller = {
+            'op': 'finalize',
+        }
+        infer_caller_details(finalize_caller, parent = parent, varval = False)
 
         #
         # Well let us see what breaks here. We used to iterate
@@ -224,6 +339,9 @@ class DataSmart(MutableMapping):
         # Then  we will handle _append and _prepend
         #
 
+        # We only want to report finalization once per variable overridden.
+        finalizes_reported = {}
+
         for o in overrides:
             # calculate '_'+override
             l = len(o) + 1
@@ -236,7 +354,19 @@ class DataSmart(MutableMapping):
             for var in vars:
                 name = var[:-l]
                 try:
-                    self.setVar(name, self.getVar(var, False))
+                    # Report only once, even if multiple changes.
+                    if name not in finalizes_reported:
+                        finalizes_reported[name] = True
+                        finalize_caller['variable'] = name
+                        finalize_caller['detail'] = 'was: ' + str(self.getVar(name, False))
+                        self.varhistory.record(**finalize_caller)
+                    # Copy history of the override over.
+                    for event in self.varhistory.variable(var):
+                        loginfo = event.copy()
+                        loginfo['variable'] = name
+                        loginfo['op'] = 'override[%s]:%s' % (o, loginfo['op'])
+                        self.varhistory.record(**loginfo)
+                    self.setVar(name, self.getVar(var, False), op = 'finalize', file = 'override[%s]' % o, line = '')
                     self.delVar(var)
                 except Exception:
                     logger.info("Untracked delVar")
@@ -267,9 +397,9 @@ class DataSmart(MutableMapping):
 
                     # We save overrides that may be applied at some later stage
                     if keep:
-                        self.setVarFlag(append, op, keep)
+                        self.setVarFlag(append, op, keep, ignore=True)
                     else:
-                        self.delVarFlag(append, op)
+                        self.delVarFlag(append, op, ignore=True)
 
     def initVar(self, var):
         self.expand_cache = {}
@@ -297,7 +427,10 @@ class DataSmart(MutableMapping):
         else:
             self.initVar(var)
 
-    def setVar(self, var, value):
+
+    def setVar(self, var, value, **loginfo):
+        if 'op' not in loginfo:
+            loginfo['op'] = "set"
         self.expand_cache = {}
         match  = __setvar_regexp__.match(var)
         if match and match.group("keyword") in __setvar_keyword__:
@@ -306,15 +439,22 @@ class DataSmart(MutableMapping):
             override = match.group('add')
             l = self.getVarFlag(base, keyword) or []
             l.append([value, override])
-            self.setVarFlag(base, keyword, l)
-
+            self.setVarFlag(base, keyword, l, ignore=True)
+            # And cause that to be recorded:
+            loginfo['detail'] = value
+            loginfo['variable'] = base
+            if override:
+                loginfo['op'] = '%s[%s]' % (keyword, override)
+            else:
+                loginfo['op'] = keyword
+            self.varhistory.record(**loginfo)
             # todo make sure keyword is not __doc__ or __module__
             # pay the cookie monster
             try:
-                self._special_values[keyword].add( base )
+                self._special_values[keyword].add(base)
             except KeyError:
                 self._special_values[keyword] = set()
-                self._special_values[keyword].add( base )
+                self._special_values[keyword].add(base)
 
             return
 
@@ -331,6 +471,7 @@ class DataSmart(MutableMapping):
 
         # setting var
         self.dict[var]["_content"] = value
+        self.varhistory.record(**loginfo)
 
     def getVar(self, var, expand=False, noweakdefault=False):
         value = self.getVarFlag(var, "_content", False, noweakdefault)
@@ -340,13 +481,17 @@ class DataSmart(MutableMapping):
             return self.expand(value, var)
         return value
 
-    def renameVar(self, key, newkey):
+    def renameVar(self, key, newkey, **loginfo):
         """
         Rename the variable key to newkey
         """
         val = self.getVar(key, 0)
         if val is not None:
-            self.setVar(newkey, val)
+            loginfo['variable'] = newkey
+            loginfo['op'] = 'rename from %s' % key
+            loginfo['detail'] = val
+            self.varhistory.record(**loginfo)
+            self.setVar(newkey, val, ignore=True)
 
         for i in ('_append', '_prepend'):
             src = self.getVarFlag(key, i)
@@ -355,23 +500,34 @@ class DataSmart(MutableMapping):
 
             dest = self.getVarFlag(newkey, i) or []
             dest.extend(src)
-            self.setVarFlag(newkey, i, dest)
+            self.setVarFlag(newkey, i, dest, ignore=True)
 
             if i in self._special_values and key in self._special_values[i]:
                 self._special_values[i].remove(key)
                 self._special_values[i].add(newkey)
 
-        self.delVar(key)
-
-    def appendVar(self, key, value):
-        value = (self.getVar(key, False) or "") + value
-        self.setVar(key, value)
-
-    def prependVar(self, key, value):
-        value = value + (self.getVar(key, False) or "")
-        self.setVar(key, value)
-
-    def delVar(self, var):
+        loginfo['variable'] = key
+        loginfo['op'] = 'rename (to)'
+        loginfo['detail'] = newkey
+        self.varhistory.record(**loginfo)
+        self.delVar(key, ignore=True)
+
+    def appendVar(self, var, value, **loginfo):
+        loginfo['op'] = 'append'
+        self.varhistory.record(**loginfo)
+        newvalue = (self.getVar(var, False) or "") + value
+        self.setVar(var, newvalue, ignore=True)
+
+    def prependVar(self, var, value, **loginfo):
+        loginfo['op'] = 'prepend'
+        self.varhistory.record(**loginfo)
+        newvalue = value + (self.getVar(var, False) or "")
+        self.setVar(var, newvalue, ignore=True)
+
+    def delVar(self, var, **loginfo):
+        loginfo['detail'] = ""
+        loginfo['op'] = 'del'
+        self.varhistory.record(**loginfo)
         self.expand_cache = {}
         self.dict[var] = {}
         if '_' in var:
@@ -379,10 +535,14 @@ class DataSmart(MutableMapping):
             if override and override in self._seen_overrides and var in self._seen_overrides[override]:
                 self._seen_overrides[override].remove(var)
 
-    def setVarFlag(self, var, flag, flagvalue):
+    def setVarFlag(self, var, flag, value, **loginfo):
+        if 'op' not in loginfo:
+            loginfo['op'] = "set"
+        loginfo['flag'] = flag
+        self.varhistory.record(**loginfo)
         if not var in self.dict:
             self._makeShadowCopy(var)
-        self.dict[var][flag] = flagvalue
+        self.dict[var][flag] = value
 
     def getVarFlag(self, var, flag, expand=False, noweakdefault=False):
         local_var = self._findVar(var)
@@ -396,7 +556,7 @@ class DataSmart(MutableMapping):
             value = self.expand(value, None)
         return value
 
-    def delVarFlag(self, var, flag):
+    def delVarFlag(self, var, flag, **loginfo):
         local_var = self._findVar(var)
         if not local_var:
             return
@@ -404,23 +564,38 @@ class DataSmart(MutableMapping):
             self._makeShadowCopy(var)
 
         if var in self.dict and flag in self.dict[var]:
-            del self.dict[var][flag]
-
-    def appendVarFlag(self, key, flag, value):
-        value = (self.getVarFlag(key, flag, False) or "") + value
-        self.setVarFlag(key, flag, value)
+            loginfo['detail'] = ""
+            loginfo['op'] = 'delFlag'
+            loginfo['flag'] = flag
+            self.varhistory.record(**loginfo)
 
-    def prependVarFlag(self, key, flag, value):
-        value = value + (self.getVarFlag(key, flag, False) or "")
-        self.setVarFlag(key, flag, value)
+            del self.dict[var][flag]
 
-    def setVarFlags(self, var, flags):
+    def appendVarFlag(self, var, flag, value, **loginfo):
+        loginfo['op'] = 'append'
+        loginfo['flag'] = flag
+        self.varhistory.record(**loginfo)
+        newvalue = (self.getVarFlag(var, flag, False) or "") + value
+        self.setVarFlag(var, flag, newvalue, ignore=True)
+
+    def prependVarFlag(self, var, flag, value, **loginfo):
+        loginfo['op'] = 'prepend'
+        loginfo['flag'] = flag
+        self.varhistory.record(**loginfo)
+        newvalue = value + (self.getVarFlag(var, flag, False) or "")
+        self.setVarFlag(var, flag, newvalue, ignore=True)
+
+    def setVarFlags(self, var, flags, **loginfo):
+        infer_caller_details(loginfo)
         if not var in self.dict:
             self._makeShadowCopy(var)
 
         for i in flags:
             if i == "_content":
                 continue
+            loginfo['flag'] = i
+            loginfo['detail'] = flags[i]
+            self.varhistory.record(**loginfo)
             self.dict[var][i] = flags[i]
 
     def getVarFlags(self, var):
@@ -438,13 +613,16 @@ class DataSmart(MutableMapping):
         return flags
 
 
-    def delVarFlags(self, var):
+    def delVarFlags(self, var, **loginfo):
         if not var in self.dict:
             self._makeShadowCopy(var)
 
         if var in self.dict:
             content = None
 
+            loginfo['op'] = 'delete flags'
+            self.varhistory.record(**loginfo)
+
             # try to save the content
             if "_content" in self.dict[var]:
                 content  = self.dict[var]["_content"]
@@ -461,8 +639,12 @@ class DataSmart(MutableMapping):
         # we really want this to be a DataSmart...
         data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy())
         data.dict["_data"] = self.dict
+        data.varhistory = self.varhistory.copy()
+        data.varhistory.datasmart = data
         data.inchistory = self.inchistory.copy()
 
+        data._tracking = self._tracking
+
         return data
 
     def expandVarref(self, variable, parents=False):