[bitbake-devel] [PATCH] data_smart: Add CoW approach for overridedata

[Richard Purdie]

Using deepcopy() caused a major performance regression. Turns
out we can work with a shallow copy as long as we force the
recreation of any list that we change, effectively a poor mans
CoW. This isn't too invasive and avoids the huge overhead of
deepcopy so this seems like the best way to have performance and
a working data store.

Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>

diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index 86cdeb5..c0a2a57 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -314,6 +314,9 @@ class DataSmart(MutableMapping):
         self.expand_cache = {}
 
         # cookie monster tribute
+        # Need to be careful about writes to overridedata as
+        # its only a shallow copy, could influence other data store
+        # copies!
         self.overridedata = {}
         self.overrides = None
         self.overridevars = set(["OVERRIDES", "FILE"])
@@ -507,6 +510,8 @@ class DataSmart(MutableMapping):
             if shortvar not in self.overridedata:
                 self.overridedata[shortvar] = []
             if [var, override] not in self.overridedata[shortvar]:
+                # Force CoW by recreating the list first
+                self.overridedata[shortvar] = list(self.overridedata[shortvar])
                 self.overridedata[shortvar].append([var, override])
             for event in self.varhistory.variable(var):
                 if 'flag' in loginfo and not loginfo['flag'].startswith("_"):
@@ -587,6 +592,8 @@ class DataSmart(MutableMapping):
             while override:
                 try:
                     if shortvar in self.overridedata:
+                        # Force CoW by recreating the list first
+                        self.overridedata[shortvar] = list(self.overridedata[shortvar])
                         self.overridedata[shortvar].remove([var, override])
                 except ValueError as e:
                     pass
@@ -806,7 +813,9 @@ class DataSmart(MutableMapping):
 
         data.overrides = None
         data.overridevars = copy.copy(self.overridevars)
-        data.overridedata = copy.deepcopy(self.overridedata)
+        # Should really be a deepcopy but has heavy overhead.
+        # Instead, we're careful with writes.
+        data.overridedata = copy.copy(self.overridedata)
 
         return data