[bitbake-devel] [PATCH 8/8] bitbake: cooker: improve inotify handling

[Robert Yang]

Each recipe calls bb_cache.add_info() and it calls add_filewatch() for every
depend file (not only existed files, but also non-exsited/attempted ones),
which caused the parsing and loading very slow. The watchers are global, so we
can only pass new depend files to add_filewatch() to improve the performance.

Here is the testing of 54 layers before and after the patch.

              Parsing       Loading
Before        35s           33s
Now           11s           9s

24 seconds are saved for parsing, and 24s are saved for loading.

Signed-off-by: Robert Yang <liezhi.yang at windriver.com>
---
 bitbake/lib/bb/cache.py  |  9 ++++++---
 bitbake/lib/bb/cooker.py | 24 +++++++++++++++---------
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 168a77a..7d65939 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -713,12 +713,15 @@ class Cache(NoCache):
     def mtime(cachefile):
         return bb.parse.cached_mtime_noerror(cachefile)
 
-    def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None):
+    def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None, watch_manager=None):
         if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
             cacheData.add_from_recipeinfo(filename, info_array)
 
-            if watcher:
-                watcher(info_array[0].file_depends)
+            if watcher and watch_manager:
+                deps_set = set()
+                for deps in info_array[0].file_depends:
+                    deps_set.add(deps[0])
+                watcher(deps_set - watch_manager.bbseen)
 
         if not self.has_cache:
             return
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index ffda225..cd98ba0 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -176,13 +176,13 @@ class BBCooker:
         self.configuration = configuration
 
         self.configwatcher = pyinotify.WatchManager()
-        self.configwatcher.bbseen = []
+        self.configwatcher.bbseen = set()
         self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
         self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
                          pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
                          pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
         self.watcher = pyinotify.WatchManager()
-        self.watcher.bbseen = []
+        self.watcher.bbseen = set()
         self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
 
         # If being called by something like tinfoil, we need to clean cached data
@@ -278,12 +278,14 @@ class BBCooker:
             watcher = self.watcher
         for i in deps:
             if dirs:
-                f = i[0]
+                f = i
             else:
-                f = os.path.dirname(i[0])
+                f = os.path.dirname(i)
             if f in watcher.bbseen:
                 continue
-            watcher.bbseen.append(f)
+            # Add both files and directories to to bbseen
+            watcher.bbseen.add(f)
+            watcher.bbseen.add(i)
             while True:
                 # We try and add watches for files that don't exist but if they did, would influence
                 # the parser. The parent directory of these files may not exist, in which case we need
@@ -296,7 +298,7 @@ class BBCooker:
                         f = os.path.dirname(f)
                         if f in watcher.bbseen:
                             break
-                        watcher.bbseen.append(f)
+                        watcher.bbseen.add(f)
                         continue
                     if 'ENOSPC' in str(e):
                         providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?")
@@ -367,7 +369,10 @@ class BBCooker:
             self.disableDataTracking()
 
         self.data.renameVar("__depends", "__base_depends")
-        self.add_filewatch(self.data.getVar("__base_depends", False), self.configwatcher)
+        deps_set = set()
+        for deps in self.data.getVar("__base_depends", False):
+            deps_set.add(deps[0])
+        self.add_filewatch(deps_set, self.configwatcher)
 
         self.baseconfig_valid = True
         self.parsecache_valid = False
@@ -1510,7 +1515,7 @@ class BBCooker:
 
             # Add inotify watches for directories searched for bb/bbappend files
             for dirent in searchdirs:
-                self.add_filewatch([[dirent]], dirs=True)
+                self.add_filewatch([dirent], dirs=True)
 
             self.parser = CookerParser(self, filelist, masked)
             self.parsecache_valid = True
@@ -2144,7 +2149,8 @@ class CookerParser(object):
                 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
             (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn)
             self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
-                                        parsed=parsed, watcher = self.cooker.add_filewatch)
+                                        parsed=parsed, watcher=self.cooker.add_filewatch,
+                                        watch_manager=self.cooker.watcher)
         return True
 
     def reparse(self, filename):
-- 
2.7.4