[bitbake-devel] [PATCH] cooker: Separate out collections handling code into its own class
Richard Purdie
richard.purdie at linuxfoundation.org
Fri May 10 15:17:06 UTC 2013
The Cooker class is too large and needs to be split up into different
functional units. Splitting out the collections code into its own class
seems like a good place to start to try and disentangle things.
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 5c52d85..da598c9 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -89,7 +89,6 @@ class BBCooker:
def __init__(self, configuration, server_registration_cb, savedenv={}):
self.status = None
- self.appendlist = {}
self.skiplist = {}
self.server_registration_cb = server_registration_cb
@@ -419,7 +418,7 @@ class BBCooker:
if fn:
try:
- envdata = bb.cache.Cache.loadDataFull(fn, self.get_file_appends(fn), self.configuration.data)
+ envdata = bb.cache.Cache.loadDataFull(fn, self.collection.get_file_appends(fn), self.configuration.data)
except Exception as e:
parselog.exception("Unable to read %s", fn)
raise
@@ -698,22 +697,13 @@ class BBCooker:
print("}", file=tdepends_file)
logger.info("Task dependencies saved to 'task-depends.dot'")
- def calc_bbfile_priority( self, filename, matched = None ):
- for _, _, regex, pri in self.status.bbfile_config_priorities:
- if regex.match(filename):
- if matched != None:
- if not regex in matched:
- matched.add(regex)
- return pri
- return 0
-
def show_appends_with_no_recipes( self ):
recipes = set(os.path.basename(f)
for f in self.status.pkg_fn.iterkeys())
recipes |= set(os.path.basename(f)
for f in self.skiplist.iterkeys())
- appended_recipes = self.appendlist.iterkeys()
- appends_without_recipes = [self.appendlist[recipe]
+ appended_recipes = self.collection.appendlist.iterkeys()
+ appends_without_recipes = [self.collection.appendlist[recipe]
for recipe in appended_recipes
if recipe not in recipes]
if appends_without_recipes:
@@ -747,32 +737,8 @@ class BBCooker:
providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.status.preferred[providee])
self.status.preferred[providee] = provider
- # Calculate priorities for each file
- matched = set()
- for p in self.status.pkg_fn:
- realfn, cls = bb.cache.Cache.virtualfn2realfn(p)
- self.status.bbfile_priority[p] = self.calc_bbfile_priority(realfn, matched)
- # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
- unmatched = set()
- for _, _, regex, pri in self.status.bbfile_config_priorities:
- if not regex in matched:
- unmatched.add(regex)
-
- def findmatch(regex):
- for bbfile in self.appendlist:
- for append in self.appendlist[bbfile]:
- if regex.match(append):
- return True
- return False
-
- for unmatch in unmatched.copy():
- if findmatch(unmatch):
- unmatched.remove(unmatch)
-
- for collection, pattern, regex, _ in self.status.bbfile_config_priorities:
- if regex in unmatched:
- collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
+ self.status.bbfile_priority = self.collection.collection_priorities(self.status.pkg_fn)
def findCoreBaseFiles(self, subdir, configfile):
corebase = self.configuration.data.getVar('COREBASE', True) or ""
@@ -1111,7 +1077,9 @@ class BBCooker:
"""
if bf.startswith("/") or bf.startswith("../"):
bf = os.path.abspath(bf)
- filelist, masked = self.collect_bbfiles()
+
+ self.collection = CookerCollectFiles(self.status.bbfile_config_priorities)
+ filelist, masked = self.collection.collect_bbfiles(self.configuration.data, self.configuration.event_data)
try:
os.stat(bf)
bf = os.path.abspath(bf)
@@ -1165,7 +1133,7 @@ class BBCooker:
self.buildSetVars()
self.status = bb.cache.CacheData(self.caches_array)
- infos = bb.cache.Cache.parse(fn, self.get_file_appends(fn), \
+ infos = bb.cache.Cache.parse(fn, self.collection.get_file_appends(fn), \
self.configuration.data,
self.caches_array)
infos = dict(infos)
@@ -1332,7 +1300,9 @@ class BBCooker:
for dep in self.configuration.extra_assume_provided:
self.status.ignored_dependencies.add(dep)
- (filelist, masked) = self.collect_bbfiles()
+ self.collection = CookerCollectFiles(self.status.bbfile_config_priorities)
+ (filelist, masked) = self.collection.collect_bbfiles(self.configuration.data, self.configuration.event_data)
+
self.configuration.data.renameVar("__depends", "__base_depends")
self.parser = CookerParser(self, filelist, masked)
@@ -1369,7 +1339,87 @@ class BBCooker:
return pkgs_to_build
- def get_bbfiles( self, path = os.getcwd() ):
+
+
+
+ def pre_serve(self):
+ # Empty the environment. The environment will be populated as
+ # necessary from the data store.
+ #bb.utils.empty_environment()
+ try:
+ prserv.serv.auto_start(self.configuration.data)
+ except prserv.serv.PRServiceConfigError:
+ bb.event.fire(CookerExit(), self.configuration.event_data)
+ return
+
+ def post_serve(self):
+ prserv.serv.auto_shutdown(self.configuration.data)
+ bb.event.fire(CookerExit(), self.configuration.event_data)
+
+ def shutdown(self):
+ self.state = state.shutdown
+
+ def stop(self):
+ self.state = state.stop
+
+ def reparseFiles(self):
+ return
+
+ def initialize(self):
+ self.state = state.initial
+ self.initConfigurationData()
+
+ def reset(self):
+ self.state = state.initial
+ self.loadConfigurationData()
+
+def server_main(cooker, func, *args):
+ cooker.pre_serve()
+
+ if cooker.configuration.profile:
+ try:
+ import cProfile as profile
+ except:
+ import profile
+ prof = profile.Profile()
+
+ ret = profile.Profile.runcall(prof, func, *args)
+
+ prof.dump_stats("profile.log")
+ bb.utils.process_profilelog("profile.log")
+ print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
+
+ else:
+ ret = func(*args)
+
+ cooker.post_serve()
+
+ return ret
+
+class CookerExit(bb.event.Event):
+ """
+ Notify clients of the Cooker shutdown
+ """
+
+ def __init__(self):
+ bb.event.Event.__init__(self)
+
+
+class CookerCollectFiles(object):
+ def __init__(self, priorities):
+ self.appendlist = {}
+ self.bbfile_config_priorities = priorities
+
+ def calc_bbfile_priority( self, filename, matched = None ):
+ for _, _, regex, pri in self.bbfile_config_priorities:
+ if regex.match(filename):
+ if matched != None:
+ if not regex in matched:
+ matched.add(regex)
+ return pri
+ return 0
+
+ def get_bbfiles(self, path = os.getcwd()):
"""Get list of default .bb files by reading out the current directory"""
contents = os.listdir(path)
bbfiles = []
@@ -1379,7 +1429,7 @@ class BBCooker:
bbfiles.append(os.path.abspath(os.path.join(os.getcwd(), f)))
return bbfiles
- def find_bbfiles( self, path ):
+ def find_bbfiles(self, path):
"""Find all the .bb and .bbappend files in a directory"""
from os.path import join
@@ -1392,14 +1442,14 @@ class BBCooker:
return found
- def collect_bbfiles( self ):
+ def collect_bbfiles(self, config, eventdata):
"""Collect all available .bb build files"""
masked = 0
collectlog.debug(1, "collecting .bb files")
- files = (data.getVar( "BBFILES", self.configuration.data, True) or "").split()
- data.setVar("BBFILES", " ".join(files), self.configuration.data)
+ files = (config.getVar( "BBFILES", True) or "").split()
+ config.setVar("BBFILES", " ".join(files))
# Sort files by priority
files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem) )
@@ -1409,7 +1459,7 @@ class BBCooker:
if not len(files):
collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
- bb.event.fire(CookerExit(), self.configuration.event_data)
+ bb.event.fire(CookerExit(), eventdata)
# Can't use set here as order is important
newfiles = []
@@ -1427,7 +1477,7 @@ class BBCooker:
if g not in newfiles:
newfiles.append(g)
- bbmask = self.configuration.data.getVar('BBMASK', True)
+ bbmask = config.getVar('BBMASK', True)
if bbmask:
try:
@@ -1475,74 +1525,44 @@ class BBCooker:
def get_file_appends(self, fn):
"""
Returns a list of .bbappend files to apply to fn
- NB: collect_bbfiles() must have been called prior to this
"""
f = os.path.basename(fn)
if f in self.appendlist:
return self.appendlist[f]
return []
- def pre_serve(self):
- # Empty the environment. The environment will be populated as
- # necessary from the data store.
- #bb.utils.empty_environment()
- try:
- prserv.serv.auto_start(self.configuration.data)
- except prserv.serv.PRServiceConfigError:
- bb.event.fire(CookerExit(), self.configuration.event_data)
- return
-
- def post_serve(self):
- prserv.serv.auto_shutdown(self.configuration.data)
- bb.event.fire(CookerExit(), self.configuration.event_data)
-
- def shutdown(self):
- self.state = state.shutdown
+ def collection_priorities(self, pkgfns):
- def stop(self):
- self.state = state.stop
-
- def reparseFiles(self):
- return
-
- def initialize(self):
- self.state = state.initial
- self.initConfigurationData()
-
- def reset(self):
- self.state = state.initial
- self.loadConfigurationData()
+ priorities = {}
-def server_main(cooker, func, *args):
- cooker.pre_serve()
-
- if cooker.configuration.profile:
- try:
- import cProfile as profile
- except:
- import profile
- prof = profile.Profile()
-
- ret = profile.Profile.runcall(prof, func, *args)
-
- prof.dump_stats("profile.log")
- bb.utils.process_profilelog("profile.log")
- print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
-
- else:
- ret = func(*args)
+ # Calculate priorities for each file
+ matched = set()
+ for p in pkgfns:
+ realfn, cls = bb.cache.Cache.virtualfn2realfn(p)
+ priorities[p] = self.calc_bbfile_priority(realfn, matched)
+
+ # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
+ unmatched = set()
+ for _, _, regex, pri in self.bbfile_config_priorities:
+ if not regex in matched:
+ unmatched.add(regex)
- cooker.post_serve()
+ def findmatch(regex):
+ for bbfile in self.appendlist:
+ for append in self.appendlist[bbfile]:
+ if regex.match(append):
+ return True
+ return False
- return ret
+ for unmatch in unmatched.copy():
+ if findmatch(unmatch):
+ unmatched.remove(unmatch)
-class CookerExit(bb.event.Event):
- """
- Notify clients of the Cooker shutdown
- """
+ for collection, pattern, regex, _ in self.bbfile_config_priorities:
+ if regex in unmatched:
+ collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
- def __init__(self):
- bb.event.Event.__init__(self)
+ return priorities
def catch_parse_error(func):
"""Exception handling bits for our parsing"""
@@ -1677,7 +1697,7 @@ class CookerParser(object):
self.fromcache = []
self.willparse = []
for filename in self.filelist:
- appends = self.cooker.get_file_appends(filename)
+ appends = self.cooker.collection.get_file_appends(filename)
if not self.bb_cache.cacheValid(filename, appends):
self.willparse.append((filename, appends, cooker.caches_array))
else:
@@ -1840,7 +1860,7 @@ class CookerParser(object):
def reparse(self, filename):
infos = self.bb_cache.parse(filename,
- self.cooker.get_file_appends(filename),
+ self.cooker.collection.get_file_appends(filename),
self.cfgdata, self.cooker.caches_array)
for vfn, info_array in infos:
self.cooker.status.add_from_recipeinfo(vfn, info_array)
More information about the bitbake-devel
mailing list