[bitbake-devel] [PATCH 49/94] bitbake: dsi: add feature to store package information
Alex DAMIAN
alexandru.damian at intel.com
Tue Sep 24 16:52:18 UTC 2013
From: Alexandru DAMIAN <alexandru.damian at intel.com>
I'm adding code to read package information and
package dependency off the buildhistory, if the
buildhistory feature is enabled, and the target is an image.
There are small changes to the events trapped since we
need to capture package information after the build is
completed.
Minor changes to internal state handling of BuildInfoHelper.
Signed-off-by: Alexandru DAMIAN <alexandru.damian at intel.com>
---
bitbake/lib/bb/ui/buildinfohelper.py | 75 ++++++++++++++++++++++++++++++++----
bitbake/lib/bb/ui/dsi.py | 30 +++++++--------
bitbake/lib/webhob/orm/models.py | 4 +-
3 files changed, 85 insertions(+), 24 deletions(-)
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 16e69c5..59b6fa3 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -8,7 +8,7 @@ import re
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "webhob.whbmain.settings")
import webhob.whbmain.settings as whb_django_settings
-from webhob.orm.models import Machine, Build, Task, Recipe, Layer_Version, Layer, Package
+from webhob.orm.models import Machine, Build, Task, Recipe, Layer_Version, Layer, Package, Package_Dependency
from webhob.orm.models import Task_Dependency
@@ -42,16 +42,14 @@ class ORMWrapper(object):
build_name=build_info['build_name'],
bitbake_version=build_info['bitbake_version'])
-
return build
- def update_build_object(self, build_obj, errors, warnings, taskfailures):
+ def update_build_object(self, build, errors, warnings, taskfailures):
outcome = Build.SUCCEEDED
if errors or taskfailures:
outcome = Build.FAILED
- build = Build.objects.get(uuid=build_obj.uuid)
build.completed_on = datetime.datetime.now()
build.errors_no = errors
build.warnings_no = warnings
@@ -116,6 +114,22 @@ class ORMWrapper(object):
return layer_object[0]
+ def save_package_information(self, build_obj, packagedict, bldpkgs, recipes):
+ for p in packagedict:
+ packagedict[p]['object'] = Package.objects.create( build = build_obj,
+ name = p,
+ size = packagedict[p]['size'])
+ if p in bldpkgs:
+ packagedict[p]['object'].version = bldpkgs[p]['version']
+ packagedict[p]['object'].recipe = recipes[bldpkgs[p]['pn']]
+ packagedict[p]['object'].save()
+
+ for p in packagedict:
+ for px in packagedict[p]['depends']:
+ Package_Dependency.objects.create( package = packagedict[p]['object'],
+ depends_on = packagedict[px]['object'] );
+
+
class BuildInfoHelper(object):
""" This class gathers the build information from the server and sends it
towards the ORM wrapper for storing in the database
@@ -123,13 +137,14 @@ class BuildInfoHelper(object):
Keeps in memory all data that needs matching before writing it to the database
"""
- def __init__(self, server):
+ def __init__(self, server, has_build_history = False):
self._configure_django()
self.internal_state = {}
self.uuid = None
self.task_order = 0
self.server = server
self.orm_wrapper = ORMWrapper()
+ self.has_build_history = has_build_history
def _configure_django(self):
# Add webhob to sys path for importing modules
@@ -343,7 +358,6 @@ class BuildInfoHelper(object):
def update_build_information(self, event, errors, warnings, taskfailures):
self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
- del self.internal_state['build']
def store_started_task(self, event):
identifier = re.split(':', event.taskfile)[-1] + event.taskname
@@ -398,8 +412,56 @@ class BuildInfoHelper(object):
self.orm_wrapper.get_update_task_object(task_information)
+ def read_package_dep_data(self, event):
+ # verify that we have something to read
+ if not self.internal_state['build'].is_image or not self.has_build_history:
+ print "not collecting package info ", self.internal_state['build'].is_image, self.has_build_history
+ return
+
+ # fair warning: code correlates to buildhistory.bbclass; anything changes there, needs to chage here too
+ TOPDIR, error = self.server.runCommand(['getVariable', 'TOPDIR'])
+ MACHINE_ARCH, error = self.server.runCommand(['getVariable', 'MACHINE_ARCH'])
+ TCLIBC, error = self.server.runCommand(['getVariable', 'TCLIBC'])
+ MULTIMACH_TARGET_SYS, error = self.server.runCommand(['getVariable', 'MULTIMACH_TARGET_SYS'])
+ SDK_NAME, error = self.server.runCommand(['getVariable', 'SDK_NAME'])
+ BUILDHISTORY_DIR = "%s/buildhistory" % TOPDIR
+ BUILDHISTORY_DIR_IMAGE = "%s/images/%s/%s/%s" % (BUILDHISTORY_DIR, MACHINE_ARCH, TCLIBC, self.internal_state['build'].target)
+
+ self.internal_state['packages'] = {}
+
+ with open("%s/installed-package-sizes.txt" % BUILDHISTORY_DIR_IMAGE, "r") as fin:
+ for line in fin:
+ line = line.rstrip(";")
+ psize, px = line.split("\t")
+ punit, pname = px.split(" ")
+ self.internal_state['packages'][pname.strip()] = {'size':psize, 'depends' : []}
+
+ with open("%s/depends.dot" % BUILDHISTORY_DIR_IMAGE, "r") as fin:
+ p = re.compile(r' -> ')
+ for line in fin:
+ line = line.rstrip(';')
+ linesplit = p.split(line)
+ if len(linesplit) == 2:
+ pname = linesplit[0]
+ dependsname = linesplit[1].split(" ")[0].strip().strip(";")
+ if not pname in self.internal_state['packages']:
+ self.internal_state['packages'][pname] = {'size': 0, 'depends' : []}
+ if not dependsname in self.internal_state['packages']:
+ self.internal_state['packages'][dependsname] = {'size': 0, 'depends' : []}
+ self.internal_state['packages'][pname]['depends'].append(dependsname)
+
+ self.orm_wrapper.save_package_information(self.internal_state['build'], self.internal_state['packages'],
+ self.internal_state['bldpkgs'], self.internal_state['recipes'])
+
def store_dependency_information(self, event):
+
+ # save build time package information
+ self.internal_state['bldpkgs'] = {}
+ for pkg in event._depgraph['packages']:
+ self.internal_state['bldpkgs'][pkg] = event._depgraph['packages'][pkg]
+
+ # save recipe information
self.internal_state['recipes'] = {}
for pn in event._depgraph['pn']:
@@ -447,4 +509,3 @@ class BuildInfoHelper(object):
dep = _save_a_task(taskdesc1)
Task_Dependency.objects.get_or_create( task = target, depends_on = dep )
- del self.internal_state['recipes']
diff --git a/bitbake/lib/bb/ui/dsi.py b/bitbake/lib/bb/ui/dsi.py
index d03a7c5..98d225b 100644
--- a/bitbake/lib/bb/ui/dsi.py
+++ b/bitbake/lib/bb/ui/dsi.py
@@ -243,6 +243,14 @@ def main(server, eventHandler, params, tf = TerminalFilter):
includelogs, loglines, consolelogfile = _log_settings_from_server(server)
+
+ # verify and warn
+ build_history_enabled = True
+ inheritlist, error = server.runCommand(["getVariable", "INHERIT"])
+ if not "buildhistory" in inheritlist.split(" "):
+ logger.warn("buildhistory is not enabled. Please enable INHERIT += \"buildhistory\" to see image details.")
+ build_history_enabled = False
+
if sys.stdin.isatty() and sys.stdout.isatty():
log_exec_tty = True
else:
@@ -275,7 +283,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
warnings = 0
taskfailures = []
- buildinfohelper = BuildInfoHelper(server)
+ buildinfohelper = BuildInfoHelper(server, build_history_enabled)
buildinfohelper.store_layer_info()
termfilter = tf(main, helper, console, format)
@@ -453,28 +461,20 @@ def main(server, eventHandler, params, tf = TerminalFilter):
# timestamp should be added for this
continue
- if isinstance(event, bb.event.OperationStarted):
- # timestamp should be added for this
+ if isinstance(event, (bb.event.TreeDataPreparationStarted, bb.event.TreeDataPreparationCompleted)):
continue
- if isinstance(event, bb.event.OperationCompleted):
- # timestamp should be added for this
- # signal a complete operation
- # calculate timing
- continue
-
- if isinstance(event, bb.event.DiskFull):
- # trigger an error
+ if isinstance(event, (bb.event.BuildCompleted)):
+ buildinfohelper.read_package_dep_data(event)
+ buildinfohelper.update_build_information(event, errors, warnings, taskfailures)
continue
- if isinstance(event, (bb.event.BuildCompleted,
- bb.command.CommandCompleted,
+ if isinstance(event, (bb.command.CommandCompleted,
bb.command.CommandFailed,
bb.command.CommandExit)):
- buildinfohelper.update_build_information(event, errors, warnings, taskfailures)
# we start a new build info
- buildinfohelper = BuildInfoHelper(server)
+ buildinfohelper = BuildInfoHelper(server, build_history_enabled)
buildinfohelper.store_layer_info()
continue
diff --git a/bitbake/lib/webhob/orm/models.py b/bitbake/lib/webhob/orm/models.py
index a4721db..774cdff 100644
--- a/bitbake/lib/webhob/orm/models.py
+++ b/bitbake/lib/webhob/orm/models.py
@@ -110,9 +110,9 @@ class Artifact(models.Model):
class Package(models.Model):
build = models.ForeignKey('Build', related_name='package_build')
- recipe = models.ForeignKey('Recipe', related_name='package_recipe')
+ recipe = models.ForeignKey('Recipe', related_name='package_recipe', null=True)
name = models.CharField(max_length=100)
- version = models.CharField(max_length=100)
+ version = models.CharField(max_length=100, default="")
size = models.IntegerField()
--
1.8.1.2
More information about the bitbake-devel
mailing list