[bitbake-devel] [PATCH 09/12] toasterui: disable autocommit for build logging

[Alex DAMIAN]

From: Alexandru DAMIAN <alexandru.damian at intel.com>

This patch disables autocommit for inserting build data,
effectively updating all build data in a single transaction.

This is a purely performance improvement patch, as the transaction
will always be commited.

Similar manual transaction handling in the layer source update
method. Added feedback messages during update method.

[YOCTO #7140]

Signed-off-by: Alexandru DAMIAN <alexandru.damian at intel.com>
---
 lib/bb/ui/buildinfohelper.py | 11 +++++++++++
 lib/toaster/orm/models.py    | 18 +++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/lib/bb/ui/buildinfohelper.py b/lib/bb/ui/buildinfohelper.py
index 1096ccf..967e4bd 100644
--- a/lib/bb/ui/buildinfohelper.py
+++ b/lib/bb/ui/buildinfohelper.py
@@ -35,6 +35,7 @@ from bb.msg import BBLogFormatter as format
 from django.db import models
 import logging
 
+from django.db import transaction
 
 logger = logging.getLogger("BitBake")
 
@@ -606,7 +607,9 @@ class BuildInfoHelper(object):
         self.internal_state = {}
         self.internal_state['taskdata'] = {}
         self.task_order = 0
+        self.autocommit_step = 1
         self.server = server
+        transaction.set_autocommit(False)
         self.orm_wrapper = ORMWrapper()
         self.has_build_history = has_build_history
         self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0]
@@ -926,6 +929,12 @@ class BuildInfoHelper(object):
                 task_information['outcome'] = Task.OUTCOME_FAILED
                 del self.internal_state['taskdata'][identifier]
 
+        # we force a sync point here, to get the progress bar to show
+        if self.autocommit_step % 3 == 0:
+            transaction.set_autocommit(True)
+            transaction.set_autocommit(False)
+        self.autocommit_step += 1
+
         self.orm_wrapper.get_update_task_object(task_information, True) # must exist
 
 
@@ -1197,3 +1206,5 @@ class BuildInfoHelper(object):
                 # we have no build, and we still have events; something amazingly wrong happend
                 for event in self.internal_state['backlog']:
                    logger.error("UNSAVED log: %s", event.msg)
+
+        transaction.set_autocommit(True)
diff --git a/lib/toaster/orm/models.py b/lib/toaster/orm/models.py
index 54cd535..9d4f4c9 100644
--- a/lib/toaster/orm/models.py
+++ b/lib/toaster/orm/models.py
@@ -664,6 +664,7 @@ class LayerIndexLayerSource(LayerSource):
         """
         assert self.apiurl is not None
         from django.db import IntegrityError
+        from django.db import transaction
 
         import httplib, urlparse, json
         import os
@@ -715,6 +716,7 @@ class LayerIndexLayerSource(LayerSource):
         # update branches; only those that we already have names listed in the Releases table
         whitelist_branch_names = map(lambda x: x.branch_name, Release.objects.all())
 
+        print "Fetching branches"
         branches_info = _get_json_response(apilinks['branches']
             + "?filter=name:%s" % "OR".join(whitelist_branch_names))
         for bi in branches_info:
@@ -727,6 +729,7 @@ class LayerIndexLayerSource(LayerSource):
 
         # update layers
         layers_info = _get_json_response(apilinks['layerItems'])
+        transaction.set_autocommit(False)
         for li in layers_info:
             l, created = Layer.objects.get_or_create(layer_source = self, name = li['name'])
             l.up_id = li['id']
@@ -738,11 +741,15 @@ class LayerIndexLayerSource(LayerSource):
             l.summary = li['summary']
             l.description = li['description']
             l.save()
+        transaction.set_autocommit(True)
 
         # update layerbranches/layer_versions
+        print "Fetching layer information"
         layerbranches_info = _get_json_response(apilinks['layerBranches']
                 + "?filter=branch:%s" % "OR".join(map(lambda x: str(x.up_id), [i for i in Branch.objects.filter(layer_source = self) if i.up_id is not None] ))
             )
+
+        transaction.set_autocommit(False)
         for lbi in layerbranches_info:
             lv, created = Layer_Version.objects.get_or_create(layer_source = self,
                     up_id = lbi['id'],
@@ -755,10 +762,12 @@ class LayerIndexLayerSource(LayerSource):
             lv.commit = lbi['actual_branch']
             lv.dirpath = lbi['vcs_subdir']
             lv.save()
+        transaction.set_autocommit(True)
 
         # update layer dependencies
         layerdependencies_info = _get_json_response(apilinks['layerDependencies'])
         dependlist = {}
+        transaction.set_autocommit(False)
         for ldi in layerdependencies_info:
             try:
                 lv = Layer_Version.objects.get(layer_source = self, up_id = ldi['layerbranch'])
@@ -777,23 +786,29 @@ class LayerIndexLayerSource(LayerSource):
             LayerVersionDependency.objects.filter(layer_version = lv).delete()
             for lvd in dependlist[lv]:
                 LayerVersionDependency.objects.get_or_create(layer_version = lv, depends_on = lvd)
+        transaction.set_autocommit(True)
 
 
         # update machines
+        print "Fetching machine information"
         machines_info = _get_json_response(apilinks['machines']
                 + "?filter=layerbranch:%s" % "OR".join(map(lambda x: str(x.up_id), Layer_Version.objects.filter(layer_source = self)))
             )
+        transaction.set_autocommit(False)
         for mi in machines_info:
             mo, created = Machine.objects.get_or_create(layer_source = self, up_id = mi['id'], layer_version = Layer_Version.objects.get(layer_source = self, up_id = mi['layerbranch']))
             mo.up_date = mi['updated']
             mo.name = mi['name']
             mo.description = mi['description']
             mo.save()
+        transaction.set_autocommit(True)
 
         # update recipes; paginate by layer version / layer branch
+        print "Fetching target information"
         recipes_info = _get_json_response(apilinks['recipes']
                 + "?filter=layerbranch:%s" % "OR".join(map(lambda x: str(x.up_id), Layer_Version.objects.filter(layer_source = self)))
             )
+        transaction.set_autocommit(False)
         for ri in recipes_info:
             try:
                 ro, created = Recipe.objects.get_or_create(layer_source = self, up_id = ri['id'], layer_version = Layer_Version.objects.get(layer_source = self, up_id = ri['layerbranch']))
@@ -809,8 +824,9 @@ class LayerIndexLayerSource(LayerSource):
                 ro.file_path = ri['filepath'] + "/" + ri['filename']
                 ro.save()
             except:
-                print "Duplicate Recipe, ignoring: ", vars(ro)
+                #print "Duplicate Recipe, ignoring: ", vars(ro)
                 pass
+        transaction.set_autocommit(True)
         pass
 
 class BitbakeVersion(models.Model):
-- 
1.9.1