[oe-commits] [bitbake] 16/20: prserv: Add dump_db()

[git at git.openembedded.org]

rpurdie pushed a commit to branch master-next
in repository bitbake.

commit de788d2c1fa7dbd1325909872ebbad71093cd7bd
Author: Haris Okanovic <haris.okanovic at ni.com>
AuthorDate: Tue Feb 23 11:36:33 2016 -0600

    prserv: Add dump_db()
    
    Returns a script (string) that reconstructs the state of the
    entire database at the time this function is called. The script
    language is defined by the backing database engine, which is a
    function of server configuration.
    Returns None if the database engine does not support dumping to
    script or if some other error is encountered in processing.
    
    The SQLite3 implementation in db.py calls iterdump() [1] to generate
    a script. iterdump() is the library equivalent of the `sqlite3 .dump`
    shell command, and the scripts are compatible. Execute the script in
    an empty SQLite3 database using the sqlite3 utility to restore a backup
    of prserv.
    
    Use case: Backup a live PR server database in a non-racy way, such
    that one could snapshot the entire database after a set of bitbake
    builds all using a shared server. I.e. All changes made prior to
    the start of a dump_db() operation should be committed and captured
    in the script. Subsequent changes made during the backup process are
    not guaranteed to be captured.
    
    Testing: ~7MB database backs up in ~1s while PR server is under load
    from 32 thread bitbake builds on two separate machines.
    
    [1] https://docs.python.org/2/library/sqlite3.html#sqlite3.Connection.iterdump
    
    Signed-off-by: Haris Okanovic <haris.okanovic at ni.com>
    Reviewed-by: Ken Sharp <ken.sharp at ni.com>
    Reviewed-by: Bill Pittman <bill.pittman at ni.com>
    Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
 lib/prserv/db.py   |  8 ++++++++
 lib/prserv/serv.py | 25 +++++++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/lib/prserv/db.py b/lib/prserv/db.py
index 36c9f7b..2a86184 100644
--- a/lib/prserv/db.py
+++ b/lib/prserv/db.py
@@ -231,6 +231,14 @@ class PRTable(object):
                 datainfo.append(col)
         return (metainfo, datainfo)
 
+    def dump_db(self, fd):
+        writeCount = 0
+        for line in self.conn.iterdump():
+            writeCount = writeCount + len(line) + 1
+            fd.write(line)
+            fd.write('\n')
+        return writeCount
+
 class PRData(object):
     """Object representing the PR database"""
     def __init__(self, filename, nohist=True):
diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py
index f588f4d..affccd9 100644
--- a/lib/prserv/serv.py
+++ b/lib/prserv/serv.py
@@ -4,6 +4,7 @@ from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
 import threading
 import Queue
 import socket
+import StringIO
 
 try:
     import sqlite3
@@ -59,6 +60,7 @@ class PRServer(SimpleXMLRPCServer):
         self.register_function(self.quit, "quit")
         self.register_function(self.ping, "ping")
         self.register_function(self.export, "export")
+        self.register_function(self.dump_db, "dump_db")
         self.register_function(self.importone, "importone")
         self.register_introspection_functions()
 
@@ -115,6 +117,26 @@ class PRServer(SimpleXMLRPCServer):
             logger.error(str(exc))
             return None
 
+    def dump_db(self):
+        """
+        Returns a script (string) that reconstructs the state of the
+        entire database at the time this function is called. The script
+        language is defined by the backing database engine, which is a
+        function of server configuration.
+        Returns None if the database engine does not support dumping to
+        script or if some other error is encountered in processing.
+        """
+        buff = StringIO.StringIO()
+        try:
+            self.table.sync()
+            self.table.dump_db(buff)
+            return buff.getvalue()
+        except Exception as exc:
+            logger.error(str(exc))
+            return None
+        finally:
+            buff.close()
+
     def importone(self, version, pkgarch, checksum, value):
         return self.table.importone(version, pkgarch, checksum, value)
 
@@ -288,6 +310,9 @@ class PRServerConnection(object):
     def export(self,version=None, pkgarch=None, checksum=None, colinfo=True):
         return self.connection.export(version, pkgarch, checksum, colinfo)
 
+    def dump_db(self):
+        return self.connection.dump_db()
+
     def importone(self, version, pkgarch, checksum, value):
         return self.connection.importone(version, pkgarch, checksum, value)
 

-- 
To stop receiving notification emails like this one, please contact
the administrator of this repository.