[OE-core] [PATCH] RFC - combo layer repo tool
Yu, Ke
ke.yu at intel.com
Mon Jun 13 13:15:04 UTC 2011
combo-layer-tool: add tool to manipulate combo layer
This patch adds the script combo-layer.py to manipulate the combo
layer repo. A combo layer repo is a repo containing multiple component
repo, e.g. oe-core, bitbake, BSP repos. The combo layer repo need to
be updated by syncing with the component repo upstream. This script
is written to assist the combo layer handling.
See https://wiki.yoctoproject.org/wiki/Layer_Tooling#Combination_repo_tool
for detail task description
Combo layer tool provides three functionalities:
- init: when the combo layer repo and component repo is not existed,
init will "git init" the combo layer repo, and also "git clone" the
component repo
- update: combo layer tool will pull the latest commit from component
repo upstream, and apply the commit since last update commit to the
combo repo. If user specify interactive mode(--interactive),
user can edit the patch list to select which commit to apply.
- splitpatch: split the combo repo commit into separate patches per
component repo. So that user can submit the patch to component repo
upstream.
Combo layer tool uses config file to define the component repo info.
please check the combo-layer.conf.example for the detail explanation
of the config file field.
CC: Richard Purdie <richard.purdie at linuxfoundation.org>
CC: Paul , Eggleton <paul.eggleton at intel.com>
Signed-off-by: Yu Ke <ke.yu at intel.com>
Note: I meeting some difficulty in push it to contrib tree. So just send it out as plain patch for review first.
diff --git a/scripts/combo-layer-hook-default.sh b/scripts/combo-layer-hook-default.sh
new file mode 100755
index 0000000..7dec9b8
--- /dev/null
+++ b/scripts/combo-layer-hook-default.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+# Take a patch from bitbake and apply to poky
+# Call as $0 revision to apply
+# Parameter:
+# $1 patch-file
+# $2 revision
+# $3 reponame
+
+patchfile=$1
+rev=$2
+reponame=$3
+
+sed -i -e "s#Subject: \[PATCH\] \(.*\)#Subject: \[PATCH\] $reponame: \1#" $patchfile
+sed -i -e "0,/Signed-off-by:/s#\(Signed-off-by:.*\)#\($reponame rev: $rev\)\n\n\1#" $patchfile
diff --git a/scripts/combo-layer.conf.example b/scripts/combo-layer.conf.example
new file mode 100644
index 0000000..13a370f
--- /dev/null
+++ b/scripts/combo-layer.conf.example
@@ -0,0 +1,38 @@
+# repo name
+[bitbake]
+# mandatory options
+# git upstream uri
+src_uri = git://git.openembedded.org/bitbake
+
+# the directory to clone the component repo
+local_repo_dir = /home/kyu3/src/test/bitbake
+
+# the relative dir to commit the repo patch
+# use "." if it is root dir
+dest_dir = bitbake
+
+# the updated revision last time.
+# leave it empty if no commit updated yet, and then the tool
+# will start from the first commit
+last_revision =
+
+# optional options
+
+# file_filter: only include the interested file
+# file_filter = [path] [path] ...
+# example:
+# file_filter = src/ : only include the subdir src
+# file_filter = src/*.c : only include the src *.c file
+# file_filter = src/main.c src/Makefile.am : only include these two files
+
+# hook: if provided, the tool will call the hook to proceed the generated patch from upstream,
+# and then apply the modified patch to combo repo
+# the hook's parameter is: ./hook patchpath revision reponame
+# example:
+# hook = combo-layer-hook-default.sh
+
+[oe-core]
+src_uri = git://git.openembedded.org/openembedded-core
+local_repo_dir = /home/kyu3/src/test/oecore
+dest_dir = .
+last_revision =
diff --git a/scripts/combo-layer.py b/scripts/combo-layer.py
new file mode 100755
index 0000000..1ebd81b
--- /dev/null
+++ b/scripts/combo-layer.py
@@ -0,0 +1,405 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright 2011 Intel Corporation
+# Authored-by: Richard Purdie <richard.purdie at intel.com>
+# Paul Eggleton <paul.eggleton at intel.com>
+# Yu Ke <ke.yu at intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os, sys, re
+import optparse
+import logging
+import subprocess,shlex
+
+__version__ = "0.1.0"
+
+def logger_create():
+ logger = logging.getLogger("")
+ loggerhandler = logging.StreamHandler()
+ loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S"))
+ logger.addHandler(loggerhandler)
+ logger.setLevel(logging.INFO)
+ return logger
+
+logger = logger_create()
+
+class Configuration(object):
+ """
+ Manages the configuration
+
+ A valid conf looks like:
+
+# repo name
+[bitbake]
+
+# mandatory options
+
+# git upstream uri
+src_uri = git://git.openembedded.org/bitbake
+
+# the directory to clone the component repo
+local_repo_dir = ~/src/bitbake
+
+# the relative dir to commit the repo patch
+# use "." if it is root dir
+dest_dir = bitbake
+
+# the updated revision last time.
+# leave it empty if no commit updated yet, and then the tool
+# will start from the first commit
+last_revision =
+
+# optional options
+
+# file_filter: only include the interested file
+# file_filter = [path] [path] ...
+# example:
+# file_filter = src/ : only include the subdir src
+# file_filter = src/*.c : only include the src *.c file
+# file_filter = src/main.c src/Makefile.am : only include these two files
+
+[oe-core]
+src_uri = git://git.openembedded.org/openembedded-core
+local_repo_dir = ~/src/oecore
+dest_dir = .
+last_revision =
+
+# more repos ...
+
+ """
+ def __init__(self, options):
+ for key, val in options.__dict__.items():
+ setattr(self, key, val)
+ self.parseconf()
+
+ def parseconf(self):
+ conffile = self.conffile
+ logger.debug("Parsing config file %s" % conffile)
+ comment = re.compile("^\s*#.*$|^\s*$") # comment: "# xxxxxx" or blank line
+ newrepo = re.compile("^\s*\[\s*([\w\-]+)\s*\]\s*$") # new repo: "[ repo_name ]"
+ newvar = re.compile("^\s*([\w\-]+)\s*=\s(\S*)\s*$") # new var : " var = value "
+ self.repos = {}
+ reponame = ""
+ lineno = 0
+ for line in open(conffile):
+ lineno += 1
+ if comment.match(line):
+ logger.debug("comment: skip")
+ continue
+
+ m = newrepo.match(line)
+ if m:
+ reponame = m.group(1)
+ self.repos[reponame] = {}
+ logger.debug("new repo: %s" % reponame)
+ continue
+
+ m = newvar.match(line)
+ if m:
+ if reponame == "":
+ logger.error("no repo specified for line:\n %s" % line)
+ raise SyntaxError(line)
+ varname = m.group(1).strip()
+ varvalue = m.group(2).strip()
+ logger.debug("new var: %s = %s" % (varname, varvalue))
+ self.repos[reponame][varname] = varvalue
+ # save last_revision line number
+ # for later usage of updating the last_revision
+ if varname == "last_revision":
+ self.repos[reponame]["last_revision_lineno"] = lineno
+ else:
+ logger.error("Syntax error in line:\n %s" % line)
+ raise SyntaxError(line)
+
+ def sanity_check(self):
+ required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"]
+ msg = ""
+ for name in self.repos:
+ for option in required_options:
+ if option not in self.repos[name]:
+ msg = "%s\nOption %s is not defined for repo %s" %(msg, option, name)
+ if msg != "":
+ logger.error("configuration file %s has the following error:%s" % (self.conffile,msg))
+ sys.exit(1)
+
+ # filterdiff is required by action_splitpatch, so check its availability
+ if subprocess.call("which filterdiff &>/dev/null", shell=True) != 0:
+ logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)")
+ sys.exit(1)
+
+def runcmd(cmd,destdir=None):
+ """
+ execute command, raise CalledProcessError if fail
+ return output if succeed
+ """
+ logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
+ out = os.tmpfile()
+ try:
+ subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True)
+ except subprocess.CalledProcessError,e:
+ out.seek(0)
+ logger.error("%s" % out.read())
+ raise e
+
+ out.seek(0)
+ output = out.read()
+ logger.debug("output: %s" % output )
+ return output
+
+def runcmd_stdout(cmd, destdir=None):
+ """
+ run command, raise CalledProcessError if fail
+ output to stdout/stderr
+ """
+ logger.debug("run cmd '%s' in dir %s" % (cmd, os.getcwd() if destdir is None else destdir))
+ subprocess.check_call(cmd, cwd=destdir, shell=True)
+
+def action_init(conf, args):
+ """
+ Clone component repositories
+ Check git initialised and working tree is clean
+ """
+ for name in conf.repos:
+ ldir = conf.repos[name]['local_repo_dir']
+ if not os.path.exists(ldir):
+ logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir))
+ runcmd_stdout("git clone %s %s" % (conf.repos[name]['src_uri'], ldir))
+ if not os.path.exists(".git"):
+ runcmd_stdout("git init")
+
+def check_repo_clean(repodir):
+ """
+ check if the repo is clean
+ exit if repo is dirty
+ """
+ try:
+ runcmd("git diff --quiet", repodir)
+ #TODO: also check the index using "git diff --cached"
+ # but this will fail in just initialized git repo
+ # so need figure out a way
+ except:
+ logger.error("git repo %s is dirty, please fix it first", repodir)
+ sys.exit(1)
+
+def action_update(conf, args):
+ """
+ update the component repo
+ generate the patch list
+ apply the generated patches
+ """
+ # make sure all repos are clean
+ for name in conf.repos:
+ check_repo_clean(conf.repos[name]['local_repo_dir'])
+ check_repo_clean(os.getcwd())
+
+ import uuid
+ patch_dir = "patch-%s" % uuid.uuid4()
+ os.mkdir(patch_dir)
+
+ for name in conf.repos:
+ repo = conf.repos[name]
+ ldir = repo['local_repo_dir']
+ dest_dir = repo['dest_dir']
+ repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name)
+
+ # Step 1: update the component repo
+ runcmd_stdout("git pull", ldir)
+
+ # Step 2: generate the patch list stored in patch dir
+ if dest_dir != ".":
+ prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir)
+ else:
+ prefix = ""
+ if repo['last_revision'] == "":
+ logger.info("Warning: last_revision of repo %s is not set, so start from the first commit" % name)
+ patch_cmd_range = "--root master"
+ rev_cmd_range = "master"
+ else:
+ patch_cmd_range = "master"
+ rev_cmd_range = "%s..master" % repo['last_revision']
+
+ file_filter = repo.get('file_filter',"")
+
+ patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \
+ (prefix,repo_patch_dir, patch_cmd_range, file_filter)
+ patchlist = runcmd(patch_cmd, ldir).splitlines()
+
+ rev_cmd = 'git log --pretty=format:"%H" ' + rev_cmd_range
+ revlist = runcmd(rev_cmd, ldir).splitlines()
+
+ # Setp 3: Call repo specific hook to adjust patch
+ if 'hook' in repo:
+ # hook parameter is: ./hook patchpath revision reponame
+ count=len(revlist)-1
+ for patch in patchlist:
+ runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name))
+ count=count-1
+
+ # Step 4: write patch list and revision list to file, for user to edit later
+ patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name)
+ repo['patchlist'] = patchlist_file
+ f = open(patchlist_file, 'w')
+ count=len(revlist)-1
+ for patch in patchlist:
+ f.write("%s %s\n" % (patch, revlist[count]))
+ count=count-1
+ f.close()
+
+ # Step 5: invoke bash for user to edit patch and patch list
+ if conf.interactive:
+ print 'Edit the patch and patch list in %s\n' \
+ 'For example, remove the unwanted patch entry from patchlist-*, so that it will be not applied later\n' \
+ 'After finish, press following command to continue\n' \
+ ' exit 0 -- exit and continue to apply the patch\n' \
+ ' exit 1 -- abort and not apply patch\n' % patch_dir
+ ret = subprocess.call(["bash"], cwd=patch_dir)
+ if ret != 0:
+ print "Abort without applying patch"
+ sys.exit(0)
+
+ # Step 6: apply the generated and revised patch
+ action_apply_patch(conf, args)
+
+def action_update_lastrev(conf,args, rev, reponame):
+ if rev == "":
+ # update for the first time, so no last update revision
+ return
+ f = open(conf.conffile)
+ lines = f.readlines()
+ f.close()
+ lines[conf.repos[reponame]['last_revision_lineno']-1] = " last_revision = %s\n" % rev
+ f = open(conf.conffile, 'w')
+ f.writelines(lines)
+ f.close()
+ logger.info("%s %s last_revision updated to %s" % (conf.conffile, reponame, rev))
+
+def action_apply_patch(conf, args):
+ """
+ apply the generated patch list to combo repo
+ """
+ for name in conf.repos:
+ repo = conf.repos[name]
+ lastrev = ""
+ for line in open(repo['patchlist']):
+ patchfile = line.split()[0]
+ cmd = "git am -s -p1 %s" % patchfile
+ logger.debug("run cmd %s" % cmd)
+ try:
+ subprocess.check_call(cmd, shell=True)
+ except subprocess.CalledProcessError:
+ print "\ncmd %s failed\n" % cmd
+ print "A new bash is invoked for you to resolve the issue"
+ print 'To resolve: patch -p1 < %s; git add -u; git am -i --resolved; exit 0' % patchfile
+ print "To abort: exit 1\n"
+ ret = subprocess.call(["bash"])
+ if ret != 0:
+ print "Abort"
+ action_update_lastrev(conf, args, lastrev, name)
+ sys.exit(0)
+ lastrev = line.split()[1]
+ action_update_lastrev(conf, args, lastrev, name)
+
+def action_splitpatch(conf, args):
+ """
+ generate the commit patch and
+ split the patch per repo
+ """
+ logger.debug("action_splitpatch")
+ if len(args) > 1:
+ commit = args[1]
+ else:
+ commit = "HEAD"
+ patchdir = "splitpatch-%s" % commit
+ if not os.path.exists(patchdir):
+ os.mkdir(patchdir)
+
+ # filerange_root is for the repo whose dest_dir is root "."
+ # and it should be specified by excluding all other repo dest dir
+ # like "-x repo1 -x repo2 -x repo3 ..."
+ filerange_root = ""
+ for name in conf.repos:
+ dest_dir = conf.repos[name]['dest_dir']
+ if dest_dir != ".":
+ filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir)
+
+ for name in conf.repos:
+ dest_dir = conf.repos[name]['dest_dir']
+ patch_filename = "%s/%s.patch" % (patchdir, name)
+ if dest_dir == ".":
+ cmd = "git format-patch -n1 --stdout %s | filterdiff -p1 %s > %s" % (commit, filerange_root, patch_filename)
+ else:
+ cmd = "git format-patch --no-prefix -n1 --stdout %s -- %s > %s" % (commit, dest_dir, patch_filename)
+ runcmd(cmd)
+ logger.info(patch_filename)
+
+def action_error(conf, args):
+ logger.info("invalide action %s" % args[0])
+
+actions = {
+ "init": action_init,
+ "update": action_update,
+ "splitpatch": action_splitpatch,
+}
+
+def main():
+ parser = optparse.OptionParser(
+ version = "Combo Layer Repo Tool version %s" % __version__,
+ usage = """%prog [options] action
+
+Create, update the combo layer according to the upstream repo.
+
+Action:
+ init init the combo layer repo
+ update get patches from upstream update, apply them to the combo repo
+ splitpatch [commit] generate commit patch and split per repo, default commit is HEAD""")
+
+ parser.add_option("-c", "--conf", help = "specify the config file. default is conf/combolayer.conf",
+ action = "store", dest = "conffile", default = "combo-layer.conf")
+
+ parser.add_option("-i", "--interactive", help = "interactive mode, user can edit the patch list and patch",
+ action = "store_true", dest = "interactive", default = False)
+
+ parser.add_option("-D", "--debug", help = "output debug information",
+ action = "store_true", dest = "debug", default = False)
+
+ options, args = parser.parse_args(sys.argv)
+
+ # Dispatch to action handler
+ if len(args) == 1:
+ logger.error("No action specified, exit...")
+ parser.print_help()
+ elif args[1] not in actions:
+ logger.error("Unsupported action %s, exit...\n" % (args[1]))
+ parser.print_help()
+ elif not os.path.exists(options.conffile):
+ logger.error("No valid config file, exit ...\n")
+ parser.print_help()
+ else:
+ if options.debug:
+ logger.setLevel(logging.DEBUG)
+ confdata = Configuration(options)
+ confdata.sanity_check()
+ actions.get(args[1], action_error)(confdata, args[1:])
+
+if __name__ == "__main__":
+ try:
+ ret = main()
+ except Exception:
+ ret = 1
+ import traceback
+ traceback.print_exc(5)
+ sys.exit(ret)
-------------- next part --------------
A non-text attachment was scrubbed...
Name: combo.patch
Type: application/octet-stream
Size: 18673 bytes
Desc: combo.patch
URL: <http://lists.openembedded.org/pipermail/openembedded-core/attachments/20110613/da2dfdaf/attachment-0002.obj>
More information about the Openembedded-core
mailing list