[OE-core] [PATCH 1/4] license_class: Reimplemented manifest creation in python
Aníbal Limón
anibal.limon at linux.intel.com
Thu Oct 30 23:08:09 UTC 2014
Richard,
I did the changes that you suggest and another small changes adding
LIC_FILES_CHKSUM in some recipes to avoid warnings in manifest creation.
I put together into this branch for review,
https://git.yoctoproject.org/cgit/cgit.cgi/poky-contrib/log/?h=alimon/license
Best regards.
On 29/10/14 15:28, Richard Purdie wrote:
> On Wed, 2014-10-29 at 12:34 -0600, Aníbal Limón wrote:
>> Reimplemented license_manifest_create from shell to python, in
>> order to use oe.license module for handle INCOMPATIBLE_LICENSE
>> and add License prorities into OR's evaluation.
>>
>> Signed-off-by: Aníbal Limón <anibal.limon at linux.intel.com>
>> ---
>> meta/classes/license.bbclass | 161 ++++++++++++++++++++++++-------------------
>> 1 file changed, 90 insertions(+), 71 deletions(-)
>>
>> diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
>> index a34ea39..c8a86ce 100644
>> --- a/meta/classes/license.bbclass
>> +++ b/meta/classes/license.bbclass
>> @@ -25,78 +25,97 @@ python write_package_manifest() {
>> 'w+').write(image_list_installed_packages(d))
>> }
>>
>> -license_create_manifest() {
>> - # Test if BUILD_IMAGES_FROM_FEEDS is defined in env
>> - if [ -n "${BUILD_IMAGES_FROM_FEEDS}" ]; then
>> - exit 0
>> - fi
>> -
>> - INSTALLED_PKGS=`cat ${LICENSE_DIRECTORY}/${IMAGE_NAME}/package.manifest`
>> - LICENSE_MANIFEST="${LICENSE_DIRECTORY}/${IMAGE_NAME}/license.manifest"
>> - # remove existing license.manifest file
>> - if [ -f ${LICENSE_MANIFEST} ]; then
>> - rm ${LICENSE_MANIFEST}
>> - fi
>> - touch ${LICENSE_MANIFEST}
>> - for pkg in ${INSTALLED_PKGS}; do
>> - filename=`ls ${PKGDATA_DIR}/runtime-reverse/${pkg}| head -1`
>> - pkged_pn="$(sed -n 's/^PN: //p' ${filename})"
>> -
>> - # check to see if the package name exists in the manifest. if so, bail.
>> - if grep -q "^PACKAGE NAME: ${pkg}" ${LICENSE_MANIFEST}; then
>> - continue
>> - fi
>> -
>> - pkged_pv="$(sed -n 's/^PV: //p' ${filename})"
>> - pkged_name="$(basename $(readlink ${filename}))"
>> - pkged_lic="$(sed -n "/^LICENSE_${pkged_name}: /{ s/^LICENSE_${pkged_name}: //; s/[|&()*]/ /g; s/ */ /g; p }" ${filename})"
>> - if [ -z ${pkged_lic} ]; then
>> - # fallback checking value of LICENSE
>> - pkged_lic="$(sed -n "/^LICENSE: /{ s/^LICENSE: //; s/[|&()*]/ /g; s/ */ /g; p }" ${filename})"
>> - fi
>> -
>> - echo "PACKAGE NAME:" ${pkg} >> ${LICENSE_MANIFEST}
>> - echo "PACKAGE VERSION:" ${pkged_pv} >> ${LICENSE_MANIFEST}
>> - echo "RECIPE NAME:" ${pkged_pn} >> ${LICENSE_MANIFEST}
>> - printf "LICENSE:" >> ${LICENSE_MANIFEST}
>> - for lic in ${pkged_lic}; do
>> - # to reference a license file trim trailing + symbol
>> - if ! [ -e "${LICENSE_DIRECTORY}/${pkged_pn}/generic_${lic%+}" ]; then
>> - bbwarn "The license listed ${lic} was not in the licenses collected for ${pkged_pn}"
>> - fi
>> - printf " ${lic}" >> ${LICENSE_MANIFEST}
>> - done
>> - printf "\n\n" >> ${LICENSE_MANIFEST}
>> - done
>> -
>> - # Two options here:
>> - # - Just copy the manifest
>> - # - Copy the manifest and the license directories
>> - # With both options set we see a .5 M increase in core-image-minimal
>> - if [ "${COPY_LIC_MANIFEST}" = "1" ]; then
>> - mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/
>> - cp ${LICENSE_MANIFEST} ${IMAGE_ROOTFS}/usr/share/common-licenses/license.manifest
>> - if [ "${COPY_LIC_DIRS}" = "1" ]; then
>> - for pkg in ${INSTALLED_PKGS}; do
>> - mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}
>> - pkged_pn="$(oe-pkgdata-util lookup-recipe ${PKGDATA_DIR} ${pkg})"
>> - for lic in `ls ${LICENSE_DIRECTORY}/${pkged_pn}`; do
>> - # Really don't need to copy the generics as they're
>> - # represented in the manifest and in the actual pkg licenses
>> - # Doing so would make your image quite a bit larger
>> - if [ "${lic#generic_}" = "${lic}" ]; then
>> - cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
>> - else
>> - if [ ! -f ${IMAGE_ROOTFS}/usr/share/common-licenses/${lic} ]; then
>> - cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/
>> - fi
>> - ln -sf ../${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
>> - fi
>> - done
>> - done
>> - fi
>> - fi
>> +python license_create_manifest() {
>> + import shutil
>> + import re
>>
>> + build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)
>> + if build_images_from_feeds == "1":
>> + return 0
>> +
>> + pkg_dic = {}
>> + package_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
>> + d.getVar('IMAGE_NAME', True), 'package.manifest')
>> + with open(package_manifest, "r") as package_file:
>> + pkg_list = package_file.read().split()
>> + for pkg in pkg_list:
>> + pkg_info = os.path.join(d.getVar('PKGDATA_DIR', True),
>> + 'runtime-reverse', pkg)
>> + with open(pkg_info, "r") as pkg_info_file:
>> + pkg_dic[pkg] = {}
>> +
>> + pkg_lic_name = os.path.basename(os.readlink(pkg_info))
>> +
>> + for line in pkg_info_file.read().split("\n"):
>> + if re.match("^PN: .*$", line):
>> + pkg_dic[pkg]["PN"] = re.search("PN: (.*)$", line).group(1)
>> +
>> + if re.match("^PV: .*$", line):
>> + pkg_dic[pkg]["PV"] = re.search("PV: (.*)$", line).group(1)
>> +
>> + if re.match("^LICENSE_%s: (.*)$" % re.escape(pkg_lic_name)
>> + , line):
>> + pkg_dic[pkg]["LICENSE"] = re.search("^LICENSE_%s: (.*)$"
>> + % pkg_lic_name, line).group(1)
>> + elif re.match("^LICENSE: (.*)$", line):
>> + pkg_dic[pkg]["LICENSE"] = re.search("^LICENSE: (.*)$",
>> + line).group(1)
> Could you have a look at oe.packagedata.read_pkgdatafile(x) and see if
> the above could use that instead please?
>
>> + license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
>> + d.getVar('IMAGE_NAME', True), 'license.manifest')
>> + with open(license_manifest, "w") as license_file:
>> + for pkg in sorted(pkg_dic):
>> + license_file.write("PACKAGE NAME: %s\n" % pkg)
>> + license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
>> + license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
>> + license_file.write("LICENSE:")
>> +
>> + licenses = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
>> + licenses = re.sub(' *', ' ', licenses)
>> + for lic in licenses.split():
>> + lic = re.sub('\+', '', lic)
>> + lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
>> + pkg_dic[pkg]["PN"], "generic_%s" % lic)
>> + if not os.path.exists(lic_file):
>> + bb.warn("The license listed %s was not in the "\
>> + "licenses collected for recipe %s"
>> + % (lic, pkg_dic[pkg]["PN"]))
>> + license_file.write(" %s" % lic)
>> + license_file.write("\n\n")
>> +
>> + # Two options here:
>> + # - Just copy the manifest
>> + # - Copy the manifest and the license directories
>> + # With both options set we see a .5 M increase in core-image-minimal
>> + copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST', True)
>> + copy_lic_dirs = d.getVar('COPY_LIC_DIRS', True)
>> + if copy_lic_manifest == "1":
>> + rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'),
>> + 'usr', 'share', 'common-licenses')
>> + os.makedirs(rootfs_license_dir)
>> + rootfs_license_manifest = os.path.join(rootfs_license_dir,
>> + 'license.manifest')
>> + shutil.copyfile(license_manifest, rootfs_license_manifest)
>> +
>> + if copy_lic_dirs == "1":
>> + for pkg in sorted(pkg_dic):
>> + pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
>> + os.makedirs(pkg_rootfs_license_dir)
>> + pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
>> + pkg_dic[pkg]["PN"])
>> + licenses = os.listdir(pkg_license_dir)
>> + for lic in licenses:
>> + rootfs_license = os.path.join(rootfs_license_dir, lic)
>> + pkg_license = os.path.join(pkg_license_dir, lic)
>> + pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
>> +
>> + if re.match("^generic_.*$", lic):
>> + if not os.path.exists(pkg_rootfs_license):
>> + shutil.copyfile(pkg_license, rootfs_license)
>> +
>> + os.symlink(os.path.join('..', lic), pkg_rootfs_license)
>> + else:
>> + shutil.copyfile(pkg_license, pkg_rootfs_license)
> One small tweak we could make here is to hardlink these files. There are
> several copies of many of them and it would reduce image size a bit. I
> appreciate the original doesn't but I've been doing this in other places
> and this is probably another we should put on the list...
>
> Otherwise looks good, thanks.
>
> Richard
>
>
>
>
More information about the Openembedded-core
mailing list