[OE-core] [PATCH 6/7] image: Add locale archive optimisation
Khem Raj
raj.khem at gmail.com
Mon Aug 13 18:28:57 UTC 2018
On Mon, Aug 13, 2018 at 7:29 AM Richard Purdie
<richard.purdie at linuxfoundation.org> wrote:
>
> Refactor the locale archive function from the SDK to also make it work during
> general image creation. This reduces the size of the locales from 900MB to 220MB
> in core-image-lsb-sdk.
>
> The exception handling around subprocess was dropped as the standard subprocess
> exception printing is better handled than the catchall exception.
>
> Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
> ---
> meta/classes/image.bbclass | 2 +-
> meta/lib/oe/package_manager.py | 49 ++++++++++++++++++++++++++++++++++
> meta/lib/oe/sdk.py | 45 ++-----------------------------
> 3 files changed, 52 insertions(+), 44 deletions(-)
>
> diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
> index c3e73676dca..c0a2714288d 100644
> --- a/meta/classes/image.bbclass
> +++ b/meta/classes/image.bbclass
> @@ -22,7 +22,7 @@ POPULATE_SDK_POST_TARGET_COMMAND += "rootfs_sysroot_relativelinks; "
>
> LICENSE ?= "MIT"
> PACKAGES = ""
> -DEPENDS += "${@' '.join(["%s-qemuwrapper-cross" % m for m in d.getVar("MULTILIB_VARIANTS").split()])} qemuwrapper-cross depmodwrapper-cross"
> +DEPENDS += "${@' '.join(["%s-qemuwrapper-cross" % m for m in d.getVar("MULTILIB_VARIANTS").split()])} qemuwrapper-cross depmodwrapper-cross cross-localedef-native"
> RDEPENDS += "${PACKAGE_INSTALL} ${LINGUAS_INSTALL}"
> RRECOMMENDS += "${PACKAGE_INSTALL_ATTEMPTONLY}"
> PATH_prepend = "${@":".join(all_multilib_tune_values(d, 'STAGING_BINDIR_CROSS').split())}:"
> diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
> index 64c8a912160..6011e873503 100644
> --- a/meta/lib/oe/package_manager.py
> +++ b/meta/lib/oe/package_manager.py
> @@ -12,6 +12,7 @@ import oe.path
> import string
> from oe.gpg_sign import get_signer
> import hashlib
> +import fnmatch
>
> # this can be used by all PM backends to create the index files in parallel
> def create_index(arg):
> @@ -89,6 +90,47 @@ def failed_postinsts_warn(pkgs, log_path):
> If deferring to first boot wasn't the intent, then scriptlet failure may mean an issue in the recipe, or a regression elsewhere.
> Details of the failure are in %s.""" %(pkgs, log_path))
>
> +def generate_locale_archive(d, rootfs, target_arch, localedir):
> + # Pretty sure we don't need this for locale archive generation but
> + # keeping it to be safe...
> + locale_arch_options = { \
> + "arm": ["--uint32-align=4", "--little-endian"],
> + "armeb": ["--uint32-align=4", "--big-endian"],
> + "aarch64": ["--uint32-align=4", "--little-endian"],
> + "aarch64_be": ["--uint32-align=4", "--big-endian"],
> + "sh4": ["--uint32-align=4", "--big-endian"],
> + "powerpc": ["--uint32-align=4", "--big-endian"],
> + "powerpc64": ["--uint32-align=4", "--big-endian"],
> + "mips": ["--uint32-align=4", "--big-endian"],
> + "mipsisa32r6": ["--uint32-align=4", "--big-endian"],
> + "mips64": ["--uint32-align=4", "--big-endian"],
> + "mipsisa64r6": ["--uint32-align=4", "--big-endian"],
> + "mipsel": ["--uint32-align=4", "--little-endian"],
> + "mipsisa32r6el": ["--uint32-align=4", "--little-endian"],
> + "mips64el": ["--uint32-align=4", "--little-endian"],
> + "mipsisa64r6el": ["--uint32-align=4", "--little-endian"],
> + "i586": ["--uint32-align=4", "--little-endian"],
> + "i686": ["--uint32-align=4", "--little-endian"],
> + "x86_64": ["--uint32-align=4", "--little-endian"]
> + }
perhaps now we should also add risc-v here since its not limited to SDK alone
"riscv64": ["--uint32-align=4", "--little-endian"]
"riscv32": ["--uint32-align=4", "--little-endian"]
> + if target_arch in locale_arch_options:
> + arch_options = locale_arch_options[target_arch]
> + else:
> + bb.error("locale_arch_options not found for target_arch=" + target_arch)
> + bb.fatal("unknown arch:" + target_arch + " for locale_arch_options")
> +
> + # Need to set this so cross-localedef knows where the archive is
> + env = dict(os.environ)
> + env["LOCALEARCHIVE"] = oe.path.join(localedir, "locale-archive")
> +
> + for name in os.listdir(localedir):
> + path = os.path.join(localedir, name)
> + if os.path.isdir(path):
> + cmd = ["cross-localedef", "--verbose"]
> + cmd += arch_options
> + cmd += ["--add-to-archive", path]
> + subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT)
> +
> class Indexer(object, metaclass=ABCMeta):
> def __init__(self, d, deploy_dir):
> self.d = d
> @@ -536,6 +578,13 @@ class PackageManager(object, metaclass=ABCMeta):
> "'%s' returned %d:\n%s" %
> (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
>
> + target_arch = self.d.getVar('TARGET_ARCH')
> + localedir = oe.path.join(self.target_rootfs, self.d.getVar("libdir"), "locale")
> + if os.path.exists(localedir) and os.listdir(localedir):
> + generate_locale_archive(self.d, self.target_rootfs, target_arch, localedir)
> + # And now delete the binary locales
> + self.remove(fnmatch.filter(self.list_installed(), "glibc-binary-localedata-*"), False)
> +
> def deploy_dir_lock(self):
> if self.deploy_dir is None:
> raise RuntimeError("deploy_dir is not set!")
> diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py
> index 0d39ea8a91e..a579d9ae77f 100644
> --- a/meta/lib/oe/sdk.py
> +++ b/meta/lib/oe/sdk.py
> @@ -7,51 +7,10 @@ import shutil
> import glob
> import traceback
>
> -def generate_locale_archive(d, rootfs):
> - # Pretty sure we don't need this for SDK archive generation but
> - # keeping it to be safe...
> +def generate_locale_archive_sdk(d, rootfs):
> target_arch = d.getVar('SDK_ARCH')
> - locale_arch_options = { \
> - "arm": ["--uint32-align=4", "--little-endian"],
> - "armeb": ["--uint32-align=4", "--big-endian"],
> - "aarch64": ["--uint32-align=4", "--little-endian"],
> - "aarch64_be": ["--uint32-align=4", "--big-endian"],
> - "sh4": ["--uint32-align=4", "--big-endian"],
> - "powerpc": ["--uint32-align=4", "--big-endian"],
> - "powerpc64": ["--uint32-align=4", "--big-endian"],
> - "mips": ["--uint32-align=4", "--big-endian"],
> - "mipsisa32r6": ["--uint32-align=4", "--big-endian"],
> - "mips64": ["--uint32-align=4", "--big-endian"],
> - "mipsisa64r6": ["--uint32-align=4", "--big-endian"],
> - "mipsel": ["--uint32-align=4", "--little-endian"],
> - "mipsisa32r6el": ["--uint32-align=4", "--little-endian"],
> - "mips64el": ["--uint32-align=4", "--little-endian"],
> - "mipsisa64r6el": ["--uint32-align=4", "--little-endian"],
> - "i586": ["--uint32-align=4", "--little-endian"],
> - "i686": ["--uint32-align=4", "--little-endian"],
> - "x86_64": ["--uint32-align=4", "--little-endian"]
> - }
> - if target_arch in locale_arch_options:
> - arch_options = locale_arch_options[target_arch]
> - else:
> - bb.error("locale_arch_options not found for target_arch=" + target_arch)
> - bb.fatal("unknown arch:" + target_arch + " for locale_arch_options")
> -
> localedir = oe.path.join(rootfs, d.getVar("libdir_nativesdk"), "locale")
> - # Need to set this so cross-localedef knows where the archive is
> - env = dict(os.environ)
> - env["LOCALEARCHIVE"] = oe.path.join(localedir, "locale-archive")
> -
> - for name in os.listdir(localedir):
> - path = os.path.join(localedir, name)
> - if os.path.isdir(path):
> - try:
> - cmd = ["cross-localedef", "--verbose"]
> - cmd += arch_options
> - cmd += ["--add-to-archive", path]
> - subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT)
> - except Exception as e:
> - bb.fatal("Cannot create locale archive: %s" % e.output)
> + generate_locale_archive(d, rootfs, target_arch, localedir)
>
> class Sdk(object, metaclass=ABCMeta):
> def __init__(self, d, manifest_dir):
> --
> 2.17.1
>
> --
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core at lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/openembedded-core
More information about the Openembedded-core
mailing list