[oe] [PATCH V2] image_types.bbclass: Set 70% RAM memory usage limit and -T for xz

[akuster808]

On 2/9/19 10:57 AM, Khem Raj wrote:
> when building with opkg backend and huge packages e.g. chromium/llvm all
> going in parallel, memory pressure causes xz to catapult with
>
> do_package_write_ipk: Failed to create package, opkg-build failed with: xz: (stdin): Cannot allocate memory
>
> since there are many tasks going on in parallel, xz adds to memory pressure
> and it wants it all, put an upper limit for memory xz can use
>
> We add a variable XZ_MAXRAM with 30% of RAM limit and can be customized
> if builders have more memory one can set it like
err, wrong mailing list.. I think you self corrected : )

- armin
> XZ_DEFAULTS = "-M 0 -T 0"
>
> Signed-off-by: Khem Raj <raj.khem at gmail.com>
> ---
> V2: Consider all locations of xz use
>
>  meta/classes/image_types.bbclass       | 9 ++++-----
>  meta/classes/package_ipk.bbclass       | 2 +-
>  meta/classes/populate_sdk_base.bbclass | 2 +-
>  meta/conf/bitbake.conf                 | 3 +++
>  4 files changed, 9 insertions(+), 7 deletions(-)
>
> diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass
> index 70bd315306..b12420310f 100644
> --- a/meta/classes/image_types.bbclass
> +++ b/meta/classes/image_types.bbclass
> @@ -56,7 +56,6 @@ def imagetypes_getdepends(d):
>  
>  XZ_COMPRESSION_LEVEL ?= "-3"
>  XZ_INTEGRITY_CHECK ?= "crc32"
> -XZ_THREADS ?= "-T 0"
>  
>  ZIP_COMPRESSION_LEVEL ?= "-9"
>  
> @@ -152,12 +151,12 @@ UBI_VOLNAME ?= "${MACHINE}-rootfs"
>  multiubi_mkfs() {
>  	local mkubifs_args="$1"
>  	local ubinize_args="$2"
> -    
> +
>          # Added prompt error message for ubi and ubifs image creation.
>          if [ -z "$mkubifs_args" ] || [ -z "$ubinize_args" ]; then
>              bbfatal "MKUBIFS_ARGS and UBINIZE_ARGS have to be set, see http://www.linux-mtd.infradead.org/faq/ubifs.html for details"
>          fi
> -    
> +
>  	if [ -z "$3" ]; then
>  		local vname=""
>  	else
> @@ -284,7 +283,7 @@ CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha25
>  CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
>  CONVERSION_CMD_gz = "pigz -f -9 -n -c ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz"
>  CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
> -CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_THREADS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz"
> +CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz"
>  CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4"
>  CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
>  CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
> @@ -317,7 +316,7 @@ CONVERSION_DEPENDS_qcow2 = "qemu-native"
>  RUNNABLE_IMAGE_TYPES ?= "ext2 ext3 ext4"
>  RUNNABLE_MACHINE_PATTERNS ?= "qemu"
>  
> -DEPLOYABLE_IMAGE_TYPES ?= "hddimg iso" 
> +DEPLOYABLE_IMAGE_TYPES ?= "hddimg iso"
>  
>  # The IMAGE_TYPES_MASKED variable will be used to mask out from the IMAGE_FSTYPES,
>  # images that will not be built at do_rootfs time: vmdk, vdi, qcow2, hdddirect, hddimg, iso, etc.
> diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
> index 508b7dcaff..d1b317b42b 100644
> --- a/meta/classes/package_ipk.bbclass
> +++ b/meta/classes/package_ipk.bbclass
> @@ -8,7 +8,7 @@ IPKGCONF_SDK =  "${WORKDIR}/opkg-sdk.conf"
>  PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"
>  
>  # Program to be used to build opkg packages
> -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "--threads 0"'
> +OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'
>  
>  OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
>  OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
> diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass
> index 80fa443e4c..ebc30d39b3 100644
> --- a/meta/classes/populate_sdk_base.bbclass
> +++ b/meta/classes/populate_sdk_base.bbclass
> @@ -226,7 +226,7 @@ fakeroot tar_sdk() {
>  	# Package it up
>  	mkdir -p ${SDKDEPLOYDIR}
>  	cd ${SDK_OUTPUT}/${SDKPATH}
> -	tar ${SDKTAROPTS} -cf - . | xz -T 0 > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz
> +	tar ${SDKTAROPTS} -cf - . | xz ${XZ_DEFAULTS} > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz
>  }
>  
>  TOOLCHAIN_SHAR_EXT_TMPL ?= "${COREBASE}/meta/files/toolchain-shar-extract.sh"
> diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
> index 9452d5ac4f..3f5d84d348 100644
> --- a/meta/conf/bitbake.conf
> +++ b/meta/conf/bitbake.conf
> @@ -791,6 +791,9 @@ BB_NUMBER_THREADS ?= "${@oe.utils.cpu_count()}"
>  # Default to setting automatically based on cpu count
>  PARALLEL_MAKE ?= "-j ${@oe.utils.cpu_count()}"
>  
> +# Default parallelism and resource usage for xz
> +XZ_DEFAULTS ?= "--memlimit=30% --threads=${BB_NUMBER_THREADS}"
> +
>  ##################################################################
>  # Magic Cookie for SANITY CHECK
>  ##################################################################