[OE-core] [PATCHv2] test-dependencies: add simple script to detect missing or autoenabled dependencies
Martin Jansa
martin.jansa at gmail.com
Wed Jul 10 12:41:15 UTC 2013
On Wed, Jul 10, 2013 at 02:38:13PM +0200, Martin Jansa wrote:
> Signed-off-by: Martin Jansa <Martin.Jansa at gmail.com>
> ---
> scripts/test-dependencies.sh | 253 +++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 253 insertions(+)
> create mode 100755 scripts/test-dependencies.sh
Build with last version of this script is still running (after 37 hours)
Building recipe: gnome-desktop (485/1848)
but I've tested it in smaller builds and I'm happy with current
functionality, I'll send found issues in tomorrow or day after that when
the build is complete.
> diff --git a/scripts/test-dependencies.sh b/scripts/test-dependencies.sh
> new file mode 100755
> index 0000000..405c14e
> --- /dev/null
> +++ b/scripts/test-dependencies.sh
> @@ -0,0 +1,253 @@
> +#!/bin/sh
> +
> +# Author: Martin Jansa <martin.jansa at gmail.com>
> +#
> +# Copyright (c) 2013 Martin Jansa <Martin.Jansa at gmail.com>
> +
> +# Used to detect missing dependencies or automagically
> +# enabled dependencies which aren't explicitly enabled
> +# or disabled.
> +
> +# It does 3 builds of <target>
> +# 1st to populate sstate-cache directory and sysroot
> +# 2nd to rebuild each recipe with every possible
> +# dependency found in sysroot (which stays populated
> +# from 1st build
> +# 3rd to rebuild each recipe only with dependencies defined
> +# in DEPENDS
> +# 4th (optional) repeat build like 3rd to make sure that
> +# minimal versions of dependencies defined in DEPENDS
> +# is also enough
> +
> +# Global vars
> +tmpdir=
> +targets=
> +recipes=
> +buildhistory=
> +buildtype=
> +default_targets="world"
> +default_buildhistory="buildhistory"
> +default_buildtype="1 2 3 c"
> +
> +usage () {
> + cat << EOF
> +Welcome to utility to detect missing or autoenabled dependencies.
> +WARNING: this utility will completely remove your tmpdir (make sure
> + you don't have important buildhistory or persistent dir there).
> +$0 <OPTION>
> +
> +Options:
> + -h, --help
> + Display this help and exit.
> +
> + --tmpdir=<tmpdir>
> + Specify tmpdir, will use the environment variable TMPDIR if it is not specified.
> + Something like /OE/oe-core/tmp-eglibc (no / at the end).
> +
> + --targets=<targets>
> + List of targets separated by space, will use the environment variable TARGETS if it is not specified.
> + It will run "bitbake <targets>" to populate sysroots.
> + Default value is "world".
> +
> + --recipes=<recipes>
> + File with list of recipes we want to rebuild with minimal and maximal sysroot.
> + Will use the environment variable RECIPES if it is not specified.
> + Default value will use all packages ever recorded in buildhistory directory.
> +
> + --buildhistory=<buildhistory>
> + Path to buildhistory directory, it needs to be enabled in your config,
> + because it's used to detect different dependencies and to create list
> + of recipes to rebuild when it's not specified.
> + Will use the environment variable BUILDHISTORY if it is not specified.
> + Default value is "buildhistory"
> +
> + --buildtype=<buildtype>
> + There are 4 types of build:
> + 1: build to populate sstate-cache directory and sysroot
> + 2: build to rebuild each recipe with every possible dep
> + 3: build to rebuild each recipe with minimal dependencies
> + 4: build to rebuild each recipe again with minimal dependencies
> + c: compare buildhistory directories from build 2 and 3
> + Will use the environment variable BUILDTYPE if it is not specified.
> + Default value is "1 2 3 c", order is important, type 4 is optional.
> +EOF
> +}
> +
> +# Print error information and exit.
> +echo_error () {
> + echo "ERROR: $1" >&2
> + exit 1
> +}
> +
> +while [ -n "$1" ]; do
> + case $1 in
> + --tmpdir=*)
> + tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e`
> + [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir"
> + shift
> + ;;
> + --targets=*)
> + targets=`echo $1 | sed -e 's#^--targets="*\([^"]*\)"*#\1#'`
> + shift
> + ;;
> + --recipes=*)
> + recipes=`echo $1 | sed -e 's#^--recipes="*\([^"]*\)"*#\1#'`
> + shift
> + ;;
> + --buildhistory=*)
> + buildhistory=`echo $1 | sed -e 's#^--buildhistory="*\([^"]*\)"*#\1#'`
> + shift
> + ;;
> + --buildtype=*)
> + buildtype=`echo $1 | sed -e 's#^--buildtype="*\([^"]*\)"*#\1#'`
> + shift
> + ;;
> + --help|-h)
> + usage
> + exit 0
> + ;;
> + *)
> + echo "Invalid arguments $*"
> + echo_error "Try '$0 -h' for more information."
> + ;;
> + esac
> +done
> +
> +# tmpdir directory, use environment variable TMPDIR
> +# if it was not specified, otherwise, error.
> +[ -n "$tmpdir" ] || tmpdir=$TMPDIR
> +[ -n "$tmpdir" ] || echo_error "No tmpdir found!"
> +[ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\""
> +[ -n "$targets" ] || targets=$TARGETS
> +[ -n "$targets" ] || targets=$default_targets
> +[ -n "$recipes" ] || recipes=$RECIPES
> +[ -n "$recipes" -a ! -f "$recipes" ] && echo_error "Invalid file with list of recipes to rebuild"
> +[ -n "$recipes" ] || echo "All packages ever recorded in buildhistory directory will be rebuilt"
> +[ -n "$buildhistory" ] || buildhistory=$BUILDHISTORY
> +[ -n "$buildhistory" ] || buildhistory=$default_buildhistory
> +[ -d "$buildhistory" ] || echo_error "Invalid buildhistory directory \"$buildhistory\""
> +[ -n "$buildtype" ] || buildtype=$BUILDTYPE
> +[ -n "$buildtype" ] || buildtype=$default_buildtype
> +echo "$buildtype" | grep -v '^[1234c ]*$' && echo_error "Invalid buildtype \"$buildtype\", only some combination of 1, 2, 3, 4, c separated by space is allowed"
> +
> +OUTPUT_BASE=test-dependencies/`date "+%s"`
> +
> +build_all() {
> + echo "===== 1st build to populate sstate-cache directory and sysroot ====="
> + OUTPUT1=${OUTPUT_BASE}/${TYPE}_all
> + mkdir -p ${OUTPUT1}
> + echo "Logs will be stored in ${OUTPUT1} directory"
> + bitbake -k $targets | tee -a ${OUTPUT1}/complete.log
> +}
> +
> +build_every_recipe() {
> + if [ "${TYPE}" = "2" ] ; then
> + echo "===== 2nd build to rebuild each recipe with every possible dep ====="
> + OUTPUT_MAX=${OUTPUT_BASE}/${TYPE}_max
> + OUTPUTB=${OUTPUT_MAX}
> + else
> + echo "===== 3rd or 4th build to rebuild each recipe with minimal dependencies ====="
> + OUTPUT_MIN=${OUTPUT_BASE}/${TYPE}_min
> + OUTPUTB=${OUTPUT_MIN}
> + fi
> +
> + mkdir -p ${OUTPUTB} ${OUTPUTB}/failed ${OUTPUTB}/ok
> + echo "Logs will be stored in ${OUTPUTB} directory"
> + if [ -z "$recipes" ]; then
> + ls -d $buildhistory/packages/*/* | xargs -n 1 basename | sort -u > ${OUTPUTB}/recipe.list
> + recipes=${OUTPUTB}/recipe.list
> + fi
> + if [ "${TYPE}" != "2" ] ; then
> + echo "!!!Removing tmpdir \"$tmpdir\"!!!"
> + rm -rf $tmpdir/cache $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null
> + fi
> + i=1
> + count=`cat $recipes | wc -l`
> + for recipe in `cat $recipes`; do
> + echo "Building recipe: ${recipe} ($i/$count)"
> + bitbake -c cleansstate ${recipe} > ${OUTPUTB}/log.${recipe} 2>&1;
> + bitbake ${recipe} >> ${OUTPUTB}/log.${recipe} 2>&1;
> + grep "ERROR: Task.*failed" ${OUTPUTB}/log.${recipe} && mv ${OUTPUTB}/log.${recipe} ${OUTPUTB}/failed/${recipe} || mv ${OUTPUTB}/log.${recipe} ${OUTPUTB}/ok/${recipe}
> + if [ "${TYPE}" != "2" ] ; then
> + rm -rf $tmpdir/cache $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null
> + fi
> + i=`expr $i + 1`
> + done
> + echo "Copying buildhistory/packages to ${OUTPUTB}"
> + cp -ra $buildhistory/packages ${OUTPUTB}
> + # This will be usefull to see which library is pulling new dependency
> + echo "Copying do_package logs to ${OUTPUTB}/do_package/"
> + mkdir ${OUTPUTB}/do_package
> + find $tmpdir/work/ -name log.do_package | while read f; do
> + # pn is 3 levels back, but we don't know if there is just one log per pn (only one arch and version)
> + # dest=`echo $f | sed 's#^.*/\([^/]*\)/\([^/]*\)/\([^/]*\)/log.do_package#\1#g'`
> + dest=`echo $f | sed "s#$tmpdir/work/##g; s#/#_#g"`
> + cp $f ${OUTPUTB}/do_package/$dest
> + done
> + grep "ERROR: Task.*failed" ${OUTPUTB}/failed/*
> +}
> +
> +compare_deps() {
> + # you can run just compare task with command like this
> + # OUTPUT_MAX=test-dependencies/1373140172/2_max \
> + # OUTPUT_MIN=test-dependencies/1373140172/3_min \
> + # OUTPUTC=test-dependencies/1373140172 \
> + # openembedded-core/scripts/test-dependencies.sh --tmpdir=tmp-eglibc --targets=glib-2.0 --recipes=recipe_list --buildtype=c
> + echo "===== Compare dependencies recorded in \"${OUTPUT_MAX}\" and \"${OUTPUT_MIN}\" ====="
> + [ -n "${OUTPUTC}" ] || OUTPUTC=${OUTPUT_BASE}
> + mkdir -p ${OUTPUTC}
> + OUTPUT_FILE=${OUTPUTC}/dependency-changes
> + echo "Differences will be stored in ${OUTPUT_FILE}, dot is shown for every 100 of checked packages"
> + echo > ${OUTPUT_FILE}
> +
> + [ -d ${OUTPUT_MAX} ] || echo_error "Directory with output from build 2 \"${OUTPUT_MAX}\" does not exist"
> + [ -d ${OUTPUT_MIN} ] || echo_error "Directory with output from build 3 \"${OUTPUT_MIN}\" does not exist"
> + [ -d ${OUTPUT_MAX}/packages/ ] || echo_error "Directory with packages from build 2 \"${OUTPUT_MAX}/packages/\" does not exist"
> + [ -d ${OUTPUT_MIN}/packages/ ] || echo_error "Directory with packages from build 3 \"${OUTPUT_MIN}/packages/\" does not exist"
> + i=0
> + find ${OUTPUT_MAX}/packages/ -name latest | sed "s#${OUTPUT_MAX}/##g" | while read pkg; do
> + max_pkg=${OUTPUT_MAX}/${pkg}
> + min_pkg=${OUTPUT_MIN}/${pkg}
> + if [ ! -f "${min_pkg}" ] ; then
> + echo "ERROR: ${min_pkg} doesn't exist" | tee -a ${OUTPUT_FILE}
> + continue
> + fi
> + # strip version information in parenthesis
> + max_deps=`grep "^RDEPENDS = " ${max_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'`
> + min_deps=`grep "^RDEPENDS = " ${min_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'`
> + if [ "$i" = 100 ] ; then
> + echo -n "." # cheap progressbar
> + i=0
> + fi
> + if [ "${max_deps}" = "${min_deps}" ] ; then
> + # it's annoying long, but at least it's showing some progress, warnings are grepped at the end
> + echo "NOTE: ${pkg} dependencies weren't changed" >> ${OUTPUT_FILE}
> + else
> + missing_deps=
> + for dep in ${max_deps}; do
> + echo "${min_deps}" | grep -q " ${dep} " || missing_deps="${missing_deps} ${dep}"
> + done
> + if [ -n "${missing_deps}" ] ; then
> + echo # to get rid of dots on last line
> + echo "WARN: ${pkg} lost dependency on ${missing_deps}" | tee -a ${OUTPUT_FILE}
> + fi
> + fi
> + i=`expr $i + 1`
> + done
> + echo # to get rid of dots on last line
> + echo "Found differences: "
> + grep "^WARN: " ${OUTPUT_FILE}
> + echo "Found errors: "
> + grep "^ERROR: " ${OUTPUT_FILE}
> +}
> +
> +for TYPE in $buildtype; do
> + case ${TYPE} in
> + 1) build_all;;
> + 2) build_every_recipe;;
> + 3) build_every_recipe;;
> + 4) build_every_recipe;;
> + c) compare_deps;;
> + *) echo_error "Invalid buildtype \"$TYPE\""
> + esac
> +done
> --
> 1.8.2.1
>
--
Martin 'JaMa' Jansa jabber: Martin.Jansa at gmail.com
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 198 bytes
Desc: Digital signature
URL: <http://lists.openembedded.org/pipermail/openembedded-core/attachments/20130710/4215a1f6/attachment-0002.sig>
More information about the Openembedded-core
mailing list