[OE-core] [PATCHv2] test-dependencies: add simple script to detect missing or autoenabled dependencies
Martin Jansa
martin.jansa at gmail.com
Wed Jul 10 12:38:13 UTC 2013
Signed-off-by: Martin Jansa <Martin.Jansa at gmail.com>
---
scripts/test-dependencies.sh | 253 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 253 insertions(+)
create mode 100755 scripts/test-dependencies.sh
diff --git a/scripts/test-dependencies.sh b/scripts/test-dependencies.sh
new file mode 100755
index 0000000..405c14e
--- /dev/null
+++ b/scripts/test-dependencies.sh
@@ -0,0 +1,253 @@
+#!/bin/sh
+
+# Author: Martin Jansa <martin.jansa at gmail.com>
+#
+# Copyright (c) 2013 Martin Jansa <Martin.Jansa at gmail.com>
+
+# Used to detect missing dependencies or automagically
+# enabled dependencies which aren't explicitly enabled
+# or disabled.
+
+# It does 3 builds of <target>
+# 1st to populate sstate-cache directory and sysroot
+# 2nd to rebuild each recipe with every possible
+# dependency found in sysroot (which stays populated
+# from 1st build
+# 3rd to rebuild each recipe only with dependencies defined
+# in DEPENDS
+# 4th (optional) repeat build like 3rd to make sure that
+# minimal versions of dependencies defined in DEPENDS
+# is also enough
+
+# Global vars
+tmpdir=
+targets=
+recipes=
+buildhistory=
+buildtype=
+default_targets="world"
+default_buildhistory="buildhistory"
+default_buildtype="1 2 3 c"
+
+usage () {
+ cat << EOF
+Welcome to utility to detect missing or autoenabled dependencies.
+WARNING: this utility will completely remove your tmpdir (make sure
+ you don't have important buildhistory or persistent dir there).
+$0 <OPTION>
+
+Options:
+ -h, --help
+ Display this help and exit.
+
+ --tmpdir=<tmpdir>
+ Specify tmpdir, will use the environment variable TMPDIR if it is not specified.
+ Something like /OE/oe-core/tmp-eglibc (no / at the end).
+
+ --targets=<targets>
+ List of targets separated by space, will use the environment variable TARGETS if it is not specified.
+ It will run "bitbake <targets>" to populate sysroots.
+ Default value is "world".
+
+ --recipes=<recipes>
+ File with list of recipes we want to rebuild with minimal and maximal sysroot.
+ Will use the environment variable RECIPES if it is not specified.
+ Default value will use all packages ever recorded in buildhistory directory.
+
+ --buildhistory=<buildhistory>
+ Path to buildhistory directory, it needs to be enabled in your config,
+ because it's used to detect different dependencies and to create list
+ of recipes to rebuild when it's not specified.
+ Will use the environment variable BUILDHISTORY if it is not specified.
+ Default value is "buildhistory"
+
+ --buildtype=<buildtype>
+ There are 4 types of build:
+ 1: build to populate sstate-cache directory and sysroot
+ 2: build to rebuild each recipe with every possible dep
+ 3: build to rebuild each recipe with minimal dependencies
+ 4: build to rebuild each recipe again with minimal dependencies
+ c: compare buildhistory directories from build 2 and 3
+ Will use the environment variable BUILDTYPE if it is not specified.
+ Default value is "1 2 3 c", order is important, type 4 is optional.
+EOF
+}
+
+# Print error information and exit.
+echo_error () {
+ echo "ERROR: $1" >&2
+ exit 1
+}
+
+while [ -n "$1" ]; do
+ case $1 in
+ --tmpdir=*)
+ tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e`
+ [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir"
+ shift
+ ;;
+ --targets=*)
+ targets=`echo $1 | sed -e 's#^--targets="*\([^"]*\)"*#\1#'`
+ shift
+ ;;
+ --recipes=*)
+ recipes=`echo $1 | sed -e 's#^--recipes="*\([^"]*\)"*#\1#'`
+ shift
+ ;;
+ --buildhistory=*)
+ buildhistory=`echo $1 | sed -e 's#^--buildhistory="*\([^"]*\)"*#\1#'`
+ shift
+ ;;
+ --buildtype=*)
+ buildtype=`echo $1 | sed -e 's#^--buildtype="*\([^"]*\)"*#\1#'`
+ shift
+ ;;
+ --help|-h)
+ usage
+ exit 0
+ ;;
+ *)
+ echo "Invalid arguments $*"
+ echo_error "Try '$0 -h' for more information."
+ ;;
+ esac
+done
+
+# tmpdir directory, use environment variable TMPDIR
+# if it was not specified, otherwise, error.
+[ -n "$tmpdir" ] || tmpdir=$TMPDIR
+[ -n "$tmpdir" ] || echo_error "No tmpdir found!"
+[ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\""
+[ -n "$targets" ] || targets=$TARGETS
+[ -n "$targets" ] || targets=$default_targets
+[ -n "$recipes" ] || recipes=$RECIPES
+[ -n "$recipes" -a ! -f "$recipes" ] && echo_error "Invalid file with list of recipes to rebuild"
+[ -n "$recipes" ] || echo "All packages ever recorded in buildhistory directory will be rebuilt"
+[ -n "$buildhistory" ] || buildhistory=$BUILDHISTORY
+[ -n "$buildhistory" ] || buildhistory=$default_buildhistory
+[ -d "$buildhistory" ] || echo_error "Invalid buildhistory directory \"$buildhistory\""
+[ -n "$buildtype" ] || buildtype=$BUILDTYPE
+[ -n "$buildtype" ] || buildtype=$default_buildtype
+echo "$buildtype" | grep -v '^[1234c ]*$' && echo_error "Invalid buildtype \"$buildtype\", only some combination of 1, 2, 3, 4, c separated by space is allowed"
+
+OUTPUT_BASE=test-dependencies/`date "+%s"`
+
+build_all() {
+ echo "===== 1st build to populate sstate-cache directory and sysroot ====="
+ OUTPUT1=${OUTPUT_BASE}/${TYPE}_all
+ mkdir -p ${OUTPUT1}
+ echo "Logs will be stored in ${OUTPUT1} directory"
+ bitbake -k $targets | tee -a ${OUTPUT1}/complete.log
+}
+
+build_every_recipe() {
+ if [ "${TYPE}" = "2" ] ; then
+ echo "===== 2nd build to rebuild each recipe with every possible dep ====="
+ OUTPUT_MAX=${OUTPUT_BASE}/${TYPE}_max
+ OUTPUTB=${OUTPUT_MAX}
+ else
+ echo "===== 3rd or 4th build to rebuild each recipe with minimal dependencies ====="
+ OUTPUT_MIN=${OUTPUT_BASE}/${TYPE}_min
+ OUTPUTB=${OUTPUT_MIN}
+ fi
+
+ mkdir -p ${OUTPUTB} ${OUTPUTB}/failed ${OUTPUTB}/ok
+ echo "Logs will be stored in ${OUTPUTB} directory"
+ if [ -z "$recipes" ]; then
+ ls -d $buildhistory/packages/*/* | xargs -n 1 basename | sort -u > ${OUTPUTB}/recipe.list
+ recipes=${OUTPUTB}/recipe.list
+ fi
+ if [ "${TYPE}" != "2" ] ; then
+ echo "!!!Removing tmpdir \"$tmpdir\"!!!"
+ rm -rf $tmpdir/cache $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null
+ fi
+ i=1
+ count=`cat $recipes | wc -l`
+ for recipe in `cat $recipes`; do
+ echo "Building recipe: ${recipe} ($i/$count)"
+ bitbake -c cleansstate ${recipe} > ${OUTPUTB}/log.${recipe} 2>&1;
+ bitbake ${recipe} >> ${OUTPUTB}/log.${recipe} 2>&1;
+ grep "ERROR: Task.*failed" ${OUTPUTB}/log.${recipe} && mv ${OUTPUTB}/log.${recipe} ${OUTPUTB}/failed/${recipe} || mv ${OUTPUTB}/log.${recipe} ${OUTPUTB}/ok/${recipe}
+ if [ "${TYPE}" != "2" ] ; then
+ rm -rf $tmpdir/cache $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null
+ fi
+ i=`expr $i + 1`
+ done
+ echo "Copying buildhistory/packages to ${OUTPUTB}"
+ cp -ra $buildhistory/packages ${OUTPUTB}
+ # This will be usefull to see which library is pulling new dependency
+ echo "Copying do_package logs to ${OUTPUTB}/do_package/"
+ mkdir ${OUTPUTB}/do_package
+ find $tmpdir/work/ -name log.do_package | while read f; do
+ # pn is 3 levels back, but we don't know if there is just one log per pn (only one arch and version)
+ # dest=`echo $f | sed 's#^.*/\([^/]*\)/\([^/]*\)/\([^/]*\)/log.do_package#\1#g'`
+ dest=`echo $f | sed "s#$tmpdir/work/##g; s#/#_#g"`
+ cp $f ${OUTPUTB}/do_package/$dest
+ done
+ grep "ERROR: Task.*failed" ${OUTPUTB}/failed/*
+}
+
+compare_deps() {
+ # you can run just compare task with command like this
+ # OUTPUT_MAX=test-dependencies/1373140172/2_max \
+ # OUTPUT_MIN=test-dependencies/1373140172/3_min \
+ # OUTPUTC=test-dependencies/1373140172 \
+ # openembedded-core/scripts/test-dependencies.sh --tmpdir=tmp-eglibc --targets=glib-2.0 --recipes=recipe_list --buildtype=c
+ echo "===== Compare dependencies recorded in \"${OUTPUT_MAX}\" and \"${OUTPUT_MIN}\" ====="
+ [ -n "${OUTPUTC}" ] || OUTPUTC=${OUTPUT_BASE}
+ mkdir -p ${OUTPUTC}
+ OUTPUT_FILE=${OUTPUTC}/dependency-changes
+ echo "Differences will be stored in ${OUTPUT_FILE}, dot is shown for every 100 of checked packages"
+ echo > ${OUTPUT_FILE}
+
+ [ -d ${OUTPUT_MAX} ] || echo_error "Directory with output from build 2 \"${OUTPUT_MAX}\" does not exist"
+ [ -d ${OUTPUT_MIN} ] || echo_error "Directory with output from build 3 \"${OUTPUT_MIN}\" does not exist"
+ [ -d ${OUTPUT_MAX}/packages/ ] || echo_error "Directory with packages from build 2 \"${OUTPUT_MAX}/packages/\" does not exist"
+ [ -d ${OUTPUT_MIN}/packages/ ] || echo_error "Directory with packages from build 3 \"${OUTPUT_MIN}/packages/\" does not exist"
+ i=0
+ find ${OUTPUT_MAX}/packages/ -name latest | sed "s#${OUTPUT_MAX}/##g" | while read pkg; do
+ max_pkg=${OUTPUT_MAX}/${pkg}
+ min_pkg=${OUTPUT_MIN}/${pkg}
+ if [ ! -f "${min_pkg}" ] ; then
+ echo "ERROR: ${min_pkg} doesn't exist" | tee -a ${OUTPUT_FILE}
+ continue
+ fi
+ # strip version information in parenthesis
+ max_deps=`grep "^RDEPENDS = " ${max_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'`
+ min_deps=`grep "^RDEPENDS = " ${min_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'`
+ if [ "$i" = 100 ] ; then
+ echo -n "." # cheap progressbar
+ i=0
+ fi
+ if [ "${max_deps}" = "${min_deps}" ] ; then
+ # it's annoying long, but at least it's showing some progress, warnings are grepped at the end
+ echo "NOTE: ${pkg} dependencies weren't changed" >> ${OUTPUT_FILE}
+ else
+ missing_deps=
+ for dep in ${max_deps}; do
+ echo "${min_deps}" | grep -q " ${dep} " || missing_deps="${missing_deps} ${dep}"
+ done
+ if [ -n "${missing_deps}" ] ; then
+ echo # to get rid of dots on last line
+ echo "WARN: ${pkg} lost dependency on ${missing_deps}" | tee -a ${OUTPUT_FILE}
+ fi
+ fi
+ i=`expr $i + 1`
+ done
+ echo # to get rid of dots on last line
+ echo "Found differences: "
+ grep "^WARN: " ${OUTPUT_FILE}
+ echo "Found errors: "
+ grep "^ERROR: " ${OUTPUT_FILE}
+}
+
+for TYPE in $buildtype; do
+ case ${TYPE} in
+ 1) build_all;;
+ 2) build_every_recipe;;
+ 3) build_every_recipe;;
+ 4) build_every_recipe;;
+ c) compare_deps;;
+ *) echo_error "Invalid buildtype \"$TYPE\""
+ esac
+done
--
1.8.2.1
More information about the Openembedded-core
mailing list