[OE-core] [PATCH 2/2] archiver/package_rpm: Drop srpm source archiver mode
Mark Hatle
mark.hatle at windriver.com
Tue Feb 5 15:28:45 UTC 2019
On 2/5/19 7:44 AM, Richard Purdie wrote:
> Creating these 'srpms' misleads users into thinking there is a way of
> rebuilding the package when there isn't. They amount to a glorified
> tarball. Worse, the code is full of races. In particular, do_package_write_rpm
> is writing out files outside of its sstate meaning the source archives
> may sometimes appear and sometimes disappear.
>
> In my view the simplest way of dealing with this is to admit these are not
> source rpms and stop creating them. The tarballs from the source archiver
> are perfectly reasonable.
To start with, yes SRPMs are absolutely a glorified archive and are NOT expected
to be buildable out of the build system.
However, we do have users who want SRMS output format for some reason. I
suspect the reason for this is that they have additional tooling that processing
both the RPMS and SRPMS to ensure that they have meet license terms by releasing
corresponding sources to any binary packages they may be publishing for
package-based upgrade situations.
Most of the users of this that I know of are using it specifically for:
- package metadata in the RPM format
- include original unmodified source and patches
- include run files for each 'operation'
- include log files for the output of each operation
What I thought was happening was:
do_fetch
do_unpack
...
do_package
do_archiver
do_package_write_rpm
do_deploy_archives
So in the sstate case you could end up skipping all the way to the do_package
(which would have the problem of the build instructions and build output being
missing.
So even in an sstate case you'd end up with something like:
do_<setscene operations>
do_<archiver setscene operations>
<some task here that has to be re-run>
do_package
do_archiver (and maybe scene operations here to restore logs/run files)
do_package_write_rpm
do_deploy_archives
Clearly that is not happening based on your previous message.
If we can generate the archiver tarball with a consistent set of contents in
parallel with the do_package operation, then we SHOULD be able to adjust it to
also be fed into the package generation in do_package_write_rpm as well.
I don't think it matters if the input to the do_package_write_rpm is an archiver
tarball or the individual items BTW. In the end, if the archiver works and can
be run -before- the do_package_write_rpm, then we should be able to continue to
do this for those people who need it. (I see no other purpose for the SRPM
other then external tooling of some kind trying to either automatically download
matching SRPMs, or doing validation that each binary package has a corresponding
SRPM.)
We can take this offline if you'd like. I won't have time in the short term
(next two weeks) to help with the archiver unfortunately.. but if the issues
with that can be corrected, then I can certainly help get the SRPM part re-working.
--Mark
> Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
> ---
> meta/classes/archiver.bbclass | 24 +----------
> meta/classes/package_rpm.bbclass | 43 --------------------
> meta/lib/oeqa/selftest/cases/archiver.py | 13 ------
> meta/lib/oeqa/selftest/cases/buildoptions.py | 15 -------
> 4 files changed, 2 insertions(+), 93 deletions(-)
>
> diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass
> index e321a0e3621..772b10a412a 100644
> --- a/meta/classes/archiver.bbclass
> +++ b/meta/classes/archiver.bbclass
> @@ -12,14 +12,12 @@
> # 5) The environment data, similar to 'bitbake -e recipe':
> # ARCHIVER_MODE[dumpdata] = "1"
> # 6) The recipe (.bb and .inc): ARCHIVER_MODE[recipe] = "1"
> -# 7) Whether output the .src.rpm package:
> -# ARCHIVER_MODE[srpm] = "1"
> -# 8) Filter the license, the recipe whose license in
> +# 7) Filter the license, the recipe whose license in
> # COPYLEFT_LICENSE_INCLUDE will be included, and in
> # COPYLEFT_LICENSE_EXCLUDE will be excluded.
> # COPYLEFT_LICENSE_INCLUDE = 'GPL* LGPL*'
> # COPYLEFT_LICENSE_EXCLUDE = 'CLOSED Proprietary'
> -# 9) The recipe type that will be archived:
> +# 8) The recipe type that will be archived:
> # COPYLEFT_RECIPE_TYPES = 'target'
> #
>
> @@ -27,7 +25,6 @@
> COPYLEFT_RECIPE_TYPES ?= 'target native nativesdk cross crosssdk cross-canadian'
> inherit copyleft_filter
>
> -ARCHIVER_MODE[srpm] ?= "0"
> ARCHIVER_MODE[src] ?= "patched"
> ARCHIVER_MODE[diff] ?= "0"
> ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches"
> @@ -112,23 +109,6 @@ python () {
>
> if ar_recipe == "1":
> d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_recipe' % pn)
> -
> - # Output the SRPM package
> - if d.getVarFlag('ARCHIVER_MODE', 'srpm') == "1" and d.getVar('PACKAGES'):
> - if "package_rpm" in d.getVar('PACKAGE_CLASSES'):
> - d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_package_write_rpm' % pn)
> - if ar_dumpdata == "1":
> - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_dumpdata' % pn)
> - if ar_recipe == "1":
> - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_recipe' % pn)
> - if ar_src == "original":
> - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_original' % pn)
> - elif ar_src == "patched":
> - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_patched' % pn)
> - elif ar_src == "configured":
> - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_configured' % pn)
> - else:
> - bb.fatal("ARCHIVER_MODE[srpm] needs package_rpm in PACKAGE_CLASSES")
> }
>
> # Take all the sources for a recipe and puts them in WORKDIR/archiver-work/.
> diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass
> index 5dafb3e7cb6..72fb2f602e6 100644
> --- a/meta/classes/package_rpm.bbclass
> +++ b/meta/classes/package_rpm.bbclass
> @@ -97,34 +97,6 @@ def write_rpm_perfiledata(srcname, d):
> python write_specfile () {
> import oe.packagedata
>
> - # append information for logs and patches to %prep
> - def add_prep(d,spec_files_bottom):
> - if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d):
> - spec_files_bottom.append('%%prep -n %s' % d.getVar('PN') )
> - spec_files_bottom.append('%s' % "echo \"include logs and patches, Please check them in SOURCES\"")
> - spec_files_bottom.append('')
> -
> - # append the name of tarball to key word 'SOURCE' in xxx.spec.
> - def tail_source(d):
> - if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d):
> - ar_outdir = d.getVar('ARCHIVER_OUTDIR')
> - if not os.path.exists(ar_outdir):
> - return
> - source_list = os.listdir(ar_outdir)
> - source_number = 0
> - for source in source_list:
> - # do_deploy_archives may have already run (from sstate) meaning a .src.rpm may already
> - # exist in ARCHIVER_OUTDIR so skip if present.
> - if source.endswith(".src.rpm"):
> - continue
> - # The rpmbuild doesn't need the root permission, but it needs
> - # to know the file's user and group name, the only user and
> - # group in fakeroot is "root" when working in fakeroot.
> - f = os.path.join(ar_outdir, source)
> - os.chown(f, 0, 0)
> - spec_preamble_top.append('Source%s: %s' % (source_number, source))
> - source_number += 1
> -
> # In RPM, dependencies are of the format: pkg <>= Epoch:Version-Release
> # This format is similar to OE, however there are restrictions on the
> # characters that can be in a field. In the Version field, "-"
> @@ -386,11 +358,6 @@ python write_specfile () {
>
> # Gather special src/first package data
> if srcname == splitname:
> - archiving = d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and \
> - bb.data.inherits_class('archiver', d)
> - if archiving and srclicense != splitlicense:
> - bb.warn("The SRPM produced may not have the correct overall source license in the License tag. This is due to the LICENSE for the primary package and SRPM conflicting.")
> -
> srclicense = splitlicense
> srcrdepends = splitrdepends
> srcrrecommends = splitrrecommends
> @@ -515,7 +482,6 @@ python write_specfile () {
>
> del localdata
>
> - add_prep(d,spec_files_bottom)
> spec_preamble_top.append('Summary: %s' % srcsummary)
> spec_preamble_top.append('Name: %s' % srcname)
> spec_preamble_top.append('Version: %s' % srcversion)
> @@ -529,7 +495,6 @@ python write_specfile () {
> spec_preamble_top.append('URL: %s' % srchomepage)
> if srccustomtagschunk:
> spec_preamble_top.append(srccustomtagschunk)
> - tail_source(d)
>
> # Replaces == Obsoletes && Provides
> robsoletes = bb.utils.explode_dep_versions2(srcrobsoletes or "")
> @@ -692,14 +657,6 @@ python do_package_rpm () {
> cmd = cmd + " --define '_unpackaged_files_terminate_build 0'"
> cmd = cmd + " --define 'debug_package %{nil}'"
> cmd = cmd + " --define '_tmppath " + workdir + "'"
> - if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d):
> - cmd = cmd + " --define '_sourcedir " + d.getVar('ARCHIVER_OUTDIR') + "'"
> - cmdsrpm = cmd + " --define '_srcrpmdir " + d.getVar('ARCHIVER_OUTDIR') + "'"
> - cmdsrpm = cmdsrpm + " -bs " + outspecfile
> - # Build the .src.rpm
> - d.setVar('SBUILDSPEC', cmdsrpm + "\n")
> - d.setVarFlag('SBUILDSPEC', 'func', '1')
> - bb.build.exec_func('SBUILDSPEC', d)
> cmd = cmd + " -bb " + outspecfile
>
> # rpm 4 creates various empty directories in _topdir, let's clean them up
> diff --git a/meta/lib/oeqa/selftest/cases/archiver.py b/meta/lib/oeqa/selftest/cases/archiver.py
> index 0a6d4e325fb..f61a5220178 100644
> --- a/meta/lib/oeqa/selftest/cases/archiver.py
> +++ b/meta/lib/oeqa/selftest/cases/archiver.py
> @@ -116,16 +116,3 @@ class Archiver(OESelftestTestCase):
>
> excluded_present = len(glob.glob(src_path_target + '/%s-*' % target_recipes[1]))
> self.assertFalse(excluded_present, 'Recipe %s was not excluded.' % target_recipes[1])
> -
> -
> -
> - def test_archiver_srpm_mode(self):
> - """
> - Test that in srpm mode, the added recipe dependencies at least exist/work [YOCTO #11121]
> - """
> -
> - features = 'INHERIT += "archiver"\n'
> - features += 'ARCHIVER_MODE[srpm] = "1"\n'
> - self.write_config(features)
> -
> - bitbake('-n core-image-sato')
> diff --git a/meta/lib/oeqa/selftest/cases/buildoptions.py b/meta/lib/oeqa/selftest/cases/buildoptions.py
> index 6a18eb83665..a7b2b13a98b 100644
> --- a/meta/lib/oeqa/selftest/cases/buildoptions.py
> +++ b/meta/lib/oeqa/selftest/cases/buildoptions.py
> @@ -152,21 +152,6 @@ class BuildhistoryTests(BuildhistoryBase):
> self.run_buildhistory_operation(target, target_config="PR = \"r1\"", change_bh_location=True)
> self.run_buildhistory_operation(target, target_config="PR = \"r0\"", change_bh_location=False, expect_error=True, error_regex=error)
>
> -class ArchiverTest(OESelftestTestCase):
> - @OETestID(926)
> - def test_arch_work_dir_and_export_source(self):
> - """
> - Test for archiving the work directory and exporting the source files.
> - """
> - self.write_config("INHERIT += \"archiver\"\nARCHIVER_MODE[src] = \"original\"\nARCHIVER_MODE[srpm] = \"1\"")
> - res = bitbake("xcursor-transparent-theme", ignore_status=True)
> - self.assertEqual(res.status, 0, "\nCouldn't build xcursortransparenttheme.\nbitbake output %s" % res.output)
> - deploy_dir_src = get_bb_var('DEPLOY_DIR_SRC')
> - pkgs_path = g.glob(str(deploy_dir_src) + "/allarch*/xcurs*")
> - src_file_glob = str(pkgs_path[0]) + "/xcursor*.src.rpm"
> - tar_file_glob = str(pkgs_path[0]) + "/xcursor*.tar.gz"
> - self.assertTrue((g.glob(src_file_glob) and g.glob(tar_file_glob)), "Couldn't find .src.rpm and .tar.gz files under %s/allarch*/xcursor*" % deploy_dir_src)
> -
> class ToolchainOptions(OESelftestTestCase):
>
> def test_toolchain_fortran(self):
>
More information about the Openembedded-core
mailing list