[oe-commits] [openembedded-core] 08/12: gpg_sign/selftest: Fix secmem parameter handling
git at git.openembedded.org
git at git.openembedded.org
Fri Jun 7 08:12:20 UTC 2019
This is an automated email from the git hooks/post-receive script.
rpurdie pushed a commit to branch master
in repository openembedded-core.
commit c7e131a76e522503df55e211dd261829feacfa28
Author: Richard Purdie <richard.purdie at linuxfoundation.org>
AuthorDate: Thu Jun 6 19:33:02 2019 +0100
gpg_sign/selftest: Fix secmem parameter handling
We keep seeing "cannot allocate memory" errors from rpm when signing packages
on the autobuilder. The following were tried:
* checking locked memory use (isn't hitting limits)
* Restricting RPM_GPG_SIGN_CHUNK to 1
* Limiting to 10 parallel do_package_write_rpm tasks
* Allowing unlimied memory overcommit
* Disabling rpm parallel compression
and the test still failed. Further invetigation showed that the --auto-expand-secmem
wasn't being passed to gpg-agent which meant the secmem couldn't be expanded hence the
errors when there was pressure on the agent.
The reason this happens is that some of the early gpg commands can start the agent
without the option and it sticks around in memory so a version with the correct
option may or may not get started.
We therefore add the option to all the key gpg calls.
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
meta/lib/oe/gpg_sign.py | 39 +++++++++++++++++++--------------
meta/lib/oeqa/selftest/cases/signing.py | 3 ++-
2 files changed, 24 insertions(+), 18 deletions(-)
diff --git a/meta/lib/oe/gpg_sign.py b/meta/lib/oe/gpg_sign.py
index a95d2ba..2fd8c3b 100644
--- a/meta/lib/oe/gpg_sign.py
+++ b/meta/lib/oe/gpg_sign.py
@@ -15,21 +15,27 @@ class LocalSigner(object):
def __init__(self, d):
self.gpg_bin = d.getVar('GPG_BIN') or \
bb.utils.which(os.getenv('PATH'), 'gpg')
+ self.gpg_cmd = [self.gpg_bin]
+ self.gpg_agent_bin = bb.utils.which(os.getenv('PATH'), "gpg-agent")
+ # Without this we see "Cannot allocate memory" errors when running processes in parallel
+ # It needs to be set for any gpg command since any agent launched can stick around in memory
+ # and this parameter must be set.
+ if self.gpg_agent_bin:
+ self.gpg_cmd += ["--agent-program=%s|--auto-expand-secmem" % (self.gpg_agent_bin)]
self.gpg_path = d.getVar('GPG_PATH')
- self.gpg_version = self.get_gpg_version()
self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmsign")
- self.gpg_agent_bin = bb.utils.which(os.getenv('PATH'), "gpg-agent")
+ self.gpg_version = self.get_gpg_version()
+
def export_pubkey(self, output_file, keyid, armor=True):
"""Export GPG public key to a file"""
- cmd = '%s --no-permission-warning --batch --yes --export -o %s ' % \
- (self.gpg_bin, output_file)
+ cmd = self.gpg_cmd + ["--no-permission-warning", "--batch", "--yes", "--export", "-o", output_file]
if self.gpg_path:
- cmd += "--homedir %s " % self.gpg_path
+ cmd += ["--homedir", self.gpg_path]
if armor:
- cmd += "--armor "
- cmd += keyid
- subprocess.check_output(shlex.split(cmd), stderr=subprocess.STDOUT)
+ cmd += ["--armor"]
+ cmd += [keyid]
+ subprocess.check_output(cmd, stderr=subprocess.STDOUT)
def sign_rpms(self, files, keyid, passphrase, digest, sign_chunk, fsk=None, fsk_password=None):
"""Sign RPM files"""
@@ -59,7 +65,7 @@ class LocalSigner(object):
if passphrase_file and passphrase:
raise Exception("You should use either passphrase_file of passphrase, not both")
- cmd = [self.gpg_bin, '--detach-sign', '--no-permission-warning', '--batch',
+ cmd = self.gpg_cmd + ['--detach-sign', '--no-permission-warning', '--batch',
'--no-tty', '--yes', '--passphrase-fd', '0', '-u', keyid]
if self.gpg_path:
@@ -72,9 +78,6 @@ class LocalSigner(object):
if self.gpg_version > (2,1,):
cmd += ['--pinentry-mode', 'loopback']
- if self.gpg_agent_bin:
- cmd += ["--agent-program=%s|--auto-expand-secmem" % (self.gpg_agent_bin)]
-
cmd += [input_file]
try:
@@ -101,7 +104,8 @@ class LocalSigner(object):
def get_gpg_version(self):
"""Return the gpg version as a tuple of ints"""
try:
- ver_str = subprocess.check_output((self.gpg_bin, "--version", "--no-permission-warning")).split()[2].decode("utf-8")
+ cmd = self.gpg_cmd + ["--version", "--no-permission-warning"]
+ ver_str = subprocess.check_output(cmd).split()[2].decode("utf-8")
return tuple([int(i) for i in ver_str.split("-")[0].split('.')])
except subprocess.CalledProcessError as e:
raise bb.build.FuncFailed("Could not get gpg version: %s" % e)
@@ -109,11 +113,12 @@ class LocalSigner(object):
def verify(self, sig_file):
"""Verify signature"""
- cmd = self.gpg_bin + " --verify --no-permission-warning "
+ cmd = self.gpg_cmd + [" --verify", "--no-permission-warning"]
if self.gpg_path:
- cmd += "--homedir %s " % self.gpg_path
- cmd += sig_file
- status = subprocess.call(shlex.split(cmd))
+ cmd += ["--homedir", self.gpg_path]
+
+ cmd += [sig_file]
+ status = subprocess.call(cmd)
ret = False if status else True
return ret
diff --git a/meta/lib/oeqa/selftest/cases/signing.py b/meta/lib/oeqa/selftest/cases/signing.py
index 9c710bd..b390f37 100644
--- a/meta/lib/oeqa/selftest/cases/signing.py
+++ b/meta/lib/oeqa/selftest/cases/signing.py
@@ -30,7 +30,8 @@ class Signing(OESelftestTestCase):
self.secret_key_path = os.path.join(self.testlayer_path, 'files', 'signing', "key.secret")
nsysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "gnupg-native")
- runCmd('gpg --batch --homedir %s --import %s %s' % (self.gpg_dir, self.pub_key_path, self.secret_key_path), native_sysroot=nsysroot)
+
+ runCmd('gpg --agent-program=`which gpg-agent`\|--auto-expand-secmem --batch --homedir %s --import %s %s' % (self.gpg_dir, self.pub_key_path, self.secret_key_path), native_sysroot=nsysroot)
return nsysroot + get_bb_var("bindir_native")
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list