[bitbake-devel] [PATCH 1/1] bb/fetch2: fix the "filename too long" error
Richard Purdie
richard.purdie at linuxfoundation.org
Wed Oct 30 14:27:31 UTC 2013
On Tue, 2013-10-29 at 22:24 +0800, Robert Yang wrote:
> When the SRC_URI is very long or the PREMIRROR/MIRROR is in a very deep
> directory, the git, svn and other fetchers will covert the path into
> the filename, so if:
>
> len(path) > NAME_MAX,
>
> then we will get the "filename too long" error, the oe-core supports to
> build when len(TMPDIR) <= 410, but the NAME_MAX is usually 256, so we
> can easily get this error when we put the PREMIRROR/MIRROR in a deep
> dir.
>
> Truncate the filename to NAME_MAX - 10 will fix the problem, the "-10"
> is used for the ".done" and ".lock", in fact, -5 would be OK, but use
> "-10" in case we will have other longer suffix in the future.
>
> [YOCTO #5389]
Aren't the "long SRC_URI" and deep PREMIRROR/MIRROR problems different
issues?
As far as I know, we don't have any long SRC_URIs in use anywhere within
the project. We could just error if we found one with a path we couldn't
cope with.
The deep PREMIRROR/MIRROR issue on the other hand sounds like something
we should adjust the code to handle properly?
Randomly truncating path names like this looks like a rather hacky
solution and it loses key information, there is a reason these long
names are being used as keys.
Put another way, the mirror tarball name should not change depending on
how deep the DL_DIR or other directories are.
So no, there is no way this patch is correct or should be merged, sorry.
Cheers,
Richard
> Signed-off-by: Robert Yang <liezhi.yang at windriver.com>
> ---
> bitbake/lib/bb/fetch2/__init__.py | 18 ++++++++++++++++++
> bitbake/lib/bb/fetch2/bzr.py | 2 ++
> bitbake/lib/bb/fetch2/git.py | 6 ++++++
> bitbake/lib/bb/fetch2/hg.py | 3 +++
> bitbake/lib/bb/fetch2/osc.py | 2 ++
> bitbake/lib/bb/fetch2/repo.py | 4 ++++
> bitbake/lib/bb/fetch2/svk.py | 2 ++
> bitbake/lib/bb/fetch2/svn.py | 3 +++
> 8 files changed, 40 insertions(+)
>
> diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
> index 451d104..f76568e 100644
> --- a/bitbake/lib/bb/fetch2/__init__.py
> +++ b/bitbake/lib/bb/fetch2/__init__.py
> @@ -959,6 +959,24 @@ def get_file_checksums(filelist, pn):
> checksums.sort(key=operator.itemgetter(1))
> return checksums
>
> +def truncate_to_name_max(dirpath, filename):
> + """
> + Truncate the filename to meet len(filename) <= NAME_MAX -10, return
> + the last NAME_MAX - 10 characters if it doesn't, the -10 is for the
> + '.done', '.lock' and perhaps others in the future.
> + """
> + if not os.path.exists(dirpath):
> + bb.utils.mkdirhier(dirpath)
> +
> + name_max = os.pathconf(dirpath, 'PC_NAME_MAX')
> + new_max = name_max - 10
> + if len(filename) > new_max:
> + new_name = filename[-new_max:]
> + logger.debug(1, "Truncate %s to %s to meet %s (NAME_MAX - 10) characters" % \
> + (filename, new_name, new_max))
> + filename = new_name
> +
> + return filename
>
> class FetchData(object):
> """
> diff --git a/bitbake/lib/bb/fetch2/bzr.py b/bitbake/lib/bb/fetch2/bzr.py
> index 5d9e5f9..f91c7ee 100644
> --- a/bitbake/lib/bb/fetch2/bzr.py
> +++ b/bitbake/lib/bb/fetch2/bzr.py
> @@ -51,6 +51,8 @@ class Bzr(FetchMethod):
> ud.revision = self.latest_revision(ud.url, ud, d)
>
> ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d)
> + # The length of the filename should be less than NAME_MAX - 10
> + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile)
>
> def _buildbzrcommand(self, ud, d, command):
> """
> diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
> index 6175e4c..19598ef 100644
> --- a/bitbake/lib/bb/fetch2/git.py
> +++ b/bitbake/lib/bb/fetch2/git.py
> @@ -135,9 +135,15 @@ class Git(FetchMethod):
> if ud.rebaseable:
> for name in ud.names:
> gitsrcname = gitsrcname + '_' + ud.revisions[name]
> +
> ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname)
> + # The length of the filename should be less than NAME_MAX - 10
> + ud.mirrortarball = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.mirrortarball)
> ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
> +
> gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/")
> + # The length of the filename should be less than NAME_MAX - 10
> + gitsrcname = bb.fetch2.truncate_to_name_max(gitdir, gitsrcname)
> ud.clonedir = os.path.join(gitdir, gitsrcname)
>
> ud.localfile = ud.clonedir
> diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
> index b1c8675..be9bb26 100644
> --- a/bitbake/lib/bb/fetch2/hg.py
> +++ b/bitbake/lib/bb/fetch2/hg.py
> @@ -66,6 +66,9 @@ class Hg(FetchMethod):
>
> ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)
>
> + # The length of the filename should be less than NAME_MAX - 10
> + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile)
> +
> def need_update(self, url, ud, d):
> revTag = ud.parm.get('rev', 'tip')
> if revTag == "tip":
> diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py
> index 1a3a7bb..2e276e0 100644
> --- a/bitbake/lib/bb/fetch2/osc.py
> +++ b/bitbake/lib/bb/fetch2/osc.py
> @@ -48,6 +48,8 @@ class Osc(FetchMethod):
> ud.revision = ""
>
> ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d)
> + # The length of the filename should be less than NAME_MAX - 10
> + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile)
>
> def _buildosccommand(self, ud, d, command):
> """
> diff --git a/bitbake/lib/bb/fetch2/repo.py b/bitbake/lib/bb/fetch2/repo.py
> index 8300da8..1e46968 100644
> --- a/bitbake/lib/bb/fetch2/repo.py
> +++ b/bitbake/lib/bb/fetch2/repo.py
> @@ -52,6 +52,8 @@ class Repo(FetchMethod):
> ud.manifest += '.xml'
>
> ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d)
> + # The length of the filename should be less than NAME_MAX - 10
> + ud.localfile = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.localfile)
>
> def download(self, loc, ud, d):
> """Fetch url"""
> @@ -62,6 +64,8 @@ class Repo(FetchMethod):
>
> gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
> repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo")
> + # The length of the filename should be less than NAME_MAX - 10
> + gitsrcname = bb.fetch2.truncate_to_name_max(repodir, gitsrcname)
> codir = os.path.join(repodir, gitsrcname, ud.manifest)
>
> if ud.user:
> diff --git a/bitbake/lib/bb/fetch2/svk.py b/bitbake/lib/bb/fetch2/svk.py
> index ee3823f..dbd78f3 100644
> --- a/bitbake/lib/bb/fetch2/svk.py
> +++ b/bitbake/lib/bb/fetch2/svk.py
> @@ -53,6 +53,8 @@ class Svk(FetchMethod):
> ud.revision = ud.parm.get('rev', "")
>
> ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
> + # The length of the filename should be less than NAME_MAX - 10
> + ud.localfile = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.localfile)
>
> def need_update(self, url, ud, d):
> if ud.date == "now":
> diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
> index 9a779d2..c155735 100644
> --- a/bitbake/lib/bb/fetch2/svn.py
> +++ b/bitbake/lib/bb/fetch2/svn.py
> @@ -65,6 +65,9 @@ class Svn(FetchMethod):
>
> ud.localfile = data.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)
>
> + # The length of the filename should be less than NAME_MAX - 10
> + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile)
> +
> def _buildsvncommand(self, ud, d, command):
> """
> Build up an svn commandline based on ud
More information about the bitbake-devel
mailing list