[oe-commits] [bitbake] branch master-next updated: fetch2: add initial Amazon AWS S3 fetcher
git at git.openembedded.org
git at git.openembedded.org
Thu Mar 2 16:55:00 UTC 2017
This is an automated email from the git hooks/post-receive script.
rpurdie pushed a commit to branch master-next
in repository bitbake.
The following commit(s) were added to refs/heads/master-next by this push:
new 46dcd09 fetch2: add initial Amazon AWS S3 fetcher
46dcd09 is described below
commit 46dcd0973e5d41ac77d8124f26f7ada827faeb96
Author: Andre McCurdy <armccurdy at gmail.com>
AuthorDate: Wed Mar 1 15:56:46 2017 -0800
fetch2: add initial Amazon AWS S3 fetcher
Class for fetching files from Amazon S3 using the AWS Command Line
Interface. The aws tool must be correctly installed and configured
prior to use.
The class supports both download() and checkstatus(), which therefore
allows S3 mirrors to be used for SSTATE_MIRRORS.
Signed-off-by: Andre McCurdy <armccurdy at gmail.com>
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
---
lib/bb/fetch2/__init__.py | 6 ++-
lib/bb/fetch2/s3.py | 96 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 100 insertions(+), 2 deletions(-)
diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index 760e24f..0205c3d 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -1196,13 +1196,13 @@ class FetchData(object):
self.sha256_name = "sha256sum"
if self.md5_name in self.parm:
self.md5_expected = self.parm[self.md5_name]
- elif self.type not in ["http", "https", "ftp", "ftps", "sftp"]:
+ elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
self.md5_expected = None
else:
self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name)
if self.sha256_name in self.parm:
self.sha256_expected = self.parm[self.sha256_name]
- elif self.type not in ["http", "https", "ftp", "ftps", "sftp"]:
+ elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
self.sha256_expected = None
else:
self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name)
@@ -1787,6 +1787,7 @@ from . import svn
from . import wget
from . import ssh
from . import sftp
+from . import s3
from . import perforce
from . import bzr
from . import hg
@@ -1804,6 +1805,7 @@ methods.append(gitannex.GitANNEX())
methods.append(cvs.Cvs())
methods.append(ssh.SSH())
methods.append(sftp.SFTP())
+methods.append(s3.S3())
methods.append(perforce.Perforce())
methods.append(bzr.Bzr())
methods.append(hg.Hg())
diff --git a/lib/bb/fetch2/s3.py b/lib/bb/fetch2/s3.py
new file mode 100644
index 0000000..27993aa
--- /dev/null
+++ b/lib/bb/fetch2/s3.py
@@ -0,0 +1,96 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementation for Amazon AWS S3.
+
+Class for fetching files from Amazon S3 using the AWS Command Line Interface.
+The aws tool must be correctly installed and configured prior to use.
+
+"""
+
+# Copyright (C) 2017, Andre McCurdy <armccurdy at gmail.com>
+#
+# Based in part on bb.fetch2.wget:
+# Copyright (C) 2003, 2004 Chris Larson
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import os
+import bb
+import urllib.request, urllib.parse, urllib.error
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import FetchError
+from bb.fetch2 import runfetchcmd
+
+class S3(FetchMethod):
+ """Class to fetch urls via 'aws s3'"""
+
+ def supports(self, ud, d):
+ """
+ Check to see if a given url can be fetched with s3.
+ """
+ return ud.type in ['s3']
+
+ def recommends_checksum(self, urldata):
+ return True
+
+ def urldata_init(self, ud, d):
+ if 'downloadfilename' in ud.parm:
+ ud.basename = ud.parm['downloadfilename']
+ else:
+ ud.basename = os.path.basename(ud.path)
+
+ ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+
+ def download(self, ud, d):
+ """
+ Fetch urls
+ Assumes localpath was called first
+ """
+
+ cmd = 'aws s3 cp s3://%s%s %s' % (ud.host, ud.path, ud.localpath)
+ bb.fetch2.check_network_access(d, cmd, ud.url)
+ runfetchcmd(cmd, d)
+
+ # Additional sanity checks copied from the wget class (although there
+ # are no known issues which mean these are required, treat the aws cli
+ # tool with a little healthy suspicion).
+
+ if not os.path.exists(ud.localpath):
+ raise FetchError("The aws cp command returned success for s3://%s%s but %s doesn't exist?!" % (ud.host, ud.path, ud.localpath))
+
+ if os.path.getsize(ud.localpath) == 0:
+ os.remove(ud.localpath)
+ raise FetchError("The aws cp command for s3://%s%s resulted in a zero size file?! Deleting and failing since this isn't right." % (ud.host, ud.path))
+
+ return True
+
+ def checkstatus(self, fetch, ud, d):
+ """
+ Check the status of a URL
+ """
+
+ cmd = 'aws s3 ls s3://%s%s' % (ud.host, ud.path)
+ bb.fetch2.check_network_access(d, cmd, ud.url)
+ output = runfetchcmd(cmd, d)
+
+ # "aws s3 ls s3://mybucket/foo" will exit with success even if the file
+ # is not found, so check output of the command to confirm success.
+
+ if not output:
+ raise FetchError("The aws ls command for s3://%s%s gave empty output" % (ud.host, ud.path))
+
+ return True
--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
More information about the Openembedded-commits
mailing list