[oe-commits] org.oe.dev ipkg-utils: Merge in tweaked version of patches from #1504 (from Paul Sokolovsky) to massively speedup ipkg-make-index
rpurdie commit
openembedded-commits at lists.openembedded.org
Sat May 26 23:17:12 UTC 2007
ipkg-utils: Merge in tweaked version of patches from #1504 (from Paul Sokolovsky) to massively speedup ipkg-make-index
Author: rpurdie at openembedded.org
Branch: org.openembedded.dev
Revision: 60f25719f044f04c78ec8cfa8af6a503b643df4e
ViewMTN: http://monotone.openembedded.org/revision.psp?id=60f25719f044f04c78ec8cfa8af6a503b643df4e
Files:
1
packages/ipkg-utils/ipkg-utils/index_speedup.patch
packages/ipkg-utils/ipkg-utils-native
packages/ipkg-utils/ipkg-utils-native/ipkg-py-sane-vercompare.patch
packages/ipkg-utils/ipkg-utils-native/ipkg-utils-fix.patch
packages/ipkg-utils/ipkg-utils/ipkg-make-index-track-stamps.patch
packages/ipkg-utils/ipkg-utils/ipkg-py-sane-vercompare.patch
packages/ipkg-utils/ipkg-utils/ipkg-py-tarfile.patch
packages/ipkg-utils/ipkg-utils/ipkg-utils-fix.patch
packages/ipkg-utils/ipkg-utils-native_1.6+cvs20050404.bb
packages/ipkg-utils/ipkg-utils_1.6+cvs20050404.bb
Diffs:
#
# mt diff -rf4e1c5aa4d9a06e35a1757adf07cdfc138c0791c -r60f25719f044f04c78ec8cfa8af6a503b643df4e
#
#
#
# delete "packages/ipkg-utils/ipkg-utils/index_speedup.patch"
#
# delete "packages/ipkg-utils/ipkg-utils-native"
#
# delete "packages/ipkg-utils/ipkg-utils-native/ipkg-py-sane-vercompare.patch"
#
# delete "packages/ipkg-utils/ipkg-utils-native/ipkg-utils-fix.patch"
#
# add_file "packages/ipkg-utils/ipkg-utils/ipkg-make-index-track-stamps.patch"
# content [13aa70d3fb4af7f44dd20be52a40d70acc6420a6]
#
# add_file "packages/ipkg-utils/ipkg-utils/ipkg-py-sane-vercompare.patch"
# content [a533f3484b87031c12b00510830579d719f4f898]
#
# add_file "packages/ipkg-utils/ipkg-utils/ipkg-py-tarfile.patch"
# content [a351c21502faa71fd5c654d9062d5ad3c0f172d1]
#
# add_file "packages/ipkg-utils/ipkg-utils/ipkg-utils-fix.patch"
# content [376af87b36ba801509d17088639cc92ecf124344]
#
# patch "packages/ipkg-utils/ipkg-utils-native_1.6+cvs20050404.bb"
# from [bd4992d548728ae583c7011fd8d2c7c0ffdd3866]
# to [991a74ee34804eb8b0aadde9d528ae65a2786d8f]
#
# patch "packages/ipkg-utils/ipkg-utils_1.6+cvs20050404.bb"
# from [f6d41b6ec89a49d9b83b2c4fd7b4825e43d158ab]
# to [23cdf8b2d949450edbc40f4123ca3d976b7ce2b4]
#
============================================================
--- packages/ipkg-utils/ipkg-utils/ipkg-make-index-track-stamps.patch 13aa70d3fb4af7f44dd20be52a40d70acc6420a6
+++ packages/ipkg-utils/ipkg-utils/ipkg-make-index-track-stamps.patch 13aa70d3fb4af7f44dd20be52a40d70acc6420a6
@@ -0,0 +1,97 @@
+---
+ ipkg-make-index | 31 +++++++++++++++++++++++++++----
+ 1 file changed, 27 insertions(+), 4 deletions(-)
+
+Index: ipkg-utils/ipkg-make-index
+===================================================================
+--- ipkg-utils.orig/ipkg-make-index 2007-05-26 23:45:56.000000000 +0100
++++ ipkg-utils/ipkg-make-index 2007-05-26 23:47:25.000000000 +0100
+@@ -40,6 +40,7 @@ def to_locale(filename, locale):
+ old_filename = None
+ packages_filename = None
+ filelist_filename = "Packages.filelist"
++stamplist_filename = "Packages.stamps"
+ opt_s = 0
+ opt_m = 0
+ (opts, remaining_args) = getopt.getopt(sys.argv[1:], "hl:p:vsmr:L:")
+@@ -50,6 +51,7 @@ for (optkey, optval) in opts:
+ opt_s = 1
+ if optkey == '-p':
+ packages_filename = optval
++ stamplist_filename = optval + ".stamps"
+ if optkey == '-l':
+ filelist_filename = optval
+ if optkey == '-v':
+@@ -72,6 +74,7 @@ old_pkg_hash = {}
+ if packages_filename and not old_filename and os.path.exists(packages_filename):
+ old_filename = packages_filename
+
++pkgsStamps = {}
+ if old_filename:
+ if (verbose):
+ sys.stderr.write("Reading package list from " + old_filename + "\n")
+@@ -80,6 +83,14 @@ if old_filename:
+ for k in old_packages.packages.keys():
+ p = old_packages.packages[k]
+ old_pkg_hash[p.filename] = p
++ try:
++ f = open(stamplist_filename, "r")
++ for l in f:
++ l = l.strip()
++ s, f = l.split(" ", 1)
++ pkgsStamps[f] = int(s)
++ except IOError:
++ pass
+
+ if (verbose):
+ sys.stderr.write("Reading in all the package info from %s\n" % (pkg_dir, ))
+@@ -87,11 +98,17 @@ files=glob(pkg_dir + '/*.ipk') + glob(pk
+ files.sort()
+ for filename in files:
+ basename = os.path.basename(filename)
++ pkg = None
++ fnameStat = os.stat(filename)
+ if old_pkg_hash.has_key(basename):
+- if (verbose):
++ if pkgsStamps.has_key(basename) and fnameStat.st_mtime == pkgsStamps[basename]:
++ if (verbose):
+ sys.stderr.write("Found %s in Packages\n" % (filename,))
+- pkg = old_pkg_hash[basename]
+- else:
++ pkg = old_pkg_hash[basename]
++ else:
++ sys.stderr.write("Found %s in Packages, but mtime differs - re-reading\n" % (filename,))
++
++ if not pkg:
+ if (verbose):
+ sys.stderr.write("Reading info for package %s\n" % (filename,))
+ pkg = ipkg.Package(filename)
+@@ -101,6 +118,7 @@ for filename in files:
+ else:
+ old_filename = ""
+ s = packages.add_package(pkg)
++ pkgsStamps[basename] = fnameStat.st_mtime
+ if s == 0:
+ if old_filename:
+ # old package was displaced by newer
+@@ -114,6 +132,11 @@ for filename in files:
+ if opt_s:
+ print filename
+
++pkgsStampsFile = open(stamplist_filename, "w")
++for f in pkgsStamps.keys():
++ pkgsStampsFile.write("%d %s\n" % (pkgsStamps[f], f))
++pkgsStampsFile.close()
++
+ if opt_s:
+ sys.exit(0)
+
+@@ -154,7 +177,7 @@ if packages_filename:
+ os.rename(tmp_packages_filename, packages_filename)
+ os.rename(tmp_gzip_filename, gzip_filename)
+
+-if verbose:
++if verbose:
+ sys.stderr.write("Generate Packages.filelist file\n")
+ files = {}
+ names = packages.packages.keys()
============================================================
--- packages/ipkg-utils/ipkg-utils/ipkg-py-sane-vercompare.patch a533f3484b87031c12b00510830579d719f4f898
+++ packages/ipkg-utils/ipkg-utils/ipkg-py-sane-vercompare.patch a533f3484b87031c12b00510830579d719f4f898
@@ -0,0 +1,51 @@
+Only in ipkg-utils: ipkg-py-sane-vercompare.patch
+diff -ur ipkg-utils.org/ipkg.py ipkg-utils/ipkg.py
+--- ipkg-utils.org/ipkg.py 2005-01-08 18:08:52.000000000 +0000
++++ ipkg-utils/ipkg.py 2007-04-04 11:52:46.000000000 +0000
+@@ -48,9 +48,9 @@
+ self.epoch = epoch
+ self.version = version
+
+- def _versioncompare(self, ref):
+- selfversion = self.version
+- refversion = ref.version
++ def _versioncompare(self, selfversion, refversion):
++ if not selfversion: selfversion = ""
++ if not refversion: refversion = ""
+ while 1:
+ ## first look for non-numeric version component
+ selfm = re.match('([^0-9]*)(.*)', selfversion)
+@@ -89,7 +89,18 @@
+ elif (self.epoch < ref.epoch):
+ return -1
+ else:
+- return self._versioncompare(ref)
++ self_ver_comps = re.match(r"(.+?)(-r.+)?$", self.version)
++ ref_ver_comps = re.match(r"(.+?)(-r.+)?$", ref.version)
++ #print (self_ver_comps.group(1), self_ver_comps.group(2))
++ #print (ref_ver_comps.group(1), ref_ver_comps.group(2))
++ r = self._versioncompare(self_ver_comps.group(1), ref_ver_comps.group(1))
++ if r == 0:
++ r = self._versioncompare(self_ver_comps.group(2), ref_ver_comps.group(2))
++ #print "compare: %s vs %s = %d" % (self, ref, r)
++ return r
++
++ def __str__(self):
++ return str(self.epoch) + ":" + self.version
+
+ def parse_version(versionstr):
+ epoch = 0
+@@ -445,6 +456,13 @@
+ return self.packages[key]
+
+ if __name__ == "__main__":
++
++ assert Version(0, "1.2.2-r1").compare(Version(0, "1.2.3-r0")) == -1
++ assert Version(0, "1.2.2-r0").compare(Version(0, "1.2.2+cvs20070308-r0")) == -1
++ assert Version(0, "1.2.2+cvs20070308").compare(Version(0, "1.2.2-r0")) == 1
++ assert Version(0, "1.2.2-r0").compare(Version(0, "1.2.2-r0")) == 0
++ assert Version(0, "1.2.2-r5").compare(Version(0, "1.2.2-r0")) == 1
++
+ package = Package()
+
+ package.set_package("FooBar")
============================================================
--- packages/ipkg-utils/ipkg-utils/ipkg-py-tarfile.patch a351c21502faa71fd5c654d9062d5ad3c0f172d1
+++ packages/ipkg-utils/ipkg-utils/ipkg-py-tarfile.patch a351c21502faa71fd5c654d9062d5ad3c0f172d1
@@ -0,0 +1,306 @@
+---
+ arfile.py | 124 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ ipkg.py | 106 ++++++++++++++++++++++++++---------------------------
+ setup.py | 2 -
+ 3 files changed, 177 insertions(+), 55 deletions(-)
+
+Index: ipkg-utils/arfile.py
+===================================================================
+--- /dev/null 1970-01-01 00:00:00.000000000 +0000
++++ ipkg-utils/arfile.py 2007-05-26 23:46:59.000000000 +0100
+@@ -0,0 +1,124 @@
++"""
++arfile - A module to parse GNU ar archives.
++
++Copyright (c) 2006-7 Paul Sokolovsky
++This file is released under the terms
++of GNU General Public License v2 or later.
++"""
++import sys
++import os
++import tarfile
++
++
++class FileSection:
++ "A class which allows to treat portion of file as separate file object."
++
++ def __init__(self, f, offset, size):
++ self.f = f
++ self.offset = offset
++ self.size = size
++ self.seek(0, 0)
++
++ def seek(self, offset, whence = 0):
++# print "seek(%x, %d)" % (offset, whence)
++ if whence == 0:
++ return self.f.seek(offset + self.offset, whence)
++ elif whence == 1:
++ return self.f.seek(offset, whence)
++ elif whence == 2:
++ return self.f.seek(self.offset + self.size + offset, 0)
++ else:
++ assert False
++
++ def tell(self):
++# print "tell()"
++ return self.f.tell() - self.offset
++
++ def read(self, size = -1):
++# print "read(%d)" % size
++ return self.f.read(size)
++
++class ArFile:
++
++ def __init__(self, f):
++ self.f = f
++ self.directory = {}
++ self.directoryRead = False
++
++ signature = self.f.readline()
++ assert signature == "!<arch>\n"
++ self.directoryOffset = self.f.tell()
++
++ def open(self, fname):
++ if self.directory.has_key(fname):
++ return FileSection(self.f, self.directory[fname][-1], int(self.directory[fname][5]))
++
++ if self.directoryRead:
++ raise IOError, (2, "AR member not found: " + fname)
++
++ f = self._scan(fname)
++ if f == None:
++ raise IOError, (2, "AR member not found: " + fname)
++ return f
++
++
++ def _scan(self, fname):
++ self.f.seek(self.directoryOffset, 0)
++
++ while True:
++ l = self.f.readline()
++ if not l:
++ self.directoryRead = True
++ return None
++
++ if l == "\n":
++ l = self.f.readline()
++ if not l: break
++ descriptor = l.split()
++# print descriptor
++ size = int(descriptor[5])
++ memberName = descriptor[0][:-1]
++ self.directory[memberName] = descriptor + [self.f.tell()]
++# print "read:", memberName
++ if memberName == fname:
++ # Record directory offset to start from next time
++ self.directoryOffset = self.f.tell() + size
++ return FileSection(self.f, self.f.tell(), size)
++
++ # Skip data and loop
++ data = self.f.seek(size, 1)
++# print hex(f.tell())
++
++
++if __name__ == "__main__":
++ if None:
++ f = open(sys.argv[1], "rb")
++
++ ar = ArFile(f)
++ tarStream = ar.open("data.tar.gz")
++ print "--------"
++ tarStream = ar.open("data.tar.gz")
++ print "--------"
++ tarStream = ar.open("control.tar.gz")
++ print "--------"
++ tarStream = ar.open("control.tar.gz2")
++
++ sys.exit(0)
++
++
++ dir = "."
++ if len(sys.argv) > 1:
++ dir = sys.argv[1]
++ for f in os.listdir(dir):
++ if not f.endswith(".ipk"): continue
++
++ print "=== %s ===" % f
++ f = open(dir + "/" + f, "rb")
++
++ ar = ArFile(f)
++ tarStream = ar.open("control.tar.gz")
++ tarf = tarfile.open("control.tar.gz", "r", tarStream)
++ #tarf.list()
++
++ f2 = tarf.extractfile("control")
++ print f2.read()
+Index: ipkg-utils/setup.py
+===================================================================
+--- ipkg-utils.orig/setup.py 2007-05-26 23:45:55.000000000 +0100
++++ ipkg-utils/setup.py 2007-05-26 23:46:59.000000000 +0100
+@@ -16,6 +16,6 @@ distutils.core.setup( name = 'ipkg-utils
+ platforms = 'POSIX',
+ keywords = 'ipkg familiar',
+ url = 'http://www.handhelds.org/sources.html/',
+- py_modules = [ 'ipkg' ],
++ py_modules = [ 'ipkg', 'arfile' ],
+ scripts = ['ipkg-compare-indexes', 'ipkg-make-index', 'ipkg-update-index', 'ipkg-build', 'ipkg-unbuild', 'ipkg-upload']
+ )
+Index: ipkg-utils/ipkg.py
+===================================================================
+--- ipkg-utils.orig/ipkg.py 2007-05-26 23:46:55.000000000 +0100
++++ ipkg-utils/ipkg.py 2007-05-26 23:45:20.000000000 +0100
+@@ -41,6 +41,8 @@ import re
+ import string
+ import commands
+ from stat import ST_SIZE
++import arfile
++import tarfile
+
+ class Version:
+ """A class for holding parsed package version information."""
+@@ -131,78 +133,58 @@ class Package:
+ self.section = None
+ self.filename_header = None
+ self.file_list = []
+- self.md5 = None
++ # md5 is lazy attribute, computed on demand
++ #self.md5 = None
+ self.size = None
+ self.installed_size = None
+ self.filename = None
+ self.isdeb = 0
++ self.fn = fn
+
+ if fn:
+ # see if it is deb format
+- f = open(fn, "r")
++ f = open(fn, "rb")
+ magic = f.read(4)
+- f.close()
++ f.seek(0, 0)
+ if (magic == "!<ar"):
+ self.isdeb = 1
+
+- # compute the MD5.
+- f = open(fn, "r")
+- sum = md5.new()
+- while 1:
+- data = f.read(1024)
+- if not data: break
+- sum.update(data)
+- f.close()
+- if sys.version[:1] > '2':
+- # when using Python 2.0 or newer
+- self.md5 = sum.hexdigest()
+- else:
+- self.md5 = string.join(map((lambda x:"%02x" % ord(x)),sum.digest()),'')
+- stat = os.stat(fn)
+- self.size = stat[ST_SIZE]
++
+ self.filename = os.path.basename(fn)
++ assert self.isdeb == 1, "Old ipk format (non-deb) is unsupported"
++
+ ## sys.stderr.write(" extracting control.tar.gz from %s\n"% (fn,))
+- if self.isdeb:
+- control = os.popen("ar p "+fn+" control.tar.gz | tar xfzO - './control'","r")
+- else:
+- control = os.popen("tar xfzO "+fn+" 'control.tar.gz' | tar xfzO - './control'","r")
+- line = control.readline()
+- while 1:
+- if not line: break
+- line = string.rstrip(line)
+- lineparts = re.match(r'([\w-]*?):\s*(.*)', line)
+- if lineparts:
+- name = string.lower(lineparts.group(1))
+- value = lineparts.group(2)
+- while 1:
+- line = control.readline()
+- if not line: break
+- if line[0] != ' ': break
+- line = string.rstrip(line)
+- value = value + '\n' + line
+- # don't allow package to override its own filename
+- if name == "filename":
+- self.filename_header = value
+- else:
+- if self.__dict__.has_key(name):
+- self.__dict__[name] = value
+- else:
+- line = control.readline()
++
++ ar = arfile.ArFile(f)
++ tarStream = ar.open("control.tar.gz")
++ tarf = tarfile.open("control.tar.gz", "r", tarStream)
++
++ control = tarf.extractfile("control")
++ self.read_control(control)
+ control.close()
+- if self.isdeb:
+- data = os.popen("ar p "+fn+" data.tar.gz | tar tfz -","r")
+- else:
+- data = os.popen("tar xfzO "+fn+" '*data.tar.gz' | tar tfz -","r")
+- while 1:
+- line = data.readline()
+- if not line: break
+- self.file_list.append(string.rstrip(line))
+- data.close()
+
+ self.scratch_dir = None
+ self.file_dir = None
+ self.meta_dir = None
+
++ def __getattr__(self, name):
++ if name == "md5":
++ self._computeFileMD5()
++ return self.md5
++ else:
++ raise AttributeError, name
++
++ def _computeFileMD5(self):
++ # compute the MD5.
++ f = open(self.fn, "rb")
++ sum = md5.new()
++ while 1:
++ data = f.%s
>>> DIFF TRUNCATED @ 16K
More information about the Openembedded-commits
mailing list