From: Michael Hanke Date: Mon, 11 May 2009 18:07:31 +0000 (+0200) Subject: Few fixes and polish. X-Git-Url: https://git.donarmstrong.com/?a=commitdiff_plain;h=52ac84470d84b54ff6792cc2c055a4d3fc2b826c;p=neurodebian.git Few fixes and polish. --- diff --git a/.gitignore b/.gitignore index b926804..f6de385 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ build cache *.swp +*-stamp diff --git a/Makefile b/Makefile index fd125e1..43e33b6 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ WWW_UPLOAD_URI = elrond:/home/hanke/public_html/archive WWW_DIR = build/html +all: html prep: if [ ! -d build ]; then mkdir build; fi @@ -12,8 +13,10 @@ pics: $(MAKE) -C artwork -html: pics prep source +html: html-stamp +html-stamp: pics prep source cd build/src && $(MAKE) html BUILDDIR=$(CURDIR)/build + touch $@ clean: @@ -26,21 +29,25 @@ distclean: clean -rm -rf cache -source: - ./reblender generate \ +source: source-stamp +source-stamp: build/db.db + tools/reblender generate \ --cfg debneuro.cfg \ --db build/db.db \ --outdir build/src \ --pkgaddenum pkgs + rm -f html-stamp + touch $@ -refresh-db: +build/db.db: mkdir -p build -# rm -f build/db.db - ./reblender refreshdb \ + tools/reblender refreshdb \ --cfg debneuro.cfg \ --db build/db.db upload-website: html rsync -rvzlhp --delete --chmod=Dg+s,g+rw $(WWW_DIR) $(WWW_UPLOAD_URI) + +.PHONY: prep diff --git a/reblender b/reblender deleted file mode 100755 index 4e1f354..0000000 --- a/reblender +++ /dev/null @@ -1,796 +0,0 @@ -#!/usr/bin/env python -"""Tell me who you are! -""" - -import urllib -import apt -from debian_bundle import deb822 -from debian_bundle import debtags -from ConfigParser import SafeConfigParser -import subprocess -import os -import sys -import shutil -import pysvn -from optparse import OptionParser, Option, OptionGroup, OptionConflictError - - -def transCodename(codename, cfg): - """Translate a known codename into a release description. - - Unknown codenames will simply be returned as is. - """ - # strip repository codename - codename = codename[codename.find('_') + 1:] - - # if we know something, tell - if codename in cfg.options('release codenames'): - return cfg.get('release codenames', codename) - else: - return codename - - -def createDir(path): - if os.path.exists(path): - return - - ps = path.split(os.path.sep) - - for i in range(1,len(ps) + 1): - p = os.path.sep.join(ps[:i]) - - if not os.path.exists(p): - os.mkdir(p) - - -class AptListsCache(object): - def __init__(self, cachedir='cache', ro_cachedirs=None, init_db=None): - self.cachedir = cachedir - - if not ro_cachedirs is None: - self.ro_cachedirs = ro_cachedirs - else: - self.ro_cachedirs = [] - - # always use system cache - #self.ro_cachedirs.append('/var/lib/apt/lists/') - - # create cachedir - createDir(self.cachedir) - - - def get(self, url, update=False): - """Looks in the cache if the file is there and takes the cached one. - Otherwise it is downloaded first. - - Knows how to deal with http:// and svn:// URLs. - - :Return: - file handler - """ - # look whether it is compressed - cext = url.split('.')[-1] - if cext in ['gz', 'bz2']: - target_url = url[:-1 * len(cext) -1] - else: - # assume not compressed - target_url = url - cext = None - - # turn url into a filename -- mimik what APT does for - # /var/lib/apt/lists/ - tfilename = '_'.join(target_url.split('/')[2:]) - - # if we need to download anyway do not search - if update: - cfilename = os.path.join(self.cachedir, tfilename) - else: - # look for the uncompressed file anywhere in the cache - cfilename = None - for cp in [self.cachedir] + self.ro_cachedirs: - if os.path.exists(os.path.join(cp, tfilename)): - cfilename = os.path.join(cp, tfilename) - - # nothing found? - if cfilename is None: - # add cache item - cfilename = os.path.join(self.cachedir, tfilename) - update = True - - # if updated needed -- download - if update: - print 'Caching file from %s' % url - - if url.startswith('svn://'): - # export from SVN - pysvn.Client().export(url, cfilename) - if url.startswith('http://'): - # download - tempfile, ignored = urllib.urlretrieve(url) - - # decompress - decompressor = None - if cext == 'gz': - decompressor = 'gzip' - elif cext == 'bz2': - decompressor = 'bzip2' - elif cext == None: - decompressor = None - else: - raise ValueError, \ - "Don't know how to decompress %s files" \ - % cext - - if not decompressor is None: - if subprocess.call([decompressor, '-d', '-q', '-f', - tempfile]) == 1: - raise RuntimeError, \ - "Something went wrong while decompressing '%s'" \ - % tempfile - - # move decompressed file into cache - shutil.move(os.path.splitext(tempfile)[0], cfilename) - - # XXX do we need that if explicit filename is provided? - urllib.urlcleanup() - - # open cached file - fh = open(cfilename, 'r') - - return fh - - - - -class DebianPkgArchive(SafeConfigParser): - """ - """ - def __init__(self, cache=None, init_db=None): - """ - :Parameter: - """ - SafeConfigParser.__init__(self) - - # read an existing database if provided - if not init_db is None: - self.read(init_db) - - # use provided file cache or use fresh one - if not cache is None: - self.cache = cache - else: - self.cache = AptListsCache() - - # init debtags DB - self.dtags = debtags.DB() - self.dtags.read(open('/var/lib/debtags/package-tags')) - - # init package filter - self.pkgfilter = None - - self._updateReleases() - - - def _updateReleases(self): - self.releases = {} - - for p in self.sections(): - if not self.has_option(p, 'releases'): - continue - - # for all releases of this package - for r in \ - [rel.strip() for rel in self.get(p, 'releases').split(',')]: - # push release code - if not self.releases.has_key(r): - self.releases[r] = [] - - # store component - component = self.get(p, 'component %s' % r) - - if not component in self.releases[r]: - self.releases[r].append(component) - - - def __repr__(self): - """Generate INI file content for current content. - """ - # make adaptor to use str as file-like (needed for ConfigParser.write() - class file2str(object): - def __init__(self): - self.__s = '' - def write(self, val): - self.__s += val - def str(self): - return self.__s - - r = file2str() - self.write(r) - - return r.str() - - - def save(self, filename): - """Write current content to a file. - """ - f = open(filename, 'w') - self.write(f) - f.close() - - - def ensureUnique(self, section, option, value): - if not self.has_option(section, option): - self.set(section, option, value) - else: - if not self.get(section, option) == value: - raise ValueError, "%s: %s is not unique (%s != %s)" \ - % (section, option, - self.get(section, option), value) - - - def appendUniqueCSV(self, section, option, value): - """ - """ - if not self.has_option(section, option): - self.set(section, option, value) - else: - l = self.get(section, option).split(', ') - if not value in l: - self.set(section, option, ', '.join(l + [value])) - - - def importRelease(self, rurl, force_update=False): - # root URL of the repository - baseurl = '/'.join(rurl.split('/')[:-1]) - # get the release file from the cache - release_file = self.cache.get(rurl, update=force_update) - - # create parser instance - rp = deb822.Release(release_file) - - # architectures on this dist - archs = rp['Architectures'].split() - components = rp['Components'].split() - # compile a new codename that also considers the repository label - # to distinguish between official and unofficial repos. - codename = '_'.join([rp['Label'], rp['Codename']]) - - # compile the list of Packages files to parse and parse them - for c in components: - for a in archs: - # compile packages URL - pkgsurl = '/'.join([baseurl, c, 'binary-' + a, 'Packages.bz2']) - - # retrieve from cache - packages_file = self.cache.get(pkgsurl, - update=force_update) - - # parse - self._parsePkgsFile(packages_file, codename, c, baseurl) - - # cleanup - packages_file.close() - - # cleanup - release_file.close() - - self._updateReleases() - - - def _parsePkgsFile(self, fh, codename, component, baseurl): - """ - :Parameters: - fh: file handler - Packages list file - codename: str - Codename of the release - component: str - The archive component this packages file corresponds to. - """ - for stanza in deb822.Packages.iter_paragraphs(fh): - self._storePkg(stanza, codename, component, baseurl) - - - def _storePkg(self, st, codename, component, baseurl): - """ - :Parameter: - st: Package section - """ - pkg = st['Package'] - - if not self.has_section(pkg): - self.add_section(pkg) - - # do nothing if package is not in filter if there is any - if not self.pkgfilter is None and not pkg in self.pkgfilter: - self.ensureUnique(pkg, 'visibility', 'shadowed') - else: - self.ensureUnique(pkg, 'visibility', 'featured') - - # which releases - self.appendUniqueCSV(pkg, "releases", codename) - - # arch listing - self.appendUniqueCSV(pkg, "archs %s" % codename, st['Architecture']) - - # versions - self.ensureUnique(pkg, - "version %s %s" % (codename, st['Architecture']), - st['Version']) - - # link to .deb - self.ensureUnique(pkg, - "file %s %s" % (codename, st['Architecture']), - '/'.join(baseurl.split('/')[:-2] + [st['Filename']])) - - # component - self.ensureUnique(pkg, 'component ' + codename, component) - - # store the pool url - self.ensureUnique(pkg, "poolurl %s" % codename, - '/'.join(baseurl.split('/')[:-2] \ - + [os.path.dirname(st['Filename'])])) - - - # now the stuff where a single variant is sufficient and where we go for - # the latest available one - if self.has_option(pkg, "newest version") \ - and apt.VersionCompare(st['Version'], - self.get(pkg, "newest version")) < 0: - return - - # everything from here will overwrite existing ones - - # we seems to have an updated package - self.set(pkg, "newest version", st['Version']) - - # description - self.set(pkg, "description", st['Description'].replace('%', '%%')) - - # maintainer - self.set(pkg, "maintainer", st['Maintainer']) - - # optional stuff - if st.has_key('Homepage'): - self.set(pkg, 'homepage', st['Homepage']) - - # query debtags - debtags = self.dtags.tagsOfPackage(pkg) - if debtags: - self.set(pkg, 'debtags', ', '.join(debtags)) - - - def writeSourcesLists(self, outdir, cfg): - createDir(outdir) - createDir(os.path.join(outdir, 'static')) - - fl = open(os.path.join(outdir, 'sources_lists'), 'w') - for trans, r in sorted([(transCodename(k, cfg), k) - for k in self.releases.keys()]): - # need to turn 'apsy_lenny' back into 'lenny' - debneuro_r = r.split('_')[1] - - f = open(os.path.join(outdir, 'static', - 'debneuro.%s.sources.list' % debneuro_r), - 'w') - f.write("deb http://apsy.gse.uni-magdeburg.de/debian %s %s\n" \ - % (debneuro_r, ' '.join(self.releases[r]))) - f.write("deb-src http://apsy.gse.uni-magdeburg.de/debian %s %s\n" \ - % (debneuro_r, ' '.join(self.releases[r]))) - # XXX use :download: role from sphinx 0.6 on - #fl.write('* `%s `_\n' \ - fl.write('* `%s <_static/debneuro.%s.sources.list>`_\n' \ - % (trans, debneuro_r)) - f.close() - fl.close() - - - def importProspectivePkgsFromTaskFile(self, url): - fh = self.cache.get(url) - - for st in deb822.Packages.iter_paragraphs(fh): - # do not stop unless we have a description - if not st.has_key('Pkg-Description'): - continue - - if st.has_key('Depends'): - pkg = st['Depends'] - elif st.has_key('Suggests'): - pkg = st['Suggests'] - else: - print 'Warning: Cannot determine name of prospective package ' \ - '... ignoring.' - continue - - # store pkg info - if not self.has_section(pkg): - self.add_section(pkg) - - # prospective ones are always featured - self.ensureUnique(pkg, 'visibility', 'featured') - - # pkg description - self.set(pkg, "description", - st['Pkg-Description'].replace('%', '%%')) - - # optional stuff - if st.has_key('Homepage'): - self.set(pkg, 'homepage', st['Homepage']) - - if st.has_key('Pkg-URL'): - self.set(pkg, 'external pkg url', st['Pkg-URL']) - - if st.has_key('WNPP'): - self.set(pkg, 'wnpp debian', st['WNPP']) - - if st.has_key('License'): - self.set(pkg, 'license', st['License']) - - # treat responsible as maintainer - if st.has_key('Responsible'): - self.set(pkg, "maintainer", st['Responsible']) - - - def setPkgFilterFromTaskFile(self, urls): - pkgs = [] - - for task in urls: - fh = self.cache.get(task) - - - # loop over all stanzas - for stanza in deb822.Packages.iter_paragraphs(fh): - if stanza.has_key('Depends'): - pkg = stanza['Depends'] - elif stanza.has_key('Suggests'): - pkg = stanza['Suggests'] - else: - continue - - # account for multiple packages per line - if pkg.count(','): - pkgs += [p.strip() for p in pkg.split(',')] - else: - pkgs.append(pkg.strip()) - - # activate filter - self.pkgfilter = pkgs - - -def genPkgPage(db, pkg, cfg): - """ - :Parameters: - db: database - pkg: str - Package name - """ - descr = db.get(pkg, 'description').split('\n') - - s = '' - - # only put index markup for featured packages - if db.get(pkg, 'visibility') == 'featured': - s = '.. index:: %s, ' % pkg - s += '\n' - - if db.has_option(pkg, 'debtags'): - # filter tags - tags = [t for t in db.get(pkg, 'debtags').split(', ') - if t.split('::')[0] in ['field', 'works-with']] - if len(tags): - s += '.. index:: %s\n\n' % ', '.join(tags) - - # main ref target for this package - s += '.. _deb_' + pkg + ':\n' - - # separate header from the rest - s += '\n\n\n' - - header = '%s -- %s' % (pkg, descr[0]) - s += '*' * (len(header) + 2) - s += '\n ' + header + '\n' - s += '*' * (len(header) + 2) + '\n\n' - - # put description - s += '\n'.join([l.lstrip(' .') for l in descr[1:]]) - s += '\n' - - if db.has_option(pkg, 'homepage'): - s += '\n**Homepage**: %s\n' % db.get(pkg, 'homepage') - - s += '\nBinary packages'\ - '\n===============\n' - - s += genMaintainerSection(db, pkg) - - if db.has_option(pkg, 'wnpp debian'): - s += """\ -A Debian packaging effort has been officially announced. Please see the -corresponding `intent-to-package bug report`_ for more information about -its current status. - -.. _intent-to-package bug report: http://bugs.debian.org/%s - -""" % db.get(pkg, 'wnpp debian') - - s += genBinaryPackageSummary(db, pkg, 'DebNeuro repository', cfg) - -# if db.has_option(pkg, 'external pkg url'): -# s += 'Other unofficial ressources\n' \ -# '---------------------------\n\n' -# s += 'An unofficial package is available from %s\ .\n\n' \ -# % db.get(pkg, 'external pkg url') - return s - - -def genMaintainerSection(db, pkg): - s = '' - - if not db.has_option(pkg, 'maintainer'): - s += """\ - -Currently, nobody seems to be responsible for creating or maintaining -Debian packages of this software. - -""" - return s - - # there is someone responsible - maintainer = db.get(pkg, 'maintainer') - - # do we have actual packages, or is it just a note - if not db.has_option(pkg, 'releases'): - s += """\ - -There are currently no binary packages available. However, the last known -packaging effort was started by %s which meanwhile might have led to an -initial unofficial Debian packaging. - -""" % maintainer - return s - - s += '\n**Maintainer**: %s\n\n' % maintainer - - if not maintainer.startswith('Michael Hanke'): - s += """\ - -.. note:: - Do not contact the original package maintainer regarding - bugs in this unofficial binary package. Instead, contact - the repository maintainer at michael.hanke@gmail.com\ . - -""" - - return s - - -def genBinaryPackageSummary(db, pkg, reposname, cfg): - # do nothing if the are no packages - if not db.has_option(pkg, 'releases'): - return '' - - s = '\n%s\n%s\n' % (reposname, '-' * len(reposname)) - - s += """\ -The repository contains binary packages for the following distribution -releases and system architectures. The corresponding source packages -are available too. Please click on the release name to access them. - -.. note:: - Do not download this package manually if you plan to use it - regularly. Instead configure your package manager to use this - repository by following the instructions on the - :ref:`front page `. - -""" - - # for all releases this package is part of - for rel in db.get(pkg, 'releases').split(', '): - # write release description and component - s += '\n`%s <%s>`_:\n ' \ - % (transCodename(rel, cfg), - db.get(pkg, 'poolurl %s' % rel)) - - s += '[%s] ' % db.get(pkg, 'component ' + rel) - - # archs this package is available for - archs = db.get(pkg, 'archs ' + rel).split(', ') - - # extract all present versions for any arch - versions = [db.get(pkg, 'version %s %s' % (rel, arch)) - for arch in archs] - - # if there is only a single version for all of them, simplify the list - single_ver = versions.count(versions[0]) == len(versions) - - if single_ver: - # only one version string for all - s += ', '.join(['`%s <%s>`_' \ - % (arch, db.get(pkg, 'file %s %s' % (rel, arch))) - for arch in archs]) - s += ' (%s)' % versions[0] - else: - # a separate version string for each arch - s += ', '.join(['`%s <%s>`_ (%s)' \ - % (arch, - db.get(pkg, 'file %s %s' % (rel, arch)), - db.get(pkg, 'version %s %s' % (rel, arch))) - for arch in archs]) - - s += '\n' - - return s - -def maintainer2email(maint): - return maint.split('<')[1].rstrip('>') - - -def writePkgsBy(db, key, value2id, outdir, heading): - createDir(outdir) - nwkey = key.replace(' ', '') - createDir(os.path.join(outdir, 'by%s' % nwkey)) - - collector = {} - - # get packages by maintainer - for p in db.sections(): - # only featured packages - if db.get(p, 'visibility') == 'shadowed': - continue - - if db.has_option(p, key): - by = db.get(p, key) - - if not collector.has_key(by): - collector[by] = (value2id(by), [p]) - else: - collector[by][1].append(p) - - toc = open(os.path.join(outdir, 'by%s.rst' % nwkey), 'w') - toc.write('.. index:: Packages by %s\n.. _by%s:\n\n' % (key, key)) - - toc_heading = 'Packages by %s' % key - toc.write('%s\n%s\n\n' % (toc_heading, '=' * len(toc_heading))) - toc.write('.. toctree::\n :maxdepth: 1\n\n') - - # summary page per maintainer - for by in sorted(collector.keys()): - toc.write(' by%s/%s\n' % (nwkey, collector[by][0])) - - fh = open(os.path.join(outdir, - 'by%s' % nwkey, - collector[by][0] + '.rst'), 'w') - - fh.write('.. index:: %s\n.. _%s:\n\n' % (by, by)) - - hdr = heading.replace('', by) - fh.write(hdr + '\n') - fh.write('=' * len(hdr) + '\n\n') - - # write sorted list of packages - for p in sorted(collector[by][1]): - fh.write('* :ref:`deb_%s`\n' % p) - - fh.close() - - toc.close() - - -def writeRst(db, outdir, cfg, addenum_dir=None): - createDir(outdir) - createDir(os.path.join(outdir, 'pkgs')) - - # open pkgs toctree - toc = open(os.path.join(outdir, 'pkgs.rst'), 'w') - # write header - toc.write('.. _full_pkg_list:\n\n') - toc.write('Archive content\n===============\n\n' - '.. toctree::\n :maxdepth: 1\n\n') - - for p in sorted(db.sections()): - print "Generating page for '%s'" % p - pf = open(os.path.join(outdir, 'pkgs', '%s.rst' % p), 'w') - pf.write(genPkgPage(db, p, cfg)) - - # check for doc addons - if addenum_dir is not None: - addenum = os.path.join(os.path.abspath(addenum_dir), '%s.rst' % p) - if os.path.exists(addenum): - pf.write('\n\n.. include:: %s\n' % addenum) - pf.close() - toc.write(' pkgs/%s\n' % p) - - - toc.close() - - -def prepOptParser(op): - # use module docstring for help output - op.usage = "%s [OPTIONS]\n\n" % sys.argv[0] + __doc__ - - op.add_option("--db", - action="store", type="string", dest="db", - default=None, - help="Database file to read. Default: None") - - op.add_option("--cfg", - action="store", type="string", dest="cfg", - default=None, - help="Repository config file.") - - op.add_option("-o", "--outdir", - action="store", type="string", dest="outdir", - default=None, - help="Target directory for ReST output. Default: None") - - op.add_option("-r", "--release-url", - action="append", dest="release_urls", - help="None") - - op.add_option("--pkgaddenum", action="store", dest="addenum_dir", - type="string", default=None, help="None") - - - -def main(): - op = OptionParser(version="%prog 0.0.1") - prepOptParser(op) - - (opts, args) = op.parse_args() - - if len(args) != 1: - print('There needs to be exactly one command') - sys.exit(1) - - cmd = args[0] - - if opts.cfg is None: - print("'--cfg' option is mandatory.") - sys.exit(1) - - - cfg = SafeConfigParser() - cfg.read(opts.cfg) - - # load existing db, unless renew is requested - if cmd == 'refreshdb': - dpa = DebianPkgArchive() - else: - dpa = DebianPkgArchive(init_db=opts.db) - - - if cmd == 'generate': - if opts.outdir is None: - print('Not output directory specified!') - sys.exit(1) - - dpa.writeSourcesLists(opts.outdir, cfg) - writeRst(dpa, opts.outdir, cfg, opts.addenum_dir) - writePkgsBy(dpa, 'maintainer', maintainer2email, opts.outdir, - 'Packages maintained by ') - # stop here - sys.exit(0) - - - if cfg.has_option('packages', 'select taskfiles'): - dpa.setPkgFilterFromTaskFile(cfg.get('packages', - 'select taskfiles').split()) - - if cfg.has_option('packages', 'select names'): - dpa.pkgfilter += cfg.get('packages', 'select names').split() - - if cfg.has_option('packages', 'prospective'): - for p in cfg.get('packages', 'prospective').split(): - dpa.importProspectivePkgsFromTaskFile(p) - - if cfg.has_option('repositories', 'releases'): - for rurl in cfg.get('repositories', 'releases').split(): - dpa.importRelease(rurl, force_update=False) - - if not opts.db is None: - dpa.save(opts.db) - - -if __name__ == "__main__": - main() - diff --git a/sphinx/index.rst b/sphinx/index.rst index d790a88..a9e111f 100644 --- a/sphinx/index.rst +++ b/sphinx/index.rst @@ -18,8 +18,8 @@ is simply useful in the neuroscience context. The featured neuroscience software can be browsed via the repository :ref:`genindex` or through the :ref:`maintainer view `. -All other packages are available through the search engine or from the -:ref:`full package list `. +All other packages are available from the :ref:`full package list +`. News diff --git a/tools/reblender b/tools/reblender new file mode 100755 index 0000000..4e1f354 --- /dev/null +++ b/tools/reblender @@ -0,0 +1,796 @@ +#!/usr/bin/env python +"""Tell me who you are! +""" + +import urllib +import apt +from debian_bundle import deb822 +from debian_bundle import debtags +from ConfigParser import SafeConfigParser +import subprocess +import os +import sys +import shutil +import pysvn +from optparse import OptionParser, Option, OptionGroup, OptionConflictError + + +def transCodename(codename, cfg): + """Translate a known codename into a release description. + + Unknown codenames will simply be returned as is. + """ + # strip repository codename + codename = codename[codename.find('_') + 1:] + + # if we know something, tell + if codename in cfg.options('release codenames'): + return cfg.get('release codenames', codename) + else: + return codename + + +def createDir(path): + if os.path.exists(path): + return + + ps = path.split(os.path.sep) + + for i in range(1,len(ps) + 1): + p = os.path.sep.join(ps[:i]) + + if not os.path.exists(p): + os.mkdir(p) + + +class AptListsCache(object): + def __init__(self, cachedir='cache', ro_cachedirs=None, init_db=None): + self.cachedir = cachedir + + if not ro_cachedirs is None: + self.ro_cachedirs = ro_cachedirs + else: + self.ro_cachedirs = [] + + # always use system cache + #self.ro_cachedirs.append('/var/lib/apt/lists/') + + # create cachedir + createDir(self.cachedir) + + + def get(self, url, update=False): + """Looks in the cache if the file is there and takes the cached one. + Otherwise it is downloaded first. + + Knows how to deal with http:// and svn:// URLs. + + :Return: + file handler + """ + # look whether it is compressed + cext = url.split('.')[-1] + if cext in ['gz', 'bz2']: + target_url = url[:-1 * len(cext) -1] + else: + # assume not compressed + target_url = url + cext = None + + # turn url into a filename -- mimik what APT does for + # /var/lib/apt/lists/ + tfilename = '_'.join(target_url.split('/')[2:]) + + # if we need to download anyway do not search + if update: + cfilename = os.path.join(self.cachedir, tfilename) + else: + # look for the uncompressed file anywhere in the cache + cfilename = None + for cp in [self.cachedir] + self.ro_cachedirs: + if os.path.exists(os.path.join(cp, tfilename)): + cfilename = os.path.join(cp, tfilename) + + # nothing found? + if cfilename is None: + # add cache item + cfilename = os.path.join(self.cachedir, tfilename) + update = True + + # if updated needed -- download + if update: + print 'Caching file from %s' % url + + if url.startswith('svn://'): + # export from SVN + pysvn.Client().export(url, cfilename) + if url.startswith('http://'): + # download + tempfile, ignored = urllib.urlretrieve(url) + + # decompress + decompressor = None + if cext == 'gz': + decompressor = 'gzip' + elif cext == 'bz2': + decompressor = 'bzip2' + elif cext == None: + decompressor = None + else: + raise ValueError, \ + "Don't know how to decompress %s files" \ + % cext + + if not decompressor is None: + if subprocess.call([decompressor, '-d', '-q', '-f', + tempfile]) == 1: + raise RuntimeError, \ + "Something went wrong while decompressing '%s'" \ + % tempfile + + # move decompressed file into cache + shutil.move(os.path.splitext(tempfile)[0], cfilename) + + # XXX do we need that if explicit filename is provided? + urllib.urlcleanup() + + # open cached file + fh = open(cfilename, 'r') + + return fh + + + + +class DebianPkgArchive(SafeConfigParser): + """ + """ + def __init__(self, cache=None, init_db=None): + """ + :Parameter: + """ + SafeConfigParser.__init__(self) + + # read an existing database if provided + if not init_db is None: + self.read(init_db) + + # use provided file cache or use fresh one + if not cache is None: + self.cache = cache + else: + self.cache = AptListsCache() + + # init debtags DB + self.dtags = debtags.DB() + self.dtags.read(open('/var/lib/debtags/package-tags')) + + # init package filter + self.pkgfilter = None + + self._updateReleases() + + + def _updateReleases(self): + self.releases = {} + + for p in self.sections(): + if not self.has_option(p, 'releases'): + continue + + # for all releases of this package + for r in \ + [rel.strip() for rel in self.get(p, 'releases').split(',')]: + # push release code + if not self.releases.has_key(r): + self.releases[r] = [] + + # store component + component = self.get(p, 'component %s' % r) + + if not component in self.releases[r]: + self.releases[r].append(component) + + + def __repr__(self): + """Generate INI file content for current content. + """ + # make adaptor to use str as file-like (needed for ConfigParser.write() + class file2str(object): + def __init__(self): + self.__s = '' + def write(self, val): + self.__s += val + def str(self): + return self.__s + + r = file2str() + self.write(r) + + return r.str() + + + def save(self, filename): + """Write current content to a file. + """ + f = open(filename, 'w') + self.write(f) + f.close() + + + def ensureUnique(self, section, option, value): + if not self.has_option(section, option): + self.set(section, option, value) + else: + if not self.get(section, option) == value: + raise ValueError, "%s: %s is not unique (%s != %s)" \ + % (section, option, + self.get(section, option), value) + + + def appendUniqueCSV(self, section, option, value): + """ + """ + if not self.has_option(section, option): + self.set(section, option, value) + else: + l = self.get(section, option).split(', ') + if not value in l: + self.set(section, option, ', '.join(l + [value])) + + + def importRelease(self, rurl, force_update=False): + # root URL of the repository + baseurl = '/'.join(rurl.split('/')[:-1]) + # get the release file from the cache + release_file = self.cache.get(rurl, update=force_update) + + # create parser instance + rp = deb822.Release(release_file) + + # architectures on this dist + archs = rp['Architectures'].split() + components = rp['Components'].split() + # compile a new codename that also considers the repository label + # to distinguish between official and unofficial repos. + codename = '_'.join([rp['Label'], rp['Codename']]) + + # compile the list of Packages files to parse and parse them + for c in components: + for a in archs: + # compile packages URL + pkgsurl = '/'.join([baseurl, c, 'binary-' + a, 'Packages.bz2']) + + # retrieve from cache + packages_file = self.cache.get(pkgsurl, + update=force_update) + + # parse + self._parsePkgsFile(packages_file, codename, c, baseurl) + + # cleanup + packages_file.close() + + # cleanup + release_file.close() + + self._updateReleases() + + + def _parsePkgsFile(self, fh, codename, component, baseurl): + """ + :Parameters: + fh: file handler + Packages list file + codename: str + Codename of the release + component: str + The archive component this packages file corresponds to. + """ + for stanza in deb822.Packages.iter_paragraphs(fh): + self._storePkg(stanza, codename, component, baseurl) + + + def _storePkg(self, st, codename, component, baseurl): + """ + :Parameter: + st: Package section + """ + pkg = st['Package'] + + if not self.has_section(pkg): + self.add_section(pkg) + + # do nothing if package is not in filter if there is any + if not self.pkgfilter is None and not pkg in self.pkgfilter: + self.ensureUnique(pkg, 'visibility', 'shadowed') + else: + self.ensureUnique(pkg, 'visibility', 'featured') + + # which releases + self.appendUniqueCSV(pkg, "releases", codename) + + # arch listing + self.appendUniqueCSV(pkg, "archs %s" % codename, st['Architecture']) + + # versions + self.ensureUnique(pkg, + "version %s %s" % (codename, st['Architecture']), + st['Version']) + + # link to .deb + self.ensureUnique(pkg, + "file %s %s" % (codename, st['Architecture']), + '/'.join(baseurl.split('/')[:-2] + [st['Filename']])) + + # component + self.ensureUnique(pkg, 'component ' + codename, component) + + # store the pool url + self.ensureUnique(pkg, "poolurl %s" % codename, + '/'.join(baseurl.split('/')[:-2] \ + + [os.path.dirname(st['Filename'])])) + + + # now the stuff where a single variant is sufficient and where we go for + # the latest available one + if self.has_option(pkg, "newest version") \ + and apt.VersionCompare(st['Version'], + self.get(pkg, "newest version")) < 0: + return + + # everything from here will overwrite existing ones + + # we seems to have an updated package + self.set(pkg, "newest version", st['Version']) + + # description + self.set(pkg, "description", st['Description'].replace('%', '%%')) + + # maintainer + self.set(pkg, "maintainer", st['Maintainer']) + + # optional stuff + if st.has_key('Homepage'): + self.set(pkg, 'homepage', st['Homepage']) + + # query debtags + debtags = self.dtags.tagsOfPackage(pkg) + if debtags: + self.set(pkg, 'debtags', ', '.join(debtags)) + + + def writeSourcesLists(self, outdir, cfg): + createDir(outdir) + createDir(os.path.join(outdir, 'static')) + + fl = open(os.path.join(outdir, 'sources_lists'), 'w') + for trans, r in sorted([(transCodename(k, cfg), k) + for k in self.releases.keys()]): + # need to turn 'apsy_lenny' back into 'lenny' + debneuro_r = r.split('_')[1] + + f = open(os.path.join(outdir, 'static', + 'debneuro.%s.sources.list' % debneuro_r), + 'w') + f.write("deb http://apsy.gse.uni-magdeburg.de/debian %s %s\n" \ + % (debneuro_r, ' '.join(self.releases[r]))) + f.write("deb-src http://apsy.gse.uni-magdeburg.de/debian %s %s\n" \ + % (debneuro_r, ' '.join(self.releases[r]))) + # XXX use :download: role from sphinx 0.6 on + #fl.write('* `%s `_\n' \ + fl.write('* `%s <_static/debneuro.%s.sources.list>`_\n' \ + % (trans, debneuro_r)) + f.close() + fl.close() + + + def importProspectivePkgsFromTaskFile(self, url): + fh = self.cache.get(url) + + for st in deb822.Packages.iter_paragraphs(fh): + # do not stop unless we have a description + if not st.has_key('Pkg-Description'): + continue + + if st.has_key('Depends'): + pkg = st['Depends'] + elif st.has_key('Suggests'): + pkg = st['Suggests'] + else: + print 'Warning: Cannot determine name of prospective package ' \ + '... ignoring.' + continue + + # store pkg info + if not self.has_section(pkg): + self.add_section(pkg) + + # prospective ones are always featured + self.ensureUnique(pkg, 'visibility', 'featured') + + # pkg description + self.set(pkg, "description", + st['Pkg-Description'].replace('%', '%%')) + + # optional stuff + if st.has_key('Homepage'): + self.set(pkg, 'homepage', st['Homepage']) + + if st.has_key('Pkg-URL'): + self.set(pkg, 'external pkg url', st['Pkg-URL']) + + if st.has_key('WNPP'): + self.set(pkg, 'wnpp debian', st['WNPP']) + + if st.has_key('License'): + self.set(pkg, 'license', st['License']) + + # treat responsible as maintainer + if st.has_key('Responsible'): + self.set(pkg, "maintainer", st['Responsible']) + + + def setPkgFilterFromTaskFile(self, urls): + pkgs = [] + + for task in urls: + fh = self.cache.get(task) + + + # loop over all stanzas + for stanza in deb822.Packages.iter_paragraphs(fh): + if stanza.has_key('Depends'): + pkg = stanza['Depends'] + elif stanza.has_key('Suggests'): + pkg = stanza['Suggests'] + else: + continue + + # account for multiple packages per line + if pkg.count(','): + pkgs += [p.strip() for p in pkg.split(',')] + else: + pkgs.append(pkg.strip()) + + # activate filter + self.pkgfilter = pkgs + + +def genPkgPage(db, pkg, cfg): + """ + :Parameters: + db: database + pkg: str + Package name + """ + descr = db.get(pkg, 'description').split('\n') + + s = '' + + # only put index markup for featured packages + if db.get(pkg, 'visibility') == 'featured': + s = '.. index:: %s, ' % pkg + s += '\n' + + if db.has_option(pkg, 'debtags'): + # filter tags + tags = [t for t in db.get(pkg, 'debtags').split(', ') + if t.split('::')[0] in ['field', 'works-with']] + if len(tags): + s += '.. index:: %s\n\n' % ', '.join(tags) + + # main ref target for this package + s += '.. _deb_' + pkg + ':\n' + + # separate header from the rest + s += '\n\n\n' + + header = '%s -- %s' % (pkg, descr[0]) + s += '*' * (len(header) + 2) + s += '\n ' + header + '\n' + s += '*' * (len(header) + 2) + '\n\n' + + # put description + s += '\n'.join([l.lstrip(' .') for l in descr[1:]]) + s += '\n' + + if db.has_option(pkg, 'homepage'): + s += '\n**Homepage**: %s\n' % db.get(pkg, 'homepage') + + s += '\nBinary packages'\ + '\n===============\n' + + s += genMaintainerSection(db, pkg) + + if db.has_option(pkg, 'wnpp debian'): + s += """\ +A Debian packaging effort has been officially announced. Please see the +corresponding `intent-to-package bug report`_ for more information about +its current status. + +.. _intent-to-package bug report: http://bugs.debian.org/%s + +""" % db.get(pkg, 'wnpp debian') + + s += genBinaryPackageSummary(db, pkg, 'DebNeuro repository', cfg) + +# if db.has_option(pkg, 'external pkg url'): +# s += 'Other unofficial ressources\n' \ +# '---------------------------\n\n' +# s += 'An unofficial package is available from %s\ .\n\n' \ +# % db.get(pkg, 'external pkg url') + return s + + +def genMaintainerSection(db, pkg): + s = '' + + if not db.has_option(pkg, 'maintainer'): + s += """\ + +Currently, nobody seems to be responsible for creating or maintaining +Debian packages of this software. + +""" + return s + + # there is someone responsible + maintainer = db.get(pkg, 'maintainer') + + # do we have actual packages, or is it just a note + if not db.has_option(pkg, 'releases'): + s += """\ + +There are currently no binary packages available. However, the last known +packaging effort was started by %s which meanwhile might have led to an +initial unofficial Debian packaging. + +""" % maintainer + return s + + s += '\n**Maintainer**: %s\n\n' % maintainer + + if not maintainer.startswith('Michael Hanke'): + s += """\ + +.. note:: + Do not contact the original package maintainer regarding + bugs in this unofficial binary package. Instead, contact + the repository maintainer at michael.hanke@gmail.com\ . + +""" + + return s + + +def genBinaryPackageSummary(db, pkg, reposname, cfg): + # do nothing if the are no packages + if not db.has_option(pkg, 'releases'): + return '' + + s = '\n%s\n%s\n' % (reposname, '-' * len(reposname)) + + s += """\ +The repository contains binary packages for the following distribution +releases and system architectures. The corresponding source packages +are available too. Please click on the release name to access them. + +.. note:: + Do not download this package manually if you plan to use it + regularly. Instead configure your package manager to use this + repository by following the instructions on the + :ref:`front page `. + +""" + + # for all releases this package is part of + for rel in db.get(pkg, 'releases').split(', '): + # write release description and component + s += '\n`%s <%s>`_:\n ' \ + % (transCodename(rel, cfg), + db.get(pkg, 'poolurl %s' % rel)) + + s += '[%s] ' % db.get(pkg, 'component ' + rel) + + # archs this package is available for + archs = db.get(pkg, 'archs ' + rel).split(', ') + + # extract all present versions for any arch + versions = [db.get(pkg, 'version %s %s' % (rel, arch)) + for arch in archs] + + # if there is only a single version for all of them, simplify the list + single_ver = versions.count(versions[0]) == len(versions) + + if single_ver: + # only one version string for all + s += ', '.join(['`%s <%s>`_' \ + % (arch, db.get(pkg, 'file %s %s' % (rel, arch))) + for arch in archs]) + s += ' (%s)' % versions[0] + else: + # a separate version string for each arch + s += ', '.join(['`%s <%s>`_ (%s)' \ + % (arch, + db.get(pkg, 'file %s %s' % (rel, arch)), + db.get(pkg, 'version %s %s' % (rel, arch))) + for arch in archs]) + + s += '\n' + + return s + +def maintainer2email(maint): + return maint.split('<')[1].rstrip('>') + + +def writePkgsBy(db, key, value2id, outdir, heading): + createDir(outdir) + nwkey = key.replace(' ', '') + createDir(os.path.join(outdir, 'by%s' % nwkey)) + + collector = {} + + # get packages by maintainer + for p in db.sections(): + # only featured packages + if db.get(p, 'visibility') == 'shadowed': + continue + + if db.has_option(p, key): + by = db.get(p, key) + + if not collector.has_key(by): + collector[by] = (value2id(by), [p]) + else: + collector[by][1].append(p) + + toc = open(os.path.join(outdir, 'by%s.rst' % nwkey), 'w') + toc.write('.. index:: Packages by %s\n.. _by%s:\n\n' % (key, key)) + + toc_heading = 'Packages by %s' % key + toc.write('%s\n%s\n\n' % (toc_heading, '=' * len(toc_heading))) + toc.write('.. toctree::\n :maxdepth: 1\n\n') + + # summary page per maintainer + for by in sorted(collector.keys()): + toc.write(' by%s/%s\n' % (nwkey, collector[by][0])) + + fh = open(os.path.join(outdir, + 'by%s' % nwkey, + collector[by][0] + '.rst'), 'w') + + fh.write('.. index:: %s\n.. _%s:\n\n' % (by, by)) + + hdr = heading.replace('', by) + fh.write(hdr + '\n') + fh.write('=' * len(hdr) + '\n\n') + + # write sorted list of packages + for p in sorted(collector[by][1]): + fh.write('* :ref:`deb_%s`\n' % p) + + fh.close() + + toc.close() + + +def writeRst(db, outdir, cfg, addenum_dir=None): + createDir(outdir) + createDir(os.path.join(outdir, 'pkgs')) + + # open pkgs toctree + toc = open(os.path.join(outdir, 'pkgs.rst'), 'w') + # write header + toc.write('.. _full_pkg_list:\n\n') + toc.write('Archive content\n===============\n\n' + '.. toctree::\n :maxdepth: 1\n\n') + + for p in sorted(db.sections()): + print "Generating page for '%s'" % p + pf = open(os.path.join(outdir, 'pkgs', '%s.rst' % p), 'w') + pf.write(genPkgPage(db, p, cfg)) + + # check for doc addons + if addenum_dir is not None: + addenum = os.path.join(os.path.abspath(addenum_dir), '%s.rst' % p) + if os.path.exists(addenum): + pf.write('\n\n.. include:: %s\n' % addenum) + pf.close() + toc.write(' pkgs/%s\n' % p) + + + toc.close() + + +def prepOptParser(op): + # use module docstring for help output + op.usage = "%s [OPTIONS]\n\n" % sys.argv[0] + __doc__ + + op.add_option("--db", + action="store", type="string", dest="db", + default=None, + help="Database file to read. Default: None") + + op.add_option("--cfg", + action="store", type="string", dest="cfg", + default=None, + help="Repository config file.") + + op.add_option("-o", "--outdir", + action="store", type="string", dest="outdir", + default=None, + help="Target directory for ReST output. Default: None") + + op.add_option("-r", "--release-url", + action="append", dest="release_urls", + help="None") + + op.add_option("--pkgaddenum", action="store", dest="addenum_dir", + type="string", default=None, help="None") + + + +def main(): + op = OptionParser(version="%prog 0.0.1") + prepOptParser(op) + + (opts, args) = op.parse_args() + + if len(args) != 1: + print('There needs to be exactly one command') + sys.exit(1) + + cmd = args[0] + + if opts.cfg is None: + print("'--cfg' option is mandatory.") + sys.exit(1) + + + cfg = SafeConfigParser() + cfg.read(opts.cfg) + + # load existing db, unless renew is requested + if cmd == 'refreshdb': + dpa = DebianPkgArchive() + else: + dpa = DebianPkgArchive(init_db=opts.db) + + + if cmd == 'generate': + if opts.outdir is None: + print('Not output directory specified!') + sys.exit(1) + + dpa.writeSourcesLists(opts.outdir, cfg) + writeRst(dpa, opts.outdir, cfg, opts.addenum_dir) + writePkgsBy(dpa, 'maintainer', maintainer2email, opts.outdir, + 'Packages maintained by ') + # stop here + sys.exit(0) + + + if cfg.has_option('packages', 'select taskfiles'): + dpa.setPkgFilterFromTaskFile(cfg.get('packages', + 'select taskfiles').split()) + + if cfg.has_option('packages', 'select names'): + dpa.pkgfilter += cfg.get('packages', 'select names').split() + + if cfg.has_option('packages', 'prospective'): + for p in cfg.get('packages', 'prospective').split(): + dpa.importProspectivePkgsFromTaskFile(p) + + if cfg.has_option('repositories', 'releases'): + for rurl in cfg.get('repositories', 'releases').split(): + dpa.importRelease(rurl, force_update=False) + + if not opts.db is None: + dpa.save(opts.db) + + +if __name__ == "__main__": + main() +