From 61b1bfd9f02bba15aba70b0cb5efbe971a2da7a9 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Sun, 5 Apr 2009 21:43:42 +0200 Subject: [PATCH 1/1] Stage one: commandline interface next. --- Makefile | 2 +- deb2rst/archive.py => reblender | 158 ++++++++++++++++++++++---------- source/index.rst | 10 +- 3 files changed, 119 insertions(+), 51 deletions(-) rename deb2rst/archive.py => reblender (83%) mode change 100644 => 100755 diff --git a/Makefile b/Makefile index 1d79dab..0e76bed 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,7 @@ linkcheck: prep update-db: - python deb2rst/archive.py + ./reblender upload-website: html diff --git a/deb2rst/archive.py b/reblender old mode 100644 new mode 100755 similarity index 83% rename from deb2rst/archive.py rename to reblender index 62d68ad..32d9932 --- a/deb2rst/archive.py +++ b/reblender @@ -1,3 +1,6 @@ +#!/usr/bin/env python + + import urllib import apt from debian_bundle import deb822 @@ -8,8 +11,6 @@ import os import shutil import pysvn -target_dir = 'build/source' - codename2descr = { 'apsy_etch': 'Debian GNU/Linux 4.0 (etch)', 'apsy_lenny': 'Debian GNU/Linux 5.0 (lenny)', @@ -35,6 +36,18 @@ def transCodename(codename): return codename +def createDir(path): + if os.path.exists(path): + return + + ps = path.split(os.path.sep) + + for i in range(1,len(ps) + 1): + p = os.path.sep.join(ps[:i]) + + if not os.path.exists(p): + os.mkdir(p) + class AptListsCache(object): def __init__(self, cachedir='cache', ro_cachedirs=None): @@ -48,6 +61,9 @@ class AptListsCache(object): # always use system cache self.ro_cachedirs.append('/var/lib/apt/lists/') + # create cachedir + createDir(self.cachedir) + def get(self, url, update=False): """Looks in the cache if the file is there and takes the cached one. @@ -266,13 +282,15 @@ class DebianPkgArchive(SafeConfigParser): """ pkg = st['Package'] - # do nothing if package is not in filter if there is any - if not self.pkgfilter is None and not pkg in self.pkgfilter: - return - if not self.has_section(pkg): self.add_section(pkg) + # do nothing if package is not in filter if there is any + if not self.pkgfilter is None and not pkg in self.pkgfilter: + self.ensureUnique(pkg, 'visibility', 'shadowed') + else: + self.ensureUnique(pkg, 'visibility', 'featured') + # which releases self.appendUniqueCSV(pkg, "releases", codename) @@ -326,12 +344,15 @@ class DebianPkgArchive(SafeConfigParser): self.set(pkg, 'debtags', ', '.join(debtags)) - def writeSourcesLists(self): - fl = open(os.path.join(target_dir, 'sources_lists'), 'w') + def writeSourcesLists(self, outdir): + createDir(outdir) + createDir(os.path.join(outdir, 'static')) + + fl = open(os.path.join(outdir, 'sources_lists'), 'w') for trans, r in sorted([(transCodename(k), k) for k in self.releases.keys()]): - f = open(os.path.join(target_dir, - "static/debneuro.%s.sources.list" % r), + f = open(os.path.join(outdir, 'static', + 'debneuro.%s.sources.list' % r), 'w') f.write("deb http://apsy.gse.uni-magdeburg.de/debian %s %s\n" \ % (r, ' '.join(self.releases[r]))) @@ -345,7 +366,7 @@ class DebianPkgArchive(SafeConfigParser): def importProspectivePkgsFromTaskFile(self, url): - fh = dpa.cache.get(url) + fh = self.cache.get(url) for st in deb822.Packages.iter_paragraphs(fh): # do not stop unless we have a description @@ -365,6 +386,9 @@ class DebianPkgArchive(SafeConfigParser): if not self.has_section(pkg): self.add_section(pkg) + # prospective ones are always featured + self.ensureUnique(pkg, 'visibility', 'featured') + # pkg description self.set(pkg, "description", st['Pkg-Description'].replace('%', '%%')) @@ -388,10 +412,11 @@ class DebianPkgArchive(SafeConfigParser): def setPkgFilterFromTaskFile(self, urls): - for task in taskfiles: - fh = dpa.cache.get(task) + pkgs = [] + + for task in urls: + fh = self.cache.get(task) - pkgs = [] # loop over all stanzas for stanza in deb822.Packages.iter_paragraphs(fh): @@ -421,17 +446,19 @@ def genPkgPage(db, pkg): """ descr = db.get(pkg, 'description').split('\n') - s = '.. index:: %s, ' % pkg -# s += db.get(pkg, 'maintainer').split('<')[0] + s = '' - s += '\n' + # only put index markup for featured packages + if db.get(pkg, 'visibility') == 'featured': + s = '.. index:: %s, ' % pkg + s += '\n' - if db.has_option(pkg, 'debtags'): - # filter tags - tags = [t for t in db.get(pkg, 'debtags').split(', ') - if t.split('::')[0] in ['field', 'works-with']] - if len(tags): - s += '.. index:: %s\n\n' % ', '.join(tags) + if db.has_option(pkg, 'debtags'): + # filter tags + tags = [t for t in db.get(pkg, 'debtags').split(', ') + if t.split('::')[0] in ['field', 'works-with']] + if len(tags): + s += '.. index:: %s\n\n' % ', '.join(tags) # main ref target for this package s += '.. _deb_' + pkg + ':\n' @@ -556,11 +583,19 @@ def maintainer2email(maint): return maint.split('<')[1].rstrip('>') -def writePkgsBy(db, key, value2id): +def writePkgsBy(db, key, value2id, outdir, heading): + createDir(outdir) + nwkey = key.replace(' ', '') + createDir(os.path.join(outdir, 'by%s' % nwkey)) + collector = {} # get packages by maintainer for p in db.sections(): + # only featured packages + if db.get(p, 'visibility') == 'shadowed': + continue + if db.has_option(p, key): by = db.get(p, key) @@ -569,34 +604,54 @@ def writePkgsBy(db, key, value2id): else: collector[by][1].append(p) - toc = open(os.path.join(target_dir, 'by%s.rst' % key), 'w') - toc.write('.. index:: Packages by %s\n.. _by%s:\n' % (key, key)) + toc = open(os.path.join(outdir, 'by%s.rst' % nwkey), 'w') + toc.write('.. index:: Packages by %s\n.. _by%s:\n\n' % (key, key)) - heading = 'Packages by %s' % key - toc.write('%s\n%s\n\n' % (heading, '=' * len(heading))) + toc_heading = 'Packages by %s' % key + toc.write('%s\n%s\n\n' % (toc_heading, '=' * len(toc_heading))) toc.write('.. toctree::\n :maxdepth: 1\n\n') # summary page per maintainer for by in sorted(collector.keys()): - toc.write(' by%s/%s\n' % (key, collector[by][0])) + toc.write(' by%s/%s\n' % (nwkey, collector[by][0])) + + fh = open(os.path.join(outdir, + 'by%s' % nwkey, + collector[by][0] + '.rst'), 'w') + + fh.write('.. index:: %s\n.. _%s:\n\n' % (by, by)) + + hdr = heading.replace('', by) + fh.write(hdr + '\n') + fh.write('=' * len(hdr) + '\n\n') + + # write sorted list of packages + for p in sorted(collector[by][1]): + fh.write('* :ref:`deb_%s`\n' % p) + + fh.close() toc.close() -def writeRst(db): +def writeRst(db, outdir): + createDir(outdir) + createDir(os.path.join(outdir, 'pkgs')) + # open pkgs toctree - toc = open(os.path.join(target_dir, 'pkgs.rst'), 'w') + toc = open(os.path.join(outdir, 'pkgs.rst'), 'w') # write header + toc.write('.. _full_pkg_list:\n\n') toc.write('Archive content\n===============\n\n' '.. toctree::\n :maxdepth: 1\n\n') for p in sorted(db.sections()): print "Generating page for '%s'" % p - pf = open(os.path.join(target_dir, 'pkgs/%s.rst' % p), 'w') + pf = open(os.path.join(outdir, 'pkgs', '%s.rst' % p), 'w') pf.write(genPkgPage(db, p)) # check for doc addons - if os.path.exists(os.path.join(target_dir, 'pkgs_addenum/%s.rst' % p)): + if os.path.exists(os.path.join(outdir, 'pkgs_addenum/%s.rst' % p)): pf.write('\n\n.. include:: ../pkgs_addenum/%s.rst\n' %p) pf.close() toc.write(' pkgs/%s\n' % p) @@ -606,10 +661,13 @@ def writeRst(db): -dpa = DebianPkgArchive() +def main(): + dpa = DebianPkgArchive() + target_dir = 'build/source' -release_urls=[ + + release_urls=[ 'http://apsy.gse.uni-magdeburg.de/debian/dists/dapper/Release', 'http://apsy.gse.uni-magdeburg.de/debian/dists/gutsy/Release', 'http://apsy.gse.uni-magdeburg.de/debian/dists/hardy/Release', @@ -620,26 +678,30 @@ release_urls=[ 'http://apsy.gse.uni-magdeburg.de/debian/dists/sid/Release', ] -taskfiles = [ - 'svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging', - 'svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging-dev', - 'svn://svn.debian.org/blends/projects/science/trunk/debian-science/tasks/neuroscience-cognitive', + taskfiles = [ + 'svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging', + 'svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging-dev', + 'svn://svn.debian.org/blends/projects/science/trunk/debian-science/tasks/neuroscience-cognitive', ] -dpa.setPkgFilterFromTaskFile(taskfiles) -dpa.pkgfilter += ['fsl-doc', 'fslview-doc', 'fsl-atlases', 'fsl-possum-data', + dpa.setPkgFilterFromTaskFile(taskfiles) + dpa.pkgfilter += ['fsl-doc', 'fslview-doc', 'fsl-atlases', 'fsl-possum-data', 'fsl-first-data', 'fsl-feeds'] -dpa.importProspectivePkgsFromTaskFile(taskfiles[0]) + dpa.importProspectivePkgsFromTaskFile(taskfiles[0]) + + for rurl in release_urls: + dpa.importRelease(rurl, force_update=False) -for rurl in release_urls: - dpa.importRelease(rurl, force_update=False) + dpa.writeSourcesLists(target_dir) -dpa.save('db.db') + writeRst(dpa, target_dir) + writePkgsBy(dpa, 'maintainer', maintainer2email, target_dir, + 'Packages maintained by ') -dpa.writeSourcesLists() + dpa.save('build/db.db') -writeRst(dpa) -writePkgsBy(dpa, 'maintainer', maintainer2email) +if __name__ == "__main__": + main() diff --git a/source/index.rst b/source/index.rst index 50b233f..1ad9ceb 100644 --- a/source/index.rst +++ b/source/index.rst @@ -12,8 +12,14 @@ This service is provided "as is". There is no guarantee that a package works as expected, so use them at your own risk. They might kill your system (although that is rather unlikely). You've been warned! -An exhaustive list of available packages is provided by the Package -:ref:`genindex`. +The repository contains both neuroscience-related packages as well as +general purpose software which is necessary to resolved dependencies, or +is simply useful in the neuroscience context. The featured neuroscience +software can be browsed via the repository :ref:`genindex` or through the +:ref:`maintainer view `. + +All other packages are available through the search engine or from the +:ref:`full package list `. How to use this repository -- 2.39.2