From 36964cd6f0a64c22e36a6fd2fbb7cd193cda887d Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 25 Aug 2009 14:44:11 -0400 Subject: [PATCH] Make it work with unicode. --- neurodebian.cfg | 8 ++- neurodebian/dde.py | 99 ++++++++++++++++++----------------- neurodebian/templates/pkg.rst | 2 +- 3 files changed, 55 insertions(+), 54 deletions(-) diff --git a/neurodebian.cfg b/neurodebian.cfg index c289d95..7fa8cff 100644 --- a/neurodebian.cfg +++ b/neurodebian.cfg @@ -1,7 +1,4 @@ [dde] -dists = debian-lenny debian-squeeze debian-sid debian-experimental - ubuntu-hardy ubuntu-intrepid ubuntu-jaunty ubuntu-karmic - pkgquery_url = http://dde.debian.net/dde/q/udd [packages] @@ -9,6 +6,8 @@ pkgquery_url = http://dde.debian.net/dde/q/udd # _if_ they are also present in the repository select taskfiles = svn://svn.debian.org/blends/projects/science/trunk/debian-science/tasks/neuroscience-cognitive + svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging + svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging-dev # Additional selection filter (similar to 'select taskfiles'), only listing # package names @@ -19,7 +18,7 @@ select names = fsl-doc fslview-doc fsl-atlases fsl-possum-data fsl-first-data prospective = svn://svn.debian.org/blends/projects/science/trunk/debian-science/tasks/neuroscience-cognitive svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging -# svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging-dev + svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging-dev [mirrors] us = http://neuro.debian.net/debian @@ -37,7 +36,6 @@ releases = http://neuro.debian.net/debian/dists/lenny/Release http://neuro.debian.net/debian/dists/squeeze/Release http://neuro.debian.net/debian/dists/sid/Release - http://neuro.debian.net/debian/dists/sid/Release [release codenames] etch = Debian GNU/Linux 4.0 (etch) diff --git a/neurodebian/dde.py b/neurodebian/dde.py index bc03b8c..0ebb0ee 100644 --- a/neurodebian/dde.py +++ b/neurodebian/dde.py @@ -13,6 +13,7 @@ import os import shutil import urllib2 import urllib +import codecs import subprocess # templating from jinja2 import Environment, PackageLoader @@ -110,7 +111,7 @@ class AptListsCache(object): urllib.urlcleanup() # open cached file - fh = open(cfilename, 'r') + fh = codecs.open(cfilename, 'r', 'utf-8') return fh @@ -167,53 +168,53 @@ def import_blendstask(db, url): task_name = st['Task'] task = (blendname, task_name, taskpage_url) - # do not stop unless we have a description - if not st.has_key('Pkg-Description'): - continue - if st.has_key('Depends'): pkg = st['Depends'] elif st.has_key('Suggests'): pkg = st['Suggests'] else: - print 'Warning: Cannot determine name of prospective package ' \ - '... ignoring.' - continue - - if not db.has_key(pkg): - print 'Ignoring blend package "%s"' % pkg +# print 'Warning: Cannot determine name of prospective package ' \ +# '... ignoring. Dump follows:' +# print st continue - info = {} + # take care of pkg lists + for p in pkg.split(', '): + if not db.has_key(p): + print 'Ignoring blend package "%s"' % p + continue - # blends info - info['tasks'] = [task] - if st.has_key('License'): - info['license'] = st['License'] - if st.has_key('Responsible'): - info['responsible'] = st['Responsible'] - - # pkg description - descr = st['Pkg-Description'].replace('%', '%%').split('\n') - info['description'] = descr[0].strip() - info['long_description'] = ' '.join([l.strip() for l in descr[1:]]) - - # charge the basic property set - db[pkg]['main']['description'] = info['description'] - db[pkg]['main']['long_description'] = info['long_description'] - if st.has_key('WNPP'): - db[pkg]['main']['debian_itp'] = st['WNPP'] - if st.has_key('Pkg-URL'): - db[pkg]['main']['other_pkg'] = st['Pkg-URL'] - if st.has_key('Homepage'): - db[pkg]['main']['homepage'] = st['Homepage'] - - # only store if there isn't something already - if not db[pkg].has_key('blends'): - db[pkg]['blends'] = info - else: - # just add this tasks name and id - db[pkg]['blends']['tasks'].append(task) + info = {} + + # blends info + info['tasks'] = [task] + if st.has_key('License'): + info['license'] = st['License'] + if st.has_key('Responsible'): + info['responsible'] = st['Responsible'] + + # pkg description + if st.has_key('Pkg-Description'): + descr = st['Pkg-Description'].replace('%', '%%').split('\n') + info['description'] = descr[0].strip() + info['long_description'] = u' '.join([l.strip() for l in descr[1:]]) + + # charge the basic property set + db[p]['main']['description'] = info['description'] + db[p]['main']['long_description'] = info['long_description'] + if st.has_key('WNPP'): + db[p]['main']['debian_itp'] = st['WNPP'] + if st.has_key('Pkg-URL'): + db[p]['main']['other_pkg'] = st['Pkg-URL'] + if st.has_key('Homepage'): + db[p]['main']['homepage'] = st['Homepage'] + + # only store if there isn't something already + if not db[p].has_key('blends'): + db[p]['blends'] = info + else: + # just add this tasks name and id + db[p]['blends']['tasks'].append(task) return db @@ -311,7 +312,7 @@ def _store_pkg(cfg, db, st, origin, codename, component, baseurl): # pkg description descr = st['Description'].replace('%', '%%').split('\n') info['description'] = descr[0].strip() - info['long_description'] = ' '.join([l.strip() for l in descr[1:]]) + info['long_description'] = u' '.join([l.strip() for l in descr[1:]]) db[pkg][distkey] = info @@ -352,12 +353,12 @@ def create_dir(path): def dde_get(url): try: return json.read(urllib2.urlopen(url+"?t=json").read())['r'] - except urllib2.HTTPError: + except (urllib2.HTTPError, StopIteration): + print "SCREWED:", url return False def import_dde(cfg, db): - dists = cfg.get('dde', 'dists').split() query_url = cfg.get('dde', 'pkgquery_url') for p in db.keys(): # get freshest @@ -440,8 +441,10 @@ def generate_pkgpage(pkg, cfg, db, template, addenum_dir): title = '%s\n %s\n%s' % (underline, title, underline) # preprocess long description + print 'rendering', pkg ld = db['main']['long_description'] - ld = ' '.join([l.lstrip(' .') for l in ld.split('\n')]) + print 'AAAAA', repr(ld), type(ld) + ld = u' '.join([l.lstrip(' .') for l in ld.split('\n')]) page = template.render(pkg=pkg, title=title, @@ -459,13 +462,13 @@ def generate_pkgpage(pkg, cfg, db, template, addenum_dir): def store_db(db, filename): pp = PrettyPrinter(indent=2) - f = open(filename, 'w') + f = codecs.open(filename, 'w', 'utf-8') f.write(pp.pformat(db)) f.close() def read_db(filename): - f = open(filename) + f = codecs.open(filename, 'r', 'utf-8') db = eval(f.read()) return db @@ -499,7 +502,7 @@ def write_pkgpages(jinja_env, cfg, db, outdir, addenum_dir): # generate the TOC with all packages toc_template = jinja_env.get_template('pkgs_toc.rst') - toc = open(os.path.join(outdir, 'pkgs.rst'), 'w') + toc = codecs.open(os.path.join(outdir, 'pkgs.rst'), 'w', 'utf-8') toc.write(toc_template.render(pkgs=db.keys())) toc.close() @@ -510,7 +513,7 @@ def write_pkgpages(jinja_env, cfg, db, outdir, addenum_dir): # when no page is available skip this package if page is None: continue - pf = open(os.path.join(outdir, 'pkgs', p + '.rst'), 'w') + pf = codecs.open(os.path.join(outdir, 'pkgs', p + '.rst'), 'w', 'utf-8') pf.write(generate_pkgpage(p, cfg, db, pkg_template, addenum_dir)) pf.close() diff --git a/neurodebian/templates/pkg.rst b/neurodebian/templates/pkg.rst index e5009f0..d990c92 100644 --- a/neurodebian/templates/pkg.rst +++ b/neurodebian/templates/pkg.rst @@ -94,7 +94,7 @@ for more information about its current status. Ubuntu ------ -{% for dist, distpkg in db.iteritems() if dist[1].startswith('ubuntu') %} +{% for dist, distpkg in db|dictsort if dist[1].startswith('ubuntu') %} {% if loop.first %} {% endif %} {{ dist[0] }} [{{ distpkg.component }}]: -- 2.39.5