Make it work with unicode.

diff --git a/neurodebian.cfg b/neurodebian.cfg
index c289d956b81fe3950e1ce045b5956afc4aef453c..7fa8cff4cdbb5cef113a2832d4c302b387c2c20c 100644 (file)
--- a/neurodebian.cfg
+++ b/neurodebian.cfg
@@ -1,7 +1,4 @@
 [dde]
-dists = debian-lenny debian-squeeze debian-sid debian-experimental
-  ubuntu-hardy ubuntu-intrepid ubuntu-jaunty ubuntu-karmic
-
 pkgquery_url = http://dde.debian.net/dde/q/udd
 
 [packages]
@@ -9,6 +6,8 @@ pkgquery_url = http://dde.debian.net/dde/q/udd
 # _if_ they are also present in the repository
 select taskfiles =
  svn://svn.debian.org/blends/projects/science/trunk/debian-science/tasks/neuroscience-cognitive
+ svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging
+ svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging-dev
 
 # Additional selection filter (similar to 'select taskfiles'), only listing
 # package names
@@ -19,7 +18,7 @@ select names = fsl-doc fslview-doc fsl-atlases fsl-possum-data fsl-first-data
 prospective =
  svn://svn.debian.org/blends/projects/science/trunk/debian-science/tasks/neuroscience-cognitive
  svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging
-# svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging-dev
+ svn://svn.debian.org/blends/projects/med/trunk/debian-med/tasks/imaging-dev
 
 [mirrors]
 us = http://neuro.debian.net/debian
@@ -37,7 +36,6 @@ releases =
  http://neuro.debian.net/debian/dists/lenny/Release
  http://neuro.debian.net/debian/dists/squeeze/Release
  http://neuro.debian.net/debian/dists/sid/Release
- http://neuro.debian.net/debian/dists/sid/Release
 
 [release codenames]
 etch = Debian GNU/Linux 4.0 (etch)

diff --git a/neurodebian/dde.py b/neurodebian/dde.py
index bc03b8c5cb370807ceeeb4f7a1c579b90ee9ae3c..0ebb0eefef9a00bfe13c30c2a7c9e945176e3a00 100644 (file)
--- a/neurodebian/dde.py
+++ b/neurodebian/dde.py
@@ -13,6 +13,7 @@ import os
 import shutil
 import urllib2
 import urllib
+import codecs
 import subprocess
 # templating
 from jinja2 import Environment, PackageLoader
@@ -110,7 +111,7 @@ class AptListsCache(object):
                 urllib.urlcleanup()
 
         # open cached file
-        fh = open(cfilename, 'r')
+        fh = codecs.open(cfilename, 'r', 'utf-8')
 
         return fh
 
@@ -167,53 +168,53 @@ def import_blendstask(db, url):
             task_name = st['Task']
             task = (blendname, task_name, taskpage_url)
 
-        # do not stop unless we have a description
-        if not st.has_key('Pkg-Description'):
-            continue
-
         if st.has_key('Depends'):
             pkg = st['Depends']
         elif st.has_key('Suggests'):
             pkg = st['Suggests']
         else:
-            print 'Warning: Cannot determine name of prospective package ' \
-                    '... ignoring.'
-            continue
-
-        if not db.has_key(pkg):
-            print 'Ignoring blend package "%s"' % pkg
+#            print 'Warning: Cannot determine name of prospective package ' \
+#                    '... ignoring. Dump follows:'
+#            print st
             continue
 
-        info = {}
+        # take care of pkg lists
+        for p in pkg.split(', '):
+            if not db.has_key(p):
+                print 'Ignoring blend package "%s"' % p
+                continue
 
-        # blends info
-        info['tasks'] = [task]
-        if st.has_key('License'):
-            info['license'] = st['License']
-        if st.has_key('Responsible'):
-            info['responsible'] = st['Responsible']
-
-        # pkg description
-        descr = st['Pkg-Description'].replace('%', '%%').split('\n')
-        info['description'] = descr[0].strip()
-        info['long_description'] = ' '.join([l.strip() for l in descr[1:]])
-
-        # charge the basic property set
-        db[pkg]['main']['description'] = info['description']
-        db[pkg]['main']['long_description'] = info['long_description']
-        if st.has_key('WNPP'):
-            db[pkg]['main']['debian_itp'] = st['WNPP']
-        if st.has_key('Pkg-URL'):
-            db[pkg]['main']['other_pkg'] = st['Pkg-URL']
-        if st.has_key('Homepage'):
-            db[pkg]['main']['homepage'] = st['Homepage']
-
-        # only store if there isn't something already
-        if not db[pkg].has_key('blends'):
-            db[pkg]['blends'] = info
-        else:
-            # just add this tasks name and id
-            db[pkg]['blends']['tasks'].append(task)
+            info = {}
+
+            # blends info
+            info['tasks'] = [task]
+            if st.has_key('License'):
+                info['license'] = st['License']
+            if st.has_key('Responsible'):
+                info['responsible'] = st['Responsible']
+
+            # pkg description
+            if st.has_key('Pkg-Description'):
+                descr = st['Pkg-Description'].replace('%', '%%').split('\n')
+                info['description'] = descr[0].strip()
+                info['long_description'] = u' '.join([l.strip() for l in descr[1:]])
+
+                # charge the basic property set
+                db[p]['main']['description'] = info['description']
+                db[p]['main']['long_description'] = info['long_description']
+            if st.has_key('WNPP'):
+                db[p]['main']['debian_itp'] = st['WNPP']
+            if st.has_key('Pkg-URL'):
+                db[p]['main']['other_pkg'] = st['Pkg-URL']
+            if st.has_key('Homepage'):
+                db[p]['main']['homepage'] = st['Homepage']
+
+            # only store if there isn't something already
+            if not db[p].has_key('blends'):
+                db[p]['blends'] = info
+            else:
+                # just add this tasks name and id
+                db[p]['blends']['tasks'].append(task)
 
     return db
 
@@ -311,7 +312,7 @@ def _store_pkg(cfg, db, st, origin, codename, component, baseurl):
     # pkg description
     descr = st['Description'].replace('%', '%%').split('\n')
     info['description'] = descr[0].strip()
-    info['long_description'] = ' '.join([l.strip() for l in descr[1:]])
+    info['long_description'] = u' '.join([l.strip() for l in descr[1:]])
 
     db[pkg][distkey] = info
 
@@ -352,12 +353,12 @@ def create_dir(path):
 def dde_get(url):
     try:
         return json.read(urllib2.urlopen(url+"?t=json").read())['r']
-    except urllib2.HTTPError:
+    except (urllib2.HTTPError, StopIteration):
+        print "SCREWED:", url
         return False
 
 
 def import_dde(cfg, db):
-    dists = cfg.get('dde', 'dists').split()
     query_url = cfg.get('dde', 'pkgquery_url')
     for p in db.keys():
         # get freshest
@@ -440,8 +441,10 @@ def generate_pkgpage(pkg, cfg, db, template, addenum_dir):
     title = '%s\n %s\n%s' % (underline, title, underline)
 
     # preprocess long description
+    print 'rendering', pkg
     ld = db['main']['long_description']
-    ld = ' '.join([l.lstrip(' .') for l in ld.split('\n')])
+    print 'AAAAA', repr(ld), type(ld)
+    ld = u' '.join([l.lstrip(' .') for l in ld.split('\n')])
 
     page = template.render(pkg=pkg,
                            title=title,
@@ -459,13 +462,13 @@ def generate_pkgpage(pkg, cfg, db, template, addenum_dir):
 
 def store_db(db, filename):
     pp = PrettyPrinter(indent=2)
-    f = open(filename, 'w')
+    f = codecs.open(filename, 'w', 'utf-8')
     f.write(pp.pformat(db))
     f.close()
 
 
 def read_db(filename):
-    f = open(filename)
+    f = codecs.open(filename, 'r', 'utf-8')
     db = eval(f.read())
     return db
 
@@ -499,7 +502,7 @@ def write_pkgpages(jinja_env, cfg, db, outdir, addenum_dir):
 
     # generate the TOC with all packages
     toc_template = jinja_env.get_template('pkgs_toc.rst')
-    toc = open(os.path.join(outdir, 'pkgs.rst'), 'w')
+    toc = codecs.open(os.path.join(outdir, 'pkgs.rst'), 'w', 'utf-8')
     toc.write(toc_template.render(pkgs=db.keys()))
     toc.close()
 
@@ -510,7 +513,7 @@ def write_pkgpages(jinja_env, cfg, db, outdir, addenum_dir):
         # when no page is available skip this package
         if page is None:
             continue
-        pf = open(os.path.join(outdir, 'pkgs', p + '.rst'), 'w')
+        pf = codecs.open(os.path.join(outdir, 'pkgs', p + '.rst'), 'w', 'utf-8')
         pf.write(generate_pkgpage(p, cfg, db, pkg_template, addenum_dir))
         pf.close()
 

diff --git a/neurodebian/templates/pkg.rst b/neurodebian/templates/pkg.rst
index e5009f02f7a6666197e41bd920432269401ef73a..d990c92bab178f753ba00238f607c78d3221c98e 100644 (file)
--- a/neurodebian/templates/pkg.rst
+++ b/neurodebian/templates/pkg.rst
@@ -94,7 +94,7 @@ for more information about its current status.
 Ubuntu
 ------
 
-{% for dist, distpkg in db.iteritems() if dist[1].startswith('ubuntu') %}
+{% for dist, distpkg in db|dictsort if dist[1].startswith('ubuntu') %}
 {% if loop.first %}
 {% endif %}
 {{ dist[0] }} [{{ distpkg.component }}]: