import codecs
import subprocess
import time
+import re
# templating
from jinja2 import Environment, PackageLoader
for stanza in deb822.Packages.iter_paragraphs(fh):
if stanza.has_key('Depends'):
pkg = stanza['Depends']
+ elif stanza.has_key('Recommends'):
+ pkg = stanza['Recommends']
elif stanza.has_key('Suggests'):
pkg = stanza['Suggests']
else:
if st.has_key('Depends'):
pkg = st['Depends']
+ elif st.has_key('Recommends'):
+ pkg = st['Recommends']
elif st.has_key('Suggests'):
pkg = st['Suggests']
else:
# Publications
if st.has_key('Published-Title'):
- pub = {'title': st['Published-Title']}
+ title = st['Published-Title']
+ if title[-1] == '.':
+ # trip trailing dot -- added later
+ pub = {'title': title[:-1]}
+ else:
+ pub = {'title': title}
if st.has_key('Published-Authors'):
pub['authors'] = st['Published-Authors']
if st.has_key('Published-Year'):
pub['doi'] = st['Published-DOI']
# need at least one URL
if not pub.has_key('url'):
- pub['url'] = st['Published-DOI']
+ pub['url'] = "http://dx.doi.org/%s" % st['Published-DOI']
db[p]['main']['publication'] = pub
if q.has_key('popcon'):
db[p]['main']['debian_popcon'] = q['popcon']
# if we have debian, need to get ubuntu
- q = dde_get(query_url + "/packages/prio-ubuntu-lucid/%s" % p)
+ q = dde_get(query_url + "/packages/prio-ubuntu-natty/%s" % p)
if q and q.has_key('popcon'):
db[p]['main']['ubuntu_popcon'] = q['popcon']
else:
return db
+def assure_unicode(s):
+ """Assure that argument is unicode
+
+ Necessary if strings are not carrying out Pythonish 'u' prefix to
+ signal UTF8 strings, but are in fact UTF8
+ """
+ if type(s) is unicode:
+ return s
+ elif type(s) is str:
+ # attempt regular unicode call and if fails -- just decode it
+ # into utf8
+ try:
+ return unicode(s)
+ except UnicodeDecodeError, e:
+ return s.decode('utf8')
+ else:
+ return assure_unicode(str(s))
+
def convert_longdescr(ld):
ld = ld.replace('% ', '%% ')
ld = ld.replace('#NEWLINEMARKER# ', '\n\n')
# cleanup any leftover (e.g. trailing markers)
ld = ld.replace('#NEWLINEMARKER#', '')
+ # safe-guard ReST active symbols
+ ld = re.sub(r'([\'`*])', r'\\\1', ld)
return ld
-def generate_pkgpage(pkg, cfg, db, template, addenum_dir):
+def generate_pkgpage(pkg, cfg, db, template, addenum_dir, extracts_dir):
+ print pkg
# local binding for ease of use
pkgdb = db[pkg]
# do nothing if there is not at least the very basic stuff
underline = '*' * (len(title) + 2)
title = '%s\n %s\n%s' % (underline, title, underline)
+ ex_dir = None
+ if 'sv' in pkgdb['main']:
+ ex_dir = os.path.join(extracts_dir, pkgdb['main']['sv'].split()[0])
+ if not os.path.exists(ex_dir):
+ ex_dir = None
page = template.render(
pkg=pkg,
title=title,
- long_description=convert_longdescr(pkgdb['main']['long_description']),
+ long_description=convert_longdescr(
+ assure_unicode(pkgdb['main']['long_description'])),
cfg=cfg,
db=pkgdb,
- fulldb=db)
+ fulldb=db,
+ extracts_dir=ex_dir,
+ op=os.path)
# the following can be replaced by something like
# {% include "sidebar.html" ignore missing %}
# in the template whenever jinja 2.2 becomes available
def write_sourceslist(jinja_env, cfg, outdir):
create_dir(outdir)
- create_dir(os.path.join(outdir, '_static'))
+ create_dir(os.path.join(outdir, 'lists'))
repos = {}
for release in cfg.options('release codenames'):
transrel = trans_codename(release, cfg)
repos[transrel] = []
for mirror in cfg.options('mirrors'):
- listname = 'neurodebian.%s.%s.sources.list' % (release, mirror)
+ listname = '%s.%s' % (release, mirror)
repos[transrel].append((mirror, listname))
- lf = open(os.path.join(outdir, '_static', listname), 'w')
+ lf = open(os.path.join(outdir, 'lists', listname), 'w')
for rel in ('data', release):
aptcfg = '%s %s main contrib non-free\n' % (cfg.get('mirrors', mirror),
rel)
lf.write('#deb-src %s' % aptcfg)
lf.close()
+ id2codename = dict([(cfg.get('release backport ids', r), r)
+ for r in cfg.options('release codenames')])
+ id2relname = dict([(cfg.get('release backport ids', r), trans_codename(r, cfg))
+ for r in cfg.options('release codenames')])
+ mirror2name = dict([(m, cfg.get('mirror names', m))
+ for m in cfg.options('mirrors')])
+ mirror2url = dict([(m, cfg.get('mirrors', m))
+ for m in cfg.options('mirrors')])
srclist_template = jinja_env.get_template('sources_lists.rst')
sl = open(os.path.join(outdir, 'sources_lists'), 'w')
- sl.write(srclist_template.render(repos=repos))
+ sl.write(srclist_template.render(id2codename=id2codename,
+ id2relname=id2relname,
+ mirror2name=mirror2name,
+ mirror2url=mirror2url))
sl.close()
-def write_pkgpages(jinja_env, cfg, db, outdir, addenum_dir):
+def write_pkgpages(jinja_env, cfg, db, outdir, addenum_dir, extracts_dir):
create_dir(outdir)
create_dir(os.path.join(outdir, 'pkgs'))
# generate the TOC with all packages
toc_template = jinja_env.get_template('pkgs_toc.rst')
toc = codecs.open(os.path.join(outdir, 'pkgs.rst'), 'w', 'utf-8')
- toc.write(toc_template.render(pkgs=db.keys()))
+ # this is a fragile test
+ toc.write(toc_template.render(
+ pkgs=[k for k in db.keys()
+ if not ('Datasets (data)', 'neurodebian-data') in db[k]]))
toc.close()
+ # and now only for dataset packages
+ toc_template = jinja_env.get_template('datasets_toc.rst')
+ toc = codecs.open(os.path.join(outdir, 'datasets.rst'), 'w', 'utf-8')
+ # this is a fragile test
+ toc.write(toc_template.render(
+ pkgs=[k for k in db.keys()
+ if ('Datasets (data)', 'neurodebian-data') in db[k]]))
+ toc.close()
+
# and now each individual package page
pkg_template = jinja_env.get_template('pkg.rst')
for p in db.keys():
- page = generate_pkgpage(p, cfg, db, pkg_template, addenum_dir)
+ page = generate_pkgpage(p, cfg, db, pkg_template, addenum_dir, extracts_dir)
# when no page is available skip this package
if page is None:
continue
op.add_option("--pkgaddenum", action="store", dest="addenum_dir",
type="string", default=None, help="None")
+ op.add_option("--extracts", action="store", dest="extracts_dir",
+ type="string", default=None, help="None")
+
def main():
op = OptionParser(version="%prog 0.0.2")
jinja_env = Environment(loader=PackageLoader('neurodebian', 'templates'))
# generate package pages and TOC and write them to files
- write_pkgpages(jinja_env, cfg, db, opts.outdir, opts.addenum_dir)
+ write_pkgpages(jinja_env, cfg, db, opts.outdir, opts.addenum_dir, opts.extracts_dir)
write_sourceslist(jinja_env, cfg, opts.outdir)