2 """Tell me who you are!
7 from debian_bundle import deb822
9 # Lets first assure no guarding (but annoying) warnings
11 warnings.simplefilter('ignore', FutureWarning)
12 warnings.filterwarnings('ignore', 'Module debian_bundle was already imported.*', UserWarning)
15 from ConfigParser import SafeConfigParser
16 from optparse import OptionParser, Option, OptionGroup, OptionConflictError
26 from jinja2 import Environment, PackageLoader
28 from pprint import PrettyPrinter
31 class AptListsCache(object):
32 def __init__(self, cachedir='build/cache',
35 self.cachedir = cachedir
37 if not ro_cachedirs is None:
38 self.ro_cachedirs = ro_cachedirs
40 self.ro_cachedirs = []
43 create_dir(self.cachedir)
45 def get(self, url, update=False):
46 """Looks in the cache if the file is there and takes the cached one.
47 Otherwise it is downloaded first.
49 Knows how to deal with http:// and svn:// URLs.
54 # look whether it is compressed
55 cext = url.split('.')[-1]
56 if cext in ['gz', 'bz2']:
57 target_url = url[:-1 * len(cext) -1]
59 # assume not compressed
63 # turn url into a filename -- mimik what APT does for
65 tfilename = '_'.join(target_url.split('/')[2:])
67 # if we need to download anyway do not search
69 cfilename = os.path.join(self.cachedir, tfilename)
71 # look for the uncompressed file anywhere in the cache
73 for cp in [self.cachedir] + self.ro_cachedirs:
74 if os.path.exists(os.path.join(cp, tfilename)):
75 cfilename = os.path.join(cp, tfilename)
80 cfilename = os.path.join(self.cachedir, tfilename)
83 # if updated needed -- download
85 #print 'Caching file from %s' % url
87 if url.startswith('svn://'):
89 pysvn.Client().export(url, cfilename)
90 if url.startswith('http://'):
92 tempfile, ignored = urllib.urlretrieve(url)
99 decompressor = 'bzip2'
104 "Don't know how to decompress %s files" \
107 if not decompressor is None:
108 if subprocess.call([decompressor, '-d', '-q', '-f',
110 raise RuntimeError, \
111 "Something went wrong while decompressing '%s'" \
114 # move decompressed file into cache
115 shutil.move(os.path.splitext(tempfile)[0], cfilename)
117 # XXX do we need that if explicit filename is provided?
121 fh = codecs.open(cfilename, 'r', 'utf-8')
126 def add_pkgfromtaskfile(db, urls):
127 cache = AptListsCache()
133 # loop over all stanzas
134 for stanza in deb822.Packages.iter_paragraphs(fh):
135 if stanza.has_key('Depends'):
136 pkg = stanza['Depends']
137 elif stanza.has_key('Suggests'):
138 pkg = stanza['Suggests']
142 # account for multiple packages per line
144 pkgs += [p.strip() for p in pkg.split(',')]
146 pkgs.append(pkg.strip())
149 if not db.has_key(p):
150 db[p] = get_emptydbentry()
154 def get_emptydbentry():
157 def import_blendstask(db, url):
158 cache = AptListsCache()
162 # figure out blend's task page URL, since they differ from blend to blend
163 urlsec = url.split('/')
164 blendname = urlsec[-3]
165 if blendname == 'debian-med':
166 taskpage_url = 'http://debian-med.alioth.debian.org/tasks/'
167 elif blendname == 'debian-science':
168 taskpage_url = 'http://blends.alioth.debian.org/science/tasks/'
170 raise ValueError('Unknown blend "%s"' % blendname)
171 taskpage_url += urlsec[-1]
173 for st in deb822.Packages.iter_paragraphs(fh):
174 if st.has_key('Task'):
175 task_name = st['Task']
176 task = (blendname, task_name, taskpage_url)
178 if st.has_key('Depends'):
180 elif st.has_key('Suggests'):
183 # print 'Warning: Cannot determine name of prospective package ' \
184 # '... ignoring. Dump follows:'
188 # take care of pkg lists
189 for p in pkg.split(', '):
190 if not db.has_key(p):
191 print 'Ignoring blend package "%s"' % p
197 info['tasks'] = [task]
198 if st.has_key('License'):
199 info['license'] = st['License']
200 if st.has_key('Responsible'):
201 info['responsible'] = st['Responsible']
204 if st.has_key('Pkg-Description'):
205 descr = st['Pkg-Description'].split('\n')
206 info['description'] = descr[0].strip()
207 info['long_description'] = \
208 u'\n'.join(descr[1:])
210 # charge the basic property set
211 db[p]['main']['description'] = info['description']
212 db[p]['main']['long_description'] = info['long_description']
213 if st.has_key('WNPP'):
214 db[p]['main']['debian_itp'] = st['WNPP']
215 if st.has_key('Pkg-URL'):
216 db[p]['main']['other_pkg'] = st['Pkg-URL']
217 if st.has_key('Homepage'):
218 db[p]['main']['homepage'] = st['Homepage']
221 if st.has_key('Published-Title'):
222 pub = {'title': st['Published-Title']}
223 if st.has_key('Published-Authors'):
224 pub['authors'] = st['Published-Authors']
225 if st.has_key('Published-Year'):
226 pub['year'] = st['Published-Year']
227 if st.has_key('Published-In'):
228 pub['in'] = st['Published-In']
229 if st.has_key('Published-URL'):
230 pub['url'] = st['Published-URL']
231 if st.has_key('Published-DOI'):
232 pub['doi'] = st['Published-DOI']
233 # need at least one URL
234 if not pub.has_key('url'):
235 pub['url'] = st['Published-DOI']
237 db[p]['main']['publication'] = pub
240 if st.has_key('Registration'):
241 db[p]['main']['registration'] = st['Registration']
244 if st.has_key('Remark'):
245 # prepend a single space to make it look like a long description
246 info['remark'] = convert_longdescr(' ' + st['Remark'])
248 # only store if there isn't something already
249 if not db[p].has_key('blends'):
250 db[p]['blends'] = info
252 # just add this tasks name and id
253 db[p]['blends']['tasks'].append(task)
258 def get_releaseinfo(rurl):
259 cache = AptListsCache()
260 # root URL of the repository
261 baseurl = '/'.join(rurl.split('/')[:-1])
262 # get the release file from the cache
263 release_file = cache.get(rurl)
265 # create parser instance
266 rp = deb822.Release(release_file)
268 # architectures on this dist
269 archs = rp['Architectures'].split()
270 components = rp['Components'].split()
271 # compile a new codename that also considers the repository label
272 # to distinguish between official and unofficial repos.
274 origin = rp['Origin']
275 codename = rp['Codename']
276 labelcode = '_'.join([rp['Label'], rp['Codename']])
281 return {'baseurl': baseurl, 'archs': archs, 'components': components,
282 'codename': codename, 'label': label, 'labelcode': labelcode,
286 def build_pkgsurl(baseurl, component, arch):
287 return '/'.join([baseurl, component, 'binary-' + arch, 'Packages.bz2'])
290 def import_release(cfg, db, rurl):
291 cache = AptListsCache()
293 ri = get_releaseinfo(rurl)
295 # compile the list of Packages files to parse and parse them
296 for c in ri['components']:
297 for a in ri['archs']:
298 # compile packages URL
299 pkgsurl = build_pkgsurl(ri['baseurl'], c, a)
301 # retrieve from cache
302 packages_file = cache.get(pkgsurl)
305 for stanza in deb822.Packages.iter_paragraphs(packages_file):
306 db = _store_pkg(cfg, db, stanza, ri['origin'], ri['codename'], c, ri['baseurl'])
309 packages_file.close()
313 def _store_pkg(cfg, db, st, origin, codename, component, baseurl):
320 # only care for known packages
321 if not db.has_key(pkg):
322 # print 'Ignoring NeuroDebian package "%s"' % pkg
325 distkey = (trans_codename(codename, cfg), 'neurodebian-' + codename)
327 if db[pkg].has_key(distkey):
328 info = db[pkg][distkey]
330 info = {'architecture': []}
333 if not st['Architecture'] in info['architecture']:
334 info['architecture'].append(st['Architecture'])
335 info['maintainer'] = st['Maintainer']
336 if st.has_key('Homepage'):
337 info['homepage'] = st['Homepage']
338 info['version'] = st['Version']
341 info['distribution'] = origin
342 info['release'] = codename
343 info['component'] = component
346 info['poolurl'] = '/'.join([os.path.dirname(st['Filename'])])
349 descr = st['Description'].replace('%', '%%').split('\n')
350 info['description'] = descr[0].strip()
351 info['long_description'] = u'\n'.join(descr[1:])
353 db[pkg][distkey] = info
355 # charge the basic property set
356 db[pkg]['main']['description'] = info['description']
357 db[pkg]['main']['long_description'] = info['long_description']
358 if st.has_key('Source'):
359 db[pkg]['main']['sv'] = "%s %s" % (st['Source'], st['Version'])
361 db[pkg]['main']['sv'] = "%s %s" % (st['Package'], st['Version'])
362 if st.has_key('Homepage'):
363 db[pkg]['main']['homepage'] = st['Homepage']
364 if st.has_key('Recommends'):
365 db[pkg]['main']['recommends'] = st['Recommends']
370 def trans_codename(codename, cfg):
371 """Translate a known codename into a release description.
373 Unknown codenames will simply be returned as is.
375 # if we know something, tell
376 if codename in cfg.options('release codenames'):
377 return cfg.get('release codenames', codename)
382 def create_dir(path):
383 if os.path.exists(path):
386 ps = path.split(os.path.sep)
388 for i in range(1,len(ps) + 1):
389 p = os.path.sep.join(ps[:i])
391 if not os.path.exists(p):
395 def dde_get(url, fail=False):
396 # enforce delay to be friendly to DDE
399 data = json.read(urllib2.urlopen(url+"?t=json").read())['r']
400 print "SUCCESS:", url
402 except urllib2.HTTPError, e:
403 print "NOINFO:", url, type(e)
405 except urllib2.URLError, e:
406 print "URLERROR:", url, type(e)
408 print "Permanant failure"
410 print "Try again after 30 seconds..."
412 return dde_get(url, fail=True)
413 except (StopIteration):
418 def import_dde(cfg, db):
419 query_url = cfg.get('dde', 'pkgquery_url')
422 q = dde_get(query_url + "/packages/all/%s" % p)
424 # copy all stuff, while preserving non-overlapping information
425 for k, v in q.iteritems():
427 # get latest popcon info for debian and ubuntu
428 # cannot use origin field itself, since it is none for few packages
430 origin = q['drc'].split()[0]
431 if origin == 'ubuntu':
432 if q.has_key('popcon'):
433 db[p]['main']['ubuntu_popcon'] = q['popcon']
434 # if we have ubuntu, need to get debian
435 q = dde_get(query_url + "/packages/prio-debian-sid/%s" % p)
436 if q and q.has_key('popcon'):
437 db[p]['main']['debian_popcon'] = q['popcon']
438 elif origin == 'debian':
439 if q.has_key('popcon'):
440 db[p]['main']['debian_popcon'] = q['popcon']
441 # if we have debian, need to get ubuntu
442 q = dde_get(query_url + "/packages/prio-ubuntu-karmic/%s" % p)
443 if q and q.has_key('popcon'):
444 db[p]['main']['ubuntu_popcon'] = q['popcon']
446 print("Ignoring unkown origin '%s' for package '%s'." \
449 # now get info for package from all releases in UDD
450 q = dde_get(query_url + "/dist/p:%s" % p)
453 # hold all info about this package per distribution release
456 distkey = (trans_codename(cp['release'], cfg),
457 "%s-%s" % (cp['distribution'], cp['release']))
458 if not info.has_key(distkey):
460 # turn into a list to append others later
461 info[distkey]['architecture'] = [info[distkey]['architecture']]
462 # accumulate data for multiple over archs
464 comp = apt.VersionCompare(cp['version'],
465 info[distkey]['version'])
466 # found another arch for the same version
468 info[distkey]['architecture'].append(cp['architecture'])
469 # found newer version, dump the old ones
472 # turn into a list to append others later
473 info[distkey]['architecture'] = [info[distkey]['architecture']]
474 # simply ignore older versions
478 # finally assign the new package data
479 for k, v in info.iteritems():
485 def convert_longdescr(ld):
486 ld = ld.replace('% ', '%% ')
488 for i, l in enumerate(ld):
490 ld[i] = ' #NEWLINEMARKER#'
491 # look for embedded lists
492 elif len(l) >=3 and l[:2] == ' ' and l[2] in '-*':
493 ld[i] = ' #NEWLINEMARKER# ' + l[2:]
495 ld = u' '.join([l[1:] for l in ld])
496 ld = ld.replace('#NEWLINEMARKER# ', '\n\n')
497 # cleanup any leftover (e.g. trailing markers)
498 ld = ld.replace('#NEWLINEMARKER#', '')
502 def generate_pkgpage(pkg, cfg, db, template, addenum_dir):
503 # local binding for ease of use
505 # do nothing if there is not at least the very basic stuff
506 if not pkgdb['main'].has_key('description'):
508 title = '**%s** -- %s' % (pkg, pkgdb['main']['description'])
509 underline = '*' * (len(title) + 2)
510 title = '%s\n %s\n%s' % (underline, title, underline)
512 page = template.render(
515 long_description=convert_longdescr(pkgdb['main']['long_description']),
519 # the following can be replaced by something like
520 # {% include "sidebar.html" ignore missing %}
521 # in the template whenever jinja 2.2 becomes available
522 addenum = os.path.join(os.path.abspath(addenum_dir), '%s.rst' % pkg)
523 if os.path.exists(addenum):
524 page += '\n\n.. include:: %s\n' % addenum
528 def store_db(db, filename):
529 pp = PrettyPrinter(indent=2)
530 f = codecs.open(filename, 'w', 'utf-8')
531 f.write(pp.pformat(db))
535 def read_db(filename):
536 f = codecs.open(filename, 'r', 'utf-8')
540 def write_sourceslist(jinja_env, cfg, outdir):
542 create_dir(os.path.join(outdir, '_static'))
545 for release in cfg.options('release codenames'):
546 transrel = trans_codename(release, cfg)
548 for mirror in cfg.options('mirrors'):
549 listname = 'neurodebian.%s.%s.sources.list' % (release, mirror)
550 repos[transrel].append((mirror, listname))
551 lf = open(os.path.join(outdir, '_static', listname), 'w')
552 aptcfg = '%s %s main contrib non-free\n' % (cfg.get('mirrors', mirror),
554 lf.write('deb %s' % aptcfg)
555 lf.write('deb-src %s' % aptcfg)
558 srclist_template = jinja_env.get_template('sources_lists.rst')
559 sl = open(os.path.join(outdir, 'sources_lists'), 'w')
560 sl.write(srclist_template.render(repos=repos))
564 def write_pkgpages(jinja_env, cfg, db, outdir, addenum_dir):
566 create_dir(os.path.join(outdir, 'pkgs'))
568 # generate the TOC with all packages
569 toc_template = jinja_env.get_template('pkgs_toc.rst')
570 toc = codecs.open(os.path.join(outdir, 'pkgs.rst'), 'w', 'utf-8')
571 toc.write(toc_template.render(pkgs=db.keys()))
574 # and now each individual package page
575 pkg_template = jinja_env.get_template('pkg.rst')
577 page = generate_pkgpage(p, cfg, db, pkg_template, addenum_dir)
578 # when no page is available skip this package
581 pf = codecs.open(os.path.join(outdir, 'pkgs', p + '.rst'), 'w', 'utf-8')
586 def prepOptParser(op):
587 # use module docstring for help output
588 op.usage = "%s [OPTIONS]\n\n" % sys.argv[0] + __doc__
590 op.add_option("--db",
591 action="store", type="string", dest="db",
593 help="Database file to read. Default: None")
595 op.add_option("--cfg",
596 action="store", type="string", dest="cfg",
598 help="Repository config file.")
600 op.add_option("-o", "--outdir",
601 action="store", type="string", dest="outdir",
603 help="Target directory for ReST output. Default: None")
605 op.add_option("-r", "--release-url",
606 action="append", dest="release_urls",
609 op.add_option("--pkgaddenum", action="store", dest="addenum_dir",
610 type="string", default=None, help="None")
614 op = OptionParser(version="%prog 0.0.2")
617 (opts, args) = op.parse_args()
620 print('There needs to be exactly one command')
626 print("'--cfg' option is mandatory.")
629 print("'--db' option is mandatory.")
633 cfg = SafeConfigParser()
636 # load existing db, unless renew is requested
637 if cmd == 'updatedb':
639 if cfg.has_option('packages', 'select taskfiles'):
640 db = add_pkgfromtaskfile(db, cfg.get('packages',
641 'select taskfiles').split())
643 # add additional package names from config file
644 if cfg.has_option('packages', 'select names'):
645 for p in cfg.get('packages', 'select names').split():
646 if not db.has_key(p):
647 db[p] = get_emptydbentry()
649 # get info from task files
650 if cfg.has_option('packages', 'prospective'):
651 for url in cfg.get('packages', 'prospective').split():
652 db = import_blendstask(db, url)
654 # parse NeuroDebian repository
655 if cfg.has_option('neurodebian', 'releases'):
656 for rurl in cfg.get('neurodebian', 'releases').split():
657 db = import_release(cfg, db, rurl)
659 # collect package information from DDE
660 db = import_dde(cfg, db)
662 store_db(db, opts.db)
666 # load the db from file
667 db = read_db(opts.db)
670 jinja_env = Environment(loader=PackageLoader('neurodebian', 'templates'))
672 # generate package pages and TOC and write them to files
673 write_pkgpages(jinja_env, cfg, db, opts.outdir, opts.addenum_dir)
675 write_sourceslist(jinja_env, cfg, opts.outdir)
677 if __name__ == "__main__":