--- /dev/null
+#!/usr/bin/python
+#emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
+#ex: set sts=4 ts=4 sw=4 noet:
+"""Script to help maintaining package definitions within Debian blends
+task files.
+
+Often it becomes necessary to duplicate the same information about a
+package within debian packaging and multiple blends task files,
+possibly of different blends. This script allows to automate:
+
+- construction of entries
+- injection into task files
+- modification of existing entries if things changed
+- removal of previously injected entries
+
+Possible TODOs:
+---------------
+
+* For every package the same task file might be re-read/written (if
+ entry changed/added) from disk.
+ That allows to replace easily original entry for 'source' package
+ (listed as Ignore:) with actual first listed binary package.
+ This should be taken into consideration if current per-package
+ handling gets changed
+
+"""
+
+"""
+Configuration
+-------------
+
+Paths to the blends top directories, containing tasks directories are
+specified in ~/.blends-inject.cfg file, e.g.::
+
+ [debian-med]
+ path = /home/yoh/deb/debian-med/
+
+ [debian-science]
+ path = /home/yoh/deb/debian-science/
+
+
+Definition of the fields for task files by default are looked up
+within debian/blends, or files provided in the command line.
+
+Format of debian/blends
+-----------------------
+
+Example::
+
+ ; If originally filed using project source name, and it is different
+ ; from the primary (first) binary package name, keep 'Source' to be
+ ; able to adopt previously included tasks entry
+Source: brian
+
+ ; Define the format on how entries should be handled.
+ ; Possible values:
+ ; extended -- whenever package is not in Debian and additional
+ ; fields should be obtained from debian/*:
+ ; * License
+ ; * WNPP
+ ; * Pkg-Description
+ ; * Responsible
+ ; * Homepage
+ ; * Vcs-*
+ ; plain [default] -- only fields listed here should be mentioned.
+ ; Common use -- whenever package is already known to UDD.
+ ;
+ ; By default, all fields specified previously propagate into following
+ ; packages as well. If that is not desired, add suffix '-clean' to
+ ; the Format
+Format: extended
+
+Tasks: debian-science/neuroscience-modeling
+
+ ; Could have Depends/Recommends/Suggests and Ignore
+ ; All those define Pkg-Name field which is not included
+ ; in the final "rendering" but is available as Pkg-Name item
+Depends: python-brian
+Pkg-URL: http://neuro.debian.net/pkgs/%(Pkg-Name)s.html
+Language: Python, C
+Published-Authors: Goodman D.F. and Brette R.
+Published-Title: Brian: a simulator for spiking neural networks in Python
+Published-In: Front. Neuroinform
+Published-Year: 2008
+Published-DOI: 10.3389/neuro.11.005.2008
+
+ ; May be some previous entry should be removed, thus say so
+Removed: python-brian-doc
+
+ ;Tasks: debian-med/imaging-dev
+ ;Why: Allows interactive development/scripting
+
+ ; ; It should be possible to switch between formats,
+ ; ; e.g. if some component is not yet in Debian
+ ;Format: extended
+ ;
+ ; ; Now some bogus one but with customizations
+ ;Tasks: debian-med/documentation
+ ;Recommends: python-brian-doc
+ ;Language:
+ ;Remark: some remark
+ ;
+
+"""
+
+
+import re, os, sys
+from os.path import join, exists, expanduser, dirname, basename
+
+from ConfigParser import ConfigParser
+from optparse import OptionParser, Option
+
+from copy import deepcopy
+#from debian_bundle import deb822
+from debian import deb822
+#import deb822
+from debian.changelog import Changelog
+
+# all files we are dealing with should be UTF8, thus
+# lets override
+import codecs
+
+def open(f, *args):
+ return codecs.open(f, *args, encoding='utf-8')
+
+__author__ = 'Yaroslav Halchenko'
+__prog__ = os.path.basename(sys.argv[0])
+__version__ = '0.0.2'
+__copyright__ = 'Copyright (c) 2010 Yaroslav Halchenko'
+__license__ = 'GPL'
+
+# What fields initiate new package description
+PKG_FIELDS = ('depends', 'recommends', 'suggests', 'ignore', 'removed')
+
+# We might need to resort to assure some what a canonical order
+FIELDS_ORDER = ('depends', 'recommends', 'suggests', 'ignore',
+ 'homepage', 'language', 'wnpp', 'responsible', 'license',
+ 'vcs-', 'pkg-url', 'pkg-description',
+ 'published-', 'x-', 'registration', 'remark')
+
+verbosity = None
+
+def error(msg, exit_code):
+ sys.stderr.write(msg + '\n')
+ sys.exit(exit_code)
+
+def verbose(level, msg):
+ if level <= verbosity:
+ print " "*level, msg
+
+
+def parse_debian_blends(f='debian/blends'):
+ """Parses debian/blends file
+
+ Returns unprocessed list of customized Deb822 entries
+ """
+ # Linearize all the paragraphs since we are not using them
+ items = []
+ for p in deb822.Deb822.iter_paragraphs(open(f)):
+ items += p.items()
+
+ verbose(6, "Got items %s" % items)
+ # Traverse and collect things
+ format_ = 'plain'
+ format_clean = False # do not propagate fields into a new pkg if True
+ pkg, source = None, None
+ pkgs = []
+ tasks = []
+
+ def new_pkg(prev_pkg, bname, sname, tasks):
+ """Helper function to create a new package
+ """
+ if format_clean or prev_pkg is None:
+ pkg = deb822.Deb822()
+ else:
+ pkg = deepcopy(prev_pkg)
+ for k_ in PKG_FIELDS: # prune older depends
+ pkg.pop(k_, None)
+ pkg['Pkg-Name'] = pkg[k] = bname
+ pkg['Pkg-Source'] = sname
+ pkgs.append(pkg)
+ pkg.tasks = dict( (t.strip(), deb822.Deb822Dict()) for t in tasks )
+ pkg.format = format_
+ return pkg
+
+ for k, v in items:
+
+ kl = k.lower()
+ if kl == 'source':
+ source = v.strip()
+ elif kl == 'format':
+ format_ = v.strip()
+ format_clean = format_.endswith('-clean')
+ if format_clean:
+ format_ = format_[:-6]
+ elif kl == 'tasks':
+ tasks = v.split(',')
+ newtasks = True # either we need to provide tune-ups
+ # for current package
+ elif kl in PKG_FIELDS: # new package
+ if source is None:
+ source = v
+ pkg = new_pkg(pkg, v, source, tasks)
+ newtasks = False
+ else:
+ if newtasks:
+ if pkg is None:
+ # So we had just source?
+ if source is None:
+ error("No package or source is known where to add %s" % k)
+ pkg = new_pkg(pkg, source, source, tasks)
+ # Since only source is available, it should be Ignore:-ed
+ pkg['Ignore'] = source
+ newtasks = False
+ # Add customization
+ for t in tasks:
+ if not t in pkg.tasks:
+ pkg.tasks[t] = deb822.Deb822Dict()
+ pkg.tasks[t][k] = v
+ else:
+ # just store the key in the pkg itself
+ pkg[k] = v
+ return pkgs
+
+
+def expand_pkgs(pkgs, topdir='.'):
+ """In-place modification of pkgs taking if necessary additional
+ information from Debian materials, and pruning empty fields
+ """
+ verbose(4, "Expanding content for %d packages" % len(pkgs))
+ debianm = None
+ # Expand packages which format is extended
+ for pkg in pkgs:
+ if pkg.format == 'extended':
+ # expanding, for that we need debian/control
+ if debianm is None:
+ debianm = DebianMaterials(topdir)
+ for k, m in (('License', lambda: debianm.get_license(pkg['Pkg-Name'])),
+ ('WNPP', debianm.get_wnpp),
+ ('Pkg-Description',
+ lambda: debianm.get_description(pkg['Pkg-Name'])),
+ ('Responsible', debianm.get_responsible),
+ ('Homepage', lambda: debianm.source.get('Homepage', None))):
+ if pkg.get(k, None):
+ continue
+ v = m()
+ if v:
+ pkg[k] = v
+ # VCS fields
+ pkg.update(debianm.get_vcsfields())
+
+def key_prefix_compare(x, y, order, strict=False, case=False):
+ """Little helper to help with sorting
+
+ Sorts according to the order of string prefixes as given by
+ `order`. If `strict`, then if no matching prefix found, would
+ raise KeyError; otherwise provides least priority to those keys
+ which were not found in `order`
+ """
+ if not case:
+ order = [v.lower() for v in order]
+
+ def prefix_index(t, order, strict=True, case=False):
+ x = t[0]
+ if not case:
+ x = x.lower()
+ for i, v in enumerate(order):
+ if x.startswith(v):
+ return i
+
+ if strict:
+ raise IndexError(
+ "Could not find location for %s as specified by %s" %
+ (x, order))
+ return 10000 # some large number ;)
+
+ cmp_res = cmp(prefix_index(x, order, strict, case),
+ prefix_index(y, order, strict, case))
+ if not cmp_res: # still unknown
+ return cmp(x, y)
+ return cmp_res
+
+
+def group_packages_into_tasks(pkgs):
+ """Given a list of packages (with .tasks) group them per each
+ task and perform necessary customizations stored in .tasks
+ """
+ # Time to take care about packages and tasks
+ # Unroll pkgs into a collection of pkgs per known task
+ tasks = {}
+ for pkg in pkgs:
+ # Lets just create deepcopies with tune-ups for each task
+ for itask, (task, fields) in enumerate(pkg.tasks.iteritems()):
+ pkg_ = deepcopy(pkg)
+ pkg_.update(fields)
+
+ # Perform string completions and removals
+ for k,v in pkg_.iteritems():
+ pkg_[k] = v % pkg_
+ if v is None or not len(v.strip()):
+ pkg_.pop(k)
+
+ # Sort the fields according to FIELDS_ORDER. Unfortunately
+ # silly Deb822* cannot create from list of tuples, so will do
+ # manually
+ pkg__ = deb822.Deb822()
+ for k,v in sorted(pkg_.items(),
+ cmp=lambda x, y:
+ key_prefix_compare(x, y, order=FIELDS_ORDER)):
+ pkg__[k] = v
+
+ # Move Pkg-source/name into attributes
+ pkg__.source = pkg__.pop('Pkg-Source')
+ pkg__.name = pkg__.pop('Pkg-name')
+
+ tasks[task] = tasks.get(task, []) + [pkg__]
+ verbose(4, "Grouped %d packages into %d tasks: %s" %
+ (len(pkgs), len(tasks), ', '.join(tasks.keys())))
+ return tasks
+
+def inject_tasks(tasks, config):
+ # Now go through task files and replace/add entries
+ for task, pkgs in tasks.iteritems():
+ verbose(2, "Task %s with %d packages" % (task, len(pkgs)))
+ blend, puretask = task.split('/')
+ taskfile = join(config.get(blend, 'path'), 'tasks', puretask)
+
+ # Load the file
+ stats = dict(Added=[], Modified=[])
+ for pkg in pkgs:
+ msgs = {'Name': pkg.name.strip(), 'Action': None}
+ # Find either it is known to the task file already
+
+ # Load entirely so we could simply manipulate
+ entries = open(taskfile).readlines()
+ known = False
+ # We need to search by name and by source
+ # We need to search for every possible type of dependecy
+ regexp = re.compile('^ *(%s) *: *(%s) *$' %
+ ('|'.join(PKG_FIELDS),
+ '|'.join((pkg.name, pkg.source))),
+ re.I)
+ for istart, e in enumerate(entries):
+ if regexp.search(e):
+ verbose(4, "Found %s in position %i: %s" %
+ (pkg.name, istart, e.rstrip()))
+ known = True
+ break
+
+ descr = ' ; Added by %s %s. [Please note here if modified manually]\n' % \
+ (__prog__, __version__)
+ # Replace existing entry
+ if known:
+ # TODO: Check if previous copy does not have our preceding comment
+ # Find the previous end
+ icount = 1
+ try:
+ while entries[istart+icount].strip() != '':
+ icount += 1
+ except IndexError, e:
+ pass # if we go beyond
+
+ # Lets not change file without necessity, if entry is identical --
+ # do nothing
+ entry = pkg.dump()
+ old_entry = entries[istart:istart+icount]
+
+ if u''.join(old_entry) == entry:
+ pass
+ else: # Rewrite the entry
+ if __prog__ in entries[istart-1]:
+ istart -= 1
+ icount += 2
+ if not 'Removed' in pkg.keys():
+ entries = entries[:istart] + [descr + entry] + entries[istart+icount:]
+ msgs['Action'] = 'Changed'
+ else:
+ while entries[istart-1].strip() == '':
+ istart -=1
+ icount +=2
+ entries = entries[:istart] + entries[istart+icount:]
+ msgs['Action'] = 'Removed'
+ output = ''.join(entries) # 'compute' first
+ open(taskfile, 'w').write(output) # then only overwrite
+ elif not 'removed' in pkg: # or Append one
+ msgs['Action'] = 'Added'
+ # could be as simple as
+ output = '\n%s%s' % (descr, pkg.dump(),)
+ open(taskfile, 'a').write(output)
+
+ if msgs['Action']:
+ verbose(3, "%(Action)s %(Name)s" % msgs)
+
+
+class DebianMaterials(object):
+ """Extract selected information from an existing debian/
+ """
+ _WNPP_RE = re.compile('^ *\* *Initial release.*closes:? #(?P<bug>[0-9]*).*', re.I)
+
+ def __init__(self, topdir):
+ #self.topdir = topdir
+ self._debiandir = join(topdir, 'debian')
+ self._source = None
+ self._binaries = None
+
+ @property
+ def source(self):
+ if self._source is None:
+ self._assign_packages()
+ return self._source
+
+ @property
+ def binaries(self):
+ if self._binaries is None:
+ self._assign_packages()
+ return self._binaries
+
+ def fpath(self, name):
+ return join(self._debiandir, name)
+
+ def _assign_packages(self):
+ try:
+ control = deb822.Deb822.iter_paragraphs(
+ open(self.fpath('control')))
+ except Exception, e:
+ raise RuntimeError(
+ "Cannot parse %s file necessary for the %s package entry. Error: %s"
+ % (control_file, pkg['Pkg-Name'], str(e)))
+ self._binaries = {}
+ self._source = None
+ for v in control:
+ if v.get('Source', None):
+ self._source = v
+ else:
+ self._binaries[v['Package']] = v
+
+ def get_license(self, package=None, first_only=True):
+ """Return a license(s). Parsed out from debian/copyright if it is
+ in machine readable format
+ """
+ licenses = []
+ # may be package should carry custom copyright file
+ copyright_file_ = self.fpath('%s.copyright' % package)
+ if package and exists(copyright_file_):
+ copyright_file = copyright_file_
+ else:
+ copyright_file = self.fpath('copyright')
+
+ try:
+ for p in deb822.Deb822.iter_paragraphs(open(copyright_file)):
+ if not 'Files' in p or p['Files'].strip().startswith('debian/'):
+ continue
+ l = p['License']
+ # Take only the short version of first line
+ l = re.sub('\n.*', '', l).strip()
+ if not len(l):
+ l = 'custom'
+ if not l in licenses:
+ licenses.append(l)
+ if first_only:
+ break
+ except Exception, e:
+ # print e
+ return None
+ return ', '.join(licenses)
+
+ def get_wnpp(self):
+ """Search for a template changelog entry closing "Initial bug
+ """
+ for l in open(self.fpath('changelog')):
+ rr = self._WNPP_RE.match(l)
+ if rr:
+ return rr.groupdict()['bug']
+ return None
+
+ def get_responsible(self):
+ """Returns responsible, atm -- maintainer
+ """
+ return self.source['Maintainer']
+
+ def get_vcsfields(self):
+ vcs = deb822.Deb822()
+ for f,v in self._source.iteritems():
+ if f.lower().startswith('vcs-'):
+ vcs[f] = v
+ return vcs
+
+ def get_description(self, pkg_name):
+ """Some logic to extract description.
+
+ If binary package matching pkg_name is found -- gets it description.
+ If no binary package with such name, and name matches source name,
+ obtain description of the first binary package.
+ """
+ if pkg_name in self.binaries:
+ pass
+ elif pkg_name.lower() == self.source['Source'].lower():
+ pkg_name = self.binaries.keys()[0]
+ else:
+ error("Name %s does not match any binary, nor source package in %s"
+ % (pkg_name, self))
+ return self.binaries[pkg_name]['Description']
+
+def main():
+
+ p = OptionParser(
+ usage="%s [OPTIONS] [blends_files]\n\n" % __prog__ + __doc__,
+ version="%prog " + __version__)
+
+ p.add_option(
+ Option("-d", "--topdir", action="store",
+ dest="topdir", default=None,
+ help="Top directory of a Debian package. It is used to locate "
+ "'debian/blends' if none is specified, and where to look for "
+ "extended information."))
+
+ p.add_option(
+ Option("-c", "--config-file", action="store",
+ dest="config_file", default=join(expanduser('~'), '.%s.cfg' % __prog__),
+ help="Noise level."))
+
+ p.add_option(
+ Option("-v", "--verbosity", action="store", type="int",
+ dest="verbosity", default=1, help="Noise level."))
+
+ (options, infiles) = p.parse_args()
+ global verbosity; verbosity = options.verbosity
+
+ if not len(infiles):
+ infiles = [join(options.topdir or './', 'debian/blends')] # default one
+
+ # Load configuration
+ config = ConfigParser()
+ config.read(options.config_file)
+
+ for blends_file in infiles:
+ verbose(1, "Processing %s" % blends_file)
+ if not exists(blends_file):
+ error("Cannot find a file %s. Either provide a file or specify top "
+ "debian directory with -d." % blends_file, 1)
+ pkgs = parse_debian_blends(blends_file)
+ if options.topdir is None:
+ if dirname(blends_file).endswith('/debian'):
+ topdir = dirname(dirname(blends_file))
+ else:
+ topdir = '.' # and hope for the best ;)
+ else:
+ topdir = options.topdir
+ expand_pkgs(pkgs, topdir=topdir)
+ tasks = group_packages_into_tasks(pkgs)
+ inject_tasks(tasks, config)
+
+
+if __name__ == '__main__':
+ main()
+
projects / neurodebian.git / commitdiff