2 #emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
3 #ex: set sts=4 ts=4 sw=4 noet:
4 """Script to help maintaining package definitions within Debian blends
7 Often it becomes necessary to duplicate the same information about a
8 package within debian packaging and multiple blends task files,
9 possibly of different blends. This script allows to automate:
11 - construction of entries
12 - injection into task files
13 - modification of existing entries if things changed
14 - removal of previously injected entries
19 * For every package the same task file might be re-read/written (if
20 entry changed/added) from disk.
21 That allows to replace easily original entry for 'source' package
22 (listed as Ignore:) with actual first listed binary package.
23 This should be taken into consideration if current per-package
32 Paths to the blends top directories, containing tasks directories are
33 specified in ~/.blends-inject.cfg file, e.g.::
36 path = /home/yoh/deb/debian-med/
39 path = /home/yoh/deb/debian-science/
42 Definition of the fields for task files by default are looked up
43 within debian/blends, or files provided in the command line.
45 Format of debian/blends
46 -----------------------
50 ; If originally filed using project source name, and it is different
51 ; from the primary (first) binary package name, keep 'Source' to be
52 ; able to adopt previously included tasks entry
55 ; Define the format on how entries should be handled.
57 ; extended -- whenever package is not in Debian and additional
58 ; fields should be obtained from debian/*:
65 ; plain [default] -- only fields listed here should be mentioned.
66 ; Common use -- whenever package is already known to UDD.
68 ; By default, all fields specified previously propagate into following
69 ; packages as well. If that is not desired, add suffix '-clean' to
73 Tasks: debian-science/neuroscience-modeling
75 ; Could have Depends/Recommends/Suggests and Ignore
76 ; All those define Pkg-Name field which is not included
77 ; in the final "rendering" but is available as Pkg-Name item
79 Pkg-URL: http://neuro.debian.net/pkgs/%(Pkg-Name)s.html
81 Published-Authors: Goodman D.F. and Brette R.
82 Published-Title: Brian: a simulator for spiking neural networks in Python
83 Published-In: Front. Neuroinform
85 Published-DOI: 10.3389/neuro.11.005.2008
87 ; May be some previous entry should be removed, thus say so
88 Removed: python-brian-doc
90 ;Tasks: debian-med/imaging-dev
91 ;Why: Allows interactive development/scripting
93 ; ; It should be possible to switch between formats,
94 ; ; e.g. if some component is not yet in Debian
97 ; ; Now some bogus one but with customizations
98 ;Tasks: debian-med/documentation
99 ;Recommends: python-brian-doc
108 from os.path import join, exists, expanduser, dirname, basename
110 from ConfigParser import ConfigParser
111 from optparse import OptionParser, Option
113 from copy import deepcopy
114 #from debian_bundle import deb822
115 from debian import deb822
117 from debian.changelog import Changelog
119 # all files we are dealing with should be UTF8, thus
124 return codecs.open(f, *args, encoding='utf-8')
126 __author__ = 'Yaroslav Halchenko'
127 __prog__ = os.path.basename(sys.argv[0])
128 __version__ = '0.0.2'
129 __copyright__ = 'Copyright (c) 2010 Yaroslav Halchenko'
132 # What fields initiate new package description
133 PKG_FIELDS = ('depends', 'recommends', 'suggests', 'ignore', 'removed')
135 # We might need to resort to assure some what a canonical order
136 FIELDS_ORDER = ('depends', 'recommends', 'suggests', 'ignore',
137 'homepage', 'language', 'wnpp', 'responsible', 'license',
138 'vcs-', 'pkg-url', 'pkg-description',
139 'published-', 'x-', 'registration', 'remark')
143 def error(msg, exit_code):
144 sys.stderr.write(msg + '\n')
147 def verbose(level, msg):
148 if level <= verbosity:
152 def parse_debian_blends(f='debian/blends'):
153 """Parses debian/blends file
155 Returns unprocessed list of customized Deb822 entries
157 # Linearize all the paragraphs since we are not using them
159 for p in deb822.Deb822.iter_paragraphs(open(f)):
162 verbose(6, "Got items %s" % items)
163 # Traverse and collect things
165 format_clean = False # do not propagate fields into a new pkg if True
166 pkg, source = None, None
170 def new_pkg(prev_pkg, bname, sname, tasks):
171 """Helper function to create a new package
173 if format_clean or prev_pkg is None:
174 pkg = deb822.Deb822()
176 pkg = deepcopy(prev_pkg)
177 for k_ in PKG_FIELDS: # prune older depends
179 pkg['Pkg-Name'] = pkg[k] = bname
180 pkg['Pkg-Source'] = sname
182 pkg.tasks = dict( (t.strip(), deb822.Deb822Dict()) for t in tasks )
193 format_clean = format_.endswith('-clean')
195 format_ = format_[:-6]
198 newtasks = True # either we need to provide tune-ups
199 # for current package
200 elif kl in PKG_FIELDS: # new package
203 pkg = new_pkg(pkg, v, source, tasks)
208 # So we had just source?
210 error("No package or source is known where to add %s" % k)
211 pkg = new_pkg(pkg, source, source, tasks)
212 # Since only source is available, it should be Ignore:-ed
213 pkg['Ignore'] = source
217 if not t in pkg.tasks:
218 pkg.tasks[t] = deb822.Deb822Dict()
221 # just store the key in the pkg itself
226 def expand_pkgs(pkgs, topdir='.'):
227 """In-place modification of pkgs taking if necessary additional
228 information from Debian materials, and pruning empty fields
230 verbose(4, "Expanding content for %d packages" % len(pkgs))
232 # Expand packages which format is extended
234 if pkg.format == 'extended':
235 # expanding, for that we need debian/control
237 debianm = DebianMaterials(topdir)
238 for k, m in (('License', lambda: debianm.get_license(pkg['Pkg-Name'])),
239 ('WNPP', debianm.get_wnpp),
241 lambda: debianm.get_description(pkg['Pkg-Name'])),
242 ('Responsible', debianm.get_responsible),
243 ('Homepage', lambda: debianm.source.get('Homepage', None))):
250 pkg.update(debianm.get_vcsfields())
252 def key_prefix_compare(x, y, order, strict=False, case=False):
253 """Little helper to help with sorting
255 Sorts according to the order of string prefixes as given by
256 `order`. If `strict`, then if no matching prefix found, would
257 raise KeyError; otherwise provides least priority to those keys
258 which were not found in `order`
261 order = [v.lower() for v in order]
263 def prefix_index(t, order, strict=True, case=False):
267 for i, v in enumerate(order):
273 "Could not find location for %s as specified by %s" %
275 return 10000 # some large number ;)
277 cmp_res = cmp(prefix_index(x, order, strict, case),
278 prefix_index(y, order, strict, case))
279 if not cmp_res: # still unknown
284 def group_packages_into_tasks(pkgs):
285 """Given a list of packages (with .tasks) group them per each
286 task and perform necessary customizations stored in .tasks
288 # Time to take care about packages and tasks
289 # Unroll pkgs into a collection of pkgs per known task
292 # Lets just create deepcopies with tune-ups for each task
293 for itask, (task, fields) in enumerate(pkg.tasks.iteritems()):
297 # Perform string completions and removals
298 for k,v in pkg_.iteritems():
300 if v is None or not len(v.strip()):
303 # Sort the fields according to FIELDS_ORDER. Unfortunately
304 # silly Deb822* cannot create from list of tuples, so will do
306 pkg__ = deb822.Deb822()
307 for k,v in sorted(pkg_.items(),
309 key_prefix_compare(x, y, order=FIELDS_ORDER)):
312 # Move Pkg-source/name into attributes
313 pkg__.source = pkg__.pop('Pkg-Source')
314 pkg__.name = pkg__.pop('Pkg-name')
316 tasks[task] = tasks.get(task, []) + [pkg__]
317 verbose(4, "Grouped %d packages into %d tasks: %s" %
318 (len(pkgs), len(tasks), ', '.join(tasks.keys())))
321 def inject_tasks(tasks, config):
322 # Now go through task files and replace/add entries
323 for task, pkgs in tasks.iteritems():
324 verbose(2, "Task %s with %d packages" % (task, len(pkgs)))
325 blend, puretask = task.split('/')
326 taskfile = join(config.get(blend, 'path'), 'tasks', puretask)
329 stats = dict(Added=[], Modified=[])
331 msgs = {'Name': pkg.name.strip(), 'Action': None}
332 # Find either it is known to the task file already
334 # Load entirely so we could simply manipulate
335 entries = open(taskfile).readlines()
337 # We need to search by name and by source
338 # We need to search for every possible type of dependecy
339 regexp = re.compile('^ *(%s) *: *(%s) *$' %
340 ('|'.join(PKG_FIELDS),
341 '|'.join((pkg.name, pkg.source))),
343 for istart, e in enumerate(entries):
345 verbose(4, "Found %s in position %i: %s" %
346 (pkg.name, istart, e.rstrip()))
350 descr = ' ; Added by %s %s. [Please note here if modified manually]\n' % \
351 (__prog__, __version__)
352 # Replace existing entry
354 # TODO: Check if previous copy does not have our preceding comment
355 # Find the previous end
358 while entries[istart+icount].strip() != '':
360 except IndexError, e:
361 pass # if we go beyond
363 # Lets not change file without necessity, if entry is identical --
366 old_entry = entries[istart:istart+icount]
368 if u''.join(old_entry) == entry:
370 else: # Rewrite the entry
371 if __prog__ in entries[istart-1]:
374 if not 'Removed' in pkg.keys():
375 entries = entries[:istart] + [descr + entry] + entries[istart+icount:]
376 msgs['Action'] = 'Changed'
378 while entries[istart-1].strip() == '':
381 entries = entries[:istart] + entries[istart+icount:]
382 msgs['Action'] = 'Removed'
383 output = ''.join(entries) # 'compute' first
384 open(taskfile, 'w').write(output) # then only overwrite
385 elif not 'removed' in pkg: # or Append one
386 msgs['Action'] = 'Added'
387 # could be as simple as
388 output = '\n%s%s' % (descr, pkg.dump(),)
389 open(taskfile, 'a').write(output)
392 verbose(3, "%(Action)s %(Name)s" % msgs)
395 class DebianMaterials(object):
396 """Extract selected information from an existing debian/
398 _WNPP_RE = re.compile('^ *\* *Initial release.*closes:? #(?P<bug>[0-9]*).*', re.I)
400 def __init__(self, topdir):
401 #self.topdir = topdir
402 self._debiandir = join(topdir, 'debian')
404 self._binaries = None
408 if self._source is None:
409 self._assign_packages()
414 if self._binaries is None:
415 self._assign_packages()
416 return self._binaries
418 def fpath(self, name):
419 return join(self._debiandir, name)
421 def _assign_packages(self):
423 control = deb822.Deb822.iter_paragraphs(
424 open(self.fpath('control')))
427 "Cannot parse %s file necessary for the %s package entry. Error: %s"
428 % (control_file, pkg['Pkg-Name'], str(e)))
432 if v.get('Source', None):
435 self._binaries[v['Package']] = v
437 def get_license(self, package=None, first_only=True):
438 """Return a license(s). Parsed out from debian/copyright if it is
439 in machine readable format
442 # may be package should carry custom copyright file
443 copyright_file_ = self.fpath('%s.copyright' % package)
444 if package and exists(copyright_file_):
445 copyright_file = copyright_file_
447 copyright_file = self.fpath('copyright')
450 for p in deb822.Deb822.iter_paragraphs(open(copyright_file)):
451 if not 'Files' in p or p['Files'].strip().startswith('debian/'):
454 # Take only the short version of first line
455 l = re.sub('\n.*', '', l).strip()
458 if not l in licenses:
465 return ', '.join(licenses)
468 """Search for a template changelog entry closing "Initial bug
470 for l in open(self.fpath('changelog')):
471 rr = self._WNPP_RE.match(l)
473 return rr.groupdict()['bug']
476 def get_responsible(self):
477 """Returns responsible, atm -- maintainer
479 return self.source['Maintainer']
481 def get_vcsfields(self):
482 vcs = deb822.Deb822()
483 for f,v in self._source.iteritems():
484 if f.lower().startswith('vcs-'):
488 def get_description(self, pkg_name):
489 """Some logic to extract description.
491 If binary package matching pkg_name is found -- gets it description.
492 If no binary package with such name, and name matches source name,
493 obtain description of the first binary package.
495 if pkg_name in self.binaries:
497 elif pkg_name.lower() == self.source['Source'].lower():
498 pkg_name = self.binaries.keys()[0]
500 error("Name %s does not match any binary, nor source package in %s"
502 return self.binaries[pkg_name]['Description']
507 usage="%s [OPTIONS] [blends_files]\n\n" % __prog__ + __doc__,
508 version="%prog " + __version__)
511 Option("-d", "--topdir", action="store",
512 dest="topdir", default=None,
513 help="Top directory of a Debian package. It is used to locate "
514 "'debian/blends' if none is specified, and where to look for "
515 "extended information."))
518 Option("-c", "--config-file", action="store",
519 dest="config_file", default=join(expanduser('~'), '.%s.cfg' % __prog__),
520 help="Noise level."))
523 Option("-v", "--verbosity", action="store", type="int",
524 dest="verbosity", default=1, help="Noise level."))
526 (options, infiles) = p.parse_args()
527 global verbosity; verbosity = options.verbosity
530 infiles = [join(options.topdir or './', 'debian/blends')] # default one
533 config = ConfigParser()
534 config.read(options.config_file)
536 for blends_file in infiles:
537 verbose(1, "Processing %s" % blends_file)
538 if not exists(blends_file):
539 error("Cannot find a file %s. Either provide a file or specify top "
540 "debian directory with -d." % blends_file, 1)
541 pkgs = parse_debian_blends(blends_file)
542 if options.topdir is None:
543 if dirname(blends_file).endswith('/debian'):
544 topdir = dirname(dirname(blends_file))
546 topdir = '.' # and hope for the best ;)
548 topdir = options.topdir
549 expand_pkgs(pkgs, topdir=topdir)
550 tasks = group_packages_into_tasks(pkgs)
551 inject_tasks(tasks, config)
554 if __name__ == '__main__':