2 #emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
3 #ex: set sts=4 ts=4 sw=4 noet:
4 """Script to help maintaining package definitions within Debian blends
7 Often it becomes necessary to duplicate the same information about a
8 package within debian packaging and multiple blends task files,
9 possibly of different blends. This script allows to automate:
11 - construction of entries
12 - injection into task files
13 - modification of existing entries if things changed
14 - removal of previously injected entries
19 * For every package the same task file might be re-read/written (if
20 entry changed/added) from disk.
21 That allows to replace easily original entry for 'source' package
22 (listed as Ignore:) with actual first listed binary package.
23 This should be taken into consideration if current per-package
32 Paths to the blends top directories, containing tasks directories are
33 specified in ~/.blends-inject.cfg file, e.g.::
36 path = /home/yoh/deb/debian-med/
39 path = /home/yoh/deb/debian-science/
42 Definition of the fields for task files by default are looked up
43 within debian/blends, or files provided in the command line.
45 Format of debian/blends
46 -----------------------
50 ; If originally filed using project source name, and it is different
51 ; from the primary (first) binary package name, keep 'Source' to be
52 ; able to adopt previously included tasks entry
55 ; Define the format on how entries should be handled.
57 ; extended -- whenever package is not in Debian and additional
58 ; fields should be obtained from debian/*:
65 ; plain [default] -- only fields listed here should be mentioned.
66 ; Common use -- whenever package is already known to UDD.
68 ; By default, all fields specified previously propagate into following
69 ; packages as well. If that is not desired, add suffix '-clean' to
73 Tasks: debian-science/neuroscience-modeling
75 ; Could have Depends/Recommends/Suggests and Ignore
76 ; All those define Pkg-Name field which is not included
77 ; in the final "rendering" but is available as Pkg-Name item
79 Pkg-URL: http://neuro.debian.net/pkgs/%(Pkg-Name)s.html
81 Published-Authors: Goodman D.F. and Brette R.
82 Published-Title: Brian: a simulator for spiking neural networks in Python
83 Published-In: Front. Neuroinform
85 Published-DOI: 10.3389/neuro.11.005.2008
87 ; May be some previous entry should be removed, thus say so
88 Remove: python-brian-doc
90 ;Tasks: debian-med/imaging-dev
91 ;Why: Allows interactive development/scripting
93 ; ; It should be possible to switch between formats,
94 ; ; e.g. if some component is not yet in Debian
97 ; ; Now some bogus one but with customizations
98 ;Tasks: debian-med/documentation
99 ;Recommends: python-brian-doc
108 from os.path import join, exists, expanduser, dirname, basename
110 from ConfigParser import ConfigParser
111 from optparse import OptionParser, Option
113 from copy import deepcopy
114 #from debian_bundle import deb822
115 from debian import deb822
117 from debian.changelog import Changelog
119 # all files we are dealing with should be UTF8, thus
124 return codecs.open(f, *args, encoding='utf-8')
126 __author__ = 'Yaroslav Halchenko'
127 __prog__ = os.path.basename(sys.argv[0])
128 __version__ = '0.0.3'
129 __copyright__ = 'Copyright (c) 2010 Yaroslav Halchenko'
132 # What fields initiate new package description
133 PKG_FIELDS = ('depends', 'recommends', 'suggests', 'ignore', 'remove')
135 # We might need to resort to assure some what a canonical order
136 # Prefixes for "standard" blends/tasks fields. Others do not get embedded
138 BLENDS_FIELDS_PREFIXES = ('depends', 'recommends', 'suggests', 'ignore',
139 'why', 'homepage', 'wnpp', 'responsible', 'license',
140 'vcs-', 'pkg-url', 'pkg-description',
141 'published-', 'x-', 'registration', 'remark')
142 # Additional fields which might come useful (e.g. for filing wnpp bugs)
143 # but are not "standard" thus should be in the trailer
144 CUSTOM_FIELDS_PREFIXES = ('author', 'language', 'pkg-name', 'pkg-source',
146 # Other fields should cause Error for consistency
148 FIELDS_ORDER = BLENDS_FIELDS_PREFIXES + CUSTOM_FIELDS_PREFIXES
152 def error(msg, exit_code=1):
153 sys.stderr.write(msg + '\n')
156 def verbose(level, msg):
157 if level <= verbosity:
161 def parse_debian_blends(f='debian/blends'):
162 """Parses debian/blends file
164 Returns unprocessed list of customized Deb822 entries
166 # Linearize all the paragraphs since we are not using them
168 for p in deb822.Deb822.iter_paragraphs(open(f)):
171 verbose(6, "Got items %s" % items)
172 # Traverse and collect things
174 format_clean = False # do not propagate fields into a new pkg if True
175 pkg, source = None, None
179 def new_pkg(prev_pkg, bname, sname, tasks):
180 """Helper function to create a new package
182 if format_clean or prev_pkg is None:
183 pkg = deb822.Deb822()
185 pkg = deepcopy(prev_pkg)
186 for k_ in PKG_FIELDS: # prune older depends
188 pkg['Pkg-Name'] = pkg[k] = bname
189 pkg['Pkg-Source'] = sname
191 pkg.tasks = dict( (t.strip(), deb822.Deb822Dict()) for t in tasks )
202 format_clean = format_.endswith('-clean')
204 format_ = format_[:-6]
207 newtasks = True # either we need to provide tune-ups
208 # for current package
209 elif kl in PKG_FIELDS: # new package
212 pkg = new_pkg(pkg, v, source, tasks)
217 # So we had just source?
219 error("No package or source is known where to add %s" % (k,), 1)
220 # TODO: just deduce source from DebianMaterials
221 pkg = new_pkg(pkg, source, source, tasks)
222 # Since only source is available, it should be Ignore:-ed
223 pkg['Ignore'] = source
227 if not t in pkg.tasks:
228 pkg.tasks[t] = deb822.Deb822Dict()
231 # just store the key in the pkg itself
236 def expand_pkgs(pkgs, topdir='.'):
237 """In-place modification of pkgs taking if necessary additional
238 information from Debian materials, and pruning empty fields
240 verbose(4, "Expanding content for %d packages" % len(pkgs))
242 # Expand packages which format is extended
244 if pkg.format == 'extended':
245 # expanding, for that we need debian/control
247 debianm = DebianMaterials(topdir)
248 for k, m in (('License', lambda: debianm.get_license(pkg['Pkg-Name'])),
249 ('WNPP', debianm.get_wnpp),
251 lambda: debianm.get_description(pkg['Pkg-Name'])),
252 ('Responsible', debianm.get_responsible),
253 ('Homepage', lambda: debianm.source.get('Homepage', None))):
260 pkg.update(debianm.get_vcsfields())
263 def prefix_index(x, entries, strict=True, case=False, default=10000):
264 """Returns an index for the x in entries
268 for i, v in enumerate(entries):
274 "Could not find location for %s as specified by %s" %
279 def key_prefix_compare(x, y, order, strict=True, case=False):
280 """Little helper to help with sorting
282 Sorts according to the order of string prefixes as given by
283 `order`. If `strict`, then if no matching prefix found, would
284 raise KeyError; otherwise provides least priority to those keys
285 which were not found in `order`
288 order = [v.lower() for v in order]
290 cmp_res = cmp(prefix_index(x[0], order, strict, case),
291 prefix_index(y[0], order, strict, case))
292 if not cmp_res: # still unknown
297 def group_packages_into_tasks(pkgs):
298 """Given a list of packages (with .tasks) group them per each
299 task and perform necessary customizations stored in .tasks
301 # Time to take care about packages and tasks
302 # Unroll pkgs into a collection of pkgs per known task
305 # Lets just create deepcopies with tune-ups for each task
306 for itask, (task, fields) in enumerate(pkg.tasks.iteritems()):
310 # Perform string completions and removals
311 for k,v in pkg_.iteritems():
313 if v is None or not len(v.strip()):
316 # Sort the fields according to FIELDS_ORDER. Unfortunately
317 # silly Deb822* cannot create from list of tuples, so will do
319 pkg__ = deb822.Deb822()
320 for k,v in sorted(pkg_.items(),
322 key_prefix_compare(x, y, order=FIELDS_ORDER)):
325 # Move Pkg-source/name into attributes
326 pkg__.source = pkg__.pop('Pkg-Source')
327 pkg__.name = pkg__.pop('Pkg-Name')
328 # Store the action taken on the package for later on actions
334 tasks[task] = tasks.get(task, []) + [pkg__]
335 verbose(4, "Grouped %d packages into %d tasks: %s" %
336 (len(pkgs), len(tasks), ', '.join(tasks.keys())))
339 def inject_tasks(tasks, config):
340 # Now go through task files and replace/add entries
341 for task, pkgs in tasks.iteritems():
342 verbose(2, "Task %s with %d packages" % (task, len(pkgs)))
343 blend, puretask = task.split('/')
344 taskfile = join(config.get(blend, 'path'), 'tasks', puretask)
347 stats = dict(Added=[], Modified=[])
349 msgs = {'Name': pkg.name.strip(), 'Action': None}
351 # Create a copy of the pkg with only valid tasks
353 # TODO: make it configurable?
356 if prefix_index(k, BLENDS_FIELDS_PREFIXES,
357 strict=False, default=None) is None:
358 pkg.pop(k) # remove it from becoming present in
361 # Find either it is known to the task file already
363 # Load entirely so we could simply manipulate
364 entries = open(taskfile).readlines()
366 # We need to search by name and by source
367 # We need to search for every possible type of dependecy
368 regexp = re.compile('^ *(%s) *: *(%s) *$' %
369 ('|'.join(PKG_FIELDS),
370 '|'.join((pkg.name, pkg.source))),
372 for istart, e in enumerate(entries):
374 verbose(4, "Found %s in position %i: %s" %
375 (pkg.name, istart, e.rstrip()))
379 descr = ' ; Added by %s %s. [Please note here if modified manually]\n' % \
380 (__prog__, __version__)
383 # Replace existing entry?
385 # TODO: Check if previous copy does not have our preceding comment
386 # Find the previous end
389 while entries[istart+icount].strip() != '':
391 except IndexError, e:
392 pass # if we go beyond
394 # Lets not change file without necessity, if entry is identical --
396 old_entry = entries[istart:istart+icount]
398 if u''.join(old_entry) == entry:
399 # no changes -- just go to the next one
401 else: # Rewrite the entry
402 if __prog__ in entries[istart-1]:
405 if 'remove' != pkg.action:
406 entry = descr + entry
407 msgs['Action'] = 'Changed'
409 while entries[istart-1].strip() == '':
413 msgs['Action'] = 'Removed'
414 entries_prior = entries[:istart]
415 entries_post = entries[istart+icount:]
416 elif not 'remove' == pkg.action: # or Append one
417 if pkg.name == 'python-brian-doc':
420 msgs['Action'] = 'Added'
421 entries_prior = entries
422 entry = descr + entry
424 # could be as simple as
425 # Lets do 'in full' for consistent handling of empty lines
427 #output = '\n%s%s' % (descr, pkg.dump(),)
428 #open(taskfile, 'a').write(output)
431 # Prepare for dumping
432 # Prune spaces before
433 while len(entries_prior) and entries_prior[-1].strip() == '':
434 entries_prior = entries_prior[:-1]
435 if len(entries_prior) and not entries_prior[-1].endswith('\n'):
436 entries_prior[-1] += '\n' # assure present trailing newline
438 while len(entries_post) and entries_post[0].strip() == '':
439 entries_post = entries_post[1:]
440 if len(entries_post) and len(entry):
441 # only then trailing empty line
443 output = ''.join(entries_prior + [ '\n' + entry ] + entries_post)
444 open(taskfile, 'w').write(output) # then only overwrite
446 verbose(3, "%(Action)s %(Name)s" % msgs)
449 class DebianMaterials(object):
450 """Extract selected information from an existing debian/
452 _WNPP_RE = re.compile('^ *\* *Initial release.*closes:? #(?P<bug>[0-9]*).*', re.I)
454 def __init__(self, topdir):
455 #self.topdir = topdir
456 self._debiandir = join(topdir, 'debian')
458 self._binaries = None
462 if self._source is None:
463 self._assign_packages()
468 if self._binaries is None:
469 self._assign_packages()
470 return self._binaries
472 def fpath(self, name):
473 return join(self._debiandir, name)
475 def _assign_packages(self):
477 control = deb822.Deb822.iter_paragraphs(
478 open(self.fpath('control')))
481 "Cannot parse %s file necessary for the %s package entry. Error: %s"
482 % (control_file, pkg['Pkg-Name'], str(e)))
486 if v.get('Source', None):
489 self._binaries[v['Package']] = v
491 def get_license(self, package=None, first_only=True):
492 """Return a license(s). Parsed out from debian/copyright if it is
493 in machine readable format
496 # may be package should carry custom copyright file
497 copyright_file_ = self.fpath('%s.copyright' % package)
498 if package and exists(copyright_file_):
499 copyright_file = copyright_file_
501 copyright_file = self.fpath('copyright')
504 for p in deb822.Deb822.iter_paragraphs(open(copyright_file)):
505 if not 'Files' in p or p['Files'].strip().startswith('debian/'):
508 # Take only the short version of first line
509 l = re.sub('\n.*', '', l).strip()
512 if not l in licenses:
519 return ', '.join(licenses)
522 """Search for a template changelog entry closing "Initial bug
524 for l in open(self.fpath('changelog')):
525 rr = self._WNPP_RE.match(l)
527 return rr.groupdict()['bug']
530 def get_responsible(self):
531 """Returns responsible, atm -- maintainer
533 return self.source['Maintainer']
535 def get_vcsfields(self):
536 vcs = deb822.Deb822()
537 for f,v in self._source.iteritems():
538 if f.lower().startswith('vcs-'):
542 def get_description(self, pkg_name):
543 """Some logic to extract description.
545 If binary package matching pkg_name is found -- gets it description.
546 If no binary package with such name, and name matches source name,
547 obtain description of the first binary package.
549 if pkg_name in self.binaries:
551 elif pkg_name.lower() == self.source['Source'].lower():
552 pkg_name = self.binaries.keys()[0]
554 error("Name %s does not match any binary, nor source package in %s"
556 return self.binaries[pkg_name]['Description']
561 usage="%s [OPTIONS] [blends_files]\n\n" % __prog__ + __doc__,
562 version="%prog " + __version__)
565 Option("-d", "--topdir", action="store",
566 dest="topdir", default=None,
567 help="Top directory of a Debian package. It is used to locate "
568 "'debian/blends' if none is specified, and where to look for "
569 "extended information."))
572 Option("-c", "--config-file", action="store",
573 dest="config_file", default=join(expanduser('~'), '.%s.cfg' % __prog__),
574 help="Noise level."))
577 Option("-v", "--verbosity", action="store", type="int",
578 dest="verbosity", default=1, help="Noise level."))
580 (options, infiles) = p.parse_args()
581 global verbosity; verbosity = options.verbosity
584 infiles = [join(options.topdir or './', 'debian/blends')] # default one
587 config = ConfigParser()
588 config.read(options.config_file)
590 for blends_file in infiles:
591 verbose(1, "Processing %s" % blends_file)
592 if not exists(blends_file):
593 error("Cannot find a file %s. Either provide a file or specify top "
594 "debian directory with -d." % blends_file, 1)
595 pkgs = parse_debian_blends(blends_file)
596 if options.topdir is None:
597 if dirname(blends_file).endswith('/debian'):
598 topdir = dirname(dirname(blends_file))
600 topdir = '.' # and hope for the best ;)
602 topdir = options.topdir
603 expand_pkgs(pkgs, topdir=topdir)
604 tasks = group_packages_into_tasks(pkgs)
605 inject_tasks(tasks, config)
608 if __name__ == '__main__':