2 #emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
3 #ex: set sts=4 ts=4 sw=4 noet:
4 """Script to help maintaining package definitions within Debian blends
7 Often it becomes necessary to duplicate the same information about a
8 package within debian packaging and multiple blends task files,
9 possibly of different blends. This script allows to automate:
11 - construction of entries
12 - injection into task files
13 - modification of existing entries if things changed
14 - removal of previously injected entries
19 * For every package the same task file might be re-read/written (if
20 entry changed/added) from disk.
21 That allows to replace easily original entry for 'source' package
22 (listed as Suggests:) with actual first listed binary package.
23 This should be taken into consideration if current per-package
32 Paths to the blends top directories, containing tasks directories are
33 specified in ~/.blends-inject.cfg file, e.g.::
36 path = /home/yoh/deb/debian-med/
39 path = /home/yoh/deb/debian-science/
42 Definition of the fields for task files by default are looked up
43 within debian/blends, or files provided in the command line.
45 Format of debian/blends
46 -----------------------
50 ; If originally filed using project source name, and it is different
51 ; from the primary (first) binary package name, keep 'Source' to be
52 ; able to adopt previously included tasks entry
55 ; Define the format on how entries should be handled.
57 ; extended -- whenever package is not in Debian and additional
58 ; fields should be obtained from debian/*:
65 ; plain [default] -- only fields listed here should be mentioned.
66 ; Common use -- whenever package is already known to UDD.
68 ; By default, all fields specified previously propagate into following
69 ; packages as well. If that is not desired, add suffix '-clean' to
73 Tasks: debian-science/neuroscience-modeling
75 ; Could have Depends/Recommends/Suggests and Ignore
76 ; All those define Pkg-Name field which is not included
77 ; in the final "rendering" but is available as Pkg-Name item
79 Pkg-URL: http://neuro.debian.net/pkgs/%(Pkg-Name)s.html
81 Published-Authors: Goodman D.F. and Brette R.
82 Published-Title: Brian: a simulator for spiking neural networks in Python
83 Published-In: Front. Neuroinform
85 Published-DOI: 10.3389/neuro.11.005.2008
87 ; May be some previous entry should be removed, thus say so
88 Remove: python-brian-doc
90 ;Tasks: debian-med/imaging-dev
91 ;Why: Allows interactive development/scripting
93 ; ; It should be possible to switch between formats,
94 ; ; e.g. if some component is not yet in Debian
97 ; ; Now some bogus one but with customizations
98 ;Tasks: debian-med/documentation
99 ;Recommends: python-brian-doc
107 import re, os, sys, tempfile
108 from os.path import join, exists, expanduser, dirname, basename
110 from ConfigParser import ConfigParser
111 from optparse import OptionParser, Option
113 from copy import deepcopy
114 #from debian_bundle import deb822
115 from debian import deb822
117 from debian.changelog import Changelog
119 # all files we are dealing with should be UTF8, thus
124 return codecs.open(f, *args, encoding='utf-8')
126 __author__ = 'Yaroslav Halchenko'
127 __prog__ = os.path.basename(sys.argv[0])
128 __version__ = '0.0.4'
129 __copyright__ = 'Copyright (c) 2010 Yaroslav Halchenko'
132 # What fields initiate new package description
133 PKG_FIELDS = ('depends', 'recommends', 'suggests', 'ignore', 'remove')
135 # We might need to resort to assure some what a canonical order
136 # Prefixes for "standard" blends/tasks fields. Others do not get embedded
138 BLENDS_FIELDS_PREFIXES = ('depends', 'recommends', 'suggests', 'ignore',
139 'why', 'homepage', 'language', 'wnpp', 'responsible', 'license',
140 'vcs-', 'pkg-url', 'pkg-description',
141 'published-', 'x-', 'registration', 'remark')
142 # Additional fields which might come useful (e.g. for filing wnpp bugs)
143 # but are not "standard" thus should be in the trailer
144 CUSTOM_FIELDS_PREFIXES = ('author', 'pkg-name', 'pkg-source',
146 # Other fields should cause Error for consistency
148 FIELDS_ORDER = BLENDS_FIELDS_PREFIXES + CUSTOM_FIELDS_PREFIXES
152 def error(msg, exit_code=1):
153 sys.stderr.write(msg + '\n')
156 def verbose(level, msg):
157 if level <= verbosity:
158 sys.stderr.write(" "*level + msg + '\n')
161 def parse_debian_blends(f='debian/blends'):
162 """Parses debian/blends file
164 Returns unprocessed list of customized Deb822 entries
166 # Linearize all the paragraphs since we are not using them
168 for p in deb822.Deb822.iter_paragraphs(open(f)):
171 verbose(6, "Got items %s" % items)
172 # Traverse and collect things
174 format_clean = False # do not propagate fields into a new pkg if True
175 pkg, source = None, None
179 def new_pkg(prev_pkg, bname, sname, tasks):
180 """Helper function to create a new package
182 if format_clean or prev_pkg is None:
183 pkg = deb822.Deb822()
185 pkg = deepcopy(prev_pkg)
186 for k_ in PKG_FIELDS: # prune older depends
188 pkg['Pkg-Name'] = pkg[k] = bname
189 pkg['Pkg-Source'] = sname
191 pkg.tasks = dict( (t.strip(), deb822.Deb822Dict()) for t in tasks )
203 format_clean = format_.endswith('-clean')
205 format_ = format_[:-6]
208 newtasks = pkg is not None # either we need to provide tune-ups
209 # for current package
210 elif kl in PKG_FIELDS: # new package
213 pkg = new_pkg(pkg, v, source, tasks)
217 # So we had just source?
219 error("No package or source is known where to add %s" % (k,), 1)
220 # TODO: just deduce source from DebianMaterials
221 pkg = new_pkg(pkg, source, source, tasks)
222 # Since only source is available, it should be only Suggest:-ed
223 pkg['Suggests'] = source
229 if not t in pkg.tasks:
230 pkg.tasks[t] = deb822.Deb822Dict()
233 # just store the key in the pkg itself
239 def expand_pkgs(pkgs, topdir='.'):
240 """In-place modification of pkgs taking if necessary additional
241 information from Debian materials, and pruning empty fields
243 verbose(4, "Expanding content for %d packages" % len(pkgs))
246 # Expand packages which format is extended
248 if pkg.format == 'extended':
249 # expanding, for that we need debian/control
251 debianm = DebianMaterials(topdir)
252 for k, m in (('License', lambda: debianm.get_license(pkg['Pkg-Name'])),
253 ('WNPP', debianm.get_wnpp),
255 lambda: debianm.get_description(pkg['Pkg-Name'])),
256 ('Responsible', debianm.get_responsible),
257 ('Homepage', lambda: debianm.source.get('Homepage', None))):
264 pkg.update(debianm.get_vcsfields())
267 def prefix_index(x, entries, strict=True, case=False, default=10000):
268 """Returns an index for the x in entries
272 for i, v in enumerate(entries):
278 "Could not find location for %s as specified by %s" %
283 def key_prefix_compare(x, y, order, strict=True, case=False):
284 """Little helper to help with sorting
286 Sorts according to the order of string prefixes as given by
287 `order`. If `strict`, then if no matching prefix found, would
288 raise KeyError; otherwise provides least priority to those keys
289 which were not found in `order`
292 order = [v.lower() for v in order]
294 cmp_res = cmp(prefix_index(x[0], order, strict, case),
295 prefix_index(y[0], order, strict, case))
296 if not cmp_res: # still unknown
301 def group_packages_into_tasks(pkgs):
302 """Given a list of packages (with .tasks) group them per each
303 task and perform necessary customizations stored in .tasks
305 # Time to take care about packages and tasks
306 # Unroll pkgs into a collection of pkgs per known task
309 # Lets just create deepcopies with tune-ups for each task
310 for itask, (task, fields) in enumerate(pkg.tasks.iteritems()):
314 # Perform string completions and removals
315 for k,v in pkg_.iteritems():
317 if v is None or not len(v.strip()):
320 # Sort the fields according to FIELDS_ORDER. Unfortunately
321 # silly Deb822* cannot create from list of tuples, so will do
323 pkg__ = deb822.Deb822()
324 for k,v in sorted(pkg_.items(),
326 key_prefix_compare(x, y, order=FIELDS_ORDER)):
329 # Move Pkg-source/name into attributes
330 pkg__.source = pkg__.pop('Pkg-Source')
331 pkg__.name = pkg__.pop('Pkg-Name')
332 # Store the action taken on the package for later on actions
338 tasks[task] = tasks.get(task, []) + [pkg__]
339 verbose(4, "Grouped %d packages into %d tasks: %s" %
340 (len(pkgs), len(tasks), ', '.join(tasks.keys())))
343 def inject_tasks(tasks, config):
344 # Now go through task files and replace/add entries
345 for task, pkgs in tasks.iteritems():
346 verbose(2, "Task %s with %d packages" % (task, len(pkgs)))
347 blend, puretask = task.split('/')
348 taskfile = join(config.get(blend, 'path'), 'tasks', puretask)
351 stats = dict(Added=[], Modified=[])
353 msgs = {'Name': pkg.name.strip(), 'Action': None}
355 # Create a copy of the pkg with only valid tasks
357 # TODO: make it configurable?
360 if prefix_index(k, BLENDS_FIELDS_PREFIXES,
361 strict=False, default=None) is None:
362 pkg.pop(k) # remove it from becoming present in
365 # Find either it is known to the task file already
367 # Load entirely so we could simply manipulate
368 entries = open(taskfile).readlines()
370 # We need to search by name and by source
371 # We need to search for every possible type of dependecy
372 regexp = re.compile('^ *(%s) *: *(%s) *$' %
373 ('|'.join(PKG_FIELDS),
374 '|'.join((pkg.name, pkg.source))),
376 for istart, e in enumerate(entries):
378 verbose(4, "Found %s in position %i: %s" %
379 (pkg.name, istart, e.rstrip()))
383 descr = ' ; Added by %s %s. [Please note here if modified manually]\n' % \
384 (__prog__, __version__)
387 # Replace existing entry?
389 # TODO: Check if previous copy does not have our preceding comment
390 # Find the previous end
393 while entries[istart+icount].strip() != '':
395 except IndexError, e:
396 pass # if we go beyond
398 # Lets not change file without necessity, if entry is identical --
400 old_entry = entries[istart:istart+icount]
402 if u''.join(old_entry) == entry:
403 # no changes -- just go to the next one
405 else: # Rewrite the entry
406 if __prog__ in entries[istart-1]:
409 if 'remove' != pkg.action:
410 entry = descr + entry
411 msgs['Action'] = 'Changed'
413 while entries[istart-1].strip() == '':
417 msgs['Action'] = 'Removed'
418 entries_prior = entries[:istart]
419 entries_post = entries[istart+icount:]
420 elif not 'remove' == pkg.action: # or Append one
421 msgs['Action'] = 'Added'
422 entries_prior = entries
423 entry = descr + entry
425 # could be as simple as
426 # Lets do 'in full' for consistent handling of empty lines
428 #output = '\n%s%s' % (descr, pkg.dump(),)
429 #open(taskfile, 'a').write(output)
432 # Prepare for dumping
433 # Prune spaces before
434 while len(entries_prior) and entries_prior[-1].strip() == '':
435 entries_prior = entries_prior[:-1]
436 if len(entries_prior) and not entries_prior[-1].endswith('\n'):
437 entries_prior[-1] += '\n' # assure present trailing newline
439 while len(entries_post) and entries_post[0].strip() == '':
440 entries_post = entries_post[1:]
441 if len(entries_post) and len(entry):
442 # only then trailing empty line
444 output = ''.join(entries_prior + [ '\n' + entry ] + entries_post)
445 open(taskfile, 'w').write(output) # then only overwrite
447 verbose(3, "%(Action)s %(Name)s" % msgs)
450 class DebianMaterials(object):
451 """Extract selected information from an existing debian/
453 _WNPP_RE = re.compile('^ *\* *Initial release.*closes:? #(?P<bug>[0-9]*).*', re.I)
455 def __init__(self, topdir):
456 #self.topdir = topdir
457 self._debiandir = join(topdir, 'debian')
459 self._binaries = None
463 if self._source is None:
464 self._assign_packages()
469 if self._binaries is None:
470 self._assign_packages()
471 return self._binaries
473 def fpath(self, name):
474 return join(self._debiandir, name)
476 def _assign_packages(self):
478 control = deb822.Deb822.iter_paragraphs(
479 open(self.fpath('control')))
482 "Cannot parse %s file necessary for the %s package entry. Error: %s"
483 % (control_file, pkg['Pkg-Name'], str(e)))
487 if v.get('Source', None):
490 self._binaries[v['Package']] = v
492 def get_license(self, package=None, first_only=True):
493 """Return a license(s). Parsed out from debian/copyright if it is
494 in machine readable format
497 # may be package should carry custom copyright file
498 copyright_file_ = self.fpath('%s.copyright' % package)
499 if package and exists(copyright_file_):
500 copyright_file = copyright_file_
502 copyright_file = self.fpath('copyright')
505 for p in deb822.Deb822.iter_paragraphs(open(copyright_file)):
506 if not 'Files' in p or p['Files'].strip().startswith('debian/'):
509 # Take only the short version of first line
510 l = re.sub('\n.*', '', l).strip()
513 if not l in licenses:
520 return ', '.join(licenses)
523 """Search for a template changelog entry closing "Initial bug
525 for l in open(self.fpath('changelog')):
526 rr = self._WNPP_RE.match(l)
528 return rr.groupdict()['bug']
531 def get_responsible(self):
532 """Returns responsible, atm -- maintainer
534 return self.source['Maintainer']
536 def get_vcsfields(self):
537 vcs = deb822.Deb822()
538 for f,v in self._source.iteritems():
539 if f.lower().startswith('vcs-'):
543 def get_description(self, pkg_name):
544 """Some logic to extract description.
546 If binary package matching pkg_name is found -- gets it description.
547 If no binary package with such name, and name matches source name,
548 obtain description of the first binary package.
550 if pkg_name in self.binaries:
552 elif pkg_name.lower() == self.source['Source'].lower():
553 pkg_name = self.binaries.keys()[0]
555 error("Name %s does not match any binary, nor source package in %s"
557 return self.binaries[pkg_name]['Description']
559 def print_wnpp(pkgs, config, wnpp_type="ITP"):
560 """Little helper to spit out formatted entry for WNPP bugreport
562 TODO: It would puke atm if any field is missing
565 pkg = pkgs[0] # everything is based on the 1st one
566 opts = dict(pkg.items())
567 opts['WNPP-Type'] = wnpp_type.upper()
568 opts['Pkg-Description-Short'] = re.sub('\n.*', '', pkg['Pkg-Description'])
570 subject = "%(WNPP-Type)s: %(Pkg-Name)s -- %(Pkg-Description-Short)s" % opts
571 body = """*** Please type your report below this line ***
573 * Package name : %(Pkg-Name)s
574 Version : %(Version)s
575 Upstream Author : %(Author)s
577 * License : %(License)s
578 Programming Lang: %(Language)s
579 Description : %(Pkg-Description)s
583 # Unfortunately could not figure out how to set the owner, so I will just print it out
585 tmpfile = tempfile.NamedTemporaryFile()
588 cmd = "reportbug -b --paranoid --subject='%s' --severity=wishlist --body-file='%s' -o /tmp/o.txt wnpp" \
589 % (subject, tmpfile.name)
590 verbose(2, "Running %s" %cmd)
593 print "Subject: %s\n\n%s" % (subject, body)
599 usage="%s [OPTIONS] [blends_files]\n\n" % __prog__ + __doc__,
600 version="%prog " + __version__)
603 Option("-d", "--topdir", action="store",
604 dest="topdir", default=None,
605 help="Top directory of a Debian package. It is used to locate "
606 "'debian/blends' if none is specified, and where to look for "
607 "extended information."))
610 Option("-c", "--config-file", action="store",
611 dest="config_file", default=join(expanduser('~'), '.%s.cfg' % __prog__),
612 help="Noise level."))
615 Option("-v", "--verbosity", action="store", type="int",
616 dest="verbosity", default=1, help="Noise level."))
618 # We might like to create a separate 'group' of options for commands
620 Option("-w", action="store_true",
621 dest="wnpp", default=False,
622 help="Operate in WNPP mode: dumps cut-paste-able entry for WNPP bugreport"))
625 Option("--wnpp", action="store",
626 dest="wnpp_mode", default=None,
627 help="Operate in WNPP mode: dumps cut-paste-able entry for WNPP bugreport"))
630 (options, infiles) = p.parse_args()
631 global verbosity; verbosity = options.verbosity
633 if options.wnpp and options.wnpp_mode is None:
634 options.wnpp_mode = 'ITP'
637 infiles = [join(options.topdir or './', 'debian/blends')] # default one
640 config = ConfigParser()
641 config.read(options.config_file)
643 for blends_file in infiles:
644 verbose(1, "Processing %s" % blends_file)
645 if not exists(blends_file):
646 error("Cannot find a file %s. Either provide a file or specify top "
647 "debian directory with -d." % blends_file, 1)
648 pkgs = parse_debian_blends(blends_file)
649 if options.topdir is None:
650 if dirname(blends_file).endswith('/debian'):
651 topdir = dirname(dirname(blends_file))
653 topdir = '.' # and hope for the best ;)
655 topdir = options.topdir
657 expand_pkgs(pkgs, topdir=topdir)
658 if options.wnpp_mode is not None:
659 print_wnpp(pkgs, config, options.wnpp_mode)
661 # by default -- operate on blends/tasks files
662 tasks = group_packages_into_tasks(pkgs)
663 inject_tasks(tasks, config)
666 if __name__ == '__main__':