2 #emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
3 #ex: set sts=4 ts=4 sw=4 noet:
4 """Script to help maintaining package definitions within Debian blends
7 Often it becomes necessary to duplicate the same information about a
8 package within debian packaging and multiple blends task files,
9 possibly of different blends. This script allows to automate:
11 - construction of entries
12 - injection into task files
13 - modification of existing entries if things changed
14 - removal of previously injected entries
19 * For every package the same task file might be re-read/written (if
20 entry changed/added) from disk.
21 That allows to replace easily original entry for 'source' package
22 (listed as Suggests:) with actual first listed binary package.
23 This should be taken into consideration if current per-package
32 Paths to the blends top directories, containing tasks directories are
33 specified in ~/.blends-inject.cfg file, e.g.::
36 path = ~/deb/debian-med/
39 path = ~/deb/debian-science/
41 Definition of the fields for task files by default are looked up
42 within debian/blends, or files provided in the command line. Also for "-a"
43 mode of operation you should define list of globs to match your debian/blends
47 all=~/deb/gits/pkg-exppsy/neurodebian/future/blends/*
48 ~/deb/gits/*/debian/blends
49 ~/deb/gits/pkg-exppsy/*/debian/blends
50 # Python regular expression on which files to skip
51 # Default is listed below
55 Format of debian/blends
56 -----------------------
60 ; If originally filed using project source name, and it is different
61 ; from the primary (first) binary package name, keep 'Source' to be
62 ; able to adopt previously included tasks entry
65 ; Define the format on how entries should be handled.
67 ; extended -- whenever package is not in Debian and additional
68 ; fields should be obtained from debian/*:
75 ; plain [default] -- only fields listed here should be mentioned.
76 ; Common use -- whenever package is already known to UDD.
78 ; By default, all fields specified previously propagate into following
79 ; packages as well. If that is not desired, add suffix '-clean' to
83 Tasks: debian-science/neuroscience-modeling
85 ; Could have Depends/Recommends/Suggests and Ignore
86 ; All those define Pkg-Name field which is not included
87 ; in the final "rendering" but is available as Pkg-Name item
89 Pkg-URL: http://neuro.debian.net/pkgs/%(Pkg-Name)s.html
91 Published-Authors: Goodman D.F. and Brette R.
92 Published-Title: Brian: a simulator for spiking neural networks in Python
93 Published-In: Front. Neuroinform
95 Published-DOI: 10.3389/neuro.11.005.2008
97 ; May be some previous entry should be removed, thus say so
98 Remove: python-brian-doc
100 ;Tasks: debian-med/imaging-dev
101 ;Why: Allows interactive development/scripting
103 ; ; It should be possible to switch between formats,
104 ; ; e.g. if some component is not yet in Debian
107 ; ; Now some bogus one but with customizations
108 ;Tasks: debian-med/documentation
109 ;Recommends: python-brian-doc
117 import re, os, sys, tempfile, glob
118 from os.path import join, exists, expanduser, dirname, basename
120 from ConfigParser import ConfigParser
121 from optparse import OptionParser, Option
123 from copy import deepcopy
124 #from debian_bundle import deb822
125 from debian import deb822
127 from debian.changelog import Changelog
129 # all files we are dealing with should be UTF8, thus
134 return codecs.open(f, *args, encoding='utf-8')
136 __author__ = 'Yaroslav Halchenko'
137 __prog__ = os.path.basename(sys.argv[0])
138 __version__ = '0.0.6'
139 __copyright__ = 'Copyright (c) 2010 Yaroslav Halchenko'
142 # What fields initiate new package description
143 PKG_FIELDS = ('depends', 'recommends', 'suggests', 'ignore', 'remove')
145 # We might need to resort to assure some what a canonical order
146 # Prefixes for "standard" blends/tasks fields. Others do not get embedded
148 BLENDS_FIELDS_PREFIXES = ('depends', 'recommends', 'suggests', 'ignore',
149 'why', 'homepage', 'language', 'wnpp', 'responsible', 'license',
150 'vcs-', 'pkg-url', 'pkg-description',
151 'published-', 'x-', 'registration', 'remark')
152 # Additional fields which might come useful (e.g. for filing wnpp bugs)
153 # but are not "standard" thus should be in the trailer
154 CUSTOM_FIELDS_PREFIXES = ('author', 'pkg-name', 'pkg-source',
156 # Other fields should cause Error for consistency
158 FIELDS_ORDER = BLENDS_FIELDS_PREFIXES + CUSTOM_FIELDS_PREFIXES
162 def error(msg, exit_code=1):
163 sys.stderr.write(msg + '\n')
166 def verbose(level, msg):
167 if level <= verbosity:
168 sys.stderr.write(" "*level + msg + '\n')
171 def parse_debian_blends(f='debian/blends'):
172 """Parses debian/blends file
174 Returns unprocessed list of customized Deb822 entries
176 # Linearize all the paragraphs since we are not using them
178 for p in deb822.Deb822.iter_paragraphs(open(f)):
181 verbose(6, "Got items %s" % items)
182 # Traverse and collect things
184 format_clean = False # do not propagate fields into a new pkg if True
185 pkg, source = None, None
189 def new_pkg(prev_pkg, bname, sname, tasks):
190 """Helper function to create a new package
192 if format_clean or prev_pkg is None:
193 pkg = deb822.Deb822()
195 pkg = deepcopy(prev_pkg)
196 for k_ in PKG_FIELDS: # prune older depends
198 pkg['Pkg-Name'] = pkg[k] = bname.lower()
199 pkg['Pkg-Source'] = sname.lower()
201 pkg.tasks = dict( (t.strip(), deb822.Deb822Dict()) for t in tasks )
213 format_clean = format_.endswith('-clean')
215 format_ = format_[:-6]
218 newtasks = pkg is not None # either we need to provide tune-ups
219 # for current package
220 elif kl in PKG_FIELDS: # new package
223 pkg = new_pkg(pkg, v, source, tasks)
227 # So we had just source?
229 error("No package or source is known where to add %s" % (k,), 1)
230 # TODO: just deduce source from DebianMaterials
231 pkg = new_pkg(pkg, source, source, tasks)
232 # Since only source is available, it should be only Suggest:-ed
233 pkg['Suggests'] = source.lower()
239 if not t in pkg.tasks:
240 pkg.tasks[t] = deb822.Deb822Dict()
243 # just store the key in the pkg itself
249 def expand_pkgs(pkgs, topdir='.'):
250 """In-place modification of pkgs taking if necessary additional
251 information from Debian materials, and pruning empty fields
253 verbose(4, "Expanding content for %d packages" % len(pkgs))
256 # Expand packages which format is extended
258 if pkg.format == 'extended':
259 # expanding, for that we need debian/control
261 debianm = DebianMaterials(topdir)
262 for k, m in (('License', lambda: debianm.get_license(pkg['Pkg-Name'])),
263 ('WNPP', debianm.get_wnpp),
265 lambda: debianm.get_description(pkg['Pkg-Name'])),
266 ('Responsible', debianm.get_responsible),
267 ('Homepage', lambda: debianm.source.get('Homepage', None))):
274 pkg.update(debianm.get_vcsfields())
277 def prefix_index(x, entries, strict=True, case=False, default=10000):
278 """Returns an index for the x in entries
282 for i, v in enumerate(entries):
288 "Could not find location for %s as specified by %s" %
293 def key_prefix_compare(x, y, order, strict=True, case=False):
294 """Little helper to help with sorting
296 Sorts according to the order of string prefixes as given by
297 `order`. If `strict`, then if no matching prefix found, would
298 raise KeyError; otherwise provides least priority to those keys
299 which were not found in `order`
302 order = [v.lower() for v in order]
304 cmp_res = cmp(prefix_index(x[0], order, strict, case),
305 prefix_index(y[0], order, strict, case))
306 if not cmp_res: # still unknown
311 def group_packages_into_tasks(pkgs):
312 """Given a list of packages (with .tasks) group them per each
313 task and perform necessary customizations stored in .tasks
315 # Time to take care about packages and tasks
316 # Unroll pkgs into a collection of pkgs per known task
319 # Lets just create deepcopies with tune-ups for each task
320 for itask, (task, fields) in enumerate(pkg.tasks.iteritems()):
324 # Perform string completions and removals
325 for k,v in pkg_.iteritems():
327 if v is None or not len(v.strip()):
330 # Sort the fields according to FIELDS_ORDER. Unfortunately
331 # silly Deb822* cannot create from list of tuples, so will do
333 pkg__ = deb822.Deb822()
334 for k,v in sorted(pkg_.items(),
336 key_prefix_compare(x, y, order=FIELDS_ORDER)):
339 # Move Pkg-source/name into attributes
340 pkg__.source = pkg__.pop('Pkg-Source')
341 pkg__.name = pkg__.pop('Pkg-Name')
342 # Store the action taken on the package for later on actions
348 tasks[task] = tasks.get(task, []) + [pkg__]
349 verbose(4, "Grouped %d packages into %d tasks: %s" %
350 (len(pkgs), len(tasks), ', '.join(tasks.keys())))
353 def inject_tasks(tasks, config):
354 # Now go through task files and replace/add entries
355 for task, pkgs in tasks.iteritems():
356 verbose(2, "Task %s with %d packages" % (task, len(pkgs)))
357 blend, puretask = task.split('/')
358 taskfile = expanduser(join(config.get(blend, 'path'), 'tasks', puretask))
361 stats = dict(Added=[], Modified=[])
363 msgs = {'Name': pkg.name.strip(), 'Action': None}
365 # Create a copy of the pkg with only valid tasks
367 # TODO: make it configurable?
370 if prefix_index(k, BLENDS_FIELDS_PREFIXES,
371 strict=False, default=None) is None:
372 pkg.pop(k) # remove it from becoming present in
375 # Find either it is known to the task file already
377 # Load entirely so we could simply manipulate
378 entries = open(taskfile).readlines()
380 # We need to search by name and by source
381 # We need to search for every possible type of dependecy
382 regexp = re.compile('^ *(%s) *: *(%s) *$' %
383 ('|'.join(PKG_FIELDS),
384 '|'.join((pkg.name, pkg.source))),
386 for istart, e in enumerate(entries):
388 verbose(4, "Found %s in position %i: %s" %
389 (pkg.name, istart, e.rstrip()))
393 descr = ' ; Added by %s %s. [Please note here if modified manually]\n' % \
394 (__prog__, __version__)
397 # Replace existing entry?
399 # TODO: Check if previous copy does not have our preceding comment
400 # Find the previous end
403 while entries[istart+icount].strip() != '':
405 except IndexError, e:
406 pass # if we go beyond
408 # Lets not change file without necessity, if entry is identical --
410 old_entry = entries[istart:istart+icount]
412 if u''.join(old_entry) == entry:
413 # no changes -- just go to the next one
415 else: # Rewrite the entry
416 if __prog__ in entries[istart-1]:
419 if 'remove' != pkg.action:
420 entry = descr + entry
421 msgs['Action'] = 'Changed'
423 while entries[istart-1].strip() == '':
427 msgs['Action'] = 'Removed'
428 entries_prior = entries[:istart]
429 entries_post = entries[istart+icount:]
430 elif not 'remove' == pkg.action: # or Append one
431 msgs['Action'] = 'Added'
432 entries_prior = entries
433 entry = descr + entry
435 # could be as simple as
436 # Lets do 'in full' for consistent handling of empty lines
438 #output = '\n%s%s' % (descr, pkg.dump(),)
439 #open(taskfile, 'a').write(output)
442 # Prepare for dumping
443 # Prune spaces before
444 while len(entries_prior) and entries_prior[-1].strip() == '':
445 entries_prior = entries_prior[:-1]
446 if len(entries_prior) and not entries_prior[-1].endswith('\n'):
447 entries_prior[-1] += '\n' # assure present trailing newline
449 while len(entries_post) and entries_post[0].strip() == '':
450 entries_post = entries_post[1:]
451 if len(entries_post) and len(entry):
452 # only then trailing empty line
454 output = ''.join(entries_prior + [ '\n' + entry ] + entries_post)
455 open(taskfile, 'w').write(output) # then only overwrite
457 verbose(3, "%(Action)s %(Name)s" % msgs)
460 class DebianMaterials(object):
461 """Extract selected information from an existing debian/
463 _WNPP_RE = re.compile('^ *\* *Initial release.*closes:? #(?P<bug>[0-9]*).*', re.I)
465 def __init__(self, topdir):
466 #self.topdir = topdir
467 self._debiandir = join(topdir, 'debian')
469 self._binaries = None
473 if self._source is None:
474 self._assign_packages()
479 if self._binaries is None:
480 self._assign_packages()
481 return self._binaries
483 def fpath(self, name):
484 return join(self._debiandir, name)
486 def _assign_packages(self):
488 control = deb822.Deb822.iter_paragraphs(
489 open(self.fpath('control')))
492 "Cannot parse %s file necessary for the %s package entry. Error: %s"
493 % (control_file, pkg['Pkg-Name'], str(e)))
497 if v.get('Source', None):
500 self._binaries[v['Package']] = v
502 def get_license(self, package=None, first_only=True):
503 """Return a license(s). Parsed out from debian/copyright if it is
504 in machine readable format
507 # may be package should carry custom copyright file
508 copyright_file_ = self.fpath('%s.copyright' % package)
509 if package and exists(copyright_file_):
510 copyright_file = copyright_file_
512 copyright_file = self.fpath('copyright')
515 for p in deb822.Deb822.iter_paragraphs(open(copyright_file)):
516 if not 'Files' in p or p['Files'].strip().startswith('debian/'):
519 # Take only the short version of first line
520 l = re.sub('\n.*', '', l).strip()
523 if not l in licenses:
530 return ', '.join(licenses)
533 """Search for a template changelog entry closing "Initial bug
535 for l in open(self.fpath('changelog')):
536 rr = self._WNPP_RE.match(l)
538 return rr.groupdict()['bug']
541 def get_responsible(self):
542 """Returns responsible, atm -- maintainer
544 return self.source['Maintainer']
546 def get_vcsfields(self):
547 vcs = deb822.Deb822()
548 for f,v in self._source.iteritems():
549 if f.lower().startswith('vcs-'):
553 def get_description(self, pkg_name):
554 """Some logic to extract description.
556 If binary package matching pkg_name is found -- gets it description.
557 If no binary package with such name, and name matches source name,
558 obtain description of the first binary package.
560 if pkg_name in self.binaries:
562 elif pkg_name.lower() == self.source['Source'].lower():
563 pkg_name = self.binaries.keys()[0]
565 error("Name %s does not match any binary, nor source package in %s"
567 return self.binaries[pkg_name]['Description']
569 def print_wnpp(pkgs, config, wnpp_type="ITP"):
570 """Little helper to spit out formatted entry for WNPP bugreport
572 TODO: It would puke atm if any field is missing
575 pkg = pkgs[0] # everything is based on the 1st one
576 opts = dict(pkg.items())
577 opts['WNPP-Type'] = wnpp_type.upper()
578 opts['Pkg-Description-Short'] = re.sub('\n.*', '', pkg['Pkg-Description'])
580 subject = "%(WNPP-Type)s: %(Pkg-Name)s -- %(Pkg-Description-Short)s" % opts
581 body = """*** Please type your report below this line ***
583 * Package name : %(Pkg-Name)s
584 Version : %(Version)s
585 Upstream Author : %(Author)s
587 * License : %(License)s
588 Programming Lang: %(Language)s
589 Description : %(Pkg-Description)s
593 # Unfortunately could not figure out how to set the owner, so I will just print it out
595 tmpfile = tempfile.NamedTemporaryFile()
598 cmd = "reportbug -b --paranoid --subject='%s' --severity=wishlist --body-file='%s' -o /tmp/o.txt wnpp" \
599 % (subject, tmpfile.name)
600 verbose(2, "Running %s" %cmd)
603 print "Subject: %s\n\n%s" % (subject, body)
607 """Helper to return true if pkg definition looks like a template
608 and should not be processed
610 # We might want to skip some which define a skeleton
611 # (no source/homepage/etc although fields are there)
612 for f in ['vcs-browser', 'pkg-url', 'pkg-description',
613 'published-Title', 'pkg-name', 'homepage',
615 if f in p and p[f] != "":
623 usage="%s [OPTIONS] [blends_files]\n\n" % __prog__ + __doc__,
624 version="%prog " + __version__)
627 Option("-d", "--topdir", action="store",
628 dest="topdir", default=None,
629 help="Top directory of a Debian package. It is used to locate "
630 "'debian/blends' if none is specified, and where to look for "
631 "extended information."))
634 Option("-c", "--config-file", action="store",
635 dest="config_file", default=join(expanduser('~'), '.%s.cfg' % __prog__),
636 help="Noise level."))
639 Option("-v", "--verbosity", action="store", type="int",
640 dest="verbosity", default=1, help="Noise level."))
642 # We might like to create a separate 'group' of options for commands
644 Option("-w", action="store_true",
645 dest="wnpp", default=False,
646 help="Operate in WNPP mode: dumps cut-paste-able entry for WNPP bugreport"))
649 Option("--wnpp", action="store",
650 dest="wnpp_mode", default=None,
651 help="Operate in WNPP mode: dumps cut-paste-able entry for WNPP bugreport"))
654 Option("-a", action="store_true",
655 dest="all_mode", default=False,
656 help="Process all files listed in paths.all"))
659 (options, infiles) = p.parse_args()
660 global verbosity; verbosity = options.verbosity
662 if options.wnpp and options.wnpp_mode is None:
663 options.wnpp_mode = 'ITP'
666 config = ConfigParser(defaults={'skip': '.*[~#]$'})
667 config.read(options.config_file)
671 raise ValueError("Do not specify any files in -a mode. Use configuration file, section paths, option all")
672 globs = config.get('paths', 'all', None).split()
673 infiles = reduce(list.__add__, (glob.glob(expanduser(f)) for f in globs))
674 verbose(1, "Found %d files in specified paths" % len(infiles))
677 infiles = [join(options.topdir or './', 'debian/blends')] # default one
679 skip_re = re.compile(config.get('paths', 'skip', None))
681 for blends_file in infiles:
682 verbose(1, "Processing %s" % blends_file)
683 if not exists(blends_file):
684 error("Cannot find a file %s. Either provide a file or specify top "
685 "debian directory with -d." % blends_file, 1)
686 if skip_re.match(blends_file):
687 verbose(2, "W: Skipped since matches paths.skip regexp")
689 pkgs = parse_debian_blends(blends_file)
690 if options.topdir is None:
691 if dirname(blends_file).endswith('/debian'):
692 topdir = dirname(dirname(blends_file))
694 topdir = '.' # and hope for the best ;)
696 topdir = options.topdir
698 expand_pkgs(pkgs, topdir=topdir)
700 pkgs = [p for p in pkgs if not is_template(p)]
702 verbose(2, "W: Skipping since seems to contain templates only")
704 if options.wnpp_mode is not None:
705 print_wnpp(pkgs, config, options.wnpp_mode)
707 # by default -- operate on blends/tasks files
708 tasks = group_packages_into_tasks(pkgs)
709 inject_tasks(tasks, config)
712 if __name__ == '__main__':