2 #emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
3 #ex: set sts=4 ts=4 sw=4 noet:
4 """Script to help maintaining package definitions within Debian blends
7 Often it becomes necessary to duplicate the same information about a
8 package within debian packaging and multiple blends task files,
9 possibly of different blends. This script allows to automate:
11 - construction of entries
12 - injection into task files
13 - modification of existing entries if things changed
14 - removal of previously injected entries
19 * For every package the same task file might be re-read/written (if
20 entry changed/added) from disk.
21 That allows to replace easily original entry for 'source' package
22 (listed as Suggests:) with actual first listed binary package.
23 This should be taken into consideration if current per-package
32 Paths to the blends top directories, containing tasks directories are
33 specified in ~/.blends-inject.cfg file, e.g.::
36 path = ~/deb/debian-med/
39 path = ~/deb/debian-science/
41 Definition of the fields for task files by default are looked up
42 within debian/blends, or files provided in the command line. Also for "-a"
43 mode of operation you should define list of globs to match your debian/blends
47 all=~/deb/gits/pkg-exppsy/neurodebian/future/blends/*
48 ~/deb/gits/*/debian/blends
49 ~/deb/gits/pkg-exppsy/*/debian/blends
50 # Python regular expression on which files to skip
51 # Default is listed below
55 Format of debian/blends
56 -----------------------
60 ; If originally filed using project source name, and it is different
61 ; from the primary (first) binary package name, keep 'Source' to be
62 ; able to adopt previously included tasks entry
65 ; Define the format on how entries should be handled.
67 ; extended -- whenever package is not in Debian and additional
68 ; fields should be obtained from debian/*:
75 ; plain [default] -- only fields listed here should be mentioned.
76 ; Common use -- whenever package is already known to UDD.
78 ; By default, all fields specified previously propagate into following
79 ; packages as well. If that is not desired, add suffix '-clean' to
83 Tasks: debian-science/neuroscience-modeling
85 ; Could have Depends/Recommends/Suggests and Ignore
86 ; All those define Pkg-Name field which is not included
87 ; in the final "rendering" but is available as Pkg-Name item
89 Pkg-URL: http://neuro.debian.net/pkgs/%(Pkg-Name)s.html
91 Published-Authors: Goodman D.F. and Brette R.
92 Published-Title: Brian: a simulator for spiking neural networks in Python
93 Published-In: Front. Neuroinform
95 Published-DOI: 10.3389/neuro.11.005.2008
97 ; May be some previous entry should be removed, thus say so
98 Remove: python-brian-doc
100 ;Tasks: debian-med/imaging-dev
101 ;Why: Allows interactive development/scripting
103 ; ; It should be possible to switch between formats,
104 ; ; e.g. if some component is not yet in Debian
107 ; ; Now some bogus one but with customizations
108 ;Tasks: debian-med/documentation
109 ;Recommends: python-brian-doc
117 import re, os, sys, tempfile, glob
118 from os.path import join, exists, expanduser, dirname, basename
120 from ConfigParser import ConfigParser
121 from optparse import OptionParser, Option
123 from copy import deepcopy
124 #from debian_bundle import deb822
125 from debian import deb822
127 from debian.changelog import Changelog
129 # all files we are dealing with should be UTF8, thus
134 return codecs.open(f, *args, encoding='utf-8')
136 __author__ = 'Yaroslav Halchenko'
137 __prog__ = os.path.basename(sys.argv[0])
138 __version__ = '0.0.7'
139 __copyright__ = 'Copyright (c) 2010 Yaroslav Halchenko'
142 # What fields initiate new package description
143 PKG_FIELDS = ('depends', 'recommends', 'suggests', 'ignore', 'remove')
145 # We might need to resort to assure some what a canonical order
146 # Prefixes for "standard" blends/tasks fields. Others do not get embedded
148 BLENDS_FIELDS_PREFIXES = ('depends', 'recommends', 'suggests', 'ignore',
149 'why', 'homepage', 'language', 'wnpp', 'responsible', 'license',
150 'vcs-', 'pkg-url', 'pkg-description',
151 'published-', 'x-', 'registration', 'remark')
152 # Additional fields which might come useful (e.g. for filing wnpp bugs)
153 # but are not "standard" thus should be in the trailer
154 CUSTOM_FIELDS_PREFIXES = ('author', 'pkg-name', 'pkg-source',
156 # Other fields should cause Error for consistency
158 FIELDS_ORDER = BLENDS_FIELDS_PREFIXES + CUSTOM_FIELDS_PREFIXES
162 def error(msg, exit_code=1):
163 sys.stderr.write(msg + '\n')
166 def verbose(level, msg):
167 if level <= verbosity:
168 sys.stderr.write(" "*level + msg + '\n')
171 def parse_debian_blends(f='debian/blends'):
172 """Parses debian/blends file
174 Returns unprocessed list of customized Deb822 entries
176 # Linearize all the paragraphs since we are not using them
178 for p in deb822.Deb822.iter_paragraphs(open(f)):
181 verbose(6, "Got items %s" % items)
182 # Traverse and collect things
184 format_clean = False # do not propagate fields into a new pkg if True
185 pkg, source = None, None
189 def new_pkg(prev_pkg, bname, sname, tasks):
190 """Helper function to create a new package
192 if format_clean or prev_pkg is None:
193 pkg = deb822.Deb822()
195 pkg = deepcopy(prev_pkg)
196 for k_ in PKG_FIELDS: # prune older depends
198 pkg['Pkg-Name'] = pkg[k] = bname.lower()
199 if sname is not None:
200 sname = sname.lower()
201 pkg['Pkg-Source'] = sname
203 pkg.tasks = dict( (t.strip(), deb822.Deb822Dict()) for t in tasks )
215 format_clean = format_.endswith('-clean')
217 format_ = format_[:-6]
219 tasks = [x.strip() for x in v.split(',')]
220 newtasks = pkg is not None # either we need to provide tune-ups
221 # for current package
222 elif kl in PKG_FIELDS: # new package
223 if source is None and not format_ in ['extended']:
225 pkg = new_pkg(pkg, v, source, tasks)
229 # So we had just source?
231 error("No package or source is known where to add %s" % (k,), 1)
232 # TODO: just deduce source from DebianMaterials
233 pkg = new_pkg(pkg, source, source, tasks)
234 # Since only source is available, it should be only Suggest:-ed
235 pkg['Suggests'] = source.lower()
241 if not t in pkg.tasks:
242 pkg.tasks[t] = deb822.Deb822Dict()
245 # just store the key in the pkg itself
251 def expand_pkgs(pkgs, topdir='.'):
252 """In-place modification of pkgs taking if necessary additional
253 information from Debian materials, and pruning empty fields
255 verbose(4, "Expanding content for %d packages" % len(pkgs))
258 # Expand packages which format is extended
260 if pkg.format == 'extended':
261 # expanding, for that we need debian/control
263 debianm = DebianMaterials(topdir)
264 for k, m in (('License', lambda: debianm.get_license(pkg['Pkg-Name'])),
265 ('WNPP', debianm.get_wnpp),
267 lambda: debianm.get_description(pkg['Pkg-Name'])),
268 ('Responsible', debianm.get_responsible),
269 ('Homepage', lambda: debianm.source.get('Homepage', None)),
270 ('Pkg-source', lambda: debianm.source.get('Source', None)),
278 pkg.update(debianm.get_vcsfields())
281 def prefix_index(x, entries, strict=True, case=False, default=10000):
282 """Returns an index for the x in entries
286 for i, v in enumerate(entries):
292 "Could not find location for %s as specified by %s" %
297 def key_prefix_compare(x, y, order, strict=True, case=False):
298 """Little helper to help with sorting
300 Sorts according to the order of string prefixes as given by
301 `order`. If `strict`, then if no matching prefix found, would
302 raise KeyError; otherwise provides least priority to those keys
303 which were not found in `order`
306 order = [v.lower() for v in order]
308 cmp_res = cmp(prefix_index(x[0], order, strict, case),
309 prefix_index(y[0], order, strict, case))
310 if not cmp_res: # still unknown
315 def group_packages_into_tasks(pkgs):
316 """Given a list of packages (with .tasks) group them per each
317 task and perform necessary customizations stored in .tasks
319 # Time to take care about packages and tasks
320 # Unroll pkgs into a collection of pkgs per known task
323 # Lets just create deepcopies with tune-ups for each task
324 for itask, (task, fields) in enumerate(pkg.tasks.iteritems()):
328 # Perform string completions and removals
329 for k,v in pkg_.iteritems():
331 if v is None or not len(v.strip()):
334 # Sort the fields according to FIELDS_ORDER. Unfortunately
335 # silly Deb822* cannot create from list of tuples, so will do
337 pkg__ = deb822.Deb822()
338 for k,v in sorted(pkg_.items(),
340 key_prefix_compare(x, y, order=FIELDS_ORDER)):
343 # Move Pkg-source/name into attributes
344 pkg__.source = pkg__.pop('Pkg-Source')
345 pkg__.name = pkg__.pop('Pkg-Name')
346 # Store the action taken on the package for later on actions
352 tasks[task] = tasks.get(task, []) + [pkg__]
353 verbose(4, "Grouped %d packages into %d tasks: %s" %
354 (len(pkgs), len(tasks), ', '.join(tasks.keys())))
357 def inject_tasks(tasks, config):
358 # Now go through task files and replace/add entries
359 for task, pkgs in tasks.iteritems():
360 verbose(2, "Task %s with %d packages" % (task, len(pkgs)))
361 blend, puretask = task.split('/')
362 taskfile = expanduser(join(config.get(blend, 'path'), 'tasks', puretask))
365 stats = dict(Added=[], Modified=[])
367 msgs = {'Name': pkg.name.strip(), 'Action': None}
369 # Create a copy of the pkg with only valid tasks
371 # TODO: make it configurable?
374 if prefix_index(k, BLENDS_FIELDS_PREFIXES,
375 strict=False, default=None) is None:
376 pkg.pop(k) # remove it from becoming present in
379 # Find either it is known to the task file already
381 # Load entirely so we could simply manipulate
382 entries = open(taskfile).readlines()
384 # We need to search by name and by source
385 # We need to search for every possible type of dependency
386 regexp_str = '^ *(%s) *: *(%s) *$' \
387 % ('|'.join(PKG_FIELDS),
388 '|'.join((pkg.name, pkg.source)).replace('+', '\+'))
389 verbose(4, "Searching for presence in %s using regexp: '%s'"
390 % (taskfile, regexp_str))
391 regexp = re.compile(regexp_str, re.I)
392 for istart, e in enumerate(entries):
394 verbose(4, "Found %s in position %i: %s" %
395 (pkg.name, istart, e.rstrip()))
399 descr = ' ; Added by %s %s. [Please note here if modified manually]\n' % \
400 (__prog__, __version__)
403 # Replace existing entry?
405 # TODO: Check if previous copy does not have our preceding comment
406 # Find the previous end
409 while entries[istart+icount].strip() != '':
411 except IndexError, e:
412 pass # if we go beyond
414 # Lets not change file without necessity, if entry is identical --
416 old_entry = entries[istart:istart+icount]
418 if u''.join(old_entry) == entry:
419 # no changes -- just go to the next one
421 else: # Rewrite the entry
422 if __prog__ in entries[istart-1]:
425 if 'remove' != pkg.action:
426 entry = descr + entry
427 msgs['Action'] = 'Changed'
429 while entries[istart-1].strip() == '':
433 msgs['Action'] = 'Removed'
434 entries_prior = entries[:istart]
435 entries_post = entries[istart+icount:]
436 elif not 'remove' == pkg.action: # or Append one
437 msgs['Action'] = 'Added'
438 entries_prior = entries
439 entry = descr + entry
441 # could be as simple as
442 # Lets do 'in full' for consistent handling of empty lines
444 #output = '\n%s%s' % (descr, pkg.dump(),)
445 #open(taskfile, 'a').write(output)
448 # Prepare for dumping
449 # Prune spaces before
450 while len(entries_prior) and entries_prior[-1].strip() == '':
451 entries_prior = entries_prior[:-1]
452 if len(entries_prior) and not entries_prior[-1].endswith('\n'):
453 entries_prior[-1] += '\n' # assure present trailing newline
455 while len(entries_post) and entries_post[0].strip() == '':
456 entries_post = entries_post[1:]
457 if len(entries_post) and len(entry):
458 # only then trailing empty line
460 output = ''.join(entries_prior + [ '\n' + entry ] + entries_post)
461 open(taskfile, 'w').write(output) # then only overwrite
463 verbose(3, "%(Action)s %(Name)s" % msgs)
466 class DebianMaterials(object):
467 """Extract selected information from an existing debian/
469 _WNPP_RE = re.compile('^ *\* *Initial release.*closes:? #(?P<bug>[0-9]*).*', re.I)
471 def __init__(self, topdir):
472 #self.topdir = topdir
473 self._debiandir = join(topdir, 'debian')
475 self._binaries = None
479 if self._source is None:
480 self._assign_packages()
485 if self._binaries is None:
486 self._assign_packages()
487 return self._binaries
489 def fpath(self, name):
490 return join(self._debiandir, name)
492 def _assign_packages(self):
494 control = deb822.Deb822.iter_paragraphs(
495 open(self.fpath('control')))
498 "Cannot parse %s file necessary for the %s package entry. Error: %s"
499 % (control_file, pkg['Pkg-Name'], str(e)))
503 if v.get('Source', None):
506 # Since it might be hash-commented out
508 self._binaries[v['Package']] = v
510 def get_license(self, package=None, first_only=True):
511 """Return a license(s). Parsed out from debian/copyright if it is
512 in machine readable format
515 # may be package should carry custom copyright file
516 copyright_file_ = self.fpath('%s.copyright' % package)
517 if package and exists(copyright_file_):
518 copyright_file = copyright_file_
520 copyright_file = self.fpath('copyright')
523 for p in deb822.Deb822.iter_paragraphs(open(copyright_file)):
524 if not 'Files' in p or p['Files'].strip().startswith('debian/'):
527 # Take only the short version of first line
528 l = re.sub('\n.*', '', l).strip()
531 if not l in licenses:
538 return ', '.join(licenses)
541 """Search for a template changelog entry closing "Initial bug
543 for l in open(self.fpath('changelog')):
544 rr = self._WNPP_RE.match(l)
546 return rr.groupdict()['bug']
549 def get_responsible(self):
550 """Returns responsible, atm -- maintainer
552 return self.source['Maintainer']
554 def get_vcsfields(self):
555 vcs = deb822.Deb822()
556 for f,v in self._source.iteritems():
557 if f.lower().startswith('vcs-'):
561 def get_description(self, pkg_name):
562 """Some logic to extract description.
564 If binary package matching pkg_name is found -- gets it description.
565 If no binary package with such name, and name matches source name,
566 obtain description of the first binary package.
568 if pkg_name in self.binaries:
570 elif pkg_name.lower() == self.source['Source'].lower():
571 pkg_name = self.binaries.keys()[0]
573 error("Name %s does not match any binary, nor source package in %s"
575 return self.binaries[pkg_name]['Description']
577 def print_wnpp(pkgs, config, wnpp_type="ITP"):
578 """Little helper to spit out formatted entry for WNPP bugreport
580 TODO: It would puke atm if any field is missing
583 pkg = pkgs[0] # everything is based on the 1st one
584 opts = dict(pkg.items())
585 opts['WNPP-Type'] = wnpp_type.upper()
586 opts['Pkg-Description-Short'] = re.sub('\n.*', '', pkg['Pkg-Description'])
588 subject = "%(WNPP-Type)s: %(Pkg-Name)s -- %(Pkg-Description-Short)s" % opts
589 body = """*** Please type your report below this line ***
591 * Package name : %(Pkg-Name)s
592 Version : %(Version)s
593 Upstream Author : %(Author)s
595 * License : %(License)s
596 Programming Lang: %(Language)s
597 Description : %(Pkg-Description)s
601 # Unfortunately could not figure out how to set the owner, so I will just print it out
603 tmpfile = tempfile.NamedTemporaryFile()
606 cmd = "reportbug -b --paranoid --subject='%s' --severity=wishlist --body-file='%s' -o /tmp/o.txt wnpp" \
607 % (subject, tmpfile.name)
608 verbose(2, "Running %s" %cmd)
611 print "Subject: %s\n\n%s" % (subject, body)
615 """Helper to return true if pkg definition looks like a template
616 and should not be processed
618 # We might want to skip some which define a skeleton
619 # (no source/homepage/etc although fields are there)
620 for f in ['vcs-browser', 'pkg-url', 'pkg-description',
621 'published-Title', 'pkg-name', 'homepage',
623 if f in p and p[f] != "":
631 usage="%s [OPTIONS] [blends_files]\n\n" % __prog__ + __doc__,
632 version="%prog " + __version__)
635 Option("-d", "--topdir", action="store",
636 dest="topdir", default=None,
637 help="Top directory of a Debian package. It is used to locate "
638 "'debian/blends' if none is specified, and where to look for "
639 "extended information."))
642 Option("-c", "--config-file", action="store",
643 dest="config_file", default=join(expanduser('~'), '.%s.cfg' % __prog__),
644 help="Noise level."))
647 Option("-v", "--verbosity", action="store", type="int",
648 dest="verbosity", default=1, help="Noise level."))
650 # We might like to create a separate 'group' of options for commands
652 Option("-w", action="store_true",
653 dest="wnpp", default=False,
654 help="Operate in WNPP mode: dumps cut-paste-able entry for WNPP bugreport"))
657 Option("--wnpp", action="store",
658 dest="wnpp_mode", default=None,
659 help="Operate in WNPP mode: dumps cut-paste-able entry for WNPP bugreport"))
662 Option("-a", action="store_true",
663 dest="all_mode", default=False,
664 help="Process all files listed in paths.all"))
667 (options, infiles) = p.parse_args()
668 global verbosity; verbosity = options.verbosity
670 if options.wnpp and options.wnpp_mode is None:
671 options.wnpp_mode = 'ITP'
674 config = ConfigParser(defaults={'skip': '.*[~#]$'})
675 config.read(options.config_file)
679 raise ValueError("Do not specify any files in -a mode. Use configuration file, section paths, option all")
680 globs = config.get('paths', 'all', None).split()
681 infiles = reduce(list.__add__, (glob.glob(expanduser(f)) for f in globs))
682 verbose(1, "Found %d files in specified paths" % len(infiles))
685 infiles = [join(options.topdir or './', 'debian/blends')] # default one
687 skip_re = re.compile(config.get('paths', 'skip', None))
689 for blends_file in infiles:
690 verbose(1, "Processing %s" % blends_file)
691 if not exists(blends_file):
692 error("Cannot find a file %s. Either provide a file or specify top "
693 "debian directory with -d." % blends_file, 1)
694 if skip_re.match(blends_file):
695 verbose(2, "W: Skipped since matches paths.skip regexp")
697 pkgs = parse_debian_blends(blends_file)
698 if options.topdir is None:
699 if dirname(blends_file).endswith('/debian'):
700 topdir = dirname(dirname(blends_file))
702 topdir = '.' # and hope for the best ;)
704 topdir = options.topdir
706 expand_pkgs(pkgs, topdir=topdir)
708 pkgs = [p for p in pkgs if not is_template(p)]
710 verbose(2, "W: Skipping since seems to contain templates only")
712 if options.wnpp_mode is not None:
713 print_wnpp(pkgs, config, options.wnpp_mode)
715 # by default -- operate on blends/tasks files
716 tasks = group_packages_into_tasks(pkgs)
717 inject_tasks(tasks, config)
720 if __name__ == '__main__':